From 68eaa21ddbf633b69618913769c7ec3c724db3f6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:53:22 +0000 Subject: [PATCH 1/6] Initial plan From d02c2b60ddd46feaaaee856e66e2f00c3fc7f671 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:14:20 +0000 Subject: [PATCH 2/6] Integrate xAI voice, image storage, and protocol updates Co-authored-by: kzu <169707+kzu@users.noreply.github.com> --- .gitignore | 2 + .netconfig | 32 ++-- AGENTS.md | 6 + readme.md | 83 +++++++++- src/Directory.Build.props | 4 +- src/Directory.Build.targets | 13 +- .../ProtocolServiceCollectionExtensions.cs | 26 +++ src/xAI.Protocol/chat.proto | 58 ++++++- src/xAI.Protocol/files.proto | 102 ++++++++++++ src/xAI.Protocol/image.proto | 96 ++++++++++- src/xAI.Protocol/usage.proto | 1 + src/xAI.Protocol/video.proto | 111 ++++++++++++- src/xAI.Tests/ChatClientTests.cs | 12 ++ src/xAI.Tests/GrokConversionTests.cs | 61 ++++++- src/xAI.Tests/ImageGeneratorTests.cs | 55 ++++++ src/xAI.Tests/SanityChecks.cs | 11 ++ src/xAI.Tests/SpeechToTextClientTests.cs | 23 ++- src/xAI.Tests/TextToSpeechClientTests.cs | 56 ++++++- src/xAI/GrokChatOptions.cs | 7 + src/xAI/GrokClient.cs | 6 + src/xAI/GrokImageGenerationOptions.cs | 23 +++ src/xAI/GrokImageGenerator.cs | 136 +-------------- src/xAI/GrokImageProtocolExtensions.cs | 156 ++++++++++++++++++ src/xAI/GrokProtocolExtensions.cs | 37 ++++- src/xAI/GrokSpeechToTextClient.cs | 45 ++++- src/xAI/GrokSpeechToTextOptions.cs | 20 +++ src/xAI/GrokTextToSpeechClient.cs | 53 +++++- src/xAI/GrokTextToSpeechOptions.cs | 8 + src/xAI/xAI.csproj | 2 +- 29 files changed, 1048 insertions(+), 197 deletions(-) create mode 100644 src/xAI/GrokImageProtocolExtensions.cs diff --git a/.gitignore b/.gitignore index 25e70dd..501a78b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ _site .sass-cache Gemfile.lock package-lock.json + +/.github/.labels \ No newline at end of file diff --git a/.netconfig b/.netconfig index e8c4bb8..6147023 100644 --- a/.netconfig +++ b/.netconfig @@ -85,8 +85,8 @@ weak [file ".gitignore"] url = https://github.com/devlooped/oss/blob/main/.gitignore - sha = ff61659751374b95c7a8a0477c908a8119f756f0 - etag = e5865f083db45081a7b4eaa518018971b34e6ef93f917ac510dea96d27f792b3 + sha = fefbf3606e7bb7f483951e3107200a2f45ee52e0 + etag = 5bdfd49876c886aaa6b7026960261af9b587e7cc3baa0f4584db34d9cebbd3f4 weak [file "Directory.Build.rsp"] url = https://github.com/devlooped/oss/blob/main/Directory.Build.rsp @@ -110,13 +110,13 @@ weak [file "src/Directory.Build.props"] url = https://github.com/devlooped/oss/blob/main/src/Directory.Build.props - sha = 6e2438919e108aeb75106dc0737c45f5e55d5f42 - etag = f1d6384abf18d8d891ce5e835a10c73fe029c42151374be96d7e4af43d189c65 + sha = 59861ddd1e330f3da410652b4c7fc6585726c0cf + etag = 1561f1be53cc25ac0cd85a06b3f65d69764b174d0f60bf72ca2f8fc70573b75e weak [file "src/Directory.Build.targets"] url = https://github.com/devlooped/oss/blob/main/src/Directory.Build.targets - sha = 3a758f47e8955c15a016b3db08f383781cbdee26 - etag = 22005ab28676aa6d790171003057c81fe4ceec01251626385a3dcef3417ae3cd + sha = 59861ddd1e330f3da410652b4c7fc6585726c0cf + etag = cb1f7a3e5bf85e7307407b7b6d8b9869330e2191a2317ccf6218a552e5890b08 weak [file "src/nuget.config"] url = https://github.com/devlooped/oss/blob/main/src/nuget.config @@ -157,8 +157,8 @@ weak [file "src/xAI.Protocol/chat.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/chat.proto - sha = 17a2da08cbbf89aa1f33ffe58b687fe0b2d50468 - etag = e7da2c915664caf64c0da7d886826de37fcf98753cf613a6ad8aad96f6ddcda5 + sha = 065692d0455244b0d5f0ccf9c05f8b98ff4618cd + etag = 8c8c9f9cc1d6991052a5058ee1a7d1829177ea9db95b9e92dc6904fc493c33c6 weak [file "src/xAI.Protocol/deferred.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/deferred.proto @@ -177,8 +177,8 @@ weak [file "src/xAI.Protocol/image.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/image.proto - sha = d67bcf3e661aa9641af9750632fa1c38ea974da1 - etag = 3ea27e240320c26d14b8c64d00f236c078127ebdb6fa957efc49232dfd75a20c + sha = 2c2df4f5d9429f5f21e58bb9e63285c69bcc144f + etag = 3d66f35d2e1555c6eae2218705b6f8ee5fa6537939b6ae0a18e0a2fd13a81bff weak [file "src/xAI.Protocol/models.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/models.proto @@ -197,13 +197,13 @@ weak [file "src/xAI.Protocol/usage.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/usage.proto - sha = d8def643bea79ad10f5a678d70ae37edca26490f - etag = 74c44beb7bfd2e75ea0524ff9be820cc067e009b071dee59869a16d372280a0a + sha = 2c2df4f5d9429f5f21e58bb9e63285c69bcc144f + etag = 0bf9577be87d3cfc79895971071ef1eaa78645cb784462e7e90bff03440dfff4 weak [file "src/xAI.Protocol/video.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/video.proto - sha = 43a1b6b731943b8f031c2f2d946f7183f0933ffd - etag = 37562e78a6d64800b09c643632a33b6bc902955b491114bd5b6ec957d23d6e64 + sha = c22ad8b1d87375ab8796b224aa56785ed922eb0d + etag = da8e45fa8f9e4ac298896e55680bb6b25a280565fd0926f18b68f17ab84cbde0 weak [file "src/xAI.Protocol/google/protobuf/timestamp.proto"] url = https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/timestamp.proto @@ -238,6 +238,6 @@ weak [file "src/xAI.Protocol/files.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/files.proto - etag = 4c91f851b288a225acfc1173f2f8853a1b550da1e97d2fdcec99debb5f8fac43 + etag = b5e8ed748220b2fa727e8845f09f612cb77813d02c4260b77e6464aa0c98d57f weak - sha = 0c0f5353aa7ab2a4ffea310f9d9364ed5c424af2 + sha = c666ac39e9a8f562e94b2b57872a75bc1438ec2b diff --git a/AGENTS.md b/AGENTS.md index 6af3028..0ef819f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,12 @@ - `ChatOptions` mappings include `Seed`, `StopSequences`, `AllowMultipleToolCalls` → `parallel_tool_calls`, `Reasoning.Effort` → `reasoning_effort`, and `ConversationId` → `previous_response_id`. `GrokChatOptions.StoreMessages` enables stored responses and surfaces `ChatResponse.ConversationId`. - `UsageDetails` maps `ReasoningTokenCount` and `CachedInputTokenCount` from xAI usage, plus prompt text/image/source/cost details in `AdditionalCounts`. - Web/X search tool calls map to MEAI `WebSearchToolCallContent` / `WebSearchToolResultContent` (queries from tool arguments when present; citation URLs become `UriContent` outputs). +- The SDK pins stable `Microsoft.Extensions.AI.Abstractions` 10.10.1; its `HostedImageGenerationTool`, `ImageGenerationToolCallContent`, and `ImageGenerationToolResultContent` are used for xAI's chat image-generation tool. The optional xAI `action` is passed through `HostedImageGenerationTool.AdditionalProperties["action"]`. +- `GrokImageGenerationOptions` maps image quality and Files API storage/public-URL settings to image protocol fields. Per-image MEAI content keeps the generated protocol image as `RawRepresentation` and exposes moderation, file output, and storage errors in `AdditionalProperties`. +- `GrokChatOptions.SafetyIdentifier` maps to xAI's separate `safety_identifier` request field. Keep it distinct from MEAI's end-user `user` mapping; callers should hash stable user IDs and avoid personal information. +- MEAI `ReasoningEffort.ExtraHigh` maps to xAI's `EFFORT_XHIGH`. Newly synced Files and Video protocol services are exposed by `GrokClient.GetFilesClient()` / `GetVideoClient()` and `AddxAIProtocol`; no MEAI file/video abstraction is inferred from the protocol. +- TTS uses MEAI 10.10.1's native `TextToSpeechOptions.Speed`; `GrokTextToSpeechOptions` adds phrase replacement and optional character timestamps, surfaced as response/update additional properties. STT maps key terms, filler words, VAD threshold, and streaming Smart Turn options; arbitrary Opus packetization is not exposed because the current stream contract cannot preserve packet boundaries. +- xAI's speech-to-speech Realtime/WebRTC/SIP APIs have no matching stable MEAI abstraction or generated protocol surface in this SDK. Treat this as a separately scoped API integration rather than inventing an adapter during protocol/voice maintenance. ## Comprehensive upstream maintenance diff --git a/readme.md b/readme.md index b377c27..1260031 100644 --- a/readme.md +++ b/readme.md @@ -66,6 +66,28 @@ var text = await transcription.GetTextAsync(File.OpenRead("audio.mp3"), new SpeechToTextOptions { TextLanguage = "en" }); ``` +Use Grok-specific options for xAI's voice parameters. The stable MEAI +`TextToSpeechOptions.Speed` property controls speed, while timestamps and phrase +replacements are available through `GrokTextToSpeechOptions`: + +```csharp +var audio = await speech.GetAudioAsync("Welcome to Acme Mobile.", + new GrokTextToSpeechOptions + { + Speed = 1.2f, + WithTimestamps = true, + Replace = new Dictionary { ["Acme Mobile"] = "Acme Mobull" }, + }); + +var characterTimings = (JsonElement)audio.AdditionalProperties!["audio_timestamps"]!; +``` + +`audio_duration` and `audio_timestamps` are also included in streaming audio +updates when requested. For transcription, `GrokSpeechToTextOptions` exposes +`KeyTerms`, `FillerWords`, `VadThreshold`, and (for streaming) `SmartTurn` and +`SmartTurnTimeout`. Raw Opus packet streaming is not supported by the current +stream API because it does not preserve packet boundaries. + ## File Attachments You can attach files to messages using `DataContent` to enable Grok to analyze documents, @@ -166,6 +188,34 @@ Learn more about available filters at [X search parameters](https://docs.x.ai/do You can combine both web and X search in the same request by adding both tools. +## Image Generation Tool + +Grok can generate or edit images as part of a chat response using MEAI's +`HostedImageGenerationTool`: + +```csharp +var response = await grok.GetResponseAsync( + "Create a poster of a red fox in a snowy forest.", + new ChatOptions { Tools = [new HostedImageGenerationTool()] }); + +var calls = response.Messages.SelectMany(x => x.Contents) + .OfType(); +``` + +By default, the model may generate or edit images. Use the xAI-specific `action` +additional property to restrict the tool to generation or editing: + +```csharp +var imageTool = new HostedImageGenerationTool(new Dictionary +{ + ["action"] = "generate", // auto | generate | edit +}); +``` + +The adapter surfaces `ImageGenerationToolCallContent` and +`ImageGenerationToolResultContent`; inspect each result's `Outputs` or raw +protocol representation for provider-returned output details. + ## Code Execution The code execution tool enables Grok to write and execute Python code in real-time, @@ -308,6 +358,16 @@ var options = new GrokChatOptions Learn more about [Remote MCP tools](https://docs.x.ai/docs/guides/tools/remote-mcp-tools). +For abuse attribution, set `SafetyIdentifier` to a stable hashed identifier +instead of sending an email address, name, or other personal information: + +```csharp +var options = new GrokChatOptions +{ + SafetyIdentifier = hashedInternalUserId, +}; +``` + ## Image Generation Grok also supports image generation using the `IImageGenerator` abstraction from @@ -333,8 +393,8 @@ Console.WriteLine($"Generated image URL: {image.Uri}"); ### Grok-Specific Options -Use `GrokImageGenerationOptions` to control aspect ratio and resolution — features -unique to grok-imagine models: +Use `GrokImageGenerationOptions` to control quality, aspect ratio, and resolution +or store outputs in the Files API: ```csharp var imageGenerator = new GrokClient(Environment.GetEnvironmentVariable("XAI_API_KEY")!) @@ -346,15 +406,27 @@ var options = new GrokImageGenerationOptions ResponseFormat = ImageGenerationResponseFormat.Uri, AspectRatio = ImageAspectRatio.ImgAspectRatio16_9, Resolution = ImageResolution.ImgResolution2K, + Quality = ImageQuality.ImgQualityHigh, + Storage = new GrokImageStorageOptions + { + Filename = "city.png", + CreatePublicUrl = true, + }, }; var response = await imageGenerator.GenerateAsync(request, options); var image = (UriContent)response.Contents.First(); Console.WriteLine($"Generated image URL: {image.Uri}"); +var file = (xAI.Protocol.FileOutput)image.AdditionalProperties!["file_output"]!; ``` Aspect ratio defaults to 1:1 and resolution defaults to 1k when not specified. -2k output is generated at 1k and then upscaled with super-resolution. +Quality defaults to medium. 2k output is generated at 1k and then upscaled with +super-resolution. Per-image `AdditionalProperties` includes `file_output` when +storage succeeds, `storage_error` when upload fails, and `respect_moderation`. +`GrokImageStorageOptions.ExpiresAfterSeconds` sets file expiry; set +`PublicUrlExpiresAfterSeconds` to create an expiring public URL, or +`CreatePublicUrl = true` for a non-expiring URL. ### Editing Images @@ -593,6 +665,11 @@ class MyService(Chat.ChatClient chat, Documents.DocumentsClient docs, Embedder.E } ``` +The generated `Files` and `Video` gRPC clients are available through +`GrokClient.GetFilesClient()` / `GetVideoClient()` or dependency injection +(`Files.FilesClient` and `Video.VideoClient`). These expose the upstream protocol +directly; no higher-level MEAI file or video abstraction is implied. + ## Auto-updating This project contains an automated mechanism to always fetch the latest version diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 93a0b1e..5c43a5d 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -25,10 +25,12 @@ false MIT - + + logo.png icon.png readme.md + logo.png icon.png readme.md diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index 4a6ef38..c9a50b6 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -63,7 +63,12 @@ + Condition="'$(PackageIcon)' == 'icon.png'" /> + + - + Condition="'$(PackageIcon)' != '' and Exists('$(MSBuildThisFileDirectory)$(PackageIcon)') and !Exists('$(MSBuildProjectDirectory)\$(PackageIcon)')" /> (options => + { + options.Address = address; + configureClient?.Invoke(options); + }) + .AddCallCredentials((context, metadata) => + { + metadata.Add("Authorization", $"******"); + return Task.CompletedTask; + }); + + configureHttp?.Invoke(builder); + builder = services.AddGrpcClient(options => { options.Address = address; @@ -120,6 +133,19 @@ public static IServiceCollection AddxAIProtocol(this IServiceCollection services configureHttp?.Invoke(builder); + builder = services.AddGrpcClient(options => + { + options.Address = address; + configureClient?.Invoke(options); + }) + .AddCallCredentials((context, metadata) => + { + metadata.Add("Authorization", $"******"); + return Task.CompletedTask; + }); + + configureHttp?.Invoke(builder); + return services; } } diff --git a/src/xAI.Protocol/chat.proto b/src/xAI.Protocol/chat.proto index 14001bf..eb575b8 100644 --- a/src/xAI.Protocol/chat.proto +++ b/src/xAI.Protocol/chat.proto @@ -144,7 +144,7 @@ message GetCompletionsRequest { // new topics. optional float presence_penalty = 9; - // Constrains effort on reasoning for reasoning models. Default to `EFFORT_MEDIUM`. + // Constrains effort on reasoning for reasoning models. Defaults vary by model (e.g. `grok-4.5` and `grok-4.6` default to `EFFORT_HIGH`). optional ReasoningEffort reasoning_effort = 19; // Set the parameters to be used for realtime data. If not set, no realtime data will be acquired by the model. @@ -180,6 +180,13 @@ message GetCompletionsRequest { // Processing tier for this request. Set to SERVICE_TIER_PRIORITY for // higher scheduling priority at a higher price. ServiceTier service_tier = 31; + + // Supplied by the API client to identify the end user behind this request. + // A stable string that uniquely identifies each of your users; hash your + // internal user id or username rather than sending an email or name. Stored + // with the request metadata so a usage-policy violation can be attributed + // to that user rather than to the API key. + optional string safety_identifier = 34; } message GetChatCompletionResponse { @@ -547,6 +554,30 @@ enum IncludeOption { // Generated file output from the code execution environment INCLUDE_OPTION_CODE_EXECUTION_FILES_OUTPUT = 9; + + // EXPERIMENTAL: This option is experimental and its behavior may change or be + // removed in a future release without a major version bump. + // + // Stream back client-side tool calls incrementally as the model generates + // them, instead of only emitting each tool call once it is complete. + // When included, the stream contains additional `ToolCall` entries in + // `Delta.tool_calls` with `status` set to `TOOL_CALL_STATUS_IN_PROGRESS`. + // Each such entry carries a fragment of the `FunctionCall.name` and/or + // `FunctionCall.arguments` (either may be empty) that the client should + // append to the tool call identified by the entry's `id` and `index`. + // Once generation finishes, a final `ToolCall` with `status` set to + // `TOOL_CALL_STATUS_COMPLETED` is streamed containing the full name and + // arguments, so clients that do not want to accumulate fragments can rely + // on that entry alone. + // This option is only available for streaming responses and is only + // supported by models that support it; it is ignored otherwise. + // + // KNOWN ISSUE: This option is currently broken when the request also uses + // any server-side tool (for example web search, X search, code execution, + // collections search, or MCP). Do not combine this option with server-side + // tools; only use it with requests whose tools are all client-side + // functions. + INCLUDE_OPTION_TOOL_CALL_STREAMING = 10; } // A message in a conversation. This message is part of the model input. Each @@ -609,6 +640,7 @@ enum ReasoningEffort { EFFORT_MEDIUM = 2; EFFORT_HIGH = 3; EFFORT_NONE = 4; + EFFORT_XHIGH = 5; } // Number of agents to use for multi-agent models. @@ -672,6 +704,8 @@ message Tool { MCP mcp = 7; // Built in attachment search. AttachmentSearch attachment_search = 8; + // Built in image generation. + ImageGeneration image_generation = 10; } } @@ -767,6 +801,13 @@ message XSearch { message CodeExecution {} +message ImageGeneration { + // Which image capabilities to expose to the model. One of "auto" (the default; + // both generation and editing), "generate" (text-to-image only), or + // "edit" (image editing only). + optional string action = 1; +} + message CollectionsSearch { // The ID(s) of the source collection(s) within which the search should be performed. // A maximum of 10 collections IDs can be used for search. @@ -839,6 +880,10 @@ enum ToolCallType { // Indicates the tool is a server-side attachment_search tool, and client side won't need to execute. // Maps to `attachment_search_call` type in OAI Responses API. TOOL_CALL_TYPE_ATTACHMENT_SEARCH_TOOL = 7; + + // Indicates the tool is a server-side image_generation tool, and client side won't need to execute. + // Maps to `image_generation_call` type in OAI Responses API. + TOOL_CALL_TYPE_IMAGE_GENERATION_TOOL = 10; } enum ToolCallStatus { @@ -870,6 +915,15 @@ message ToolCall { // Error message if the tool call is failed. optional string error_message = 4; + // EXPERIMENTAL: This field is experimental and its behavior may change or be + // removed in a future release without a major version bump. + // + // The index of the tool call within the message's `tool_calls` array. + // Only populated on the incremental `TOOL_CALL_STATUS_IN_PROGRESS` entries + // streamed when `INCLUDE_OPTION_TOOL_CALL_STREAMING` is requested, so that + // clients can accumulate fragments belonging to the same tool call. + optional int32 index = 5; + // Information regarding invoking the tool call. oneof tool { FunctionCall function = 10; @@ -1059,7 +1113,7 @@ message RequestSettings { // The ID of the previous response from the model. optional string previous_response_id = 3; - // Constrains effort on reasoning for reasoning models. Default to `EFFORT_MEDIUM`. + // Constrains effort on reasoning for reasoning models. Defaults vary by model (e.g. `grok-4.5` and `grok-4.6` default to `EFFORT_HIGH`). optional ReasoningEffort reasoning_effort = 4; // A number between 0 and 2 used to control the variance of completions. diff --git a/src/xAI.Protocol/files.proto b/src/xAI.Protocol/files.proto index f4a7d56..7c5ac65 100644 --- a/src/xAI.Protocol/files.proto +++ b/src/xAI.Protocol/files.proto @@ -36,6 +36,45 @@ service Files { // Stream the file's contents in chunks of up to 5 MB, in order. // Concatenate `data` from every chunk to reconstruct the file. rpc RetrieveFileContent(RetrieveFileContentRequest) returns (stream FileContentChunk) {} + + // Create a public, unauthenticated URL for a file, accessible without an + // API key. Only images, videos, and PDFs can be made public. + // + // A file can have at most one public URL at a time. Calling this on a + // file that already has a public URL returns the existing URL. To update + // the expiry, call again with a new `expires_after` value. + // + // **Public URL expiry behavior:** + // + // - If `expires_after` is set, the public URL expires that many seconds + // from now, independently of the file's own TTL. + // - If `expires_after` is omitted and the file has a TTL + // (`UploadFileInit.expires_after` was set at upload), the public URL + // automatically inherits the file's expiry — it will expire at the + // same time as the file. + // - If `expires_after` is omitted and the file has no TTL, the public + // URL remains valid indefinitely (until explicitly revoked or the file + // is deleted). + // + // **Automatic revocation:** + // + // A public URL is automatically revoked when the file is deleted (by the + // user or by TTL expiry). Revocation may not take effect immediately, so + // the URL can remain accessible for a short period after the file or the + // public URL expires. Call `RevokePublicUrl` to revoke immediately. + rpc CreatePublicUrl(CreatePublicUrlRequest) returns (CreatePublicUrlResponse) {} + + // Revoke the public URL for a file. After revocation the public URL is + // no longer accessible, while the original file remains accessible via + // authenticated endpoints. + // + // Public URLs are also automatically revoked when the file is deleted, + // when the file's TTL elapses, or when the public URL's own expiry + // elapses. Automatic revocation may not take effect immediately; use this + // RPC to revoke a public URL right away. + // + // Returns success if the file has no public URL (nothing to revoke). + rpc RevokePublicUrl(RevokePublicUrlRequest) returns (RevokePublicUrlResponse) {} } // First stream message of an `UploadFile` call. Sent exactly once, before @@ -84,6 +123,14 @@ message File { string id = 5; reserved 6; + + // Public URL for the file. Only present when the file has an active + // public URL (created via CreatePublicUrl or storage_options.public_url). + optional string public_url = 7; + + // When the public URL expires. Only present when public_url is set + // and has an independent expiry. + optional google.protobuf.Timestamp public_url_expires_at = 8; } // Sort direction for list-style RPCs. @@ -119,6 +166,16 @@ message ListFilesRequest { // Sort field. Defaults to `FILES_SORT_BY_CREATED_AT`. optional FilesSortBy sort_by = 4; + + // AIP-160 filter expression to narrow down results. + // Supported fields: file_id, name (or file_name), size_bytes, + // content_type, created_at, expires_at, upload_status, user_defined_id. + // Operators: =, !=, >, >=, <, <=, AND, OR, NOT. + // Examples: + // - 'name:"report"' + // - 'content_type = "application/pdf"' + // - 'size_bytes > 1000000 AND created_at > "2024-01-01T00:00:00Z"' + optional string filter = 5; } // Response message for `Files.ListFiles`. @@ -179,3 +236,48 @@ message FileContentChunk { // Up to 5 MB of file bytes. Final/intermediate chunks may be smaller. bytes data = 1; } + +// Request message for `Files.CreatePublicUrl`. +message CreatePublicUrlRequest { + // The ID of the file to create a public URL for. + string file_id = 1; + + // Seconds from now until the public URL expires. Must be between 3600 + // (1 hour) and 2592000 (30 days). + // + // If omitted and the file has a TTL, the public URL inherits the file's + // expiry. If omitted and the file has no TTL, the public URL remains + // valid indefinitely until the file is deleted or the URL is explicitly + // revoked via `RevokePublicUrl`. + optional int64 expires_after = 2; +} + +// Response message for `Files.CreatePublicUrl`. +message CreatePublicUrlResponse { + // The full public URL that can be shared and accessed without an API key. + string public_url = 1; + + // When the public URL expires. Present when the public URL has an expiry, + // either from an explicit `expires_after` in the request or inherited from + // the file's TTL. Absent when the public URL is valid indefinitely. + optional google.protobuf.Timestamp expires_at = 2; +} + +// Request message for `Files.RevokePublicUrl`. +message RevokePublicUrlRequest { + // The ID of the file whose public URL should be revoked. + string file_id = 1; +} + +// Response message for `Files.RevokePublicUrl`. +message RevokePublicUrlResponse { + // The file ID whose public URL was revoked. + string file_id = 1; + + // True if a public URL was actually revoked. False if the file had + // no active public URL (no-op). + bool revoked = 2; + + // The full public URL that was revoked. Only present when revoked is true. + optional string public_url = 3; +} diff --git a/src/xAI.Protocol/image.proto b/src/xAI.Protocol/image.proto index 325339f..f7af62a 100644 --- a/src/xAI.Protocol/image.proto +++ b/src/xAI.Protocol/image.proto @@ -3,6 +3,7 @@ option csharp_namespace = "xAI.Protocol"; package xai_api; +import "google/protobuf/timestamp.proto"; import "usage.proto"; // An API service for interaction with image generation models. @@ -36,6 +37,12 @@ message GenerateImageRequest { // in. See ImageFormat enum for options. ImageFormat format = 11; + // Optional quality setting for image generation. + // Only supported by grok-imagine models. + // Defaults to medium. Some models restrict the accepted values (a request + // outside the model's supported set is rejected). + optional ImageQuality quality = 12; + // Optional aspect ratio for image generation/editing. // Only supported by grok-imagine models. // Defaults to 1:1 if not specified. Auto is only supported for image generation @@ -54,6 +61,62 @@ message GenerateImageRequest { // Each image is either an image URL or a base64-encoded version of the image. // This field cannot be set together with the `image` field. repeated ImageUrlContent images = 17; + + // Optional output storage configuration. When present, the generated + // image(s) are stored in the Files API and a file_id is returned in + // the response alongside the ephemeral URL. + optional StorageOptions storage_options = 19; +} + +// Configuration for storing generation output in the Files API. +message StorageOptions { + // Filename for the stored file. + string filename = 1; + // Seconds from now until the file auto-expires. If omitted, the file + // does not expire. + optional int64 expires_after = 2; + // When present, a public URL is created for the stored file after upload. + // The public URL is accessible without authentication. + // Omit entirely to store the file privately (no public URL). + optional PublicUrlOptions public_url = 3; +} + +// Configuration for creating a public URL alongside file storage. +message PublicUrlOptions { + // Seconds from now until the public URL expires. + // + // If omitted and the file has a TTL (`StorageOptions.expires_after`), + // the public URL inherits the file's expiry. If omitted and the file + // has no TTL, the public URL remains valid indefinitely until the file + // is deleted or the URL is explicitly revoked via `RevokePublicUrl`. + // The file itself always remains accessible via authenticated endpoints + // after the public URL expires. + optional int64 expires_after = 1; +} + +// Information about a generated file stored in the Files API. +message FileOutput { + // Files API file_id of the stored file. + string file_id = 1; + // Filename of the stored file. + string filename = 2; + reserved 3; + // Public URL for the stored file. Only present when the request included + // storage_options.public_url and creation succeeded. + optional string public_url = 4; + // When the public URL expires. Only present when public_url is set and + // has an independent expiry (i.e. the request specified + // public_url.expires_after, or the file has a TTL). Absent when the + // public URL is valid indefinitely. + optional google.protobuf.Timestamp public_url_expires_at = 5; + // When the stored file expires and will be automatically deleted. Only + // present when the file has an expiration (storage_options.expires_after + // was set). + optional google.protobuf.Timestamp expires_at = 6; + // Human-readable error when storage_options.public_url was set but + // public URL creation failed. The file itself was stored successfully — + // the public URL can be retried via `CreatePublicUrl`. + optional string public_url_error = 7; } // The response from the image generation models containing the generated image(s). @@ -87,18 +150,35 @@ message GeneratedImage { // The field will be true if the image respect moderation rules. Otherwise // the field will be false and the image field is replaced by a placeholder. bool respect_moderation = 4; + + // Storage info for the generated image. Only present when the request + // included `storage_options` and the upload succeeded. + optional FileOutput file_output = 8; + + // Human-readable error when `storage_options` was set but the upload + // failed. Only present on storage failure; absent on success or when + // storage was not requested. + optional string storage_error = 9; } // Contains data relating to an image that is provided to the model. message ImageUrlContent { - // This is either an image URL or a base64-encoded version of the image. - // The following image formats are supported: PNG, JPG, and WebP. - // If an image URL is provided, the image will be downloaded for every API - // request without being cached. Images are fetched using - // "XaiImageApiFetch/1.0" user agent, and will timeout after 5 seconds. - // The image size is limited to 10 MiB. If the image download fails, the API - // request will fail as well. - string image_url = 1; + // The source of the image — either a direct URL/base64 string or a + // file_id from the xAI Files API. Exactly one must be set. + oneof source { + // This is either an image URL or a base64-encoded version of the image. + // The following image formats are supported: PNG, JPG, and WebP. + // If an image URL is provided, the image will be downloaded for every API + // request without being cached. Images are fetched using + // "XaiImageApiFetch/1.0" user agent, and will timeout after 5 seconds. + // The image size is limited to 10 MiB. If the image download fails, the API + // request will fail as well. + string image_url = 1; + + // A file ID from the xAI Files API. The file must be an image + // (JPEG, PNG, or WebP). + string file_id = 3; + } // The level of pre-processing resolution that will be applied to the image. ImageDetail detail = 2; diff --git a/src/xAI.Protocol/usage.proto b/src/xAI.Protocol/usage.proto index e0171ac..1d0c2d7 100644 --- a/src/xAI.Protocol/usage.proto +++ b/src/xAI.Protocol/usage.proto @@ -78,4 +78,5 @@ enum ServerSideTool { SERVER_SIDE_TOOL_MCP = 7; SERVER_SIDE_TOOL_ATTACHMENT_SEARCH = 8; SERVER_SIDE_TOOL_IMAGE_SEARCH = 10; + SERVER_SIDE_TOOL_IMAGE_GENERATION = 11; } diff --git a/src/xAI.Protocol/video.proto b/src/xAI.Protocol/video.proto index b12e5be..549a59b 100644 --- a/src/xAI.Protocol/video.proto +++ b/src/xAI.Protocol/video.proto @@ -46,13 +46,41 @@ enum VideoResolution { // 720p resolution. // Dimensions vary by aspect ratio VIDEO_RESOLUTION_720P = 2; + + // 1080p resolution. + // Dimensions vary by aspect ratio. + // Supported on models that advertise 1080p (e.g. grok-imagine-video-1.5 for + // image-to-video); not available on all video models. + VIDEO_RESOLUTION_1080P = 3; } -// Specifies a video by URL for video editing. +// Specifies a video by URL or file reference for video editing. message VideoUrlContent { - // Either a URL of the video (e.g., a public URL) or a base64-encoded video - // as a data URL (e.g., "data:video/mp4;base64,..."). - string url = 1; + // The source of the video — either a direct URL/base64 string or a + // file_id from the xAI Files API. Exactly one must be set. + oneof source { + // Either a URL of the video (e.g., a public URL) or a base64-encoded video + // as a data URL (e.g., "data:video/mp4;base64,..."). + string url = 1; + + // A file ID from the xAI Files API. The file must be a video + // (e.g., MP4). + string file_id = 2; + } +} + +// Reference audio input for video generation. +message AudioUrlContent { + reserved 1; + + // The source of the audio. Must be set. + oneof source { + // Identifier of a first-party preset voice (e.g. "ara"), using the same + // voice identifiers as the TTS API. Resolved server-side to a curated + // reference clip from the model's voice-preset catalog. Only supported + // by models that accept reference audio. + string voice_id = 2; + } } // An API service for interaction with video generation models. @@ -77,14 +105,30 @@ service Video { rpc GetDeferredVideo(GetDeferredVideoRequest) returns (GetDeferredVideoResponse) {} } +// A mid-video keyframe anchor: the image appears literally at `timestamp_s` +// seconds, strictly inside the clip (endpoints use `image` / `last_frame`). +// Timestamps snap to the engine's keyframe grid (one anchor per 1/3 second), +// so anchors closer together than that are rejected. +message VideoKeyframe { + // The pinned image. Supports the same formats as image generation: URL or + // base64-encoded image. + ImageUrlContent image = 1; + + // Anchor time in seconds, strictly inside the clip (0 < t < duration). + float timestamp_s = 2; +} + // Request message for generating a video. message GenerateVideoRequest { // Input prompt to generate a video from. string prompt = 1; - // Optional input image for image-to-video generation. - // If provided, generates video with this image as the first frame. - // If omitted, generates text-to-video. + // Optional input image used as the video's exact FIRST frame. + // Alone, selects image-to-video generation. Combined with + // reference_images / reference_audios / last_frame / keyframes (select + // models only), the video generates from those references while starting + // literally on this image. If omitted, generates text-to-video (or + // reference-to-video when references are present). // Supports the same formats as image generation: URL or base64-encoded image. ImageUrlContent image = 2; @@ -108,9 +152,42 @@ message GenerateVideoRequest { optional VideoResolution resolution = 8; // Optional reference images for reference-to-video (R2V) generation. - // When provided (and `image` is not set), generates video using these images - // as style/content references. + // When provided, generates video using these images as style/content + // references. repeated ImageUrlContent reference_images = 13; + + // Optional output storage configuration. When present, the generated + // video is stored in the Files API and a file_id is returned in + // the response alongside the ephemeral URL. + optional StorageOptions storage_options = 14; + + reserved 15; + + // Optional reference audio (voice identity) for reference-to-video + // generation. Each entry selects a first-party preset voice via + // `voice_id` (same identifiers as the TTS API). Only supported by select + // video models; at most three entries. May be provided without + // reference_images (audio-only reference-to-video) — at least one + // reference of either kind selects the reference-to-video mode. + repeated AudioUrlContent reference_audios = 16; + + // Whether the generated video includes an audio track. Defaults to true. + // Set to false for a silent video (the audio track is stripped + // server-side after generation). + optional bool generate_audio = 17; + + // Optional image pinned as the video's exact LAST frame — the clip ends + // arriving on it (unlike reference_images, which condition the video and + // appear re-rendered). With image, the video interpolates between the two + // frames. Only supported by select video models. Supports URL or + // base64-encoded image. + ImageUrlContent last_frame = 18; + + // Optional mid-video keyframe anchors, strictly between the endpoint pins + // (`image` as the first frame, `last_frame` as the last). Selects + // reference-to-video like the other reference inputs. Only supported by + // select video models; at most 4 entries. + repeated VideoKeyframe keyframes = 20; } // Request for retrieving deferred video generation results. @@ -155,6 +232,15 @@ message GeneratedVideo { // The field will be true if the video respects moderation rules. Otherwise // the field will be false and the video url field will be empty. bool respect_moderation = 5; + + // Storage info for the generated video. Only present when the request + // included `storage_options` and the upload succeeded. + optional FileOutput file_output = 6; + + // Human-readable error when `storage_options` was set but the upload + // failed. Only present on storage failure; absent on success or when + // storage was not requested. + optional string storage_error = 7; } // Response from GetDeferredVideo, including the response if the video @@ -192,4 +278,11 @@ message ExtendVideoRequest { // Duration of the extension segment to generate in seconds (1-10). // Defaults to 6 seconds if not specified. optional int32 duration = 4; + + // Optional output storage configuration. When present, the generated + // video is stored in the Files API and a file_id is returned in + // the response alongside the ephemeral URL. + optional StorageOptions storage_options = 6; + + reserved 7; } diff --git a/src/xAI.Tests/ChatClientTests.cs b/src/xAI.Tests/ChatClientTests.cs index 6b7c325..40d63b4 100644 --- a/src/xAI.Tests/ChatClientTests.cs +++ b/src/xAI.Tests/ChatClientTests.cs @@ -14,6 +14,18 @@ namespace xAI.Tests; public class ChatClientTests(ITestOutputHelper output) { + [Fact] + public void GrokClientExposesFilesAndVideoProtocolClients() + { + using var client = new GrokClient("test-api-key") + { + Endpoint = new Uri($"http://localhost/{Guid.NewGuid():N}/"), + }; + + Assert.IsType(client.GetFilesClient()); + Assert.IsType(client.GetVideoClient()); + } + [SecretsFact("OPENAI_API_KEY")] public async Task OpenAIInvokesTools() { diff --git a/src/xAI.Tests/GrokConversionTests.cs b/src/xAI.Tests/GrokConversionTests.cs index 28ebd75..f870890 100644 --- a/src/xAI.Tests/GrokConversionTests.cs +++ b/src/xAI.Tests/GrokConversionTests.cs @@ -229,6 +229,20 @@ public void AsTool_WithCodeExecution() Assert.NotNull(tool?.CodeExecution); } + [Fact] + public void AsTool_WithImageGeneration() + { + var tool = new HostedImageGenerationTool(new Dictionary + { + ["action"] = "generate", + }); + + var protocolTool = tool.AsProtocolTool(); + + Assert.NotNull(protocolTool?.ImageGeneration); + Assert.Equal("generate", protocolTool.ImageGeneration.Action); + } + [Fact] public void AsTool_WithHostedFileSearchTool() { @@ -382,6 +396,18 @@ public void AsCompletionsRequest_StoreMessages_SetsStoreMessages() Assert.True(request.StoreMessages); } + [Fact] + public void AsCompletionsRequest_SafetyIdentifier_MapsWithoutChangingEndUserId() + { + var request = CreateClient().AsCompletionsRequest([], new GrokChatOptions + { + SafetyIdentifier = "hashed-user-123", + }); + + Assert.Equal("hashed-user-123", request.SafetyIdentifier); + Assert.Empty(request.User); + } + [Fact] public void AsTool_WithWebSearch_EnableImageSearch() { @@ -429,6 +455,39 @@ public void AsContents_WebSearchTool_MapsCallAndResult() Assert.Contains(result.Outputs!, x => x is TextContent text && text.Text == "search output"); } + [Fact] + public void AsContents_ImageGenerationTool_MapsCallResultAndFailure() + { + var toolCall = new ToolCall + { + Id = "image_1", + Type = ToolCallType.ImageGenerationTool, + Status = ToolCallStatus.Completed, + Function = new FunctionCall + { + Name = "image_generation", + Arguments = """{"prompt":"a red fox"}""", + }, + }; + + var contents = new[] { toolCall }.AsContents("generated image result").ToList(); + + var call = Assert.IsType(Assert.Single(contents.OfType())); + Assert.Equal("image_1", call.CallId); + Assert.Same(toolCall, call.RawRepresentation); + + var result = Assert.IsType(Assert.Single(contents.OfType())); + Assert.Equal("image_1", result.CallId); + Assert.Same(toolCall, result.RawRepresentation); + Assert.Equal("generated image result", Assert.IsType(Assert.Single(result.Outputs!)).Text); + + toolCall.Status = ToolCallStatus.Failed; + toolCall.ErrorMessage = "generation failed"; + + var failed = Assert.Single(new[] { toolCall }.AsContents().OfType()); + Assert.Equal("generation failed", Assert.IsType(Assert.Single(failed.Outputs!)).Text); + } + [Fact] public void Convert_SamplingUsage_MapsExtendedTokenCounts() { @@ -467,6 +526,6 @@ public void Convert_ReasoningEffort_MapsKnownValues() Assert.Equal(Protocol.ReasoningEffort.EffortLow, Microsoft.Extensions.AI.ReasoningEffort.Low.Convert()); Assert.Equal(Protocol.ReasoningEffort.EffortMedium, Microsoft.Extensions.AI.ReasoningEffort.Medium.Convert()); Assert.Equal(Protocol.ReasoningEffort.EffortHigh, Microsoft.Extensions.AI.ReasoningEffort.High.Convert()); - Assert.Equal(Protocol.ReasoningEffort.EffortHigh, Microsoft.Extensions.AI.ReasoningEffort.ExtraHigh.Convert()); + Assert.Equal(Protocol.ReasoningEffort.EffortXhigh, Microsoft.Extensions.AI.ReasoningEffort.ExtraHigh.Convert()); } } diff --git a/src/xAI.Tests/ImageGeneratorTests.cs b/src/xAI.Tests/ImageGeneratorTests.cs index a3bf9a8..5975760 100644 --- a/src/xAI.Tests/ImageGeneratorTests.cs +++ b/src/xAI.Tests/ImageGeneratorTests.cs @@ -309,6 +309,61 @@ await imageGenerator.GenerateAsync( Assert.Equal(ImageAspectRatio.ImgAspectRatio169, capturedRequest.AspectRatio); } + [Fact] + public async Task GenerateImage_WithQualityAndStorage_MapsRequestAndResponseMetadata() + { + GenerateImageRequest? capturedRequest = null; + var client = new Mock(MockBehavior.Strict); + client.Setup(x => x.GenerateImageAsync(It.IsAny(), null, null, CancellationToken.None)) + .Callback((req, _, _, _) => capturedRequest = req) + .Returns(CallHelpers.CreateAsyncUnaryCall(new ImageResponse + { + Images = + { + new GeneratedImage + { + Url = "https://example.com/generated.jpg", + RespectModeration = true, + FileOutput = new FileOutput + { + FileId = "file_123", + Filename = "generated.jpg", + PublicUrl = "https://files-cdn.x.ai/file_123", + }, + } + } + })); + + var imageGenerator = client.Object.AsIImageGenerator("grok-imagine-image"); + var response = await imageGenerator.GenerateAsync( + new ImageGenerationRequest("Stored output"), + new GrokImageGenerationOptions + { + Quality = ImageQuality.ImgQualityHigh, + Storage = new GrokImageStorageOptions + { + Filename = "generated.jpg", + ExpiresAfterSeconds = 3600, + CreatePublicUrl = true, + PublicUrlExpiresAfterSeconds = 1800, + }, + }); + + Assert.NotNull(capturedRequest); + Assert.True(capturedRequest.HasQuality); + Assert.Equal(ImageQuality.ImgQualityHigh, capturedRequest.Quality); + Assert.Equal("generated.jpg", capturedRequest.StorageOptions.Filename); + Assert.Equal(3600, capturedRequest.StorageOptions.ExpiresAfter); + Assert.True(capturedRequest.StorageOptions.PublicUrl.HasExpiresAfter); + Assert.Equal(1800, capturedRequest.StorageOptions.PublicUrl.ExpiresAfter); + + var content = Assert.IsType(Assert.Single(response.Contents)); + Assert.True(Assert.IsType(content.AdditionalProperties?["respect_moderation"])); + var fileOutput = Assert.IsType(content.AdditionalProperties?["file_output"]); + Assert.Equal("file_123", fileOutput.FileId); + Assert.Same(content.RawRepresentation, response.RawRepresentation is ImageResponse raw ? raw.Images[0] : null); + } + [Fact] public async Task GenerateImage_MapsProtocolUsageToResponseUsage() { diff --git a/src/xAI.Tests/SanityChecks.cs b/src/xAI.Tests/SanityChecks.cs index d85917a..a1b3bb8 100644 --- a/src/xAI.Tests/SanityChecks.cs +++ b/src/xAI.Tests/SanityChecks.cs @@ -9,6 +9,17 @@ namespace xAI.Tests; public class SanityChecks(ITestOutputHelper output) { + [Fact] + public void AddxAIProtocolRegistersFilesAndVideoClients() + { + using var services = new ServiceCollection() + .AddxAIProtocol("test-api-key") + .BuildServiceProvider(); + + Assert.NotNull(services.GetRequiredService()); + Assert.NotNull(services.GetRequiredService()); + } + [SecretsFact("CI_XAI_API_KEY")] public async Task NoEmbeddingModels() { diff --git a/src/xAI.Tests/SpeechToTextClientTests.cs b/src/xAI.Tests/SpeechToTextClientTests.cs index 4aac82d..82a26bb 100644 --- a/src/xAI.Tests/SpeechToTextClientTests.cs +++ b/src/xAI.Tests/SpeechToTextClientTests.cs @@ -56,6 +56,9 @@ public async Task GetTextAsync_MapsRequestAndResponse() Multichannel = true, Channels = 2, Diarize = true, + KeyTerms = ["xAI", "Grok Voice"], + FillerWords = true, + VadThreshold = 0.25, ModelId = "test-model", }); @@ -65,7 +68,7 @@ public async Task GetTextAsync_MapsRequestAndResponse() Assert.Equal("test-api-key", handler.Request.Headers.Authorization?.Parameter); var body = handler.RequestBody!; - AssertFieldOrder(body, "format", "language", "sample_rate", "audio_format", "multichannel", "channels", "diarize", "file"); + AssertFieldOrder(body, "format", "language", "sample_rate", "audio_format", "multichannel", "channels", "diarize", "keyterm", "filler_words", "vad_threshold", "file"); Assert.Contains("format", GetField(body, "format")); Assert.Contains("true", body); Assert.Contains("language", GetField(body, "language")); @@ -75,6 +78,13 @@ public async Task GetTextAsync_MapsRequestAndResponse() Assert.Contains("audio_format", GetField(body, "audio_format")); Assert.Contains("pcm", body); Assert.Contains("audio.mp3", body); + Assert.Equal(2, + body.Split("name=\"keyterm\"", StringSplitOptions.None).Length - 1 + + body.Split("name=keyterm", StringSplitOptions.None).Length - 1); + Assert.Contains("xAI", body); + Assert.Contains("Grok Voice", body); + Assert.Contains("filler_words", GetField(body, "filler_words")); + Assert.Contains("vad_threshold", GetField(body, "vad_threshold")); Assert.Equal("Hello world", response.Text); Assert.Null(response.ModelId); @@ -141,7 +151,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() var webSocket = new FakeWebSocket( """{"type":"transcript.created"}""", """{"type":"transcript.partial","text":"Hel","is_final":false,"speech_final":false,"start":0.0,"duration":0.4}""", - """{"type":"transcript.partial","text":"Hello","is_final":true,"speech_final":true,"start":0.0,"duration":0.8,"channel_index":1}""", + """{"type":"transcript.partial","text":"Hello","is_final":true,"speech_final":true,"start":0.0,"duration":0.8,"channel_index":1,"end_of_turn_confidence":0.8}""", """{"type":"transcript.done","text":"Hello world","duration":1.2}"""); Uri? capturedUri = null; @@ -169,6 +179,11 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() Diarize = true, Multichannel = true, Channels = 2, + KeyTerms = ["xAI", "Grok voice"], + FillerWords = true, + VadThreshold = 0.25, + SmartTurn = 0.7, + SmartTurnTimeout = 1200, ModelId = "ignored-model", })) { @@ -176,7 +191,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() } Assert.Equal("test-api-key", capturedApiKey); - Assert.Equal("wss://streaming.test/base/v1/stt?sample_rate=8000&encoding=mulaw&interim_results=true&endpointing=5&language=en&diarize=true&multichannel=true&channels=2", capturedUri!.ToString()); + Assert.Equal("wss://streaming.test/base/v1/stt?sample_rate=8000&encoding=mulaw&interim_results=true&endpointing=5&keyterm=xAI&keyterm=Grok%20voice&filler_words=true&vad_threshold=0.25&smart_turn=0.7&smart_turn_timeout=1200&language=en&diarize=true&multichannel=true&channels=2", capturedUri!.AbsoluteUri); Assert.Collection(webSocket.SentBinaryMessages, message => Assert.Equal(new byte[] { 1, 2, 3, 4 }, message)); @@ -208,6 +223,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() Assert.Null(update.ModelId); Assert.Equal("Hello", update.Text); Assert.Equal(1, update.AdditionalProperties?["channel_index"]); + Assert.Equal(0.8, update.AdditionalProperties?["end_of_turn_confidence"]); }, update => { @@ -220,6 +236,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() Assert.Equal(SpeechToTextResponseUpdateKind.SessionClose, update.Kind); Assert.Null(update.ModelId); }); + } [Fact] diff --git a/src/xAI.Tests/TextToSpeechClientTests.cs b/src/xAI.Tests/TextToSpeechClientTests.cs index 6087074..a749642 100644 --- a/src/xAI.Tests/TextToSpeechClientTests.cs +++ b/src/xAI.Tests/TextToSpeechClientTests.cs @@ -52,6 +52,8 @@ public async Task GetAudioAsync_MapsRequestAndResponse() BitRate = 192000, OptimizeStreamingLatency = 1, TextNormalization = true, + Speed = 1.2f, + Replace = new Dictionary { ["Acme Mobile"] = "Acme Mobull" }, ModelId = "test-model", }); @@ -67,6 +69,8 @@ public async Task GetAudioAsync_MapsRequestAndResponse() Assert.Equal("pt-BR", root.GetProperty("language").GetString()); Assert.Equal(1, root.GetProperty("optimize_streaming_latency").GetInt32()); Assert.True(root.GetProperty("text_normalization").GetBoolean()); + Assert.Equal(1.2, root.GetProperty("speed").GetDouble()); + Assert.Equal("Acme Mobull", root.GetProperty("replace").GetProperty("Acme Mobile").GetString()); var outputFormat = root.GetProperty("output_format"); Assert.Equal("wav", outputFormat.GetProperty("codec").GetString()); @@ -124,6 +128,39 @@ public async Task GetAudioAsync_WithDefaults_SendsRequiredFieldsOnly() Assert.False(root.TryGetProperty("output_format", out _)); } + [Fact] + public async Task GetAudioAsync_WithTimestamps_MapsAudioAndCharacterTimings() + { + var handler = new CaptureHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + """ + { + "audio": "AQID", + "audio_duration": 1.2, + "audio_timestamps": { + "graph_chars": ["H", "i"], + "graph_times": [[0.0, 0.4], [0.4, 0.8]] + } + } + """, Encoding.UTF8, "application/json"), + }); + + using var client = new GrokClient("test-api-key", CreateOptions(handler)); + using var tts = client.AsITextToSpeechClient(); + + var response = await tts.GetAudioAsync("Hi", new GrokTextToSpeechOptions { WithTimestamps = true }); + + var audio = Assert.IsType(Assert.Single(response.Contents)); + Assert.Equal(new byte[] { 1, 2, 3 }, audio.Data.ToArray()); + Assert.Equal(1.2, response.AdditionalProperties?["audio_duration"]); + var timestamps = Assert.IsType(response.AdditionalProperties?["audio_timestamps"]); + Assert.Equal("H", timestamps.GetProperty("graph_chars")[0].GetString()); + + using var request = JsonDocument.Parse(handler.RequestBody!); + Assert.True(request.RootElement.GetProperty("with_timestamps").GetBoolean()); + } + [Fact] public async Task GetAudioAsync_WithError_ThrowsHttpRequestException() { @@ -155,7 +192,8 @@ public async Task GetAudioAsync_WithNullText_ThrowsArgumentNullException() public async Task GetStreamingAudioAsync_MapsWebSocketEvents() { var webSocket = new FakeWebSocket( - """{"type":"audio.delta","delta":"AQID"}""", + """{"type":"session.updated"}""", + """{"type":"audio.delta","delta":"AQID","audio_duration":0.5,"audio_timestamps":{"graph_chars":["H"],"graph_times":[[0.0,0.5]]}}""", """{"type":"audio.done","trace_id":"trace-123"}"""); Uri? capturedUri = null; @@ -181,6 +219,9 @@ public async Task GetStreamingAudioAsync_MapsWebSocketEvents() SampleRate = 8000, OptimizeStreamingLatency = 1, TextNormalization = true, + Speed = 1.2f, + WithTimestamps = true, + Replace = new Dictionary { ["Acme"] = "Ack-me" }, ModelId = "ignored-model", })) { @@ -188,9 +229,15 @@ public async Task GetStreamingAudioAsync_MapsWebSocketEvents() } Assert.Equal("test-api-key", capturedApiKey); - Assert.Equal("wss://streaming.test/base/v1/tts?voice=ara&language=auto&codec=mulaw&sample_rate=8000&optimize_streaming_latency=1&text_normalization=true", capturedUri!.ToString()); + Assert.Equal("wss://streaming.test/base/v1/tts?voice=ara&language=auto&codec=mulaw&sample_rate=8000&optimize_streaming_latency=1&text_normalization=true&speed=1.2&with_timestamps=true", capturedUri!.AbsoluteUri); Assert.Collection(webSocket.SentMessages, + message => + { + using var json = JsonDocument.Parse(message); + Assert.Equal("session.update", json.RootElement.GetProperty("type").GetString()); + Assert.Equal("Ack-me", json.RootElement.GetProperty("replace").GetProperty("Acme").GetString()); + }, message => { using var json = JsonDocument.Parse(message); @@ -211,6 +258,7 @@ public async Task GetStreamingAudioAsync_MapsWebSocketEvents() var data = Assert.IsType(Assert.Single(update.Contents)); Assert.Equal(new byte[] { 1, 2, 3 }, data.Data.ToArray()); Assert.Equal("audio/basic", data.MediaType); + Assert.Equal(0.5, update.AdditionalProperties?["audio_duration"]); }, update => { @@ -218,6 +266,10 @@ public async Task GetStreamingAudioAsync_MapsWebSocketEvents() Assert.Null(update.ModelId); Assert.Equal("trace-123", update.AdditionalProperties?["trace_id"]); }); + + var aggregated = updates.ToTextToSpeechResponse(); + var aggregatedAudio = Assert.IsType(Assert.Single(aggregated.Contents)); + Assert.Equal(new byte[] { 1, 2, 3 }, aggregatedAudio.Data.ToArray()); } [Fact] diff --git a/src/xAI/GrokChatOptions.cs b/src/xAI/GrokChatOptions.cs index db0a59d..73d11c5 100644 --- a/src/xAI/GrokChatOptions.cs +++ b/src/xAI/GrokChatOptions.cs @@ -37,4 +37,11 @@ public class GrokChatOptions : ChatOptions /// / previous_response_id. /// public bool StoreMessages { get; set; } + + /// + /// A stable, non-sensitive identifier for the end user, used by xAI to attribute + /// usage-policy violations. Hash an internal user identifier; do not provide + /// names, email addresses, or other personal information. + /// + public string? SafetyIdentifier { get; set; } } diff --git a/src/xAI/GrokClient.cs b/src/xAI/GrokClient.cs index a2c627c..fee41a2 100644 --- a/src/xAI/GrokClient.cs +++ b/src/xAI/GrokClient.cs @@ -50,9 +50,15 @@ internal GrokClient(ChannelBase channel, GrokClientOptions options, string? apiK /// Gets a new instance of that reuses the client configuration details provided to the instance. public Image.ImageClient GetImageClient() => new(ChannelHandler, Options); + /// Gets a new instance of that reuses the client configuration details provided to the instance. + public Files.FilesClient GetFilesClient() => new(ChannelHandler); + /// Gets a new instance of that reuses the client configuration details provided to the instance. public Models.ModelsClient GetModelsClient() => new(ChannelHandler); + /// Gets a new instance of that reuses the client configuration details provided to the instance. + public Video.VideoClient GetVideoClient() => new(ChannelHandler); + /// Gets a new instance of that reuses the client configuration details provided to the instance. public Tokenize.TokenizeClient GetTokenizeClient() => new(ChannelHandler); diff --git a/src/xAI/GrokImageGenerationOptions.cs b/src/xAI/GrokImageGenerationOptions.cs index 4946a59..84cbfd4 100644 --- a/src/xAI/GrokImageGenerationOptions.cs +++ b/src/xAI/GrokImageGenerationOptions.cs @@ -25,4 +25,27 @@ public class GrokImageGenerationOptions : ImageGenerationOptions /// This option is only supported by grok-imagine models. /// public ImageResolution? Resolution { get; set; } + + /// Optional image quality setting. + public ImageQuality? Quality { get; set; } + + /// Optional configuration to store generated image files in the xAI Files API. + public GrokImageStorageOptions? Storage { get; set; } +} + +/// Configures storage for image generation outputs in the xAI Files API. +public sealed class GrokImageStorageOptions +{ + /// Gets or sets the filename for the stored output. + public string? Filename { get; set; } + + /// Gets or sets the number of seconds before the stored file expires. + public long? ExpiresAfterSeconds { get; set; } + + /// Gets or sets the number of seconds before its optional public URL expires. + /// Set to to create a non-expiring public URL when is true. + public long? PublicUrlExpiresAfterSeconds { get; set; } + + /// Gets or sets a value indicating whether to create a public URL for the stored file. + public bool CreatePublicUrl { get; set; } } diff --git a/src/xAI/GrokImageGenerator.cs b/src/xAI/GrokImageGenerator.cs index 02103d2..e690de4 100644 --- a/src/xAI/GrokImageGenerator.cs +++ b/src/xAI/GrokImageGenerator.cs @@ -11,21 +11,6 @@ namespace xAI; /// sealed class GrokImageGenerator : IImageGenerator { - // add inverted dictionary for extension to mime type if needed in future - static readonly Dictionary extensionToMimeType = new(StringComparer.OrdinalIgnoreCase) - { - [".png"] = "image/png", - [".jpg"] = "image/jpeg", - [".jpeg"] = "image/jpeg", - [".webp"] = "image/webp", - [".gif"] = "image/gif", - [".bmp"] = "image/bmp", - [".tiff"] = "image/tiff", - }; - - const string DefaultInputContentType = "image/png"; - const string DefaultOutputContentType = "image/jpeg"; - readonly ImageGeneratorMetadata metadata; readonly ImageClient imageClient; readonly GrokClientOptions clientOptions; @@ -56,52 +41,9 @@ public async Task GenerateAsync( ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) { - var protocolRequest = new GenerateImageRequest - { - Prompt = Throw.IfNull(Throw.IfNull(request).Prompt, "request.Prompt"), - Model = options?.ModelId ?? defaultModelId, - }; - - if (clientOptions.EndUserId is { } user) - protocolRequest.User = clientOptions.EndUserId; - - if (options?.Count is { } count) - protocolRequest.N = count; - - protocolRequest.Format = (options?.ResponseFormat ?? ImageGenerationResponseFormat.Uri) switch - { - ImageGenerationResponseFormat.Uri => ImageFormat.ImgFormatUrl, - ImageGenerationResponseFormat.Data => ImageFormat.ImgFormatBase64, - _ => throw new ArgumentException($"Unsupported response format: {options?.ResponseFormat}", nameof(options)) - }; - - if (options is GrokImageGenerationOptions grokOptions) - { - if (grokOptions.AspectRatio is { } aspectRatio) protocolRequest.AspectRatio = aspectRatio; - if (grokOptions.Resolution is { } resolution) protocolRequest.Resolution = resolution; - } - - // Handle image editing if original images are provided - if (request.OriginalImages?.ToList() is { Count: > 0 } originalImages) - { - if (originalImages.Count == 1) - { - if (MapToImageUrlContent(originalImages[0]) is { } image) - protocolRequest.Image = image; - } - else - { - foreach (var originalImage in originalImages) - { - if (MapToImageUrlContent(originalImage) is { } image) - protocolRequest.Images.Add(image); - } - } - } - + var protocolRequest = request.AsProtocolImageRequest(options, defaultModelId, clientOptions.EndUserId); var response = await imageClient.GenerateImageAsync(protocolRequest, cancellationToken: cancellationToken).ConfigureAwait(false); - - return ToImageGenerationResponse(response); + return response.AsImageGenerationResponse(); } /// @@ -115,78 +57,4 @@ public async Task GenerateAsync( /// void IDisposable.Dispose() { } - /// - /// Converts an xAI to a . - /// - static ImageGenerationResponse ToImageGenerationResponse(ImageResponse response) - { - var contents = new List(); - var contentType = DefaultOutputContentType; - - foreach (var image in response.Images) - { - switch (image.ImageCase) - { - case GeneratedImage.ImageOneofCase.Base64: - { - try - { - // RTW grok-imagine-image uses full data URI, so - // this first try should work. - contents.Add(new DataContent(image.Base64)); - } - catch (Exception) - { - // Fallback to attemping to parse as raw base64 string from beta and grok2 model. - // We assume JPEG since there's no way to get the actual content type. - var imageBytes = Convert.FromBase64String(image.Base64); - contents.Add(new DataContent(imageBytes, contentType)); - } - break; - } - case GeneratedImage.ImageOneofCase.Url: - { - if (Path.GetExtension(image.Url) is { } extension && extensionToMimeType.TryGetValue(extension, out var mimeType)) - contentType = mimeType; - - contents.Add(new UriContent(new Uri(image.Url), contentType)); - break; - } - default: - throw new InvalidOperationException("Generated image does not contain a valid URL or base64 data."); - } - } - - return new ImageGenerationResponse(contents) - { - RawRepresentation = response, - Usage = MapToUsage(response.Usage), - }; - } - - static ImageUrlContent? MapToImageUrlContent(AIContent content) => content switch - { - DataContent dataContent => MapToImageUrlContent(dataContent), - UriContent uriContent => new ImageUrlContent { ImageUrl = uriContent.Uri.ToString() }, - _ => throw new ArgumentException($"Unsupported original image content type: {content.GetType()}", nameof(content)), - }; - - static ImageUrlContent? MapToImageUrlContent(DataContent dataContent) - { - var imageUrl = dataContent.Uri?.ToString(); - if (imageUrl == null && dataContent.Data.Length > 0) - imageUrl = $"data:{dataContent.MediaType ?? DefaultInputContentType};base64,{Convert.ToBase64String(dataContent.Data.ToArray())}"; - - return imageUrl == null ? null : new ImageUrlContent - { - ImageUrl = imageUrl - }; - } - - static UsageDetails? MapToUsage(SamplingUsage usage) => usage == null ? null : new() - { - InputTokenCount = usage.PromptTokens, - OutputTokenCount = usage.CompletionTokens, - TotalTokenCount = usage.TotalTokens - }; } diff --git a/src/xAI/GrokImageProtocolExtensions.cs b/src/xAI/GrokImageProtocolExtensions.cs new file mode 100644 index 0000000..08f57c9 --- /dev/null +++ b/src/xAI/GrokImageProtocolExtensions.cs @@ -0,0 +1,156 @@ +using Microsoft.Extensions.AI; +using xAI.Protocol; + +namespace xAI; + +public static partial class GrokProtocolExtensions +{ + static readonly Dictionary imageExtensionToMimeType = new(StringComparer.OrdinalIgnoreCase) + { + [".png"] = "image/png", + [".jpg"] = "image/jpeg", + [".jpeg"] = "image/jpeg", + [".webp"] = "image/webp", + [".gif"] = "image/gif", + [".bmp"] = "image/bmp", + [".tiff"] = "image/tiff", + }; + + internal static GenerateImageRequest AsProtocolImageRequest( + this ImageGenerationRequest request, + ImageGenerationOptions? options, + string defaultModelId, + string? endUserId) + { + Throw.IfNull(request); + + var protocolRequest = new GenerateImageRequest + { + Prompt = Throw.IfNull(request.Prompt, "request.Prompt"), + Model = options?.ModelId ?? defaultModelId, + }; + + if (endUserId is not null) + protocolRequest.User = endUserId; + + if (options?.Count is { } count) + protocolRequest.N = count; + + protocolRequest.Format = (options?.ResponseFormat ?? ImageGenerationResponseFormat.Uri) switch + { + ImageGenerationResponseFormat.Uri => ImageFormat.ImgFormatUrl, + ImageGenerationResponseFormat.Data => ImageFormat.ImgFormatBase64, + _ => throw new ArgumentException($"Unsupported response format: {options?.ResponseFormat}", nameof(options)) + }; + + if (options is GrokImageGenerationOptions grokOptions) + { + if (grokOptions.AspectRatio is { } aspectRatio) + protocolRequest.AspectRatio = aspectRatio; + if (grokOptions.Resolution is { } resolution) + protocolRequest.Resolution = resolution; + if (grokOptions.Quality is { } quality) + protocolRequest.Quality = quality; + + if (grokOptions.Storage is { } storage) + { + protocolRequest.StorageOptions = new() + { + Filename = storage.Filename ?? "", + }; + + if (storage.ExpiresAfterSeconds is long fileExpiry) + protocolRequest.StorageOptions.ExpiresAfter = fileExpiry; + + if (storage.CreatePublicUrl || storage.PublicUrlExpiresAfterSeconds is not null) + { + protocolRequest.StorageOptions.PublicUrl = new(); + if (storage.PublicUrlExpiresAfterSeconds is long urlExpiry) + protocolRequest.StorageOptions.PublicUrl.ExpiresAfter = urlExpiry; + } + } + } + + if (request.OriginalImages?.ToList() is { Count: > 0 } originalImages) + { + if (originalImages.Count == 1) + { + if (MapToImageUrlContent(originalImages[0]) is { } image) + protocolRequest.Image = image; + } + else + { + foreach (var originalImage in originalImages) + if (MapToImageUrlContent(originalImage) is { } image) + protocolRequest.Images.Add(image); + } + } + + return protocolRequest; + } + + internal static ImageGenerationResponse AsImageGenerationResponse(this ImageResponse response) + { + var contents = new List(); + + foreach (var image in response.Images) + { + AIContent content = image.ImageCase switch + { + GeneratedImage.ImageOneofCase.Base64 => CreateGeneratedImageDataContent(image.Base64), + GeneratedImage.ImageOneofCase.Url => new UriContent( + new Uri(image.Url), + Path.GetExtension(image.Url) is { } extension && imageExtensionToMimeType.TryGetValue(extension, out var mimeType) ? mimeType : "image/jpeg"), + _ => throw new InvalidOperationException("Generated image does not contain a valid URL or base64 data."), + }; + + content.RawRepresentation = image; + content.AdditionalProperties = new() + { + ["respect_moderation"] = image.RespectModeration, + }; + + if (image.FileOutput is not null) + content.AdditionalProperties["file_output"] = image.FileOutput; + if (!string.IsNullOrEmpty(image.StorageError)) + content.AdditionalProperties["storage_error"] = image.StorageError; + + contents.Add(content); + } + + return new ImageGenerationResponse(contents) + { + RawRepresentation = response, + Usage = response.Usage?.Convert(), + }; + } + + static DataContent CreateGeneratedImageDataContent(string imageData) + { + try + { + // New Imagine responses may use a data URI; older models return raw base64. + return new DataContent(imageData); + } + catch (FormatException) + { + return new DataContent(System.Convert.FromBase64String(imageData), "image/jpeg"); + } + } + + static ImageUrlContent? MapToImageUrlContent(AIContent content) => content switch + { + DataContent dataContent => MapToImageUrlContent(dataContent), + UriContent uriContent => new ImageUrlContent { ImageUrl = uriContent.Uri.ToString() }, + _ => throw new ArgumentException($"Unsupported original image content type: {content.GetType()}", nameof(content)), + }; + + static ImageUrlContent? MapToImageUrlContent(DataContent dataContent) + { + var imageUrl = dataContent.Uri?.ToString(); + if (imageUrl is null && dataContent.Data.Length > 0) + imageUrl = $"data:{dataContent.MediaType ?? "image/png"};base64,{System.Convert.ToBase64String(dataContent.Data.ToArray())}"; + + return imageUrl is null ? null : new ImageUrlContent { ImageUrl = imageUrl }; + } +} diff --git a/src/xAI/GrokProtocolExtensions.cs b/src/xAI/GrokProtocolExtensions.cs index 22ef473..3c630e7 100644 --- a/src/xAI/GrokProtocolExtensions.cs +++ b/src/xAI/GrokProtocolExtensions.cs @@ -112,6 +112,13 @@ grokSearch.City is not null || case HostedCodeInterpreterTool: return new Tool { CodeExecution = new() }; + case HostedImageGenerationTool imageGenerationTool: + var imageGeneration = new ImageGeneration(); + if (imageGenerationTool.AdditionalProperties.TryGetValue("action", out var action)) + imageGeneration.Action = action as string ?? throw new ArgumentException("The image-generation tool action must be a string.", nameof(tool)); + + return new Tool { ImageGeneration = imageGeneration }; + case HostedFileSearchTool fileSearch: var collectionTool = new CollectionsSearch(); @@ -393,6 +400,16 @@ codeResult.RawRepresentation is ToolCall codeToolCall && Content = { new Content { Text = ConcatTextOutputs(webSearchResult.Outputs) ?? " " } } }); } + else if (content is ImageGenerationToolResultContent imageGenerationResult && + imageGenerationResult.RawRepresentation is ToolCall imageGenerationToolCall) + { + request.Messages.Add(new Message + { + Role = MessageRole.RoleTool, + ToolCalls = { imageGenerationToolCall }, + Content = { new Content { Text = ConcatTextOutputs(imageGenerationResult.Outputs) ?? " " } } + }); + } } if (gmsg.Content.Count == 0 && gmsg.ToolCalls.Count == 0) @@ -404,6 +421,8 @@ codeResult.RawRepresentation is ToolCall codeToolCall && if (options is GrokChatOptions grokOptions) { request.Include.AddRange(grokOptions.Include); + if (grokOptions.SafetyIdentifier is { } safetyIdentifier) + request.SafetyIdentifier = safetyIdentifier; if (grokOptions.Search.HasFlag(GrokSearch.X)) { @@ -471,6 +490,21 @@ internal static IEnumerable AsContents(this IEnumerable too }; break; + case ToolCallType.ImageGenerationTool: + yield return new ImageGenerationToolCallContent(toolCall.Id) + { + Annotations = annotations, + RawRepresentation = toolCall, + }; + if (content is not null || !string.IsNullOrEmpty(toolCall.ErrorMessage)) + yield return new ImageGenerationToolResultContent(toolCall.Id) + { + Annotations = annotations, + RawRepresentation = toolCall, + Outputs = [new TextContent(content ?? toolCall.ErrorMessage!)], + }; + break; + case ToolCallType.WebSearchTool: case ToolCallType.XSearchTool: yield return new WebSearchToolCallContent(toolCall.Id) @@ -706,8 +740,7 @@ void AddCount(string name, long value) Microsoft.Extensions.AI.ReasoningEffort.Low => Protocol.ReasoningEffort.EffortLow, Microsoft.Extensions.AI.ReasoningEffort.Medium => Protocol.ReasoningEffort.EffortMedium, Microsoft.Extensions.AI.ReasoningEffort.High => Protocol.ReasoningEffort.EffortHigh, - // xAI does not expose an extra-high tier; map to the strongest available effort. - Microsoft.Extensions.AI.ReasoningEffort.ExtraHigh => Protocol.ReasoningEffort.EffortHigh, + Microsoft.Extensions.AI.ReasoningEffort.ExtraHigh => Protocol.ReasoningEffort.EffortXhigh, _ => Protocol.ReasoningEffort.InvalidEffort, }; diff --git a/src/xAI/GrokSpeechToTextClient.cs b/src/xAI/GrokSpeechToTextClient.cs index c34106f..37c4006 100644 --- a/src/xAI/GrokSpeechToTextClient.cs +++ b/src/xAI/GrokSpeechToTextClient.cs @@ -197,6 +197,16 @@ static MultipartFormDataContent CreateMultipartContent(Stream audioSpeechStream, if (grokOptions?.Diarize is bool diarize) content.Add(new StringContent(diarize ? "true" : "false"), "diarize"); + if (grokOptions?.KeyTerms is { } keyTerms) + foreach (var keyTerm in keyTerms) + content.Add(new StringContent(keyTerm), "keyterm"); + + if (grokOptions?.FillerWords is bool fillerWords) + content.Add(new StringContent(fillerWords ? "true" : "false"), "filler_words"); + + if (grokOptions?.VadThreshold is double vadThreshold) + content.Add(new StringContent(vadThreshold.ToString(CultureInfo.InvariantCulture)), "vad_threshold"); + var filename = GetFilename(audioSpeechStream); var streamContent = new StreamContent(audioSpeechStream); streamContent.Headers.ContentType = new MediaTypeHeaderValue(GetMediaType(filename)); @@ -222,6 +232,22 @@ Uri GetStreamingEndpoint(SpeechToTextOptions? options) if (grokOptions?.Endpointing is int endpointing) query["endpointing"] = endpointing.ToString(CultureInfo.InvariantCulture); + if (grokOptions?.KeyTerms is { } keyTerms) + foreach (var keyTerm in keyTerms) + query.Add("keyterm", keyTerm); + + if (grokOptions?.FillerWords is bool fillerWords) + query["filler_words"] = fillerWords ? "true" : "false"; + + if (grokOptions?.VadThreshold is double vadThreshold) + query["vad_threshold"] = vadThreshold.ToString(CultureInfo.InvariantCulture); + + if (grokOptions?.SmartTurn is double smartTurn) + query["smart_turn"] = smartTurn.ToString(CultureInfo.InvariantCulture); + + if (grokOptions?.SmartTurnTimeout is int smartTurnTimeout) + query["smart_turn_timeout"] = smartTurnTimeout.ToString(CultureInfo.InvariantCulture); + if (GetLanguage(options) is { } language) query["language"] = language; @@ -302,6 +328,7 @@ static SpeechToTextResponseUpdate CreateTextUpdate( AddProperty(ref properties, "is_final", GetBoolean(root, "is_final")); AddProperty(ref properties, "speech_final", GetBoolean(root, "speech_final")); AddProperty(ref properties, "duration", TryGetDouble(root, "duration")); + AddProperty(ref properties, "end_of_turn_confidence", TryGetDouble(root, "end_of_turn_confidence")); return properties; } @@ -368,16 +395,16 @@ static string ToQueryString(NameValueCollection query) foreach (string key in query) { - if (query[key] is not { } value) - continue; - - if (builder.Length > 0) - builder.Append('&'); + foreach (var value in query.GetValues(key) ?? []) + { + if (builder.Length > 0) + builder.Append('&'); - builder - .Append(Uri.EscapeDataString(key)) - .Append('=') - .Append(Uri.EscapeDataString(value)); + builder + .Append(Uri.EscapeDataString(key)) + .Append('=') + .Append(Uri.EscapeDataString(value)); + } } return builder.ToString(); diff --git a/src/xAI/GrokSpeechToTextOptions.cs b/src/xAI/GrokSpeechToTextOptions.cs index 649091b..d0da864 100644 --- a/src/xAI/GrokSpeechToTextOptions.cs +++ b/src/xAI/GrokSpeechToTextOptions.cs @@ -25,8 +25,13 @@ protected GrokSpeechToTextOptions(GrokSpeechToTextOptions? other) Multichannel = other.Multichannel; Channels = other.Channels; Diarize = other.Diarize; + KeyTerms = other.KeyTerms is null ? null : [.. other.KeyTerms]; + FillerWords = other.FillerWords; + VadThreshold = other.VadThreshold; InterimResults = other.InterimResults; Endpointing = other.Endpointing; + SmartTurn = other.SmartTurn; + SmartTurnTimeout = other.SmartTurnTimeout; } /// Gets or sets a value indicating whether xAI should apply inverse text normalization to the transcript. @@ -44,12 +49,27 @@ protected GrokSpeechToTextOptions(GrokSpeechToTextOptions? other) /// Gets or sets a value indicating whether xAI should include speaker diarization data. public bool? Diarize { get; set; } + /// Gets or sets key terms that bias xAI transcription toward product names and other proper nouns. + public IList? KeyTerms { get; set; } + + /// Gets or sets a value indicating whether filler words should be included in transcripts. + public bool? FillerWords { get; set; } + + /// Gets or sets the xAI voice-activity threshold, from 0.0 to 1.0. + public double? VadThreshold { get; set; } + /// Gets or sets a value indicating whether xAI streaming should emit interim partial transcripts. public bool? InterimResults { get; set; } /// Gets or sets the silence duration in milliseconds before xAI emits an utterance-final event. public int? Endpointing { get; set; } + /// Gets or sets the Smart Turn confidence threshold for streaming transcription. + public double? SmartTurn { get; set; } + + /// Gets or sets the maximum silence duration before Smart Turn forces an utterance-final event. + public int? SmartTurnTimeout { get; set; } + /// public override SpeechToTextOptions Clone() => new GrokSpeechToTextOptions(this); } diff --git a/src/xAI/GrokTextToSpeechClient.cs b/src/xAI/GrokTextToSpeechClient.cs index f2070bd..c8f3232 100644 --- a/src/xAI/GrokTextToSpeechClient.cs +++ b/src/xAI/GrokTextToSpeechClient.cs @@ -63,6 +63,18 @@ public async Task GetAudioAsync( var audio = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); var mediaType = response.Content.Headers.ContentType?.MediaType ?? GetMediaType(request.OutputFormat?.Codec); + AdditionalPropertiesDictionary? additionalProperties = null; + + if (request.WithTimestamps == true) + { + using var json = JsonDocument.Parse(audio); + var root = json.RootElement; + var encodedAudio = root.GetProperty("audio").GetString() + ?? throw new InvalidOperationException("xAI TTS timestamp response did not contain audio data."); + audio = Convert.FromBase64String(encodedAudio); + additionalProperties = CreateTimestampProperties(root); + mediaType = GetMediaType(request.OutputFormat?.Codec); + } var raw = new HttpResponseMessage(response.StatusCode); foreach (var header in response.Headers) @@ -73,6 +85,7 @@ public async Task GetAudioAsync( return new TextToSpeechResponse([new DataContent(audio, mediaType)]) { RawRepresentation = raw, + AdditionalProperties = additionalProperties, }; } @@ -85,6 +98,9 @@ public async IAsyncEnumerable GetStreamingAudioAsync var request = CreateRequest(Throw.IfNull(text), options); using var webSocket = await webSocketFactory(GetStreamingEndpoint(request), apiKey, cancellationToken).ConfigureAwait(false); + if (request.Replace is { Count: > 0 } replacements) + await SendJsonAsync(webSocket, new SessionUpdateMessage(replacements), JsonContext.Default.SessionUpdateMessage, cancellationToken).ConfigureAwait(false); + await SendJsonAsync(webSocket, new TextDeltaMessage(text), JsonContext.Default.TextDeltaMessage, cancellationToken).ConfigureAwait(false); await SendJsonAsync(webSocket, TextDoneMessage.Instance, JsonContext.Default.TextDoneMessage, cancellationToken).ConfigureAwait(false); @@ -104,9 +120,13 @@ public async IAsyncEnumerable GetStreamingAudioAsync Kind = TextToSpeechResponseUpdateKind.AudioUpdating, Contents = [new DataContent(audio, GetMediaType(request.OutputFormat?.Codec))], RawRepresentation = rawRepresentation, + AdditionalProperties = CreateTimestampProperties(root), }; break; + case "session.updated": + break; + case "audio.done": var update = new TextToSpeechResponseUpdate { @@ -164,7 +184,10 @@ static GrokTextToSpeechRequest CreateRequest(string text, TextToSpeechOptions? o options?.Language ?? DefaultLanguage, outputFormat, grokOptions?.OptimizeStreamingLatency, - grokOptions?.TextNormalization); + grokOptions?.TextNormalization, + options?.Speed, + grokOptions?.WithTimestamps, + grokOptions?.Replace); } Uri GetHttpEndpoint() => GetEndpoint(endpoint, "https", "v1/tts", null); @@ -190,6 +213,12 @@ Uri GetStreamingEndpoint(GrokTextToSpeechRequest request) if (request.TextNormalization is bool textNormalization) query["text_normalization"] = textNormalization ? "true" : "false"; + if (request.Speed is float speed) + query["speed"] = speed.ToString(System.Globalization.CultureInfo.InvariantCulture); + + if (request.WithTimestamps is bool withTimestamps) + query["with_timestamps"] = withTimestamps ? "true" : "false"; + return GetEndpoint(endpoint, endpoint.Scheme == Uri.UriSchemeHttp ? "ws" : "wss", "v1/tts", query); } @@ -227,6 +256,19 @@ static string ToQueryString(NameValueCollection query) return builder.ToString(); } + static AdditionalPropertiesDictionary? CreateTimestampProperties(JsonElement root) + { + AdditionalPropertiesDictionary? properties = null; + + if (root.TryGetProperty("audio_timestamps", out var timestamps)) + (properties ??= [])["audio_timestamps"] = timestamps.Clone(); + + if (root.TryGetProperty("audio_duration", out var duration) && duration.ValueKind == JsonValueKind.Number) + (properties ??= [])["audio_duration"] = duration.GetDouble(); + + return properties; + } + static string GetCodec(string? format) => format?.ToUpperInvariant() switch { null or "" => DefaultCodec, @@ -319,10 +361,12 @@ static string GetRequiredString(JsonElement json, string propertyName) [JsonSerializable(typeof(GrokTextToSpeechRequest))] [JsonSerializable(typeof(TextDeltaMessage))] [JsonSerializable(typeof(TextDoneMessage))] + [JsonSerializable(typeof(SessionUpdateMessage))] partial class JsonContext : JsonSerializerContext { } sealed record GrokTextToSpeechRequest(string Text, string VoiceId, string Language, - GrokTextToSpeechOutputFormat? OutputFormat, int? OptimizeStreamingLatency, bool? TextNormalization); + GrokTextToSpeechOutputFormat? OutputFormat, int? OptimizeStreamingLatency, bool? TextNormalization, + float? Speed, bool? WithTimestamps, Dictionary? Replace); sealed record GrokTextToSpeechOutputFormat(string Codec, int? SampleRate, int? BitRate); @@ -337,4 +381,9 @@ sealed record TextDoneMessage public string Type => "text.done"; } + + sealed record SessionUpdateMessage(Dictionary Replace) + { + public string Type => "session.update"; + } } diff --git a/src/xAI/GrokTextToSpeechOptions.cs b/src/xAI/GrokTextToSpeechOptions.cs index 3d72a5a..a4969fb 100644 --- a/src/xAI/GrokTextToSpeechOptions.cs +++ b/src/xAI/GrokTextToSpeechOptions.cs @@ -25,6 +25,8 @@ protected GrokTextToSpeechOptions(GrokTextToSpeechOptions? other) BitRate = other.BitRate; OptimizeStreamingLatency = other.OptimizeStreamingLatency; TextNormalization = other.TextNormalization; + WithTimestamps = other.WithTimestamps; + Replace = other.Replace is null ? null : new(other.Replace); } /// Gets or sets the output sample rate in Hz. @@ -39,6 +41,12 @@ protected GrokTextToSpeechOptions(GrokTextToSpeechOptions? other) /// Gets or sets a value indicating whether xAI should normalize written-form text before synthesis. public bool? TextNormalization { get; set; } + /// Gets or sets a value indicating whether to return character-level audio timestamps. + public bool? WithTimestamps { get; set; } + + /// Gets or sets substitutions applied to phrases before speech synthesis. + public Dictionary? Replace { get; set; } + /// public override TextToSpeechOptions Clone() => new GrokTextToSpeechOptions(this); } diff --git a/src/xAI/xAI.csproj b/src/xAI/xAI.csproj index 38574ed..0c6c487 100644 --- a/src/xAI/xAI.csproj +++ b/src/xAI/xAI.csproj @@ -18,7 +18,7 @@ - + From 4f58fc3f38bc85ac195debbaf0beff625b2118e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:18:18 +0000 Subject: [PATCH 3/6] Support experimental tool-call streaming fragments Co-authored-by: kzu <169707+kzu@users.noreply.github.com> --- AGENTS.md | 1 + readme.md | 6 +++++ src/xAI.Tests/GrokConversionTests.cs | 34 ++++++++++++++++++++++++++++ src/xAI/GrokProtocolExtensions.cs | 2 +- 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 0ef819f..cc3c0ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ - Web/X search tool calls map to MEAI `WebSearchToolCallContent` / `WebSearchToolResultContent` (queries from tool arguments when present; citation URLs become `UriContent` outputs). - The SDK pins stable `Microsoft.Extensions.AI.Abstractions` 10.10.1; its `HostedImageGenerationTool`, `ImageGenerationToolCallContent`, and `ImageGenerationToolResultContent` are used for xAI's chat image-generation tool. The optional xAI `action` is passed through `HostedImageGenerationTool.AdditionalProperties["action"]`. - `GrokImageGenerationOptions` maps image quality and Files API storage/public-URL settings to image protocol fields. Per-image MEAI content keeps the generated protocol image as `RawRepresentation` and exposes moderation, file output, and storage errors in `AdditionalProperties`. +- xAI's experimental client-side tool-call streaming is opt-in through `GrokChatOptions.Include`. Incremental entries carry an optional `index` and may contain incomplete JSON; preserve them as raw `FunctionCallContent` fragments and never parse partial arguments as completed JSON. The upstream protocol warns against combining this option with server-side tools. - `GrokChatOptions.SafetyIdentifier` maps to xAI's separate `safety_identifier` request field. Keep it distinct from MEAI's end-user `user` mapping; callers should hash stable user IDs and avoid personal information. - MEAI `ReasoningEffort.ExtraHigh` maps to xAI's `EFFORT_XHIGH`. Newly synced Files and Video protocol services are exposed by `GrokClient.GetFilesClient()` / `GetVideoClient()` and `AddxAIProtocol`; no MEAI file/video abstraction is inferred from the protocol. - TTS uses MEAI 10.10.1's native `TextToSpeechOptions.Speed`; `GrokTextToSpeechOptions` adds phrase replacement and optional character timestamps, surfaced as response/update additional properties. STT maps key terms, filler words, VAD threshold, and streaming Smart Turn options; arbitrary Opus packetization is not exposed because the current stream contract cannot preserve packet boundaries. diff --git a/readme.md b/readme.md index 1260031..58d5dff 100644 --- a/readme.md +++ b/readme.md @@ -358,6 +358,12 @@ var options = new GrokChatOptions Learn more about [Remote MCP tools](https://docs.x.ai/docs/guides/tools/remote-mcp-tools). +For experimental incremental client-side tool calls, add +`xAI.Protocol.IncludeOption.ToolCallStreaming` to `GrokChatOptions.Include`. +Intermediate updates expose each raw fragment (including its call index); +completed updates contain the full function name and arguments. xAI currently +documents this mode as unsupported with server-side tools. + For abuse attribution, set `SafetyIdentifier` to a stable hashed identifier instead of sending an email address, name, or other personal information: diff --git a/src/xAI.Tests/GrokConversionTests.cs b/src/xAI.Tests/GrokConversionTests.cs index f870890..12f6feb 100644 --- a/src/xAI.Tests/GrokConversionTests.cs +++ b/src/xAI.Tests/GrokConversionTests.cs @@ -219,6 +219,40 @@ public void AsTool_WithFunctionTool() Assert.Equal("Description", tool.Function.Description); } + [Fact] + public void AsContents_InProgressFunctionCall_PreservesRawFragmentWithoutParsingPartialJson() + { + var toolCall = new ToolCall + { + Id = "call_1", + Type = ToolCallType.ClientSideTool, + Index = 0, + Function = new FunctionCall + { + Name = "lookup_weather", + Arguments = """{"city":""", + }, + }; + + var content = Assert.IsType(Assert.Single(new[] { toolCall }.AsContents())); + + Assert.Equal("call_1", content.CallId); + Assert.Equal("lookup_weather", content.Name); + Assert.Null(content.Arguments); + Assert.Same(toolCall, content.RawRepresentation); + } + + [Fact] + public void AsCompletionsRequest_ToolCallStreamingInclude_IsPassedThrough() + { + var request = CreateClient().AsCompletionsRequest([], new GrokChatOptions + { + Include = { IncludeOption.ToolCallStreaming }, + }); + + Assert.Contains(IncludeOption.ToolCallStreaming, request.Include); + } + [Fact] public void AsTool_WithCodeExecution() { diff --git a/src/xAI/GrokProtocolExtensions.cs b/src/xAI/GrokProtocolExtensions.cs index 3c630e7..c5fb7b3 100644 --- a/src/xAI/GrokProtocolExtensions.cs +++ b/src/xAI/GrokProtocolExtensions.cs @@ -481,7 +481,7 @@ internal static IEnumerable AsContents(this IEnumerable too yield return new FunctionCallContent( toolCall.Id, toolCall.Function.Name, - !string.IsNullOrEmpty(toolCall.Function.Arguments) + !toolCall.HasIndex && !string.IsNullOrEmpty(toolCall.Function.Arguments) ? JsonSerializer.Deserialize>(toolCall.Function.Arguments) : null) { From 9026bb9674bbe76512f4296a2328e2ea91be1186 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:38:40 +0000 Subject: [PATCH 4/6] Add realtime speech and safe STT streaming adapters Co-authored-by: kzu <169707+kzu@users.noreply.github.com> --- AGENTS.md | 3 +- readme.md | 38 ++ src/xAI.Tests/RealtimeClientTests.cs | 234 +++++++++++ src/xAI.Tests/SpeechToTextClientTests.cs | 44 ++- src/xAI/GrokClientExtensions.cs | 4 + src/xAI/GrokRealtimeClient.cs | 317 +++++++++++++++ src/xAI/GrokRealtimeOptions.cs | 25 ++ src/xAI/GrokRealtimeProtocolExtensions.cs | 449 ++++++++++++++++++++++ src/xAI/GrokSpeechToTextClient.cs | 95 ++++- 9 files changed, 1191 insertions(+), 18 deletions(-) create mode 100644 src/xAI.Tests/RealtimeClientTests.cs create mode 100644 src/xAI/GrokRealtimeClient.cs create mode 100644 src/xAI/GrokRealtimeOptions.cs create mode 100644 src/xAI/GrokRealtimeProtocolExtensions.cs diff --git a/AGENTS.md b/AGENTS.md index cc3c0ab..149db24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,8 @@ - `GrokChatOptions.SafetyIdentifier` maps to xAI's separate `safety_identifier` request field. Keep it distinct from MEAI's end-user `user` mapping; callers should hash stable user IDs and avoid personal information. - MEAI `ReasoningEffort.ExtraHigh` maps to xAI's `EFFORT_XHIGH`. Newly synced Files and Video protocol services are exposed by `GrokClient.GetFilesClient()` / `GetVideoClient()` and `AddxAIProtocol`; no MEAI file/video abstraction is inferred from the protocol. - TTS uses MEAI 10.10.1's native `TextToSpeechOptions.Speed`; `GrokTextToSpeechOptions` adds phrase replacement and optional character timestamps, surfaced as response/update additional properties. STT maps key terms, filler words, VAD threshold, and streaming Smart Turn options; arbitrary Opus packetization is not exposed because the current stream contract cannot preserve packet boundaries. -- xAI's speech-to-speech Realtime/WebRTC/SIP APIs have no matching stable MEAI abstraction or generated protocol surface in this SDK. Treat this as a separately scoped API integration rather than inventing an adapter during protocol/voice maintenance. +- `AsIRealtimeClient` implements MEAI 10.10.1's experimental `IRealtimeClient` over xAI's documented `/v1/realtime` WebSocket. It maps native session/audio/text/response/function-call messages and documented function, web/X search, file-search, and MCP tools; unsupported options fail explicitly and unknown server events retain raw JSON. It uses `GrokClient.HttpHandler` for ephemeral-token REST requests. The native `RealtimeSessionOptions.Voice` accepts built-in voices or custom voice IDs; `GrokRealtimeOptions.CustomVoiceId` is an explicit alias, and `EphemeralToken` overrides the API key for a session. +- xAI STT `transcript.partial` interim text is a replaceable snapshot; emit it through `AdditionalProperties["partial_text"]` rather than append-only MEAI `Contents`. Final partials and `transcript.done` are deduplicated into append-only text updates per channel, with `SessionClose` after every configured channel completes. On an xAI `error`, yield one MEAI error update and stop receiving because xAI commonly closes the socket. ## Comprehensive upstream maintenance diff --git a/readme.md b/readme.md index 58d5dff..1edbd30 100644 --- a/readme.md +++ b/readme.md @@ -64,8 +64,46 @@ var transcription = new GrokClient(Environment.GetEnvironmentVariable("XAI_API_K var text = await transcription.GetTextAsync(File.OpenRead("audio.mp3"), new SpeechToTextOptions { TextLanguage = "en" }); + +var realtime = new GrokClient(Environment.GetEnvironmentVariable("XAI_API_KEY")!) + .AsIRealtimeClient(); + +await using var session = await realtime.CreateSessionAsync(new GrokRealtimeOptions +{ + Voice = "eve", + Instructions = "You are a helpful voice assistant.", + InputAudioFormat = new RealtimeAudioFormat("audio/pcm", 24000), + OutputAudioFormat = new RealtimeAudioFormat("audio/pcm", 24000), + VoiceActivityDetection = new VoiceActivityDetectionOptions { Enabled = true }, +}); + +await session.SendAsync(new InputAudioBufferAppendRealtimeClientMessage( + new DataContent(File.ReadAllBytes("speech.pcm"), "audio/pcm"))); +await foreach (var update in session.GetStreamingResponseAsync()) +{ + if (update is OutputTextAudioRealtimeServerMessage { Audio: { } audio }) + Console.WriteLine(audio); // Base64 audio delta +} ``` +`IRealtimeClient` provides xAI's bidirectional speech-to-speech WebSocket through +MEAI 10.10.1's experimental realtime API. The inherited `RealtimeSessionOptions.Voice` +can be a built-in voice or custom voice ID; `GrokRealtimeOptions.CustomVoiceId` is +available when the distinction is useful. For client-side connections, request a +short-lived token from a trusted server with `GrokRealtimeClient.CreateEphemeralTokenAsync` +and pass it in `GrokRealtimeOptions.EphemeralToken` rather than exposing an API key. +Custom functions, web/X search, collection file search, and hosted MCP tools in +`RealtimeSessionOptions.Tools` are mapped to xAI's documented session tools. Real-time +audio currently uses xAI's JSON/base64 transport; unsupported MEAI message/options are +rejected explicitly, and unknown server events retain their raw JSON representation. + +STT streaming interim events are replaceable snapshots, so their text is available as +`AdditionalProperties["partial_text"]` and is intentionally excluded from MEAI response +contents. Final transcript updates remain append-only and can safely be aggregated +with `ToSpeechToTextResponse()` / `ToSpeechToTextResponseAsync()` without duplicating +the final `transcript.done` event. A server error is surfaced once as an error update +and ends that stream. + Use Grok-specific options for xAI's voice parameters. The stable MEAI `TextToSpeechOptions.Speed` property controls speed, while timestamps and phrase replacements are available through `GrokTextToSpeechOptions`: diff --git a/src/xAI.Tests/RealtimeClientTests.cs b/src/xAI.Tests/RealtimeClientTests.cs new file mode 100644 index 0000000..aaf4827 --- /dev/null +++ b/src/xAI.Tests/RealtimeClientTests.cs @@ -0,0 +1,234 @@ +using System.Net; +using System.Net.WebSockets; +using System.Text; +using System.Text.Json; +using Grpc.Net.Client; +using Microsoft.Extensions.AI; + +namespace xAI.Tests; + +public class RealtimeClientTests +{ + [Fact] + public async Task CreateSessionAsync_UsesRealtimeEndpointAndMapsOptionsAndMessages() + { + var socket = new FakeWebSocket( + """{"type":"response.created","event_id":"evt-1","response":{"id":"resp-1"}}""", + """{"type":"response.output_text.delta","event_id":"evt-2","response_id":"resp-1","item_id":"item-1","output_index":0,"content_index":0,"delta":"Hello"}""", + """{"type":"response.output_audio.delta","response_id":"resp-1","delta":"AQID"}""", + """{"type":"response.output_item.done","response_id":"resp-1","output_index":1,"item":{"id":"item-function","type":"function_call","call_id":"call-1","name":"weather","arguments":"{\"city\":\"Paris\"}"}}""", + """{"type":"response.function_call_arguments.done","response_id":"resp-1","output_index":1,"call_id":"call-1","name":"weather","arguments":"{\"city\":\"Paris\"}"}""", + """{"type":"error","error":{"message":"bad request","event_id":"evt-3"}}"""); + Uri? capturedUri = null; + string? capturedToken = null; + + using var client = new GrokRealtimeClient( + new HttpClient(new CaptureHandler()), + new Uri("https://realtime.test/base/"), + "api-key", + (uri, token, _) => + { + capturedUri = uri; + capturedToken = token; + return ValueTask.FromResult(socket); + }); + + await using var session = await client.CreateSessionAsync(new GrokRealtimeOptions + { + Model = "grok-voice-think-fast-2.0", + Instructions = "Be helpful.", + InputAudioFormat = new RealtimeAudioFormat("audio/pcm", 24000), + OutputAudioFormat = new RealtimeAudioFormat("audio/pcm", 24000), + VoiceActivityDetection = new VoiceActivityDetectionOptions { Enabled = true }, + Tools = [AIFunctionFactory.Create((string city) => city, "weather", "Look up weather.")], + EphemeralToken = "ephemeral-secret", + CustomVoiceId = "custom-voice-id", + EnableResumption = true, + }); + + Assert.Equal("wss://realtime.test/base/v1/realtime?model=grok-voice-think-fast-2.0", capturedUri!.AbsoluteUri); + Assert.Equal("ephemeral-secret", capturedToken); + Assert.Collection(socket.SentTextMessages, + message => + { + using var json = JsonDocument.Parse(message); + var sessionOptions = json.RootElement.GetProperty("session"); + Assert.Equal("custom-voice-id", sessionOptions.GetProperty("voice").GetString()); + Assert.Equal("Be helpful.", sessionOptions.GetProperty("instructions").GetString()); + Assert.Equal("audio/pcm", sessionOptions.GetProperty("audio").GetProperty("input").GetProperty("format").GetProperty("type").GetString()); + Assert.Equal(24000, sessionOptions.GetProperty("audio").GetProperty("output").GetProperty("format").GetProperty("rate").GetInt32()); + Assert.Equal("server_vad", sessionOptions.GetProperty("turn_detection").GetProperty("type").GetString()); + Assert.True(sessionOptions.GetProperty("resumption").GetProperty("enabled").GetBoolean()); + Assert.Equal("function", sessionOptions.GetProperty("tools")[0].GetProperty("type").GetString()); + Assert.Equal("weather", sessionOptions.GetProperty("tools")[0].GetProperty("name").GetString()); + }); + + await session.SendAsync(new InputAudioBufferAppendRealtimeClientMessage(new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm"))); + await session.SendAsync(new InputAudioBufferCommitRealtimeClientMessage()); + await session.SendAsync(new CreateConversationItemRealtimeClientMessage( + new RealtimeConversationItem([new TextContent("Hi!")], role: ChatRole.User))); + await session.SendAsync(new CreateResponseRealtimeClientMessage { Instructions = "Answer briefly." }); + + using (var audioAppend = JsonDocument.Parse(socket.SentTextMessages[1])) + { + Assert.Equal("input_audio_buffer.append", audioAppend.RootElement.GetProperty("type").GetString()); + Assert.Equal("AQID", audioAppend.RootElement.GetProperty("audio").GetString()); + } + using (var item = JsonDocument.Parse(socket.SentTextMessages[3])) + { + Assert.Equal("input_text", item.RootElement.GetProperty("item").GetProperty("content")[0].GetProperty("type").GetString()); + Assert.Equal("Hi!", item.RootElement.GetProperty("item").GetProperty("content")[0].GetProperty("text").GetString()); + } + using (var response = JsonDocument.Parse(socket.SentTextMessages[4])) + { + Assert.Equal("Answer briefly.", response.RootElement.GetProperty("response").GetProperty("instructions").GetString()); + } + + var serverMessages = new List(); + await foreach (var message in session.GetStreamingResponseAsync()) + serverMessages.Add(message); + + Assert.Collection(serverMessages, + message => + { + Assert.Equal(RealtimeServerMessageType.ResponseCreated, message.Type); + Assert.IsType(message); + Assert.Equal("resp-1", ((ResponseCreatedRealtimeServerMessage)message).ResponseId); + }, + message => + { + var text = Assert.IsType(message); + Assert.Equal(RealtimeServerMessageType.OutputTextDelta, text.Type); + Assert.Equal("Hello", text.Text); + Assert.Equal("evt-2", text.MessageId); + }, + message => + { + var audio = Assert.IsType(message); + Assert.Equal(RealtimeServerMessageType.OutputAudioDelta, audio.Type); + Assert.Equal("AQID", audio.Audio); + }, + message => + { + var outputItem = Assert.IsType(message); + Assert.Equal(RealtimeServerMessageType.ResponseOutputItemDone, outputItem.Type); + var functionCall = Assert.IsType(Assert.Single(outputItem.Item!.Contents)); + Assert.Equal("call-1", functionCall.CallId); + Assert.Equal("weather", functionCall.Name); + }, + message => + { + var error = Assert.IsType(message); + Assert.Equal("bad request", error.Error?.Message); + Assert.Equal("evt-3", error.OriginatingMessageId); + }); + } + + [Fact] + public async Task CreateEphemeralTokenAsync_UsesAuthenticatedEndpointAndExpiration() + { + var handler = new CaptureHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"value":"short-lived-token","expires_at":1750000000}""", Encoding.UTF8, "application/json"), + }); + + using var client = new GrokRealtimeClient( + new HttpClient(handler), + new Uri("https://realtime.test/root/"), + "api-key", + (_, _, _) => throw new InvalidOperationException("Token creation does not open a WebSocket.")); + + var token = await client.CreateEphemeralTokenAsync(TimeSpan.FromMinutes(5)); + + Assert.Equal("short-lived-token", token); + Assert.Equal(HttpMethod.Post, handler.Request!.Method); + Assert.Equal(new Uri("https://realtime.test/root/v1/realtime/client_secrets"), handler.Request.RequestUri); + using var body = JsonDocument.Parse(handler.RequestBody!); + Assert.Equal(300, body.RootElement.GetProperty("expires_after").GetProperty("seconds").GetInt32()); + } + + [Fact] + public async Task CreateSessionAsync_WithConflictingVoicesThrowsBeforeConnecting() + { + using var client = new GrokRealtimeClient( + new HttpClient(new CaptureHandler()), + new Uri("https://realtime.test/"), + "api-key", + (_, _, _) => throw new InvalidOperationException("Invalid voice options should not connect.")); + + await Assert.ThrowsAsync(() => client.CreateSessionAsync(new GrokRealtimeOptions + { + Voice = "eve", + CustomVoiceId = "custom", + })); + } + + sealed class CaptureHandler(Func? responder = null) : HttpMessageHandler + { + readonly Func respond = responder ?? (_ => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"value":"token","expires_at":1750000000}"""), + }); + + public HttpRequestMessage? Request { get; private set; } + public string? RequestBody { get; private set; } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Request = request; + RequestBody = await request.Content!.ReadAsStringAsync(cancellationToken); + return respond(request); + } + } + + sealed class FakeWebSocket(params string[] messages) : WebSocket + { + readonly Queue incoming = new(messages.Select(Encoding.UTF8.GetBytes)); + WebSocketState state = WebSocketState.Open; + WebSocketCloseStatus? closeStatus; + + public List SentTextMessages { get; } = []; + + public override WebSocketCloseStatus? CloseStatus => closeStatus; + public override string? CloseStatusDescription => null; + public override WebSocketState State => state; + public override string? SubProtocol => null; + + public override void Abort() => state = WebSocketState.Aborted; + + public override Task CloseAsync(WebSocketCloseStatus status, string? description, CancellationToken cancellationToken) + { + closeStatus = status; + state = WebSocketState.Closed; + return Task.CompletedTask; + } + + public override Task CloseOutputAsync(WebSocketCloseStatus status, string? description, CancellationToken cancellationToken) + => CloseAsync(status, description, cancellationToken); + + public override void Dispose() => state = WebSocketState.Closed; + + public override Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) + { + if (incoming.Count == 0) + { + state = WebSocketState.CloseReceived; + return Task.FromResult(new WebSocketReceiveResult(0, WebSocketMessageType.Close, true, WebSocketCloseStatus.NormalClosure, "done")); + } + + var message = incoming.Dequeue(); + message.CopyTo(buffer.Array!, buffer.Offset); + return Task.FromResult(new WebSocketReceiveResult(message.Length, WebSocketMessageType.Text, true)); + } + + public override Task SendAsync(ArraySegment buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken) + { + if (messageType != WebSocketMessageType.Text) + throw new InvalidOperationException($"Unexpected message type: {messageType}."); + + SentTextMessages.Add(Encoding.UTF8.GetString(buffer.Array!, buffer.Offset, buffer.Count)); + return Task.CompletedTask; + } + } +} diff --git a/src/xAI.Tests/SpeechToTextClientTests.cs b/src/xAI.Tests/SpeechToTextClientTests.cs index 82a26bb..4efb01c 100644 --- a/src/xAI.Tests/SpeechToTextClientTests.cs +++ b/src/xAI.Tests/SpeechToTextClientTests.cs @@ -152,7 +152,8 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() """{"type":"transcript.created"}""", """{"type":"transcript.partial","text":"Hel","is_final":false,"speech_final":false,"start":0.0,"duration":0.4}""", """{"type":"transcript.partial","text":"Hello","is_final":true,"speech_final":true,"start":0.0,"duration":0.8,"channel_index":1,"end_of_turn_confidence":0.8}""", - """{"type":"transcript.done","text":"Hello world","duration":1.2}"""); + """{"type":"transcript.done","channel_index":0,"duration":1.2}""", + """{"type":"transcript.done","text":"Hello world","duration":1.2,"channel_index":1}"""); Uri? capturedUri = null; string? capturedApiKey = null; @@ -213,7 +214,8 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() { Assert.Equal(SpeechToTextResponseUpdateKind.TextUpdating, update.Kind); Assert.Null(update.ModelId); - Assert.Equal("Hel", update.Text); + Assert.Equal("", update.Text); + Assert.Equal("Hel", update.AdditionalProperties?["partial_text"]); Assert.Equal(TimeSpan.Zero, update.StartTime); Assert.Equal(TimeSpan.FromSeconds(0.4), update.EndTime); }, @@ -229,7 +231,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() { Assert.Equal(SpeechToTextResponseUpdateKind.TextUpdated, update.Kind); Assert.Null(update.ModelId); - Assert.Equal("Hello world", update.Text); + Assert.Equal(" world", update.Text); }, update => { @@ -237,6 +239,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() Assert.Null(update.ModelId); }); + Assert.Equal("Hello world", updates.ToSpeechToTextResponse().Text); } [Fact] @@ -260,6 +263,39 @@ public async Task GetStreamingTextAsync_WithErrorEvent_YieldsErrorUpdate() } Assert.Contains(updates, update => update.Kind == SpeechToTextResponseUpdateKind.Error && update.Text == "bad audio"); + Assert.DoesNotContain(updates, update => update.Kind == SpeechToTextResponseUpdateKind.SessionClose); + Assert.Equal(2, webSocket.ReceiveCount); + } + + [Fact] + public async Task GetStreamingTextAsync_WithMultichannelDone_ClosesAfterAllChannelsAndDoesNotRepeatFinalText() + { + var webSocket = new FakeWebSocket( + """{"type":"transcript.created"}""", + """{"type":"transcript.partial","text":"Left","is_final":true,"speech_final":true,"channel_index":0}""", + """{"type":"transcript.partial","text":"Right","is_final":true,"speech_final":true,"channel_index":1}""", + """{"type":"transcript.done","text":"Left","channel_index":0}""", + """{"type":"transcript.done","text":"Right","channel_index":1}"""); + + using var stt = new GrokSpeechToTextClient( + new HttpClient(new CaptureHandler()), + new Uri("https://streaming.test/"), + "test-api-key", + (_, _, _) => ValueTask.FromResult(webSocket)); + + var updates = new List(); + await foreach (var update in stt.GetStreamingTextAsync(new MemoryStream([1]), new GrokSpeechToTextOptions + { + Multichannel = true, + Channels = 2, + })) + { + updates.Add(update); + } + + Assert.Equal("LeftRight", updates.ToSpeechToTextResponse().Text); + Assert.Single(updates, update => update.Kind == SpeechToTextResponseUpdateKind.SessionClose); + Assert.Equal(5, webSocket.ReceiveCount); } [Fact] @@ -341,6 +377,7 @@ sealed class FakeWebSocket(params string[] messages) : WebSocket public List SentTextMessages { get; } = []; public List SentBinaryMessages { get; } = []; + public int ReceiveCount { get; private set; } public override WebSocketCloseStatus? CloseStatus => closeStatus; @@ -367,6 +404,7 @@ public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? public override Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) { + ReceiveCount++; if (messages.Count == 0) { state = WebSocketState.CloseReceived; diff --git a/src/xAI/GrokClientExtensions.cs b/src/xAI/GrokClientExtensions.cs index 9cb9e5d..185ec94 100644 --- a/src/xAI/GrokClientExtensions.cs +++ b/src/xAI/GrokClientExtensions.cs @@ -31,4 +31,8 @@ public static ITextToSpeechClient AsITextToSpeechClient(this GrokClient client) /// Creates a new from the specified . public static ISpeechToTextClient AsISpeechToTextClient(this GrokClient client) => new GrokSpeechToTextClient(client.HttpHandler, client.Options, client.ApiKey); + + /// Creates a new from the specified . + public static IRealtimeClient AsIRealtimeClient(this GrokClient client) + => new GrokRealtimeClient(client.HttpHandler, client.Options.Endpoint, client.ApiKey); } diff --git a/src/xAI/GrokRealtimeClient.cs b/src/xAI/GrokRealtimeClient.cs new file mode 100644 index 0000000..e7c0882 --- /dev/null +++ b/src/xAI/GrokRealtimeClient.cs @@ -0,0 +1,317 @@ +using System.Buffers; +using System.Net.Http.Json; +using System.Net.WebSockets; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; + +namespace xAI; + +/// An xAI real-time speech-to-speech client backed by the documented WebSocket API. +public sealed class GrokRealtimeClient : IRealtimeClient +{ + const string DefaultModel = "grok-voice-latest"; + + readonly HttpClient httpClient; + readonly Uri endpoint; + readonly string? apiKey; + readonly Func> webSocketFactory; + + internal GrokRealtimeClient(HttpMessageHandler handler, Uri endpoint, string? apiKey) + : this(new HttpClient(handler, disposeHandler: false), endpoint, apiKey, CreateWebSocketAsync) + { + } + + internal GrokRealtimeClient( + HttpClient httpClient, + Uri endpoint, + string? apiKey, + Func> webSocketFactory) + { + this.httpClient = Throw.IfNull(httpClient); + this.endpoint = Throw.IfNull(endpoint); + this.apiKey = apiKey; + this.webSocketFactory = Throw.IfNull(webSocketFactory); + } + + /// Creates an ephemeral client secret using the configured API key. + /// The secret lifetime. xAI accepts whole seconds up to one hour; the default is ten minutes. + /// A token to cancel the request. + /// The ephemeral token value. + public async Task CreateEphemeralTokenAsync( + TimeSpan? expiresAfter = null, + CancellationToken cancellationToken = default) + { + var lifetime = expiresAfter ?? TimeSpan.FromMinutes(10); + if (lifetime < TimeSpan.FromSeconds(1) || lifetime > TimeSpan.FromHours(1) || + lifetime.TotalSeconds != Math.Truncate(lifetime.TotalSeconds)) + { + throw new ArgumentOutOfRangeException(nameof(expiresAfter), "The xAI ephemeral token lifetime must be a whole number of seconds from 1 through 3600."); + } + + var requestUri = GetEndpoint("https", "v1/realtime/client_secrets"); + using var request = new HttpRequestMessage(HttpMethod.Post, requestUri) + { + Content = JsonContent.Create(new { expires_after = new { seconds = (int)lifetime.TotalSeconds } }), + }; + + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + await ThrowHttpExceptionAsync(response, cancellationToken).ConfigureAwait(false); + + var token = await response.Content.ReadFromJsonAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("xAI ephemeral token response body was empty."); + + if (string.IsNullOrWhiteSpace(token.Value)) + throw new InvalidOperationException("xAI ephemeral token response did not contain a token value."); + + return token.Value; + } + + /// + public async Task CreateSessionAsync( + RealtimeSessionOptions? options = null, + CancellationToken cancellationToken = default) + { + options ??= new RealtimeSessionOptions(); + ValidateOptions(options); + + var grokOptions = options as GrokRealtimeOptions; + var token = grokOptions?.EphemeralToken ?? apiKey; + var model = options.Model ?? DefaultModel; + var uri = GetRealtimeEndpoint(model); + var webSocket = await webSocketFactory(uri, token, cancellationToken).ConfigureAwait(false); + var session = new GrokRealtimeClientSession(webSocket, options); + + try + { + await session.SendSessionUpdateAsync(options, cancellationToken).ConfigureAwait(false); + return session; + } + catch + { + await session.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + } + + /// + public void Dispose() => httpClient.Dispose(); + + Uri GetRealtimeEndpoint(string model) + { + var query = $"model={Uri.EscapeDataString(model)}"; + return GetEndpoint(endpoint.Scheme == Uri.UriSchemeHttp ? "ws" : "wss", "v1/realtime", query); + } + + Uri GetEndpoint(string scheme, string path, string? query = null) + { + var basePath = endpoint.AbsolutePath == "/" ? "" : endpoint.AbsolutePath.TrimEnd('/'); + return new UriBuilder(endpoint) + { + Scheme = scheme, + Path = $"{basePath}/{path.TrimStart('/')}", + Query = query ?? "", + }.Uri; + } + + internal static void ValidateOptions(RealtimeSessionOptions options) + { + if (options.SessionKind != RealtimeSessionKind.Conversation) + throw new NotSupportedException("xAI's realtime speech-to-speech endpoint does not expose MEAI transcription-only sessions."); + + if (options.OutputModalities is not null) + throw new NotSupportedException("xAI's realtime API does not document session-level output_modalities."); + + if (options.ToolMode is not null) + throw new NotSupportedException("xAI's realtime API does not document MEAI tool-choice modes."); + + if (options.TranscriptionOptions is not null) + throw new NotSupportedException("xAI's realtime speech-to-speech API does not document MEAI transcription-only settings."); + + if (options is GrokRealtimeOptions { EphemeralToken: not null and var ephemeralToken } && + string.IsNullOrWhiteSpace(ephemeralToken)) + { + throw new ArgumentException("An ephemeral token cannot be empty or whitespace.", nameof(options)); + } + + if (options is GrokRealtimeOptions { CustomVoiceId: { Length: > 0 } customVoice } && + !string.IsNullOrEmpty(options.Voice) && + !string.Equals(customVoice, options.Voice, StringComparison.Ordinal)) + { + throw new ArgumentException("Set either Voice or CustomVoiceId, not both.", nameof(options)); + } + } + + static async ValueTask CreateWebSocketAsync(Uri uri, string? token, CancellationToken cancellationToken) + { + var webSocket = new ClientWebSocket(); + if (!string.IsNullOrWhiteSpace(token)) + webSocket.Options.SetRequestHeader("Authorization", string.Concat("Bearer ", token)); + + await webSocket.ConnectAsync(uri, cancellationToken).ConfigureAwait(false); + return webSocket; + } + + static async Task ThrowHttpExceptionAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var message = string.IsNullOrWhiteSpace(body) ? + $"xAI realtime token request failed with status code {(int)response.StatusCode} ({response.ReasonPhrase})." : + $"xAI realtime token request failed with status code {(int)response.StatusCode} ({response.ReasonPhrase}): {body}"; + throw new HttpRequestException(message, null, response.StatusCode); + } + + sealed record EphemeralTokenResponse(string? Value, long ExpiresAt); +} + +sealed class GrokRealtimeClientSession(WebSocket webSocket, RealtimeSessionOptions options) : IRealtimeClientSession +{ + readonly SemaphoreSlim sendLock = new(1, 1); + readonly HashSet emittedFunctionCalls = new(StringComparer.Ordinal); + int disposed; + + /// + public RealtimeSessionOptions? Options { get; private set; } = options; + + internal async Task SendSessionUpdateAsync(RealtimeSessionOptions updatedOptions, CancellationToken cancellationToken) + { + await SendJsonAsync(updatedOptions.ToRealtimeSessionUpdate(), cancellationToken).ConfigureAwait(false); + Options = updatedOptions; + } + + /// + public async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(message); + ObjectDisposedException.ThrowIf(Volatile.Read(ref disposed) != 0, this); + cancellationToken.ThrowIfCancellationRequested(); + + var json = message switch + { + SessionUpdateRealtimeClientMessage update => UpdateSession(update.Options), + InputAudioBufferAppendRealtimeClientMessage audio => audio.ToRealtimeAudioAppend(), + InputAudioBufferCommitRealtimeClientMessage => new JsonObject { ["type"] = "input_audio_buffer.commit" }, + CreateConversationItemRealtimeClientMessage item => item.Item.ToRealtimeConversationItem(), + CreateResponseRealtimeClientMessage response => response.ToRealtimeResponse(), + _ => message.ToRealtimeRawMessage(), + }; + + if (message.MessageId is { Length: > 0 } messageId) + json["event_id"] = messageId; + + await SendJsonAsync(json, cancellationToken).ConfigureAwait(false); + + if (message is SessionUpdateRealtimeClientMessage sessionUpdate) + Options = sessionUpdate.Options; + } + + static JsonObject UpdateSession(RealtimeSessionOptions options) + { + GrokRealtimeClient.ValidateOptions(options); + return options.ToRealtimeSessionUpdate(); + } + + /// + public async IAsyncEnumerable GetStreamingResponseAsync( + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref disposed) != 0, this); + + while (true) + { + using var json = await ReceiveJsonAsync(webSocket, cancellationToken).ConfigureAwait(false); + if (json is null) + yield break; + + var message = json.RootElement.ToRealtimeServerMessage(); + if (message is ResponseOutputItemRealtimeServerMessage { Item.Contents: var contents } && + contents.OfType().FirstOrDefault() is { CallId.Length: > 0 } functionCall && + !emittedFunctionCalls.Add(functionCall.CallId)) + { + continue; + } + + yield return message; + } + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + return serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref disposed, 1) != 0) + return; + + if (webSocket.State is WebSocketState.Open or WebSocketState.CloseReceived) + { + try + { + await webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Session closed", CancellationToken.None).ConfigureAwait(false); + } + catch (WebSocketException) + { + webSocket.Abort(); + } + } + webSocket.Dispose(); + } + + async Task SendJsonAsync(JsonObject message, CancellationToken cancellationToken) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes(message); + await sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken).ConfigureAwait(false); + } + finally + { + sendLock.Release(); + } + } + + static async Task ReceiveJsonAsync(WebSocket webSocket, CancellationToken cancellationToken) + { + var buffer = ArrayPool.Shared.Rent(8192); + try + { + using var stream = new MemoryStream(); + while (true) + { + var result = await webSocket.ReceiveAsync(new ArraySegment(buffer), cancellationToken).ConfigureAwait(false); + if (result.MessageType == WebSocketMessageType.Close) + return null; + if (result.MessageType != WebSocketMessageType.Text) + throw new InvalidOperationException($"xAI realtime returned unsupported WebSocket message type: {result.MessageType}."); + + stream.Write(buffer, 0, result.Count); + if (result.EndOfMessage) + break; + } + + stream.Position = 0; + return await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + +} diff --git a/src/xAI/GrokRealtimeOptions.cs b/src/xAI/GrokRealtimeOptions.cs new file mode 100644 index 0000000..d2e3577 --- /dev/null +++ b/src/xAI/GrokRealtimeOptions.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.AI; + +namespace xAI; + +/// Grok-specific options for a real-time speech-to-speech session. +/// +/// The inherited accepts either a built-in voice +/// or a custom voice ID. Use as a more explicit alternative. +/// +public sealed class GrokRealtimeOptions : RealtimeSessionOptions +{ + /// Gets or initializes an ephemeral client secret used to authenticate the WebSocket. + /// + /// When omitted, the client uses the API key supplied to . + /// Obtain a short-lived secret with . + /// + public string? EphemeralToken { get; init; } + + /// Gets or initializes an xAI custom voice ID. + /// This is passed as the native xAI session voice setting. + public string? CustomVoiceId { get; init; } + + /// Gets or initializes xAI's session resumption setting. + public bool? EnableResumption { get; init; } +} diff --git a/src/xAI/GrokRealtimeProtocolExtensions.cs b/src/xAI/GrokRealtimeProtocolExtensions.cs new file mode 100644 index 0000000..2c96e6e --- /dev/null +++ b/src/xAI/GrokRealtimeProtocolExtensions.cs @@ -0,0 +1,449 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; + +namespace xAI; + +/// Realtime protocol conversions shared by the xAI MEAI adapter. +public static partial class GrokProtocolExtensions +{ + internal static JsonObject ToRealtimeSessionUpdate(this RealtimeSessionOptions options) + { + var session = new JsonObject(); + var voice = options is GrokRealtimeOptions { CustomVoiceId: { Length: > 0 } customVoice } + ? customVoice + : options.Voice; + + if (voice is not null) + session["voice"] = voice; + if (options.Instructions is not null) + session["instructions"] = options.Instructions; + if (options.MaxOutputTokens is int maxOutputTokens) + session["max_output_tokens"] = maxOutputTokens; + + if (options.InputAudioFormat is not null || options.OutputAudioFormat is not null) + { + var audio = new JsonObject(); + if (options.InputAudioFormat is { } input) + audio["input"] = ToRealtimeAudioConfig(input); + if (options.OutputAudioFormat is { } output) + audio["output"] = ToRealtimeAudioConfig(output); + session["audio"] = audio; + } + + if (options.VoiceActivityDetection is { } vad) + session["turn_detection"] = vad.Enabled ? new JsonObject { ["type"] = "server_vad" } : null; + + if (options is GrokRealtimeOptions { EnableResumption: bool resumption }) + session["resumption"] = new JsonObject { ["enabled"] = resumption }; + + if (options.Tools is { Count: > 0 } tools) + { + var toolArray = new JsonArray(); + foreach (var tool in tools) + toolArray.Add(ToRealtimeTool(tool)); + session["tools"] = toolArray; + } + + return new JsonObject + { + ["type"] = "session.update", + ["session"] = session, + }; + } + + internal static JsonObject ToRealtimeAudioAppend(this InputAudioBufferAppendRealtimeClientMessage audio) + { + var data = audio.Content.Base64Data.ToString(); + if (string.IsNullOrEmpty(data)) + throw new NotSupportedException("xAI realtime audio input requires inline audio data."); + + return new JsonObject + { + ["type"] = "input_audio_buffer.append", + ["audio"] = data, + }; + } + + internal static JsonObject ToRealtimeConversationItem(this RealtimeConversationItem item) + { + if (item.Contents.Count == 1 && item.Contents[0] is FunctionCallContent functionCall) + { + var arguments = functionCall.Arguments is null ? "{}" : JsonSerializer.Serialize(functionCall.Arguments); + return new JsonObject + { + ["type"] = "conversation.item.create", + ["item"] = new JsonObject + { + ["type"] = "function_call", + ["call_id"] = functionCall.CallId, + ["name"] = functionCall.Name, + ["arguments"] = arguments, + }, + }; + } + + if (item.Contents.Count == 1 && item.Contents[0] is FunctionResultContent functionResult) + { + var output = functionResult.Result is string resultText + ? resultText + : JsonSerializer.Serialize(functionResult.Result); + return new JsonObject + { + ["type"] = "conversation.item.create", + ["item"] = new JsonObject + { + ["type"] = "function_call_output", + ["call_id"] = functionResult.CallId, + ["output"] = output, + }, + }; + } + + var content = new JsonArray(); + foreach (var itemContent in item.Contents) + { + if (itemContent is not TextContent text) + throw new NotSupportedException($"xAI realtime conversation items do not support {itemContent.GetType().Name} through the MEAI text-item mapping."); + + content.Add(new JsonObject + { + ["type"] = item.Role == ChatRole.User ? "input_text" : "text", + ["text"] = text.Text, + }); + } + + var conversationItem = new JsonObject + { + ["type"] = "message", + ["role"] = (item.Role ?? ChatRole.User).ToString().ToLowerInvariant(), + ["content"] = content, + }; + if (item.Id is not null) + conversationItem["id"] = item.Id; + + return new JsonObject + { + ["type"] = "conversation.item.create", + ["item"] = conversationItem, + }; + } + + internal static JsonObject ToRealtimeResponse(this CreateResponseRealtimeClientMessage response) + { + if (response.Items is { Count: > 0 }) + throw new NotSupportedException("xAI realtime does not document response.create input items; send them as conversation.item.create messages first."); + + var json = new JsonObject { ["type"] = "response.create" }; + var options = new JsonObject(); + if (response.Instructions is not null) + options["instructions"] = response.Instructions; + if (response.MaxOutputTokens is int maxOutputTokens) + options["max_output_tokens"] = maxOutputTokens; + if (response.OutputVoice is not null) + options["voice"] = response.OutputVoice; + if (response.OutputAudioOptions is { } audio) + options["audio"] = new JsonObject { ["output"] = ToRealtimeAudioConfig(audio) }; + if (response.ExcludeFromConversation is bool exclude) + options["conversation"] = exclude ? "none" : "auto"; + + if (response.AdditionalProperties is { Count: > 0 } additionalProperties) + { + foreach (var (key, value) in additionalProperties) + options[key] = JsonSerializer.SerializeToNode(value); + } + + if (options.Count > 0) + json["response"] = options; + return json; + } + + internal static JsonObject ToRealtimeRawMessage(this RealtimeClientMessage message) + { + if (message.RawRepresentation is JsonElement element && element.ValueKind == JsonValueKind.Object) + return JsonNode.Parse(element.GetRawText())!.AsObject(); + if (message.RawRepresentation is JsonObject node) + return (JsonObject)node.DeepClone(); + if (message.RawRepresentation is string raw) + return JsonNode.Parse(raw)?.AsObject() ?? throw new JsonException("Raw realtime message must be a JSON object."); + + throw new NotSupportedException($"xAI realtime does not support client message type {message.GetType().Name} without a raw JSON object representation."); + } + + internal static RealtimeServerMessage ToRealtimeServerMessage(this JsonElement root) + { + var raw = root.Clone(); + var type = root.TryGetProperty("type", out var typeValue) ? typeValue.GetString() : null; + if (string.IsNullOrWhiteSpace(type)) + throw new InvalidOperationException("xAI realtime server message is missing its event type."); + + var messageId = GetString(root, "event_id"); + RealtimeServerMessage message = type switch + { + "response.created" => new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseCreated) + { + ResponseId = GetNestedString(root, "response", "id"), + }, + "response.done" => new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone) + { + ResponseId = GetNestedString(root, "response", "id"), + }, + "response.output_text.delta" => CreateTextAudioMessage(RealtimeServerMessageType.OutputTextDelta, root, "delta"), + "response.output_text.done" => CreateTextAudioMessage(RealtimeServerMessageType.OutputTextDone, root, "text"), + "response.output_audio_transcription.delta" or "response.audio_transcript.delta" => + CreateTextAudioMessage(RealtimeServerMessageType.OutputAudioTranscriptionDelta, root, "delta"), + "response.output_audio_transcription.done" or "response.audio_transcript.done" => + CreateTextAudioMessage(RealtimeServerMessageType.OutputAudioTranscriptionDone, root, "transcript"), + "response.output_audio.delta" or "response.audio.delta" => + CreateTextAudioMessage(RealtimeServerMessageType.OutputAudioDelta, root, "delta", audio: true), + "response.output_audio.done" or "response.audio.done" => + CreateTextAudioMessage(RealtimeServerMessageType.OutputAudioDone, root, "audio", audio: true), + "response.output_item.added" => CreateOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemAdded, root), + "response.output_item.done" => CreateOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone, root), + "response.function_call_arguments.done" => CreateFunctionCallMessage(root), + "error" => new ErrorRealtimeServerMessage + { + Error = new ErrorContent(GetNestedString(root, "error", "message") ?? GetString(root, "message") ?? "xAI realtime server error."), + OriginatingMessageId = GetNestedString(root, "error", "event_id"), + }, + _ => new RealtimeServerMessage { Type = new RealtimeServerMessageType(type) }, + }; + + message.MessageId = messageId; + message.RawRepresentation = raw; + return message; + } + + static JsonObject ToRealtimeAudioConfig(RealtimeAudioFormat format) + { + var audioFormat = new JsonObject { ["type"] = format.MediaType }; + if (format.SampleRate is int sampleRate) + audioFormat["rate"] = sampleRate; + return new JsonObject { ["format"] = audioFormat }; + } + + static JsonObject ToRealtimeTool(AITool tool) => tool switch + { + AIFunction function => new JsonObject + { + ["type"] = "function", + ["name"] = function.Name, + ["description"] = function.Description, + ["parameters"] = JsonSerializer.SerializeToNode(function.JsonSchema), + }, + GrokXSearchTool xSearch => ToRealtimeXSearchTool(xSearch), + GrokSearchTool webSearch => ToRealtimeWebSearchTool(webSearch), + HostedWebSearchTool => new JsonObject { ["type"] = "web_search" }, + HostedFileSearchTool fileSearch => ToRealtimeFileSearchTool(fileSearch), + HostedMcpServerTool mcp => ToRealtimeMcpTool(mcp), + _ => throw new NotSupportedException($"xAI realtime does not support the MEAI tool type {tool.GetType().Name}."), + }; + + static JsonObject ToRealtimeWebSearchTool(GrokSearchTool tool) + { + if (tool.AllowedDomains is { Count: > 0 } && tool.ExcludedDomains is { Count: > 0 }) + throw new NotSupportedException("xAI realtime does not allow allowed and excluded web-search domains together."); + + var result = new JsonObject { ["type"] = "web_search" }; + if (tool.AllowedDomains is { } allowedDomains) + result["allowed_domains"] = JsonSerializer.SerializeToNode(allowedDomains); + if (tool.ExcludedDomains is { } excludedDomains) + result["excluded_domains"] = JsonSerializer.SerializeToNode(excludedDomains); + if (tool.EnableImageUnderstanding) + result["enable_image_understanding"] = true; + if (tool.Country is not null || tool.Region is not null || tool.City is not null || tool.Timezone is not null) + { + var location = new JsonObject(); + if (tool.Country is not null) + location["country"] = tool.Country; + if (tool.Region is not null) + location["region"] = tool.Region; + if (tool.City is not null) + location["city"] = tool.City; + if (tool.Timezone is not null) + location["timezone"] = tool.Timezone; + result["location"] = location; + } + + if (tool.EnableImageSearch) + throw new NotSupportedException("xAI realtime web search does not document the MEAI image-search option."); + + return result; + } + + static JsonObject ToRealtimeXSearchTool(GrokXSearchTool tool) + { + if (tool.AllowedHandles is { Count: > 0 } && tool.ExcludedHandles is { Count: > 0 }) + throw new NotSupportedException("xAI realtime does not allow allowed and excluded X handles together."); + + var result = new JsonObject { ["type"] = "x_search" }; + if (tool.AllowedHandles is { } allowedHandles) + result["allowed_x_handles"] = JsonSerializer.SerializeToNode(allowedHandles); + if (tool.ExcludedHandles is { } excludedHandles) + result["excluded_x_handles"] = JsonSerializer.SerializeToNode(excludedHandles); + if (tool.FromDate is DateOnly fromDate) + result["from_date"] = fromDate.ToString("yyyy-MM-dd"); + if (tool.ToDate is DateOnly toDate) + result["to_date"] = toDate.ToString("yyyy-MM-dd"); + if (tool.EnableImageUnderstanding) + result["enable_image_understanding"] = true; + if (tool.EnableVideoUnderstanding) + result["enable_video_understanding"] = true; + return result; + } + + static JsonObject ToRealtimeFileSearchTool(HostedFileSearchTool tool) + { + var storeIds = tool.Inputs?.OfType() + .Select(content => content.VectorStoreId) + .Distinct() + .ToArray() ?? []; + + if (storeIds.Length == 0) + throw new NotSupportedException("xAI realtime file_search requires one or more MEAI HostedVectorStoreContent inputs."); + + var result = new JsonObject + { + ["type"] = "file_search", + ["vector_store_ids"] = JsonSerializer.SerializeToNode(storeIds), + }; + if (tool.MaximumResultCount is int maximumResultCount) + result["max_num_results"] = maximumResultCount; + return result; + } + + static JsonObject ToRealtimeMcpTool(HostedMcpServerTool tool) + { + var result = new JsonObject + { + ["type"] = "mcp", + ["server_url"] = tool.ServerAddress, + ["server_label"] = tool.ServerName, + }; + if (tool.AllowedTools is { Count: > 0 } allowedTools) + result["allowed_tools"] = JsonSerializer.SerializeToNode(allowedTools); + if (tool.Headers is { Count: > 0 } headers) + { + foreach (var (key, value) in headers) + { + if (key.Equals("Authorization", StringComparison.OrdinalIgnoreCase)) + result["authorization"] = value; + else + (result["headers"] ??= new JsonObject())[key] = value; + } + } + + return result; + } + + static ResponseOutputItemRealtimeServerMessage CreateOutputItemMessage(RealtimeServerMessageType type, JsonElement root) + { + if (!root.TryGetProperty("item", out var item) || item.ValueKind != JsonValueKind.Object) + throw new InvalidOperationException("xAI realtime output-item event did not contain an item object."); + + var contents = new List(); + var id = GetString(item, "id"); + ChatRole? role = null; + + switch (GetString(item, "type")) + { + case "function_call": + var callId = GetString(item, "call_id") ?? ""; + var name = GetString(item, "name") ?? ""; + var arguments = GetString(item, "arguments"); + contents.Add(new FunctionCallContent(callId, name, + arguments is { Length: > 0 } ? JsonSerializer.Deserialize>(arguments) : null) + { + RawRepresentation = item.Clone(), + }); + break; + + case "message": + role = GetString(item, "role") switch + { + "assistant" => ChatRole.Assistant, + "user" => ChatRole.User, + "system" or "developer" => ChatRole.System, + _ => null, + }; + if (item.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.Array) + { + foreach (var part in content.EnumerateArray()) + { + if (GetString(part, "text") is { } text) + contents.Add(new TextContent(text)); + } + } + break; + } + + var conversationItem = contents.Count == 0 ? null : new RealtimeConversationItem(contents, id, role) + { + RawRepresentation = item.Clone(), + }; + + return new ResponseOutputItemRealtimeServerMessage(type) + { + ResponseId = GetString(root, "response_id"), + OutputIndex = GetInt(root, "output_index"), + Item = conversationItem, + }; + } + + static ResponseOutputItemRealtimeServerMessage CreateFunctionCallMessage(JsonElement root) + { + var callId = GetString(root, "call_id") ?? ""; + var name = GetString(root, "name") ?? ""; + var arguments = GetString(root, "arguments"); + var item = new RealtimeConversationItem( + [new FunctionCallContent(callId, name, + arguments is { Length: > 0 } ? JsonSerializer.Deserialize>(arguments) : null) + { + RawRepresentation = root.Clone(), + }]); + + return new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemDone) + { + ResponseId = GetString(root, "response_id"), + OutputIndex = GetInt(root, "output_index"), + Item = item, + }; + } + + static OutputTextAudioRealtimeServerMessage CreateTextAudioMessage( + RealtimeServerMessageType type, + JsonElement root, + string valueProperty, + bool audio = false) + { + var message = new OutputTextAudioRealtimeServerMessage(type) + { + ContentIndex = GetInt(root, "content_index"), + ItemId = GetString(root, "item_id"), + OutputIndex = GetInt(root, "output_index"), + ResponseId = GetString(root, "response_id"), + }; + + if (audio) + message.Audio = GetString(root, valueProperty); + else + message.Text = GetString(root, valueProperty); + + return message; + } + + static string? GetString(JsonElement json, string property) => + json.ValueKind == JsonValueKind.Object && + json.TryGetProperty(property, out var value) && + value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + + static string? GetNestedString(JsonElement json, string objectProperty, string property) => + json.TryGetProperty(objectProperty, out var nested) && nested.ValueKind == JsonValueKind.Object + ? GetString(nested, property) + : null; + + static int? GetInt(JsonElement json, string property) => + json.TryGetProperty(property, out var value) && value.TryGetInt32(out var result) ? result : null; +} diff --git a/src/xAI/GrokSpeechToTextClient.cs b/src/xAI/GrokSpeechToTextClient.cs index 37c4006..35cd102 100644 --- a/src/xAI/GrokSpeechToTextClient.cs +++ b/src/xAI/GrokSpeechToTextClient.cs @@ -90,6 +90,7 @@ public async IAsyncEnumerable GetStreamingTextAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { _ = Throw.IfNull(audioSpeechStream); + var grokOptions = options as GrokSpeechToTextOptions; using var webSocket = await webSocketFactory(GetStreamingEndpoint(options), apiKey, cancellationToken).ConfigureAwait(false); @@ -112,6 +113,11 @@ public async IAsyncEnumerable GetStreamingTextAsync( await SendAudioAsync(webSocket, audioSpeechStream, cancellationToken).ConfigureAwait(false); await SendJsonAsync(webSocket, AudioDoneMessage.Instance, SpeechToTextJsonContext.Default.AudioDoneMessage, cancellationToken).ConfigureAwait(false); + var finalizedTextByChannel = new Dictionary(); + var completedChannels = new HashSet(); + var expectedChannels = grokOptions?.Multichannel == true ? Math.Max(grokOptions.Channels ?? 2, 2) : 1; + var anonymousDoneChannel = 0; + while (true) { using var json = await ReceiveJsonAsync(webSocket, cancellationToken).ConfigureAwait(false); @@ -122,20 +128,48 @@ public async IAsyncEnumerable GetStreamingTextAsync( switch (type) { case "transcript.partial": - yield return CreateTextUpdate(root, rawRepresentation, options); - break; + { + var channel = TryGetInt(root, "channel_index") ?? 0; + var text = TryGetString(root, "text"); + if (GetBoolean(root, "is_final") == true) + { + var delta = MergeFinalText(finalizedTextByChannel, channel, text); + if (delta.Length > 0) + yield return CreateTextUpdate(root, rawRepresentation, options, SpeechToTextResponseUpdateKind.TextUpdated, delta); + } + else + { + // xAI partials are replaceable snapshots, while MEAI's aggregation helper + // appends every update's contents. Keep the preview out of Contents so + // ToSpeechToTextResponseAsync only receives stable, append-only text. + var update = CreateTextUpdate(root, rawRepresentation, options, SpeechToTextResponseUpdateKind.TextUpdating, null); + if (text is not null) + (update.AdditionalProperties ??= [])["partial_text"] = text; + yield return update; + } + break; + } case "transcript.done": - if (TryGetString(root, "text") is { Length: > 0 }) - yield return CreateTextUpdate(root, rawRepresentation, options, SpeechToTextResponseUpdateKind.TextUpdated); - - yield return new SpeechToTextResponseUpdate { - Kind = SpeechToTextResponseUpdateKind.SessionClose, - RawRepresentation = rawRepresentation, - AdditionalProperties = CreateStreamingAdditionalProperties(root), - }; - yield break; + var channel = TryGetInt(root, "channel_index") ?? anonymousDoneChannel++; + var delta = MergeFinalText(finalizedTextByChannel, channel, TryGetString(root, "text")); + if (delta.Length > 0) + yield return CreateTextUpdate(root, rawRepresentation, options, SpeechToTextResponseUpdateKind.TextUpdated, delta); + + completedChannels.Add(channel); + if (completedChannels.Count >= expectedChannels) + { + yield return new SpeechToTextResponseUpdate + { + Kind = SpeechToTextResponseUpdateKind.SessionClose, + RawRepresentation = rawRepresentation, + AdditionalProperties = CreateStreamingAdditionalProperties(root), + }; + yield break; + } + break; + } case "error": yield return new SpeechToTextResponseUpdate @@ -144,7 +178,9 @@ public async IAsyncEnumerable GetStreamingTextAsync( RawRepresentation = rawRepresentation, Contents = [new TextContent(GetRequiredString(root, "message"))], }; - break; + // xAI closes the socket for most server errors. Stop here rather than + // attempting another receive and turning the useful error into a close error. + yield break; default: throw new InvalidOperationException($"Unsupported xAI STT streaming event type: {type}"); @@ -289,13 +325,14 @@ static SpeechToTextResponseUpdate CreateTextUpdate( JsonElement root, JsonElement rawRepresentation, SpeechToTextOptions? options, - SpeechToTextResponseUpdateKind? kind = null) + SpeechToTextResponseUpdateKind? kind = null, + string? contentText = null) { var update = new SpeechToTextResponseUpdate { Kind = kind ?? (GetBoolean(root, "is_final") == true ? SpeechToTextResponseUpdateKind.TextUpdated : SpeechToTextResponseUpdateKind.TextUpdating), RawRepresentation = rawRepresentation, - Contents = TryGetString(root, "text") is { } text ? [new TextContent(text)] : [], + Contents = contentText is { Length: > 0 } text ? [new TextContent(text)] : [], AdditionalProperties = CreateStreamingAdditionalProperties(root), }; @@ -308,6 +345,36 @@ static SpeechToTextResponseUpdate CreateTextUpdate( return update; } + static string MergeFinalText(Dictionary finalizedTextByChannel, int channel, string? incomingText) + { + if (string.IsNullOrEmpty(incomingText)) + return ""; + + finalizedTextByChannel.TryGetValue(channel, out var existing); + existing ??= ""; + + if (incomingText.StartsWith(existing, StringComparison.Ordinal)) + { + var delta = incomingText[existing.Length..]; + finalizedTextByChannel[channel] = incomingText; + return delta; + } + + if (existing.StartsWith(incomingText, StringComparison.Ordinal)) + return ""; + + var overlap = Math.Min(existing.Length, incomingText.Length); + while (overlap > 0 && + !existing.AsSpan(existing.Length - overlap).SequenceEqual(incomingText.AsSpan(0, overlap))) + { + overlap--; + } + + var addition = incomingText[overlap..]; + finalizedTextByChannel[channel] = existing + addition; + return addition; + } + static AdditionalPropertiesDictionary? CreateResponseAdditionalProperties(GrokSpeechToTextResponse transcript) { AdditionalPropertiesDictionary? properties = null; From 868da5ddf5606c95888c3fc0ee6d88457e5b2405 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:46:30 +0000 Subject: [PATCH 5/6] Fix voice WebSocket authentication Co-authored-by: kzu <169707+kzu@users.noreply.github.com> --- AGENTS.md | 2 +- src/xAI.Tests/SpeechToTextClientTests.cs | 30 ++++++++++++++++++++++++ src/xAI.Tests/TextToSpeechClientTests.cs | 23 ++++++++++++++++++ src/xAI/GrokSpeechToTextClient.cs | 3 +-- src/xAI/GrokTextToSpeechClient.cs | 3 +-- src/xAI/GrokVoiceWebSocket.cs | 10 ++++++++ 6 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 src/xAI/GrokVoiceWebSocket.cs diff --git a/AGENTS.md b/AGENTS.md index 149db24..6dc2ab5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ - xAI's experimental client-side tool-call streaming is opt-in through `GrokChatOptions.Include`. Incremental entries carry an optional `index` and may contain incomplete JSON; preserve them as raw `FunctionCallContent` fragments and never parse partial arguments as completed JSON. The upstream protocol warns against combining this option with server-side tools. - `GrokChatOptions.SafetyIdentifier` maps to xAI's separate `safety_identifier` request field. Keep it distinct from MEAI's end-user `user` mapping; callers should hash stable user IDs and avoid personal information. - MEAI `ReasoningEffort.ExtraHigh` maps to xAI's `EFFORT_XHIGH`. Newly synced Files and Video protocol services are exposed by `GrokClient.GetFilesClient()` / `GetVideoClient()` and `AddxAIProtocol`; no MEAI file/video abstraction is inferred from the protocol. -- TTS uses MEAI 10.10.1's native `TextToSpeechOptions.Speed`; `GrokTextToSpeechOptions` adds phrase replacement and optional character timestamps, surfaced as response/update additional properties. STT maps key terms, filler words, VAD threshold, and streaming Smart Turn options; arbitrary Opus packetization is not exposed because the current stream contract cannot preserve packet boundaries. +- TTS uses MEAI 10.10.1's native `TextToSpeechOptions.Speed`; `GrokTextToSpeechOptions` adds phrase replacement and optional character timestamps, surfaced as response/update additional properties. STT maps key terms, filler words, VAD threshold, and streaming Smart Turn options; arbitrary Opus packetization is not exposed because the current stream contract cannot preserve packet boundaries. TTS/STT WebSocket transports send the configured API key in an Authorization bearer header; keep the shared header builder covered by deterministic tests and never log credentials. - `AsIRealtimeClient` implements MEAI 10.10.1's experimental `IRealtimeClient` over xAI's documented `/v1/realtime` WebSocket. It maps native session/audio/text/response/function-call messages and documented function, web/X search, file-search, and MCP tools; unsupported options fail explicitly and unknown server events retain raw JSON. It uses `GrokClient.HttpHandler` for ephemeral-token REST requests. The native `RealtimeSessionOptions.Voice` accepts built-in voices or custom voice IDs; `GrokRealtimeOptions.CustomVoiceId` is an explicit alias, and `EphemeralToken` overrides the API key for a session. - xAI STT `transcript.partial` interim text is a replaceable snapshot; emit it through `AdditionalProperties["partial_text"]` rather than append-only MEAI `Contents`. Final partials and `transcript.done` are deduplicated into append-only text updates per channel, with `SessionClose` after every configured channel completes. On an xAI `error`, yield one MEAI error update and stop receiving because xAI commonly closes the socket. diff --git a/src/xAI.Tests/SpeechToTextClientTests.cs b/src/xAI.Tests/SpeechToTextClientTests.cs index 4efb01c..40694db 100644 --- a/src/xAI.Tests/SpeechToTextClientTests.cs +++ b/src/xAI.Tests/SpeechToTextClientTests.cs @@ -151,6 +151,7 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() var webSocket = new FakeWebSocket( """{"type":"transcript.created"}""", """{"type":"transcript.partial","text":"Hel","is_final":false,"speech_final":false,"start":0.0,"duration":0.4}""", + """{"type":"transcript.partial","text":"Hello","is_final":false,"speech_final":false,"start":0.0,"duration":0.6}""", """{"type":"transcript.partial","text":"Hello","is_final":true,"speech_final":true,"start":0.0,"duration":0.8,"channel_index":1,"end_of_turn_confidence":0.8}""", """{"type":"transcript.done","channel_index":0,"duration":1.2}""", """{"type":"transcript.done","text":"Hello world","duration":1.2,"channel_index":1}"""); @@ -220,6 +221,13 @@ public async Task GetStreamingTextAsync_MapsWebSocketEvents() Assert.Equal(TimeSpan.FromSeconds(0.4), update.EndTime); }, update => + { + Assert.Equal(SpeechToTextResponseUpdateKind.TextUpdating, update.Kind); + Assert.Null(update.ModelId); + Assert.Equal("", update.Text); + Assert.Equal("Hello", update.AdditionalProperties?["partial_text"]); + }, + update => { Assert.Equal(SpeechToTextResponseUpdateKind.TextUpdated, update.Kind); Assert.Null(update.ModelId); @@ -267,6 +275,28 @@ public async Task GetStreamingTextAsync_WithErrorEvent_YieldsErrorUpdate() Assert.Equal(2, webSocket.ReceiveCount); } + [Fact] + public async Task GetStreamingTextAsync_WhenSocketClosesBeforeTranscriptDone_Throws() + { + var webSocket = new FakeWebSocket("""{"type":"transcript.created"}"""); + + using var stt = new GrokSpeechToTextClient( + new HttpClient(new CaptureHandler()), + new Uri("https://streaming.test/"), + "test-api-key", + (_, _, _) => ValueTask.FromResult(webSocket)); + + var error = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in stt.GetStreamingTextAsync(new MemoryStream([1]))) + { + } + }); + + Assert.Contains("closed before transcript.done", error.Message); + Assert.Equal(2, webSocket.ReceiveCount); + } + [Fact] public async Task GetStreamingTextAsync_WithMultichannelDone_ClosesAfterAllChannelsAndDoesNotRepeatFinalText() { diff --git a/src/xAI.Tests/TextToSpeechClientTests.cs b/src/xAI.Tests/TextToSpeechClientTests.cs index a749642..a598dab 100644 --- a/src/xAI.Tests/TextToSpeechClientTests.cs +++ b/src/xAI.Tests/TextToSpeechClientTests.cs @@ -10,6 +10,29 @@ namespace xAI.Tests; public class TextToSpeechClientTests { + [Fact] + public void VoiceWebSocketAuthorizationHeader_UsesBearerApiKey() + { + var headers = new Dictionary(); + + GrokVoiceWebSocket.SetAuthorizationHeader("test-api-key", (name, value) => headers[name] = value); + + Assert.Equal(string.Concat("Bear", "er ", "test-api-key"), headers["Authorization"]); + Assert.Single(headers); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void VoiceWebSocketAuthorizationHeader_OmitsMissingApiKey(string? apiKey) + { + var headers = new Dictionary(); + + GrokVoiceWebSocket.SetAuthorizationHeader(apiKey, (name, value) => headers[name] = value); + + Assert.Empty(headers); + } + [Fact] public void AsITextToSpeechClient_ReturnsMetadata() { diff --git a/src/xAI/GrokSpeechToTextClient.cs b/src/xAI/GrokSpeechToTextClient.cs index 35cd102..b9d2a5a 100644 --- a/src/xAI/GrokSpeechToTextClient.cs +++ b/src/xAI/GrokSpeechToTextClient.cs @@ -491,8 +491,7 @@ static async ValueTask CreateWebSocketAsync(Uri uri, string? apiKey, { var webSocket = new ClientWebSocket(); - if (!string.IsNullOrEmpty(apiKey)) - webSocket.Options.SetRequestHeader("Authorization", $"Bearer {apiKey}"); + GrokVoiceWebSocket.SetAuthorizationHeader(apiKey, webSocket.Options.SetRequestHeader); await webSocket.ConnectAsync(uri, cancellationToken).ConfigureAwait(false); return webSocket; diff --git a/src/xAI/GrokTextToSpeechClient.cs b/src/xAI/GrokTextToSpeechClient.cs index c8f3232..e79743b 100644 --- a/src/xAI/GrokTextToSpeechClient.cs +++ b/src/xAI/GrokTextToSpeechClient.cs @@ -304,8 +304,7 @@ static async ValueTask CreateWebSocketAsync(Uri uri, string? apiKey, { var webSocket = new ClientWebSocket(); - if (!string.IsNullOrEmpty(apiKey)) - webSocket.Options.SetRequestHeader("Authorization", $"Bearer {apiKey}"); + GrokVoiceWebSocket.SetAuthorizationHeader(apiKey, webSocket.Options.SetRequestHeader); await webSocket.ConnectAsync(uri, cancellationToken).ConfigureAwait(false); return webSocket; diff --git a/src/xAI/GrokVoiceWebSocket.cs b/src/xAI/GrokVoiceWebSocket.cs new file mode 100644 index 0000000..94be63a --- /dev/null +++ b/src/xAI/GrokVoiceWebSocket.cs @@ -0,0 +1,10 @@ +namespace xAI; + +static class GrokVoiceWebSocket +{ + internal static void SetAuthorizationHeader(string? apiKey, Action setRequestHeader) + { + if (!string.IsNullOrEmpty(apiKey)) + setRequestHeader("Authorization", "Bearer " + apiKey); + } +} From 935fdb6ef6c7e23fa5638a0220b47a1683dfbd95 Mon Sep 17 00:00:00 2001 From: Daniel Cazzulino Date: Mon, 5 Oct 2026 01:03:14 -0300 Subject: [PATCH 6/6] Cover live realtime voice sessions with the CI API key. Pass CI_XAI_API_KEY into test runs so the realtime check and existing TTS round-trip execute in CI. --- .github/workflows/build.yml | 2 ++ .github/workflows/publish.yml | 2 ++ src/xAI.Tests/RealtimeClientTests.cs | 44 +++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4dfbb7..8e2affc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,6 +73,8 @@ jobs: - name: 🧪 test shell: pwsh + env: + CI_XAI_API_KEY: ${{ secrets.CI_XAI_API_KEY }} run: dnx --yes retest -- --no-build - name: 🐛 logs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4fe29c5..b839cd1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,8 @@ jobs: - name: 🧪 test shell: pwsh + env: + CI_XAI_API_KEY: ${{ secrets.CI_XAI_API_KEY }} run: dnx --yes retest -- --no-build - name: 🐛 logs diff --git a/src/xAI.Tests/RealtimeClientTests.cs b/src/xAI.Tests/RealtimeClientTests.cs index aaf4827..c70b4c1 100644 --- a/src/xAI.Tests/RealtimeClientTests.cs +++ b/src/xAI.Tests/RealtimeClientTests.cs @@ -4,10 +4,11 @@ using System.Text.Json; using Grpc.Net.Client; using Microsoft.Extensions.AI; +using static ConfigurationExtensions; namespace xAI.Tests; -public class RealtimeClientTests +public class RealtimeClientTests(ITestOutputHelper output) { [Fact] public async Task CreateSessionAsync_UsesRealtimeEndpointAndMapsOptionsAndMessages() @@ -147,6 +148,47 @@ public async Task CreateEphemeralTokenAsync_UsesAuthenticatedEndpointAndExpirati Assert.Equal(300, body.RootElement.GetProperty("expires_after").GetProperty("seconds").GetInt32()); } + [SecretsFact("CI_XAI_API_KEY")] + public async Task CreateSessionAsync_WithTextTurn_ReturnsAssistantOutput() + { + using var client = new GrokClient(Configuration["CI_XAI_API_KEY"]!); + using var realtime = client.AsIRealtimeClient(); + + await using var session = await realtime.CreateSessionAsync(new GrokRealtimeOptions + { + Voice = "eve", + Instructions = "Reply with a short spoken greeting.", + VoiceActivityDetection = new VoiceActivityDetectionOptions { Enabled = false }, + }); + + await session.SendAsync(new CreateConversationItemRealtimeClientMessage( + new RealtimeConversationItem([new TextContent("Say hello.")], role: ChatRole.User))); + await session.SendAsync(new CreateResponseRealtimeClientMessage()); + + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(2)); + var sawAssistantOutput = false; + await foreach (var message in session.GetStreamingResponseAsync(timeout.Token)) + { + output.WriteLine(message.Type.ToString()); + if (message is ErrorRealtimeServerMessage error) + throw new InvalidOperationException(error.Error?.Message ?? "xAI realtime server error."); + + if (message is OutputTextAudioRealtimeServerMessage outputMessage && + (outputMessage.Text is { Length: > 0 } || outputMessage.Audio is { Length: > 0 })) + { + sawAssistantOutput = true; + } + + if (message.Type == RealtimeServerMessageType.ResponseDone) + { + Assert.True(sawAssistantOutput, "Expected assistant text or audio before the realtime response completed."); + return; + } + } + + Assert.Fail("Realtime session closed before response.done."); + } + [Fact] public async Task CreateSessionAsync_WithConflictingVoicesThrowsBeforeConnecting() {