diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39a32910a..54311aa99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,9 @@ jobs: run: npm install -g pnpm@10.32.1 - name: Install dependencies - run: pnpm install --frozen-lockfile --filter liveagent... + # GUI frontend tests load shared Gateway WebUI modules (including STT adapters). + # Install that workspace and its transitive dependencies in this isolated job. + run: pnpm install --frozen-lockfile --filter liveagent... --filter @liveagent/gateway-webui... - name: Typecheck and build GUI frontend working-directory: crates/agent-gui @@ -169,7 +171,7 @@ jobs: tauri-rust: name: Tauri Rust Check runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v6 diff --git a/.gitignore b/.gitignore index 705ae5fc2..de135aab5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ chatroom !.vscode/extensions.json .idea .DS_Store +._* *.suo *.ntvs* *.njsproj diff --git a/Cargo.lock b/Cargo.lock index ec2764e4f..48ce66c51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1412,6 +1412,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -2551,7 +2552,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac", + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -3376,20 +3386,26 @@ name = "liveagent" version = "1.3.0-dev.0" dependencies = [ "arboard", + "async-trait", "axum", "base64 0.22.1", + "block2 0.6.2", "chardetng", "chrono", "dirs", "encoding_rs", + "flate2", "futures-util", "globset", + "hmac 0.12.1", "ignore", "image", "leveldb-core", "lopdf", "notify", + "objc2 0.6.4", "objc2-app-kit 0.3.2", + "objc2-av-foundation", "percent-encoding", "portable-pty", "prost", @@ -3406,6 +3422,7 @@ dependencies = [ "semver", "serde", "serde_json", + "sha1 0.10.6", "sha2 0.11.0", "tauri", "tauri-build", @@ -3421,6 +3438,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite 0.29.0", "toml 0.9.12+spec-1.1.0", + "urlencoding", "uuid", "wait-timeout", "walkdir", @@ -3854,6 +3872,18 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -4410,7 +4440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest 0.11.3", - "hmac", + "hmac 0.13.0", ] [[package]] @@ -5200,7 +5230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ "crypto-bigint", - "hmac", + "hmac 0.13.0", ] [[package]] @@ -5366,7 +5396,7 @@ dependencies = [ "getrandom 0.4.3", "ghash", "hex-literal", - "hmac", + "hmac 0.13.0", "inout", "internal-russh-num-bigint", "keccak", @@ -5482,7 +5512,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6239,7 +6269,7 @@ dependencies = [ "ctutils", "ed25519-dalek", "hex", - "hmac", + "hmac 0.13.0", "p256", "p384", "p521", @@ -7480,6 +7510,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "urlpattern" version = "0.3.0" diff --git a/crates/agent-gateway/cmd/gateway/main.go b/crates/agent-gateway/cmd/gateway/main.go index a76686236..3817e2525 100644 --- a/crates/agent-gateway/cmd/gateway/main.go +++ b/crates/agent-gateway/cmd/gateway/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "log/slog" "net/http" @@ -16,6 +17,7 @@ import ( "github.com/liveagent/agent-gateway/internal/observability" "github.com/liveagent/agent-gateway/internal/server" "github.com/liveagent/agent-gateway/internal/session" + "github.com/liveagent/agent-gateway/internal/stt" ) // fatal 记录错误并以非零码退出（slog 没有 Fatal 级别，集中在此处理）。 @@ -43,10 +45,22 @@ func main() { } slog.Info("agent registry db ready", "path", cfg.AgentDB) slog.Info("agent authentication accepts gateway token or per-agent token") + sttStore, err := stt.NewStore(database) + if err != nil { + fatal("init STT settings store failed", "err", err) + } + sm.SetSTTSettingsSyncHandler(func(ctx context.Context, raw json.RawMessage) (any, error) { + var settings stt.Settings + if err := json.Unmarshal(raw, &settings); err != nil { + return nil, err + } + return sttStore.SyncFromDesktop(ctx, settings) + }) + sttManager := stt.NewManager(sttStore) httpServer := &http.Server{ Addr: cfg.HTTPAddr, - Handler: server.NewHTTPServer(cfg, sm, tokens), + Handler: server.NewHTTPServer(cfg, sm, tokens, sttManager), ReadHeaderTimeout: 10 * time.Second, // 空闲 keep-alive 连接必须回收，否则 REST/静态资源访问方挂住连接会把 fd // 慢性耗尽到 ulimit。刻意不设全局 Read/WriteTimeout：流式上传与隧道长响应 diff --git a/crates/agent-gateway/internal/proto/v2/capabilities.go b/crates/agent-gateway/internal/proto/v2/capabilities.go index cf721ff74..e03d09778 100644 --- a/crates/agent-gateway/internal/proto/v2/capabilities.go +++ b/crates/agent-gateway/internal/proto/v2/capabilities.go @@ -3,3 +3,6 @@ package gatewayv2 // ChatIngressV1Capability is the capability identifier for reliable desktop // chat mirroring into the gateway. const ChatIngressV1Capability = "CHAT_INGRESS_V1" + +// SttStreamV1Capability identifies the gateway-owned browser STT data plane. +const SttStreamV1Capability = "STT_STREAM_V1" diff --git a/crates/agent-gateway/internal/proto/v2/gateway_ws.pb.go b/crates/agent-gateway/internal/proto/v2/gateway_ws.pb.go index 9014834d7..6d83aacb4 100644 --- a/crates/agent-gateway/internal/proto/v2/gateway_ws.pb.go +++ b/crates/agent-gateway/internal/proto/v2/gateway_ws.pb.go @@ -415,6 +415,698 @@ func (x *AckResult) GetOk() bool { return false } +type SttClientHello struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` + Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttClientHello) Reset() { + *x = SttClientHello{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttClientHello) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttClientHello) ProtoMessage() {} + +func (x *SttClientHello) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttClientHello.ProtoReflect.Descriptor instead. +func (*SttClientHello) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{5} +} + +func (x *SttClientHello) GetProtocolVersion() uint32 { + if x != nil { + return x.ProtocolVersion + } + return 0 +} + +func (x *SttClientHello) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + +type SttStart struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Provider string `protobuf:"bytes,2,opt,name=provider,proto3" json:"provider,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttStart) Reset() { + *x = SttStart{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttStart) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttStart) ProtoMessage() {} + +func (x *SttStart) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttStart.ProtoReflect.Descriptor instead. +func (*SttStart) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{6} +} + +func (x *SttStart) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *SttStart) GetProvider() string { + if x != nil { + return x.Provider + } + return "" +} + +type SttAudio struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Sequence uint32 `protobuf:"varint,2,opt,name=sequence,proto3" json:"sequence,omitempty"` + Pcm []byte `protobuf:"bytes,3,opt,name=pcm,proto3" json:"pcm,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttAudio) Reset() { + *x = SttAudio{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttAudio) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttAudio) ProtoMessage() {} + +func (x *SttAudio) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttAudio.ProtoReflect.Descriptor instead. +func (*SttAudio) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{7} +} + +func (x *SttAudio) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *SttAudio) GetSequence() uint32 { + if x != nil { + return x.Sequence + } + return 0 +} + +func (x *SttAudio) GetPcm() []byte { + if x != nil { + return x.Pcm + } + return nil +} + +type SttSessionControl struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttSessionControl) Reset() { + *x = SttSessionControl{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttSessionControl) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttSessionControl) ProtoMessage() {} + +func (x *SttSessionControl) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttSessionControl.ProtoReflect.Descriptor instead. +func (*SttSessionControl) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{8} +} + +func (x *SttSessionControl) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +type SttClientFrame struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Payload: + // + // *SttClientFrame_Hello + // *SttClientFrame_Start + // *SttClientFrame_Audio + // *SttClientFrame_Stop + // *SttClientFrame_Cancel + Payload isSttClientFrame_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttClientFrame) Reset() { + *x = SttClientFrame{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttClientFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttClientFrame) ProtoMessage() {} + +func (x *SttClientFrame) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttClientFrame.ProtoReflect.Descriptor instead. +func (*SttClientFrame) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{9} +} + +func (x *SttClientFrame) GetPayload() isSttClientFrame_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SttClientFrame) GetHello() *SttClientHello { + if x != nil { + if x, ok := x.Payload.(*SttClientFrame_Hello); ok { + return x.Hello + } + } + return nil +} + +func (x *SttClientFrame) GetStart() *SttStart { + if x != nil { + if x, ok := x.Payload.(*SttClientFrame_Start); ok { + return x.Start + } + } + return nil +} + +func (x *SttClientFrame) GetAudio() *SttAudio { + if x != nil { + if x, ok := x.Payload.(*SttClientFrame_Audio); ok { + return x.Audio + } + } + return nil +} + +func (x *SttClientFrame) GetStop() *SttSessionControl { + if x != nil { + if x, ok := x.Payload.(*SttClientFrame_Stop); ok { + return x.Stop + } + } + return nil +} + +func (x *SttClientFrame) GetCancel() *SttSessionControl { + if x != nil { + if x, ok := x.Payload.(*SttClientFrame_Cancel); ok { + return x.Cancel + } + } + return nil +} + +type isSttClientFrame_Payload interface { + isSttClientFrame_Payload() +} + +type SttClientFrame_Hello struct { + Hello *SttClientHello `protobuf:"bytes,1,opt,name=hello,proto3,oneof"` +} + +type SttClientFrame_Start struct { + Start *SttStart `protobuf:"bytes,2,opt,name=start,proto3,oneof"` +} + +type SttClientFrame_Audio struct { + Audio *SttAudio `protobuf:"bytes,3,opt,name=audio,proto3,oneof"` +} + +type SttClientFrame_Stop struct { + Stop *SttSessionControl `protobuf:"bytes,4,opt,name=stop,proto3,oneof"` +} + +type SttClientFrame_Cancel struct { + Cancel *SttSessionControl `protobuf:"bytes,5,opt,name=cancel,proto3,oneof"` +} + +func (*SttClientFrame_Hello) isSttClientFrame_Payload() {} + +func (*SttClientFrame_Start) isSttClientFrame_Payload() {} + +func (*SttClientFrame_Audio) isSttClientFrame_Payload() {} + +func (*SttClientFrame_Stop) isSttClientFrame_Payload() {} + +func (*SttClientFrame_Cancel) isSttClientFrame_Payload() {} + +type SttServerHello struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttServerHello) Reset() { + *x = SttServerHello{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttServerHello) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttServerHello) ProtoMessage() {} + +func (x *SttServerHello) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttServerHello.ProtoReflect.Descriptor instead. +func (*SttServerHello) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{10} +} + +func (x *SttServerHello) GetOk() bool { + if x != nil { + return x.Ok + } + return false +} + +func (x *SttServerHello) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +type SttTextEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Text string `protobuf:"bytes,2,opt,name=text,proto3" json:"text,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttTextEvent) Reset() { + *x = SttTextEvent{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttTextEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttTextEvent) ProtoMessage() {} + +func (x *SttTextEvent) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttTextEvent.ProtoReflect.Descriptor instead. +func (*SttTextEvent) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{11} +} + +func (x *SttTextEvent) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *SttTextEvent) GetText() string { + if x != nil { + return x.Text + } + return "" +} + +type SttError struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Code string `protobuf:"bytes,2,opt,name=code,proto3" json:"code,omitempty"` + Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttError) Reset() { + *x = SttError{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttError) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttError) ProtoMessage() {} + +func (x *SttError) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttError.ProtoReflect.Descriptor instead. +func (*SttError) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{12} +} + +func (x *SttError) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *SttError) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +func (x *SttError) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +type SttClosed struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttClosed) Reset() { + *x = SttClosed{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttClosed) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttClosed) ProtoMessage() {} + +func (x *SttClosed) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttClosed.ProtoReflect.Descriptor instead. +func (*SttClosed) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{13} +} + +func (x *SttClosed) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +type SttServerFrame struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Payload: + // + // *SttServerFrame_Hello + // *SttServerFrame_Ready + // *SttServerFrame_Partial + // *SttServerFrame_Final + // *SttServerFrame_Error + // *SttServerFrame_Closed + Payload isSttServerFrame_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SttServerFrame) Reset() { + *x = SttServerFrame{} + mi := &file_proto_v2_gateway_ws_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SttServerFrame) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SttServerFrame) ProtoMessage() {} + +func (x *SttServerFrame) ProtoReflect() protoreflect.Message { + mi := &file_proto_v2_gateway_ws_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SttServerFrame.ProtoReflect.Descriptor instead. +func (*SttServerFrame) Descriptor() ([]byte, []int) { + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{14} +} + +func (x *SttServerFrame) GetPayload() isSttServerFrame_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SttServerFrame) GetHello() *SttServerHello { + if x != nil { + if x, ok := x.Payload.(*SttServerFrame_Hello); ok { + return x.Hello + } + } + return nil +} + +func (x *SttServerFrame) GetReady() *SttTextEvent { + if x != nil { + if x, ok := x.Payload.(*SttServerFrame_Ready); ok { + return x.Ready + } + } + return nil +} + +func (x *SttServerFrame) GetPartial() *SttTextEvent { + if x != nil { + if x, ok := x.Payload.(*SttServerFrame_Partial); ok { + return x.Partial + } + } + return nil +} + +func (x *SttServerFrame) GetFinal() *SttTextEvent { + if x != nil { + if x, ok := x.Payload.(*SttServerFrame_Final); ok { + return x.Final + } + } + return nil +} + +func (x *SttServerFrame) GetError() *SttError { + if x != nil { + if x, ok := x.Payload.(*SttServerFrame_Error); ok { + return x.Error + } + } + return nil +} + +func (x *SttServerFrame) GetClosed() *SttClosed { + if x != nil { + if x, ok := x.Payload.(*SttServerFrame_Closed); ok { + return x.Closed + } + } + return nil +} + +type isSttServerFrame_Payload interface { + isSttServerFrame_Payload() +} + +type SttServerFrame_Hello struct { + Hello *SttServerHello `protobuf:"bytes,1,opt,name=hello,proto3,oneof"` +} + +type SttServerFrame_Ready struct { + Ready *SttTextEvent `protobuf:"bytes,2,opt,name=ready,proto3,oneof"` +} + +type SttServerFrame_Partial struct { + Partial *SttTextEvent `protobuf:"bytes,3,opt,name=partial,proto3,oneof"` +} + +type SttServerFrame_Final struct { + Final *SttTextEvent `protobuf:"bytes,4,opt,name=final,proto3,oneof"` +} + +type SttServerFrame_Error struct { + Error *SttError `protobuf:"bytes,5,opt,name=error,proto3,oneof"` +} + +type SttServerFrame_Closed struct { + Closed *SttClosed `protobuf:"bytes,6,opt,name=closed,proto3,oneof"` +} + +func (*SttServerFrame_Hello) isSttServerFrame_Payload() {} + +func (*SttServerFrame_Ready) isSttServerFrame_Payload() {} + +func (*SttServerFrame_Partial) isSttServerFrame_Payload() {} + +func (*SttServerFrame_Final) isSttServerFrame_Payload() {} + +func (*SttServerFrame_Error) isSttServerFrame_Payload() {} + +func (*SttServerFrame_Closed) isSttServerFrame_Payload() {} + // WebClientFrame 为浏览器 → 网关方向的帧。除 agent_request 直通臂外，其余臂均为 // 网关本地操作（由网关自身状态应答，不经桌面端往返）。 type WebClientFrame struct { @@ -445,7 +1137,7 @@ type WebClientFrame struct { func (x *WebClientFrame) Reset() { *x = WebClientFrame{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[5] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -457,7 +1149,7 @@ func (x *WebClientFrame) String() string { func (*WebClientFrame) ProtoMessage() {} func (x *WebClientFrame) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[5] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -470,7 +1162,7 @@ func (x *WebClientFrame) ProtoReflect() protoreflect.Message { // Deprecated: Use WebClientFrame.ProtoReflect.Descriptor instead. func (*WebClientFrame) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{5} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{15} } func (x *WebClientFrame) GetRequestId() string { @@ -721,7 +1413,7 @@ type WebServerFrame struct { func (x *WebServerFrame) Reset() { *x = WebServerFrame{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[6] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -733,7 +1425,7 @@ func (x *WebServerFrame) String() string { func (*WebServerFrame) ProtoMessage() {} func (x *WebServerFrame) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[6] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -746,7 +1438,7 @@ func (x *WebServerFrame) ProtoReflect() protoreflect.Message { // Deprecated: Use WebServerFrame.ProtoReflect.Descriptor instead. func (*WebServerFrame) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{6} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{16} } func (x *WebServerFrame) GetRequestId() string { @@ -1132,7 +1824,7 @@ type AgentListRequest struct { func (x *AgentListRequest) Reset() { *x = AgentListRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[7] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1144,7 +1836,7 @@ func (x *AgentListRequest) String() string { func (*AgentListRequest) ProtoMessage() {} func (x *AgentListRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[7] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1157,7 +1849,7 @@ func (x *AgentListRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentListRequest.ProtoReflect.Descriptor instead. func (*AgentListRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{7} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{17} } // AgentListResult 返回全部已登记 Agent 的状态（含离线项，供目录渲染）， @@ -1171,7 +1863,7 @@ type AgentListResult struct { func (x *AgentListResult) Reset() { *x = AgentListResult{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[8] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1183,7 +1875,7 @@ func (x *AgentListResult) String() string { func (*AgentListResult) ProtoMessage() {} func (x *AgentListResult) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[8] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1196,7 +1888,7 @@ func (x *AgentListResult) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentListResult.ProtoReflect.Descriptor instead. func (*AgentListResult) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{8} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{18} } func (x *AgentListResult) GetAgents() []*StatusEvent { @@ -1220,7 +1912,7 @@ type AgentClientFrame struct { func (x *AgentClientFrame) Reset() { *x = AgentClientFrame{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[9] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1232,7 +1924,7 @@ func (x *AgentClientFrame) String() string { func (*AgentClientFrame) ProtoMessage() {} func (x *AgentClientFrame) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[9] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1245,7 +1937,7 @@ func (x *AgentClientFrame) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentClientFrame.ProtoReflect.Descriptor instead. func (*AgentClientFrame) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{9} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{19} } func (x *AgentClientFrame) GetPayload() isAgentClientFrame_Payload { @@ -1303,7 +1995,7 @@ type AgentServerFrame struct { func (x *AgentServerFrame) Reset() { *x = AgentServerFrame{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[10] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1315,7 +2007,7 @@ func (x *AgentServerFrame) String() string { func (*AgentServerFrame) ProtoMessage() {} func (x *AgentServerFrame) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[10] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1328,7 +2020,7 @@ func (x *AgentServerFrame) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentServerFrame.ProtoReflect.Descriptor instead. func (*AgentServerFrame) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{10} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{20} } func (x *AgentServerFrame) GetPayload() isAgentServerFrame_Payload { @@ -1386,7 +2078,7 @@ type TerminalClientFrame struct { func (x *TerminalClientFrame) Reset() { *x = TerminalClientFrame{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[11] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1398,7 +2090,7 @@ func (x *TerminalClientFrame) String() string { func (*TerminalClientFrame) ProtoMessage() {} func (x *TerminalClientFrame) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[11] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1411,7 +2103,7 @@ func (x *TerminalClientFrame) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminalClientFrame.ProtoReflect.Descriptor instead. func (*TerminalClientFrame) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{11} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{21} } func (x *TerminalClientFrame) GetPayload() isTerminalClientFrame_Payload { @@ -1469,7 +2161,7 @@ type TerminalServerFrame struct { func (x *TerminalServerFrame) Reset() { *x = TerminalServerFrame{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[12] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1481,7 +2173,7 @@ func (x *TerminalServerFrame) String() string { func (*TerminalServerFrame) ProtoMessage() {} func (x *TerminalServerFrame) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[12] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1494,7 +2186,7 @@ func (x *TerminalServerFrame) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminalServerFrame.ProtoReflect.Descriptor instead. func (*TerminalServerFrame) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{12} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{22} } func (x *TerminalServerFrame) GetPayload() isTerminalServerFrame_Payload { @@ -1547,7 +2239,7 @@ type StatusGetRequest struct { func (x *StatusGetRequest) Reset() { *x = StatusGetRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[13] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1559,7 +2251,7 @@ func (x *StatusGetRequest) String() string { func (*StatusGetRequest) ProtoMessage() {} func (x *StatusGetRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[13] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1572,7 +2264,7 @@ func (x *StatusGetRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StatusGetRequest.ProtoReflect.Descriptor instead. func (*StatusGetRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{13} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{23} } // StatusEvent 镜像 session.Status 的 JSON 形状（字段一一对应）。 @@ -1599,7 +2291,7 @@ type StatusEvent struct { func (x *StatusEvent) Reset() { *x = StatusEvent{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[14] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1611,7 +2303,7 @@ func (x *StatusEvent) String() string { func (*StatusEvent) ProtoMessage() {} func (x *StatusEvent) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[14] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1624,7 +2316,7 @@ func (x *StatusEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead. func (*StatusEvent) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{14} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{24} } func (x *StatusEvent) GetOnline() bool { @@ -1735,7 +2427,7 @@ type ChatPrepareRequest struct { func (x *ChatPrepareRequest) Reset() { *x = ChatPrepareRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[15] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1747,7 +2439,7 @@ func (x *ChatPrepareRequest) String() string { func (*ChatPrepareRequest) ProtoMessage() {} func (x *ChatPrepareRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[15] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1760,7 +2452,7 @@ func (x *ChatPrepareRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatPrepareRequest.ProtoReflect.Descriptor instead. func (*ChatPrepareRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{15} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{25} } func (x *ChatPrepareRequest) GetReason() string { @@ -1784,7 +2476,7 @@ type ChatSubscribeRequest struct { func (x *ChatSubscribeRequest) Reset() { *x = ChatSubscribeRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[16] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1796,7 +2488,7 @@ func (x *ChatSubscribeRequest) String() string { func (*ChatSubscribeRequest) ProtoMessage() {} func (x *ChatSubscribeRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[16] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1809,7 +2501,7 @@ func (x *ChatSubscribeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatSubscribeRequest.ProtoReflect.Descriptor instead. func (*ChatSubscribeRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{16} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{26} } func (x *ChatSubscribeRequest) GetConversationId() string { @@ -1854,7 +2546,7 @@ type ChatRunActivity struct { func (x *ChatRunActivity) Reset() { *x = ChatRunActivity{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[17] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1866,7 +2558,7 @@ func (x *ChatRunActivity) String() string { func (*ChatRunActivity) ProtoMessage() {} func (x *ChatRunActivity) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[17] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1879,7 +2571,7 @@ func (x *ChatRunActivity) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatRunActivity.ProtoReflect.Descriptor instead. func (*ChatRunActivity) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{17} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{27} } func (x *ChatRunActivity) GetRunId() string { @@ -1968,7 +2660,7 @@ type ChatRunSnapshot struct { func (x *ChatRunSnapshot) Reset() { *x = ChatRunSnapshot{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[18] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1980,7 +2672,7 @@ func (x *ChatRunSnapshot) String() string { func (*ChatRunSnapshot) ProtoMessage() {} func (x *ChatRunSnapshot) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[18] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1993,7 +2685,7 @@ func (x *ChatRunSnapshot) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatRunSnapshot.ProtoReflect.Descriptor instead. func (*ChatRunSnapshot) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{18} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{28} } func (x *ChatRunSnapshot) GetRunId() string { @@ -2056,7 +2748,7 @@ type ChatSubscribeResult struct { func (x *ChatSubscribeResult) Reset() { *x = ChatSubscribeResult{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[19] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2068,7 +2760,7 @@ func (x *ChatSubscribeResult) String() string { func (*ChatSubscribeResult) ProtoMessage() {} func (x *ChatSubscribeResult) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[19] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2081,7 +2773,7 @@ func (x *ChatSubscribeResult) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatSubscribeResult.ProtoReflect.Descriptor instead. func (*ChatSubscribeResult) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{19} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{29} } func (x *ChatSubscribeResult) GetConversationId() string { @@ -2143,7 +2835,7 @@ type ChatUnsubscribeRequest struct { func (x *ChatUnsubscribeRequest) Reset() { *x = ChatUnsubscribeRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[20] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2155,7 +2847,7 @@ func (x *ChatUnsubscribeRequest) String() string { func (*ChatUnsubscribeRequest) ProtoMessage() {} func (x *ChatUnsubscribeRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[20] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2168,7 +2860,7 @@ func (x *ChatUnsubscribeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatUnsubscribeRequest.ProtoReflect.Descriptor instead. func (*ChatUnsubscribeRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{20} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{30} } func (x *ChatUnsubscribeRequest) GetConversationId() string { @@ -2187,7 +2879,7 @@ type ChatActivitiesRequest struct { func (x *ChatActivitiesRequest) Reset() { *x = ChatActivitiesRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[21] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2199,7 +2891,7 @@ func (x *ChatActivitiesRequest) String() string { func (*ChatActivitiesRequest) ProtoMessage() {} func (x *ChatActivitiesRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[21] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2212,7 +2904,7 @@ func (x *ChatActivitiesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatActivitiesRequest.ProtoReflect.Descriptor instead. func (*ChatActivitiesRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{21} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{31} } type ChatActivitiesResult struct { @@ -2224,7 +2916,7 @@ type ChatActivitiesResult struct { func (x *ChatActivitiesResult) Reset() { *x = ChatActivitiesResult{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[22] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2236,7 +2928,7 @@ func (x *ChatActivitiesResult) String() string { func (*ChatActivitiesResult) ProtoMessage() {} func (x *ChatActivitiesResult) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[22] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2249,7 +2941,7 @@ func (x *ChatActivitiesResult) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatActivitiesResult.ProtoReflect.Descriptor instead. func (*ChatActivitiesResult) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{22} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{32} } func (x *ChatActivitiesResult) GetRunningConversations() []*ChatRunActivity { @@ -2272,7 +2964,7 @@ type ChatStreamEvent struct { func (x *ChatStreamEvent) Reset() { *x = ChatStreamEvent{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[23] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2284,7 +2976,7 @@ func (x *ChatStreamEvent) String() string { func (*ChatStreamEvent) ProtoMessage() {} func (x *ChatStreamEvent) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[23] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2297,7 +2989,7 @@ func (x *ChatStreamEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatStreamEvent.ProtoReflect.Descriptor instead. func (*ChatStreamEvent) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{23} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{33} } func (x *ChatStreamEvent) GetConversationId() string { @@ -2334,7 +3026,7 @@ type ChatCommandAccepted struct { func (x *ChatCommandAccepted) Reset() { *x = ChatCommandAccepted{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[24] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2346,7 +3038,7 @@ func (x *ChatCommandAccepted) String() string { func (*ChatCommandAccepted) ProtoMessage() {} func (x *ChatCommandAccepted) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[24] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2359,7 +3051,7 @@ func (x *ChatCommandAccepted) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatCommandAccepted.ProtoReflect.Descriptor instead. func (*ChatCommandAccepted) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{24} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{34} } func (x *ChatCommandAccepted) GetRunId() string { @@ -2405,7 +3097,7 @@ type ChatCommandUpdate struct { func (x *ChatCommandUpdate) Reset() { *x = ChatCommandUpdate{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[25] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2417,7 +3109,7 @@ func (x *ChatCommandUpdate) String() string { func (*ChatCommandUpdate) ProtoMessage() {} func (x *ChatCommandUpdate) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[25] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2430,7 +3122,7 @@ func (x *ChatCommandUpdate) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatCommandUpdate.ProtoReflect.Descriptor instead. func (*ChatCommandUpdate) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{25} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{35} } func (x *ChatCommandUpdate) GetRunId() string { @@ -2485,7 +3177,7 @@ type ChatSubscriptionReset struct { func (x *ChatSubscriptionReset) Reset() { *x = ChatSubscriptionReset{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[26] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2497,7 +3189,7 @@ func (x *ChatSubscriptionReset) String() string { func (*ChatSubscriptionReset) ProtoMessage() {} func (x *ChatSubscriptionReset) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[26] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2510,7 +3202,7 @@ func (x *ChatSubscriptionReset) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatSubscriptionReset.ProtoReflect.Descriptor instead. func (*ChatSubscriptionReset) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{26} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{36} } func (x *ChatSubscriptionReset) GetConversationId() string { @@ -2532,7 +3224,7 @@ type ChatCancelResult struct { func (x *ChatCancelResult) Reset() { *x = ChatCancelResult{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[27] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2544,7 +3236,7 @@ func (x *ChatCancelResult) String() string { func (*ChatCancelResult) ProtoMessage() {} func (x *ChatCancelResult) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[27] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2557,7 +3249,7 @@ func (x *ChatCancelResult) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatCancelResult.ProtoReflect.Descriptor instead. func (*ChatCancelResult) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{27} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{37} } func (x *ChatCancelResult) GetOk() bool { @@ -2597,7 +3289,7 @@ type ChatActivityEvent struct { func (x *ChatActivityEvent) Reset() { *x = ChatActivityEvent{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[28] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2609,7 +3301,7 @@ func (x *ChatActivityEvent) String() string { func (*ChatActivityEvent) ProtoMessage() {} func (x *ChatActivityEvent) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[28] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2622,7 +3314,7 @@ func (x *ChatActivityEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use ChatActivityEvent.ProtoReflect.Descriptor instead. func (*ChatActivityEvent) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{28} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{38} } func (x *ChatActivityEvent) GetConversationId() string { @@ -2684,7 +3376,7 @@ type WorkspaceSubscribeRequest struct { func (x *WorkspaceSubscribeRequest) Reset() { *x = WorkspaceSubscribeRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[29] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2696,7 +3388,7 @@ func (x *WorkspaceSubscribeRequest) String() string { func (*WorkspaceSubscribeRequest) ProtoMessage() {} func (x *WorkspaceSubscribeRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[29] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2709,7 +3401,7 @@ func (x *WorkspaceSubscribeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceSubscribeRequest.ProtoReflect.Descriptor instead. func (*WorkspaceSubscribeRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{29} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{39} } func (x *WorkspaceSubscribeRequest) GetWorkdir() string { @@ -2729,7 +3421,7 @@ type WorkspaceUnsubscribeRequest struct { func (x *WorkspaceUnsubscribeRequest) Reset() { *x = WorkspaceUnsubscribeRequest{} - mi := &file_proto_v2_gateway_ws_proto_msgTypes[30] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2741,7 +3433,7 @@ func (x *WorkspaceUnsubscribeRequest) String() string { func (*WorkspaceUnsubscribeRequest) ProtoMessage() {} func (x *WorkspaceUnsubscribeRequest) ProtoReflect() protoreflect.Message { - mi := &file_proto_v2_gateway_ws_proto_msgTypes[30] + mi := &file_proto_v2_gateway_ws_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2754,7 +3446,7 @@ func (x *WorkspaceUnsubscribeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkspaceUnsubscribeRequest.ProtoReflect.Descriptor instead. func (*WorkspaceUnsubscribeRequest) Descriptor() ([]byte, []int) { - return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{30} + return file_proto_v2_gateway_ws_proto_rawDescGZIP(), []int{40} } func (x *WorkspaceUnsubscribeRequest) GetWorkdir() string { @@ -2794,7 +3486,52 @@ const file_proto_v2_gateway_ws_proto_rawDesc = "" + "\tPongFrame\x12\x1c\n" + "\ttimestamp\x18\x01 \x01(\x03R\ttimestamp\"\x1b\n" + "\tAckResult\x12\x0e\n" + - "\x02ok\x18\x01 \x01(\bR\x02ok\"\x9b\b\n" + + "\x02ok\x18\x01 \x01(\bR\x02ok\"Q\n" + + "\x0eSttClientHello\x12)\n" + + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x14\n" + + "\x05token\x18\x02 \x01(\tR\x05token\"E\n" + + "\bSttStart\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\x12\x1a\n" + + "\bprovider\x18\x02 \x01(\tR\bprovider\"W\n" + + "\bSttAudio\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\x12\x1a\n" + + "\bsequence\x18\x02 \x01(\rR\bsequence\x12\x10\n" + + "\x03pcm\x18\x03 \x01(\fR\x03pcm\"2\n" + + "\x11SttSessionControl\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\"\xcb\x02\n" + + "\x0eSttClientFrame\x12<\n" + + "\x05hello\x18\x01 \x01(\v2$.liveagent.gateway.v2.SttClientHelloH\x00R\x05hello\x126\n" + + "\x05start\x18\x02 \x01(\v2\x1e.liveagent.gateway.v2.SttStartH\x00R\x05start\x126\n" + + "\x05audio\x18\x03 \x01(\v2\x1e.liveagent.gateway.v2.SttAudioH\x00R\x05audio\x12=\n" + + "\x04stop\x18\x04 \x01(\v2'.liveagent.gateway.v2.SttSessionControlH\x00R\x04stop\x12A\n" + + "\x06cancel\x18\x05 \x01(\v2'.liveagent.gateway.v2.SttSessionControlH\x00R\x06cancelB\t\n" + + "\apayload\":\n" + + "\x0eSttServerHello\x12\x0e\n" + + "\x02ok\x18\x01 \x01(\bR\x02ok\x12\x18\n" + + "\amessage\x18\x02 \x01(\tR\amessage\"A\n" + + "\fSttTextEvent\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\x12\x12\n" + + "\x04text\x18\x02 \x01(\tR\x04text\"W\n" + + "\bSttError\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\x12\x12\n" + + "\x04code\x18\x02 \x01(\tR\x04code\x12\x18\n" + + "\amessage\x18\x03 \x01(\tR\amessage\"*\n" + + "\tSttClosed\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\"\x84\x03\n" + + "\x0eSttServerFrame\x12<\n" + + "\x05hello\x18\x01 \x01(\v2$.liveagent.gateway.v2.SttServerHelloH\x00R\x05hello\x12:\n" + + "\x05ready\x18\x02 \x01(\v2\".liveagent.gateway.v2.SttTextEventH\x00R\x05ready\x12>\n" + + "\apartial\x18\x03 \x01(\v2\".liveagent.gateway.v2.SttTextEventH\x00R\apartial\x12:\n" + + "\x05final\x18\x04 \x01(\v2\".liveagent.gateway.v2.SttTextEventH\x00R\x05final\x126\n" + + "\x05error\x18\x05 \x01(\v2\x1e.liveagent.gateway.v2.SttErrorH\x00R\x05error\x129\n" + + "\x06closed\x18\x06 \x01(\v2\x1f.liveagent.gateway.v2.SttClosedH\x00R\x06closedB\t\n" + + "\apayload\"\x9b\b\n" + "\x0eWebClientFrame\x12\x1d\n" + "\n" + "request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" + @@ -2983,7 +3720,7 @@ func file_proto_v2_gateway_ws_proto_rawDescGZIP() []byte { } var file_proto_v2_gateway_ws_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_proto_v2_gateway_ws_proto_msgTypes = make([]protoimpl.MessageInfo, 31) +var file_proto_v2_gateway_ws_proto_msgTypes = make([]protoimpl.MessageInfo, 41) var file_proto_v2_gateway_ws_proto_goTypes = []any{ (ClientRole)(0), // 0: liveagent.gateway.v2.ClientRole (*ClientHello)(nil), // 1: liveagent.gateway.v2.ClientHello @@ -2991,100 +3728,121 @@ var file_proto_v2_gateway_ws_proto_goTypes = []any{ (*PingFrame)(nil), // 3: liveagent.gateway.v2.PingFrame (*PongFrame)(nil), // 4: liveagent.gateway.v2.PongFrame (*AckResult)(nil), // 5: liveagent.gateway.v2.AckResult - (*WebClientFrame)(nil), // 6: liveagent.gateway.v2.WebClientFrame - (*WebServerFrame)(nil), // 7: liveagent.gateway.v2.WebServerFrame - (*AgentListRequest)(nil), // 8: liveagent.gateway.v2.AgentListRequest - (*AgentListResult)(nil), // 9: liveagent.gateway.v2.AgentListResult - (*AgentClientFrame)(nil), // 10: liveagent.gateway.v2.AgentClientFrame - (*AgentServerFrame)(nil), // 11: liveagent.gateway.v2.AgentServerFrame - (*TerminalClientFrame)(nil), // 12: liveagent.gateway.v2.TerminalClientFrame - (*TerminalServerFrame)(nil), // 13: liveagent.gateway.v2.TerminalServerFrame - (*StatusGetRequest)(nil), // 14: liveagent.gateway.v2.StatusGetRequest - (*StatusEvent)(nil), // 15: liveagent.gateway.v2.StatusEvent - (*ChatPrepareRequest)(nil), // 16: liveagent.gateway.v2.ChatPrepareRequest - (*ChatSubscribeRequest)(nil), // 17: liveagent.gateway.v2.ChatSubscribeRequest - (*ChatRunActivity)(nil), // 18: liveagent.gateway.v2.ChatRunActivity - (*ChatRunSnapshot)(nil), // 19: liveagent.gateway.v2.ChatRunSnapshot - (*ChatSubscribeResult)(nil), // 20: liveagent.gateway.v2.ChatSubscribeResult - (*ChatUnsubscribeRequest)(nil), // 21: liveagent.gateway.v2.ChatUnsubscribeRequest - (*ChatActivitiesRequest)(nil), // 22: liveagent.gateway.v2.ChatActivitiesRequest - (*ChatActivitiesResult)(nil), // 23: liveagent.gateway.v2.ChatActivitiesResult - (*ChatStreamEvent)(nil), // 24: liveagent.gateway.v2.ChatStreamEvent - (*ChatCommandAccepted)(nil), // 25: liveagent.gateway.v2.ChatCommandAccepted - (*ChatCommandUpdate)(nil), // 26: liveagent.gateway.v2.ChatCommandUpdate - (*ChatSubscriptionReset)(nil), // 27: liveagent.gateway.v2.ChatSubscriptionReset - (*ChatCancelResult)(nil), // 28: liveagent.gateway.v2.ChatCancelResult - (*ChatActivityEvent)(nil), // 29: liveagent.gateway.v2.ChatActivityEvent - (*WorkspaceSubscribeRequest)(nil), // 30: liveagent.gateway.v2.WorkspaceSubscribeRequest - (*WorkspaceUnsubscribeRequest)(nil), // 31: liveagent.gateway.v2.WorkspaceUnsubscribeRequest - (*GatewayEnvelope)(nil), // 32: liveagent.gateway.v2.GatewayEnvelope - (*ChatCommandRequest)(nil), // 33: liveagent.gateway.v2.ChatCommandRequest - (*AgentEnvelope)(nil), // 34: liveagent.gateway.v2.AgentEnvelope - (*ErrorResponse)(nil), // 35: liveagent.gateway.v2.ErrorResponse - (*HistorySyncEvent)(nil), // 36: liveagent.gateway.v2.HistorySyncEvent - (*SettingsSyncEvent)(nil), // 37: liveagent.gateway.v2.SettingsSyncEvent - (*TerminalEvent)(nil), // 38: liveagent.gateway.v2.TerminalEvent - (*SftpEvent)(nil), // 39: liveagent.gateway.v2.SftpEvent - (*ChatQueueEvent)(nil), // 40: liveagent.gateway.v2.ChatQueueEvent - (*TunnelStateSnapshot)(nil), // 41: liveagent.gateway.v2.TunnelStateSnapshot - (*ManagedProcessSnapshot)(nil), // 42: liveagent.gateway.v2.ManagedProcessSnapshot - (*WorkspaceActivityEvent)(nil), // 43: liveagent.gateway.v2.WorkspaceActivityEvent - (*TerminalStreamFrame)(nil), // 44: liveagent.gateway.v2.TerminalStreamFrame + (*SttClientHello)(nil), // 6: liveagent.gateway.v2.SttClientHello + (*SttStart)(nil), // 7: liveagent.gateway.v2.SttStart + (*SttAudio)(nil), // 8: liveagent.gateway.v2.SttAudio + (*SttSessionControl)(nil), // 9: liveagent.gateway.v2.SttSessionControl + (*SttClientFrame)(nil), // 10: liveagent.gateway.v2.SttClientFrame + (*SttServerHello)(nil), // 11: liveagent.gateway.v2.SttServerHello + (*SttTextEvent)(nil), // 12: liveagent.gateway.v2.SttTextEvent + (*SttError)(nil), // 13: liveagent.gateway.v2.SttError + (*SttClosed)(nil), // 14: liveagent.gateway.v2.SttClosed + (*SttServerFrame)(nil), // 15: liveagent.gateway.v2.SttServerFrame + (*WebClientFrame)(nil), // 16: liveagent.gateway.v2.WebClientFrame + (*WebServerFrame)(nil), // 17: liveagent.gateway.v2.WebServerFrame + (*AgentListRequest)(nil), // 18: liveagent.gateway.v2.AgentListRequest + (*AgentListResult)(nil), // 19: liveagent.gateway.v2.AgentListResult + (*AgentClientFrame)(nil), // 20: liveagent.gateway.v2.AgentClientFrame + (*AgentServerFrame)(nil), // 21: liveagent.gateway.v2.AgentServerFrame + (*TerminalClientFrame)(nil), // 22: liveagent.gateway.v2.TerminalClientFrame + (*TerminalServerFrame)(nil), // 23: liveagent.gateway.v2.TerminalServerFrame + (*StatusGetRequest)(nil), // 24: liveagent.gateway.v2.StatusGetRequest + (*StatusEvent)(nil), // 25: liveagent.gateway.v2.StatusEvent + (*ChatPrepareRequest)(nil), // 26: liveagent.gateway.v2.ChatPrepareRequest + (*ChatSubscribeRequest)(nil), // 27: liveagent.gateway.v2.ChatSubscribeRequest + (*ChatRunActivity)(nil), // 28: liveagent.gateway.v2.ChatRunActivity + (*ChatRunSnapshot)(nil), // 29: liveagent.gateway.v2.ChatRunSnapshot + (*ChatSubscribeResult)(nil), // 30: liveagent.gateway.v2.ChatSubscribeResult + (*ChatUnsubscribeRequest)(nil), // 31: liveagent.gateway.v2.ChatUnsubscribeRequest + (*ChatActivitiesRequest)(nil), // 32: liveagent.gateway.v2.ChatActivitiesRequest + (*ChatActivitiesResult)(nil), // 33: liveagent.gateway.v2.ChatActivitiesResult + (*ChatStreamEvent)(nil), // 34: liveagent.gateway.v2.ChatStreamEvent + (*ChatCommandAccepted)(nil), // 35: liveagent.gateway.v2.ChatCommandAccepted + (*ChatCommandUpdate)(nil), // 36: liveagent.gateway.v2.ChatCommandUpdate + (*ChatSubscriptionReset)(nil), // 37: liveagent.gateway.v2.ChatSubscriptionReset + (*ChatCancelResult)(nil), // 38: liveagent.gateway.v2.ChatCancelResult + (*ChatActivityEvent)(nil), // 39: liveagent.gateway.v2.ChatActivityEvent + (*WorkspaceSubscribeRequest)(nil), // 40: liveagent.gateway.v2.WorkspaceSubscribeRequest + (*WorkspaceUnsubscribeRequest)(nil), // 41: liveagent.gateway.v2.WorkspaceUnsubscribeRequest + (*GatewayEnvelope)(nil), // 42: liveagent.gateway.v2.GatewayEnvelope + (*ChatCommandRequest)(nil), // 43: liveagent.gateway.v2.ChatCommandRequest + (*AgentEnvelope)(nil), // 44: liveagent.gateway.v2.AgentEnvelope + (*ErrorResponse)(nil), // 45: liveagent.gateway.v2.ErrorResponse + (*HistorySyncEvent)(nil), // 46: liveagent.gateway.v2.HistorySyncEvent + (*SettingsSyncEvent)(nil), // 47: liveagent.gateway.v2.SettingsSyncEvent + (*TerminalEvent)(nil), // 48: liveagent.gateway.v2.TerminalEvent + (*SftpEvent)(nil), // 49: liveagent.gateway.v2.SftpEvent + (*ChatQueueEvent)(nil), // 50: liveagent.gateway.v2.ChatQueueEvent + (*TunnelStateSnapshot)(nil), // 51: liveagent.gateway.v2.TunnelStateSnapshot + (*ManagedProcessSnapshot)(nil), // 52: liveagent.gateway.v2.ManagedProcessSnapshot + (*WorkspaceActivityEvent)(nil), // 53: liveagent.gateway.v2.WorkspaceActivityEvent + (*TerminalStreamFrame)(nil), // 54: liveagent.gateway.v2.TerminalStreamFrame } var file_proto_v2_gateway_ws_proto_depIdxs = []int32{ 0, // 0: liveagent.gateway.v2.ClientHello.role:type_name -> liveagent.gateway.v2.ClientRole - 1, // 1: liveagent.gateway.v2.WebClientFrame.hello:type_name -> liveagent.gateway.v2.ClientHello - 32, // 2: liveagent.gateway.v2.WebClientFrame.agent_request:type_name -> liveagent.gateway.v2.GatewayEnvelope - 14, // 3: liveagent.gateway.v2.WebClientFrame.status_get:type_name -> liveagent.gateway.v2.StatusGetRequest - 33, // 4: liveagent.gateway.v2.WebClientFrame.chat_command:type_name -> liveagent.gateway.v2.ChatCommandRequest - 16, // 5: liveagent.gateway.v2.WebClientFrame.chat_prepare:type_name -> liveagent.gateway.v2.ChatPrepareRequest - 17, // 6: liveagent.gateway.v2.WebClientFrame.chat_subscribe:type_name -> liveagent.gateway.v2.ChatSubscribeRequest - 21, // 7: liveagent.gateway.v2.WebClientFrame.chat_unsubscribe:type_name -> liveagent.gateway.v2.ChatUnsubscribeRequest - 22, // 8: liveagent.gateway.v2.WebClientFrame.chat_activities:type_name -> liveagent.gateway.v2.ChatActivitiesRequest - 30, // 9: liveagent.gateway.v2.WebClientFrame.workspace_subscribe:type_name -> liveagent.gateway.v2.WorkspaceSubscribeRequest - 31, // 10: liveagent.gateway.v2.WebClientFrame.workspace_unsubscribe:type_name -> liveagent.gateway.v2.WorkspaceUnsubscribeRequest - 4, // 11: liveagent.gateway.v2.WebClientFrame.pong:type_name -> liveagent.gateway.v2.PongFrame - 8, // 12: liveagent.gateway.v2.WebClientFrame.agent_list:type_name -> liveagent.gateway.v2.AgentListRequest - 2, // 13: liveagent.gateway.v2.WebServerFrame.hello:type_name -> liveagent.gateway.v2.ServerHello - 34, // 14: liveagent.gateway.v2.WebServerFrame.agent_response:type_name -> liveagent.gateway.v2.AgentEnvelope - 35, // 15: liveagent.gateway.v2.WebServerFrame.local_error:type_name -> liveagent.gateway.v2.ErrorResponse - 3, // 16: liveagent.gateway.v2.WebServerFrame.ping:type_name -> liveagent.gateway.v2.PingFrame - 15, // 17: liveagent.gateway.v2.WebServerFrame.status:type_name -> liveagent.gateway.v2.StatusEvent - 20, // 18: liveagent.gateway.v2.WebServerFrame.chat_subscribed:type_name -> liveagent.gateway.v2.ChatSubscribeResult - 25, // 19: liveagent.gateway.v2.WebServerFrame.chat_accepted:type_name -> liveagent.gateway.v2.ChatCommandAccepted - 23, // 20: liveagent.gateway.v2.WebServerFrame.chat_activities:type_name -> liveagent.gateway.v2.ChatActivitiesResult - 24, // 21: liveagent.gateway.v2.WebServerFrame.chat_event:type_name -> liveagent.gateway.v2.ChatStreamEvent - 26, // 22: liveagent.gateway.v2.WebServerFrame.chat_command_update:type_name -> liveagent.gateway.v2.ChatCommandUpdate - 27, // 23: liveagent.gateway.v2.WebServerFrame.chat_subscription_reset:type_name -> liveagent.gateway.v2.ChatSubscriptionReset - 29, // 24: liveagent.gateway.v2.WebServerFrame.chat_activity:type_name -> liveagent.gateway.v2.ChatActivityEvent - 5, // 25: liveagent.gateway.v2.WebServerFrame.ack:type_name -> liveagent.gateway.v2.AckResult - 28, // 26: liveagent.gateway.v2.WebServerFrame.chat_cancelled:type_name -> liveagent.gateway.v2.ChatCancelResult - 9, // 27: liveagent.gateway.v2.WebServerFrame.agent_list:type_name -> liveagent.gateway.v2.AgentListResult - 36, // 28: liveagent.gateway.v2.WebServerFrame.history_event:type_name -> liveagent.gateway.v2.HistorySyncEvent - 37, // 29: liveagent.gateway.v2.WebServerFrame.settings_event:type_name -> liveagent.gateway.v2.SettingsSyncEvent - 38, // 30: liveagent.gateway.v2.WebServerFrame.terminal_event:type_name -> liveagent.gateway.v2.TerminalEvent - 39, // 31: liveagent.gateway.v2.WebServerFrame.sftp_event:type_name -> liveagent.gateway.v2.SftpEvent - 40, // 32: liveagent.gateway.v2.WebServerFrame.chat_queue_event:type_name -> liveagent.gateway.v2.ChatQueueEvent - 41, // 33: liveagent.gateway.v2.WebServerFrame.tunnel_state:type_name -> liveagent.gateway.v2.TunnelStateSnapshot - 42, // 34: liveagent.gateway.v2.WebServerFrame.process_state:type_name -> liveagent.gateway.v2.ManagedProcessSnapshot - 43, // 35: liveagent.gateway.v2.WebServerFrame.workspace_activity:type_name -> liveagent.gateway.v2.WorkspaceActivityEvent - 15, // 36: liveagent.gateway.v2.AgentListResult.agents:type_name -> liveagent.gateway.v2.StatusEvent - 1, // 37: liveagent.gateway.v2.AgentClientFrame.hello:type_name -> liveagent.gateway.v2.ClientHello - 34, // 38: liveagent.gateway.v2.AgentClientFrame.envelope:type_name -> liveagent.gateway.v2.AgentEnvelope - 2, // 39: liveagent.gateway.v2.AgentServerFrame.hello:type_name -> liveagent.gateway.v2.ServerHello - 32, // 40: liveagent.gateway.v2.AgentServerFrame.envelope:type_name -> liveagent.gateway.v2.GatewayEnvelope - 1, // 41: liveagent.gateway.v2.TerminalClientFrame.hello:type_name -> liveagent.gateway.v2.ClientHello - 44, // 42: liveagent.gateway.v2.TerminalClientFrame.frame:type_name -> liveagent.gateway.v2.TerminalStreamFrame - 2, // 43: liveagent.gateway.v2.TerminalServerFrame.hello:type_name -> liveagent.gateway.v2.ServerHello - 44, // 44: liveagent.gateway.v2.TerminalServerFrame.frame:type_name -> liveagent.gateway.v2.TerminalStreamFrame - 18, // 45: liveagent.gateway.v2.ChatSubscribeResult.activity:type_name -> liveagent.gateway.v2.ChatRunActivity - 19, // 46: liveagent.gateway.v2.ChatSubscribeResult.snapshot:type_name -> liveagent.gateway.v2.ChatRunSnapshot - 18, // 47: liveagent.gateway.v2.ChatActivitiesResult.running_conversations:type_name -> liveagent.gateway.v2.ChatRunActivity - 48, // [48:48] is the sub-list for method output_type - 48, // [48:48] is the sub-list for method input_type - 48, // [48:48] is the sub-list for extension type_name - 48, // [48:48] is the sub-list for extension extendee - 0, // [0:48] is the sub-list for field type_name + 6, // 1: liveagent.gateway.v2.SttClientFrame.hello:type_name -> liveagent.gateway.v2.SttClientHello + 7, // 2: liveagent.gateway.v2.SttClientFrame.start:type_name -> liveagent.gateway.v2.SttStart + 8, // 3: liveagent.gateway.v2.SttClientFrame.audio:type_name -> liveagent.gateway.v2.SttAudio + 9, // 4: liveagent.gateway.v2.SttClientFrame.stop:type_name -> liveagent.gateway.v2.SttSessionControl + 9, // 5: liveagent.gateway.v2.SttClientFrame.cancel:type_name -> liveagent.gateway.v2.SttSessionControl + 11, // 6: liveagent.gateway.v2.SttServerFrame.hello:type_name -> liveagent.gateway.v2.SttServerHello + 12, // 7: liveagent.gateway.v2.SttServerFrame.ready:type_name -> liveagent.gateway.v2.SttTextEvent + 12, // 8: liveagent.gateway.v2.SttServerFrame.partial:type_name -> liveagent.gateway.v2.SttTextEvent + 12, // 9: liveagent.gateway.v2.SttServerFrame.final:type_name -> liveagent.gateway.v2.SttTextEvent + 13, // 10: liveagent.gateway.v2.SttServerFrame.error:type_name -> liveagent.gateway.v2.SttError + 14, // 11: liveagent.gateway.v2.SttServerFrame.closed:type_name -> liveagent.gateway.v2.SttClosed + 1, // 12: liveagent.gateway.v2.WebClientFrame.hello:type_name -> liveagent.gateway.v2.ClientHello + 42, // 13: liveagent.gateway.v2.WebClientFrame.agent_request:type_name -> liveagent.gateway.v2.GatewayEnvelope + 24, // 14: liveagent.gateway.v2.WebClientFrame.status_get:type_name -> liveagent.gateway.v2.StatusGetRequest + 43, // 15: liveagent.gateway.v2.WebClientFrame.chat_command:type_name -> liveagent.gateway.v2.ChatCommandRequest + 26, // 16: liveagent.gateway.v2.WebClientFrame.chat_prepare:type_name -> liveagent.gateway.v2.ChatPrepareRequest + 27, // 17: liveagent.gateway.v2.WebClientFrame.chat_subscribe:type_name -> liveagent.gateway.v2.ChatSubscribeRequest + 31, // 18: liveagent.gateway.v2.WebClientFrame.chat_unsubscribe:type_name -> liveagent.gateway.v2.ChatUnsubscribeRequest + 32, // 19: liveagent.gateway.v2.WebClientFrame.chat_activities:type_name -> liveagent.gateway.v2.ChatActivitiesRequest + 40, // 20: liveagent.gateway.v2.WebClientFrame.workspace_subscribe:type_name -> liveagent.gateway.v2.WorkspaceSubscribeRequest + 41, // 21: liveagent.gateway.v2.WebClientFrame.workspace_unsubscribe:type_name -> liveagent.gateway.v2.WorkspaceUnsubscribeRequest + 4, // 22: liveagent.gateway.v2.WebClientFrame.pong:type_name -> liveagent.gateway.v2.PongFrame + 18, // 23: liveagent.gateway.v2.WebClientFrame.agent_list:type_name -> liveagent.gateway.v2.AgentListRequest + 2, // 24: liveagent.gateway.v2.WebServerFrame.hello:type_name -> liveagent.gateway.v2.ServerHello + 44, // 25: liveagent.gateway.v2.WebServerFrame.agent_response:type_name -> liveagent.gateway.v2.AgentEnvelope + 45, // 26: liveagent.gateway.v2.WebServerFrame.local_error:type_name -> liveagent.gateway.v2.ErrorResponse + 3, // 27: liveagent.gateway.v2.WebServerFrame.ping:type_name -> liveagent.gateway.v2.PingFrame + 25, // 28: liveagent.gateway.v2.WebServerFrame.status:type_name -> liveagent.gateway.v2.StatusEvent + 30, // 29: liveagent.gateway.v2.WebServerFrame.chat_subscribed:type_name -> liveagent.gateway.v2.ChatSubscribeResult + 35, // 30: liveagent.gateway.v2.WebServerFrame.chat_accepted:type_name -> liveagent.gateway.v2.ChatCommandAccepted + 33, // 31: liveagent.gateway.v2.WebServerFrame.chat_activities:type_name -> liveagent.gateway.v2.ChatActivitiesResult + 34, // 32: liveagent.gateway.v2.WebServerFrame.chat_event:type_name -> liveagent.gateway.v2.ChatStreamEvent + 36, // 33: liveagent.gateway.v2.WebServerFrame.chat_command_update:type_name -> liveagent.gateway.v2.ChatCommandUpdate + 37, // 34: liveagent.gateway.v2.WebServerFrame.chat_subscription_reset:type_name -> liveagent.gateway.v2.ChatSubscriptionReset + 39, // 35: liveagent.gateway.v2.WebServerFrame.chat_activity:type_name -> liveagent.gateway.v2.ChatActivityEvent + 5, // 36: liveagent.gateway.v2.WebServerFrame.ack:type_name -> liveagent.gateway.v2.AckResult + 38, // 37: liveagent.gateway.v2.WebServerFrame.chat_cancelled:type_name -> liveagent.gateway.v2.ChatCancelResult + 19, // 38: liveagent.gateway.v2.WebServerFrame.agent_list:type_name -> liveagent.gateway.v2.AgentListResult + 46, // 39: liveagent.gateway.v2.WebServerFrame.history_event:type_name -> liveagent.gateway.v2.HistorySyncEvent + 47, // 40: liveagent.gateway.v2.WebServerFrame.settings_event:type_name -> liveagent.gateway.v2.SettingsSyncEvent + 48, // 41: liveagent.gateway.v2.WebServerFrame.terminal_event:type_name -> liveagent.gateway.v2.TerminalEvent + 49, // 42: liveagent.gateway.v2.WebServerFrame.sftp_event:type_name -> liveagent.gateway.v2.SftpEvent + 50, // 43: liveagent.gateway.v2.WebServerFrame.chat_queue_event:type_name -> liveagent.gateway.v2.ChatQueueEvent + 51, // 44: liveagent.gateway.v2.WebServerFrame.tunnel_state:type_name -> liveagent.gateway.v2.TunnelStateSnapshot + 52, // 45: liveagent.gateway.v2.WebServerFrame.process_state:type_name -> liveagent.gateway.v2.ManagedProcessSnapshot + 53, // 46: liveagent.gateway.v2.WebServerFrame.workspace_activity:type_name -> liveagent.gateway.v2.WorkspaceActivityEvent + 25, // 47: liveagent.gateway.v2.AgentListResult.agents:type_name -> liveagent.gateway.v2.StatusEvent + 1, // 48: liveagent.gateway.v2.AgentClientFrame.hello:type_name -> liveagent.gateway.v2.ClientHello + 44, // 49: liveagent.gateway.v2.AgentClientFrame.envelope:type_name -> liveagent.gateway.v2.AgentEnvelope + 2, // 50: liveagent.gateway.v2.AgentServerFrame.hello:type_name -> liveagent.gateway.v2.ServerHello + 42, // 51: liveagent.gateway.v2.AgentServerFrame.envelope:type_name -> liveagent.gateway.v2.GatewayEnvelope + 1, // 52: liveagent.gateway.v2.TerminalClientFrame.hello:type_name -> liveagent.gateway.v2.ClientHello + 54, // 53: liveagent.gateway.v2.TerminalClientFrame.frame:type_name -> liveagent.gateway.v2.TerminalStreamFrame + 2, // 54: liveagent.gateway.v2.TerminalServerFrame.hello:type_name -> liveagent.gateway.v2.ServerHello + 54, // 55: liveagent.gateway.v2.TerminalServerFrame.frame:type_name -> liveagent.gateway.v2.TerminalStreamFrame + 28, // 56: liveagent.gateway.v2.ChatSubscribeResult.activity:type_name -> liveagent.gateway.v2.ChatRunActivity + 29, // 57: liveagent.gateway.v2.ChatSubscribeResult.snapshot:type_name -> liveagent.gateway.v2.ChatRunSnapshot + 28, // 58: liveagent.gateway.v2.ChatActivitiesResult.running_conversations:type_name -> liveagent.gateway.v2.ChatRunActivity + 59, // [59:59] is the sub-list for method output_type + 59, // [59:59] is the sub-list for method input_type + 59, // [59:59] is the sub-list for extension type_name + 59, // [59:59] is the sub-list for extension extendee + 0, // [0:59] is the sub-list for field type_name } func init() { file_proto_v2_gateway_ws_proto_init() } @@ -3093,7 +3851,22 @@ func file_proto_v2_gateway_ws_proto_init() { return } file_proto_v2_gateway_proto_init() - file_proto_v2_gateway_ws_proto_msgTypes[5].OneofWrappers = []any{ + file_proto_v2_gateway_ws_proto_msgTypes[9].OneofWrappers = []any{ + (*SttClientFrame_Hello)(nil), + (*SttClientFrame_Start)(nil), + (*SttClientFrame_Audio)(nil), + (*SttClientFrame_Stop)(nil), + (*SttClientFrame_Cancel)(nil), + } + file_proto_v2_gateway_ws_proto_msgTypes[14].OneofWrappers = []any{ + (*SttServerFrame_Hello)(nil), + (*SttServerFrame_Ready)(nil), + (*SttServerFrame_Partial)(nil), + (*SttServerFrame_Final)(nil), + (*SttServerFrame_Error)(nil), + (*SttServerFrame_Closed)(nil), + } + file_proto_v2_gateway_ws_proto_msgTypes[15].OneofWrappers = []any{ (*WebClientFrame_Hello)(nil), (*WebClientFrame_AgentRequest)(nil), (*WebClientFrame_StatusGet)(nil), @@ -3107,7 +3880,7 @@ func file_proto_v2_gateway_ws_proto_init() { (*WebClientFrame_Pong)(nil), (*WebClientFrame_AgentList)(nil), } - file_proto_v2_gateway_ws_proto_msgTypes[6].OneofWrappers = []any{ + file_proto_v2_gateway_ws_proto_msgTypes[16].OneofWrappers = []any{ (*WebServerFrame_Hello)(nil), (*WebServerFrame_AgentResponse)(nil), (*WebServerFrame_LocalError)(nil), @@ -3132,19 +3905,19 @@ func file_proto_v2_gateway_ws_proto_init() { (*WebServerFrame_ProcessState)(nil), (*WebServerFrame_WorkspaceActivity)(nil), } - file_proto_v2_gateway_ws_proto_msgTypes[9].OneofWrappers = []any{ + file_proto_v2_gateway_ws_proto_msgTypes[19].OneofWrappers = []any{ (*AgentClientFrame_Hello)(nil), (*AgentClientFrame_Envelope)(nil), } - file_proto_v2_gateway_ws_proto_msgTypes[10].OneofWrappers = []any{ + file_proto_v2_gateway_ws_proto_msgTypes[20].OneofWrappers = []any{ (*AgentServerFrame_Hello)(nil), (*AgentServerFrame_Envelope)(nil), } - file_proto_v2_gateway_ws_proto_msgTypes[11].OneofWrappers = []any{ + file_proto_v2_gateway_ws_proto_msgTypes[21].OneofWrappers = []any{ (*TerminalClientFrame_Hello)(nil), (*TerminalClientFrame_Frame)(nil), } - file_proto_v2_gateway_ws_proto_msgTypes[12].OneofWrappers = []any{ + file_proto_v2_gateway_ws_proto_msgTypes[22].OneofWrappers = []any{ (*TerminalServerFrame_Hello)(nil), (*TerminalServerFrame_Frame)(nil), } @@ -3154,7 +3927,7 @@ func file_proto_v2_gateway_ws_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_v2_gateway_ws_proto_rawDesc), len(file_proto_v2_gateway_ws_proto_rawDesc)), NumEnums: 1, - NumMessages: 31, + NumMessages: 41, NumExtensions: 0, NumServices: 0, }, diff --git a/crates/agent-gateway/internal/protocol/pbws/handshake.go b/crates/agent-gateway/internal/protocol/pbws/handshake.go index 69169344f..eabc61341 100644 --- a/crates/agent-gateway/internal/protocol/pbws/handshake.go +++ b/crates/agent-gateway/internal/protocol/pbws/handshake.go @@ -65,7 +65,10 @@ func (s *Server) serverHello(ok bool, message string, sessionID string, maxMessa ServerTime: time.Now().Unix(), HeartbeatPeriodSeconds: uint32(s.heartbeatPeriod() / time.Second), MaxMessageBytes: uint64(maxMessageBytes), - Capabilities: []string{gatewayv2.ChatIngressV1Capability}, + Capabilities: []string{ + gatewayv2.ChatIngressV1Capability, + gatewayv2.SttStreamV1Capability, + }, } } diff --git a/crates/agent-gateway/internal/protocol/pbws/handshake_test.go b/crates/agent-gateway/internal/protocol/pbws/handshake_test.go index a960aa124..c4a9a25b5 100644 --- a/crates/agent-gateway/internal/protocol/pbws/handshake_test.go +++ b/crates/agent-gateway/internal/protocol/pbws/handshake_test.go @@ -6,10 +6,11 @@ import ( gatewayv2 "github.com/liveagent/agent-gateway/internal/proto/v2" ) -func TestServerHelloAdvertisesChatIngressV1(t *testing.T) { +func TestServerHelloAdvertisesGatewayCapabilities(t *testing.T) { hello := (&Server{}).serverHello(true, "", "session-1", 1024) - if got := hello.GetCapabilities(); len(got) != 1 || got[0] != gatewayv2.ChatIngressV1Capability { - t.Fatalf("server hello capabilities = %v, want [%q]", got, gatewayv2.ChatIngressV1Capability) + got := hello.GetCapabilities() + if len(got) != 2 || got[0] != gatewayv2.ChatIngressV1Capability || got[1] != gatewayv2.SttStreamV1Capability { + t.Fatalf("server hello capabilities = %v", got) } } diff --git a/crates/agent-gateway/internal/server/http.go b/crates/agent-gateway/internal/server/http.go index 416508c82..d88418cff 100644 --- a/crates/agent-gateway/internal/server/http.go +++ b/crates/agent-gateway/internal/server/http.go @@ -20,10 +20,11 @@ import ( gatewayv2 "github.com/liveagent/agent-gateway/internal/proto/v2" "github.com/liveagent/agent-gateway/internal/protocol/pbws" "github.com/liveagent/agent-gateway/internal/session" + "github.com/liveagent/agent-gateway/internal/stt" ) // NewHTTPServer 构造 HTTP 路由；生产启动时 tokens 始终是已初始化的 Agent 目录与凭证存储。 -func NewHTTPServer(cfg *config.Config, sm *session.Manager, tokens *agenttoken.Store) http.Handler { +func NewHTTPServer(cfg *config.Config, sm *session.Manager, tokens *agenttoken.Store, sttManagers ...*stt.Manager) http.Handler { rootMux := http.NewServeMux() rootMux.HandleFunc("GET /healthz", handler.Health()) @@ -32,6 +33,13 @@ func NewHTTPServer(cfg *config.Config, sm *session.Manager, tokens *agenttoken.S rootMux.Handle("/ws/v2", v2.BrowserHandler()) rootMux.Handle("/ws/v2/agent", v2.AgentHandler()) rootMux.Handle("/ws/v2/terminal", v2.TerminalHandler()) + var sttManager *stt.Manager + if len(sttManagers) > 0 { + sttManager = sttManagers[0] + } + if sttManager != nil { + rootMux.Handle("/ws/v2/stt", sttManager.WebSocketHandler(cfg.Token)) + } rootMux.HandleFunc("/t/", publicTunnelProxy(sm)) rootMux.HandleFunc("GET /image-proxy", handler.ImageProxy(cfg.RequestTimeout)) @@ -46,6 +54,10 @@ func NewHTTPServer(cfg *config.Config, sm *session.Manager, tokens *agenttoken.S apiMux.HandleFunc("POST /api/agents/{id}/token", handler.IssueAgentToken(sm, tokens)) apiMux.HandleFunc("PATCH /api/agents/{id}", handler.UpdateAgentName(tokens)) apiMux.HandleFunc("DELETE /api/agents/{id}", handler.DeleteAgent(sm, tokens)) + if sttManager != nil { + apiMux.Handle("/api/v2/stt/settings", sttManager.SettingsHandler()) + apiMux.Handle("/api/v2/stt/settings/test", sttManager.TestHandler()) + } rootMux.Handle("/api/", auth.HTTPMiddleware(cfg.Token, apiMux)) webFS, err := fs.Sub(gateway.WebUIAssets, "web/dist") diff --git a/crates/agent-gateway/internal/session/manager.go b/crates/agent-gateway/internal/session/manager.go index 76ac947af..c3c75697f 100644 --- a/crates/agent-gateway/internal/session/manager.go +++ b/crates/agent-gateway/internal/session/manager.go @@ -1,6 +1,8 @@ package session import ( + "context" + "encoding/json" "errors" "sync" "time" @@ -34,6 +36,7 @@ type Manager struct { workspaceHub *workspaceActivityHub managedProcesses *managedProcessHub statusSubs *statusSubscriberHub + sttSettingsSync func(context.Context, json.RawMessage) (any, error) } type AgentSession struct { diff --git a/crates/agent-gateway/internal/session/manager_settings_sync.go b/crates/agent-gateway/internal/session/manager_settings_sync.go index 307c30dfb..e141d9438 100644 --- a/crates/agent-gateway/internal/session/manager_settings_sync.go +++ b/crates/agent-gateway/internal/session/manager_settings_sync.go @@ -1,12 +1,64 @@ package session import ( + "context" "encoding/json" + "log/slog" "strings" gatewayv2 "github.com/liveagent/agent-gateway/internal/proto/v2" ) +const sttSecretSyncField = "sttSecretSync" + +// SetSTTSettingsSyncHandler wires the trusted desktop-agent credential sync to +// the Gateway STT store. The private field is always removed before snapshots +// are cached or events are fanned out to browser subscribers. +func (m *Manager) SetSTTSettingsSyncHandler( + handler func(context.Context, json.RawMessage) (any, error), +) { + m.sttSettingsSync = handler +} + +func (m *Manager) consumePrivateSTTSettings(event *gatewayv2.SettingsSyncEvent) *gatewayv2.SettingsSyncEvent { + if event == nil { + return nil + } + payload, ok := parseSettingsJSON(event.GetSettingsJson()) + if !ok { + return event + } + private, exists := payload[sttSecretSyncField] + if !exists { + return event + } + // Delete first and on every path: private credentials must never enter a + // browser-visible snapshot, including when persistence or decoding fails. + delete(payload, sttSecretSyncField) + if m.sttSettingsSync != nil { + raw, err := json.Marshal(private) + if err == nil { + var redacted any + redacted, err = m.sttSettingsSync(context.Background(), raw) + if err == nil { + payload["stt"] = redacted + } + } + if err != nil { + slog.Warn("sync desktop STT settings to gateway failed", "err", err) + } + } + settingsJSON, err := json.Marshal(payload) + if err != nil { + // The parsed payload was valid JSON and deleting a map key cannot make it + // unmarshalable in normal operation. Drop the event if that invariant is + // ever violated rather than risk forwarding the original private field. + slog.Error("sanitize STT settings sync payload failed", "err", err) + return nil + } + return &gatewayv2.SettingsSyncEvent{SettingsJson: string(settingsJSON)} +} + func (m *Manager) SubscribeSettingsSync() (<-chan Tagged[*gatewayv2.SettingsSyncEvent], func()) { ch := make(chan Tagged[*gatewayv2.SettingsSyncEvent], 64) @@ -108,6 +160,7 @@ func (m *Manager) ApplySettingsJSONPreservingRemote(agentID, settingsJSON string } func (m *Manager) broadcastSettingsSync(agentID string, event *gatewayv2.SettingsSyncEvent) { + event = m.consumePrivateSTTSettings(event) if event == nil { return } diff --git a/crates/agent-gateway/internal/session/manager_settings_sync_stt_test.go b/crates/agent-gateway/internal/session/manager_settings_sync_stt_test.go new file mode 100644 index 000000000..7841566c8 --- /dev/null +++ b/crates/agent-gateway/internal/session/manager_settings_sync_stt_test.go @@ -0,0 +1,65 @@ +package session + +import ( + "context" + "encoding/json" + "testing" + "time" + + gatewayv2 "github.com/liveagent/agent-gateway/internal/proto/v2" +) + +func TestSettingsSyncConsumesSTTSecretsBeforeBrowserBroadcast(t *testing.T) { + manager := NewManager() + var receivedSecret string + manager.SetSTTSettingsSyncHandler(func(_ context.Context, raw json.RawMessage) (any, error) { + var payload map[string]any + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, err + } + providers := payload["providers"].(map[string]any) + aliyun := providers["aliyun_dashscope"].(map[string]any) + receivedSecret = aliyun["apiKey"].(string) + return map[string]any{ + "provider": "aliyun_dashscope", + "providers": map[string]any{ + "aliyun_dashscope": map[string]any{ + "id": "aliyun_dashscope", + "configured": true, + "apiKey": "", + }, + }, + }, nil + }) + + subscription, cleanup := manager.SubscribeSettingsSync() + defer cleanup() + manager.broadcastSettingsSync("desktop-a", &gatewayv2.SettingsSyncEvent{SettingsJson: `{ + "theme":"dark", + "sttSecretSync":{ + "provider":"aliyun_dashscope", + "providers":{"aliyun_dashscope":{"apiKey":"server-only-secret"}} + } +}`}) + + if receivedSecret != "server-only-secret" { + t.Fatalf("private handler received secret %q", receivedSecret) + } + select { + case tagged := <-subscription: + var public map[string]any + if err := json.Unmarshal([]byte(tagged.Event.GetSettingsJson()), &public); err != nil { + t.Fatal(err) + } + if _, leaked := public[sttSecretSyncField]; leaked { + t.Fatal("private STT field leaked to settings subscriber") + } + sttPayload := public["stt"].(map[string]any) + provider := sttPayload["providers"].(map[string]any)["aliyun_dashscope"].(map[string]any) + if provider["apiKey"] != "" || provider["configured"] != true { + t.Fatalf("unexpected public STT payload: %#v", provider) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for settings broadcast") + } +} diff --git a/crates/agent-gateway/internal/stt/aliyun_dashscope.go b/crates/agent-gateway/internal/stt/aliyun_dashscope.go new file mode 100644 index 000000000..2338f1cdb --- /dev/null +++ b/crates/agent-gateway/internal/stt/aliyun_dashscope.go @@ -0,0 +1,187 @@ +package stt + +import ( + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/gorilla/websocket" +) + +type AliyunDashScopeAdapter struct{} + +const dashScopeEndpoint = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/" + +func dashScopeModel(configured string) string { + model := strings.TrimSpace(configured) + if model == "" || model == "paraformer-realtime-8k-v2" { + return "paraformer-realtime-v2" + } + return model +} + +func dashScopeFinish(taskID string) map[string]any { + return map[string]any{ + "header": map[string]any{"action": "finish-task", "task_id": taskID, "streaming": "duplex"}, + "payload": map[string]any{"input": map[string]any{}}, + } +} + +func (a *AliyunDashScopeAdapter) Test(ctx context.Context, cfg map[string]any) (string, error) { + return readyProtocolTest(ctx, a, cfg) +} +func (a *AliyunDashScopeAdapter) Run(ctx context.Context, id string, cfg map[string]any, commands <-chan Command, events chan<- Event) error { + endpoint, err := websocketEndpoint(cfg, dashScopeEndpoint) + if err != nil { + return stageError("DashScope", "validate", err) + } + model := dashScopeModel(value(cfg, "model")) + var taskBytes [16]byte + if _, err := rand.Read(taskBytes[:]); err != nil { + return stageError("DashScope", "start", err) + } + wireTaskID := fmt.Sprintf("%x", taskBytes) + header := http.Header{} + header.Set("Authorization", "Bearer "+value(cfg, "apiKey")) + header.Set("X-DashScope-DataInspection", "enable") + conn, response, err := websocket.DefaultDialer.DialContext(ctx, endpoint, header) + if err != nil { + return stageError("DashScope", "connect", websocketConnectError(response, err)) + } + defer func() { _ = conn.Close() }() + start := map[string]any{ + "header": map[string]any{"action": "run-task", "task_id": wireTaskID, "streaming": "duplex"}, + "payload": map[string]any{ + "task_group": "audio", "task": "asr", "function": "recognition", + "model": model, + "parameters": map[string]any{ + "format": "pcm", "sample_rate": 16000, + "language_hints": []string{"zh", "en"}, + "max_sentence_silence": 2000, + "disfluency_removal_enabled": false, + }, + "input": map[string]any{}, + }, + } + if err = writeProviderJSON(conn, start); err != nil { + return stageError("DashScope", "start", err) + } + incoming := make(chan map[string]any, 8) + readErr := make(chan error, 1) + go func() { + for { + var msg map[string]any + if err := conn.ReadJSON(&msg); err != nil { + readErr <- stageError("DashScope", "receive", err) + return + } + if !emitIncoming(ctx, incoming, msg) { + return + } + } + }() + finishing := false + finishSent := false + ready := false + pending := make([][]byte, 0, 32) + finals := "" + for { + select { + case <-ctx.Done(): + return nil + case err := <-readErr: + if finishing && finishSent && isWebSocketCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + return stageError("DashScope", "close", errors.New("connection closed before task-finished")) + } + return err + case cmd := <-commands: + if cmd.Cancel { + return nil + } + if cmd.Audio != nil { + if !ready { + pending = append(pending, append([]byte(nil), cmd.Audio.PCM...)) + } else if err := writeProviderMessage(conn, websocket.BinaryMessage, cmd.Audio.PCM); err != nil { + return stageError("DashScope", "send_audio", err) + } + } + if cmd.Finish { + finishing = true + if ready && !finishSent { + if err := writeProviderJSON(conn, dashScopeFinish(wireTaskID)); err != nil { + return stageError("DashScope", "finish", err) + } + finishSent = true + } + } + case msg := <-incoming: + body, _ := json.Marshal(msg) + var wire struct { + Header struct { + Event string `json:"event"` + ErrorCode string `json:"error_code"` + ErrorMessage string `json:"error_message"` + } `json:"header"` + Payload struct { + Output struct { + Sentence struct { + Text string `json:"text"` + End bool `json:"end"` + SentenceEnd bool `json:"sentence_end"` + } `json:"sentence"` + } `json:"output"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &wire); err != nil { + return stageError("DashScope", "parse", err) + } + switch wire.Header.Event { + case "task-started": + if !ready { + ready = true + if !emitEvent(ctx, events, Event{Type: "ready", SessionID: id}) { + return ctx.Err() + } + for _, pcm := range pending { + if err := writeProviderMessage(conn, websocket.BinaryMessage, pcm); err != nil { + return stageError("DashScope", "send_audio", err) + } + } + pending = nil + if finishing && !finishSent { + if err := writeProviderJSON(conn, dashScopeFinish(wireTaskID)); err != nil { + return stageError("DashScope", "finish", err) + } + finishSent = true + } + } + case "result-generated": + if wire.Payload.Output.Sentence.End || wire.Payload.Output.Sentence.SentenceEnd { + finals += wire.Payload.Output.Sentence.Text + if !emitEvent(ctx, events, Event{Type: "partial", SessionID: id, Text: finals}) { + return ctx.Err() + } + } else { + if !emitEvent(ctx, events, Event{Type: "partial", SessionID: id, Text: finals + wire.Payload.Output.Sentence.Text}) { + return ctx.Err() + } + } + case "task-finished": + if finishing && finishSent { + if finals != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: finals}) { + return ctx.Err() + } + } + return nil + } + case "task-failed": + return stageError("DashScope", "provider_response", providerFailure("DashScope", wire.Header.ErrorCode, wire.Header.ErrorMessage)) + } + } + } +} diff --git a/crates/agent-gateway/internal/stt/baidu_cloud.go b/crates/agent-gateway/internal/stt/baidu_cloud.go new file mode 100644 index 000000000..7929514b2 --- /dev/null +++ b/crates/agent-gateway/internal/stt/baidu_cloud.go @@ -0,0 +1,195 @@ +package stt + +import ( + "context" + "errors" + "io" + "net/url" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +type BaiduCloudAdapter struct{} + +const baiduRealtimeEndpoint = "wss://vop.baidu.com/realtime_asr" + +func baiduNoSpeech(errNo float64) bool { return errNo == 3301 || errNo == -3005 } + +func baiduFinishedConnectionClosed(err error) bool { + return errors.Is(err, io.EOF) || isWebSocketCloseError( + err, + websocket.CloseNormalClosure, + websocket.CloseGoingAway, + websocket.CloseNoStatusReceived, + ) +} + +func baiduStartMessage(appid uint64, devPID uint32, appKey, cuid string) map[string]any { + return map[string]any{ + "type": "START", + "data": map[string]any{ + "appid": appid, "appkey": appKey, "dev_pid": devPID, + "cuid": cuid, "format": "pcm", "sample": 16000, + }, + } +} + +func baiduEndpoint(configured string) (string, error) { + endpoint, err := websocketEndpoint(map[string]any{"websocketUrl": configured}, baiduRealtimeEndpoint) + if err != nil { + return "", err + } + parsed, err := url.Parse(endpoint) + if err != nil { + return "", err + } + query := parsed.Query() + query.Set("sn", uuid.NewString()) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func baiduResultText(result any) string { + switch typed := result.(type) { + case string: + return strings.TrimSpace(typed) + case []any: + var merged strings.Builder + for _, item := range typed { + if part, ok := item.(string); ok { + merged.WriteString(part) + } + } + return strings.TrimSpace(merged.String()) + default: + return "" + } +} + +func (a *BaiduCloudAdapter) Test(ctx context.Context, cfg map[string]any) (string, error) { + return silentProtocolTest(ctx, a, cfg) +} +func (a *BaiduCloudAdapter) Run(ctx context.Context, id string, cfg map[string]any, commands <-chan Command, events chan<- Event) error { + endpoint, endpointErr := baiduEndpoint(value(cfg, "websocketUrl")) + if endpointErr != nil { + return stageError("Baidu", "validate", endpointErr) + } + appid, err := strconv.ParseUint(value(cfg, "baiduAppId"), 10, 64) + if err != nil { + return stageError("Baidu", "validate", &ResultError{Result: "authentication_failed", Err: errors.New("appid must be numeric")}) + } + pid, err := strconv.ParseUint(value(cfg, "devPid"), 10, 32) + if err != nil { + return stageError("Baidu", "validate", &ResultError{Result: "protocol_failed", Err: errors.New("dev_pid is required")}) + } + conn, response, err := websocket.DefaultDialer.DialContext(ctx, endpoint, nil) + if err != nil { + return stageError("Baidu", "connect", websocketConnectError(response, err)) + } + defer func() { _ = conn.Close() }() + if err = writeProviderJSON(conn, baiduStartMessage(appid, uint32(pid), value(cfg, "baiduApiKey"), "LiveAgent-"+uuid.NewString())); err != nil { + return stageError("Baidu", "start", err) + } + if !emitEvent(ctx, events, Event{Type: "ready", SessionID: id}) { + return ctx.Err() + } + incoming := make(chan map[string]any, 8) + readErr := make(chan error, 1) + go func() { + for { + var msg map[string]any + if e := conn.ReadJSON(&msg); e != nil { + readErr <- stageError("Baidu", "receive", e) + return + } + if !emitIncoming(ctx, incoming, msg) { + return + } + } + }() + finishSent := false + noSpeechSeen := false + finalText := "" + for { + select { + case <-ctx.Done(): + return nil + case e := <-readErr: + if finishSent && baiduFinishedConnectionClosed(e) { + if finalText != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: finalText}) { + return ctx.Err() + } + } + return nil + } + if finishSent { + return stageError("Baidu", "close", e) + } + return e + case cmd := <-commands: + if cmd.Cancel { + return nil + } + if cmd.Audio != nil { + if e := writeProviderMessage(conn, websocket.BinaryMessage, cmd.Audio.PCM); e != nil { + return stageError("Baidu", "send_audio", e) + } + } + if cmd.Finish { + finishSent = true + if e := writeProviderJSON(conn, map[string]any{"type": "FINISH"}); e != nil { + return stageError("Baidu", "finish", e) + } + if noSpeechSeen { + return nil + } + } + case msg := <-incoming: + if n, ok := msg["err_no"].(float64); ok && n != 0 { + if baiduNoSpeech(n) { + noSpeechSeen = true + if finishSent { + return nil + } + continue + } + return stageError("Baidu", "provider_response", providerFailure("Baidu", strconv.Itoa(int(n)), valueString(msg, "err_msg"))) + } + switch msg["type"] { + case "MID_TEXT": + textValue := baiduResultText(msg["result"]) + eventsEvent := Event{Type: "partial", SessionID: id, Text: finalText + textValue} + if !emitEvent(ctx, events, eventsEvent) { + return ctx.Err() + } + case "FIN_TEXT": + textValue := baiduResultText(msg["result"]) + finalText += textValue + if !emitEvent(ctx, events, Event{Type: "partial", SessionID: id, Text: finalText}) { + return ctx.Err() + } + if finishSent { + if finalText != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: finalText}) { + return ctx.Err() + } + } + return nil + } + case "FINISH": + if finishSent { + if finalText != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: finalText}) { + return ctx.Err() + } + } + return nil + } + } + } + } +} diff --git a/crates/agent-gateway/internal/stt/common.go b/crates/agent-gateway/internal/stt/common.go new file mode 100644 index 000000000..8a788e0a8 --- /dev/null +++ b/crates/agent-gateway/internal/stt/common.go @@ -0,0 +1,315 @@ +package stt + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gorilla/websocket" +) + +const providerWriteTimeout = 10 * time.Second + +// emitEvent never lets a provider goroutine block forever when its consumer +// (usually the WebSocket writer) has stopped reading. Cancellation must be +// able to unwind the provider and close its upstream connection. +func emitEvent(ctx context.Context, events chan<- Event, event Event) bool { + select { + case events <- event: + return true + case <-ctx.Done(): + return false + } +} + +// emitIncoming is the adapter-local counterpart of emitEvent: the provider +// read goroutine must not stay blocked on a full incoming channel after the +// session context is cancelled. +func emitIncoming(ctx context.Context, incoming chan<- map[string]any, msg map[string]any) bool { + select { + case incoming <- msg: + return true + case <-ctx.Done(): + return false + } +} + +func writeProviderMessage(conn *websocket.Conn, messageType int, data []byte) error { + if err := conn.SetWriteDeadline(time.Now().Add(providerWriteTimeout)); err != nil { + return err + } + return conn.WriteMessage(messageType, data) +} + +func writeProviderJSON(conn *websocket.Conn, value any) error { + if err := conn.SetWriteDeadline(time.Now().Add(providerWriteTimeout)); err != nil { + return err + } + return conn.WriteJSON(value) +} + +type ResultError struct { + Result string + Err error +} + +func (e *ResultError) Error() string { return e.Err.Error() } +func (e *ResultError) Unwrap() error { return e.Err } + +type StageError struct { + Provider string + Stage string + Err error +} + +func (e *StageError) Error() string { + return fmt.Sprintf("[%s/%s] %s", e.Provider, e.Stage, e.Err) +} + +func (e *StageError) Unwrap() error { return e.Err } + +func stageError(provider, stage string, err error) error { + if err == nil { + return nil + } + return &StageError{Provider: provider, Stage: stage, Err: err} +} + +func isWebSocketCloseError(err error, codes ...int) bool { + var closeErr *websocket.CloseError + if !errors.As(err, &closeErr) { + return false + } + for _, code := range codes { + if closeErr.Code == code { + return true + } + } + return false +} + +func websocketConnectError(response *http.Response, err error) error { + if response == nil { + return err + } + if response.Body != nil { + _ = response.Body.Close() + } + result := "protocol_failed" + if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden { + result = "authentication_failed" + } else if response.StatusCode >= http.StatusInternalServerError { + result = "network_failed" + } + return &ResultError{ + Result: result, + Err: fmt.Errorf("STT provider WebSocket handshake failed with HTTP %d", response.StatusCode), + } +} + +func resultForError(err error) string { + var classified *ResultError + if errors.As(err, &classified) && classified.Result != "" { + return classified.Result + } + return classifyError(err) +} + +func classifyError(err error) string { + if err == nil { + return "connected_no_speech" + } + if noSpeechError(err) { + return "connected_no_speech" + } + message := strings.ToLower(err.Error()) + if strings.Contains(message, "timeout") || strings.Contains(message, "deadline") { + return "timeout" + } + var netErr net.Error + if errors.As(err, &netErr) { + return "network_failed" + } + if strings.Contains(message, "auth") || strings.Contains(message, "鉴权") || strings.Contains(message, "unauthor") || strings.Contains(message, "invalid api") || strings.Contains(message, "appid") { + return "authentication_failed" + } + return "protocol_failed" +} + +func noSpeechError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "no valid speeches") || + strings.Contains(message, "no speech") || + strings.Contains(message, "未检测到有效语音") || + strings.Contains(message, "未发现有效语音") || + strings.Contains(message, "3301") || + strings.Contains(message, "-3005") || + strings.Contains(message, "1013") +} + +func value(cfg map[string]any, key string) string { v, _ := cfg[key].(string); return v } + +func websocketEndpoint(cfg map[string]any, fallback string) (string, error) { + endpoint := strings.TrimSpace(value(cfg, "websocketUrl")) + if endpoint == "" { + endpoint = fallback + } + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Scheme != "wss" || parsed.Host == "" || parsed.User != nil { + return "", &ResultError{Result: "protocol_failed", Err: errors.New("STT WebSocket URL must be an absolute wss:// URL without user information")} + } + return endpoint, nil +} + +func providerFailure(provider, code, message string) error { + detail := strings.TrimSpace(code) + if text := strings.TrimSpace(message); text != "" { + if detail != "" { + detail += ": " + } + detail += text + } + if detail == "" { + detail = "provider rejected the request" + } + result := "protocol_failed" + lower := strings.ToLower(detail) + if noSpeechError(errors.New(detail)) { + result = "connected_no_speech" + } else if strings.Contains(lower, "auth") || strings.Contains(lower, "unauthor") || strings.Contains(lower, "forbidden") || strings.Contains(lower, "api key") || strings.Contains(lower, "access key") || strings.Contains(lower, "token") || strings.Contains(lower, "signature") || strings.Contains(lower, "secret") || strings.Contains(lower, "鉴权") { + result = "authentication_failed" + } + return &ResultError{Result: result, Err: errors.New(provider + " rejected the request (" + detail + ")")} +} + +// readyProtocolTest mirrors the working desktop reference implementation: +// providers with an explicit ready state have already proved endpoint, +// authentication, and start-request compatibility at that point. Their +// provider-specific finish protocol remains enforced for real recognition, +// but is not used to decide whether saved configuration is valid. +func readyProtocolTest(parent context.Context, adapter Adapter, cfg map[string]any) (string, error) { + ctx, cancel := context.WithTimeout(parent, 10*time.Second) + defer cancel() + commands := make(chan Command, 1) + events := make(chan Event, 8) + done := make(chan error, 1) + go func() { done <- adapter.Run(ctx, "connection-test", cfg, commands, events) }() + for { + select { + case event := <-events: + switch event.Type { + case "ready": + commands <- Command{Cancel: true} + select { + case err := <-done: + if err != nil { + return resultForError(err), err + } + case <-time.After(time.Second): + // Authentication and the provider start request are already + // proven. Context cancellation below releases a slow close. + } + return "connected", nil + case "error": + err := errors.New(event.Message) + return resultForError(err), err + } + case err := <-done: + if err != nil { + return resultForError(err), err + } + return "protocol_failed", errors.New("STT provider closed before reporting ready") + case <-ctx.Done(): + return "timeout", errors.New("waiting for STT provider ready state timed out") + } + } +} + +func silentProtocolTest(ctx context.Context, adapter Adapter, cfg map[string]any) (string, error) { + // The ready and provider-finish windows are independent. A slow but valid + // handshake must not consume the time reserved for the finish acknowledgement. + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + commands := make(chan Command, 16) + events := make(chan Event, 16) + done := make(chan error, 1) + go func() { done <- adapter.Run(ctx, "connection-test", cfg, commands, events) }() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + ready := false + finishSent := false + recognized := false + sequence := uint32(0) + readyTimeout := time.After(10 * time.Second) + var finishTimeout <-chan time.Time + for { + select { + case event := <-events: + if event.Type == "error" { + err := errors.New(event.Message) + if finishSent && noSpeechError(err) { + return "connected_no_speech", nil + } + return resultForError(err), err + } + if event.Type == "ready" { + ready = true + readyTimeout = nil + } + if (event.Type == "partial" || event.Type == "final") && strings.TrimSpace(event.Text) != "" { + recognized = true + } + case <-ticker.C: + if !ready || finishSent { + continue + } + if sequence < 10 { + select { + case commands <- Command{Audio: &AudioChunk{Sequence: sequence, PCM: make([]byte, 3200)}}: + sequence++ + case <-ctx.Done(): + return "timeout", ctx.Err() + } + continue + } + select { + case commands <- Command{Finish: true}: + finishSent = true + finishTimeout = time.After(5 * time.Second) + case <-ctx.Done(): + return "timeout", ctx.Err() + } + case err := <-done: + if err != nil { + if finishSent && noSpeechError(err) { + return "connected_no_speech", nil + } + return resultForError(err), err + } + if !ready { + return "protocol_failed", errors.New("STT provider closed before reporting ready") + } + if !finishSent { + return "protocol_failed", errors.New("STT provider closed before the test audio was finished") + } + if recognized { + return "connected", nil + } + return "connected_no_speech", nil + case <-readyTimeout: + return "timeout", errors.New("waiting for STT provider ready state timed out") + case <-finishTimeout: + return "timeout", errors.New("STT provider connected but finish acknowledgement timed out") + case <-ctx.Done(): + return "timeout", ctx.Err() + } + } +} diff --git a/crates/agent-gateway/internal/stt/http.go b/crates/agent-gateway/internal/stt/http.go new file mode 100644 index 000000000..197976841 --- /dev/null +++ b/crates/agent-gateway/internal/stt/http.go @@ -0,0 +1,217 @@ +package stt + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "time" + + "github.com/gorilla/websocket" + "github.com/liveagent/agent-gateway/internal/auth" + gatewayv2 "github.com/liveagent/agent-gateway/internal/proto/v2" + "github.com/liveagent/agent-gateway/internal/protocol/pbws" + "github.com/liveagent/agent-gateway/internal/protocol/shared" + "google.golang.org/protobuf/proto" +) + +const connectionTestHTTPTimeout = 22 * time.Second + +// sttHelloTimeout is the maximum time a client may wait after the WebSocket +// upgrade before sending a hello frame. It is a package var so tests can +// shorten it without exposing an unauthenticated Slowloris window in production. +var sttHelloTimeout = 10 * time.Second + +func (m *Manager) SettingsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.Method { + case http.MethodGet: + settings, err := m.store.Get(r.Context()) + if err != nil { + http.Error(w, "STT settings unavailable", 500) + return + } + _ = json.NewEncoder(w).Encode(settings) + case http.MethodPut: + var incoming Settings + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&incoming); err != nil { + http.Error(w, "invalid STT settings", 400) + return + } + settings, err := m.store.Update(r.Context(), incoming) + if err != nil { + http.Error(w, "STT settings update failed", 400) + return + } + _ = json.NewEncoder(w).Encode(settings) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + }) +} + +func (m *Manager) TestHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + provider := strings.TrimSpace(r.URL.Query().Get("provider")) + // The provider probe reserves up to 10 seconds for ready and another + // 5 seconds for finish acknowledgement. Keep the HTTP request alive for + // the complete protocol lifecycle plus scheduling overhead. + ctx, cancel := context.WithTimeout(r.Context(), connectionTestHTTPTimeout) + defer cancel() + result, err := m.Test(ctx, provider) + if err != nil && result == "" { + result = "protocol_failed" + } + w.Header().Set("Content-Type", "application/json") + response := map[string]string{"result": result} + if err != nil { + message := []rune(err.Error()) + if len(message) > 240 { + message = message[:240] + } + response["message"] = string(message) + } + _ = json.NewEncoder(w).Encode(response) + }) +} + +func (m *Manager) WebSocketHandler(token string) http.Handler { + upgrader := websocket.Upgrader{Subprotocols: []string{pbws.Subprotocol}, CheckOrigin: shared.OriginAllowed} + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer func() { _ = conn.Close() }() + conn.SetReadLimit(64 << 10) + _ = conn.SetReadDeadline(time.Now().Add(sttHelloTimeout)) + messageType, data, err := conn.ReadMessage() + if err != nil || messageType != websocket.BinaryMessage { + return + } + var helloFrame gatewayv2.SttClientFrame + if proto.Unmarshal(data, &helloFrame) != nil || helloFrame.GetHello() == nil || + helloFrame.GetHello().GetProtocolVersion() != pbws.ProtocolVersion || + !auth.ValidateToken(helloFrame.GetHello().GetToken(), token) { + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(4401, "unauthorized"), time.Now().Add(time.Second)) + return + } + if !writeSttFrame(conn, &gatewayv2.SttServerFrame{Payload: &gatewayv2.SttServerFrame_Hello{Hello: &gatewayv2.SttServerHello{Ok: true}}}) { + return + } + _ = conn.SetReadDeadline(time.Time{}) + events := make(chan Event, 64) + writerStop := make(chan struct{}) + writerDone := make(chan struct{}) + var activeID string + var nextSequence uint32 + go func() { + defer close(writerDone) + for { + select { + case <-writerStop: + return + case event := <-events: + if !writeSttEvent(conn, event) { + // Stop the read loop as soon as the client can no longer + // receive events; its deferred cleanup cancels the adapter + // context so event delivery cannot block the provider. + _ = conn.Close() + return + } + } + } + }() + defer func() { + if activeID != "" { + m.Cancel(activeID) + } + close(writerStop) + <-writerDone + }() + windowStart := time.Now() + frames := 0 + for { + kind, payload, readErr := conn.ReadMessage() + if readErr != nil { + return + } + if time.Since(windowStart) >= time.Second { + windowStart = time.Now() + frames = 0 + } + frames++ + if frames > 120 { + return + } + if kind != websocket.BinaryMessage { + return + } + var frame gatewayv2.SttClientFrame + if proto.Unmarshal(payload, &frame) != nil { + return + } + switch payload := frame.GetPayload().(type) { + case *gatewayv2.SttClientFrame_Start: + if activeID != "" || payload.Start.GetSessionId() == "" { + return + } + if err := m.Start(r.Context(), payload.Start.GetSessionId(), payload.Start.GetProvider(), events); err != nil { + // Writer is idle until the adapter emits; send the error + // directly so the client is not left with a silent close. + _ = writeSttEvent(conn, Event{ + Type: "error", + SessionID: payload.Start.GetSessionId(), + Code: resultForError(err), + Message: err.Error(), + }) + return + } + activeID = payload.Start.GetSessionId() + nextSequence = 0 + case *gatewayv2.SttClientFrame_Audio: + if activeID == "" || payload.Audio.GetSessionId() != activeID || payload.Audio.GetSequence() != nextSequence || len(payload.Audio.GetPcm()) == 0 || len(payload.Audio.GetPcm())%2 != 0 || len(payload.Audio.GetPcm()) > 6400 || m.Send(activeID, Command{Audio: &AudioChunk{Sequence: payload.Audio.GetSequence(), PCM: append([]byte(nil), payload.Audio.GetPcm()...)}}) != nil { + return + } + nextSequence++ + case *gatewayv2.SttClientFrame_Stop: + if activeID == "" || payload.Stop.GetSessionId() != activeID || m.Send(activeID, Command{Finish: true}) != nil { + return + } + case *gatewayv2.SttClientFrame_Cancel: + if activeID != "" && payload.Cancel.GetSessionId() == activeID { + m.Cancel(activeID) + activeID = "" + } + } + } + }) +} + +func writeSttFrame(conn *websocket.Conn, frame *gatewayv2.SttServerFrame) bool { + data, err := proto.Marshal(frame) + if err != nil || conn.SetWriteDeadline(time.Now().Add(10*time.Second)) != nil { + return false + } + return conn.WriteMessage(websocket.BinaryMessage, data) == nil +} + +func writeSttEvent(conn *websocket.Conn, event Event) bool { + var frame *gatewayv2.SttServerFrame + switch event.Type { + case "ready": + frame = &gatewayv2.SttServerFrame{Payload: &gatewayv2.SttServerFrame_Ready{Ready: &gatewayv2.SttTextEvent{SessionId: event.SessionID}}} + case "partial": + frame = &gatewayv2.SttServerFrame{Payload: &gatewayv2.SttServerFrame_Partial{Partial: &gatewayv2.SttTextEvent{SessionId: event.SessionID, Text: event.Text}}} + case "final": + frame = &gatewayv2.SttServerFrame{Payload: &gatewayv2.SttServerFrame_Final{Final: &gatewayv2.SttTextEvent{SessionId: event.SessionID, Text: event.Text}}} + case "error": + frame = &gatewayv2.SttServerFrame{Payload: &gatewayv2.SttServerFrame_Error{Error: &gatewayv2.SttError{SessionId: event.SessionID, Code: event.Code, Message: event.Message}}} + case "closed": + frame = &gatewayv2.SttServerFrame{Payload: &gatewayv2.SttServerFrame_Closed{Closed: &gatewayv2.SttClosed{SessionId: event.SessionID}}} + default: + return true + } + return writeSttFrame(conn, frame) +} diff --git a/crates/agent-gateway/internal/stt/live_provider_test.go b/crates/agent-gateway/internal/stt/live_provider_test.go new file mode 100644 index 000000000..ceaa00719 --- /dev/null +++ b/crates/agent-gateway/internal/stt/live_provider_test.go @@ -0,0 +1,63 @@ +package stt + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// TestConfiguredDesktopProvidersLive is an opt-in diagnostic for real provider +// credentials. Normal test runs never access a user's desktop settings or the +// network; set LIVEAGENT_STT_LIVE_DB to an explicit config.sqlite path to run it. +func TestConfiguredDesktopProvidersLive(t *testing.T) { + databasePath := os.Getenv("LIVEAGENT_STT_LIVE_DB") + if databasePath == "" { + t.Skip("set LIVEAGENT_STT_LIVE_DB to run real STT provider probes") + } + + database, err := sql.Open("sqlite", "file:"+databasePath+"?mode=ro") + if err != nil { + t.Fatalf("open desktop STT settings: %v", err) + } + defer database.Close() + + var payload string + if err := database.QueryRow(`SELECT payload_json FROM stt_settings WHERE config_id = 'default'`).Scan(&payload); err != nil { + t.Fatalf("load desktop STT settings: %v", err) + } + var settings Settings + if err := json.Unmarshal([]byte(payload), &settings); err != nil { + t.Fatalf("decode desktop STT settings: %v", err) + } + + tested := 0 + for _, providerID := range providerIDs { + config := settings.Providers[providerID] + if !configured(providerID, config) { + continue + } + adapter := adapterFor(providerID) + if adapter == nil { + t.Fatalf("missing adapter for %s", providerID) + } + tested++ + t.Run(providerID, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + result, probeErr := adapter.Test(ctx, config) + if probeErr != nil { + t.Errorf("result=%s provider_message=%s", resultForError(probeErr), sanitizeError(probeErr.Error(), config)) + return + } + t.Logf("result=%s", result) + }) + } + if tested == 0 { + t.Fatal("desktop STT settings contain no configured providers") + } +} diff --git a/crates/agent-gateway/internal/stt/manager.go b/crates/agent-gateway/internal/stt/manager.go new file mode 100644 index 000000000..bcff9f145 --- /dev/null +++ b/crates/agent-gateway/internal/stt/manager.go @@ -0,0 +1,165 @@ +package stt + +import ( + "context" + "errors" + "regexp" + "sync" +) + +type AudioChunk struct { + Sequence uint32 + PCM []byte +} +type Command struct { + Audio *AudioChunk + Finish bool + Cancel bool +} +type Event struct { + Type string `json:"type"` + SessionID string `json:"sessionId"` + Text string `json:"text,omitempty"` + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} +type Adapter interface { + Run(context.Context, string, map[string]any, <-chan Command, chan<- Event) error + Test(context.Context, map[string]any) (string, error) +} +type activeSession struct { + cancel context.CancelFunc + commands chan Command +} +type Manager struct { + store *Store + mu sync.Mutex + sessions map[string]*activeSession +} + +func NewManager(store *Store) *Manager { + return &Manager{store: store, sessions: map[string]*activeSession{}} +} +func adapterFor(id string) Adapter { + switch id { + case "aliyun_dashscope": + return &AliyunDashScopeAdapter{} + case "tencent_cloud": + return &TencentCloudAdapter{} + case "volcengine_v2": + return &VolcengineV2Adapter{} + case "volcengine_seed_v3": + return &VolcengineSeedV3Adapter{} + case "baidu_cloud": + return &BaiduCloudAdapter{} + } + return nil +} + +// adapterFactory is kept indirect so protocol/session tests can inject a +// deterministic adapter without opening a real provider connection. +var adapterFactory = adapterFor + +func (m *Manager) Store() *Store { return m.store } +func (m *Manager) Start(parent context.Context, sessionID, provider string, events chan<- Event) error { + adapter := adapterFactory(provider) + if adapter == nil { + return errors.New("unknown STT provider") + } + cfg, err := m.store.Provider(parent, provider) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.sessions[sessionID]; exists { + return errors.New("STT session already exists") + } + ctx, cancel := context.WithCancel(parent) + active := &activeSession{cancel: cancel, commands: make(chan Command, 128)} + m.sessions[sessionID] = active + go func() { + err := adapter.Run(ctx, sessionID, cfg, active.commands, events) + if err != nil { + emitEvent(ctx, events, Event{ + Type: "error", + SessionID: sessionID, + Code: resultForError(err), + Message: sanitizeError(err.Error(), cfg), + }) + } + emitEvent(ctx, events, Event{Type: "closed", SessionID: sessionID}) + m.mu.Lock() + delete(m.sessions, sessionID) + m.mu.Unlock() + }() + return nil +} +func (m *Manager) Send(sessionID string, command Command) error { + m.mu.Lock() + active := m.sessions[sessionID] + m.mu.Unlock() + if active == nil { + return errors.New("STT session not found") + } + select { + case active.commands <- command: + return nil + default: + return errors.New("STT write queue full") + } +} +func (m *Manager) Cancel(sessionID string) { + m.mu.Lock() + active := m.sessions[sessionID] + m.mu.Unlock() + if active != nil { + active.cancel() + } +} +func (m *Manager) Test(ctx context.Context, provider string) (string, error) { + adapter := adapterFactory(provider) + if adapter == nil { + return "protocol_failed", errors.New("unknown provider") + } + cfg, err := m.store.Provider(ctx, provider) + if err != nil { + return "authentication_failed", err + } + result, testErr := adapter.Test(ctx, cfg) + if testErr != nil { + return resultForError(testErr), errors.New(sanitizeError(testErr.Error(), cfg)) + } + return result, nil +} + +var providerURLPattern = regexp.MustCompile(`(?i)\b(?:wss?|https?)://\S+`) + +func sanitizeError(message string, cfg map[string]any) string { + for _, field := range secretFields { + if value, ok := cfg[field].(string); ok && value != "" { + message = stringReplaceAll(message, value, "[redacted]") + } + } + return providerURLPattern.ReplaceAllString(message, "[provider endpoint]") +} +func stringReplaceAll(s, old, new string) string { + for { + next := replaceOnce(s, old, new) + if next == s { + return s + } + s = next + } +} +func replaceOnce(s, old, new string) string { + if old == "" { + return s + } + for i := 0; i+len(old) <= len(s); i++ { + if s[i:i+len(old)] == old { + return s[:i] + new + s[i+len(old):] + } + } + return s +} diff --git a/crates/agent-gateway/internal/stt/settings.go b/crates/agent-gateway/internal/stt/settings.go new file mode 100644 index 000000000..b122b2a17 --- /dev/null +++ b/crates/agent-gateway/internal/stt/settings.go @@ -0,0 +1,307 @@ +package stt + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "strconv" + "strings" + + "github.com/liveagent/agent-gateway/internal/db" +) + +var providerIDs = []string{"aliyun_dashscope", "tencent_cloud", "volcengine_v2", "volcengine_seed_v3", "baidu_cloud"} +var secretFields = []string{"apiKey", "secretId", "secretKey", "accessToken", "baiduApiKey"} + +func providerDefaults(id string) map[string]any { + provider := map[string]any{"id": id, "configured": false, "websocketUrl": "", "model": "", "apiKey": "", "appId": "", "secretId": "", "secretKey": "", "accessToken": "", "cluster": "", "resourceId": "", "engineModelType": "16k_zh", "baiduAppId": "", "baiduApiKey": "", "devPid": ""} + switch id { + case "aliyun_dashscope": + provider["websocketUrl"] = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/" + provider["model"] = "paraformer-realtime-v2" + case "volcengine_v2": + provider["websocketUrl"] = "wss://openspeech.bytedance.com/api/v2/asr" + case "volcengine_seed_v3": + provider["websocketUrl"] = volcengineSeedV3Endpoint + case "baidu_cloud": + provider["websocketUrl"] = "wss://vop.baidu.com/realtime_asr" + } + return provider +} + +type Settings struct { + Enabled bool `json:"enabled"` + Provider *string `json:"provider"` + Providers map[string]map[string]any `json:"providers"` + AllowIncomplete bool `json:"allowIncomplete,omitempty"` +} +type Store struct{ pool *sql.DB } + +func NewStore(database *db.DB) (*Store, error) { + if database == nil || !database.Enabled() { + return nil, errors.New("gateway database is required") + } + s := &Store{pool: database.Pool()} + _, err := s.pool.Exec(`CREATE TABLE IF NOT EXISTS stt_settings (config_id TEXT PRIMARY KEY, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL DEFAULT (unixepoch('subsec') * 1000))`) + return s, err +} + +func defaults() Settings { + providers := map[string]map[string]any{} + for _, id := range providerIDs { + providers[id] = providerDefaults(id) + } + return Settings{Enabled: false, Providers: providers} +} + +func (s *Store) raw(ctx context.Context) (Settings, error) { + current := defaults() + var payload string + err := s.pool.QueryRowContext(ctx, `SELECT payload_json FROM stt_settings WHERE config_id='default'`).Scan(&payload) + if errors.Is(err, sql.ErrNoRows) { + return current, nil + } + if err != nil { + return current, err + } + if err := json.Unmarshal([]byte(payload), ¤t); err != nil { + return defaults(), err + } + if current.Providers == nil { + current.Providers = map[string]map[string]any{} + } + for _, id := range providerIDs { + normalized := providerDefaults(id) + for key, value := range current.Providers[id] { + normalized[key] = value + } + if id == "aliyun_dashscope" && normalized["model"] == "paraformer-realtime-8k-v2" { + normalized["model"] = "paraformer-realtime-v2" + } + current.Providers[id] = normalized + } + return current, nil +} + +func configured(id string, p map[string]any) bool { + return validateProvider(id, p) == nil +} + +func requiredProviderValue(p map[string]any, field, label string) (string, error) { + value := strings.TrimSpace(stringValue(p, field)) + if value == "" { + return "", errors.New(label + " is required") + } + return value, nil +} + +func validateProviderWebSocket(p map[string]any) error { + _, err := websocketEndpoint(p, "") + return err +} + +func requirePositiveProviderInteger(p map[string]any, field, label string) error { + value, err := requiredProviderValue(p, field, label) + if err != nil { + return err + } + numeric, err := strconv.ParseUint(value, 10, 64) + if err != nil || numeric == 0 { + return errors.New(label + " must be a positive decimal integer") + } + return nil +} + +func validateProvider(id string, p map[string]any) error { + switch id { + case "aliyun_dashscope": + if err := validateProviderWebSocket(p); err != nil { + return err + } + if _, err := requiredProviderValue(p, "model", "DashScope model"); err != nil { + return err + } + _, err := requiredProviderValue(p, "apiKey", "DashScope API key") + return err + case "tencent_cloud": + if err := requirePositiveProviderInteger(p, "appId", "Tencent AppId"); err != nil { + return err + } + for _, item := range [][2]string{{"engineModelType", "Tencent engine model"}, {"secretId", "Tencent SecretId"}, {"secretKey", "Tencent SecretKey"}} { + if _, err := requiredProviderValue(p, item[0], item[1]); err != nil { + return err + } + } + return nil + case "volcengine_v2": + if err := validateProviderWebSocket(p); err != nil { + return err + } + for _, item := range [][2]string{{"appId", "Volcengine v2 App ID"}, {"cluster", "Volcengine v2 Cluster"}, {"accessToken", "Volcengine v2 Access Token"}} { + if _, err := requiredProviderValue(p, item[0], item[1]); err != nil { + return err + } + } + return nil + case "volcengine_seed_v3": + if err := validateProviderWebSocket(p); err != nil { + return err + } + for _, item := range [][2]string{{"appId", "Volcengine Seed v3 App ID"}, {"accessToken", "Volcengine Seed v3 Access Token"}, {"resourceId", "Volcengine Seed v3 Resource ID"}} { + if _, err := requiredProviderValue(p, item[0], item[1]); err != nil { + return err + } + } + return nil + case "baidu_cloud": + if err := validateProviderWebSocket(p); err != nil { + return err + } + if err := requirePositiveProviderInteger(p, "baiduAppId", "Baidu App ID"); err != nil { + return err + } + if err := requirePositiveProviderInteger(p, "devPid", "Baidu dev_pid"); err != nil { + return err + } + _, err := requiredProviderValue(p, "baiduApiKey", "Baidu API key") + return err + } + return errors.New("unknown STT provider") +} + +func stringValue(provider map[string]any, field string) string { + value, _ := provider[field].(string) + return value +} + +func redact(settings Settings) Settings { + settings.AllowIncomplete = false + for id, provider := range settings.Providers { + provider["configured"] = configured(id, provider) + for _, field := range secretFields { + provider[field] = "" + } + delete(provider, "clearSecrets") + } + return settings +} +func (s *Store) Get(ctx context.Context) (Settings, error) { + current, err := s.raw(ctx) + if err != nil { + return current, err + } + return redact(current), nil +} + +func (s *Store) Update(ctx context.Context, incoming Settings) (Settings, error) { + current, err := s.raw(ctx) + if err != nil { + return Settings{}, err + } + if incoming.Providers == nil { + return Settings{}, errors.New("providers are required") + } + selectedCleared := false + for _, id := range providerIDs { + next, ok := incoming.Providers[id] + if !ok { + next = current.Providers[id] + } + old := current.Providers[id] + clear, _ := next["clearSecrets"].(bool) + if incoming.Provider != nil && *incoming.Provider == id { + selectedCleared = clear + } + delete(next, "clearSecrets") + delete(next, "configured") + for _, field := range secretFields { + value, _ := next[field].(string) + if clear { + next[field] = "" + } else if strings.TrimSpace(value) == "" { + next[field] = old[field] + } else { + next[field] = strings.TrimSpace(value) + } + } + next["id"] = id + current.Providers[id] = next + } + if incoming.Provider != nil && !selectedCleared && !incoming.AllowIncomplete { + provider, ok := current.Providers[*incoming.Provider] + if !ok { + return Settings{}, errors.New("selected STT provider is unknown") + } + if err := validateProvider(*incoming.Provider, provider); err != nil { + return Settings{}, err + } + } + current.Enabled = incoming.Enabled + current.Provider = incoming.Provider + current.AllowIncomplete = false + payload, err := json.Marshal(current) + if err != nil { + return Settings{}, err + } + _, err = s.pool.ExecContext(ctx, `INSERT INTO stt_settings(config_id,payload_json,updated_at) VALUES('default',?,unixepoch('subsec')*1000) ON CONFLICT(config_id) DO UPDATE SET payload_json=excluded.payload_json,updated_at=excluded.updated_at`, string(payload)) + if err != nil { + return Settings{}, err + } + return redact(current), nil +} + +// SyncFromDesktop replaces the Gateway runtime copy with the desktop's raw, +// authoritative STT settings. Unlike Update, blank secrets are real values +// here (including an explicit clear already consumed by the desktop store), +// so they must not preserve stale Gateway credentials. +func (s *Store) SyncFromDesktop(ctx context.Context, incoming Settings) (Settings, error) { + if incoming.Providers == nil { + return Settings{}, errors.New("providers are required") + } + next := defaults() + if incoming.Provider != nil { + if _, known := next.Providers[*incoming.Provider]; !known { + return Settings{}, errors.New("selected STT provider is unknown") + } + provider := strings.TrimSpace(*incoming.Provider) + incoming.Provider = &provider + } + for _, id := range providerIDs { + provider := next.Providers[id] + for key, value := range incoming.Providers[id] { + provider[key] = value + } + provider["id"] = id + delete(provider, "configured") + delete(provider, "clearSecrets") + for _, field := range secretFields { + provider[field] = strings.TrimSpace(stringValue(provider, field)) + } + next.Providers[id] = provider + } + next.Provider = incoming.Provider + next.Enabled = incoming.Enabled + next.AllowIncomplete = false + payload, err := json.Marshal(next) + if err != nil { + return Settings{}, err + } + _, err = s.pool.ExecContext(ctx, `INSERT INTO stt_settings(config_id,payload_json,updated_at) VALUES('default',?,unixepoch('subsec')*1000) ON CONFLICT(config_id) DO UPDATE SET payload_json=excluded.payload_json,updated_at=excluded.updated_at`, string(payload)) + if err != nil { + return Settings{}, err + } + return redact(next), nil +} + +func (s *Store) Provider(ctx context.Context, id string) (map[string]any, error) { + current, err := s.raw(ctx) + if err != nil { + return nil, err + } + p, ok := current.Providers[id] + if !ok || !configured(id, p) { + return nil, errors.New("STT provider is not configured") + } + return p, nil +} diff --git a/crates/agent-gateway/internal/stt/stt_test.go b/crates/agent-gateway/internal/stt/stt_test.go new file mode 100644 index 000000000..0b76cbf95 --- /dev/null +++ b/crates/agent-gateway/internal/stt/stt_test.go @@ -0,0 +1,833 @@ +package stt + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/binary" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/liveagent/agent-gateway/internal/db" + gatewayv2 "github.com/liveagent/agent-gateway/internal/proto/v2" + "github.com/liveagent/agent-gateway/internal/protocol/pbws" + "google.golang.org/protobuf/proto" +) + +type fixtureAdapter struct{ consume bool } + +type errorAdapter struct{} + +type probeAdapter struct { + commands chan Command +} + +func (errorAdapter) Test(context.Context, map[string]any) (string, error) { + return "protocol_failed", errors.New("provider fixture-secret") +} +func (errorAdapter) Run(context.Context, string, map[string]any, <-chan Command, chan<- Event) error { + return errors.New("provider fixture-secret") +} + +func (a probeAdapter) Test(context.Context, map[string]any) (string, error) { + return "connected", nil +} + +func (a probeAdapter) Run(ctx context.Context, id string, _ map[string]any, commands <-chan Command, events chan<- Event) error { + events <- Event{Type: "ready", SessionID: id} + for { + select { + case <-ctx.Done(): + return nil + case command := <-commands: + a.commands <- command + if command.Cancel || command.Finish { + return nil + } + } + } +} + +func (a fixtureAdapter) Test(context.Context, map[string]any) (string, error) { + return "connected_no_speech", nil +} +func (a fixtureAdapter) Run(ctx context.Context, id string, cfg map[string]any, commands <-chan Command, events chan<- Event) error { + events <- Event{Type: "ready", SessionID: id} + if !a.consume { + <-ctx.Done() + return ctx.Err() + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case command := <-commands: + if command.Audio != nil { + events <- Event{Type: "partial", SessionID: id, Text: "fixture"} + } + if command.Finish { + events <- Event{Type: "final", SessionID: id, Text: "fixture final"} + return nil + } + } + } +} + +func fixtureStore(t *testing.T) *Store { + t.Helper() + database, err := db.Open(t.TempDir() + "/stt.db") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + store, err := NewStore(database) + if err != nil { + t.Fatal(err) + } + settings := defaults() + settings.Enabled = true + settings.Providers["aliyun_dashscope"]["apiKey"] = "fixture-secret" + if _, err := store.Update(context.Background(), settings); err != nil { + t.Fatal(err) + } + return store +} + +func TestSyncFromDesktopClearsStaleGatewaySecrets(t *testing.T) { + store := fixtureStore(t) + settings := defaults() + settings.Enabled = true + provider := "aliyun_dashscope" + settings.Provider = &provider + settings.Providers[provider]["apiKey"] = "" + if _, err := store.SyncFromDesktop(context.Background(), settings); err != nil { + t.Fatal(err) + } + redacted, err := store.Get(context.Background()) + if err != nil { + t.Fatal(err) + } + if redacted.Providers[provider]["configured"] != false { + t.Fatalf("expected cleared provider to be unconfigured: %#v", redacted.Providers[provider]) + } + if !redacted.Enabled { + t.Fatal("STT enabled flag was not synchronized from desktop") + } + if _, err := store.Provider(context.Background(), provider); err == nil { + t.Fatal("cleared desktop secret remained usable in Gateway runtime") + } +} + +func withFixtureAdapter(t *testing.T, adapter Adapter) { + t.Helper() + previous := adapterFactory + adapterFactory = func(string) Adapter { return adapter } + t.Cleanup(func() { adapterFactory = previous }) +} + +func TestConnectionProbeUsesCompleteSilentAudioProtocol(t *testing.T) { + commands := make(chan Command, 16) + result, err := silentProtocolTest(context.Background(), probeAdapter{commands: commands}, map[string]any{}) + if err != nil || result != "connected_no_speech" { + t.Fatalf("silent protocol probe failed: result=%q err=%v", result, err) + } + for sequence := uint32(0); sequence < 10; sequence++ { + command := <-commands + if command.Audio == nil || command.Audio.Sequence != sequence || len(command.Audio.PCM) != 3200 { + t.Fatalf("unexpected silent probe packet %d: %#v", sequence, command) + } + } + if command := <-commands; !command.Finish || command.Audio != nil || command.Cancel { + t.Fatalf("silent probe must finish after ten packets: %#v", command) + } +} + +func TestReadyConnectionProbeStopsAfterProviderReady(t *testing.T) { + commands := make(chan Command, 1) + result, err := readyProtocolTest(context.Background(), probeAdapter{commands: commands}, map[string]any{}) + if err != nil || result != "connected" { + t.Fatalf("ready protocol probe failed: result=%q err=%v", result, err) + } + select { + case command := <-commands: + if !command.Cancel || command.Audio != nil || command.Finish { + t.Fatalf("ready probe must cancel without sending synthetic audio: %#v", command) + } + case <-time.After(time.Second): + t.Fatal("ready probe did not cancel the provider session") + } +} + +func TestManagerRejectsDuplicateUnknownAndMissingSessions(t *testing.T) { + manager := NewManager(fixtureStore(t)) + withFixtureAdapter(t, fixtureAdapter{consume: false}) + events := make(chan Event, 256) + if err := manager.Start(context.Background(), "session", "aliyun_dashscope", events); err != nil { + t.Fatal(err) + } + if err := manager.Start(context.Background(), "session", "aliyun_dashscope", events); err == nil { + t.Fatal("duplicate session id must be rejected") + } + if err := manager.Start(context.Background(), "other", "unknown", events); err == nil { + t.Fatal("unknown provider must be rejected") + } + if err := manager.Send("missing", Command{}); err == nil { + t.Fatal("unknown session must be rejected") + } + for index := 0; index < 128; index++ { + if err := manager.Send("session", Command{Audio: &AudioChunk{Sequence: uint32(index), PCM: []byte{0, 0}}}); err != nil { + t.Fatalf("queue fill failed at %d: %v", index, err) + } + } + if err := manager.Send("session", Command{Audio: &AudioChunk{Sequence: 128, PCM: []byte{0, 0}}}); err == nil { + t.Fatal("full write queue must be rejected") + } + manager.Cancel("session") +} + +func TestEmitEventUnblocksWhenConsumerIsCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + events := make(chan Event) + done := make(chan bool, 1) + go func() { + done <- emitEvent(ctx, events, Event{Type: "partial", SessionID: "session"}) + }() + cancel() + select { + case emitted := <-done: + if emitted { + t.Fatal("emitEvent reported delivery after context cancellation") + } + case <-time.After(time.Second): + t.Fatal("emitEvent remained blocked after context cancellation") + } +} + +func TestEmitIncomingUnblocksWhenConsumerIsCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + incoming := make(chan map[string]any) + done := make(chan bool, 1) + go func() { + done <- emitIncoming(ctx, incoming, map[string]any{"type": "partial"}) + }() + cancel() + select { + case emitted := <-done: + if emitted { + t.Fatal("emitIncoming reported delivery after context cancellation") + } + case <-time.After(time.Second): + t.Fatal("emitIncoming remained blocked after context cancellation") + } +} + +func TestSttHelloTimesOutWithoutFirstFrame(t *testing.T) { + previous := sttHelloTimeout + sttHelloTimeout = 50 * time.Millisecond + t.Cleanup(func() { sttHelloTimeout = previous }) + + manager := NewManager(fixtureStore(t)) + server := httptest.NewServer(manager.WebSocketHandler("token")) + t.Cleanup(server.Close) + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + conn.SetReadDeadline(time.Now().Add(time.Second)) + if _, _, err := conn.ReadMessage(); err == nil { + t.Fatal("hello timeout must close the connection before any server frame") + } +} + +func TestManagerCancelsAdapterAndRedactsProviderErrors(t *testing.T) { + manager := NewManager(fixtureStore(t)) + withFixtureAdapter(t, errorAdapter{}) + events := make(chan Event, 256) + if err := manager.Start(context.Background(), "session", "aliyun_dashscope", events); err != nil { + t.Fatal(err) + } + deadline := time.After(time.Second) + seenError := false + for { + select { + case event := <-events: + if event.Type == "error" { + seenError = true + if strings.Contains(event.Message, "fixture-secret") { + t.Fatalf("provider secret leaked in event: %q", event.Message) + } + } + if event.Type == "closed" { + if !seenError { + t.Fatal("provider error event was not emitted") + } + return + } + case <-deadline: + t.Fatal("cancel did not close adapter") + } + } +} + +func TestSttServerEventMappingAndWebSocketSequenceValidation(t *testing.T) { + manager := NewManager(fixtureStore(t)) + withFixtureAdapter(t, fixtureAdapter{consume: true}) + // Exercise every server-event oneof directly so a regression cannot silently + // collapse final/error/closed into the same frame shape. + eventServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + for _, event := range []Event{ + {Type: "ready", SessionID: "s"}, + {Type: "partial", SessionID: "s", Text: "p"}, + {Type: "final", SessionID: "s", Text: "f"}, + {Type: "error", SessionID: "s", Code: "bad", Message: "safe"}, + {Type: "closed", SessionID: "s"}, + } { + if !writeSttEvent(conn, event) { + return + } + } + })) + t.Cleanup(eventServer.Close) + eventURL := "ws" + strings.TrimPrefix(eventServer.URL, "http") + eventConn, _, err := websocket.DefaultDialer.Dial(eventURL, nil) + if err != nil { + t.Fatal(err) + } + for _, want := range []func(*gatewayv2.SttServerFrame) bool{ + func(frame *gatewayv2.SttServerFrame) bool { return frame.GetReady().GetSessionId() == "s" }, + func(frame *gatewayv2.SttServerFrame) bool { return frame.GetPartial().GetText() == "p" }, + func(frame *gatewayv2.SttServerFrame) bool { return frame.GetFinal().GetText() == "f" }, + func(frame *gatewayv2.SttServerFrame) bool { + return frame.GetError().GetCode() == "bad" && frame.GetError().GetMessage() == "safe" + }, + func(frame *gatewayv2.SttServerFrame) bool { return frame.GetClosed().GetSessionId() == "s" }, + } { + kind, data, readErr := eventConn.ReadMessage() + if readErr != nil || kind != websocket.BinaryMessage { + t.Fatalf("event frame read failed: kind=%d err=%v", kind, readErr) + } + var frame gatewayv2.SttServerFrame + if err := proto.Unmarshal(data, &frame); err != nil || !want(&frame) { + t.Fatalf("unexpected event frame: %v %#v", err, frame.GetPayload()) + } + } + _ = eventConn.Close() + + server := httptest.NewServer(manager.WebSocketHandler("token")) + t.Cleanup(server.Close) + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + hello := &gatewayv2.SttClientFrame{Payload: &gatewayv2.SttClientFrame_Hello{Hello: &gatewayv2.SttClientHello{ProtocolVersion: pbws.ProtocolVersion, Token: "token"}}} + if err := conn.WriteMessage(websocket.BinaryMessage, mustProto(hello)); err != nil { + t.Fatal(err) + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Fatal("hello response: ", err) + } + if err := conn.WriteMessage(websocket.BinaryMessage, mustProto(&gatewayv2.SttClientFrame{Payload: &gatewayv2.SttClientFrame_Start{Start: &gatewayv2.SttStart{SessionId: "s", Provider: "aliyun_dashscope"}}})); err != nil { + t.Fatal(err) + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Fatal("ready response: ", err) + } + audio := func(sequence uint32) []byte { + return mustProto(&gatewayv2.SttClientFrame{Payload: &gatewayv2.SttClientFrame_Audio{Audio: &gatewayv2.SttAudio{SessionId: "s", Sequence: sequence, Pcm: []byte{0, 0}}}}) + } + if err := conn.WriteMessage(websocket.BinaryMessage, audio(0)); err != nil { + t.Fatal(err) + } + if _, _, err := conn.ReadMessage(); err != nil { + t.Fatal("partial response: ", err) + } + if err := conn.WriteMessage(websocket.BinaryMessage, audio(0)); err != nil { + t.Fatal(err) + } + conn.SetReadDeadline(time.Now().Add(time.Second)) + if _, _, err := conn.ReadMessage(); err == nil { + t.Fatal("duplicate sequence must close the connection") + } +} + +func mustProto(message proto.Message) []byte { + data, err := proto.Marshal(message) + if err != nil { + panic(err) + } + return data +} + +func TestVolcV2FrameRoundTrip(t *testing.T) { + payload, err := gzipJSON(map[string]any{"code": float64(1000), "message": "Success"}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeVolcV2Frame(volcV2Frame(1, 0, 1, 1, payload)) + if err != nil { + t.Fatal(err) + } + if !volcV2ResponseOK(decoded) { + t.Fatalf("code 1000/Success should be accepted: %#v", decoded) + } + if _, err := decodeVolcV2Frame([]byte{0x11}); err == nil { + t.Fatal("short frame should fail") + } + end := volcV2Frame(2, 2, 0, 0, nil) + if end[1] != 0x22 || len(end) != 8 { + t.Fatalf("unexpected v2 end frame: %#v", end) + } +} + +func TestVolcV2AndSeedV3EndpointsRemainDistinct(t *testing.T) { + if strings.Contains("wss://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash", "/api/v2/asr") { + t.Fatal("v3 endpoint must not be v2 endpoint") + } +} + +func TestTencentSignedURL(t *testing.T) { + signed := tencentSignedURL("123", "16k_zh", "sid", "secret", "voice", 1700000000) + u, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + if u.Path != "/asr/v2/123" { + t.Fatalf("unexpected path: %s", u.Path) + } + q := u.Query() + for _, field := range []string{"convert_num_mode", "engine_model_type", "expired", "filter_dirty", "filter_modal", "filter_punc", "needvad", "nonce", "secretid", "timestamp", "voice_format", "voice_id", "word_info", "signature"} { + if q.Get(field) == "" { + t.Fatalf("missing %s", field) + } + } + if len(q.Get("signature")) < 20 { + t.Fatal("signature is not HMAC-SHA1 base64") + } + if strings.Contains(signed, "wss://asr.cloud.tencent.com/asr/v2/123?wss://") { + t.Fatal("signature query must not be duplicated") + } + if nonce, err := strconv.ParseUint(q.Get("nonce"), 10, 32); err != nil || nonce == 0 { + t.Fatalf("Tencent nonce must be a non-zero decimal integer: %q", q.Get("nonce")) + } +} + +func TestProviderProtocolFixtures(t *testing.T) { + if got := dashScopeModel(""); got != "paraformer-realtime-v2" { + t.Fatalf("DashScope default model must accept 16 kHz PCM: %q", got) + } + if got := dashScopeModel("paraformer-realtime-8k-v2"); got != "paraformer-realtime-v2" { + t.Fatalf("legacy incorrect DashScope model was not migrated: %q", got) + } + if dashScopeFinish("fixture-task")["payload"] == nil { + t.Fatal("DashScope finish-task must include payload.input") + } + if got := mergeTencentFragments(map[int]string{4: "后", 1: "前", 2: "中"}); got != "前中后" { + t.Fatalf("乱序片段未按 index 合并: %q", got) + } + if !baiduNoSpeech(3301) || !baiduNoSpeech(-3005) || baiduNoSpeech(3300) { + t.Fatal("百度无语音错误码分类错误") + } + if got := volcV2Authorization("token"); got != "Bearer; token" { + t.Fatalf("火山 v2 authorization 格式错误: %q", got) + } + if volcengineV2Endpoint == volcengineSeedV3Endpoint { + t.Fatal("火山 v2/v3 endpoint must remain isolated") + } + if !tencentMessageComplete(map[string]any{"final": float64(1)}) || tencentMessageComplete(map[string]any{"final": float64(0)}) { + t.Fatal("腾讯结束响应必须按 final=1 判定") + } + index, transcript, ok := tencentResult(map[string]any{"index": float64(99), "result": map[string]any{"index": float64(7), "voice_text_str": "嵌套结果"}}) + if !ok || index != 7 || transcript != "嵌套结果" { + t.Fatalf("Tencent result index must come from result.index: %d %q %v", index, transcript, ok) + } +} + +func TestProviderFailureKeepsNoSpeechOutOfProtocolErrors(t *testing.T) { + err := providerFailure("Volcengine v2", "1013", "No valid speeches found in input audio") + if result := resultForError(err); result != "connected_no_speech" { + t.Fatalf("resultForError() = %q, want connected_no_speech", result) + } + + wrapped := stageError("VolcengineV2", "provider_response", err) + if result := resultForError(wrapped); result != "connected_no_speech" { + t.Fatalf("wrapped resultForError() = %q, want connected_no_speech", result) + } + if !strings.Contains(wrapped.Error(), "[VolcengineV2/provider_response]") { + t.Fatalf("stage context missing from %q", wrapped.Error()) + } + + if !volcV2ResponseNoSpeech(map[string]any{"code": float64(1013)}) { + t.Fatal("火山 v2 code 1013 must be classified as no speech") + } + if volcV2ResponseNoSpeech(map[string]any{"code": float64(45000000)}) { + t.Fatal("火山 v2 non-1013 errors must remain protocol failures") + } +} + +func TestStageErrorPreservesWebSocketCloseCodes(t *testing.T) { + closed := stageError("DashScope", "receive", &websocket.CloseError{Code: websocket.CloseNormalClosure}) + if !isWebSocketCloseError(closed, websocket.CloseNormalClosure) { + t.Fatal("stage error must preserve a normal WebSocket close code") + } + if isWebSocketCloseError(closed, websocket.CloseProtocolError) { + t.Fatal("normal closure must not match a protocol close code") + } +} + +func TestVolcengineV2RequiresExplicitLastResponseToComplete(t *testing.T) { + textOnly := map[string]any{"result": map[string]any{"text": "partial"}} + if volcV2ResponseComplete(textOnly, true) { + t.Fatal("text without a last marker must not complete the v2 session") + } + if !volcV2ResponseComplete(map[string]any{"_last": true}, true) { + t.Fatal("negative-sequence response must complete a finishing v2 session") + } + if volcV2ResponseComplete(map[string]any{"_last": true}, false) { + t.Fatal("last response must not complete before the client starts finishing") + } +} + +func TestProviderFieldDefaultsMatchRuntimeProtocols(t *testing.T) { + settings := defaults() + aliyun := settings.Providers["aliyun_dashscope"] + if aliyun["websocketUrl"] != dashScopeEndpoint || aliyun["model"] != "paraformer-realtime-v2" { + t.Fatalf("unexpected DashScope defaults: %#v", aliyun) + } + v2 := settings.Providers["volcengine_v2"] + if v2["websocketUrl"] != volcengineV2Endpoint || v2["cluster"] != "" { + t.Fatalf("unexpected Volcengine v2 defaults: %#v", v2) + } + if settings.Providers["volcengine_seed_v3"]["websocketUrl"] != volcengineSeedV3Endpoint { + t.Fatal("Seed v3 WebSocket default does not match the adapter") + } + if settings.Providers["baidu_cloud"]["websocketUrl"] != baiduRealtimeEndpoint { + t.Fatal("Baidu WebSocket default does not match the adapter") + } +} + +func TestVolcengineAudioAndSequenceFrames(t *testing.T) { + start := volcV2StartRequest(map[string]any{"appId": "app", "accessToken": "token", "cluster": "cluster"}, "session", "request") + app, _ := start["app"].(map[string]any) + audioRequest, _ := start["audio"].(map[string]any) + request, _ := start["request"].(map[string]any) + if app["token"] != "token" || audioRequest["format"] != "raw" || audioRequest["codec"] != "raw" || request["workflow"] == "" || request["show_utterances"] != true || request["result_type"] != "full" { + t.Fatalf("Volcengine v2 full request is incomplete: %#v", start) + } + compressed, err := gzipBytes([]byte{0, 1, 2, 3}) + if err != nil { + t.Fatal(err) + } + audio := volcV2Frame(2, 0, 0, 1, compressed) + if audio[1]>>4 != 2 || audio[2]&0x0f != 1 { + t.Fatalf("v2 audio frame must be audio-only gzip: %x", audio[:4]) + } + payload, err := gzipJSON(map[string]any{"code": 1000, "message": "Success"}) + if err != nil { + t.Fatal(err) + } + header := []byte{0x11, 0x91, 0x11, 0} + sequence := make([]byte, 4) + binary.BigEndian.PutUint32(sequence, 7) + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(payload))) + response := append(append(append(header, sequence...), size...), payload...) + decoded, err := decodeVolcV2Frame(response) + if err != nil || !volcV2ResponseOK(decoded) { + t.Fatalf("v2 sequence response was not decoded: %#v %v", decoded, err) + } + negativeSequence := make([]byte, 4) + binary.BigEndian.PutUint32(negativeSequence, uint32(0xffffffff)) + lastResponse := append(append(append(header, negativeSequence...), size...), payload...) + lastDecoded, err := decodeVolcV2Frame(lastResponse) + if err != nil || lastDecoded["_last"] != true { + t.Fatalf("v2 negative sequence must mark the final response: %#v %v", lastDecoded, err) + } + if got := volcV2ResultText(map[string]any{"result": []any{map[string]any{"text": "fixture"}}}); got != "fixture" { + t.Fatalf("v2 array result text was not decoded: %q", got) + } +} + +func TestVolcengineSeedV3RequestAndFrames(t *testing.T) { + if volcengineSeedV3Endpoint != "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" { + t.Fatalf("Seed v3 must use the streaming async endpoint, got %q", volcengineSeedV3Endpoint) + } + headers := seedV3Headers(map[string]any{ + "appId": "app-id", "accessToken": "access-token", "resourceId": "resource-id", + }, "connect-id") + if headers.Get("X-Api-App-Key") != "app-id" || + headers.Get("X-Api-Access-Key") != "access-token" || + headers.Get("X-Api-Resource-Id") != "resource-id" || + headers.Get("X-Api-Connect-Id") != "connect-id" { + t.Fatalf("Seed v3 connection headers are incomplete: %#v", headers) + } + if headers.Get("X-Api-Request-Id") != "" { + t.Fatal("Seed v3 must use X-Api-Connect-Id, not X-Api-Request-Id") + } + request := seedV3StartRequest("fixture-session") + audio, ok := request["audio"].(map[string]any) + if !ok || audio["format"] != "pcm" || audio["codec"] != "raw" || audio["rate"] != 16000 { + t.Fatalf("unexpected Seed v3 audio request: %#v", request["audio"]) + } + options, ok := request["request"].(map[string]any) + if !ok || options["model_name"] != "bigmodel" { + t.Fatalf("Seed v3 model_name is required: %#v", request["request"]) + } + if options["show_utterances"] != true || options["result_type"] != "full" { + t.Fatal("Seed v3 must request full utterance results") + } + compressed, err := gzipBytes([]byte{0, 1, 2, 3}) + if err != nil { + t.Fatal(err) + } + audioFrame := seedV3AudioFrame(false, compressed) + if audioFrame[1] != 0x20 || audioFrame[2] != 0x01 || int(binary.BigEndian.Uint32(audioFrame[4:8])) != len(compressed) || len(audioFrame) != 8+len(compressed) { + t.Fatalf("Seed v3 audio frame must omit the client sequence: %x", audioFrame[:8]) + } + lastFrame := seedV3AudioFrame(true, compressed) + if lastFrame[1] != 0x22 || lastFrame[2] != 0x01 || int(binary.BigEndian.Uint32(lastFrame[4:8])) != len(compressed) || len(lastFrame) != 8+len(compressed) { + t.Fatalf("Seed v3 final frame must use the last-audio flag without a client sequence: %x", lastFrame[:8]) + } +} + +func TestResultClassification(t *testing.T) { + if got := resultForError(&ResultError{Result: "authentication_failed", Err: context.Canceled}); got != "authentication_failed" { + t.Fatal(got) + } + if got := resultForError(&net.DNSError{IsTimeout: true}); got != "network_failed" { + t.Fatal(got) + } + if got := resultForError(context.DeadlineExceeded); got != "timeout" { + t.Fatal(got) + } + for status, expected := range map[int]string{ + http.StatusUnauthorized: "authentication_failed", + http.StatusForbidden: "authentication_failed", + http.StatusBadRequest: "protocol_failed", + http.StatusInternalServerError: "network_failed", + http.StatusServiceUnavailable: "network_failed", + } { + response := &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader("must not be exposed"))} + classified := websocketConnectError(response, errors.New("websocket: bad handshake")) + if got := resultForError(classified); got != expected { + t.Fatalf("HTTP %d classified as %q, want %q", status, got, expected) + } + if strings.Contains(classified.Error(), "must not be exposed") { + t.Fatal("handshake response body must not be included in diagnostics") + } + } + diagnostic := sanitizeError( + "request wss://example.invalid/asr?secretid=fixture-id&signature=fixture-signature failed with fixture-key", + map[string]any{"secretKey": "fixture-key"}, + ) + if diagnostic != "request [provider endpoint] failed with [redacted]" { + t.Fatalf("provider diagnostic was not safely redacted: %q", diagnostic) + } +} + +func TestSettingsRedactPreserveAndClearSecrets(t *testing.T) { + database, err := db.Open(t.TempDir() + "/stt.db") + if err != nil { + t.Fatal(err) + } + defer database.Close() + store, err := NewStore(database) + if err != nil { + t.Fatal(err) + } + incoming := defaults() + incoming.Providers["aliyun_dashscope"]["apiKey"] = "secret-value" + if _, err := store.Update(context.Background(), incoming); err != nil { + t.Fatal(err) + } + redacted, err := store.Get(context.Background()) + if err != nil { + t.Fatal(err) + } + if redacted.Providers["aliyun_dashscope"]["apiKey"] != "" || !redacted.Providers["aliyun_dashscope"]["configured"].(bool) { + t.Fatalf("secret leaked or provider not configured: %#v", redacted.Providers["aliyun_dashscope"]) + } + keep := defaults() + keep.Providers["aliyun_dashscope"] = map[string]any{"apiKey": "", "id": "aliyun_dashscope"} + if _, err := store.Update(context.Background(), keep); err != nil { + t.Fatal(err) + } + raw, err := store.Provider(context.Background(), "aliyun_dashscope") + if err != nil || raw["apiKey"] != "secret-value" { + t.Fatalf("blank update should preserve secret: %#v %v", raw, err) + } + clear := defaults() + clear.Providers["aliyun_dashscope"] = map[string]any{"apiKey": "", "clearSecrets": true, "id": "aliyun_dashscope"} + if _, err := store.Update(context.Background(), clear); err != nil { + t.Fatal(err) + } + if _, err := store.Provider(context.Background(), "aliyun_dashscope"); err == nil { + t.Fatal("cleared provider should be unavailable") + } +} + +func TestBaiduConfiguredRequiresNumericIDs(t *testing.T) { + p := map[string]any{ + "websocketUrl": baiduRealtimeEndpoint, + "baiduAppId": "abc", + "baiduApiKey": "key", + "devPid": "1537", + } + if configured("baidu_cloud", p) { + t.Fatal("non-numeric appid must not configure provider") + } + p["baiduAppId"] = "123" + if !configured("baidu_cloud", p) { + t.Fatal("numeric appid and dev_pid should configure provider") + } + p["devPid"] = "0" + if configured("baidu_cloud", p) { + t.Fatal("zero dev_pid must not configure provider") + } +} + +func TestSettingsUpdateValidatesSelectedProviderOnly(t *testing.T) { + store := fixtureStore(t) + invalid := defaults() + provider := "tencent_cloud" + invalid.Provider = &provider + invalid.Providers[provider]["appId"] = "not-numeric" + invalid.Providers[provider]["engineModelType"] = "16k_zh" + invalid.Providers[provider]["secretId"] = "id" + invalid.Providers[provider]["secretKey"] = "key" + if _, err := store.Update(context.Background(), invalid); err == nil || !strings.Contains(err.Error(), "positive decimal integer") { + t.Fatalf("invalid Tencent AppId must be rejected before persistence: %v", err) + } + + clear := defaults() + clear.Provider = &provider + clear.Providers[provider]["clearSecrets"] = true + if _, err := store.Update(context.Background(), clear); err != nil { + t.Fatalf("explicit secret clearing must remain valid: %v", err) + } +} + +func TestSettingsUpdateAllowsVoiceToggleWithIncompleteProvider(t *testing.T) { + store := fixtureStore(t) + provider := "tencent_cloud" + settings := defaults() + settings.Provider = &provider + settings.AllowIncomplete = true + settings.Providers[provider]["appId"] = "" + settings.Providers[provider]["engineModelType"] = "16k_zh" + settings.Providers[provider]["secretId"] = "" + settings.Providers[provider]["secretKey"] = "" + if _, err := store.Update(context.Background(), settings); err != nil { + t.Fatalf("voice-input toggle should persist with incomplete provider: %v", err) + } +} + +func TestBaiduStartUsesNumericIDsAndNestedData(t *testing.T) { + message := baiduStartMessage(123, 1537, "app-key", "session") + data, ok := message["data"].(map[string]any) + if !ok { + t.Fatal("START data must be an object") + } + if _, ok := data["appid"].(uint64); !ok { + t.Fatalf("appid must be JSON number, got %T", data["appid"]) + } + if _, ok := data["dev_pid"].(uint32); !ok { + t.Fatalf("dev_pid must be numeric, got %T", data["dev_pid"]) + } + if _, ok := message["appid"]; ok { + t.Fatal("appid must be nested under START data") + } +} + +func TestBaiduAcceptsNoStatusCloseAfterFinish(t *testing.T) { + err := &websocket.CloseError{Code: websocket.CloseNoStatusReceived, Text: "no status"} + if !baiduFinishedConnectionClosed(err) { + t.Fatal("Baidu FINISH must accept a provider close without a status frame") + } + if baiduFinishedConnectionClosed(&websocket.CloseError{Code: websocket.CloseProtocolError, Text: "bad frame"}) { + t.Fatal("Baidu must not hide a real protocol close") + } +} + +func TestBaiduEndpointAndResultFixtures(t *testing.T) { + endpoint, err := baiduEndpoint(baiduRealtimeEndpoint) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Query().Get("sn") == "" { + t.Fatalf("Baidu WebSocket URL must include a unique sn: %q %v", endpoint, err) + } + if text := baiduResultText([]any{"你好，", "世界。"}); text != "你好，世界。" { + t.Fatalf("Baidu array result was not joined: %q", text) + } + if text := baiduResultText(" 单句 "); text != "单句" { + t.Fatalf("Baidu string result was not normalized: %q", text) + } +} + +func TestSeedV3FrameRoundTrip(t *testing.T) { + payload, err := seedV3JSON(map[string]any{"is_last_package": true, "result": map[string]any{"text": "ok"}}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeSeedV3Frame(seedV3Frame(9, 0, 1, payload)) + if err != nil { + t.Fatal(err) + } + if decoded["is_last_package"] != true { + t.Fatalf("unexpected Seed v3 response: %#v", decoded) + } +} + +func TestGzipPayloadIsJSON(t *testing.T) { + payload, err := gzipJSON(map[string]string{"x": "y"}) + if err != nil { + t.Fatal(err) + } + reader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + var value map[string]string + if json.Unmarshal(data, &value) != nil || value["x"] != "y" { + t.Fatalf("invalid gzip JSON: %s", data) + } +} + +func TestSttProtobufAudioRoundTrip(t *testing.T) { + want := &gatewayv2.SttClientFrame{Payload: &gatewayv2.SttClientFrame_Audio{Audio: &gatewayv2.SttAudio{SessionId: "session", Sequence: 7, Pcm: []byte{0, 1, 2, 3}}}} + data, err := proto.Marshal(want) + if err != nil { + t.Fatal(err) + } + var got gatewayv2.SttClientFrame + if err := proto.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if got.GetAudio().GetSessionId() != "session" || got.GetAudio().GetSequence() != 7 || !bytes.Equal(got.GetAudio().GetPcm(), []byte{0, 1, 2, 3}) { + t.Fatalf("unexpected STT frame: %#v", got.GetAudio()) + } +} diff --git a/crates/agent-gateway/internal/stt/tencent_cloud.go b/crates/agent-gateway/internal/stt/tencent_cloud.go new file mode 100644 index 000000000..0339661f7 --- /dev/null +++ b/crates/agent-gateway/internal/stt/tencent_cloud.go @@ -0,0 +1,161 @@ +package stt + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +type TencentCloudAdapter struct{} + +const tencentEndpoint = "wss://asr.cloud.tencent.com/asr/v2/" + +func (a *TencentCloudAdapter) Test(ctx context.Context, cfg map[string]any) (string, error) { + return readyProtocolTest(ctx, a, cfg) +} + +func tencentSignedURL(appID, engine, secretID, secretKey, voiceID string, timestamp int64) string { + path := fmt.Sprintf("asr.cloud.tencent.com/asr/v2/%s", appID) + q := url.Values{ + "convert_num_mode": {"1"}, + "engine_model_type": {engine}, + "expired": {strconv.FormatInt(timestamp+24*60*60, 10)}, + "filter_dirty": {"1"}, + "filter_modal": {"2"}, + "filter_punc": {"0"}, + "needvad": {"0"}, + "nonce": {strconv.FormatUint(uint64(uuid.New().ID()), 10)}, + "secretid": {secretID}, + "timestamp": {strconv.FormatInt(timestamp, 10)}, + "voice_format": {"1"}, + "voice_id": {voiceID}, + "word_info": {"0"}, + } + signed := path + "?" + q.Encode() + mac := hmac.New(sha1.New, []byte(secretKey)) + _, _ = mac.Write([]byte(signed)) + q.Set("signature", base64.StdEncoding.EncodeToString(mac.Sum(nil))) + return tencentEndpoint + appID + "?" + q.Encode() +} + +func (a *TencentCloudAdapter) Run(ctx context.Context, id string, cfg map[string]any, commands <-chan Command, events chan<- Event) error { + voiceID := uuid.NewString() + conn, response, err := websocket.DefaultDialer.DialContext(ctx, tencentSignedURL(value(cfg, "appId"), value(cfg, "engineModelType"), value(cfg, "secretId"), value(cfg, "secretKey"), voiceID, time.Now().Unix()), nil) + if err != nil { + return stageError("Tencent", "connect", websocketConnectError(response, err)) + } + defer func() { _ = conn.Close() }() + time.Sleep(25 * time.Millisecond) + incoming := make(chan map[string]any, 8) + readErr := make(chan error, 1) + go func() { + for { + var msg map[string]any + if e := conn.ReadJSON(&msg); e != nil { + readErr <- stageError("Tencent", "receive", e) + return + } + if !emitIncoming(ctx, incoming, msg) { + return + } + } + }() + if !emitEvent(ctx, events, Event{Type: "ready", SessionID: id}) { + return ctx.Err() + } + fragments := map[int]string{} + finishSent := false + for { + select { + case <-ctx.Done(): + return nil + case e := <-readErr: + if finishSent && isWebSocketCloseError(e, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + if len(fragments) > 0 { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: mergeTencentFragments(fragments)}) { + return ctx.Err() + } + } + return nil + } + if finishSent { + return stageError("Tencent", "close", e) + } + return e + case cmd := <-commands: + if cmd.Cancel { + return nil + } + if cmd.Audio != nil { + if e := writeProviderMessage(conn, websocket.BinaryMessage, cmd.Audio.PCM); e != nil { + return stageError("Tencent", "send_audio", e) + } + } + if cmd.Finish { + finishSent = true + if e := writeProviderJSON(conn, map[string]any{"type": "end"}); e != nil { + return stageError("Tencent", "finish", e) + } + } + case msg := <-incoming: + if code, ok := msg["code"].(float64); ok && code != 0 { + return stageError("Tencent", "provider_response", providerFailure("Tencent Cloud", strconv.Itoa(int(code)), valueString(msg, "message"))) + } + if idx, transcript, ok := tencentResult(msg); ok { + fragments[idx] = transcript + if !emitEvent(ctx, events, Event{Type: "partial", SessionID: id, Text: mergeTencentFragments(fragments)}) { + return ctx.Err() + } + } + if finishSent && tencentMessageComplete(msg) { + if len(fragments) > 0 { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: mergeTencentFragments(fragments)}) { + return ctx.Err() + } + } + return nil + } + } + } +} +func number(v any) float64 { n, _ := v.(float64); return n } +func valueString(message map[string]any, key string) string { + text, _ := message[key].(string) + return text +} + +func tencentMessageComplete(message map[string]any) bool { + return message["type"] == "end" || number(message["final"]) == 1 +} + +func tencentResult(message map[string]any) (int, string, bool) { + result, ok := message["result"].(map[string]any) + if !ok { + return 0, "", false + } + text, ok := result["voice_text_str"].(string) + return int(number(result["index"])), text, ok +} + +func mergeTencentFragments(fragments map[int]string) string { + keys := make([]int, 0, len(fragments)) + for key := range fragments { + keys = append(keys, key) + } + sort.Ints(keys) + var merged strings.Builder + for _, key := range keys { + merged.WriteString(fragments[key]) + } + return merged.String() +} diff --git a/crates/agent-gateway/internal/stt/volcengine_seed_v3.go b/crates/agent-gateway/internal/stt/volcengine_seed_v3.go new file mode 100644 index 000000000..97d08ef63 --- /dev/null +++ b/crates/agent-gateway/internal/stt/volcengine_seed_v3.go @@ -0,0 +1,293 @@ +package stt + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/google/uuid" + "github.com/gorilla/websocket" + "github.com/klauspost/compress/gzip" +) + +type VolcengineSeedV3Adapter struct{} + +const volcengineSeedV3Endpoint = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" + +func (a *VolcengineSeedV3Adapter) Test(ctx context.Context, cfg map[string]any) (string, error) { + return readyProtocolTest(ctx, a, cfg) +} + +func seedV3JSON(value any) ([]byte, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var output bytes.Buffer + writer := gzip.NewWriter(&output) + if _, err = writer.Write(raw); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func seedV3Frame(messageType, flags, serialization byte, payload []byte) []byte { + frame := []byte{0x11, messageType<<4 | flags, serialization<<4 | 0x01, 0} + length := make([]byte, 4) + binary.BigEndian.PutUint32(length, uint32(len(payload))) + return append(append(frame, length...), payload...) +} + +func seedV3AudioFrame(last bool, payload []byte) []byte { + flags := byte(0) + if last { + flags = 2 + } + return seedV3Frame(2, flags, 0, payload) +} + +func seedV3StartRequest(id string) map[string]any { + return map[string]any{ + "user": map[string]any{"uid": id}, + "audio": map[string]any{"format": "pcm", "codec": "raw", "rate": 16000, "bits": 16, "channel": 1}, + "request": map[string]any{ + "model_name": "bigmodel", + "enable_itn": true, + "enable_punc": true, + "enable_ddc": true, + "show_utterances": true, + "result_type": "full", + }, + } +} + +func seedV3Headers(cfg map[string]any, connectID string) http.Header { + header := http.Header{} + header.Set("X-Api-App-Key", value(cfg, "appId")) + header.Set("X-Api-Access-Key", value(cfg, "accessToken")) + header.Set("X-Api-Resource-Id", value(cfg, "resourceId")) + header.Set("X-Api-Connect-Id", connectID) + return header +} + +func decodeSeedV3Frame(data []byte) (map[string]any, error) { + if len(data) < 8 || data[0]&0x0f < 1 { + return nil, errors.New("seed v3 invalid frame") + } + headerLength := int(data[0]&0x0f) * 4 + if len(data) < headerLength+4 { + return nil, errors.New("seed v3 invalid header") + } + messageType := data[1] >> 4 + flags := data[1] & 0x0f + offset := headerLength + var errorCode uint32 + var sequence int32 + if messageType == 0x0f { + if len(data) < offset+8 { + return nil, errors.New("seed v3 invalid error frame") + } + errorCode = binary.BigEndian.Uint32(data[offset : offset+4]) + offset += 4 + } else if flags&0x01 != 0 { + if len(data) < offset+8 { + return nil, errors.New("seed v3 invalid sequence frame") + } + sequence = int32(binary.BigEndian.Uint32(data[offset : offset+4])) + offset += 4 + } + payloadLength := int(binary.BigEndian.Uint32(data[offset : offset+4])) + if len(data) < offset+4+payloadLength { + return nil, errors.New("seed v3 invalid payload") + } + payload := data[offset+4 : offset+4+payloadLength] + if data[2]&0x0f == 1 { + reader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + return nil, err + } + payload, err = io.ReadAll(reader) + _ = reader.Close() + if err != nil { + return nil, err + } + } + if messageType == 0x0f { + return map[string]any{"code": float64(errorCode), "message": string(payload)}, nil + } + var message map[string]any + if err := json.Unmarshal(payload, &message); err != nil { + return nil, err + } + message["_sequence"] = float64(sequence) + message["_last"] = flags == 2 || flags == 3 || sequence < 0 + return message, nil +} + +func (a *VolcengineSeedV3Adapter) Run(ctx context.Context, id string, cfg map[string]any, commands <-chan Command, events chan<- Event) error { + connectID := uuid.NewString() + header := seedV3Headers(cfg, connectID) + endpoint, err := websocketEndpoint(cfg, volcengineSeedV3Endpoint) + if err != nil { + return stageError("VolcengineSeedV3", "validate", err) + } + conn, response, err := websocket.DefaultDialer.DialContext(ctx, endpoint, header) + if err != nil { + return stageError("VolcengineSeedV3", "connect", websocketConnectError(response, err)) + } + defer func() { _ = conn.Close() }() + start, err := seedV3JSON(seedV3StartRequest(id)) + if err != nil { + return stageError("VolcengineSeedV3", "start", err) + } + if err = writeProviderMessage(conn, websocket.BinaryMessage, seedV3Frame(1, 0, 1, start)); err != nil { + return stageError("VolcengineSeedV3", "start", err) + } + incoming := make(chan map[string]any, 8) + readErr := make(chan error, 1) + go func() { + for { + kind, data, readError := conn.ReadMessage() + if readError != nil { + readErr <- stageError("VolcengineSeedV3", "receive", readError) + return + } + if kind != websocket.BinaryMessage { + readErr <- stageError("VolcengineSeedV3", "parse", errors.New("expected binary frame")) + return + } + message, decodeError := decodeSeedV3Frame(data) + if decodeError != nil { + readErr <- stageError("VolcengineSeedV3", "parse", decodeError) + return + } + if !emitIncoming(ctx, incoming, message) { + return + } + } + }() + ready := false + finishing := false + finishSent := false + pending := make([][]byte, 0, 32) + pendingSequences := make([]uint32, 0, 32) + var heldAudio *AudioChunk + lastText := "" + sendAudio := func(chunk *AudioChunk, last bool) error { + compressed, compressError := gzipBytes(chunk.PCM) + if compressError != nil { + return stageError("VolcengineSeedV3", "send_audio", compressError) + } + frame := seedV3AudioFrame(last, compressed) + if err := writeProviderMessage(conn, websocket.BinaryMessage, frame); err != nil { + stage := "send_audio" + if last { + stage = "finish" + } + return stageError("VolcengineSeedV3", stage, err) + } + return nil + } + holdAudio := func(chunk AudioChunk) error { + if heldAudio != nil { + if err := sendAudio(heldAudio, false); err != nil { + return err + } + } + copyChunk := AudioChunk{Sequence: chunk.Sequence, PCM: append([]byte(nil), chunk.PCM...)} + heldAudio = ©Chunk + return nil + } + finishAudio := func() error { + if heldAudio == nil { + heldAudio = &AudioChunk{Sequence: 0} + } + err := sendAudio(heldAudio, true) + heldAudio = nil + return err + } + for { + select { + case <-ctx.Done(): + return nil + case readError := <-readErr: + if finishing && finishSent { + return stageError("VolcengineSeedV3", "close", readError) + } + return readError + case command := <-commands: + if command.Cancel { + return nil + } + if command.Audio != nil { + if !ready { + pending = append(pending, append([]byte(nil), command.Audio.PCM...)) + pendingSequences = append(pendingSequences, command.Audio.Sequence) + } else if err := holdAudio(*command.Audio); err != nil { + return err + } + } + if command.Finish { + finishing = true + if ready && !finishSent { + if err := finishAudio(); err != nil { + return err + } + finishSent = true + } + } + case message := <-incoming: + if errorValue, ok := message["error"].(string); ok && errorValue != "" { + return stageError("VolcengineSeedV3", "provider_response", providerFailure("Volcengine Seed v3", fmt.Sprint(message["code"]), errorValue)) + } + if code := int(number(message["code"])); code != 0 && code != 1000 { + return stageError("VolcengineSeedV3", "provider_response", providerFailure("Volcengine Seed v3", fmt.Sprint(code), valueString(message, "message"))) + } + if !ready { + ready = true + if !emitEvent(ctx, events, Event{Type: "ready", SessionID: id}) { + return ctx.Err() + } + for index, pcm := range pending { + if err := holdAudio(AudioChunk{Sequence: pendingSequences[index], PCM: pcm}); err != nil { + return err + } + } + pending = nil + pendingSequences = nil + if finishing && !finishSent { + if err := finishAudio(); err != nil { + return err + } + finishSent = true + } + } + if result, ok := message["result"].(map[string]any); ok { + if text, ok := result["text"].(string); ok { + lastText = text + if !emitEvent(ctx, events, Event{Type: "partial", SessionID: id, Text: text}) { + return ctx.Err() + } + } + } + lastPackage, _ := message["is_last_package"].(bool) + lastFrame, _ := message["_last"].(bool) + if (lastPackage || lastFrame) && finishing { + if lastText != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: lastText}) { + return ctx.Err() + } + } + return nil + } + } + } +} diff --git a/crates/agent-gateway/internal/stt/volcengine_v2.go b/crates/agent-gateway/internal/stt/volcengine_v2.go new file mode 100644 index 000000000..c18ac651e --- /dev/null +++ b/crates/agent-gateway/internal/stt/volcengine_v2.go @@ -0,0 +1,314 @@ +package stt + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/google/uuid" + "github.com/gorilla/websocket" + "github.com/klauspost/compress/gzip" +) + +type VolcengineV2Adapter struct{} + +const volcengineV2Endpoint = "wss://openspeech.bytedance.com/api/v2/asr" + +func volcV2Authorization(token string) string { return "Bearer; " + token } + +func (a *VolcengineV2Adapter) Test(ctx context.Context, cfg map[string]any) (string, error) { + return readyProtocolTest(ctx, a, cfg) +} +func gzipJSON(v any) ([]byte, error) { + raw, e := json.Marshal(v) + if e != nil { + return nil, e + } + var out bytes.Buffer + writer := gzip.NewWriter(&out) + if _, e = writer.Write(raw); e != nil { + return nil, e + } + if e = writer.Close(); e != nil { + return nil, e + } + return out.Bytes(), nil +} + +func gzipBytes(payload []byte) ([]byte, error) { + var out bytes.Buffer + writer := gzip.NewWriter(&out) + if _, err := writer.Write(payload); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +func volcV2Frame(messageType, flags, serialization, compression byte, payload []byte) []byte { + frame := []byte{0x11, messageType<<4 | flags, serialization<<4 | compression, 0} + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(payload))) + return append(append(frame, size...), payload...) +} + +func volcV2ResponseOK(message map[string]any) bool { + code := int(number(message["code"])) + text, _ := message["message"].(string) + return code == 0 || code == 1000 || text == "Success" +} + +func volcV2ResponseNoSpeech(message map[string]any) bool { + return int(number(message["code"])) == 1013 +} + +func decodeVolcV2Frame(data []byte) (map[string]any, error) { + if len(data) < 8 { + return nil, errors.New("volcengine v2 short frame") + } + headerLen := int(data[0]&0x0f) * 4 + if headerLen < 4 || len(data) < headerLen+4 { + return nil, errors.New("volcengine v2 invalid header") + } + messageType := data[1] >> 4 + flags := data[1] & 0x0f + offset := headerLen + var errorCode uint32 + var sequence int32 + if messageType == 0x0f { + if len(data) < offset+8 { + return nil, errors.New("volcengine v2 invalid error frame") + } + errorCode = binary.BigEndian.Uint32(data[offset : offset+4]) + offset += 4 + } else if messageType == 0x0b || flags&0x01 != 0 { + if len(data) < offset+8 { + return nil, errors.New("volcengine v2 invalid sequence frame") + } + sequence = int32(binary.BigEndian.Uint32(data[offset : offset+4])) + offset += 4 + } + payloadLen := int(binary.BigEndian.Uint32(data[offset : offset+4])) + if payloadLen < 0 || len(data) < offset+4+payloadLen { + return nil, errors.New("volcengine v2 invalid payload") + } + payload := append([]byte(nil), data[offset+4:offset+4+payloadLen]...) + if data[2]&0x0f == 1 { + reader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + return nil, err + } + payload, err = io.ReadAll(reader) + _ = reader.Close() + if err != nil { + return nil, err + } + } + if messageType == 0x0f { + return map[string]any{"code": float64(errorCode), "message": string(payload)}, nil + } + var message map[string]any + if err := json.Unmarshal(payload, &message); err != nil { + return nil, err + } + message["_sequence"] = float64(sequence) + message["_last"] = flags == 2 || flags == 3 || sequence < 0 || number(message["sequence"]) < 0 + return message, nil +} + +func volcV2ResultText(message map[string]any) string { + result := message["result"] + if object, ok := result.(map[string]any); ok { + return valueString(object, "text") + } + if items, ok := result.([]any); ok && len(items) > 0 { + if object, ok := items[0].(map[string]any); ok { + return valueString(object, "text") + } + } + for _, key := range []string{"payload", "data"} { + if wrapper, ok := message[key].(map[string]any); ok { + if nested, ok := wrapper["result"].(map[string]any); ok { + if text := valueString(nested, "text"); text != "" { + return text + } + } + } + } + return "" +} + +func volcV2ResponseComplete(message map[string]any, finishing bool) bool { + if !finishing { + return false + } + lastPackage, _ := message["is_last_package"].(bool) + lastFrame, _ := message["_last"].(bool) + return lastPackage || lastFrame +} + +func volcV2StartRequest(cfg map[string]any, id, requestID string) map[string]any { + return map[string]any{ + "app": map[string]any{ + "appid": value(cfg, "appId"), "token": value(cfg, "accessToken"), "cluster": value(cfg, "cluster"), + }, + "user": map[string]any{"uid": id}, + "audio": map[string]any{"format": "raw", "rate": 16000, "bits": 16, "channel": 1, "codec": "raw"}, + "request": map[string]any{ + "reqid": requestID, "nbest": 1, + "workflow": "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate", + "show_utterances": true, "result_type": "full", "sequence": 1, + }, + } +} +func (a *VolcengineV2Adapter) Run(ctx context.Context, id string, cfg map[string]any, commands <-chan Command, events chan<- Event) error { + endpoint, err := websocketEndpoint(cfg, volcengineV2Endpoint) + if err != nil { + return stageError("VolcengineV2", "validate", err) + } + header := http.Header{} + header.Set("Authorization", volcV2Authorization(value(cfg, "accessToken"))) + conn, response, err := websocket.DefaultDialer.DialContext(ctx, endpoint, header) + if err != nil { + return stageError("VolcengineV2", "connect", websocketConnectError(response, err)) + } + defer func() { _ = conn.Close() }() + requestID := uuid.NewString() + first, err := gzipJSON(volcV2StartRequest(cfg, id, requestID)) + if err != nil { + return stageError("VolcengineV2", "start", err) + } + if err = writeProviderMessage(conn, websocket.BinaryMessage, volcV2Frame(1, 0, 1, 1, first)); err != nil { + return stageError("VolcengineV2", "start", err) + } + return runVolcV2Loop(ctx, conn, id, commands, events) +} +func runVolcV2Loop(ctx context.Context, conn *websocket.Conn, id string, commands <-chan Command, events chan<- Event) error { + incoming := make(chan map[string]any, 8) + readErr := make(chan error, 1) + go func() { + for { + messageType, data, e := conn.ReadMessage() + if e != nil { + readErr <- stageError("VolcengineV2", "receive", e) + return + } + if messageType != websocket.BinaryMessage { + readErr <- stageError("VolcengineV2", "parse", errors.New("expected binary frame")) + return + } + msg, e := decodeVolcV2Frame(data) + if e != nil { + readErr <- stageError("VolcengineV2", "parse", e) + return + } + if !emitIncoming(ctx, incoming, msg) { + return + } + } + }() + finishing := false + finishSent := false + ready := false + pending := make([][]byte, 0, 32) + lastText := "" + for { + select { + case <-ctx.Done(): + return nil + case e := <-readErr: + if finishing && finishSent && isWebSocketCloseError(e, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + if lastText != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: lastText}) { + return ctx.Err() + } + } + return nil + } + if finishing && finishSent { + return stageError("VolcengineV2", "close", e) + } + return e + case cmd := <-commands: + if cmd.Cancel { + return nil + } + if cmd.Audio != nil { + if !ready { + pending = append(pending, append([]byte(nil), cmd.Audio.PCM...)) + } else if compressed, e := gzipBytes(cmd.Audio.PCM); e != nil { + return stageError("VolcengineV2", "send_audio", e) + } else if e := writeProviderMessage(conn, websocket.BinaryMessage, volcV2Frame(2, 0, 0, 1, compressed)); e != nil { + return stageError("VolcengineV2", "send_audio", e) + } + } + if cmd.Finish { + finishing = true + if ready && !finishSent { + compressed, compressErr := gzipBytes(nil) + if compressErr != nil { + return stageError("VolcengineV2", "finish", compressErr) + } + if e := writeProviderMessage(conn, websocket.BinaryMessage, volcV2Frame(2, 2, 0, 1, compressed)); e != nil { + return stageError("VolcengineV2", "finish", e) + } + finishSent = true + } + } + case msg := <-incoming: + if volcV2ResponseNoSpeech(msg) { + return nil + } + if !volcV2ResponseOK(msg) { + return stageError("VolcengineV2", "provider_response", providerFailure("Volcengine v2", fmt.Sprint(msg["code"]), valueString(msg, "message"))) + } + if !ready { + ready = true + if !emitEvent(ctx, events, Event{Type: "ready", SessionID: id}) { + return ctx.Err() + } + for _, pcm := range pending { + compressed, compressErr := gzipBytes(pcm) + if compressErr != nil { + return stageError("VolcengineV2", "send_audio", compressErr) + } + if e := writeProviderMessage(conn, websocket.BinaryMessage, volcV2Frame(2, 0, 0, 1, compressed)); e != nil { + return stageError("VolcengineV2", "send_audio", e) + } + } + pending = nil + if finishing && !finishSent { + compressed, compressErr := gzipBytes(nil) + if compressErr != nil { + return stageError("VolcengineV2", "finish", compressErr) + } + if e := writeProviderMessage(conn, websocket.BinaryMessage, volcV2Frame(2, 2, 0, 1, compressed)); e != nil { + return stageError("VolcengineV2", "finish", e) + } + finishSent = true + } + } + if text := volcV2ResultText(msg); text != "" { + lastText = text + if !emitEvent(ctx, events, Event{Type: "partial", SessionID: id, Text: text}) { + return ctx.Err() + } + } + if volcV2ResponseComplete(msg, finishing) { + if lastText != "" { + if !emitEvent(ctx, events, Event{Type: "final", SessionID: id, Text: lastText}) { + return ctx.Err() + } + } + return nil + } + } + } +} diff --git a/crates/agent-gateway/proto/v2/gateway_ws.proto b/crates/agent-gateway/proto/v2/gateway_ws.proto index 06c06fa34..1d798c6a1 100644 --- a/crates/agent-gateway/proto/v2/gateway_ws.proto +++ b/crates/agent-gateway/proto/v2/gateway_ws.proto @@ -69,6 +69,71 @@ message AckResult { bool ok = 1; } +// --------------------------------------------------------------------------- +// 独立语音识别链路（/ws/v2/stt） +// --------------------------------------------------------------------------- + +message SttClientHello { + uint32 protocol_version = 1; + string token = 2; +} + +message SttStart { + string session_id = 1; + string provider = 2; +} + +message SttAudio { + string session_id = 1; + uint32 sequence = 2; + bytes pcm = 3; +} + +message SttSessionControl { + string session_id = 1; +} + +message SttClientFrame { + oneof payload { + SttClientHello hello = 1; + SttStart start = 2; + SttAudio audio = 3; + SttSessionControl stop = 4; + SttSessionControl cancel = 5; + } +} + +message SttServerHello { + bool ok = 1; + string message = 2; +} + +message SttTextEvent { + string session_id = 1; + string text = 2; +} + +message SttError { + string session_id = 1; + string code = 2; + string message = 3; +} + +message SttClosed { + string session_id = 1; +} + +message SttServerFrame { + oneof payload { + SttServerHello hello = 1; + SttTextEvent ready = 2; + SttTextEvent partial = 3; + SttTextEvent final = 4; + SttError error = 5; + SttClosed closed = 6; + } +} + // --------------------------------------------------------------------------- // 浏览器链路（/ws/v2） // --------------------------------------------------------------------------- diff --git a/crates/agent-gateway/test/webui/web-settings.test.mjs b/crates/agent-gateway/test/webui/web-settings.test.mjs index 3340d960f..6c17f4411 100644 --- a/crates/agent-gateway/test/webui/web-settings.test.mjs +++ b/crates/agent-gateway/test/webui/web-settings.test.mjs @@ -28,6 +28,74 @@ async function withNavigator(value, task) { } } +test("gateway settings sync publishes redacted STT state and enables WebUI STT", () => { + const desktop = settings.normalizeSettings({ + stt: { + enabled: true, + provider: "aliyun_dashscope", + providers: { + aliyun_dashscope: { + id: "aliyun_dashscope", + configured: true, + websocketUrl: "wss://dashscope.aliyuncs.com/api-ws/v1/inference/", + model: "paraformer-realtime-v2", + apiKey: "desktop-secret", + }, + }, + }, + }); + + const payload = settingsSync.buildGatewaySettingsSyncPayload(desktop); + assert.equal(payload.stt.provider, "aliyun_dashscope"); + assert.equal(payload.stt.enabled, true); + assert.equal(payload.stt.providers.aliyun_dashscope.configured, true); + assert.equal(payload.stt.providers.aliyun_dashscope.apiKey, ""); + + const web = settingsSync.applyGatewaySettingsSyncPayload(settings.normalizeSettings({}), payload); + assert.equal(web.stt.provider, "aliyun_dashscope"); + assert.equal(web.stt.enabled, true); + assert.equal(web.stt.providers.aliyun_dashscope.configured, true); + assert.equal(web.stt.providers.aliyun_dashscope.apiKey, ""); +}); + +test("WebUI STT secret sidecar reaches desktop state but never enters public payload", () => { + const current = settings.normalizeSettings({ + stt: { + provider: "aliyun_dashscope", + providers: { + aliyun_dashscope: { + id: "aliyun_dashscope", + configured: true, + apiKey: "existing-desktop-secret", + }, + }, + }, + }); + const incoming = settings.normalizeSettings({ + stt: { + provider: "aliyun_dashscope", + providers: { + aliyun_dashscope: { + id: "aliyun_dashscope", + configured: true, + apiKey: "", + clearSecrets: true, + }, + }, + }, + }).stt; + + const desktop = settingsSync.applyGatewaySettingsSyncPayload(current, { + sttSecretUpdate: incoming, + }); + assert.equal(desktop.stt.providers.aliyun_dashscope.clearSecrets, true); + assert.equal(desktop.stt.providers.aliyun_dashscope.apiKey, ""); + + const publicPayload = settingsSync.buildGatewaySettingsSyncPayload(desktop); + assert.equal(publicPayload.stt.providers.aliyun_dashscope.clearSecrets, undefined); + assert.equal(publicPayload.stt.providers.aliyun_dashscope.apiKey, ""); +}); + test("web settings normalize and preserve workspace project groups", () => { const normalized = settings.normalizeSettings({ system: { diff --git a/crates/agent-gateway/web/src/app/GatewayAppView.tsx b/crates/agent-gateway/web/src/app/GatewayAppView.tsx index d0f04139a..3ee7485a6 100644 --- a/crates/agent-gateway/web/src/app/GatewayAppView.tsx +++ b/crates/agent-gateway/web/src/app/GatewayAppView.tsx @@ -48,11 +48,14 @@ import { } from "react"; import { createGatewayTrajectoryHost } from "@/agent-ui-adapters/trajectory"; import { GatewayTranscript } from "@/components/GatewayTranscript"; +import type { SttProviderId } from "@/lib/settings"; import { getNextTheme, updateExecutionModeFromChatSelection, updateWorkspaceResourceSettings, } from "@/lib/settings"; +import { createWebSttSettingsService } from "@/lib/stt/webSttSettingsService"; +import { webSttTransport } from "@/lib/stt/webSttTransport"; import { liveTrajectoryAuthoritativeRevision, liveTrajectoryEvents, @@ -326,6 +329,19 @@ export function GatewayAppView({ viewModel }: { viewModel: GatewayAppViewModel } workspaceSshTerminalOpen, workspaceSshTerminalOpenRequest, } = viewModel; + const [sttProviderOverride, setSttProviderOverride] = useState(null); + // biome-ignore lint/correctness/useExhaustiveDependencies: Saved provider changes invalidate the temporary card selection. + useEffect(() => { + setSttProviderOverride(null); + }, [settings.stt.provider]); + const sttSettingsService = useMemo( + () => + createWebSttSettingsService(async (sttSecretUpdate) => { + if (!api) throw new Error("桌面 Agent 未连接，无法同步 STT 配置"); + await api.updateSettings({ sttSecretUpdate }); + }), + [api], + ); const [activeConversationView, setActiveConversationView] = useState("conversation"); @@ -357,6 +373,8 @@ export function GatewayAppView({ viewModel }: { viewModel: GatewayAppViewModel } setSettings((prev) => updateExecutionModeFromChatSelection(prev, mode)), [setSettings], ); + // 语音输入失败（麦克风不可用等）以 toast 提示，不占用输入框区域。 + const handleSttError = useCallback((message: string) => addNotify("error", message), [addNotify]); const resolveCheckpointAuthorizedRoots = useCallback(async () => { const roots: string[] = []; const push = (value?: string | null) => { @@ -744,6 +762,20 @@ export function GatewayAppView({ viewModel }: { viewModel: GatewayAppViewModel } isSending={composerIsSending} isUploadingFiles={isUploadingFiles} isInputDisabled={composerInputDisabled} + // 麦克风在开启语音输入后显示；点击设置卡片会立即切换当前供应商。 + sttSessionKey={displayedConversationId} + sttProvider={ + settings.stt.enabled + ? (sttProviderOverride ?? settings.stt.provider ?? "tencent_cloud") + : null + } + sttProviderConfigured={ + settings.stt.providers[ + sttProviderOverride ?? settings.stt.provider ?? "tencent_cloud" + ]?.configured + } + sttTransport={webSttTransport} + onSttError={handleSttError} inputPlaceholder={composerPlaceholder} workdir={displayedConversationWorkdir} enabledSkills={enabledComposerSkills} @@ -1034,6 +1066,8 @@ export function GatewayAppView({ viewModel }: { viewModel: GatewayAppViewModel } initialSection={settingsSection} initialProviderId={settingsProviderId} hiddenSections={["remote"]} + sttSettingsService={sttSettingsService} + onSttProviderChange={setSttProviderOverride} onAgentDirectoryChanged={async () => { if (!api) return; await api.listAgents(); diff --git a/crates/agent-gateway/web/src/app/hooks/useGatewaySettingsSync.ts b/crates/agent-gateway/web/src/app/hooks/useGatewaySettingsSync.ts index e85b2a3a8..9808310e4 100644 --- a/crates/agent-gateway/web/src/app/hooks/useGatewaySettingsSync.ts +++ b/crates/agent-gateway/web/src/app/hooks/useGatewaySettingsSync.ts @@ -9,7 +9,7 @@ import { setPreferredMonacoNlsLocale } from "@liveagent/ui/lib/monacoNls"; import { applyGatewaySettingsSyncPayload, buildGatewaySettingsSyncUpdatePayload, - type GatewaySettingsSyncPayload, + type GatewaySettingsSyncUpdatePayload, redactSettingsForWebStorage, } from "@liveagent/ui/lib/settings/sync"; import { applyFontFamilies } from "@liveagent/ui/lib/shared/fontFamily"; @@ -24,6 +24,7 @@ import { subscribeToSystemThemePreference, } from "@/lib/settings"; import { loadToken } from "@/lib/storage"; +import { webSttSettingsService } from "@/lib/stt/webSttSettingsService"; import { loadWebSettings, persistWebSettings, type WebSettingsSaveState } from "@/lib/webSettings"; import { asErrorMessage } from "../chatEventUtils"; @@ -156,33 +157,40 @@ export function useGatewaySettingsSync(params: { [api], ); - const applyGatewaySettings = useCallback( - (payload: GatewaySettingsSyncPayload) => { - // Automation snapshots ride along on the settings-sync channel but are - // desktop-owned state with their own revision — feed them straight into - // the automation store instead of the settings state. - const automation = payload as { - automationCron?: CronSnapshot; - automationHooks?: HooksSnapshot; - }; - if (automation.automationCron) { - feedCronSnapshot(automation.automationCron); - } - if (automation.automationHooks) { - feedHooksSnapshot(automation.automationHooks); - } - const prev = settingsRef.current; - const rawNext = resolveAppWorkspaceProjects(applyGatewaySettingsSyncPayload(prev, payload)); - const next = redactSettingsForWebStorage(rawNext); - if (!hasSettingsSyncChanged(prev, next)) { - return; - } - settingsRef.current = next; - setSettingsState(next); - queueSettingsSave(prev, next, t("app.desktopSettingsSyncFailed", next.locale), false); - }, - [queueSettingsSave], - ); + const applyGatewaySettings = useCallback((payload: GatewaySettingsSyncUpdatePayload) => { + // Automation snapshots ride along on the settings-sync channel but are + // desktop-owned state with their own revision — feed them straight into + // the automation store instead of the settings state. + const automation = payload as { + automationCron?: CronSnapshot; + automationHooks?: HooksSnapshot; + }; + if (automation.automationCron) { + feedCronSnapshot(automation.automationCron); + } + if (automation.automationHooks) { + feedHooksSnapshot(automation.automationHooks); + } + const prev = settingsRef.current; + const rawNext = resolveAppWorkspaceProjects(applyGatewaySettingsSyncPayload(prev, payload)); + const next = redactSettingsForWebStorage(rawNext); + if (!hasSettingsSyncChanged(prev, next)) { + return; + } + settingsRef.current = next; + setSettingsState(next); + // Gateway/desktop pushes are hydration, not user edits. Persist the + // already-redacted browser cache directly so opening Settings never + // flashes “saving” or sends the hydrated value back to the desktop. + try { + persistWebSettings(next); + } catch (error) { + setSettingsSaveState({ + status: "error", + message: asErrorMessage(error, "缓存桌面端设置失败。"), + }); + } + }, []); const setSettings = useCallback( (updater: (prev: AppSettings) => AppSettings) => { @@ -211,19 +219,41 @@ export function useGatewaySettingsSync(params: { // Best-effort: the desktop may be offline; the settings-sync push // populates the store once it connects. void initAutomation().catch(() => undefined); - const unsubscribe = api.subscribeSettings((payload) => { + const liveSyncEpochRef = { current: 0 }; + const applySyncedSettings = (payload: GatewaySettingsSyncUpdatePayload) => { if (cancelled) { return; } + liveSyncEpochRef.current += 1; applyGatewaySettings(payload); setSettingsSyncError(null); - }); + }; + const unsubscribe = api.subscribeSettings(applySyncedSettings); void api .getSettings() - .then((payload) => { + .then(async (payload) => { + if (!cancelled) { + // A live WS push that arrived while GET was in flight is newer. + if (liveSyncEpochRef.current === 0) { + applySyncedSettings(payload); + } + // The Gateway STT store is the WebUI runtime authority for whether + // redacted credentials are configured. A cached desktop snapshot may + // contain an older configured=false value even though Gateway still + // has the credentials, so hydrate this once before the app is ready. + // Skip the HTTP result if a newer settings push landed during fetch. + try { + const sttEpoch = liveSyncEpochRef.current; + const stt = await webSttSettingsService.get(); + if (!cancelled && liveSyncEpochRef.current === sttEpoch) { + applyGatewaySettings({ stt }); + } + } catch { + // General settings sync remains usable when STT is unavailable. + } + } if (!cancelled) { - applyGatewaySettings(payload); setSettingsSyncReady(true); setSettingsSyncError(null); } diff --git a/crates/agent-gateway/web/src/lib/proto/gen/proto/v2/gateway_ws_pb.ts b/crates/agent-gateway/web/src/lib/proto/gen/proto/v2/gateway_ws_pb.ts index 6b6155610..80b38b8c3 100644 --- a/crates/agent-gateway/web/src/lib/proto/gen/proto/v2/gateway_ws_pb.ts +++ b/crates/agent-gateway/web/src/lib/proto/gen/proto/v2/gateway_ws_pb.ts @@ -17,7 +17,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file proto/v2/gateway_ws.proto. */ export const file_proto_v2_gateway_ws: GenFile = /*@__PURE__*/ - fileDesc("Chlwcm90by92Mi9nYXRld2F5X3dzLnByb3RvEhRsaXZlYWdlbnQuZ2F0ZXdheS52MiLSAQoLQ2xpZW50SGVsbG8SGAoQcHJvdG9jb2xfdmVyc2lvbhgBIAEoDRIuCgRyb2xlGAIgASgOMiAubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2xpZW50Um9sZRINCgV0b2tlbhgDIAEoCRIQCghhZ2VudF9pZBgEIAEoCRIVCg1hZ2VudF92ZXJzaW9uGAUgASgJEhMKC2NsaWVudF9uYW1lGAYgASgJEhYKDmNsaWVudF92ZXJzaW9uGAcgASgJEhQKDGNhcGFiaWxpdGllcxgIIAMoCSKmAQoLU2VydmVySGVsbG8SCgoCb2sYASABKAgSDwoHbWVzc2FnZRgCIAEoCRISCgpzZXNzaW9uX2lkGAMgASgJEhMKC3NlcnZlcl90aW1lGAQgASgDEiAKGGhlYXJ0YmVhdF9wZXJpb2Rfc2Vjb25kcxgFIAEoDRIZChFtYXhfbWVzc2FnZV9ieXRlcxgGIAEoBBIUCgxjYXBhYmlsaXRpZXMYByADKAkiHgoJUGluZ0ZyYW1lEhEKCXRpbWVzdGFtcBgBIAEoAyIeCglQb25nRnJhbWUSEQoJdGltZXN0YW1wGAEgASgDIhcKCUFja1Jlc3VsdBIKCgJvaxgBIAEoCCLiBgoOV2ViQ2xpZW50RnJhbWUSEgoKcmVxdWVzdF9pZBgBIAEoCRIQCghhZ2VudF9pZBgNIAEoCRIyCgVoZWxsbxgCIAEoCzIhLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNsaWVudEhlbGxvSAASPgoNYWdlbnRfcmVxdWVzdBgDIAEoCzIlLmxpdmVhZ2VudC5nYXRld2F5LnYyLkdhdGV3YXlFbnZlbG9wZUgAEjwKCnN0YXR1c19nZXQYBCABKAsyJi5saXZlYWdlbnQuZ2F0ZXdheS52Mi5TdGF0dXNHZXRSZXF1ZXN0SAASQAoMY2hhdF9jb21tYW5kGAUgASgLMigubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdENvbW1hbmRSZXF1ZXN0SAASQAoMY2hhdF9wcmVwYXJlGAYgASgLMigubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFByZXBhcmVSZXF1ZXN0SAASRAoOY2hhdF9zdWJzY3JpYmUYByABKAsyKi5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0U3Vic2NyaWJlUmVxdWVzdEgAEkgKEGNoYXRfdW5zdWJzY3JpYmUYCCABKAsyLC5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0VW5zdWJzY3JpYmVSZXF1ZXN0SAASRgoPY2hhdF9hY3Rpdml0aWVzGAkgASgLMisubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdEFjdGl2aXRpZXNSZXF1ZXN0SAASTgoTd29ya3NwYWNlX3N1YnNjcmliZRgKIAEoCzIvLmxpdmVhZ2VudC5nYXRld2F5LnYyLldvcmtzcGFjZVN1YnNjcmliZVJlcXVlc3RIABJSChV3b3Jrc3BhY2VfdW5zdWJzY3JpYmUYCyABKAsyMS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5Xb3Jrc3BhY2VVbnN1YnNjcmliZVJlcXVlc3RIABIvCgRwb25nGAwgASgLMh8ubGl2ZWFnZW50LmdhdGV3YXkudjIuUG9uZ0ZyYW1lSAASPAoKYWdlbnRfbGlzdBgOIAEoCzImLmxpdmVhZ2VudC5nYXRld2F5LnYyLkFnZW50TGlzdFJlcXVlc3RIAEIJCgdwYXlsb2FkIv8LCg5XZWJTZXJ2ZXJGcmFtZRISCgpyZXF1ZXN0X2lkGAEgASgJEhAKCGFnZW50X2lkGBAgASgJEjIKBWhlbGxvGAIgASgLMiEubGl2ZWFnZW50LmdhdGV3YXkudjIuU2VydmVySGVsbG9IABI9Cg5hZ2VudF9yZXNwb25zZRgDIAEoCzIjLmxpdmVhZ2VudC5nYXRld2F5LnYyLkFnZW50RW52ZWxvcGVIABI6Cgtsb2NhbF9lcnJvchgEIAEoCzIjLmxpdmVhZ2VudC5nYXRld2F5LnYyLkVycm9yUmVzcG9uc2VIABIvCgRwaW5nGAUgASgLMh8ubGl2ZWFnZW50LmdhdGV3YXkudjIuUGluZ0ZyYW1lSAASMwoGc3RhdHVzGAYgASgLMiEubGl2ZWFnZW50LmdhdGV3YXkudjIuU3RhdHVzRXZlbnRIABJECg9jaGF0X3N1YnNjcmliZWQYByABKAsyKS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0U3Vic2NyaWJlUmVzdWx0SAASQgoNY2hhdF9hY2NlcHRlZBgIIAEoCzIpLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRDb21tYW5kQWNjZXB0ZWRIABJFCg9jaGF0X2FjdGl2aXRpZXMYCSABKAsyKi5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0QWN0aXZpdGllc1Jlc3VsdEgAEjsKCmNoYXRfZXZlbnQYCiABKAsyJS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0U3RyZWFtRXZlbnRIABJGChNjaGF0X2NvbW1hbmRfdXBkYXRlGAsgASgLMicubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdENvbW1hbmRVcGRhdGVIABJOChdjaGF0X3N1YnNjcmlwdGlvbl9yZXNldBgMIAEoCzIrLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRTdWJzY3JpcHRpb25SZXNldEgAEkAKDWNoYXRfYWN0aXZpdHkYDSABKAsyJy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0QWN0aXZpdHlFdmVudEgAEi4KA2FjaxgOIAEoCzIfLmxpdmVhZ2VudC5nYXRld2F5LnYyLkFja1Jlc3VsdEgAEkAKDmNoYXRfY2FuY2VsbGVkGA8gASgLMiYubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdENhbmNlbFJlc3VsdEgAEjsKCmFnZW50X2xpc3QYESABKAsyJS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5BZ2VudExpc3RSZXN1bHRIABI/Cg1oaXN0b3J5X2V2ZW50GBQgASgLMiYubGl2ZWFnZW50LmdhdGV3YXkudjIuSGlzdG9yeVN5bmNFdmVudEgAEkEKDnNldHRpbmdzX2V2ZW50GBUgASgLMicubGl2ZWFnZW50LmdhdGV3YXkudjIuU2V0dGluZ3NTeW5jRXZlbnRIABI9Cg50ZXJtaW5hbF9ldmVudBgWIAEoCzIjLmxpdmVhZ2VudC5nYXRld2F5LnYyLlRlcm1pbmFsRXZlbnRIABI1CgpzZnRwX2V2ZW50GBcgASgLMh8ubGl2ZWFnZW50LmdhdGV3YXkudjIuU2Z0cEV2ZW50SAASQAoQY2hhdF9xdWV1ZV9ldmVudBgYIAEoCzIkLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRRdWV1ZUV2ZW50SAASQQoMdHVubmVsX3N0YXRlGBkgASgLMikubGl2ZWFnZW50LmdhdGV3YXkudjIuVHVubmVsU3RhdGVTbmFwc2hvdEgAEkUKDXByb2Nlc3Nfc3RhdGUYGiABKAsyLC5saXZlYWdlbnQuZ2F0ZXdheS52Mi5NYW5hZ2VkUHJvY2Vzc1NuYXBzaG90SAASSgoSd29ya3NwYWNlX2FjdGl2aXR5GBsgASgLMiwubGl2ZWFnZW50LmdhdGV3YXkudjIuV29ya3NwYWNlQWN0aXZpdHlFdmVudEgAQgkKB3BheWxvYWQiEgoQQWdlbnRMaXN0UmVxdWVzdCJECg9BZ2VudExpc3RSZXN1bHQSMQoGYWdlbnRzGAEgAygLMiEubGl2ZWFnZW50LmdhdGV3YXkudjIuU3RhdHVzRXZlbnQiigEKEEFnZW50Q2xpZW50RnJhbWUSMgoFaGVsbG8YASABKAsyIS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DbGllbnRIZWxsb0gAEjcKCGVudmVsb3BlGAIgASgLMiMubGl2ZWFnZW50LmdhdGV3YXkudjIuQWdlbnRFbnZlbG9wZUgAQgkKB3BheWxvYWQijAEKEEFnZW50U2VydmVyRnJhbWUSMgoFaGVsbG8YASABKAsyIS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5TZXJ2ZXJIZWxsb0gAEjkKCGVudmVsb3BlGAIgASgLMiUubGl2ZWFnZW50LmdhdGV3YXkudjIuR2F0ZXdheUVudmVsb3BlSABCCQoHcGF5bG9hZCKQAQoTVGVybWluYWxDbGllbnRGcmFtZRIyCgVoZWxsbxgBIAEoCzIhLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNsaWVudEhlbGxvSAASOgoFZnJhbWUYAiABKAsyKS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5UZXJtaW5hbFN0cmVhbUZyYW1lSABCCQoHcGF5bG9hZCKQAQoTVGVybWluYWxTZXJ2ZXJGcmFtZRIyCgVoZWxsbxgBIAEoCzIhLmxpdmVhZ2VudC5nYXRld2F5LnYyLlNlcnZlckhlbGxvSAASOgoFZnJhbWUYAiABKAsyKS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5UZXJtaW5hbFN0cmVhbUZyYW1lSABCCQoHcGF5bG9hZCISChBTdGF0dXNHZXRSZXF1ZXN0ItcCCgtTdGF0dXNFdmVudBIOCgZvbmxpbmUYASABKAgSEwoLYWdlbnRfcmVhZHkYAiABKAgSGgoSY2hhdF9ydW50aW1lX3JlYWR5GAMgASgIEhAKCGFnZW50X2lkGAQgASgJEhUKDWFnZW50X3ZlcnNpb24YBSABKAkSEgoKc2Vzc2lvbl9pZBgGIAEoCRIXCg9jb25uZWN0ZWRfc2luY2UYByABKAMSFgoObGFzdF9oZWFydGJlYXQYCCABKAMSFQoNcnVudGltZV9zdGF0ZRgJIAEoCRIeChZydW50aW1lX2xhc3RfaGVhcnRiZWF0GAogASgDEhkKEXJ1bnRpbWVfd29ya2VyX2lkGAsgASgJEhcKD3J1bnRpbWVfdmlzaWJsZRgMIAEoCBIgChhydW50aW1lX2FjdGl2ZV9ydW5fY291bnQYDSABKA0SDAoEbmFtZRgOIAEoCSIkChJDaGF0UHJlcGFyZVJlcXVlc3QSDgoGcmVhc29uGAEgASgJIlgKFENoYXRTdWJzY3JpYmVSZXF1ZXN0EhcKD2NvbnZlcnNhdGlvbl9pZBgBIAEoCRIRCglhZnRlcl9zZXEYAiABKAMSFAoMc3RyZWFtX2Vwb2NoGAMgASgJIusBCg9DaGF0UnVuQWN0aXZpdHkSDgoGcnVuX2lkGAEgASgJEg0KBXN0YXRlGAIgASgJEhMKC3N0YXJ0ZWRfc2VxGAMgASgDEhUKDXVwZGF0ZWRfYXRfbXMYBCABKAMSEwoLdG9vbF9zdGF0dXMYBSABKAkSIQoZdG9vbF9zdGF0dXNfaXNfY29tcGFjdGlvbhgGIAEoCBIZChFjbGllbnRfcmVxdWVzdF9pZBgHIAEoCRIXCg9jb252ZXJzYXRpb25faWQYCCABKAkSDwoHd29ya2RpchgJIAEoCRIQCghhZ2VudF9pZBgKIAEoCSKUAQoPQ2hhdFJ1blNuYXBzaG90Eg4KBnJ1bl9pZBgBIAEoCRIQCghyZXZpc2lvbhgCIAEoAxIUCgxlbnRyaWVzX2pzb24YAyABKAkSEwoLdG9vbF9zdGF0dXMYBCABKAkSIQoZdG9vbF9zdGF0dXNfaXNfY29tcGFjdGlvbhgFIAEoCBIRCglhc19vZl9zZXEYBiABKAMi7gEKE0NoYXRTdWJzY3JpYmVSZXN1bHQSFwoPY29udmVyc2F0aW9uX2lkGAEgASgJEhQKDHN0cmVhbV9lcG9jaBgCIAEoCRISCgpsYXRlc3Rfc2VxGAMgASgDEg0KBXJlc2V0GAQgASgIEjcKCGFjdGl2aXR5GAUgASgLMiUubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFJ1bkFjdGl2aXR5EjcKCHNuYXBzaG90GAYgASgLMiUubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFJ1blNuYXBzaG90EhMKC2V2ZW50c19qc29uGAcgAygMIjEKFkNoYXRVbnN1YnNjcmliZVJlcXVlc3QSFwoPY29udmVyc2F0aW9uX2lkGAEgASgJIhcKFUNoYXRBY3Rpdml0aWVzUmVxdWVzdCJcChRDaGF0QWN0aXZpdGllc1Jlc3VsdBJEChVydW5uaW5nX2NvbnZlcnNhdGlvbnMYASADKAsyJS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0UnVuQWN0aXZpdHkiTQoPQ2hhdFN0cmVhbUV2ZW50EhcKD2NvbnZlcnNhdGlvbl9pZBgBIAEoCRILCgNzZXEYAiABKAMSFAoMcGF5bG9hZF9qc29uGAMgASgMImUKE0NoYXRDb21tYW5kQWNjZXB0ZWQSDgoGcnVuX2lkGAEgASgJEhcKD2NvbnZlcnNhdGlvbl9pZBgCIAEoCRIUCgxhY2NlcHRlZF9zZXEYAyABKAMSDwoHZGVkdXBlZBgEIAEoCCKLAQoRQ2hhdENvbW1hbmRVcGRhdGUSDgoGcnVuX2lkGAEgASgJEhkKEWNsaWVudF9yZXF1ZXN0X2lkGAIgASgJEhcKD2NvbnZlcnNhdGlvbl9pZBgDIAEoCRINCgVwaGFzZRgEIAEoCRISCgplcnJvcl9jb2RlGAUgASgJEg8KB21lc3NhZ2UYBiABKAkiMAoVQ2hhdFN1YnNjcmlwdGlvblJlc2V0EhcKD2NvbnZlcnNhdGlvbl9pZBgBIAEoCSJHChBDaGF0Q2FuY2VsUmVzdWx0EgoKAm9rGAEgASgIEg4KBnJ1bl9pZBgCIAEoCRIXCg9jb252ZXJzYXRpb25faWQYAyABKAkinwEKEUNoYXRBY3Rpdml0eUV2ZW50EhcKD2NvbnZlcnNhdGlvbl9pZBgBIAEoCRIOCgZydW5faWQYAiABKAkSGQoRY2xpZW50X3JlcXVlc3RfaWQYAyABKAkSDwoHcnVubmluZxgEIAEoCBINCgVzdGF0ZRgFIAEoCRIPCgd3b3JrZGlyGAYgASgJEhUKDXVwZGF0ZWRfYXRfbXMYByABKAMiLAoZV29ya3NwYWNlU3Vic2NyaWJlUmVxdWVzdBIPCgd3b3JrZGlyGAEgASgJIi4KG1dvcmtzcGFjZVVuc3Vic2NyaWJlUmVxdWVzdBIPCgd3b3JrZGlyGAEgASgJKlkKCkNsaWVudFJvbGUSGwoXQ0xJRU5UX1JPTEVfVU5TUEVDSUZJRUQQABIXChNDTElFTlRfUk9MRV9CUk9XU0VSEAESFQoRQ0xJRU5UX1JPTEVfQUdFTlQQAkJAWj5naXRodWIuY29tL2xpdmVhZ2VudC9hZ2VudC1nYXRld2F5L2ludGVybmFsL3Byb3RvL3YyO2dhdGV3YXl2MmIGcHJvdG8z", [file_proto_v2_gateway]); + fileDesc("Chlwcm90by92Mi9nYXRld2F5X3dzLnByb3RvEhRsaXZlYWdlbnQuZ2F0ZXdheS52MiLSAQoLQ2xpZW50SGVsbG8SGAoQcHJvdG9jb2xfdmVyc2lvbhgBIAEoDRIuCgRyb2xlGAIgASgOMiAubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2xpZW50Um9sZRINCgV0b2tlbhgDIAEoCRIQCghhZ2VudF9pZBgEIAEoCRIVCg1hZ2VudF92ZXJzaW9uGAUgASgJEhMKC2NsaWVudF9uYW1lGAYgASgJEhYKDmNsaWVudF92ZXJzaW9uGAcgASgJEhQKDGNhcGFiaWxpdGllcxgIIAMoCSKmAQoLU2VydmVySGVsbG8SCgoCb2sYASABKAgSDwoHbWVzc2FnZRgCIAEoCRISCgpzZXNzaW9uX2lkGAMgASgJEhMKC3NlcnZlcl90aW1lGAQgASgDEiAKGGhlYXJ0YmVhdF9wZXJpb2Rfc2Vjb25kcxgFIAEoDRIZChFtYXhfbWVzc2FnZV9ieXRlcxgGIAEoBBIUCgxjYXBhYmlsaXRpZXMYByADKAkiHgoJUGluZ0ZyYW1lEhEKCXRpbWVzdGFtcBgBIAEoAyIeCglQb25nRnJhbWUSEQoJdGltZXN0YW1wGAEgASgDIhcKCUFja1Jlc3VsdBIKCgJvaxgBIAEoCCI5Cg5TdHRDbGllbnRIZWxsbxIYChBwcm90b2NvbF92ZXJzaW9uGAEgASgNEg0KBXRva2VuGAIgASgJIjAKCFN0dFN0YXJ0EhIKCnNlc3Npb25faWQYASABKAkSEAoIcHJvdmlkZXIYAiABKAkiPQoIU3R0QXVkaW8SEgoKc2Vzc2lvbl9pZBgBIAEoCRIQCghzZXF1ZW5jZRgCIAEoDRILCgNwY20YAyABKAwiJwoRU3R0U2Vzc2lvbkNvbnRyb2wSEgoKc2Vzc2lvbl9pZBgBIAEoCSKoAgoOU3R0Q2xpZW50RnJhbWUSNQoFaGVsbG8YASABKAsyJC5saXZlYWdlbnQuZ2F0ZXdheS52Mi5TdHRDbGllbnRIZWxsb0gAEi8KBXN0YXJ0GAIgASgLMh4ubGl2ZWFnZW50LmdhdGV3YXkudjIuU3R0U3RhcnRIABIvCgVhdWRpbxgDIAEoCzIeLmxpdmVhZ2VudC5nYXRld2F5LnYyLlN0dEF1ZGlvSAASNwoEc3RvcBgEIAEoCzInLmxpdmVhZ2VudC5nYXRld2F5LnYyLlN0dFNlc3Npb25Db250cm9sSAASOQoGY2FuY2VsGAUgASgLMicubGl2ZWFnZW50LmdhdGV3YXkudjIuU3R0U2Vzc2lvbkNvbnRyb2xIAEIJCgdwYXlsb2FkIi0KDlN0dFNlcnZlckhlbGxvEgoKAm9rGAEgASgIEg8KB21lc3NhZ2UYAiABKAkiMAoMU3R0VGV4dEV2ZW50EhIKCnNlc3Npb25faWQYASABKAkSDAoEdGV4dBgCIAEoCSI9CghTdHRFcnJvchISCgpzZXNzaW9uX2lkGAEgASgJEgwKBGNvZGUYAiABKAkSDwoHbWVzc2FnZRgDIAEoCSIfCglTdHRDbG9zZWQSEgoKc2Vzc2lvbl9pZBgBIAEoCSLXAgoOU3R0U2VydmVyRnJhbWUSNQoFaGVsbG8YASABKAsyJC5saXZlYWdlbnQuZ2F0ZXdheS52Mi5TdHRTZXJ2ZXJIZWxsb0gAEjMKBXJlYWR5GAIgASgLMiIubGl2ZWFnZW50LmdhdGV3YXkudjIuU3R0VGV4dEV2ZW50SAASNQoHcGFydGlhbBgDIAEoCzIiLmxpdmVhZ2VudC5nYXRld2F5LnYyLlN0dFRleHRFdmVudEgAEjMKBWZpbmFsGAQgASgLMiIubGl2ZWFnZW50LmdhdGV3YXkudjIuU3R0VGV4dEV2ZW50SAASLwoFZXJyb3IYBSABKAsyHi5saXZlYWdlbnQuZ2F0ZXdheS52Mi5TdHRFcnJvckgAEjEKBmNsb3NlZBgGIAEoCzIfLmxpdmVhZ2VudC5nYXRld2F5LnYyLlN0dENsb3NlZEgAQgkKB3BheWxvYWQi4gYKDldlYkNsaWVudEZyYW1lEhIKCnJlcXVlc3RfaWQYASABKAkSEAoIYWdlbnRfaWQYDSABKAkSMgoFaGVsbG8YAiABKAsyIS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DbGllbnRIZWxsb0gAEj4KDWFnZW50X3JlcXVlc3QYAyABKAsyJS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5HYXRld2F5RW52ZWxvcGVIABI8CgpzdGF0dXNfZ2V0GAQgASgLMiYubGl2ZWFnZW50LmdhdGV3YXkudjIuU3RhdHVzR2V0UmVxdWVzdEgAEkAKDGNoYXRfY29tbWFuZBgFIAEoCzIoLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRDb21tYW5kUmVxdWVzdEgAEkAKDGNoYXRfcHJlcGFyZRgGIAEoCzIoLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRQcmVwYXJlUmVxdWVzdEgAEkQKDmNoYXRfc3Vic2NyaWJlGAcgASgLMioubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFN1YnNjcmliZVJlcXVlc3RIABJIChBjaGF0X3Vuc3Vic2NyaWJlGAggASgLMiwubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFVuc3Vic2NyaWJlUmVxdWVzdEgAEkYKD2NoYXRfYWN0aXZpdGllcxgJIAEoCzIrLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRBY3Rpdml0aWVzUmVxdWVzdEgAEk4KE3dvcmtzcGFjZV9zdWJzY3JpYmUYCiABKAsyLy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5Xb3Jrc3BhY2VTdWJzY3JpYmVSZXF1ZXN0SAASUgoVd29ya3NwYWNlX3Vuc3Vic2NyaWJlGAsgASgLMjEubGl2ZWFnZW50LmdhdGV3YXkudjIuV29ya3NwYWNlVW5zdWJzY3JpYmVSZXF1ZXN0SAASLwoEcG9uZxgMIAEoCzIfLmxpdmVhZ2VudC5nYXRld2F5LnYyLlBvbmdGcmFtZUgAEjwKCmFnZW50X2xpc3QYDiABKAsyJi5saXZlYWdlbnQuZ2F0ZXdheS52Mi5BZ2VudExpc3RSZXF1ZXN0SABCCQoHcGF5bG9hZCL/CwoOV2ViU2VydmVyRnJhbWUSEgoKcmVxdWVzdF9pZBgBIAEoCRIQCghhZ2VudF9pZBgQIAEoCRIyCgVoZWxsbxgCIAEoCzIhLmxpdmVhZ2VudC5nYXRld2F5LnYyLlNlcnZlckhlbGxvSAASPQoOYWdlbnRfcmVzcG9uc2UYAyABKAsyIy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5BZ2VudEVudmVsb3BlSAASOgoLbG9jYWxfZXJyb3IYBCABKAsyIy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5FcnJvclJlc3BvbnNlSAASLwoEcGluZxgFIAEoCzIfLmxpdmVhZ2VudC5nYXRld2F5LnYyLlBpbmdGcmFtZUgAEjMKBnN0YXR1cxgGIAEoCzIhLmxpdmVhZ2VudC5nYXRld2F5LnYyLlN0YXR1c0V2ZW50SAASRAoPY2hhdF9zdWJzY3JpYmVkGAcgASgLMikubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFN1YnNjcmliZVJlc3VsdEgAEkIKDWNoYXRfYWNjZXB0ZWQYCCABKAsyKS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0Q29tbWFuZEFjY2VwdGVkSAASRQoPY2hhdF9hY3Rpdml0aWVzGAkgASgLMioubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdEFjdGl2aXRpZXNSZXN1bHRIABI7CgpjaGF0X2V2ZW50GAogASgLMiUubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFN0cmVhbUV2ZW50SAASRgoTY2hhdF9jb21tYW5kX3VwZGF0ZRgLIAEoCzInLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRDb21tYW5kVXBkYXRlSAASTgoXY2hhdF9zdWJzY3JpcHRpb25fcmVzZXQYDCABKAsyKy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0U3Vic2NyaXB0aW9uUmVzZXRIABJACg1jaGF0X2FjdGl2aXR5GA0gASgLMicubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdEFjdGl2aXR5RXZlbnRIABIuCgNhY2sYDiABKAsyHy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5BY2tSZXN1bHRIABJACg5jaGF0X2NhbmNlbGxlZBgPIAEoCzImLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRDYW5jZWxSZXN1bHRIABI7CgphZ2VudF9saXN0GBEgASgLMiUubGl2ZWFnZW50LmdhdGV3YXkudjIuQWdlbnRMaXN0UmVzdWx0SAASPwoNaGlzdG9yeV9ldmVudBgUIAEoCzImLmxpdmVhZ2VudC5nYXRld2F5LnYyLkhpc3RvcnlTeW5jRXZlbnRIABJBCg5zZXR0aW5nc19ldmVudBgVIAEoCzInLmxpdmVhZ2VudC5nYXRld2F5LnYyLlNldHRpbmdzU3luY0V2ZW50SAASPQoOdGVybWluYWxfZXZlbnQYFiABKAsyIy5saXZlYWdlbnQuZ2F0ZXdheS52Mi5UZXJtaW5hbEV2ZW50SAASNQoKc2Z0cF9ldmVudBgXIAEoCzIfLmxpdmVhZ2VudC5nYXRld2F5LnYyLlNmdHBFdmVudEgAEkAKEGNoYXRfcXVldWVfZXZlbnQYGCABKAsyJC5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DaGF0UXVldWVFdmVudEgAEkEKDHR1bm5lbF9zdGF0ZRgZIAEoCzIpLmxpdmVhZ2VudC5nYXRld2F5LnYyLlR1bm5lbFN0YXRlU25hcHNob3RIABJFCg1wcm9jZXNzX3N0YXRlGBogASgLMiwubGl2ZWFnZW50LmdhdGV3YXkudjIuTWFuYWdlZFByb2Nlc3NTbmFwc2hvdEgAEkoKEndvcmtzcGFjZV9hY3Rpdml0eRgbIAEoCzIsLmxpdmVhZ2VudC5nYXRld2F5LnYyLldvcmtzcGFjZUFjdGl2aXR5RXZlbnRIAEIJCgdwYXlsb2FkIhIKEEFnZW50TGlzdFJlcXVlc3QiRAoPQWdlbnRMaXN0UmVzdWx0EjEKBmFnZW50cxgBIAMoCzIhLmxpdmVhZ2VudC5nYXRld2F5LnYyLlN0YXR1c0V2ZW50IooBChBBZ2VudENsaWVudEZyYW1lEjIKBWhlbGxvGAEgASgLMiEubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2xpZW50SGVsbG9IABI3CghlbnZlbG9wZRgCIAEoCzIjLmxpdmVhZ2VudC5nYXRld2F5LnYyLkFnZW50RW52ZWxvcGVIAEIJCgdwYXlsb2FkIowBChBBZ2VudFNlcnZlckZyYW1lEjIKBWhlbGxvGAEgASgLMiEubGl2ZWFnZW50LmdhdGV3YXkudjIuU2VydmVySGVsbG9IABI5CghlbnZlbG9wZRgCIAEoCzIlLmxpdmVhZ2VudC5nYXRld2F5LnYyLkdhdGV3YXlFbnZlbG9wZUgAQgkKB3BheWxvYWQikAEKE1Rlcm1pbmFsQ2xpZW50RnJhbWUSMgoFaGVsbG8YASABKAsyIS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5DbGllbnRIZWxsb0gAEjoKBWZyYW1lGAIgASgLMikubGl2ZWFnZW50LmdhdGV3YXkudjIuVGVybWluYWxTdHJlYW1GcmFtZUgAQgkKB3BheWxvYWQikAEKE1Rlcm1pbmFsU2VydmVyRnJhbWUSMgoFaGVsbG8YASABKAsyIS5saXZlYWdlbnQuZ2F0ZXdheS52Mi5TZXJ2ZXJIZWxsb0gAEjoKBWZyYW1lGAIgASgLMikubGl2ZWFnZW50LmdhdGV3YXkudjIuVGVybWluYWxTdHJlYW1GcmFtZUgAQgkKB3BheWxvYWQiEgoQU3RhdHVzR2V0UmVxdWVzdCLXAgoLU3RhdHVzRXZlbnQSDgoGb25saW5lGAEgASgIEhMKC2FnZW50X3JlYWR5GAIgASgIEhoKEmNoYXRfcnVudGltZV9yZWFkeRgDIAEoCBIQCghhZ2VudF9pZBgEIAEoCRIVCg1hZ2VudF92ZXJzaW9uGAUgASgJEhIKCnNlc3Npb25faWQYBiABKAkSFwoPY29ubmVjdGVkX3NpbmNlGAcgASgDEhYKDmxhc3RfaGVhcnRiZWF0GAggASgDEhUKDXJ1bnRpbWVfc3RhdGUYCSABKAkSHgoWcnVudGltZV9sYXN0X2hlYXJ0YmVhdBgKIAEoAxIZChFydW50aW1lX3dvcmtlcl9pZBgLIAEoCRIXCg9ydW50aW1lX3Zpc2libGUYDCABKAgSIAoYcnVudGltZV9hY3RpdmVfcnVuX2NvdW50GA0gASgNEgwKBG5hbWUYDiABKAkiJAoSQ2hhdFByZXBhcmVSZXF1ZXN0Eg4KBnJlYXNvbhgBIAEoCSJYChRDaGF0U3Vic2NyaWJlUmVxdWVzdBIXCg9jb252ZXJzYXRpb25faWQYASABKAkSEQoJYWZ0ZXJfc2VxGAIgASgDEhQKDHN0cmVhbV9lcG9jaBgDIAEoCSLrAQoPQ2hhdFJ1bkFjdGl2aXR5Eg4KBnJ1bl9pZBgBIAEoCRINCgVzdGF0ZRgCIAEoCRITCgtzdGFydGVkX3NlcRgDIAEoAxIVCg11cGRhdGVkX2F0X21zGAQgASgDEhMKC3Rvb2xfc3RhdHVzGAUgASgJEiEKGXRvb2xfc3RhdHVzX2lzX2NvbXBhY3Rpb24YBiABKAgSGQoRY2xpZW50X3JlcXVlc3RfaWQYByABKAkSFwoPY29udmVyc2F0aW9uX2lkGAggASgJEg8KB3dvcmtkaXIYCSABKAkSEAoIYWdlbnRfaWQYCiABKAkilAEKD0NoYXRSdW5TbmFwc2hvdBIOCgZydW5faWQYASABKAkSEAoIcmV2aXNpb24YAiABKAMSFAoMZW50cmllc19qc29uGAMgASgJEhMKC3Rvb2xfc3RhdHVzGAQgASgJEiEKGXRvb2xfc3RhdHVzX2lzX2NvbXBhY3Rpb24YBSABKAgSEQoJYXNfb2Zfc2VxGAYgASgDIu4BChNDaGF0U3Vic2NyaWJlUmVzdWx0EhcKD2NvbnZlcnNhdGlvbl9pZBgBIAEoCRIUCgxzdHJlYW1fZXBvY2gYAiABKAkSEgoKbGF0ZXN0X3NlcRgDIAEoAxINCgVyZXNldBgEIAEoCBI3CghhY3Rpdml0eRgFIAEoCzIlLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRSdW5BY3Rpdml0eRI3CghzbmFwc2hvdBgGIAEoCzIlLmxpdmVhZ2VudC5nYXRld2F5LnYyLkNoYXRSdW5TbmFwc2hvdBITCgtldmVudHNfanNvbhgHIAMoDCIxChZDaGF0VW5zdWJzY3JpYmVSZXF1ZXN0EhcKD2NvbnZlcnNhdGlvbl9pZBgBIAEoCSIXChVDaGF0QWN0aXZpdGllc1JlcXVlc3QiXAoUQ2hhdEFjdGl2aXRpZXNSZXN1bHQSRAoVcnVubmluZ19jb252ZXJzYXRpb25zGAEgAygLMiUubGl2ZWFnZW50LmdhdGV3YXkudjIuQ2hhdFJ1bkFjdGl2aXR5Ik0KD0NoYXRTdHJlYW1FdmVudBIXCg9jb252ZXJzYXRpb25faWQYASABKAkSCwoDc2VxGAIgASgDEhQKDHBheWxvYWRfanNvbhgDIAEoDCJlChNDaGF0Q29tbWFuZEFjY2VwdGVkEg4KBnJ1bl9pZBgBIAEoCRIXCg9jb252ZXJzYXRpb25faWQYAiABKAkSFAoMYWNjZXB0ZWRfc2VxGAMgASgDEg8KB2RlZHVwZWQYBCABKAgiiwEKEUNoYXRDb21tYW5kVXBkYXRlEg4KBnJ1bl9pZBgBIAEoCRIZChFjbGllbnRfcmVxdWVzdF9pZBgCIAEoCRIXCg9jb252ZXJzYXRpb25faWQYAyABKAkSDQoFcGhhc2UYBCABKAkSEgoKZXJyb3JfY29kZRgFIAEoCRIPCgdtZXNzYWdlGAYgASgJIjAKFUNoYXRTdWJzY3JpcHRpb25SZXNldBIXCg9jb252ZXJzYXRpb25faWQYASABKAkiRwoQQ2hhdENhbmNlbFJlc3VsdBIKCgJvaxgBIAEoCBIOCgZydW5faWQYAiABKAkSFwoPY29udmVyc2F0aW9uX2lkGAMgASgJIp8BChFDaGF0QWN0aXZpdHlFdmVudBIXCg9jb252ZXJzYXRpb25faWQYASABKAkSDgoGcnVuX2lkGAIgASgJEhkKEWNsaWVudF9yZXF1ZXN0X2lkGAMgASgJEg8KB3J1bm5pbmcYBCABKAgSDQoFc3RhdGUYBSABKAkSDwoHd29ya2RpchgGIAEoCRIVCg11cGRhdGVkX2F0X21zGAcgASgDIiwKGVdvcmtzcGFjZVN1YnNjcmliZVJlcXVlc3QSDwoHd29ya2RpchgBIAEoCSIuChtXb3Jrc3BhY2VVbnN1YnNjcmliZVJlcXVlc3QSDwoHd29ya2RpchgBIAEoCSpZCgpDbGllbnRSb2xlEhsKF0NMSUVOVF9ST0xFX1VOU1BFQ0lGSUVEEAASFwoTQ0xJRU5UX1JPTEVfQlJPV1NFUhABEhUKEUNMSUVOVF9ST0xFX0FHRU5UEAJCQFo+Z2l0aHViLmNvbS9saXZlYWdlbnQvYWdlbnQtZ2F0ZXdheS9pbnRlcm5hbC9wcm90by92MjtnYXRld2F5djJiBnByb3RvMw", [file_proto_v2_gateway]); /** * ClientHello 是所有 v2 连接的第一帧。 @@ -197,6 +197,282 @@ export type AckResult = Message<"liveagent.gateway.v2.AckResult"> & { export const AckResultSchema: GenMessage = /*@__PURE__*/ messageDesc(file_proto_v2_gateway_ws, 4); +/** + * @generated from message liveagent.gateway.v2.SttClientHello + */ +export type SttClientHello = Message<"liveagent.gateway.v2.SttClientHello"> & { + /** + * @generated from field: uint32 protocol_version = 1; + */ + protocolVersion: number; + + /** + * @generated from field: string token = 2; + */ + token: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttClientHello. + * Use `create(SttClientHelloSchema)` to create a new message. + */ +export const SttClientHelloSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 5); + +/** + * @generated from message liveagent.gateway.v2.SttStart + */ +export type SttStart = Message<"liveagent.gateway.v2.SttStart"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string provider = 2; + */ + provider: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttStart. + * Use `create(SttStartSchema)` to create a new message. + */ +export const SttStartSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 6); + +/** + * @generated from message liveagent.gateway.v2.SttAudio + */ +export type SttAudio = Message<"liveagent.gateway.v2.SttAudio"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: uint32 sequence = 2; + */ + sequence: number; + + /** + * @generated from field: bytes pcm = 3; + */ + pcm: Uint8Array; +}; + +/** + * Describes the message liveagent.gateway.v2.SttAudio. + * Use `create(SttAudioSchema)` to create a new message. + */ +export const SttAudioSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 7); + +/** + * @generated from message liveagent.gateway.v2.SttSessionControl + */ +export type SttSessionControl = Message<"liveagent.gateway.v2.SttSessionControl"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttSessionControl. + * Use `create(SttSessionControlSchema)` to create a new message. + */ +export const SttSessionControlSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 8); + +/** + * @generated from message liveagent.gateway.v2.SttClientFrame + */ +export type SttClientFrame = Message<"liveagent.gateway.v2.SttClientFrame"> & { + /** + * @generated from oneof liveagent.gateway.v2.SttClientFrame.payload + */ + payload: { + /** + * @generated from field: liveagent.gateway.v2.SttClientHello hello = 1; + */ + value: SttClientHello; + case: "hello"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttStart start = 2; + */ + value: SttStart; + case: "start"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttAudio audio = 3; + */ + value: SttAudio; + case: "audio"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttSessionControl stop = 4; + */ + value: SttSessionControl; + case: "stop"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttSessionControl cancel = 5; + */ + value: SttSessionControl; + case: "cancel"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message liveagent.gateway.v2.SttClientFrame. + * Use `create(SttClientFrameSchema)` to create a new message. + */ +export const SttClientFrameSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 9); + +/** + * @generated from message liveagent.gateway.v2.SttServerHello + */ +export type SttServerHello = Message<"liveagent.gateway.v2.SttServerHello"> & { + /** + * @generated from field: bool ok = 1; + */ + ok: boolean; + + /** + * @generated from field: string message = 2; + */ + message: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttServerHello. + * Use `create(SttServerHelloSchema)` to create a new message. + */ +export const SttServerHelloSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 10); + +/** + * @generated from message liveagent.gateway.v2.SttTextEvent + */ +export type SttTextEvent = Message<"liveagent.gateway.v2.SttTextEvent"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string text = 2; + */ + text: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttTextEvent. + * Use `create(SttTextEventSchema)` to create a new message. + */ +export const SttTextEventSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 11); + +/** + * @generated from message liveagent.gateway.v2.SttError + */ +export type SttError = Message<"liveagent.gateway.v2.SttError"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string code = 2; + */ + code: string; + + /** + * @generated from field: string message = 3; + */ + message: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttError. + * Use `create(SttErrorSchema)` to create a new message. + */ +export const SttErrorSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 12); + +/** + * @generated from message liveagent.gateway.v2.SttClosed + */ +export type SttClosed = Message<"liveagent.gateway.v2.SttClosed"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; +}; + +/** + * Describes the message liveagent.gateway.v2.SttClosed. + * Use `create(SttClosedSchema)` to create a new message. + */ +export const SttClosedSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 13); + +/** + * @generated from message liveagent.gateway.v2.SttServerFrame + */ +export type SttServerFrame = Message<"liveagent.gateway.v2.SttServerFrame"> & { + /** + * @generated from oneof liveagent.gateway.v2.SttServerFrame.payload + */ + payload: { + /** + * @generated from field: liveagent.gateway.v2.SttServerHello hello = 1; + */ + value: SttServerHello; + case: "hello"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttTextEvent ready = 2; + */ + value: SttTextEvent; + case: "ready"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttTextEvent partial = 3; + */ + value: SttTextEvent; + case: "partial"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttTextEvent final = 4; + */ + value: SttTextEvent; + case: "final"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttError error = 5; + */ + value: SttError; + case: "error"; + } | { + /** + * @generated from field: liveagent.gateway.v2.SttClosed closed = 6; + */ + value: SttClosed; + case: "closed"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message liveagent.gateway.v2.SttServerFrame. + * Use `create(SttServerFrameSchema)` to create a new message. + */ +export const SttServerFrameSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_proto_v2_gateway_ws, 14); + /** * WebClientFrame 为浏览器 → 网关方向的帧。除 agent_request 直通臂外，其余臂均为 * 网关本地操作（由网关自身状态应答，不经桌面端往返）。 @@ -306,7 +582,7 @@ export type WebClientFrame = Message<"liveagent.gateway.v2.WebClientFrame"> & { * Use `create(WebClientFrameSchema)` to create a new message. */ export const WebClientFrameSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 5); + messageDesc(file_proto_v2_gateway_ws, 15); /** * WebServerFrame 为网关 → 浏览器方向的帧。 @@ -487,7 +763,7 @@ export type WebServerFrame = Message<"liveagent.gateway.v2.WebServerFrame"> & { * Use `create(WebServerFrameSchema)` to create a new message. */ export const WebServerFrameSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 6); + messageDesc(file_proto_v2_gateway_ws, 16); /** * AgentListRequest 查询 Agent 目录；响应为 AgentListResult。 @@ -502,7 +778,7 @@ export type AgentListRequest = Message<"liveagent.gateway.v2.AgentListRequest"> * Use `create(AgentListRequestSchema)` to create a new message. */ export const AgentListRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 7); + messageDesc(file_proto_v2_gateway_ws, 17); /** * AgentListResult 返回全部已登记 Agent 的状态（含离线项，供目录渲染）， @@ -522,7 +798,7 @@ export type AgentListResult = Message<"liveagent.gateway.v2.AgentListResult"> & * Use `create(AgentListResultSchema)` to create a new message. */ export const AgentListResultSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 8); + messageDesc(file_proto_v2_gateway_ws, 18); /** * AgentClientFrame 为桌面端 → 网关方向的帧。 @@ -553,7 +829,7 @@ export type AgentClientFrame = Message<"liveagent.gateway.v2.AgentClientFrame"> * Use `create(AgentClientFrameSchema)` to create a new message. */ export const AgentClientFrameSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 9); + messageDesc(file_proto_v2_gateway_ws, 19); /** * AgentServerFrame 为网关 → 桌面端方向的帧。 @@ -584,7 +860,7 @@ export type AgentServerFrame = Message<"liveagent.gateway.v2.AgentServerFrame"> * Use `create(AgentServerFrameSchema)` to create a new message. */ export const AgentServerFrameSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 10); + messageDesc(file_proto_v2_gateway_ws, 20); /** * TerminalClientFrame 为客户端（浏览器或桌面端）→ 网关方向的帧。 @@ -615,7 +891,7 @@ export type TerminalClientFrame = Message<"liveagent.gateway.v2.TerminalClientFr * Use `create(TerminalClientFrameSchema)` to create a new message. */ export const TerminalClientFrameSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 11); + messageDesc(file_proto_v2_gateway_ws, 21); /** * TerminalServerFrame 为网关 → 客户端方向的帧。 @@ -646,7 +922,7 @@ export type TerminalServerFrame = Message<"liveagent.gateway.v2.TerminalServerFr * Use `create(TerminalServerFrameSchema)` to create a new message. */ export const TerminalServerFrameSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 12); + messageDesc(file_proto_v2_gateway_ws, 22); /** * StatusGetRequest 请求网关侧运行状态快照（操作类型："status.get"）。 @@ -661,7 +937,7 @@ export type StatusGetRequest = Message<"liveagent.gateway.v2.StatusGetRequest"> * Use `create(StatusGetRequestSchema)` to create a new message. */ export const StatusGetRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 13); + messageDesc(file_proto_v2_gateway_ws, 23); /** * StatusEvent 镜像 session.Status 的 JSON 形状（字段一一对应）。 @@ -747,7 +1023,7 @@ export type StatusEvent = Message<"liveagent.gateway.v2.StatusEvent"> & { * Use `create(StatusEventSchema)` to create a new message. */ export const StatusEventSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 14); + messageDesc(file_proto_v2_gateway_ws, 24); /** * ChatPrepareRequest 唤醒/探活桌面端 chat 运行时（操作类型："chat.prepare"）；响应为 StatusEvent。 @@ -766,7 +1042,7 @@ export type ChatPrepareRequest = Message<"liveagent.gateway.v2.ChatPrepareReques * Use `create(ChatPrepareRequestSchema)` to create a new message. */ export const ChatPrepareRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 15); + messageDesc(file_proto_v2_gateway_ws, 25); /** * ChatSubscribeRequest 订阅会话事件流（操作类型："chat.subscribe"）；外层 WebClientFrame.agent_id 必须非空， @@ -797,7 +1073,7 @@ export type ChatSubscribeRequest = Message<"liveagent.gateway.v2.ChatSubscribeRe * Use `create(ChatSubscribeRequestSchema)` to create a new message. */ export const ChatSubscribeRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 16); + messageDesc(file_proto_v2_gateway_ws, 26); /** * ChatRunActivity 镜像 session.RunActivity 的 JSON 形状。 @@ -865,7 +1141,7 @@ export type ChatRunActivity = Message<"liveagent.gateway.v2.ChatRunActivity"> & * Use `create(ChatRunActivitySchema)` to create a new message. */ export const ChatRunActivitySchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 17); + messageDesc(file_proto_v2_gateway_ws, 27); /** * ChatRunSnapshot 镜像 session.RunSnapshot 的 JSON 形状。 @@ -911,7 +1187,7 @@ export type ChatRunSnapshot = Message<"liveagent.gateway.v2.ChatRunSnapshot"> & * Use `create(ChatRunSnapshotSchema)` to create a new message. */ export const ChatRunSnapshotSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 18); + messageDesc(file_proto_v2_gateway_ws, 28); /** * ChatSubscribeResult 是 chat_subscribe 的响应。 @@ -963,7 +1239,7 @@ export type ChatSubscribeResult = Message<"liveagent.gateway.v2.ChatSubscribeRes * Use `create(ChatSubscribeResultSchema)` to create a new message. */ export const ChatSubscribeResultSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 19); + messageDesc(file_proto_v2_gateway_ws, 29); /** * ChatUnsubscribeRequest 取消订阅（操作类型："chat.unsubscribe"）；响应 AckResult。 @@ -982,7 +1258,7 @@ export type ChatUnsubscribeRequest = Message<"liveagent.gateway.v2.ChatUnsubscri * Use `create(ChatUnsubscribeRequestSchema)` to create a new message. */ export const ChatUnsubscribeRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 20); + messageDesc(file_proto_v2_gateway_ws, 30); /** * ChatActivitiesRequest 查询运行中会话（操作类型："chat.activities"）；仅由网关状态应答，桌面端离线时亦可用。 @@ -997,7 +1273,7 @@ export type ChatActivitiesRequest = Message<"liveagent.gateway.v2.ChatActivities * Use `create(ChatActivitiesRequestSchema)` to create a new message. */ export const ChatActivitiesRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 21); + messageDesc(file_proto_v2_gateway_ws, 31); /** * @generated from message liveagent.gateway.v2.ChatActivitiesResult @@ -1014,7 +1290,7 @@ export type ChatActivitiesResult = Message<"liveagent.gateway.v2.ChatActivitiesR * Use `create(ChatActivitiesResultSchema)` to create a new message. */ export const ChatActivitiesResultSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 22); + messageDesc(file_proto_v2_gateway_ws, 32); /** * ChatStreamEvent 是订阅后推送的单条会话事件。 @@ -1045,7 +1321,7 @@ export type ChatStreamEvent = Message<"liveagent.gateway.v2.ChatStreamEvent"> & * Use `create(ChatStreamEventSchema)` to create a new message. */ export const ChatStreamEventSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 23); + messageDesc(file_proto_v2_gateway_ws, 33); /** * ChatCommandAccepted 是 chat_command 提交被接受的响应（chat_command 的接受应答）。 @@ -1079,7 +1355,7 @@ export type ChatCommandAccepted = Message<"liveagent.gateway.v2.ChatCommandAccep * Use `create(ChatCommandAcceptedSchema)` to create a new message. */ export const ChatCommandAcceptedSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 24); + messageDesc(file_proto_v2_gateway_ws, 34); /** * ChatCommandUpdate 推送命令的前置阶段结果（bound / queued_in_gui / failed），镜像 session.ChatCommandUpdate。 @@ -1123,7 +1399,7 @@ export type ChatCommandUpdate = Message<"liveagent.gateway.v2.ChatCommandUpdate" * Use `create(ChatCommandUpdateSchema)` to create a new message. */ export const ChatCommandUpdateSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 25); + messageDesc(file_proto_v2_gateway_ws, 35); /** * ChatSubscriptionReset 通知客户端某会话流已被限流丢弃，需重新订阅（after_seq 断点续传）。 @@ -1142,7 +1418,7 @@ export type ChatSubscriptionReset = Message<"liveagent.gateway.v2.ChatSubscripti * Use `create(ChatSubscriptionResetSchema)` to create a new message. */ export const ChatSubscriptionResetSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 26); + messageDesc(file_proto_v2_gateway_ws, 36); /** * ChatCancelResult 是 chat.cancel 的响应。 @@ -1171,7 +1447,7 @@ export type ChatCancelResult = Message<"liveagent.gateway.v2.ChatCancelResult"> * Use `create(ChatCancelResultSchema)` to create a new message. */ export const ChatCancelResultSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 27); + messageDesc(file_proto_v2_gateway_ws, 37); /** * ChatActivityEvent 广播会话活动状态变化，镜像 session.ConversationActivityEvent。 @@ -1220,7 +1496,7 @@ export type ChatActivityEvent = Message<"liveagent.gateway.v2.ChatActivityEvent" * Use `create(ChatActivityEventSchema)` to create a new message. */ export const ChatActivityEventSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 28); + messageDesc(file_proto_v2_gateway_ws, 38); /** * WorkspaceSubscribeRequest 订阅工作区活动（操作类型："workspace.subscribe"）；响应 AckResult，事件经 workspace_activity 臂广播。 @@ -1239,7 +1515,7 @@ export type WorkspaceSubscribeRequest = Message<"liveagent.gateway.v2.WorkspaceS * Use `create(WorkspaceSubscribeRequestSchema)` to create a new message. */ export const WorkspaceSubscribeRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 29); + messageDesc(file_proto_v2_gateway_ws, 39); /** * WorkspaceUnsubscribeRequest 取消订阅；响应 AckResult。 @@ -1258,7 +1534,7 @@ export type WorkspaceUnsubscribeRequest = Message<"liveagent.gateway.v2.Workspac * Use `create(WorkspaceUnsubscribeRequestSchema)` to create a new message. */ export const WorkspaceUnsubscribeRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_proto_v2_gateway_ws, 30); + messageDesc(file_proto_v2_gateway_ws, 40); /** * ClientRole 区分 /ws/v2/terminal 上连接的所属端（该链路两端共用一条路径，靠 hello.role 区分）。 diff --git a/crates/agent-gateway/web/src/lib/stt/webSttSettingsService.ts b/crates/agent-gateway/web/src/lib/stt/webSttSettingsService.ts new file mode 100644 index 000000000..22b314391 --- /dev/null +++ b/crates/agent-gateway/web/src/lib/stt/webSttSettingsService.ts @@ -0,0 +1,49 @@ +import type { SttConnectionTestResponse, SttSettingsService } from "@liveagent/ui/lib/stt/types"; +import { type AppSettings, normalizeSttSettings } from "@/lib/settings"; +import { loadToken } from "@/lib/storage"; + +function authorizationHeaders() { + return { authorization: `Bearer ${loadToken()}` }; +} + +async function readSettingsResponse(response: Response) { + if (!response.ok) throw new Error("STT 配置请求失败"); + return normalizeSttSettings((await response.json()) as AppSettings["stt"]); +} + +export function createWebSttSettingsService( + syncDesktop?: (settings: AppSettings["stt"]) => Promise, +): SttSettingsService { + return { + runtimeLabel: "WebUI（与桌面端同步，凭据由 Gateway 安全托管）", + // Browser settings never receive stored credential values. The shared + // password control displays the field name when the eye button is active. + secretRevealMode: "field-name", + async get() { + return readSettingsResponse( + await fetch("/api/v2/stt/settings", { headers: authorizationHeaders() }), + ); + }, + async update(settings) { + const redacted = await readSettingsResponse( + await fetch("/api/v2/stt/settings", { + method: "PUT", + headers: { ...authorizationHeaders(), "content-type": "application/json" }, + body: JSON.stringify(settings), + }), + ); + await syncDesktop?.(settings); + return redacted; + }, + async test(provider) { + const response = await fetch( + `/api/v2/stt/settings/test?provider=${encodeURIComponent(provider)}`, + { method: "POST", headers: authorizationHeaders() }, + ); + if (!response.ok) throw new Error("STT 连接测试失败"); + return (await response.json()) as SttConnectionTestResponse; + }, + }; +} + +export const webSttSettingsService = createWebSttSettingsService(); diff --git a/crates/agent-gateway/web/src/lib/stt/webSttTransport.ts b/crates/agent-gateway/web/src/lib/stt/webSttTransport.ts new file mode 100644 index 000000000..b1057539b --- /dev/null +++ b/crates/agent-gateway/web/src/lib/stt/webSttTransport.ts @@ -0,0 +1,157 @@ +import { create, fromBinary, toBinary } from "@bufbuild/protobuf"; +import type { + SttRuntimeEvent, + SttTransport, + SttTransportOpenOptions, +} from "@liveagent/ui/lib/stt/types"; +import { + SttAudioSchema, + type SttClientFrame, + SttClientFrameSchema, + SttClientHelloSchema, + type SttServerFrame, + SttServerFrameSchema, + SttSessionControlSchema, + SttStartSchema, +} from "@/lib/proto/gen/proto/v2/gateway_ws_pb"; +import { loadToken } from "@/lib/storage"; + +const STT_SUBPROTOCOL = "liveagent.v2.pb"; + +type SttClientPayload = SttClientFrame["payload"]; + +class WebSttTransport implements SttTransport { + private socket: WebSocket | null = null; + private handler: ((event: SttRuntimeEvent) => void) | null = null; + + async open(options: SttTransportOpenOptions) { + this.dispose(); + this.handler = options.onEvent; + const protocol = location.protocol === "https:" ? "wss:" : "ws:"; + const socket = new WebSocket(`${protocol}//${location.host}/ws/v2/stt`, STT_SUBPROTOCOL); + socket.binaryType = "arraybuffer"; + this.socket = socket; + + await new Promise((resolve, reject) => { + let settled = false; + const timer = window.setTimeout(() => { + settled = true; + socket.close(); + reject(new Error("Gateway STT 连接超时")); + }, 10_000); + const settleError = (message: string) => { + if (settled) return; + settled = true; + window.clearTimeout(timer); + reject(new Error(message)); + }; + + socket.onopen = () => { + this.sendFrame({ + case: "hello", + value: create(SttClientHelloSchema, { + protocolVersion: 2, + token: loadToken(), + }), + }); + }; + socket.onerror = () => settleError("Gateway STT 网络连接失败"); + socket.onmessage = (message) => { + if (!(message.data instanceof ArrayBuffer)) return; + let frame: SttServerFrame; + try { + frame = fromBinary(SttServerFrameSchema, new Uint8Array(message.data)); + } catch { + settleError("Gateway STT 返回了无效协议帧"); + return; + } + const payload = frame.payload; + if (payload.case === "hello") { + if (!payload.value.ok) { + settleError(payload.value.message || "Gateway STT 鉴权失败"); + return; + } + if (!settled) { + settled = true; + window.clearTimeout(timer); + this.sendFrame({ + case: "start", + value: create(SttStartSchema, { + sessionId: options.sessionId, + provider: options.provider, + }), + }); + resolve(); + } + return; + } + if (payload.case === "ready") { + this.handler?.({ type: "ready", sessionId: payload.value.sessionId }); + } else if (payload.case === "partial") { + this.handler?.({ + type: "partial", + sessionId: payload.value.sessionId, + text: payload.value.text, + }); + } else if (payload.case === "final") { + this.handler?.({ + type: "final", + sessionId: payload.value.sessionId, + text: payload.value.text, + }); + } else if (payload.case === "error") { + this.handler?.({ + type: "error", + sessionId: payload.value.sessionId, + code: payload.value.code, + message: payload.value.message, + }); + } else if (payload.case === "closed") { + this.handler?.({ type: "closed", sessionId: payload.value.sessionId }); + } + }; + socket.onclose = () => { + settleError("Gateway STT 连接已关闭"); + this.handler?.({ type: "closed", sessionId: options.sessionId }); + }; + }); + } + + async sendAudio(sessionId: string, sequence: number, pcm: Uint8Array) { + this.sendFrame({ + case: "audio", + value: create(SttAudioSchema, { sessionId, sequence, pcm }), + }); + } + + async stop(sessionId: string) { + this.sendControl("stop", sessionId); + } + + async cancel(sessionId: string) { + this.sendControl("cancel", sessionId); + } + + private sendControl(type: "stop" | "cancel", sessionId: string) { + this.sendFrame({ + case: type, + value: create(SttSessionControlSchema, { sessionId }), + }); + } + + private sendFrame(payload: SttClientPayload) { + const socket = this.socket; + if (!socket || socket.readyState !== WebSocket.OPEN) { + throw new Error("Gateway STT 未连接"); + } + socket.send(toBinary(SttClientFrameSchema, create(SttClientFrameSchema, { payload }))); + } + + dispose() { + this.handler = null; + this.socket?.close(); + this.socket = null; + } +} + +export const webSttTransport = new WebSttTransport(); diff --git a/crates/agent-gateway/web/src/pages/settings/types.ts b/crates/agent-gateway/web/src/pages/settings/types.ts index 470b32fdd..d04298452 100644 --- a/crates/agent-gateway/web/src/pages/settings/types.ts +++ b/crates/agent-gateway/web/src/pages/settings/types.ts @@ -1,4 +1,5 @@ -import type { AppSettings } from "../../lib/settings"; +import type { SttSettingsService } from "@liveagent/ui/lib/stt/types"; +import type { AppSettings, SttProviderId } from "../../lib/settings"; import type { WebSettingsSaveState } from "../../lib/webSettings"; export type SetSettingsFn = (updater: (prev: AppSettings) => AppSettings) => void; @@ -6,6 +7,7 @@ export type SetSettingsFn = (updater: (prev: AppSettings) => AppSettings) => voi export type SectionId = | "system" | "systemTools" + | "stt" | "providers" | "agents" | "ssh" @@ -24,6 +26,9 @@ export type SettingsPageProps = { initialProviderId?: string; hiddenSections?: SectionId[]; onAgentDirectoryChanged?: () => void | Promise; + sttSettingsService: SttSettingsService; + /** 临时切换语音输入运行供应商，不触发配置保存。 */ + onSttProviderChange?: (provider: SttProviderId) => void; }; export type SettingsSectionProps = { diff --git a/crates/agent-gui/src-tauri/Cargo.toml b/crates/agent-gui/src-tauri/Cargo.toml index e8c6629f6..d284ab05c 100644 --- a/crates/agent-gui/src-tauri/Cargo.toml +++ b/crates/agent-gui/src-tauri/Cargo.toml @@ -28,11 +28,16 @@ serde_json = "1.0.150" reqwest = { version = "0.13.4", features = ["blocking", "json", "stream", "socks"] } rquickjs = { version = "0.8", features = ["array-buffer", "classes", "bindgen"] } percent-encoding = "2.3.2" +urlencoding = "2.1.3" axum = "0.8.9" tokio = { version = "1.52.3", features = ["macros", "net", "sync", "time", "io-util"] } tokio-stream = "0.1.18" tokio-tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"] } futures-util = "0.3.32" +async-trait = "0.1.89" +hmac = "0.12.1" +sha1 = "0.10.6" +flate2 = "1.1.8" uuid = { version = "1.23.4", features = ["v4"] } base64 = "0.22.1" globset = "0.4.18" @@ -73,6 +78,12 @@ chardetng = "0.1.17" [target.'cfg(target_os = "macos")'.dependencies] objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSButton", "NSControl", "NSView", "NSWindow"] } +objc2-av-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "AVCaptureDevice", "AVMediaFormat"] } +objc2 = { version = "0.6", default-features = false, features = ["std"] } +block2 = "0.6" [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Threading"] } + +[dev-dependencies] +tauri = { version = "2.11.5", features = ["test"] } diff --git a/crates/agent-gui/src-tauri/Entitlements.plist b/crates/agent-gui/src-tauri/Entitlements.plist new file mode 100644 index 000000000..f35f2d428 --- /dev/null +++ b/crates/agent-gui/src-tauri/Entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.device.audio-input + + + diff --git a/crates/agent-gui/src-tauri/Info.plist b/crates/agent-gui/src-tauri/Info.plist new file mode 100644 index 000000000..2ac60386b --- /dev/null +++ b/crates/agent-gui/src-tauri/Info.plist @@ -0,0 +1,8 @@ + + + + + NSMicrophoneUsageDescription + LiveAgent 使用麦克风将语音转换为输入文字。 + + diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/commands.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/commands.rs index b1d208beb..76e608a22 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/commands.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/commands.rs @@ -10,6 +10,7 @@ pub async fn settings_load_all() -> Result { agents: load_agents(&conn)?, ssh: load_ssh(&conn)?, remote: load_remote(&conn)?, + stt: load_stt_redacted(&conn)?, memory: load_memory(&conn)?, model_failover: load_model_failover(&conn)?, default_workdir, @@ -92,6 +93,37 @@ pub async fn settings_save_model_failover(payload: Value) -> Result<(), String> .map_err(|e| format!("settings_save_model_failover join 失败：{e}"))? } +#[tauri::command] +pub async fn settings_save_stt( + payload: Value, + gateway_controller: tauri::State<'_, Arc>, +) -> Result { + let saved = tauri::async_runtime::spawn_blocking(move || { + let mut conn = open_db()?; + save_stt(&mut conn, payload)?; + load_stt_redacted(&conn).map(|value| value.unwrap_or_else(|| json!({}))) + }) + .await + .map_err(|e| format!("settings_save_stt join failed: {e}"))??; + if let Err(error) = gateway_controller.publish_current_settings_sync().await { + eprintln!("publish STT settings sync failed: {error}"); + } + Ok(saved) +} + +#[tauri::command] +pub async fn settings_reveal_stt_secret( + provider: String, + field: String, +) -> Result { + tauri::async_runtime::spawn_blocking(move || { + let conn = open_db()?; + load_stt_secret(&conn, &provider, &field) + }) + .await + .map_err(|e| format!("settings_reveal_stt_secret join failed: {e}"))? +} + #[tauri::command] pub async fn settings_save_agents(payload: Value) -> Result<(), String> { tauri::async_runtime::spawn_blocking(move || { diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/db.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/db.rs index 895b09d1e..e46d0c76a 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/db.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/db.rs @@ -100,6 +100,11 @@ pub(crate) fn initialize_schema(conn: &Connection) -> Result<(), String> { payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL ); + CREATE TABLE IF NOT EXISTS stt_settings ( + config_id TEXT PRIMARY KEY, + payload_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); -- WebDAV 同步配置。独立成表是刻意的：见 mod.rs 上 BACKUP_SYNC_SETTINGS_TABLE 的注释。 CREATE TABLE IF NOT EXISTS backup_sync_settings ( config_id TEXT PRIMARY KEY, diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/gateway_sync.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/gateway_sync.rs index dcaca3f41..76e55775c 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/gateway_sync.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/gateway_sync.rs @@ -24,6 +24,10 @@ pub(crate) fn load_gateway_settings_sync_snapshot(conn: &Connection) -> Result Result Result, provider: &Map) -> bool { + validate_provider(provider_id.unwrap_or_default(), provider).is_ok() +} + +fn provider_text<'a>(provider: &'a Map, field: &str) -> &'a str { + provider + .get(field) + .and_then(Value::as_str) + .unwrap_or_default() + .trim() +} + +fn require_provider_text<'a>( + provider: &'a Map, + field: &str, + label: &str, +) -> Result<&'a str, String> { + let value = provider_text(provider, field); + if value.is_empty() { + Err(format!("{label} 不能为空")) + } else { + Ok(value) + } +} + +fn validate_websocket_url(provider: &Map) -> Result<(), String> { + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + + let endpoint = require_provider_text(provider, "websocketUrl", "WebSocket 地址")?; + let request = endpoint + .into_client_request() + .map_err(|_| "WebSocket 地址必须是完整的 wss:// 地址".to_string())?; + let has_user_info = request + .uri() + .authority() + .is_some_and(|authority| authority.as_str().contains('@')); + if request.uri().scheme_str() != Some("wss") + || request.uri().host().is_none() + || has_user_info + { + return Err("WebSocket 地址必须是完整的 wss:// 地址".to_string()); + } + Ok(()) +} + +fn require_positive_integer( + provider: &Map, + field: &str, + label: &str, +) -> Result<(), String> { + let value = require_provider_text(provider, field, label)?; + if value.parse::().is_ok_and(|value| value > 0) { + Ok(()) + } else { + Err(format!("{label} 必须是大于 0 的数字")) + } +} + +fn validate_provider(provider_id: &str, provider: &Map) -> Result<(), String> { + match provider_id { + "aliyun_dashscope" => { + validate_websocket_url(provider)?; + require_provider_text(provider, "model", "模型名称")?; + require_provider_text(provider, "apiKey", "API Key")?; + } + "tencent_cloud" => { + require_positive_integer(provider, "appId", "腾讯 AppId")?; + require_provider_text(provider, "engineModelType", "腾讯引擎模型")?; + require_provider_text(provider, "secretId", "腾讯 SecretId")?; + require_provider_text(provider, "secretKey", "腾讯 SecretKey")?; + } + "volcengine_v2" => { + validate_websocket_url(provider)?; + require_provider_text(provider, "appId", "火山 v2 App ID")?; + require_provider_text(provider, "cluster", "火山 v2 Cluster")?; + require_provider_text(provider, "accessToken", "火山 v2 Access Token")?; + } + "volcengine_seed_v3" => { + validate_websocket_url(provider)?; + require_provider_text(provider, "appId", "火山 Seed v3 App ID")?; + require_provider_text(provider, "accessToken", "火山 Seed v3 Access Token")?; + require_provider_text(provider, "resourceId", "火山 Seed v3 Resource ID")?; + } + "baidu_cloud" => { + validate_websocket_url(provider)?; + require_positive_integer(provider, "baiduAppId", "百度 App ID")?; + require_positive_integer(provider, "devPid", "百度 dev_pid")?; + require_provider_text(provider, "baiduApiKey", "百度 API Key")?; + } + _ => return Err("未知 STT 供应商".to_string()), + } + Ok(()) +} + +pub(crate) fn load_stt_raw(conn: &Connection) -> Result, String> { + let payload: Option = conn + .query_row( + &format!("SELECT payload_json FROM {STT_SETTINGS_TABLE} WHERE config_id = ?1"), + params![STT_CONFIG_ID], + |row| row.get(0), + ) + .optional() + .map_err(|error| format!("load STT settings failed: {error}"))?; + payload + .map(|raw| { + serde_json::from_str(&raw) + .map_err(|error| format!("decode STT settings failed: {error}")) + }) + .transpose() +} + +pub(crate) fn redact_stt_secrets(value: &mut Value) { + let Some(providers) = value.get_mut("providers").and_then(Value::as_object_mut) else { + return; + }; + for provider in providers.values_mut().filter_map(Value::as_object_mut) { + let provider_id = provider.get("id").and_then(Value::as_str); + let configured = provider_configured(provider_id, provider); + for field in STT_SECRET_FIELDS { + if provider.contains_key(*field) { + provider.insert((*field).to_string(), Value::String(String::new())); + } + } + provider.insert("configured".to_string(), Value::Bool(configured)); + provider.remove("clearSecrets"); + } +} + +pub(crate) fn load_stt_redacted(conn: &Connection) -> Result, String> { + let mut value = load_stt_raw(conn)?; + if let Some(payload) = value.as_mut() { + redact_stt_secrets(payload); + } + Ok(value) +} + +pub(crate) fn load_stt_secret( + conn: &Connection, + provider_id: &str, + field: &str, +) -> Result { + let provider_id = provider_id.trim(); + let field = field.trim(); + if !STT_SECRET_FIELDS.contains(&field) { + return Err("不允许查看该 STT 字段".to_string()); + } + let payload = load_stt_raw(conn)?.ok_or_else(|| "STT 尚未配置".to_string())?; + let provider = payload + .get("providers") + .and_then(Value::as_object) + .and_then(|providers| providers.get(provider_id)) + .and_then(Value::as_object) + .ok_or_else(|| "STT 供应商配置不存在".to_string())?; + Ok(provider_text(provider, field).to_string()) +} + +pub(crate) fn load_stt_provider_runtime(provider_id: &str) -> Result, String> { + let conn = open_db()?; + let payload = load_stt_raw(&conn)?.ok_or_else(|| "STT 尚未配置".to_string())?; + let mut provider = payload + .get("providers") + .and_then(Value::as_object) + .and_then(|providers| providers.get(provider_id)) + .and_then(Value::as_object) + .cloned() + .ok_or_else(|| "STT 供应商配置不存在".to_string())?; + if provider_id == "volcengine_seed_v3" && provider_text(&provider, "websocketUrl").is_empty() { + provider.insert( + "websocketUrl".to_string(), + Value::String(VOLCENGINE_SEED_V3_ENDPOINT.to_string()), + ); + } + if !provider_configured(Some(provider_id), &provider) { + return Err("STT 供应商配置不完整".to_string()); + } + Ok(provider) +} + +pub(crate) fn save_stt(conn: &mut Connection, payload: Value) -> Result<(), String> { + let mut next = expect_object(payload, "settings_save_stt payload")?; + let allow_incomplete = next + .remove("allowIncomplete") + .and_then(|value| value.as_bool()) + .unwrap_or(false); + let selected_provider = next + .get("provider") + .and_then(Value::as_str) + .map(str::to_string); + let mut current = load_stt_raw(conn)? + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + let current_providers = current + .remove("providers") + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + let Some(next_providers) = next.get_mut("providers").and_then(Value::as_object_mut) else { + return Err("settings_save_stt payload.providers must be an object".to_string()); + }; + let mut selected_secrets_cleared = false; + for (provider_id, provider_value) in next_providers.iter_mut() { + let Some(provider) = provider_value.as_object_mut() else { + return Err(format!("STT provider {provider_id} must be an object")); + }; + let old = current_providers + .get(provider_id) + .and_then(Value::as_object); + let clear = provider + .remove("clearSecrets") + .and_then(|value| value.as_bool()) + .unwrap_or(false); + if selected_provider.as_deref() == Some(provider_id) { + selected_secrets_cleared = clear; + } + provider.remove("configured"); + for field in STT_SECRET_FIELDS { + let incoming = provider + .get(*field) + .and_then(Value::as_str) + .unwrap_or_default() + .trim() + .to_string(); + if clear { + provider.insert((*field).to_string(), Value::String(String::new())); + } else if incoming.is_empty() { + if let Some(existing) = old.and_then(|item| item.get(*field)).cloned() { + provider.insert((*field).to_string(), existing); + } + } else { + provider.insert((*field).to_string(), Value::String(incoming)); + } + } + } + if let Some(provider_id) = selected_provider.as_deref() { + if !selected_secrets_cleared && !allow_incomplete { + let provider = next_providers + .get(provider_id) + .and_then(Value::as_object) + .ok_or_else(|| "当前 STT 供应商配置不存在".to_string())?; + validate_provider(provider_id, provider)?; + } + } + let payload_json = serde_json::to_string(&Value::Object(next)) + .map_err(|error| format!("encode STT settings failed: {error}"))?; + conn.execute( + &format!( + "INSERT INTO {STT_SETTINGS_TABLE} (config_id, payload_json, updated_at) VALUES (?1, ?2, ?3) \ + ON CONFLICT(config_id) DO UPDATE SET payload_json = excluded.payload_json, updated_at = excluded.updated_at" + ), + params![STT_CONFIG_ID, payload_json, now_ms()], + ) + .map_err(|error| format!("save STT settings failed: {error}"))?; + Ok(()) +} + +#[cfg(test)] +mod stt_tests { + use super::*; + use serde_json::json; + + fn provider(id: &str) -> Map { + let mut value = Map::new(); + value.insert("id".into(), Value::String(id.into())); + value + } + + #[test] + fn provider_validation_requires_each_secret_and_numeric_baidu_ids() { + let mut aliyun = provider("aliyun_dashscope"); + aliyun.insert("websocketUrl".into(), json!("wss://example.com/stt")); + aliyun.insert("model".into(), json!("model")); + aliyun.insert("apiKey".into(), json!("key")); + assert!(provider_configured(Some("aliyun_dashscope"), &aliyun)); + + let mut tencent = provider("tencent_cloud"); + for (key, value) in [ + ("appId", "1"), + ("secretId", "id"), + ("secretKey", "key"), + ("engineModelType", "16k_zh"), + ] { + tencent.insert(key.into(), json!(value)); + } + assert!(provider_configured(Some("tencent_cloud"), &tencent)); + tencent.insert("appId".into(), json!("not-numeric")); + assert!(!provider_configured(Some("tencent_cloud"), &tencent)); + tencent.insert("appId".into(), json!("1")); + tencent.remove("secretKey"); + assert!(!provider_configured(Some("tencent_cloud"), &tencent)); + + let mut v2 = provider("volcengine_v2"); + v2.insert("websocketUrl".into(), json!("wss://example.com/stt")); + for (key, value) in [("appId", "app"), ("accessToken", "token"), ("cluster", "router")] { + v2.insert(key.into(), json!(value)); + } + assert!(provider_configured(Some("volcengine_v2"), &v2)); + + let mut v3 = provider("volcengine_seed_v3"); + v3.insert("websocketUrl".into(), json!(VOLCENGINE_SEED_V3_ENDPOINT)); + for (key, value) in [("appId", "app"), ("accessToken", "token"), ("resourceId", "resource")] { + v3.insert(key.into(), json!(value)); + } + assert!(provider_configured(Some("volcengine_seed_v3"), &v3)); + + let mut baidu = provider("baidu_cloud"); + baidu.insert("websocketUrl".into(), json!("wss://example.com/stt")); + baidu.insert("baiduAppId".into(), json!("123")); + baidu.insert("baiduApiKey".into(), json!("key")); + baidu.insert("devPid".into(), json!("1537")); + assert!(provider_configured(Some("baidu_cloud"), &baidu)); + baidu.insert("devPid".into(), json!("model-name")); + assert!(!provider_configured(Some("baidu_cloud"), &baidu)); + + aliyun.insert("websocketUrl".into(), json!("https://example.com/stt")); + assert!(!provider_configured(Some("aliyun_dashscope"), &aliyun)); + v3.insert("websocketUrl".into(), json!("wss://user@example.com/stt")); + assert!(!provider_configured(Some("volcengine_seed_v3"), &v3)); + } + + #[test] + fn redaction_marks_configured_without_exposing_secrets() { + let mut payload = json!({ + "providers": { + "aliyun_dashscope": { + "id": "aliyun_dashscope", + "websocketUrl": "wss://example.com/stt", + "model": "model", + "apiKey": "secret", + "clearSecrets": true + } + } + }); + redact_stt_secrets(&mut payload); + let provider = &payload["providers"]["aliyun_dashscope"]; + assert_eq!(provider["apiKey"], ""); + assert_eq!(provider["configured"], true); + assert!(provider.get("clearSecrets").is_none()); + assert!(!payload.to_string().contains("secret")); + } + + #[test] + fn error_sanitization_replaces_all_supported_credentials() { + let config = serde_json::from_value(json!({ + "apiKey": "dash-secret", + "secretId": "tencent-id", + "secretKey": "tencent-secret", + "accessToken": "volc-token", + "baiduApiKey": "baidu-secret" + })) + .expect("map"); + let sanitized = crate::services::stt::sanitize_error( + "dash-secret/tencent-id/tencent-secret/volc-token/baidu-secret", + &config, + ); + assert!(!sanitized.contains("dash-secret")); + assert!(!sanitized.contains("tencent-secret")); + assert_eq!(sanitized.matches("[redacted]").count(), 5); + } + + #[test] + fn save_rejects_invalid_selected_provider_but_allows_explicit_clear() { + let mut conn = Connection::open_in_memory().expect("in-memory STT database"); + conn.execute_batch( + "CREATE TABLE stt_settings ( + config_id TEXT PRIMARY KEY, + payload_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + )", + ) + .expect("create STT settings table"); + + let invalid = json!({ + "provider": "tencent_cloud", + "providers": { + "tencent_cloud": { + "id": "tencent_cloud", + "appId": "not-numeric", + "engineModelType": "16k_zh", + "secretId": "id", + "secretKey": "key" + } + } + }); + let error = save_stt(&mut conn, invalid).expect_err("invalid Tencent AppId must fail"); + assert!(error.contains("大于 0 的数字")); + + let clear = json!({ + "provider": "tencent_cloud", + "providers": { + "tencent_cloud": { + "id": "tencent_cloud", + "clearSecrets": true + } + } + }); + save_stt(&mut conn, clear).expect("explicit secret clearing must remain valid"); + assert!(load_stt_raw(&conn).expect("load cleared settings").is_some()); + + // The UI may toggle the voice-input switch after the clear request. + // That follow-up metadata write must be allowed without re-validating + // the now intentionally incomplete provider. + let follow_up = json!({ + "provider": "tencent_cloud", + "enabled": false, + "allowIncomplete": true, + "providers": { + "tencent_cloud": { + "id": "tencent_cloud", + "appId": "", + "engineModelType": "16k_zh", + "secretId": "", + "secretKey": "" + } + } + }); + save_stt(&mut conn, follow_up).expect("voice-input toggle persistence must remain valid"); + } +} diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs index 49d06ee83..c0e34a1f8 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs @@ -382,6 +382,57 @@ mod tests { assert_eq!(redacted.get("providerUsageQuerySecretUpdates"), None); } + #[test] + fn gateway_settings_snapshot_redacts_stt_and_private_sync_field() { + let mut conn = open_memory_db(); + save_stt( + &mut conn, + json!({ + "provider": "aliyun_dashscope", + "providers": { + "aliyun_dashscope": { + "id": "aliyun_dashscope", + "websocketUrl": "wss://example.com/stt", + "model": "paraformer-realtime-v2", + "apiKey": "desktop-only-secret" + } + } + }), + ) + .expect("save STT settings"); + + let snapshot = + load_gateway_settings_sync_snapshot(&conn).expect("load gateway settings snapshot"); + assert_eq!(snapshot["stt"]["provider"], "aliyun_dashscope"); + assert_eq!( + snapshot["stt"]["providers"]["aliyun_dashscope"]["configured"], + true + ); + assert_eq!( + snapshot["stt"]["providers"]["aliyun_dashscope"]["apiKey"], + "" + ); + assert_eq!( + load_stt_secret(&conn, "aliyun_dashscope", "apiKey") + .expect("reveal local STT secret"), + "desktop-only-secret" + ); + assert!(load_stt_secret(&conn, "aliyun_dashscope", "websocketUrl").is_err()); + + let redacted = redact_gateway_settings_sync_payload(json!({ + "sttSecretSync": { + "providers": {"aliyun_dashscope": {"apiKey": "must-not-leak"}} + }, + "stt": load_stt_raw(&conn).expect("load raw STT settings") + })) + .expect("redact gateway STT payload"); + assert_eq!(redacted.get(STT_SECRET_SYNC_FIELD), None); + assert_eq!( + redacted["stt"]["providers"]["aliyun_dashscope"]["apiKey"], + "" + ); + } + #[test] fn save_ssh_persists_hosts_and_redacts_sync_snapshot() { let mut conn = open_memory_db(); diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/types.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/types.rs index 4391cea0f..f3989ac92 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/types.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/types.rs @@ -7,6 +7,7 @@ pub struct SettingsLoadResponse { pub agents: Option, pub ssh: Option, pub remote: Option, + pub stt: Option, pub memory: Option, pub model_failover: Option, pub default_workdir: String, diff --git a/crates/agent-gui/src-tauri/src/lib.rs b/crates/agent-gui/src-tauri/src/lib.rs index cc9bf16ed..d738ed671 100644 --- a/crates/agent-gui/src-tauri/src/lib.rs +++ b/crates/agent-gui/src-tauri/src/lib.rs @@ -154,6 +154,14 @@ macro_rules! app_invoke_handler { commands::settings::settings_save_remote, commands::settings::settings_save_memory, commands::settings::settings_save_model_failover, + commands::settings::settings_save_stt, + commands::settings::settings_reveal_stt_secret, + services::stt::settings_test_stt, + services::stt::stt_request_microphone_permission, + services::stt::stt_start, + services::stt::stt_send_audio, + services::stt::stt_stop, + services::stt::stt_cancel, commands::settings::settings_backup_export, commands::settings::settings_backup_peek_import, commands::settings::settings_backup_apply_import, @@ -711,6 +719,7 @@ pub fn run() { let close_window_behavior = Arc::new(commands::app::CloseWindowBehaviorState::new( commands::app::CLOSE_WINDOW_BEHAVIOR_MINIMIZE, )); + let stt_manager = Arc::new(services::stt::SttManager::default()); let app = tauri::Builder::default() .plugin(tauri_plugin_opener::init()) @@ -747,6 +756,7 @@ pub fn run() { .manage(Arc::clone(&automation_store)) .manage(Arc::clone(&automation_scheduler)) .manage(Arc::new(commands::hook::HookScopeRegistry::default())) + .manage(stt_manager) .setup({ let terminal_registry = Arc::clone(&terminal_registry); let sftp_registry = Arc::clone(&sftp_registry); diff --git a/crates/agent-gui/src-tauri/src/services/gateway/controller.rs b/crates/agent-gui/src-tauri/src/services/gateway/controller.rs index 23507b243..4715cc62c 100644 --- a/crates/agent-gui/src-tauri/src/services/gateway/controller.rs +++ b/crates/agent-gui/src-tauri/src/services/gateway/controller.rs @@ -381,7 +381,11 @@ impl GatewayController { return Ok(()); } - let envelope = build_settings_sync_envelope(snapshot)?; + // The cached/browser-visible snapshot remains redacted. Only the + // authenticated desktop-to-Gateway envelope receives the raw STT + // sidecar, which Gateway consumes before broadcasting the snapshot. + let outbound = attach_current_stt_secret_sync(snapshot).await?; + let envelope = build_settings_sync_envelope(outbound)?; self.send_agent_envelope(envelope).await } } diff --git a/crates/agent-gui/src-tauri/src/services/gateway/settings_sync.rs b/crates/agent-gui/src-tauri/src/services/gateway/settings_sync.rs index 6003c2dd9..260ede8a0 100644 --- a/crates/agent-gui/src-tauri/src/services/gateway/settings_sync.rs +++ b/crates/agent-gui/src-tauri/src/services/gateway/settings_sync.rs @@ -4,9 +4,10 @@ use serde_json::Value; use uuid::Uuid; use crate::commands::settings::{ - load_gateway_settings_sync_snapshot, open_db, redact_gateway_settings_sync_payload, - PROVIDER_API_KEY_UPDATES_FIELD, PROVIDER_USAGE_QUERY_SECRET_UPDATES_FIELD, SSH_PATCH_FIELD, - SSH_SECRET_UPDATES_FIELD, SYSTEM_PROXY_PASSWORD_UPDATE_FIELD, + load_gateway_settings_sync_snapshot, load_stt_raw, open_db, + redact_gateway_settings_sync_payload, PROVIDER_API_KEY_UPDATES_FIELD, + PROVIDER_USAGE_QUERY_SECRET_UPDATES_FIELD, SSH_PATCH_FIELD, SSH_SECRET_UPDATES_FIELD, + STT_SECRET_SYNC_FIELD, STT_SECRET_UPDATE_FIELD, SYSTEM_PROXY_PASSWORD_UPDATE_FIELD, }; use super::*; @@ -94,6 +95,30 @@ pub(crate) fn merge_settings_update_into_snapshot( Ok(Value::Object(merged)) } +pub(crate) fn attach_stt_secret_sync( + mut snapshot: Value, + raw_stt: Option, +) -> Result { + let settings = snapshot + .as_object_mut() + .ok_or_else(|| "gateway settings sync payload must be an object".to_string())?; + settings.remove(STT_SECRET_SYNC_FIELD); + if let Some(stt) = raw_stt { + settings.insert(STT_SECRET_SYNC_FIELD.to_string(), stt); + } + Ok(snapshot) +} + +pub(crate) async fn attach_current_stt_secret_sync(snapshot: Value) -> Result { + let raw_stt = tauri::async_runtime::spawn_blocking(move || { + let conn = open_db()?; + load_stt_raw(&conn) + }) + .await + .map_err(|e| format!("load private STT sync join failed: {e}"))??; + attach_stt_secret_sync(snapshot, raw_stt) +} + fn workspace_project_ids( payload: &Value, payload_label: &str, @@ -179,6 +204,7 @@ pub(crate) fn build_local_settings_update_event_payload(payload: Value) -> Resul event.remove(PROVIDER_USAGE_QUERY_SECRET_UPDATES_FIELD); let ssh_secret_updates = event.remove(SSH_SECRET_UPDATES_FIELD); let system_proxy_password_update = event.remove(SYSTEM_PROXY_PASSWORD_UPDATE_FIELD); + let stt_secret_update = event.remove(STT_SECRET_UPDATE_FIELD); event.remove("remote"); let mut public_event = match redact_gateway_settings_sync_payload(Value::Object(event))? { Value::Object(map) => map, @@ -199,6 +225,9 @@ pub(crate) fn build_local_settings_update_event_payload(payload: Value) -> Resul if let Some(update) = system_proxy_password_update { public_event.insert(SYSTEM_PROXY_PASSWORD_UPDATE_FIELD.to_string(), update); } + if let Some(update) = stt_secret_update { + public_event.insert(STT_SECRET_UPDATE_FIELD.to_string(), update); + } Ok(Value::Object(public_event)) } @@ -215,6 +244,7 @@ pub(crate) fn build_local_settings_update_event_payload_with_ssh( event.remove(PROVIDER_USAGE_QUERY_SECRET_UPDATES_FIELD); let ssh_secret_updates = event.remove(SSH_SECRET_UPDATES_FIELD); let system_proxy_password_update = event.remove(SYSTEM_PROXY_PASSWORD_UPDATE_FIELD); + let stt_secret_update = event.remove(STT_SECRET_UPDATE_FIELD); event.remove("remote"); event.remove(SSH_PATCH_FIELD); event.insert("ssh".to_string(), ssh); @@ -237,6 +267,9 @@ pub(crate) fn build_local_settings_update_event_payload_with_ssh( if let Some(update) = system_proxy_password_update { public_event.insert(SYSTEM_PROXY_PASSWORD_UPDATE_FIELD.to_string(), update); } + if let Some(update) = stt_secret_update { + public_event.insert(STT_SECRET_UPDATE_FIELD.to_string(), update); + } Ok(Value::Object(public_event)) } diff --git a/crates/agent-gui/src-tauri/src/services/gateway/tests.rs b/crates/agent-gui/src-tauri/src/services/gateway/tests.rs index dc83e612a..1b049294f 100644 --- a/crates/agent-gui/src-tauri/src/services/gateway/tests.rs +++ b/crates/agent-gui/src-tauri/src/services/gateway/tests.rs @@ -1,7 +1,8 @@ use super::{ - build_gateway_runtime_status_envelope, build_local_settings_update_event_payload, - effective_agent_id, gateway_connection_needs_restart, gateway_connection_stale_after, - gateway_reconnect_backoff, history_share_resolve_error_code, is_chat_runtime_wake_request_id, + attach_stt_secret_sync, build_gateway_runtime_status_envelope, + build_local_settings_update_event_payload, effective_agent_id, + gateway_connection_needs_restart, gateway_connection_stale_after, gateway_reconnect_backoff, + history_share_resolve_error_code, is_chat_runtime_wake_request_id, merge_settings_sync_snapshot, merge_settings_update_into_snapshot, proto, removed_workspace_project_ids, required_terminal_project_path_key, set_disconnected_status, GatewayChatRequestEvent, GatewayController, GatewayStatusSnapshot, RemoteChatInboxRecord, @@ -381,6 +382,46 @@ fn merge_settings_sync_snapshot_without_cache_leaves_ui_only_fields_absent() { assert_eq!(merged["system"], json!({ "executionMode": "agent-dev" })); } +#[test] +fn settings_sync_attaches_private_stt_only_to_the_outbound_payload() { + let public_snapshot = json!({ + "theme": "dark", + "stt": { + "provider": "tencent_cloud", + "providers": { + "tencent_cloud": { + "id": "tencent_cloud", + "configured": true, + "appId": "123", + "secretId": "", + "secretKey": "" + } + } + } + }); + let raw_stt = json!({ + "provider": "tencent_cloud", + "providers": { + "tencent_cloud": { + "id": "tencent_cloud", + "appId": "123", + "secretId": "desktop-secret-id", + "secretKey": "desktop-secret-key" + } + } + }); + + let outbound = attach_stt_secret_sync(public_snapshot.clone(), Some(raw_stt.clone())) + .expect("attach private STT settings"); + + assert_eq!(outbound["sttSecretSync"], raw_stt); + assert_eq!( + public_snapshot["stt"]["providers"]["tencent_cloud"]["secretId"], + "" + ); + assert!(public_snapshot.get("sttSecretSync").is_none()); +} + #[test] fn merge_settings_update_into_snapshot_keeps_unrelated_fields() { let full_snapshot = json!({ diff --git a/crates/agent-gui/src-tauri/src/services/mod.rs b/crates/agent-gui/src-tauri/src/services/mod.rs index b51ea6911..bb39f9755 100644 --- a/crates/agent-gui/src-tauri/src/services/mod.rs +++ b/crates/agent-gui/src-tauri/src/services/mod.rs @@ -8,6 +8,7 @@ pub mod provider_models; pub mod provider_usage; pub mod proxy; pub mod skills; +pub mod stt; pub mod system_proxy; pub mod tray; pub mod tunnel; diff --git a/crates/agent-gui/src-tauri/src/services/stt/aliyun_dashscope.rs b/crates/agent-gui/src-tauri/src/services/stt/aliyun_dashscope.rs new file mode 100644 index 000000000..29ed5d6fe --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/stt/aliyun_dashscope.rs @@ -0,0 +1,134 @@ +use super::{ + close_provider_socket, emit, provider_failure, send_provider_message, stage_failure, text, + websocket_endpoint, SttCommand, SttEvent, +}; +use futures_util::StreamExt; +use serde_json::Value; +use tauri::{AppHandle, Runtime}; +use tokio::sync::mpsc::Receiver; +use tokio_tungstenite::{ + connect_async, + tungstenite::{client::IntoClientRequest, http::header, Message}, +}; + +fn dashscope_model(configured: String) -> String { + if configured.is_empty() || configured == "paraformer-realtime-8k-v2" { + "paraformer-realtime-v2".to_string() + } else { + configured + } +} + +fn finish_message(task_id: &str) -> Value { + serde_json::json!({ + "header":{"action":"finish-task","task_id":task_id,"streaming":"duplex"}, + "payload":{"input":{}} + }) +} + +pub async fn run( + app: AppHandle, + session: String, + config: serde_json::Map, + mut rx: Receiver, +) -> Result<(), String> { + let key = text(&config, "apiKey"); + let endpoint = websocket_endpoint(&config, "wss://dashscope.aliyuncs.com/api-ws/v1/inference/") + .map_err(|e| stage_failure("DashScope", "validate", e))?; + let model = dashscope_model(text(&config, "model")); + let task_id = uuid::Uuid::new_v4().simple().to_string(); + let mut request = endpoint + .into_client_request() + .map_err(|e| stage_failure("DashScope", "connect", e.to_string()))?; + request.headers_mut().insert( + header::AUTHORIZATION, + format!("Bearer {key}") + .parse() + .map_err(|_| "无效 API Key")?, + ); + request.headers_mut().insert( + "X-DashScope-DataInspection", + "enable" + .parse() + .map_err(|_| "DashScope 数据检查请求头无效")?, + ); + let (socket, _) = connect_async(request) + .await + .map_err(|e| stage_failure("DashScope", "connect", format!("网络错误: {e}")))?; + let (mut write, mut read) = socket.split(); + send_provider_message(&mut write, Message::Text( serde_json::json!({ "header":{"action":"run-task","task_id":task_id,"streaming":"duplex"}, "payload":{ "task_group":"audio","task":"asr","function":"recognition", "model":&model, "parameters":{ "format":"pcm", "sample_rate":16000, "language_hints":["zh", "en"], "max_sentence_silence":2000, "disfluency_removal_enabled":false }, "input":{} } }) .to_string() .into(), ), "DashScope", "start").await?; + let mut ready = false; + let mut finishing = false; + let mut finish_sent = false; + let mut pending = Vec::>::new(); + let mut finals = String::new(); + loop { + tokio::select! { + Some(command) = rx.recv() => match command { + SttCommand::Audio { pcm, .. } => { if ready { send_provider_message(&mut write, Message::Binary(pcm.into()), "DashScope", "send_audio").await?; } else { pending.push(pcm); } } + SttCommand::Finish => { + if !finishing { finishing = true; if ready { send_provider_message(&mut write, Message::Text(finish_message(&task_id).to_string().into()), "DashScope", "finish").await?; finish_sent = true; } } + } + SttCommand::Cancel => { close_provider_socket(&mut write).await; return Ok(()); } + }, + Some(message) = read.next() => { + let message = message.map_err(|e| stage_failure("DashScope", "receive", e.to_string()))?; + let Message::Text(body) = message else { continue }; + let value: Value = serde_json::from_str(&body).map_err(|_| stage_failure("DashScope", "parse", "返回内容不是有效 JSON"))?; + match value.pointer("/header/event").and_then(Value::as_str).unwrap_or_default() { + "task-started" => { + if !ready { + ready = true; + emit(&app, SttEvent::Ready { session_id: session.clone() }); + for pcm in pending.drain(..) { send_provider_message(&mut write, Message::Binary(pcm.into()), "DashScope", "send_audio").await?; } + if finishing && !finish_sent { + send_provider_message(&mut write, Message::Text(finish_message(&task_id).to_string().into()), "DashScope", "finish").await?; + finish_sent = true; + } + } + } + "result-generated" => { + let sentence = value.pointer("/payload/output/sentence"); + let sentence_text = sentence.and_then(|s| s.get("text")).and_then(Value::as_str).unwrap_or_default(); + let ended = sentence.and_then(|s| s.get("end")).and_then(Value::as_bool).unwrap_or(false) || sentence.and_then(|s| s.get("sentence_end")).and_then(Value::as_bool).unwrap_or(false); + if ended { finals.push_str(sentence_text); } + let partial = if ended { finals.clone() } else { format!("{finals}{sentence_text}") }; + emit(&app, SttEvent::Partial { session_id: session.clone(), text: partial }); + } + "task-finished" if finish_sent => { + if !finals.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: finals.clone() }); } + close_provider_socket(&mut write).await; + return Ok(()); + } + "task-failed" => return Err(stage_failure("DashScope", "provider_response", provider_failure( + "DashScope", + value.pointer("/header/error_code").and_then(Value::as_str).unwrap_or_default(), + value.pointer("/header/error_message").and_then(Value::as_str).unwrap_or_default(), + ))), + _ => {} + } + } + else => return Err(stage_failure("DashScope", "close", "连接在 task-finished 前关闭")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn uses_16_khz_model_and_complete_finish_message() { + assert_eq!(dashscope_model(String::new()), "paraformer-realtime-v2"); + assert_eq!( + dashscope_model("paraformer-realtime-8k-v2".to_string()), + "paraformer-realtime-v2" + ); + let finish = finish_message("task"); + assert_eq!( + finish.pointer("/header/streaming").and_then(Value::as_str), + Some("duplex") + ); + assert!(finish.pointer("/payload/input").is_some()); + } +} diff --git a/crates/agent-gui/src-tauri/src/services/stt/baidu_cloud.rs b/crates/agent-gui/src-tauri/src/services/stt/baidu_cloud.rs new file mode 100644 index 000000000..c09b1f03e --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/stt/baidu_cloud.rs @@ -0,0 +1,152 @@ +use super::{ + close_provider_socket, emit, provider_failure, send_provider_message, stage_failure, text, + websocket_endpoint, SttCommand, SttEvent, +}; +use futures_util::StreamExt; +use serde_json::Value; +use tauri::{AppHandle, Runtime}; +use tokio::sync::mpsc::Receiver; +use tokio_tungstenite::{ + connect_async, + tungstenite::{error::ProtocolError, Error as WebSocketError, Message}, +}; + +fn baidu_finished_connection_closed(error: &WebSocketError) -> bool { + matches!( + error, + WebSocketError::ConnectionClosed + | WebSocketError::AlreadyClosed + | WebSocketError::Protocol(ProtocolError::ResetWithoutClosingHandshake) + ) +} + +fn result_text(value: &Value) -> String { + match value { + Value::String(text) => text.trim().to_string(), + Value::Array(parts) => parts + .iter() + .filter_map(Value::as_str) + .collect::>() + .join("") + .trim() + .to_string(), + _ => String::new(), + } +} + +pub async fn run( + app: AppHandle, + session: String, + config: serde_json::Map, + mut rx: Receiver, +) -> Result<(), String> { + let app_id = text(&config, "baiduAppId") + .parse::() + .map_err(|_| stage_failure("Baidu", "validate", "App ID 必须是数字"))?; + let dev_pid = text(&config, "devPid") + .parse::() + .map_err(|_| stage_failure("Baidu", "validate", "dev_pid 必须是数字"))?; + let app_key = text(&config, "baiduApiKey"); + let endpoint = websocket_endpoint(&config, "wss://vop.baidu.com/realtime_asr") + .map_err(|e| stage_failure("Baidu", "validate", e))?; + let endpoint = format!( + "{endpoint}{}sn={}", + if endpoint.contains('?') { "&" } else { "?" }, + uuid::Uuid::new_v4() + ); + let (socket, _) = connect_async(endpoint) + .await + .map_err(|e| stage_failure("Baidu", "connect", format!("网络错误: {e}")))?; + let (mut write, mut read) = socket.split(); + send_provider_message(&mut write, Message::Text( serde_json::json!({ "type":"START", "data": { "appid":app_id, "appkey":app_key, "dev_pid":dev_pid, "cuid":format!("LiveAgent-{}", uuid::Uuid::new_v4()), "format":"pcm", "sample":16000 } }) .to_string() .into(), ), "Baidu", "start").await?; + // 百度协议没有 READY 消息，START 写入后即可发送已缓存音频。 + emit( + &app, + SttEvent::Ready { + session_id: session.clone(), + }, + ); + let mut finish_sent = false; + let mut no_speech_seen = false; + let mut final_text = String::new(); + loop { + tokio::select! { + Some(command) = rx.recv() => match command { + SttCommand::Audio { pcm, .. } => { if !finish_sent { send_provider_message(&mut write, Message::Binary(pcm.into()), "Baidu", "send_audio").await?; } } + SttCommand::Finish => { if !finish_sent { finish_sent = true; send_provider_message(&mut write, Message::Text(serde_json::json!({"type":"FINISH"}).to_string().into()), "Baidu", "finish").await?; if no_speech_seen { return Ok(()); } } } + SttCommand::Cancel => { close_provider_socket(&mut write).await; return Ok(()); } + }, + Some(message) = read.next() => { + let message = match message { + Ok(message) => message, + Err(error) if finish_sent && baidu_finished_connection_closed(&error) => { + if !final_text.is_empty() { + emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text.clone() }); + } + return Ok(()); + } + Err(error) => return Err(stage_failure("Baidu", "receive", error.to_string())), + }; + match message { + Message::Text(body) => { + let value: Value = serde_json::from_str(&body).map_err(|_| stage_failure("Baidu", "parse", "返回内容不是有效 JSON"))?; + let err_no = value.get("err_no").and_then(Value::as_i64).unwrap_or(0); + let err_msg = value.get("err_msg").and_then(Value::as_str).unwrap_or("百度识别失败"); + if err_no == 3301 || err_no == -3005 { + no_speech_seen = true; + if finish_sent { return Ok(()); } + continue; + } + if err_no != 0 { return Err(stage_failure("Baidu", "provider_response", provider_failure("百度", &err_no.to_string(), err_msg))); } + match value.get("type").and_then(Value::as_str) { + Some("MID_TEXT") => emit(&app, SttEvent::Partial { session_id: session.clone(), text: format!("{final_text}{}", result_text(value.get("result").unwrap_or(&Value::Null))) }), + Some("FIN_TEXT") => { + let text = result_text(value.get("result").unwrap_or(&Value::Null)); + if !text.is_empty() { final_text.push_str(&text); } + emit(&app, SttEvent::Partial { session_id: session.clone(), text: final_text.clone() }); + if finish_sent { + if !final_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text.clone() }); } + close_provider_socket(&mut write).await; + return Ok(()); + } + } + Some("FINISH") if finish_sent => { if !final_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text.clone() }); } close_provider_socket(&mut write).await; return Ok(()); } + _ => {} + } + } + Message::Close(_) => return if finish_sent { if !final_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text.clone() }); } Ok(()) } else { Err(stage_failure("Baidu", "close", "连接在 FINISH 前关闭")) }, + _ => {} + } + } + else => return if finish_sent { if !final_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text.clone() }); } Ok(()) } else { Err(stage_failure("Baidu", "close", "连接在 FINISH 前关闭")) }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn response_text_accepts_string_and_array_results() { + assert_eq!(result_text(&json!(" 单句 ")), "单句"); + assert_eq!(result_text(&json!(["你好，", "世界。"])), "你好，世界。"); + } + + #[test] + fn finish_accepts_only_normal_websocket_shutdown_errors() { + assert!(baidu_finished_connection_closed( + &WebSocketError::ConnectionClosed + )); + assert!(baidu_finished_connection_closed( + &WebSocketError::AlreadyClosed + )); + assert!(baidu_finished_connection_closed(&WebSocketError::Protocol( + ProtocolError::ResetWithoutClosingHandshake + ))); + assert!(!baidu_finished_connection_closed( + &WebSocketError::Protocol(ProtocolError::InvalidOpcode(3)) + )); + } +} diff --git a/crates/agent-gui/src-tauri/src/services/stt/mod.rs b/crates/agent-gui/src-tauri/src/services/stt/mod.rs new file mode 100644 index 000000000..c9b7bcda2 --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/stt/mod.rs @@ -0,0 +1,918 @@ +use serde::Serialize; +use std::{ + collections::HashMap, + sync::{Arc, LazyLock, Mutex}, + time::Duration, +}; +use tauri::{AppHandle, Emitter, Runtime, State}; +use tokio::sync::{mpsc, oneshot}; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; + +mod aliyun_dashscope; +mod baidu_cloud; +mod tencent_cloud; +mod volcengine_seed_v3; +mod volcengine_v2; + +#[derive(Debug, Clone)] +pub enum SttCommand { + Audio { sequence: u32, pcm: Vec }, + Finish, + Cancel, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum SttEvent { + Ready { + #[serde(rename = "sessionId")] + session_id: String, + }, + Partial { + #[serde(rename = "sessionId")] + session_id: String, + text: String, + }, + Final { + #[serde(rename = "sessionId")] + session_id: String, + text: String, + }, + Error { + #[serde(rename = "sessionId")] + session_id: String, + code: String, + message: String, + }, + Closed { + #[serde(rename = "sessionId")] + session_id: String, + }, +} + +impl SttEvent { + fn session_id(&self) -> &str { + match self { + Self::Ready { session_id } + | Self::Partial { session_id, .. } + | Self::Final { session_id, .. } + | Self::Error { session_id, .. } + | Self::Closed { session_id } => session_id, + } + } +} + +static TEST_OBSERVERS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +fn ensure_stt_crypto_provider() { + crate::services::gateway::ensure_rustls_crypto_provider(); +} + +struct ActiveSttSession { + sender: mpsc::Sender, + next_sequence: u32, + cancel: Option>, +} + +#[derive(Default)] +pub struct SttManager { + sessions: Arc>>, +} + +impl SttManager { + fn emit(app: &AppHandle, event: SttEvent) { + let session_id = event.session_id().to_string(); + let _ = app.emit("stt:event", event.clone()); + if let Ok(observers) = TEST_OBSERVERS.lock() { + if let Some(observer) = observers.get(&session_id) { + let _ = observer.send(event); + } + } + } + pub async fn start( + &self, + app: AppHandle, + session_id: String, + provider: String, + ) -> Result<(), String> { + self.start_observed(app, session_id, provider, None).await + } + async fn start_observed( + &self, + app: AppHandle, + session_id: String, + provider: String, + observer: Option>, + ) -> Result<(), String> { + // Desktop STT is independent of the Gateway connection, so it must + // initialize rustls before any provider WebSocket is opened. + ensure_stt_crypto_provider(); + let config = crate::commands::settings::load_stt_provider_runtime(&provider)?; + let secrets = config.clone(); + let (tx, rx) = mpsc::channel(64); + let (cancel_tx, cancel_rx) = oneshot::channel(); + let mut sessions_guard = self + .sessions + .lock() + .map_err(|_| "STT session lock poisoned")?; + if sessions_guard.contains_key(&session_id) { + return Err("STT session already exists".to_string()); + } + sessions_guard.insert( + session_id.clone(), + ActiveSttSession { + sender: tx, + next_sequence: 0, + cancel: Some(cancel_tx), + }, + ); + drop(sessions_guard); + if let Some(observer) = observer { + TEST_OBSERVERS + .lock() + .map_err(|_| "STT test observer lock poisoned")? + .insert(session_id.clone(), observer); + } + let sessions = self.sessions.clone(); + tokio::spawn(async move { + let run_fut = async { + match provider.as_str() { + "aliyun_dashscope" => { + aliyun_dashscope::run(app.clone(), session_id.clone(), config, rx).await + } + "tencent_cloud" => { + tencent_cloud::run(app.clone(), session_id.clone(), config, rx).await + } + "volcengine_v2" => { + volcengine_v2::run(app.clone(), session_id.clone(), config, rx).await + } + "volcengine_seed_v3" => { + volcengine_seed_v3::run(app.clone(), session_id.clone(), config, rx).await + } + "baidu_cloud" => { + baidu_cloud::run(app.clone(), session_id.clone(), config, rx).await + } + _ => Err("未知 STT 供应商".to_string()), + } + }; + let result = tokio::select! { + biased; + _ = cancel_rx => Ok(()), + result = run_fut => result, + }; + if let Err(message) = result { + let safe_message = sanitize_error(&message, &secrets); + let event = SttEvent::Error { + session_id: session_id.clone(), + code: classify_test_error(&safe_message).into(), + message: safe_message, + }; + Self::emit(&app, event.clone()); + } + let closed = SttEvent::Closed { + session_id: session_id.clone(), + }; + Self::emit(&app, closed.clone()); + if let Ok(mut observers) = TEST_OBSERVERS.lock() { + observers.remove(&session_id); + } + if let Ok(mut all) = sessions.lock() { + all.remove(&session_id); + } + }); + Ok(()) + } + pub async fn send(&self, session_id: &str, command: SttCommand) -> Result<(), String> { + let tx = { + let mut sessions = self + .sessions + .lock() + .map_err(|_| "STT session lock poisoned")?; + let active = sessions + .get_mut(session_id) + .ok_or_else(|| "STT session 不存在".to_string())?; + if let SttCommand::Audio { sequence, .. } = &command { + if *sequence != active.next_sequence { + return Err("STT 音频序号不连续".to_string()); + } + active.next_sequence = active + .next_sequence + .checked_add(1) + .ok_or_else(|| "STT 音频序号溢出".to_string())?; + } + active.sender.clone() + }; + tx.send(command) + .await + .map_err(|_| "STT session 已关闭".to_string()) + } + pub async fn cancel(&self, session_id: &str) -> Result<(), String> { + let (sender, cancel) = { + let mut sessions = self + .sessions + .lock() + .map_err(|_| "STT session lock poisoned")?; + let active = sessions + .get_mut(session_id) + .ok_or_else(|| "STT session 不存在".to_string())?; + (active.sender.clone(), active.cancel.take()) + }; + if let Some(cancel) = cancel { + let _ = cancel.send(()); + } + let _ = sender.try_send(SttCommand::Cancel); + Ok(()) + } +} + +#[tauri::command] +pub async fn stt_start( + app: AppHandle, + manager: State<'_, Arc>, + session_id: String, + provider: String, +) -> Result<(), String> { + manager.start(app, session_id, provider).await +} + +#[tauri::command] +pub async fn stt_send_audio( + manager: State<'_, Arc>, + session_id: String, + sequence: u32, + pcm: Vec, +) -> Result<(), String> { + manager + .send(&session_id, SttCommand::Audio { sequence, pcm }) + .await +} + +#[tauri::command] +pub async fn stt_stop( + manager: State<'_, Arc>, + session_id: String, +) -> Result<(), String> { + manager.send(&session_id, SttCommand::Finish).await +} + +#[tauri::command] +pub async fn stt_cancel( + manager: State<'_, Arc>, + session_id: String, +) -> Result<(), String> { + manager.cancel(&session_id).await +} + +#[tauri::command] +#[cfg(target_os = "macos")] +pub async fn stt_request_microphone_permission(app: AppHandle) -> Result<(), String> { + crate::services::stt::macos::request_microphone_permission(app).await +} + +#[tauri::command] +#[cfg(not(target_os = "macos"))] +pub async fn stt_request_microphone_permission(_app: AppHandle) -> Result<(), String> { + Ok(()) +} + +#[derive(Serialize)] +pub struct SttTestResponse { + result: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, +} + +fn classify_test_error(message: &str) -> &'static str { + let lower = message.to_ascii_lowercase(); + if lower.contains("no valid speeches") + || lower.contains("no speech") + || lower.contains("未检测到有效语音") + || lower.contains("未发现有效语音") + || lower.contains("3301") + || lower.contains("-3005") + || lower.contains("1013") + { + "connected_no_speech" + } else if lower.starts_with("timeout:") + || lower.contains("超时") + || lower.contains("timeout") + || lower.contains("deadline") + { + "timeout" + } else if lower.starts_with("authentication_failed:") + || lower.contains("authentication") + || lower.contains("鉴权") + || lower.contains("认证") + || lower.contains("api key") + || lower.contains("appkey") + || lower.contains("token") + || lower.contains("appid") + || lower.contains("unauthorized") + || lower.contains("forbidden") + || lower.contains("http error: 401") + || lower.contains("http error: 403") + { + "authentication_failed" + } else if lower.starts_with("network_failed:") + || lower.contains("网络") + || lower.contains("network") + || lower.contains("dns") + || lower.contains("connection refused") + { + "network_failed" + } else { + "protocol_failed" + } +} + +fn safe_test_message(message: &str) -> String { + static URL: LazyLock = LazyLock::new(|| { + regex::Regex::new(r"(?i)\b(?:wss?|https?)://\S+").expect("valid STT URL redaction regex") + }); + URL.replace_all(message, "[provider endpoint]") + .chars() + .take(240) + .collect() +} + +fn connection_test_requires_audio(provider: &str) -> bool { + provider == "baidu_cloud" +} + +#[tauri::command] +pub async fn settings_test_stt( + app: AppHandle, + manager: State<'_, Arc>, + provider: String, +) -> Result { + test_stt_provider(app, manager.inner(), provider).await +} + +async fn test_stt_provider( + app: AppHandle, + manager: &SttManager, + provider: String, +) -> Result { + let requires_audio_probe = connection_test_requires_audio(&provider); + let session_id = format!("test-{}", uuid::Uuid::new_v4()); + let (events_tx, mut events_rx) = mpsc::unbounded_channel(); + manager + .start_observed(app, session_id.clone(), provider, Some(events_tx)) + .await?; + let (result, message) = async { + let mut ready = false; + let mut finish_sent = false; + let mut sequence = 0_u32; + let mut recognized = false; + let ready_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + let mut finish_deadline = None; + let mut interval = tokio::time::interval(std::time::Duration::from_millis(100)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + let phase_deadline = if !ready { + Some(ready_deadline) + } else if finish_sent { + finish_deadline + } else { + None + }; + tokio::select! { + biased; + event = events_rx.recv() => match event { + Some(SttEvent::Error { message, .. }) => { + let result = classify_test_error(&message); + return if result == "connected_no_speech" && finish_sent { + (result, None) + } else { + (result, Some(safe_test_message(&message))) + }; + } + Some(SttEvent::Ready { .. }) if !ready => { + ready = true; + // PlanForDesktop treats a provider-specific ready state as + // sufficient proof that configuration and authentication work. + // Baidu has no server READY event, so its adapter emits ready + // after START and still needs the synthetic-audio probe below. + if !requires_audio_probe { + let _ = manager.cancel(&session_id).await; + return ("connected", None); + } + } + Some(SttEvent::Partial { text, .. } | SttEvent::Final { text, .. }) => { + recognized |= !text.trim().is_empty(); + } + Some(SttEvent::Closed { .. }) if ready && finish_sent => { + return if recognized { ("connected", None) } else { ("connected_no_speech", None) }; + } + Some(SttEvent::Closed { .. }) => { + return ( + "protocol_failed", + Some("供应商在完成连接测试前关闭了会话".to_string()), + ); + } + Some(_) => {} + None => { + return ( + "protocol_failed", + Some("供应商连接测试未返回结果".to_string()), + ); + } + }, + _ = interval.tick(), if ready && !finish_sent => { + let result = if sequence < 10 { + let current = sequence; + sequence += 1; + manager.send( + &session_id, + SttCommand::Audio { sequence: current, pcm: vec![0; 3_200] }, + ).await + } else { + finish_sent = true; + finish_deadline = Some( + tokio::time::Instant::now() + std::time::Duration::from_secs(5), + ); + manager.send(&session_id, SttCommand::Finish).await + }; + if let Err(message) = result { + let result = classify_test_error(&message); + return (result, Some(safe_test_message(&message))); + } + }, + _ = tokio::time::sleep_until( + phase_deadline.unwrap_or( + ready_deadline + std::time::Duration::from_secs(60), + ), + ), if phase_deadline.is_some() => { + let message = if ready { + "供应商已连接，但等待结束确认超时" + } else { + "等待供应商进入可发送音频状态超时" + }; + return ("timeout", Some(message.to_string())); + }, + } + } + } + .await; + if result == "timeout" { + let _ = manager.cancel(&session_id).await; + } + Ok(SttTestResponse { result, message }) +} + +#[cfg(target_os = "macos")] +mod macos { + use block2::RcBlock; + use objc2::runtime::Bool; + use objc2_av_foundation::{AVAuthorizationStatus, AVCaptureDevice, AVMediaTypeAudio}; + use std::sync::{Arc, Mutex}; + use tauri::AppHandle; + + pub async fn request_microphone_permission(app: AppHandle) -> Result<(), String> { + let (sender, receiver) = tokio::sync::oneshot::channel(); + let sender_cell = Arc::new(Mutex::new(Some(sender))); + let sender_for_main = sender_cell.clone(); + app.run_on_main_thread(move || { + let send_result = |result| { + if let Ok(mut sender) = sender_for_main.lock() { + if let Some(sender) = sender.take() { + let _ = sender.send(result); + } + } + }; + let media_type = unsafe { AVMediaTypeAudio.as_ref() }; + let Some(media_type) = media_type else { + send_result(Err("macOS 音频媒体类型不可用".to_string())); + return; + }; + let status = unsafe { AVCaptureDevice::authorizationStatusForMediaType(media_type) }; + match status { + AVAuthorizationStatus::Authorized => { + send_result(Ok(())); + } + AVAuthorizationStatus::Denied => { + send_result(Err("麦克风权限已拒绝，请在系统设置中允许访问".to_string())); + } + AVAuthorizationStatus::Restricted => { + send_result(Err("麦克风权限受系统限制".to_string())); + } + AVAuthorizationStatus::NotDetermined => { + let sender_for_callback = sender_cell.clone(); + let callback = RcBlock::new(move |granted: Bool| { + let result = if granted.as_bool() { + Ok(()) + } else { + Err("麦克风权限被拒绝".to_string()) + }; + if let Ok(mut sender) = sender_for_callback.lock() { + if let Some(sender) = sender.take() { + let _ = sender.send(result); + } + } + }); + unsafe { + AVCaptureDevice::requestAccessForMediaType_completionHandler( + media_type, &callback, + ) + }; + } + _ => { + send_result(Err("未知的 macOS 麦克风权限状态".to_string())); + } + } + }) + .map_err(|error| format!("无法调度 macOS 麦克风权限请求: {error}"))?; + receiver + .await + .map_err(|_| "macOS 麦克风权限请求未返回".to_string())? + } +} + +fn text(config: &serde_json::Map, key: &str) -> String { + config + .get(key) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .trim() + .to_string() +} + +fn websocket_endpoint( + config: &serde_json::Map, + fallback: &str, +) -> Result { + let configured = text(config, "websocketUrl"); + let endpoint = if configured.is_empty() { + fallback.to_string() + } else { + configured + }; + let request = endpoint + .as_str() + .into_client_request() + .map_err(|_| "STT WebSocket 地址无效".to_string())?; + let has_user_info = request + .uri() + .authority() + .is_some_and(|authority| authority.as_str().contains('@')); + if request.uri().scheme_str() != Some("wss") || request.uri().host().is_none() || has_user_info + { + return Err("STT WebSocket 地址必须是完整的 wss:// 地址".to_string()); + } + Ok(endpoint) +} + +fn provider_failure(provider: &str, code: &str, message: &str) -> String { + let detail = match (code.trim(), message.trim()) { + ("", "") => "供应商拒绝了请求".to_string(), + ("", message) => message.to_string(), + (code, "") => code.to_string(), + (code, message) => format!("{code}: {message}"), + }; + let lower = detail.to_ascii_lowercase(); + let category = if [ + "auth", + "unauthor", + "forbidden", + "api key", + "access key", + "token", + "signature", + "secret", + "鉴权", + ] + .iter() + .any(|needle| lower.contains(needle)) + { + "authentication_failed" + } else { + "protocol_failed" + }; + format!("{category}: {provider} 拒绝了请求（{detail}）") +} + +/// Keep the provider and lifecycle stage visible in the UI while leaving +/// credentials to the manager's existing redaction pass. +pub(crate) fn stage_failure(provider: &str, stage: &str, message: impl AsRef) -> String { + format!("[{provider}/{stage}] {}", message.as_ref().trim()) +} + +pub(crate) const PROVIDER_WRITE_TIMEOUT: Duration = Duration::from_secs(10); + +pub(crate) async fn send_provider_message( + write: &mut W, + message: tokio_tungstenite::tungstenite::Message, + provider: &str, + stage: &str, +) -> Result<(), String> +where + W: futures_util::Sink + Unpin, + W::Error: std::fmt::Display, +{ + use futures_util::SinkExt; + tokio::time::timeout(PROVIDER_WRITE_TIMEOUT, write.send(message)) + .await + .map_err(|_| stage_failure(provider, stage, "写入供应商超时"))? + .map_err(|error| stage_failure(provider, stage, error.to_string())) +} + +pub(crate) async fn close_provider_socket(write: &mut W) +where + W: futures_util::Sink + Unpin, +{ + use futures_util::SinkExt; + let _ = tokio::time::timeout(PROVIDER_WRITE_TIMEOUT, write.close()).await; +} + +pub(crate) fn sanitize_error( + message: &str, + config: &serde_json::Map, +) -> String { + [ + "apiKey", + "secretId", + "secretKey", + "accessToken", + "baiduApiKey", + ] + .into_iter() + .fold(message.to_string(), |sanitized, field| { + let secret = config + .get(field) + .and_then(|value| value.as_str()) + .filter(|value| !value.is_empty()); + match secret { + Some(secret) => sanitized.replace(secret, "[redacted]"), + None => sanitized, + } + }) +} + +fn emit(app: &AppHandle, event: SttEvent) { + SttManager::emit(app, event); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn live_audio_pcm() -> Vec { + let Ok(path) = std::env::var("LIVEAGENT_STT_LIVE_AUDIO_FILE") else { + return vec![0; 30_720]; + }; + let wave = std::fs::read(path).expect("read live STT WAVE fixture"); + assert!(wave.starts_with(b"RIFF") && wave.get(8..12) == Some(b"WAVE")); + let mut offset = 12; + while offset + 8 <= wave.len() { + let chunk_id = &wave[offset..offset + 4]; + let chunk_len = u32::from_le_bytes( + wave[offset + 4..offset + 8] + .try_into() + .expect("WAVE chunk length"), + ) as usize; + let start = offset + 8; + let end = start.checked_add(chunk_len).expect("WAVE chunk overflow"); + assert!(end <= wave.len(), "WAVE chunk exceeds file length"); + if chunk_id == b"data" { + let mut pcm = wave[start..end].to_vec(); + pcm.truncate(160_000); // Five seconds at 16 kHz mono PCM16. + assert!(!pcm.is_empty() && pcm.len() % 2 == 0); + return pcm; + } + offset = end + (chunk_len % 2); + } + panic!("live STT WAVE fixture has no data chunk"); + } + + #[test] + fn runtime_events_use_frontend_session_id_field() { + let events = [ + SttEvent::Ready { + session_id: "session-1".to_string(), + }, + SttEvent::Partial { + session_id: "session-1".to_string(), + text: "partial".to_string(), + }, + SttEvent::Final { + session_id: "session-1".to_string(), + text: "final".to_string(), + }, + SttEvent::Error { + session_id: "session-1".to_string(), + code: "protocol_failed".to_string(), + message: "failure".to_string(), + }, + SttEvent::Closed { + session_id: "session-1".to_string(), + }, + ]; + for event in events { + let value = serde_json::to_value(event).expect("serialize STT runtime event"); + assert_eq!( + value.get("sessionId").and_then(|value| value.as_str()), + Some("session-1") + ); + assert!(value.get("session_id").is_none()); + } + } + + #[test] + fn connection_test_preserves_adapter_error_categories() { + assert_eq!( + classify_test_error("authentication_failed: invalid appkey"), + "authentication_failed" + ); + assert_eq!( + classify_test_error("network_failed: connection refused"), + "network_failed" + ); + assert_eq!( + classify_test_error("protocol_failed: malformed response"), + "protocol_failed" + ); + } + + #[test] + fn only_baidu_connection_test_requires_synthetic_audio() { + assert!(connection_test_requires_audio("baidu_cloud")); + for provider in [ + "aliyun_dashscope", + "tencent_cloud", + "volcengine_v2", + "volcengine_seed_v3", + ] { + assert!(!connection_test_requires_audio(provider)); + } + } + + #[test] + fn connection_test_diagnostic_redacts_provider_urls() { + let diagnostic = safe_test_message( + "request wss://example.invalid/asr?secretid=id&signature=signature failed", + ); + assert_eq!(diagnostic, "request [provider endpoint] failed"); + assert!(!diagnostic.contains("signature")); + } + + #[test] + fn stt_installs_tls_crypto_without_a_gateway_connection() { + ensure_stt_crypto_provider(); + assert!(rustls::crypto::CryptoProvider::get_default().is_some()); + } + + #[tokio::test] + async fn configured_desktop_providers_live() { + if std::env::var("LIVEAGENT_STT_LIVE").as_deref() != Ok("1") { + return; + } + let app = tauri::test::mock_app(); + let manager = SttManager::default(); + let mut tested = 0; + for provider in [ + "aliyun_dashscope", + "tencent_cloud", + "volcengine_v2", + "volcengine_seed_v3", + "baidu_cloud", + ] { + if crate::commands::settings::load_stt_provider_runtime(provider).is_err() { + continue; + } + tested += 1; + let response = test_stt_provider(app.handle().clone(), &manager, provider.to_string()) + .await + .expect("run live desktop STT probe"); + eprintln!( + "{provider}: result={} message={}", + response.result, + response.message.as_deref().unwrap_or("") + ); + assert!( + matches!(response.result, "connected" | "connected_no_speech"), + "{provider} live probe failed: {:?}", + response.message + ); + } + assert!( + tested > 0, + "desktop STT settings contain no configured providers" + ); + } + + #[tokio::test] + async fn configured_desktop_providers_audio_roundtrip_live() { + if std::env::var("LIVEAGENT_STT_LIVE_AUDIO").as_deref() != Ok("1") { + return; + } + let app = tauri::test::mock_app(); + let manager = SttManager::default(); + let pcm = live_audio_pcm(); + let require_transcript = std::env::var_os("LIVEAGENT_STT_LIVE_AUDIO_FILE").is_some(); + let mut tested = 0; + for provider in [ + "aliyun_dashscope", + "tencent_cloud", + "volcengine_v2", + "volcengine_seed_v3", + "baidu_cloud", + ] { + if crate::commands::settings::load_stt_provider_runtime(provider).is_err() { + continue; + } + tested += 1; + let session_id = format!("live-audio-{}", uuid::Uuid::new_v4()); + let (events_tx, mut events_rx) = mpsc::unbounded_channel(); + manager + .start_observed( + app.handle().clone(), + session_id.clone(), + provider.to_string(), + Some(events_tx), + ) + .await + .expect("start live STT audio roundtrip"); + let outcome = tokio::time::timeout(std::time::Duration::from_secs(20), async { + let mut sent = false; + let mut recognized = false; + while let Some(event) = events_rx.recv().await { + match event { + SttEvent::Ready { .. } if !sent => { + sent = true; + let chunks = pcm.chunks(3_200).collect::>(); + for (sequence, chunk) in chunks.iter().enumerate() { + if let Err(send_error) = manager + .send( + &session_id, + SttCommand::Audio { + sequence: sequence as u32, + pcm: chunk.to_vec(), + }, + ) + .await + { + let mut detail = send_error; + while let Some(event) = events_rx.recv().await { + match event { + SttEvent::Error { message, .. } => { + detail.push_str("; provider: "); + detail.push_str(&message); + } + SttEvent::Closed { .. } => return Err(detail), + _ => {} + } + } + return Err(detail); + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + manager + .send( + &session_id, + SttCommand::Audio { + sequence: chunks.len() as u32, + pcm: vec![0; 12_800], + }, + ) + .await + .expect("send live STT trailing silence"); + manager + .send(&session_id, SttCommand::Finish) + .await + .expect("finish live STT audio roundtrip"); + } + SttEvent::Partial { ref text, .. } | SttEvent::Final { ref text, .. } => { + recognized |= !text.trim().is_empty(); + } + SttEvent::Error { message, .. } => return Err(message), + SttEvent::Closed { .. } if sent => return Ok(recognized), + SttEvent::Closed { .. } => { + return Err("provider closed before ready".to_string()); + } + _ => {} + } + } + Err("provider event stream ended before closed".to_string()) + }) + .await; + match outcome { + Ok(Ok(recognized)) if require_transcript && !recognized => { + panic!("{provider} audio roundtrip returned no transcript") + } + Ok(Ok(recognized)) => { + eprintln!("{provider}: audio_roundtrip=connected transcript={recognized}") + } + Ok(Err(message)) => panic!("{provider} audio roundtrip failed: {message}"), + Err(_) => { + let _ = manager.cancel(&session_id).await; + panic!("{provider} audio roundtrip timed out"); + } + } + } + assert!( + tested > 0, + "desktop STT settings contain no configured providers" + ); + } +} diff --git a/crates/agent-gui/src-tauri/src/services/stt/tencent_cloud.rs b/crates/agent-gui/src-tauri/src/services/stt/tencent_cloud.rs new file mode 100644 index 000000000..40d5ce9fd --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/stt/tencent_cloud.rs @@ -0,0 +1,183 @@ +use super::{ + close_provider_socket, emit, provider_failure, send_provider_message, stage_failure, text, + SttCommand, SttEvent, +}; +use futures_util::StreamExt; +use hmac::{Hmac, Mac}; +use serde_json::Value; +use sha1::Sha1; +use tauri::{AppHandle, Runtime}; +use tokio::sync::mpsc::Receiver; +use tokio_tungstenite::{ + connect_async, + tungstenite::{protocol::frame::coding::CloseCode, Message}, +}; + +type HmacSha1 = Hmac; + +fn numeric_nonce() -> String { + let bytes = uuid::Uuid::new_v4().into_bytes(); + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + .max(1) + .to_string() +} + +fn signed_url(config: &serde_json::Map, voice_id: &str) -> Result { + let app_id = text(config, "appId"); + let secret_id = text(config, "secretId"); + let secret_key = text(config, "secretKey"); + let path = format!("asr.cloud.tencent.com/asr/v2/{app_id}"); + let timestamp = chrono::Utc::now().timestamp(); + let mut parameters = std::collections::BTreeMap::from([ + ("convert_num_mode", "1".to_string()), + ("engine_model_type", text(config, "engineModelType")), + ("expired", (timestamp + 24 * 60 * 60).to_string()), + ("filter_dirty", "1".to_string()), + ("filter_modal", "2".to_string()), + ("filter_punc", "0".to_string()), + ("needvad", "0".to_string()), + ("nonce", numeric_nonce()), + ("secretid", secret_id), + ("timestamp", timestamp.to_string()), + ("voice_format", "1".to_string()), + ("voice_id", voice_id.to_string()), + ("word_info", "0".to_string()), + ]); + let query = parameters + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join("&"); + let sign_text = format!("{path}?{query}"); + let mut mac = HmacSha1::new_from_slice(secret_key.as_bytes()).map_err(|_| "腾讯签名失败")?; + mac.update(sign_text.as_bytes()); + let signature = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + mac.finalize().into_bytes(), + ); + parameters.insert("signature", signature); + let encoded = parameters + .iter() + .map(|(key, value)| format!("{key}={}", urlencoding::encode(value))) + .collect::>() + .join("&"); + Ok(format!("wss://{path}?{encoded}")) +} + +pub async fn run( + app: AppHandle, + session: String, + config: serde_json::Map, + mut rx: Receiver, +) -> Result<(), String> { + let voice_id = uuid::Uuid::new_v4().to_string(); + let endpoint = + signed_url(&config, &voice_id).map_err(|e| stage_failure("Tencent", "validate", e))?; + let (socket, _) = connect_async(endpoint) + .await + .map_err(|e| stage_failure("Tencent", "connect", format!("网络错误: {e}")))?; + let (mut write, mut read) = socket.split(); + let mut fragments = std::collections::BTreeMap::::new(); + let mut finish_sent = false; + emit( + &app, + SttEvent::Ready { + session_id: session.clone(), + }, + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + loop { + tokio::select! { + Some(command) = rx.recv() => match command { + SttCommand::Audio { pcm, .. } => { if !finish_sent { send_provider_message(&mut write, Message::Binary(pcm.into()), "Tencent", "send_audio").await?; } } + SttCommand::Finish => { if !finish_sent { finish_sent = true; send_provider_message(&mut write, Message::Text(serde_json::json!({"type":"end"}).to_string().into()), "Tencent", "finish").await?; } } + SttCommand::Cancel => { close_provider_socket(&mut write).await; return Ok(()); } + }, + Some(message) = read.next() => { + let message = message.map_err(|e| stage_failure("Tencent", "receive", e.to_string()))?; + match message { + Message::Text(body) => { + let value: Value = serde_json::from_str(&body).map_err(|_| stage_failure("Tencent", "parse", "返回内容不是有效 JSON"))?; + if let Some(code) = value.get("code").and_then(Value::as_i64).filter(|code| *code != 0) { + return Err(stage_failure("Tencent", "provider_response", provider_failure( + "腾讯云", + &code.to_string(), + value.get("message").and_then(Value::as_str).unwrap_or_default(), + ))); + } + let index = value.pointer("/result/index").and_then(Value::as_i64).unwrap_or(0); + if let Some(text_value) = value.pointer("/result/voice_text_str").and_then(Value::as_str) { + fragments.insert(index, text_value.to_string()); + let merged = fragments.values().cloned().collect::(); + emit(&app, SttEvent::Partial { session_id: session.clone(), text: merged }); + } + let completed = value.get("type").and_then(Value::as_str) == Some("end") + || value.get("final").and_then(Value::as_i64) == Some(1); + if completed && finish_sent { + let final_text = fragments.values().cloned().collect::(); + if !final_text.is_empty() { + emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text }); + } + close_provider_socket(&mut write).await; + return Ok(()); + } + } + Message::Close(frame) if finish_sent && frame.as_ref().is_none_or(|frame| frame.code == CloseCode::Normal) => { + let final_text = fragments.values().cloned().collect::(); + if !final_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text }); } + return Ok(()); + } + Message::Close(_) => return Err(stage_failure("Tencent", "close", "连接在结束协议前异常关闭")), + _ => {} + } + } + else => { + if finish_sent { + let final_text = fragments.values().cloned().collect::(); + if !final_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: final_text }); } + return Ok(()); + } + return Err(stage_failure("Tencent", "close", "连接在结束协议前关闭")); + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn signed_url_contains_full_realtime_asr_parameter_set() { + let config = serde_json::from_value(json!({ + "appId": "123", + "secretId": "fixture-id", + "secretKey": "fixture-key", + "engineModelType": "16k_zh" + })) + .expect("config map"); + let endpoint = signed_url(&config, "voice-id").expect("signed URL"); + for field in [ + "convert_num_mode=1", + "engine_model_type=16k_zh", + "filter_dirty=1", + "filter_modal=2", + "filter_punc=0", + "needvad=0", + "voice_format=1", + "voice_id=voice-id", + "word_info=0", + "signature=", + ] { + assert!(endpoint.contains(field), "missing Tencent field {field}"); + } + let parsed = reqwest::Url::parse(&endpoint).expect("parse signed URL"); + let nonce = parsed + .query_pairs() + .find_map(|(key, value)| (key == "nonce").then(|| value.into_owned())) + .expect("Tencent nonce"); + assert!(nonce.parse::().is_ok(), "nonce must be decimal"); + assert_ne!(nonce, "0"); + } +} diff --git a/crates/agent-gui/src-tauri/src/services/stt/volcengine_seed_v3.rs b/crates/agent-gui/src-tauri/src/services/stt/volcengine_seed_v3.rs new file mode 100644 index 000000000..526b1bf02 --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/stt/volcengine_seed_v3.rs @@ -0,0 +1,333 @@ +use super::{ + close_provider_socket, emit, provider_failure, send_provider_message, stage_failure, text, + websocket_endpoint, SttCommand, SttEvent, +}; +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use futures_util::StreamExt; +use serde_json::Value; +use std::io::{Read, Write}; +use tauri::{AppHandle, Runtime}; +use tokio::sync::mpsc::Receiver; +use tokio_tungstenite::{ + connect_async, + tungstenite::{ + client::IntoClientRequest, http::Request, protocol::frame::coding::CloseCode, + Error as WebSocketError, Message, + }, +}; + +const SEED_V3_ENDPOINT: &str = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"; + +fn seed_v3_connect_failure(error: WebSocketError) -> String { + match error { + WebSocketError::Http(response) => { + let status = response.status(); + let category = if status.as_u16() == 401 || status.as_u16() == 403 { + "authentication_failed" + } else if status.is_server_error() { + "network_failed" + } else { + "protocol_failed" + }; + format!( + "{category}: 火山 Seed v3 WebSocket 握手失败（HTTP {}）", + status.as_u16() + ) + } + other => format!("network_failed: {other}"), + } +} + +fn seed_v3_websocket_request( + config: &serde_json::Map, + connect_id: &str, +) -> Result, String> { + let endpoint = websocket_endpoint(config, SEED_V3_ENDPOINT) + .map_err(|error| stage_failure("VolcengineSeedV3", "validate", error))?; + let mut request = endpoint + .into_client_request() + .map_err(|_| stage_failure("VolcengineSeedV3", "validate", "WebSocket 地址无效"))?; + for (name, value, label) in [ + ("X-Api-App-Key", text(config, "appId"), "App ID"), + ( + "X-Api-Access-Key", + text(config, "accessToken"), + "Access Token", + ), + ( + "X-Api-Resource-Id", + text(config, "resourceId"), + "Resource ID", + ), + ("X-Api-Connect-Id", connect_id.to_string(), "连接 ID"), + ] { + request.headers_mut().insert( + name, + value.parse().map_err(|_| { + stage_failure("VolcengineSeedV3", "connect", format!("{label} 无效")) + })?, + ); + } + Ok(request) +} + +fn seed_v3_json(value: Value) -> Result, String> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(value.to_string().as_bytes()) + .map_err(|e| stage_failure("VolcengineSeedV3", "connect", e.to_string()))?; + encoder.finish().map_err(|e| e.to_string()) +} + +fn seed_v3_start_request(session: &str) -> Value { + serde_json::json!({ + "user": {"uid": session}, + "audio": {"format":"pcm", "codec":"raw", "rate":16000, "bits":16, "channel":1}, + "request": { + "model_name":"bigmodel", + "enable_itn":true, + "enable_punc":true, + "enable_ddc":true, + "show_utterances":true, + "result_type":"full" + } + }) +} +fn seed_v3_audio(payload: &[u8]) -> Result, String> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(payload).map_err(|e| e.to_string())?; + encoder.finish().map_err(|e| e.to_string()) +} +fn seed_v3_frame(message_type: u8, flags: u8, serialization: u8, payload: &[u8]) -> Vec { + let mut output = vec![ + 0x11, + (message_type << 4) | flags, + (serialization << 4) | 0x01, + 0, + ]; + output.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + output.extend_from_slice(payload); + output +} +fn seed_v3_audio_frame(last: bool, payload: &[u8]) -> Vec { + // bigmodel_async assigns request sequences server-side. Audio-only client + // frames therefore omit the optional sequence field entirely. + seed_v3_frame(2, if last { 2 } else { 0 }, 0, payload) +} +fn decode_seed_v3_frame(data: &[u8]) -> Result { + if data.len() < 8 || data[0] & 0x0f < 1 { + return Err("火山 Seed v3 帧无效".into()); + } + let header_len = (data[0] & 0x0f) as usize * 4; + if data.len() < header_len + 4 { + return Err("火山 Seed v3 帧头无效".into()); + } + let message_type = data[1] >> 4; + let flags = data[1] & 0x0f; + let mut offset = header_len; + let mut error_code = 0_u32; + let mut sequence = 0_i32; + if message_type == 0x0f { + if data.len() < offset + 8 { + return Err("火山 Seed v3 错误帧无效".into()); + } + error_code = u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()); + offset += 4; + } else if flags & 1 != 0 { + if data.len() < offset + 8 { + return Err("火山 Seed v3 序列帧无效".into()); + } + sequence = i32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()); + offset += 4; + } + let payload_len = u32::from_be_bytes( + data[offset..offset + 4] + .try_into() + .map_err(|_| "火山 Seed v3 负载长度无效")?, + ) as usize; + if data.len() < offset + 4 + payload_len { + return Err("火山 Seed v3 负载长度无效".into()); + } + let mut payload = data[offset + 4..offset + 4 + payload_len].to_vec(); + if data[2] & 0x0f == 1 { + let mut decoder = GzDecoder::new(payload.as_slice()); + let mut decoded = Vec::new(); + decoder + .read_to_end(&mut decoded) + .map_err(|_| "火山 Seed v3 gzip 负载无效")?; + payload = decoded; + } + if message_type == 0x0f { + return Ok( + serde_json::json!({"code": error_code, "message": String::from_utf8_lossy(&payload)}), + ); + } + let mut value: Value = + serde_json::from_slice(&payload).map_err(|_| "火山 Seed v3 响应协议错误")?; + if let Some(object) = value.as_object_mut() { + object.insert("_sequence".into(), Value::from(sequence)); + object.insert( + "_last".into(), + Value::from(flags == 2 || flags == 3 || sequence < 0), + ); + } + Ok(value) +} + +pub async fn run( + app: AppHandle, + session: String, + config: serde_json::Map, + mut rx: Receiver, +) -> Result<(), String> { + let connect_id = uuid::Uuid::new_v4().to_string(); + let request = seed_v3_websocket_request(&config, &connect_id)?; + let (socket, _) = connect_async(request) + .await + .map_err(|e| stage_failure("VolcengineSeedV3", "connect", seed_v3_connect_failure(e)))?; + let (mut write, mut read) = socket.split(); + let start = seed_v3_json(seed_v3_start_request(&session))?; + send_provider_message( + &mut write, + Message::Binary(seed_v3_frame(1, 0, 1, &start).into()), + "VolcengineSeedV3", + "start", + ) + .await?; + // The WebSocket upgrade only validates the transport. Wait for the first + // successful v3 binary response before releasing buffered microphone data. + let mut ready = false; + let mut finishing = false; + let mut finish_sent = false; + let mut pending = Vec::<(u32, Vec)>::new(); + let mut held_audio = None::<(u32, Vec)>; + let mut last_text = String::new(); + loop { + tokio::select! { + Some(command) = rx.recv() => match command { + SttCommand::Audio { sequence, pcm } => { + if ready { + if let Some((_previous_sequence, previous_pcm)) = held_audio.replace((sequence, pcm)) { + let compressed = seed_v3_audio(&previous_pcm)?; + let frame = seed_v3_audio_frame(false, &compressed); + send_provider_message(&mut write, Message::Binary(frame.into()), "VolcengineSeedV3", "send_audio").await?; + } + } else { + pending.push((sequence, pcm)); + } + } + SttCommand::Finish => { + finishing = true; + if ready && !finish_sent { + let (_sequence, pcm) = held_audio.take().unwrap_or_default(); + let compressed = seed_v3_audio(&pcm)?; + let frame = seed_v3_audio_frame(true, &compressed); + send_provider_message(&mut write, Message::Binary(frame.into()), "VolcengineSeedV3", "finish").await?; + finish_sent = true; + } + } + SttCommand::Cancel => { close_provider_socket(&mut write).await; return Ok(()); } + }, + Some(message) = read.next() => { let message = message.map_err(|e| stage_failure("VolcengineSeedV3", "receive", e.to_string()))?; let body = match message { Message::Binary(body) => body, Message::Close(frame) if finishing && finish_sent && frame.as_ref().is_none_or(|frame| frame.code == CloseCode::Normal) => { if !last_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: last_text.clone() }); } return Ok(()); }, Message::Close(_) => return Err(stage_failure("VolcengineSeedV3", "close", "连接异常关闭")), _ => continue }; let value = decode_seed_v3_frame(&body).map_err(|e| stage_failure("VolcengineSeedV3", "parse", e))?; if let Some(error) = value.get("error").and_then(Value::as_str) { return Err(stage_failure("VolcengineSeedV3", "provider_response", provider_failure("火山 Seed v3", &value.get("code").and_then(Value::as_i64).unwrap_or_default().to_string(), error))); } let code = value.get("code").and_then(Value::as_i64).unwrap_or_default(); if code != 0 && code != 1000 { return Err(stage_failure("VolcengineSeedV3", "provider_response", provider_failure("火山 Seed v3", &code.to_string(), value.get("message").and_then(Value::as_str).unwrap_or_default()))); } if !ready { ready = true; emit(&app, SttEvent::Ready { session_id: session.clone() }); for (sequence, pcm) in pending.drain(..) { if let Some((_previous_sequence, previous_pcm)) = held_audio.replace((sequence, pcm)) { let compressed = seed_v3_audio(&previous_pcm).map_err(|e| stage_failure("VolcengineSeedV3", "send_audio", e))?; let frame = seed_v3_audio_frame(false, &compressed); send_provider_message(&mut write, Message::Binary(frame.into()), "VolcengineSeedV3", "send_audio").await?; } } if finishing && !finish_sent { let (_sequence, pcm) = held_audio.take().unwrap_or_default(); let compressed = seed_v3_audio(&pcm).map_err(|e| stage_failure("VolcengineSeedV3", "finish", e))?; let frame = seed_v3_audio_frame(true, &compressed); send_provider_message(&mut write, Message::Binary(frame.into()), "VolcengineSeedV3", "finish").await?; finish_sent = true; } } if let Some(result_text) = value.pointer("/result/text").and_then(Value::as_str) { last_text = result_text.to_string(); emit(&app, SttEvent::Partial { session_id: session.clone(), text: last_text.clone() }); } let completed = value.get("is_last_package").and_then(Value::as_bool).unwrap_or(false) || value.get("_last").and_then(Value::as_bool).unwrap_or(false); if finishing && completed { if !last_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: last_text.clone() }); } close_provider_socket(&mut write).await; return Ok(()); } }, + else => return if finishing && finish_sent { if !last_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: last_text.clone() }); } Ok(()) } else { Err(stage_failure("VolcengineSeedV3", "close", "连接意外关闭")) }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn websocket_request_uses_seed_v3_connection_headers() { + let config = serde_json::from_value(json!({ + "websocketUrl": "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async", + "appId": "app-id", + "accessToken": "access-token", + "resourceId": "resource-id" + })) + .expect("config map"); + let request = seed_v3_websocket_request(&config, "connect-id").expect("v3 request"); + assert_eq!( + request.uri().to_string(), + "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async" + ); + assert_eq!(request.headers()["X-Api-App-Key"], "app-id"); + assert_eq!(request.headers()["X-Api-Access-Key"], "access-token"); + assert_eq!(request.headers()["X-Api-Resource-Id"], "resource-id"); + assert_eq!(request.headers()["X-Api-Connect-Id"], "connect-id"); + assert!(!request.headers().contains_key("X-Api-Request-Id")); + } + + #[test] + fn websocket_handshake_errors_keep_their_result_category() { + for (status, expected) in [ + (400, "protocol_failed:"), + (401, "authentication_failed:"), + (403, "authentication_failed:"), + (500, "network_failed:"), + (503, "network_failed:"), + ] { + let response = tokio_tungstenite::tungstenite::http::Response::builder() + .status(status) + .body(None) + .expect("HTTP response"); + let failure = seed_v3_connect_failure(WebSocketError::Http(Box::new(response))); + assert!( + failure.starts_with(expected), + "HTTP {status} classified as {failure:?}" + ); + assert!(!failure.contains("access-token")); + } + + let failure = seed_v3_connect_failure(WebSocketError::ConnectionClosed); + assert!(failure.starts_with("network_failed:")); + } + + #[test] + fn websocket_request_uses_configured_endpoint() { + let config = serde_json::from_value(json!({ + "websocketUrl": "wss://example.com/custom-v3", + "appId": "app-id", + "accessToken": "access-token", + "resourceId": "resource-id" + })) + .expect("config map"); + let request = seed_v3_websocket_request(&config, "connect-id").expect("v3 request"); + assert_eq!(request.uri().to_string(), "wss://example.com/custom-v3"); + } + + #[test] + fn audio_frames_rely_on_server_assigned_sequences() { + let audio = seed_v3_audio_frame(false, &[1, 2]); + assert_eq!(&audio[..4], &[0x11, 0x20, 0x01, 0x00]); + assert_eq!(&audio[4..8], &2_u32.to_be_bytes()); + assert_eq!(&audio[8..], &[1, 2]); + let finish = seed_v3_audio_frame(true, &[3, 4]); + assert_eq!(&finish[..4], &[0x11, 0x22, 0x01, 0x00]); + assert_eq!(&finish[4..8], &2_u32.to_be_bytes()); + assert_eq!(&finish[8..], &[3, 4]); + let start = seed_v3_frame(1, 0, 1, &[1, 2]); + assert_eq!(&start[..4], &[0x11, 0x10, 0x11, 0x00]); + } + + #[test] + fn start_request_uses_seed_v3_full_result_options() { + let request = seed_v3_start_request("fixture-session"); + assert_eq!(request.pointer("/audio/format"), Some(&json!("pcm"))); + assert_eq!(request.pointer("/audio/rate"), Some(&json!(16000))); + assert_eq!( + request.pointer("/request/model_name"), + Some(&json!("bigmodel")) + ); + assert_eq!( + request.pointer("/request/show_utterances"), + Some(&json!(true)) + ); + assert_eq!( + request.pointer("/request/result_type"), + Some(&json!("full")) + ); + } +} diff --git a/crates/agent-gui/src-tauri/src/services/stt/volcengine_v2.rs b/crates/agent-gui/src-tauri/src/services/stt/volcengine_v2.rs new file mode 100644 index 000000000..9dcd107db --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/stt/volcengine_v2.rs @@ -0,0 +1,291 @@ +use super::{ + close_provider_socket, emit, provider_failure, send_provider_message, stage_failure, text, + websocket_endpoint, SttCommand, SttEvent, +}; +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use futures_util::StreamExt; +use serde_json::Value; +use std::io::{Read, Write}; +use tauri::{AppHandle, Runtime}; +use tokio::sync::mpsc::Receiver; +use tokio_tungstenite::{ + connect_async, + tungstenite::{ + client::IntoClientRequest, http::header, protocol::frame::coding::CloseCode, Message, + }, +}; + +fn gzip_json(value: Value) -> Result, String> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(value.to_string().as_bytes()) + .map_err(|e| e.to_string())?; + encoder.finish().map_err(|e| e.to_string()) +} + +fn gzip_bytes(payload: &[u8]) -> Result, String> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(payload).map_err(|e| e.to_string())?; + encoder.finish().map_err(|e| e.to_string()) +} + +fn frame( + message_type: u8, + flags: u8, + serialization: u8, + compression: u8, + payload: &[u8], +) -> Vec { + let mut output = vec![ + 0x11, + (message_type << 4) | flags, + (serialization << 4) | compression, + 0, + ]; + output.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + output.extend_from_slice(payload); + output +} + +fn decode_frame(data: &[u8]) -> Result { + if data.len() < 8 { + return Err("火山 v2 帧长度无效".into()); + } + let header_len = ((data[0] & 0x0f) as usize) * 4; + if header_len < 4 || data.len() < header_len + 4 { + return Err("火山 v2 帧头无效".into()); + } + let message_type = data[1] >> 4; + let flags = data[1] & 0x0f; + let mut offset = header_len; + let mut error_code = 0_u32; + let mut sequence = 0_i32; + if message_type == 0x0f { + if data.len() < offset + 8 { + return Err("火山 v2 错误帧无效".into()); + } + error_code = u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()); + offset += 4; + } else if message_type == 0x0b || flags & 1 != 0 { + if data.len() < offset + 8 { + return Err("火山 v2 序列帧无效".into()); + } + sequence = i32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()); + offset += 4; + } + let payload_len = u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()) as usize; + if data.len() < offset + 4 + payload_len { + return Err("火山 v2 负载长度无效".into()); + } + let mut payload = data[offset + 4..offset + 4 + payload_len].to_vec(); + if data[2] & 0x0f == 1 { + let mut decoder = GzDecoder::new(payload.as_slice()); + let mut decoded = Vec::new(); + decoder + .read_to_end(&mut decoded) + .map_err(|_| "火山 v2 gzip 负载无效")?; + payload = decoded; + } + if message_type == 0x0f { + return Ok( + serde_json::json!({"code": error_code, "message": String::from_utf8_lossy(&payload)}), + ); + } + let mut value: Value = serde_json::from_slice(&payload).map_err(|_| "火山 v2 响应协议错误")?; + if let Some(object) = value.as_object_mut() { + let payload_sequence = object + .get("sequence") + .and_then(Value::as_i64) + .unwrap_or_default(); + object.insert("_sequence".into(), Value::from(sequence)); + object.insert( + "_last".into(), + Value::from(flags == 2 || flags == 3 || sequence < 0 || payload_sequence < 0), + ); + } + Ok(value) +} + +fn response_is_no_speech(value: &Value) -> bool { + value.get("code").and_then(Value::as_i64) == Some(1013) +} + +fn start_request( + config: &serde_json::Map, + session: &str, + request_id: &str, +) -> Value { + serde_json::json!({ + "app": {"appid": text(config, "appId"), "token": text(config, "accessToken"), "cluster": text(config, "cluster")}, + "user": {"uid": session}, + "audio": {"format": "raw", "rate": 16000, "bits": 16, "channel": 1, "codec": "raw"}, + "request": { + "reqid": request_id, + "nbest": 1, + "workflow": "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate", + "show_utterances": true, + "result_type": "full", + "sequence": 1 + } + }) +} + +pub async fn run( + app: AppHandle, + session: String, + config: serde_json::Map, + mut rx: Receiver, +) -> Result<(), String> { + let token = text(&config, "accessToken"); + let endpoint = websocket_endpoint(&config, "wss://openspeech.bytedance.com/api/v2/asr") + .map_err(|e| stage_failure("VolcengineV2", "validate", e))?; + let mut request = endpoint + .into_client_request() + .map_err(|e| stage_failure("VolcengineV2", "connect", e.to_string()))?; + request.headers_mut().insert( + header::AUTHORIZATION, + format!("Bearer; {token}") + .parse() + .map_err(|_| stage_failure("VolcengineV2", "connect", "Access Token 无效"))?, + ); + let (socket, _) = connect_async(request) + .await + .map_err(|e| stage_failure("VolcengineV2", "connect", format!("网络错误: {e}")))?; + let (mut write, mut read) = socket.split(); + let request_id = uuid::Uuid::new_v4().to_string(); + let start = gzip_json(start_request(&config, &session, &request_id))?; + send_provider_message( + &mut write, + Message::Binary(frame(1, 0, 1, 1, &start).into()), + "VolcengineV2", + "start", + ) + .await?; + + let mut ready = false; + let mut pending = Vec::>::new(); + let mut finishing = false; + let mut finish_sent = false; + let mut last_text = String::new(); + loop { + tokio::select! { + Some(command) = rx.recv() => match command { + SttCommand::Audio { pcm, .. } => { + if ready { let compressed = gzip_bytes(&pcm).map_err(|e| stage_failure("VolcengineV2", "send_audio", e))?; send_provider_message(&mut write, Message::Binary(frame(2, 0, 0, 1, &compressed).into()), "VolcengineV2", "send_audio").await?; } + else { pending.push(pcm); } + } + SttCommand::Finish => { + finishing = true; + if ready && !finish_sent { + let compressed = gzip_bytes(&[]).map_err(|e| stage_failure("VolcengineV2", "finish", e))?; + send_provider_message(&mut write, Message::Binary(frame(2, 2, 0, 1, &compressed).into()), "VolcengineV2", "finish").await?; + finish_sent = true; + } + } + SttCommand::Cancel => { close_provider_socket(&mut write).await; return Ok(()); } + }, + Some(message) = read.next() => { + let message = message.map_err(|e| stage_failure("VolcengineV2", "receive", e.to_string()))?; + let body = match message { + Message::Binary(body) => body, + Message::Close(frame) if finishing && finish_sent && frame.as_ref().is_none_or(|frame| frame.code == CloseCode::Normal) => { + if !last_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: last_text.clone() }); } + return Ok(()); + } + Message::Close(_) => return Err(stage_failure("VolcengineV2", "close", "在结束协议前异常关闭")), + _ => continue, + }; + let value = decode_frame(&body).map_err(|e| stage_failure("VolcengineV2", "parse", e))?; + let code = value.get("code").and_then(Value::as_i64).unwrap_or(0); + let success = code == 0 || code == 1000 || value.get("message").and_then(Value::as_str) == Some("Success"); + if response_is_no_speech(&value) { + close_provider_socket(&mut write).await; + return Ok(()); + } + if !success { return Err(stage_failure("VolcengineV2", "provider_response", provider_failure("火山 v2", &code.to_string(), value.get("message").and_then(Value::as_str).unwrap_or_default()))); } + if !ready { + ready = true; + emit(&app, SttEvent::Ready { session_id: session.clone() }); + for pcm in pending.drain(..) { let compressed = gzip_bytes(&pcm).map_err(|e| stage_failure("VolcengineV2", "send_audio", e))?; send_provider_message(&mut write, Message::Binary(frame(2, 0, 0, 1, &compressed).into()), "VolcengineV2", "send_audio").await?; } + if finishing && !finish_sent { + let compressed = gzip_bytes(&[]).map_err(|e| stage_failure("VolcengineV2", "finish", e))?; + send_provider_message(&mut write, Message::Binary(frame(2, 2, 0, 1, &compressed).into()), "VolcengineV2", "finish").await?; + finish_sent = true; + } + } + let result_text = value.pointer("/result/text").and_then(Value::as_str) + .or_else(|| value.pointer("/result/0/text").and_then(Value::as_str)) + .or_else(|| value.pointer("/payload/result/text").and_then(Value::as_str)) + .or_else(|| value.pointer("/data/result/text").and_then(Value::as_str)); + if let Some(result_text) = result_text { + last_text = result_text.to_string(); + emit(&app, SttEvent::Partial { session_id: session.clone(), text: last_text.clone() }); + } + let completed = value.get("is_last_package").and_then(Value::as_bool).unwrap_or(false) + || value.get("_last").and_then(Value::as_bool).unwrap_or(false); + if finishing && completed { if !last_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: last_text.clone() }); } close_provider_socket(&mut write).await; return Ok(()); } + } + else => { + if finishing && finish_sent { + if !last_text.is_empty() { emit(&app, SttEvent::Final { session_id: session.clone(), text: last_text.clone() }); } + return Ok(()); + } + break; + }, + } + } + Err(stage_failure("VolcengineV2", "close", "在结束前关闭")) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn full_request_contains_v2_auth_audio_and_workflow_fields() { + let config = serde_json::from_value(json!({ + "appId": "app", + "accessToken": "token", + "cluster": "cluster" + })) + .expect("config map"); + let request = start_request(&config, "session", "request"); + assert_eq!( + request.pointer("/app/token").and_then(Value::as_str), + Some("token") + ); + assert_eq!( + request.pointer("/audio/format").and_then(Value::as_str), + Some("raw") + ); + assert_eq!( + request.pointer("/audio/codec").and_then(Value::as_str), + Some("raw") + ); + assert_eq!( + request + .pointer("/request/show_utterances") + .and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + request + .pointer("/request/result_type") + .and_then(Value::as_str), + Some("full") + ); + } + + #[test] + fn classifies_1013_as_no_speech_only() { + assert!(response_is_no_speech(&json!({ + "code": 1013, + "message": "No valid speeches found in input audio" + }))); + assert!(!response_is_no_speech(&json!({ + "code": 45000000, + "message": "invalid request" + }))); + } +} diff --git a/crates/agent-gui/src-tauri/tauri.macos.conf.json b/crates/agent-gui/src-tauri/tauri.macos.conf.json index acdb5053a..c94eea321 100644 --- a/crates/agent-gui/src-tauri/tauri.macos.conf.json +++ b/crates/agent-gui/src-tauri/tauri.macos.conf.json @@ -27,7 +27,9 @@ "icons/icon.icns" ], "macOS": { - "hardenedRuntime": true + "hardenedRuntime": true, + "entitlements": "Entitlements.plist", + "infoPlist": "Info.plist" } } } diff --git a/crates/agent-gui/src-tauri/tauri.macos.release.conf.json b/crates/agent-gui/src-tauri/tauri.macos.release.conf.json index ff364db55..9d32e2502 100644 --- a/crates/agent-gui/src-tauri/tauri.macos.release.conf.json +++ b/crates/agent-gui/src-tauri/tauri.macos.release.conf.json @@ -14,7 +14,9 @@ "icons/icon.icns" ], "macOS": { - "hardenedRuntime": true + "hardenedRuntime": true, + "entitlements": "Entitlements.plist", + "infoPlist": "Info.plist" } } } diff --git a/crates/agent-gui/src/App.tsx b/crates/agent-gui/src/App.tsx index 22cf5443d..ff49df3e6 100644 --- a/crates/agent-gui/src/App.tsx +++ b/crates/agent-gui/src/App.tsx @@ -28,6 +28,7 @@ import { normalizeSettings, resolveEffectiveTheme, resolveWorkspaceProjects, + type SttProviderId, subscribeToSystemThemePreference, THEME_OPTIONS, type Theme, @@ -40,6 +41,7 @@ import { type SettingsSaveState, } from "./lib/settings/storage"; import { applyStoredGlobalShortcuts } from "./lib/shortcuts/globalShortcuts"; +import { desktopSttSettingsService } from "./lib/stt/desktopSttSettingsService"; import { ChatPage } from "./pages/ChatPage"; import type { SectionId } from "./pages/settings/types"; @@ -104,8 +106,16 @@ function hasSensitiveSettingsUpdatesPayload(payload: unknown) { providerApiKeyUpdates?: unknown; providerUsageQuerySecretUpdates?: unknown; sshSecretUpdates?: unknown; + sttSecretUpdate?: unknown; }) : {}; + if ( + source.sttSecretUpdate && + typeof source.sttSecretUpdate === "object" && + !Array.isArray(source.sttSecretUpdate) + ) { + return true; + } const providerUpdates = source.providerApiKeyUpdates; if ( providerUpdates && @@ -182,6 +192,7 @@ export default function App() { const [settingsProviderId, setSettingsProviderId] = useState(); const [settingsReady, setSettingsReady] = useState(false); const [settings, setSettingsState] = useState(() => getDefaultSettings()); + const [sttProviderOverride, setSttProviderOverride] = useState(null); const [settingsSaveState, setSettingsSaveState] = useState({ status: "idle", }); @@ -199,6 +210,10 @@ export default function App() { // crypto.randomUUID() inside caller updaters) twice per call. const settingsRef = useRef(settings); settingsRef.current = settings; + // biome-ignore lint/correctness/useExhaustiveDependencies: Saved provider changes invalidate the temporary card selection. + useEffect(() => { + setSttProviderOverride(null); + }, [settings.stt.provider]); const [systemThemeVersion, setSystemThemeVersion] = useState(0); const effectiveTheme = useMemo( () => resolveEffectiveTheme(settings.theme), @@ -332,16 +347,19 @@ export default function App() { .catch(() => undefined) .then(() => persistSettings(prev, next)) .then(async (persistResult) => { - const publishTarget = persistResult.ssh - ? normalizeSettings({ - ...next, - ssh: persistResult.ssh, - }) - : next; - if (persistResult.ssh && saveSequenceRef.current === saveSequence) { + const publishTarget = normalizeSettings({ + ...next, + ...(persistResult.ssh ? { ssh: persistResult.ssh } : {}), + ...(persistResult.stt ? { stt: persistResult.stt } : {}), + }); + if ( + (persistResult.ssh || persistResult.stt) && + saveSequenceRef.current === saveSequence + ) { const merged = normalizeSettings({ ...settingsRef.current, - ssh: persistResult.ssh, + ...(persistResult.ssh ? { ssh: persistResult.ssh } : {}), + ...(persistResult.stt ? { stt: persistResult.stt } : {}), }); settingsRef.current = merged; setSettingsState(merged); @@ -631,6 +649,7 @@ export default function App() { diff --git a/crates/agent-gui/src/lib/settings/storage.ts b/crates/agent-gui/src/lib/settings/storage.ts index 33949439d..f137aba6c 100644 --- a/crates/agent-gui/src/lib/settings/storage.ts +++ b/crates/agent-gui/src/lib/settings/storage.ts @@ -37,6 +37,7 @@ type PersistedSettingsResponse = { agents?: unknown | null; ssh?: unknown | null; remote?: unknown | null; + stt?: unknown | null; memory?: unknown | null; modelFailover?: unknown | null; defaultWorkdir?: unknown | null; @@ -67,6 +68,7 @@ type SshPatchApplyResponse = { export type PersistSettingsResult = { ssh?: AppSettings["ssh"]; + stt?: AppSettings["stt"]; conflict?: "ssh_settings_changed"; }; @@ -252,6 +254,7 @@ export async function loadPersistedSettingsWithDefaults(): Promise("settings_save_stt", { payload: next.stt } as any).then((response) => { + if (response) { + result.stt = normalizeSettings({ stt: response as AppSettings["stt"] }).stt; + } + }), + ); + } + if ( hasChanged(prev.skills, next.skills) || hasChanged(prev.chatRuntimeControls, next.chatRuntimeControls) || diff --git a/crates/agent-gui/src/lib/stt/desktopSttSettingsService.ts b/crates/agent-gui/src/lib/stt/desktopSttSettingsService.ts new file mode 100644 index 000000000..2aa18aab6 --- /dev/null +++ b/crates/agent-gui/src/lib/stt/desktopSttSettingsService.ts @@ -0,0 +1,30 @@ +import type { + SttConnectionTestResponse, + SttSecretField, + SttSettingsService, +} from "@liveagent/ui/lib/stt/types"; +import { invoke } from "@tauri-apps/api/core"; +import { type AppSettings, normalizeSttSettings } from "../../lib/settings"; + +export const desktopSttSettingsService: SttSettingsService = { + runtimeLabel: "桌面端（同步到 Gateway WebUI）", + secretRevealMode: "value", + revealSecret(provider, field: SttSecretField) { + return invoke("settings_reveal_stt_secret", { provider, field }); + }, + async get() { + const response = await invoke<{ stt?: AppSettings["stt"] }>("settings_load_all"); + if (!response.stt) throw new Error("无法读取 STT 配置"); + return normalizeSttSettings(response.stt); + }, + async update(settings) { + return normalizeSttSettings( + await invoke("settings_save_stt", { payload: settings }), + ); + }, + async test(provider) { + return invoke("settings_test_stt", { + provider, + }); + }, +}; diff --git a/crates/agent-gui/src/lib/stt/desktopSttTransport.ts b/crates/agent-gui/src/lib/stt/desktopSttTransport.ts new file mode 100644 index 000000000..9bdc8fbda --- /dev/null +++ b/crates/agent-gui/src/lib/stt/desktopSttTransport.ts @@ -0,0 +1,39 @@ +import type { + SttRuntimeEvent, + SttTransport, + SttTransportOpenOptions, +} from "@liveagent/ui/lib/stt/types"; +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; + +class DesktopSttTransport implements SttTransport { + private unlisten: UnlistenFn | null = null; + private handler: ((event: SttRuntimeEvent) => void) | null = null; + async requestPermission() { + await invoke("stt_request_microphone_permission"); + } + async open(options: SttTransportOpenOptions) { + this.handler = options.onEvent; + if (!this.unlisten) + this.unlisten = await listen("stt:event", (event) => + this.handler?.(event.payload), + ); + await invoke("stt_start", { sessionId: options.sessionId, provider: options.provider }); + } + async sendAudio(sessionId: string, sequence: number, pcm: Uint8Array) { + await invoke("stt_send_audio", { sessionId, sequence, pcm: Array.from(pcm) }); + } + async stop(sessionId: string) { + await invoke("stt_stop", { sessionId }); + } + async cancel(sessionId: string) { + await invoke("stt_cancel", { sessionId }); + } + dispose() { + this.handler = null; + this.unlisten?.(); + this.unlisten = null; + } +} + +export const desktopSttTransport = new DesktopSttTransport(); diff --git a/crates/agent-gui/src/pages/ChatPage.tsx b/crates/agent-gui/src/pages/ChatPage.tsx index 2422a8ddc..e5f84da78 100644 --- a/crates/agent-gui/src/pages/ChatPage.tsx +++ b/crates/agent-gui/src/pages/ChatPage.tsx @@ -99,6 +99,7 @@ import { } from "../lib/settings"; import { tauriSftpClient } from "../lib/sftp/tauriSftpClient"; import { createGuiSidebarBackend } from "../lib/sidebar/guiSidebarBackend"; +import { desktopSttTransport } from "../lib/stt/desktopSttTransport"; import { createSubagentStoreManager } from "../lib/subagents"; import { tauriTerminalClient } from "../lib/terminal/tauriTerminalClient"; import { cancelPendingAskUserQuestionsForConversation } from "../lib/tools/askUserQuestionTools"; @@ -168,6 +169,7 @@ export function ChatPage(props: ChatPageProps) { const { settings, setSettings, + sttProviderOverride, getMcpSettings, getToolPolicies, context, @@ -698,6 +700,8 @@ export function ChatPage(props: ChatPageProps) { (message: string) => addNotify("error", message), [addNotify], ); + // 语音输入失败（麦克风不可用等）以 toast 提示，不占用输入框区域。 + const handleSttError = useCallback((message: string) => addNotify("error", message), [addNotify]); const handleOpenChatFileLink = useChatFileLinkNavigation({ conversationId: currentConversationId, conversationWorkdir: displayedConversationWorkdir, @@ -1972,6 +1976,20 @@ export function ChatPage(props: ChatPageProps) { isSending={isSending} isUploadingFiles={isUploadingFiles} isInputDisabled={isComposerInputDisabled} + // 麦克风在开启语音输入后显示；点击设置卡片会立即切换当前供应商。 + sttSessionKey={currentConversationId} + sttProvider={ + settings.stt.enabled + ? (sttProviderOverride ?? settings.stt.provider ?? "tencent_cloud") + : null + } + sttProviderConfigured={ + settings.stt.providers[ + sttProviderOverride ?? settings.stt.provider ?? "tencent_cloud" + ]?.configured + } + sttTransport={desktopSttTransport} + onSttError={handleSttError} inputPlaceholder={composerPlaceholder} workdir={displayedConversationWorkdir} enabledSkills={enabledComposerSkills} diff --git a/crates/agent-gui/src/pages/chat/chatPageTypes.ts b/crates/agent-gui/src/pages/chat/chatPageTypes.ts index 081003cea..48cb5a05c 100644 --- a/crates/agent-gui/src/pages/chat/chatPageTypes.ts +++ b/crates/agent-gui/src/pages/chat/chatPageTypes.ts @@ -1,11 +1,12 @@ import type { Context } from "@earendil-works/pi-ai"; import type { AppUpdateController } from "../../lib/appUpdates"; -import type { AppSettings } from "../../lib/settings"; +import type { AppSettings, SttProviderId } from "../../lib/settings"; import type { SectionId } from "../settings/types"; export type ChatPageProps = { settings: AppSettings; setSettings: (updater: (prev: AppSettings) => AppSettings) => void; + sttProviderOverride?: SttProviderId | null; getMcpSettings: () => AppSettings["mcp"]; getToolPolicies: () => AppSettings["system"]["toolPolicies"]; context: Context; diff --git a/crates/agent-gui/src/pages/settings/types.ts b/crates/agent-gui/src/pages/settings/types.ts index ee1c5697c..3b448a2b3 100644 --- a/crates/agent-gui/src/pages/settings/types.ts +++ b/crates/agent-gui/src/pages/settings/types.ts @@ -1,5 +1,6 @@ +import type { SttSettingsService } from "@liveagent/ui/lib/stt/types"; import type { AppUpdateController } from "../../lib/appUpdates"; -import type { AppSettings } from "../../lib/settings"; +import type { AppSettings, SttProviderId } from "../../lib/settings"; import type { SettingsSaveState } from "../../lib/settings/storage"; export type SetSettingsFn = (updater: (prev: AppSettings) => AppSettings) => void; @@ -8,6 +9,7 @@ export type SectionId = | "system" | "shortcuts" | "systemTools" + | "stt" | "providers" | "agents" | "ssh" @@ -26,6 +28,9 @@ export type SettingsPageProps = { initialProviderId?: string; hiddenSections?: SectionId[]; appUpdate: AppUpdateController; + sttSettingsService: SttSettingsService; + /** 临时切换语音输入运行供应商，不触发配置保存。 */ + onSttProviderChange?: (provider: SttProviderId) => void; /** 绕过 setSettings 从 SQLite 重新载入（备份还原后用，见 SettingsSectionProps）。 */ reloadSettings?: () => Promise; }; diff --git a/crates/agent-gui/test/chat/composer-stt-lifecycle.test.mjs b/crates/agent-gui/test/chat/composer-stt-lifecycle.test.mjs new file mode 100644 index 000000000..d92557ce8 --- /dev/null +++ b/crates/agent-gui/test/chat/composer-stt-lifecycle.test.mjs @@ -0,0 +1,357 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { createWebModuleLoader } from "../../../agent-gateway/test/helpers/load-web-module.mjs"; + +function createHarness(options = {}) { + const hangSendAudio = options.hangSendAudio === true; + let hookCursor = 0; + const refs = []; + const state = []; + const effects = []; + const layoutEffects = []; + const react = { + useCallback(fn) { + hookCursor += 1; + return fn; + }, + useEffect(fn) { + hookCursor += 1; + effects.push(fn); + }, + useLayoutEffect(fn) { + hookCursor += 1; + layoutEffects.push(fn); + }, + useRef(initial) { + return (refs[hookCursor] ??= { current: initial }); + }, + useState(initial) { + const index = hookCursor++; + state[index] ??= initial; + return [state[index], (next) => { + state[index] = typeof next === "function" ? next(state[index]) : next; + }]; + }, + render() { + hookCursor = 0; + layoutEffects.length = 0; + }, + effects, + }; + + const captures = []; + let deferCaptureStop = false; + let releaseCaptureStop = null; + class FakeCapture { + constructor(options) { + this.options = options; + this.events = []; + captures.push(this); + } + async start() { + this.events.push("capture.start"); + } + stop() { + this.events.push("capture.stop"); + if (deferCaptureStop) { + return new Promise((resolve) => { + releaseCaptureStop = resolve; + }); + } + } + chunk(sequence, values = [sequence]) { + this.options.onChunk({ sequence, pcm: new Int16Array(values), durationMs: 100 }); + } + silence() { + this.options.onSilenceTimeout?.(); + } + resetSilenceClock() { + this.events.push("capture.resetSilenceClock"); + } + } + class FakeFifo { + constructor() { + this.items = []; + } + push(value) { + this.items.push(value); + return true; + } + drain() { + return this.items.splice(0); + } + clear() { + this.items = []; + } + } + + const loader = createWebModuleLoader({ + rootDir: fileURLToPath(new URL("../../../agent-gateway/web/", import.meta.url)), + mocks: { + react, + "@liveagent/ui/lib/stt/audio": { + appendTailSilence: () => new Int16Array(6400), + pcm16ToLittleEndianBytes: (pcm) => new Uint8Array(pcm.buffer), + STT_CONNECT_TIMEOUT_MS: 10_000, + STT_SAMPLES_PER_CHUNK: 1600, + SttAudioCapture: FakeCapture, + SttPcmFifo: FakeFifo, + }, + }, + }); + const { useComposerStt } = loader.loadModule("@liveagent/ui/pages/chat/useComposerStt"); + const calls = []; + const callbacks = []; + const sessionIds = []; + const transport = { + async requestPermission() { + calls.push("permission"); + }, + async open(options) { + calls.push("open"); + sessionIds.push(options.sessionId); + callbacks.push(options.onEvent); + }, + async sendAudio(sessionId, sequence, bytes) { + calls.push(["audio", sessionId, sequence, bytes.byteLength]); + if (hangSendAudio) { + return new Promise(() => {}); + } + }, + async stop(sessionId) { + calls.push(["stop", sessionId]); + }, + async cancel(sessionId) { + calls.push(["cancel", sessionId]); + }, + dispose() { + calls.push("dispose"); + }, + }; + const composerEvents = []; + const composer = { + beginTransientText() { + composerEvents.push("begin"); + return true; + }, + updateTransientText(text) { + composerEvents.push(["partial", text]); + }, + commitTransientText(text) { + composerEvents.push(["final", text]); + }, + cancelTransientText(options) { + composerEvents.push(["cancel", options]); + }, + }; + const composerRef = { current: composer }; + const timeoutCallbacks = new Map(); + let timeoutId = 0; + globalThis.window = { + setTimeout: (fn) => { + const id = ++timeoutId; + timeoutCallbacks.set(id, fn); + return id; + }, + clearTimeout: (id) => { + timeoutCallbacks.delete(id); + }, + }; + const render = (overrides = {}) => { + react.render(); + return useComposerStt({ + composerRef, + provider: "aliyun_dashscope", + transport, + disabled: false, + sessionKey: "conversation-a", + hidden: false, + ...overrides, + }); + }; + return { + render, + calls, + callbacks, + sessionIds, + captures, + composerEvents, + effects, + deferCaptureStop() { + deferCaptureStop = true; + }, + releaseCaptureStop() { + releaseCaptureStop?.(); + releaseCaptureStop = null; + deferCaptureStop = false; + }, + flushLayoutEffects() { + for (const effect of [...layoutEffects]) effect(); + }, + fireTimeout(id) { + const fn = timeoutCallbacks.get(id); + timeoutCallbacks.delete(id); + fn?.(); + }, + pendingTimeoutIds() { + return [...timeoutCallbacks.keys()]; + }, + }; +} + +async function settle() { + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); +} + +test("useComposerStt starts permission, capture, then cloud open and drains FIFO in order", async () => { + const harness = createHarness(); + let result = harness.render(); + await result.toggle(); + await settle(); + assert.deepEqual(harness.calls.slice(0, 2), ["permission", "open"]); + assert.deepEqual(harness.captures[0].events, ["capture.start"]); + + harness.captures[0].chunk(0, [1]); + harness.captures[0].chunk(1, [2]); + assert.deepEqual(harness.calls.filter((call) => Array.isArray(call)), []); + const sessionId = harness.sessionIds[0]; + harness.callbacks[0]({ type: "ready", sessionId }); + await settle(); + assert.ok(harness.captures[0].events.includes("capture.resetSilenceClock")); + const audioCalls = harness.calls.filter((call) => Array.isArray(call) && call[0] === "audio"); + assert.deepEqual(audioCalls.map((call) => call[2]), [0, 1]); + + harness.captures[0].chunk(2, [3]); + await settle(); + assert.deepEqual( + harness.calls.filter((call) => Array.isArray(call) && call[0] === "audio").map((call) => call[2]), + [0, 1, 2], + ); + assert.match(sessionId, /^[0-9a-f-]{16,}$/); + result = harness.render(); + assert.equal(result.state, "recognizing"); +}); + +test("stop halts capture, sends four 100 ms tail chunks, then finishes transport", async () => { + const harness = createHarness(); + let result = harness.render(); + await result.toggle(); + await settle(); + harness.callbacks[0]({ type: "ready", sessionId: harness.sessionIds[0] }); + await settle(); + result = harness.render(); + await result.toggle(); + await settle(); + assert.deepEqual( + harness.captures[0].events.filter((event) => event !== "capture.resetSilenceClock"), + ["capture.start", "capture.stop"], + ); + const audio = harness.calls.filter((call) => Array.isArray(call) && call[0] === "audio"); + assert.deepEqual(audio.map((call) => call[2]), [0, 1, 2, 3]); + assert.deepEqual(harness.calls.filter((call) => Array.isArray(call) && call[0] === "stop").length, 1); +}); + +test("ready racing with capture stop waits for tail audio before finish", async () => { + const harness = createHarness(); + let result = harness.render(); + await result.toggle(); + await settle(); + harness.deferCaptureStop(); + result = harness.render(); + const stopping = result.toggle(); + await settle(); + + harness.callbacks[0]({ type: "ready", sessionId: harness.sessionIds[0] }); + await settle(); + assert.equal(harness.calls.some((call) => Array.isArray(call) && call[0] === "stop"), false); + + harness.releaseCaptureStop(); + await stopping; + await settle(); + const audioSequences = harness.calls + .filter((call) => Array.isArray(call) && call[0] === "audio") + .map((call) => call[2]); + assert.deepEqual(audioSequences, [0, 1, 2, 3]); + assert.equal(harness.calls.filter((call) => Array.isArray(call) && call[0] === "stop").length, 1); +}); + +test("final accepts only the active session and stale events cannot mutate the composer", async () => { + const harness = createHarness(); + let result = harness.render(); + await result.toggle(); + await settle(); + harness.callbacks[0]({ type: "partial", sessionId: "old-session", text: "stale" }); + assert.deepEqual(harness.composerEvents, ["begin"]); + harness.callbacks[0]({ type: "partial", sessionId: harness.sessionIds[0], text: "live" }); + assert.deepEqual(harness.composerEvents.at(-1), ["partial", "live"]); + harness.callbacks[0]({ type: "final", sessionId: harness.sessionIds[0], text: "done" }); + await settle(); + assert.deepEqual(harness.composerEvents.at(-1), ["cancel", { preserveLastText: true }]); + assert.ok(harness.composerEvents.some((event) => event[0] === "final" && event[1] === "done")); +}); + +test("silence during buffering does not finish; ready resets the clock then silence stops", async () => { + const harness = createHarness(); + let result = harness.render(); + await result.toggle(); + await settle(); + harness.captures[0].silence(); + await settle(); + assert.equal(harness.calls.some((call) => Array.isArray(call) && call[0] === "stop"), false); + + harness.callbacks[0]({ type: "ready", sessionId: harness.sessionIds[0] }); + await settle(); + assert.ok(harness.captures[0].events.includes("capture.resetSilenceClock")); + harness.captures[0].silence(); + await settle(); + assert.equal(harness.calls.filter((call) => Array.isArray(call) && call[0] === "stop").length, 1); + result = harness.render(); + assert.equal(result.state, "stopping"); +}); + +test("changing sessionKey or hiding the composer cancels the active session", async () => { + const harness = createHarness(); + let result = harness.render({ sessionKey: "conversation-a" }); + await result.toggle(); + await settle(); + harness.callbacks[0]({ type: "partial", sessionId: harness.sessionIds[0], text: "keep" }); + + harness.render({ sessionKey: "conversation-b" }); + harness.flushLayoutEffects(); + await settle(); + assert.equal(harness.calls.filter((call) => Array.isArray(call) && call[0] === "cancel").length, 1); + assert.deepEqual(harness.composerEvents.at(-1), ["cancel", { preserveLastText: true }]); + + result = harness.render({ sessionKey: "conversation-b" }); + await result.toggle(); + await settle(); + harness.render({ sessionKey: "conversation-b", hidden: true }); + harness.flushLayoutEffects(); + await settle(); + assert.equal(harness.calls.filter((call) => Array.isArray(call) && call[0] === "cancel").length, 2); +}); + +test("a second stop while audio send is stuck force-cancels instead of remaining in stopping", async () => { + const harness = createHarness({ hangSendAudio: true }); + let result = harness.render(); + await result.toggle(); + await settle(); + harness.callbacks[0]({ type: "ready", sessionId: harness.sessionIds[0] }); + await settle(); + harness.captures[0].chunk(0, [1]); + await settle(); + result = harness.render(); + result.toggle(); + await settle(); + result = harness.render(); + assert.equal(result.state, "stopping"); + assert.equal(harness.calls.filter((call) => Array.isArray(call) && call[0] === "stop").length, 0); + + result.toggle(); + await settle(); + result = harness.render(); + assert.equal(result.state, "idle"); + assert.equal(harness.calls.filter((call) => Array.isArray(call) && call[0] === "cancel").length, 1); +}); diff --git a/crates/agent-gui/test/chat/mention-composer-selection.test.mjs b/crates/agent-gui/test/chat/mention-composer-selection.test.mjs index cdb3d2c89..c4c5e6082 100644 --- a/crates/agent-gui/test/chat/mention-composer-selection.test.mjs +++ b/crates/agent-gui/test/chat/mention-composer-selection.test.mjs @@ -24,7 +24,7 @@ test("the shared composer restores the last editor selection before external men assert.match(composer, /document\.addEventListener\("selectionchange", rememberEditorSelection\)/); assert.equal( (composer.match(/focusEditorAtSavedSelection\(\);/g) ?? []).length, - 5, + 6, ); } }); @@ -85,3 +85,27 @@ test("composer caret measurement never splits text nodes and restores the select assert.match(scrollBodies[0], /measureComposerCaretRect\(range\)/); assert.doesNotMatch(scrollBodies[0], /cloneRange\(\)/); }); + +test("composer transient text exposes an anchored marker transaction", () => { + const composer = composerSource(sourceRoots[0]); + const transient = composer.slice(composer.indexOf("beginTransientText:"), composer.indexOf("focus: ()")); + assert.match(transient, /clearTransientText\(false\)/); + assert.match(transient, /activeRange\.insertNode\(end\)/); + assert.match(transient, /activeRange\.insertNode\(textNode\)/); + assert.match(transient, /activeRange\.insertNode\(start\)/); + assert.match(transient, /transientTextRef\.current = \{ textNode, start, end \}/); + assert.match(transient, /active\.textNode\.data = normalizeLogicalLineEndings\(text\)/); + assert.match(transient, /range\.setStartAfter\(active\.end\)/); + assert.match(transient, /clearTransientText\(true\)/); + assert.match(transient, /clearTransientText\(options\?\.preserveLastText === true\)/); +}); + +test("transient cleanup protects mention nodes and locks composer controls during STT", () => { + const composer = composerSource(sourceRoots[0]); + assert.match(composer, /clearTransientText\(false\);[\s\S]*?setBusy\(false\)/); + assert.match(composer, /closestComposerChipFromNode/); + const bar = source(new URL("../../../agent-ui/src/pages/chat/", import.meta.url), "ChatComposerBar.tsx"); + assert.match(bar, /disabled=\{isInputDisabled \|\| stt\.active\}/); + assert.match(bar, /disabled=\{controlsDisabled\}/); + assert.match(bar, /const sendDisabled = isInputDisabled \|\| stt\.active \|\| isUploadingFiles \|\| !hasSendableDraft/); +}); diff --git a/crates/agent-gui/test/chat/stt-audio.test.mjs b/crates/agent-gui/test/chat/stt-audio.test.mjs new file mode 100644 index 000000000..f5c254626 --- /dev/null +++ b/crates/agent-gui/test/chat/stt-audio.test.mjs @@ -0,0 +1,152 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { createWebModuleLoader } from "../../../agent-gateway/test/helpers/load-web-module.mjs"; + +const loader = createWebModuleLoader({ + rootDir: fileURLToPath(new URL("../../../agent-gateway/web/", import.meta.url)), +}); +const { + STT_MAX_BUFFER_BYTES, + STT_SAMPLES_PER_CHUNK, + STT_TAIL_SILENCE_MS, + SttStreamingResampler, + SttPcmFifo, + appendTailSilence, + floatToPcm16, + pcm16ToLittleEndianBytes, + resampleTo16k, +} = loader.loadModule("@liveagent/ui/lib/stt/audio"); + +function concatenateFloat32(parts) { + const output = new Float32Array(parts.reduce((length, part) => length + part.length, 0)); + let offset = 0; + for (const part of parts) { + output.set(part, offset); + offset += part.length; + } + return output; +} + +function chunk(sequence, samples, durationMs = 100) { + return { sequence, pcm: new Int16Array(samples), durationMs }; +} + +test("STT audio constants describe 100 ms chunks and a 400 ms zero tail", () => { + assert.equal(STT_SAMPLES_PER_CHUNK, 1600); + assert.equal(STT_TAIL_SILENCE_MS, 400); + const tail = appendTailSilence(); + assert.equal(tail.length, 6400); + assert.ok(tail.every((sample) => sample === 0)); +}); + +test("resampleTo16k preserves boundaries and handles empty or invalid input", () => { + const source = new Float32Array([-1, 0, 1, 0]); + const output = resampleTo16k(source, 8_000); + assert.equal(output.length, 8); + assert.equal(output[0], -1); + assert.equal(output.at(-1), 0); + assert.deepEqual([...resampleTo16k(source, 16_000)], [...source]); + assert.deepEqual([...resampleTo16k(new Float32Array(), 48_000)], [0]); + assert.deepEqual([...resampleTo16k(source, Number.NaN)], [...source]); + assert.doesNotThrow(() => resampleTo16k(source, -1)); +}); + +test("streaming resampler preserves phase across arbitrary 48 kHz blocks", () => { + const source = Float32Array.from( + { length: 48_000 }, + (_, index) => Math.sin((2 * Math.PI * 440 * index) / 48_000), + ); + const whole = new SttStreamingResampler(48_000); + const expected = concatenateFloat32([whole.process(source), whole.flush()]); + const split = new SttStreamingResampler(48_000); + const actualParts = []; + for (let offset = 0; offset < source.length; offset += 4096) { + actualParts.push(split.process(source.slice(offset, offset + 4096))); + } + actualParts.push(split.flush()); + const actual = concatenateFloat32(actualParts); + + assert.equal(actual.length, 16_000); + assert.deepEqual(actual, expected); +}); + +test("streaming resampler has the correct long-run count at 44.1 kHz", () => { + const source = Float32Array.from({ length: 44_100 }, (_, index) => index / 44_100); + const resampler = new SttStreamingResampler(44_100); + const parts = []; + for (let offset = 0; offset < source.length; offset += 997) { + parts.push(resampler.process(source.slice(offset, offset + 997))); + } + parts.push(resampler.flush()); + const output = concatenateFloat32(parts); + + assert.equal(output.length, 16_000); + assert.equal(output[0], 0); + assert.ok(output.at(-1) > 0.99); +}); + +test("streaming resampler rejects invalid rates and handles empty blocks", () => { + assert.throws(() => new SttStreamingResampler(0), /采样率/); + assert.throws(() => new SttStreamingResampler(Number.NaN), /采样率/); + const resampler = new SttStreamingResampler(16_000); + assert.deepEqual(resampler.process(new Float32Array()), new Float32Array()); + assert.deepEqual(resampler.process(new Float32Array([0.25, -0.25])), new Float32Array([0.25, -0.25])); + assert.deepEqual(resampler.flush(), new Float32Array()); +}); + +test("floatToPcm16 clamps positive and negative saturation", () => { + assert.deepEqual( + [...floatToPcm16(new Float32Array([-2, -1, -0.5, 0, 0.5, 1, 2]))], + [-32768, -32768, -16384, 0, 16384, 32767, 32767], + ); +}); + +test("PCM16 encoding is signed little-endian", () => { + assert.deepEqual( + [...pcm16ToLittleEndianBytes(new Int16Array([0x1234, -2, -32768, 32767]))], + [0x34, 0x12, 0xfe, 0xff, 0x00, 0x80, 0xff, 0x7f], + ); +}); + +test("SttPcmFifo preserves order, rejects overflow, and drains or clears", () => { + const fifo = new SttPcmFifo(8); + assert.equal(fifo.push(chunk(3, [3, 3])), true); + assert.equal(fifo.push(chunk(4, [4, 4])), true); + assert.equal(fifo.push(chunk(5, [5])), false); + assert.equal(fifo.sizeBytes, 8); + assert.equal(fifo.length, 2); + assert.deepEqual(fifo.drain().map((value) => value.sequence), [3, 4]); + assert.equal(fifo.sizeBytes, 0); + assert.equal(fifo.length, 0); + assert.equal(fifo.push(chunk(6, [6])), true); + fifo.clear(); + assert.equal(fifo.sizeBytes, 0); + assert.equal(fifo.length, 0); + const defaultFifo = new SttPcmFifo(); + assert.equal(defaultFifo.push(chunk(0, new Array(STT_MAX_BUFFER_BYTES / 2))), true); + assert.equal(defaultFifo.push(chunk(1, [1])), false); +}); + +test("partial audio fragments do not make FIFO helpers throw", () => { + assert.doesNotThrow(() => { + const fifo = new SttPcmFifo(0); + fifo.push(chunk(0, [])); + fifo.drain(); + fifo.clear(); + appendTailSilence(0.5); + }); +}); + +test("microphone capture is wired through a muted gain node instead of speakers", () => { + const source = readFileSync( + fileURLToPath(new URL("../../../agent-ui/src/lib/stt/audio.ts", import.meta.url)), + "utf8", + ); + assert.match(source, /this\.sink = this\.context\.createGain\(\)/); + assert.match(source, /this\.sink\.gain\.value = 0/); + assert.match(source, /this\.processor\.connect\(this\.sink\)/); + assert.match(source, /this\.sink\.connect\(this\.context\.destination\)/); + assert.doesNotMatch(source, /this\.processor\.connect\(this\.context\.destination\)/); +}); diff --git a/crates/agent-gui/test/settings/stt-settings.test.mjs b/crates/agent-gui/test/settings/stt-settings.test.mjs new file mode 100644 index 000000000..d3fe40c58 --- /dev/null +++ b/crates/agent-gui/test/settings/stt-settings.test.mjs @@ -0,0 +1,242 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { createWebModuleLoader } from "../../../agent-gateway/test/helpers/load-web-module.mjs"; + +const loader = createWebModuleLoader({ + rootDir: fileURLToPath(new URL("../../../agent-gateway/web/", import.meta.url)), +}); +const { getDefaultSttSettings, normalizeSttSettings } = loader.loadModule( + "@liveagent/ui/lib/settings/index.ts", +); + +test("STT provider defaults match the four provider configuration forms", () => { + const defaults = getDefaultSttSettings(); + assert.equal(defaults.enabled, false); + const providers = defaults.providers; + assert.deepEqual(Object.keys(providers), [ + "tencent_cloud", + "volcengine_seed_v3", + "aliyun_dashscope", + "baidu_cloud", + ]); + assert.equal( + providers.aliyun_dashscope.websocketUrl, + "wss://dashscope.aliyuncs.com/api-ws/v1/inference/", + ); + assert.equal(providers.aliyun_dashscope.model, "paraformer-realtime-v2"); + assert.equal(providers.tencent_cloud.engineModelType, "16k_zh"); + assert.equal( + providers.volcengine_seed_v3.websocketUrl, + "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async", + ); + assert.equal(providers.baidu_cloud.websocketUrl, "wss://vop.baidu.com/realtime_asr"); + assert.equal(providers.baidu_cloud.devPid, ""); +}); + +test("legacy STT settings gain protocol defaults without changing saved values", () => { + const normalized = normalizeSttSettings({ + enabled: true, + provider: "volcengine_v2", + providers: { + aliyun_dashscope: { + id: "aliyun_dashscope", + configured: true, + model: "paraformer-realtime-8k-v2", + }, + tencent_cloud: { id: "tencent_cloud", engineModelType: "16k_en" }, + volcengine_v2: { id: "volcengine_v2", cluster: "custom_cluster" }, + }, + }); + assert.equal(normalized.enabled, true); + assert.equal(normalized.provider, null); + assert.equal(normalized.aliyun_dashscope, undefined); + assert.equal( + normalized.providers.aliyun_dashscope.websocketUrl, + "wss://dashscope.aliyuncs.com/api-ws/v1/inference/", + ); + assert.equal(normalized.providers.aliyun_dashscope.model, "paraformer-realtime-v2"); + assert.equal(normalized.providers.tencent_cloud.engineModelType, "16k_en"); + assert.equal(normalized.providers.volcengine_v2, undefined); + assert.equal( + normalized.providers.volcengine_seed_v3.websocketUrl, + "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async", + ); + assert.equal(normalized.providers.baidu_cloud.websocketUrl, "wss://vop.baidu.com/realtime_asr"); +}); + +test("STT connection test saves the current form and identifies the active runtime", () => { + const section = readFileSync( + fileURLToPath(new URL("../../../agent-ui/src/pages/settings/SttSection.tsx", import.meta.url)), + "utf8", + ); + const desktopService = readFileSync( + fileURLToPath(new URL("../../src/lib/stt/desktopSttSettingsService.ts", import.meta.url)), + "utf8", + ); + const settingsPage = readFileSync( + fileURLToPath(new URL("../../../agent-ui/src/pages/settings/SettingsPage.tsx", import.meta.url)), + "utf8", + ); + const zhCNSettings = readFileSync( + fileURLToPath( + new URL("../../../agent-ui/src/i18n/translations/zhCNSettings.ts", import.meta.url), + ), + "utf8", + ); + const webService = readFileSync( + fileURLToPath( + new URL( + "../../../agent-gateway/web/src/lib/stt/webSttSettingsService.ts", + import.meta.url, + ), + ), + "utf8", + ); + const gatewayView = readFileSync( + fileURLToPath( + new URL("../../../agent-gateway/web/src/app/GatewayAppView.tsx", import.meta.url), + ), + "utf8", + ); + const desktopChatPage = readFileSync( + fileURLToPath(new URL("../../src/pages/ChatPage.tsx", import.meta.url)), + "utf8", + ); + const desktopApp = readFileSync( + fileURLToPath(new URL("../../src/App.tsx", import.meta.url)), + "utf8", + ); + const composerBar = readFileSync( + fileURLToPath( + new URL("../../../agent-ui/src/pages/chat/ChatComposerBar.tsx", import.meta.url), + ), + "utf8", + ); + const webSettingsSync = readFileSync( + fileURLToPath( + new URL( + "../../../agent-gateway/web/src/app/hooks/useGatewaySettingsSync.ts", + import.meta.url, + ), + ), + "utf8", + ); + + assert.match(section, /if \(!\(await save\(\)\)\) return;/); + assert.match(section, />语音输入<\/h3>/); + assert.doesNotMatch(section, /语音转文字/); + assert.match(zhCNSettings, /"settings\.navStt": "语音输入"/); + assert.doesNotMatch(zhCNSettings, /语音转文字/); + assert.match(section, /保存并测试连接/); + assert.match(section, /当前运行端：\{service\.runtimeLabel\}/); + assert.match(section, /fields: \["websocketUrl", "model", "apiKey"\]/); + assert.match( + section, + /fields: \["appId", "engineModelType", "secretId", "secretKey"\]/, + ); + assert.match( + section, + /fields: \["websocketUrl", "appId", "accessToken", "resourceId"\]/, + ); + assert.match( + section, + /fields: \["websocketUrl", "baiduAppId", "devPid", "baiduApiKey"\]/, + ); + assert.match(section, /provider === "tencent_cloud" \? "AppId" : "App ID"/); + assert.match(section, /delete next\[definition\.id\]/); + assert.doesNotMatch(section, /service\s*\.get\(\)/); + assert.match(section, /const displayedStt = settings\.stt/); + assert.match(section, /checked=\{displayedStt\.enabled\}/); + assert.match(section, /stt: \{ \.\.\.previous\.stt, enabled, allowIncomplete: true \}/); + assert.match(section, /className="w-full min-w-0 space-y-5"/); + assert.doesNotMatch(webSettingsSync, /receivedSyncedStt/); + assert.match(webSettingsSync, /const stt = await webSttSettingsService\.get\(\)/); + assert.match( + webSettingsSync, + /if \(!cancelled && liveSyncEpochRef\.current === sttEpoch\)/, + ); + assert.match(webSettingsSync, /if \(liveSyncEpochRef\.current === 0\)/); + assert.match(webSettingsSync, /persistWebSettings\(next\)/); + assert.doesNotMatch( + webSettingsSync, + /queueSettingsSave\(prev, next, "同步桌面端设置失败。", false\)/, + ); + assert.match(desktopService, /runtimeLabel: "桌面端（同步到 Gateway WebUI）"/); + assert.match(desktopService, /secretRevealMode: "value"/); + assert.match(desktopService, /settings_reveal_stt_secret/); + assert.match(webService, /runtimeLabel: "WebUI（与桌面端同步，凭据由 Gateway 安全托管）"/); + assert.match(webService, /secretRevealMode: "field-name"/); + assert.match(section, /WebUI 的查看按钮只显示字段名/); + assert.match(section, /toggleSecretVisibility/); + assert.match(section, /const SAVED_SECRET_MASK = "saved-secret-placeholder"/); + assert.match(section, /selectedProvider: SttProviderId/); + assert.match(section, /onSelectedProviderChange\(id\)/); + assert.match(settingsPage, /const \[sttSelectedProvider, setSttSelectedProvider\]/); + assert.match(settingsPage, /STT_SELECTED_PROVIDER_CACHE = new WeakMap/); + assert.match(settingsPage, /STT_SELECTED_PROVIDER_CACHE\.set\(sttSettingsService, provider\)/); + assert.match(settingsPage, /selectedProvider=\{sttSelectedProvider\}/); + assert.match(settingsPage, /onSelectedProviderChange=\{handleSttProviderChange\}/); + assert.match(section, /setDraftProviders/); + assert.doesNotMatch(section, /volcengine_v2/); + assert.doesNotMatch(section, /火山引擎实时语音识别 v2/); + assert.match(section, /label: "火山引擎实时语音识别"/); + assert.ok(section.indexOf('id: "tencent_cloud"') < section.indexOf('id: "volcengine_seed_v3"')); + assert.ok(section.indexOf('id: "volcengine_seed_v3"') < section.indexOf('id: "aliyun_dashscope"')); + assert.ok(section.indexOf('id: "aliyun_dashscope"') < section.indexOf('id: "baidu_cloud"')); + assert.doesNotMatch(section, />保存配置 void clearProviderSecrets\(\)\}/); + assert.match(section, /clearSecrets: true/); + const selectProviderBlock = section.slice( + section.indexOf("const selectProvider"), + section.indexOf("const save"), + ); + assert.doesNotMatch(selectProviderBlock, /setSettings/); + const saveBlock = section.slice(section.indexOf("const save ="), section.indexOf("const clearProviderSecrets")); + assert.match(saveBlock, /delete nextProvider\.clearSecrets/); + assert.match( + gatewayView, + /settings\.stt\.enabled\s*\?\s*\(\s*sttProviderOverride\s*\?\?\s*settings\.stt\.provider\s*\?\?\s*"tencent_cloud"\s*\)\s*:\s*null/, + ); + assert.match(gatewayView, /onSttProviderChange=\{setSttProviderOverride\}/); + assert.match(gatewayView, /setSttProviderOverride\(null\)/); + assert.doesNotMatch(gatewayView, /settings\.stt\.providers\[settings\.stt\.provider\]\.configured/); + assert.match( + desktopChatPage, + /settings\.stt\.enabled\s*\?\s*\(\s*sttProviderOverride\s*\?\?\s*settings\.stt\.provider\s*\?\?\s*"tencent_cloud"\s*\)\s*:\s*null/, + ); + assert.match(desktopChatPage, /sttProviderOverride/); + assert.match(desktopApp, /sttProviderOverride=\{sttProviderOverride\}/); + assert.match(desktopApp, /onSttProviderChange=\{setSttProviderOverride\}/); + assert.match(desktopApp, /setSttProviderOverride\(null\)/); + assert.match(desktopChatPage, /sttSessionKey=\{currentConversationId\}/); + assert.match(gatewayView, /sttSessionKey=\{displayedConversationId\}/); + assert.match(desktopChatPage, /sttProviderConfigured=/); + assert.doesNotMatch( + desktopChatPage, + /settings\.stt\.providers\[settings\.stt\.provider\]\.configured/, + ); + assert.match(desktopChatPage, /onSttError=\{handleSttError\}/); + assert.match(gatewayView, /onSttError=\{handleSttError\}/); + assert.match(gatewayView, /sttProviderConfigured=/); + assert.match(composerBar, /onError: onSttError/); + assert.match(composerBar, /providerConfigured: sttProviderConfigured/); + assert.match(composerBar, /sessionKey: sttSessionKey/); + assert.match(composerBar, /hidden,/); + assert.doesNotMatch(composerBar, /stt\.error \? \(/); + const composerStt = readFileSync( + fileURLToPath(new URL("../../../agent-ui/src/pages/chat/useComposerStt.ts", import.meta.url)), + "utf8", + ); + assert.match(composerStt, /STT供应商配置不完整/); + assert.match(composerStt, /resetSilenceClock/); + assert.match(composerStt, /abortActiveSession/); + assert.match(composerStt, /if \(!current\?\.ready \|\| current\.stopping\) return;/); + assert.match(composerStt, /if \(active\.stopping\) \{/); + assert.match(composerStt, /STT_SEND_QUEUE_TIMEOUT_MS/); + assert.doesNotMatch( + composerBar, + /disabled=\{isInputDisabled \|\| stt\.state === "stopping"\}/, + ); +}); diff --git a/crates/agent-ui/src/components/IconSet.tsx b/crates/agent-ui/src/components/IconSet.tsx index d017c6cc4..62b0ef1e5 100644 --- a/crates/agent-ui/src/components/IconSet.tsx +++ b/crates/agent-ui/src/components/IconSet.tsx @@ -15,6 +15,7 @@ import BanSource from "~icons/lucide/ban"; import BlendSource from "~icons/lucide/blend"; import BookOpenSource from "~icons/lucide/book-open"; import BotSource from "~icons/lucide/bot"; +import BracketsSource from "~icons/lucide/brackets"; import BrainSource from "~icons/lucide/brain"; import BrushCleaningSource from "~icons/lucide/brush-cleaning"; import CableSource from "~icons/lucide/cable"; @@ -44,6 +45,7 @@ import EyeOffSource from "~icons/lucide/eye-off"; import FileSource from "~icons/lucide/file"; import FilePenLineSource from "~icons/lucide/file-pen-line"; import FileTextSource from "~icons/lucide/file-text"; +import FlameSource from "~icons/lucide/flame"; import FolderSource from "~icons/lucide/folder"; import FolderClosedSource from "~icons/lucide/folder-closed"; import FolderOpenSource from "~icons/lucide/folder-open"; @@ -77,6 +79,7 @@ import Maximize2Source from "~icons/lucide/maximize-2"; import MessageCircleSource from "~icons/lucide/message-circle"; import MessageSquareSource from "~icons/lucide/message-square"; import MessageSquareTextSource from "~icons/lucide/message-square-text"; +import MicSource from "~icons/lucide/mic"; import Minimize2Source from "~icons/lucide/minimize-2"; import MinusSource from "~icons/lucide/minus"; import MonitorSmartphoneSource from "~icons/lucide/monitor-smartphone"; @@ -88,6 +91,7 @@ import PanelLeftCloseSource from "~icons/lucide/panel-left-close"; import PanelRightCloseSource from "~icons/lucide/panel-right-close"; import PanelRightOpenSource from "~icons/lucide/panel-right-open"; import PaperclipSource from "~icons/lucide/paperclip"; +import PawPrintSource from "~icons/lucide/paw-print"; import Edit3Source from "~icons/lucide/pen-line"; import PencilSource from "~icons/lucide/pencil"; import PinSource from "~icons/lucide/pin"; @@ -483,6 +487,7 @@ export const Ban = createIcon(BanSource); export const Blend = createIcon(BlendSource); export const BookOpen = createIcon(BookOpenSource); export const Bot = createIcon(BotSource); +export const Brackets = createIcon(BracketsSource); export const Brain = createIcon(BrainSource); export const BrushCleaning = createIcon(BrushCleaningSource); export const Cable = createIcon(CableSource); @@ -509,6 +514,7 @@ export const EyeOff = createIcon(EyeOffSource); export const File = createIcon(FileSource); export const FilePenLine = createIcon(FilePenLineSource); export const FileText = createIcon(FileTextSource); +export const Flame = createIcon(FlameSource); export const Folder = createIcon(FolderSource); export const FolderClosed = createIcon(FolderClosedSource); export const FolderOpen = createIcon(FolderOpenSource); @@ -544,6 +550,7 @@ export const Maximize2 = createIcon(Maximize2Source); export const MessageCircle = createIcon(MessageCircleSource); export const MessageSquare = createIcon(MessageSquareSource); export const MessageSquareText = createIcon(MessageSquareTextSource); +export const Mic = createIcon(MicSource); export const McpLogo = createIcon(McpLogoSource); export const Minimize2 = createIcon(Minimize2Source); export const Minus = createIcon(MinusSource); @@ -556,6 +563,7 @@ export const PanelLeftClose = createIcon(PanelLeftCloseSource); export const PanelRightClose = createIcon(PanelRightCloseSource); export const PanelRightOpen = createIcon(PanelRightOpenSource); export const Paperclip = createIcon(PaperclipSource); +export const PawPrint = createIcon(PawPrintSource); export const Pencil = createIcon(PencilSource); export const Package = createIcon(PackageSource); export const Palette = createIcon(PaletteSource); diff --git a/crates/agent-ui/src/components/chat/MentionComposer.tsx b/crates/agent-ui/src/components/chat/MentionComposer.tsx index 0a519ad40..f22c927d1 100644 --- a/crates/agent-ui/src/components/chat/MentionComposer.tsx +++ b/crates/agent-ui/src/components/chat/MentionComposer.tsx @@ -203,6 +203,11 @@ export const MentionComposer = memo( }, []); const lastEditorSelectionRef = useRef(null); + const transientTextRef = useRef<{ + textNode: Text; + start: HTMLElement; + end: HTMLElement; + } | null>(null); const rememberEditorSelection = useCallback(() => { const editor = editorRef.current; if (!editor) return; @@ -369,19 +374,6 @@ export const MentionComposer = memo( closeMentionSession(); }, [normalizedWorkdir, closeMentionSession]); - useEffect(() => { - return () => { - mentionSessionRequestSeqRef.current += 1; - if (mentionRefetchTimerRef.current !== null) { - window.clearTimeout(mentionRefetchTimerRef.current); - } - if (busyReleaseTimerRef.current !== null) { - window.clearTimeout(busyReleaseTimerRef.current); - } - setBusy(false); - }; - }, [setBusy]); - useEffect(() => { if (!disabled) return; closeMentionSession(); @@ -501,6 +493,33 @@ export const MentionComposer = memo( applyEmptyState(editorTextIsEmpty(el), editorHasNoContent(el)); }, [applyEmptyState]); + const clearTransientText = useCallback( + (preserveLastText: boolean) => { + const active = transientTextRef.current; + transientTextRef.current = null; + if (!active) return; + if (!preserveLastText) active.textNode.remove(); + active.start.remove(); + active.end.remove(); + refreshEmptyState(); + }, + [refreshEmptyState], + ); + + useEffect(() => { + return () => { + mentionSessionRequestSeqRef.current += 1; + if (mentionRefetchTimerRef.current !== null) { + window.clearTimeout(mentionRefetchTimerRef.current); + } + if (busyReleaseTimerRef.current !== null) { + window.clearTimeout(busyReleaseTimerRef.current); + } + clearTransientText(false); + setBusy(false); + }; + }, [clearTransientText, setBusy]); + // ---- Typewriter (typeText) ---- // While a run is active the editor drops contentEditable so keyboard and // IME input cannot interleave user text with the scripted text. @@ -834,9 +853,74 @@ export const MentionComposer = memo( closeMentionSession(); refreshEmptyState(); }, + beginTransientText: () => { + const el = editorRef.current; + if (!el || disabled || isTypewriting) return false; + clearTransientText(false); + finishTypewriter(); + resetPromptHistoryRecall(); + focusEditorAtSavedSelection(); + const selection = window.getSelection(); + const range = + selection && selection.rangeCount > 0 ? selection.getRangeAt(0).cloneRange() : null; + if (!range || !editorRangeIsInsideRoot(el, range)) { + const fallback = document.createRange(); + fallback.selectNodeContents(el); + fallback.collapse(false); + if (!selection) return false; + selection.removeAllRanges(); + selection.addRange(fallback); + } + const activeRange = selection?.getRangeAt(0); + if (!activeRange) return false; + activeRange.collapse(true); + const start = document.createElement("span"); + const end = document.createElement("span"); + start.dataset.sttMarker = "start"; + end.dataset.sttMarker = "end"; + start.contentEditable = "false"; + end.contentEditable = "false"; + start.setAttribute("aria-hidden", "true"); + end.setAttribute("aria-hidden", "true"); + start.style.display = "none"; + end.style.display = "none"; + const textNode = document.createTextNode(""); + activeRange.insertNode(end); + activeRange.insertNode(textNode); + activeRange.insertNode(start); + transientTextRef.current = { textNode, start, end }; + refreshEmptyState(); + return true; + }, + updateTransientText: (text: string) => { + const active = transientTextRef.current; + if (!active?.textNode.isConnected) return; + active.textNode.data = normalizeLogicalLineEndings(text); + refreshEmptyState(); + }, + commitTransientText: (text?: string) => { + const active = transientTextRef.current; + if (text !== undefined) { + if (active?.textNode.isConnected) + active.textNode.data = normalizeLogicalLineEndings(text); + } + if (active?.end.isConnected) { + const range = document.createRange(); + range.setStartAfter(active.end); + range.collapse(true); + const selection = window.getSelection(); + selection?.removeAllRanges(); + selection?.addRange(range); + } + clearTransientText(true); + }, + cancelTransientText: (options?: { preserveLastText?: boolean }) => { + clearTransientText(options?.preserveLastText === true); + }, clear: () => { const el = editorRef.current; if (!el) return; + clearTransientText(false); cancelTypewriter(); resetPromptHistoryRecall(); el.innerHTML = ""; @@ -934,6 +1018,7 @@ export const MentionComposer = memo( [ buildDraft, cancelTypewriter, + clearTransientText, closeCommitTooltip, closeComposerContextMenu, closeMentionSession, @@ -943,6 +1028,8 @@ export const MentionComposer = memo( placeCaretAtEditorEnd, refreshEmptyState, resetPromptHistoryRecall, + disabled, + isTypewriting, ], ); diff --git a/crates/agent-ui/src/components/chat/MentionComposerModel.ts b/crates/agent-ui/src/components/chat/MentionComposerModel.ts index 5cfa3519f..9755f5f9d 100644 --- a/crates/agent-ui/src/components/chat/MentionComposerModel.ts +++ b/crates/agent-ui/src/components/chat/MentionComposerModel.ts @@ -100,6 +100,10 @@ export interface MentionComposerHandle { * has landed in the editor (or the run was cancelled). */ typeText: (text: string) => Promise; + beginTransientText: () => boolean; + updateTransientText: (text: string) => void; + commitTransientText: (text?: string) => void; + cancelTransientText: (options?: { preserveLastText?: boolean }) => void; } export type MentionComposerLargePaste = { diff --git a/crates/agent-ui/src/i18n/translations/enUSSettings.ts b/crates/agent-ui/src/i18n/translations/enUSSettings.ts index 71fe6d889..c11ba91ce 100644 --- a/crates/agent-ui/src/i18n/translations/enUSSettings.ts +++ b/crates/agent-ui/src/i18n/translations/enUSSettings.ts @@ -1,6 +1,7 @@ export const EN_US_SETTINGS_TRANSLATIONS = { "settings.navSystem": "System", "settings.navSystemTools": "System Tools", + "settings.navStt": "Speech to Text", "settings.toolPermissionsOverridden": "{count} customized", "settings.toolPolicy.allow": "Allow", "settings.toolPolicy.ask": "Ask", diff --git a/crates/agent-ui/src/i18n/translations/zhCNSettings.ts b/crates/agent-ui/src/i18n/translations/zhCNSettings.ts index 6c382d1a5..832c00afe 100644 --- a/crates/agent-ui/src/i18n/translations/zhCNSettings.ts +++ b/crates/agent-ui/src/i18n/translations/zhCNSettings.ts @@ -1,6 +1,7 @@ export const ZH_CN_SETTINGS_TRANSLATIONS = { "settings.navSystem": "系统设置", "settings.navSystemTools": "系统工具", + "settings.navStt": "语音输入", "settings.toolPermissionsOverridden": "{count} 项已自定义", "settings.toolPolicy.allow": "允许", "settings.toolPolicy.ask": "询问", diff --git a/crates/agent-ui/src/lib/settings/index.ts b/crates/agent-ui/src/lib/settings/index.ts index af3316f86..f392b6124 100644 --- a/crates/agent-ui/src/lib/settings/index.ts +++ b/crates/agent-ui/src/lib/settings/index.ts @@ -93,6 +93,9 @@ import type { SshProxyConfig, SshProxyType, SshSettings, + SttProviderId, + SttProviderSettings, + SttSettings, SystemProxyConfig, SystemSettings, ToolPolicy, @@ -584,6 +587,109 @@ export function normalizeRemoteSettings(input: unknown): RemoteSettings { }; } +export const STT_PROVIDER_IDS: readonly SttProviderId[] = [ + "tencent_cloud", + "volcengine_seed_v3", + "aliyun_dashscope", + "baidu_cloud", +]; + +function defaultSttProvider(id: SttProviderId): SttProviderSettings { + const providerDefaults: Partial = + id === "aliyun_dashscope" + ? { + websocketUrl: "wss://dashscope.aliyuncs.com/api-ws/v1/inference/", + model: "paraformer-realtime-v2", + } + : id === "volcengine_seed_v3" + ? { + websocketUrl: "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async", + } + : id === "baidu_cloud" + ? { websocketUrl: "wss://vop.baidu.com/realtime_asr" } + : {}; + return { + id, + configured: false, + websocketUrl: "", + model: "", + apiKey: "", + appId: "", + secretId: "", + secretKey: "", + accessToken: "", + cluster: "", + resourceId: "", + engineModelType: "16k_zh", + baiduAppId: "", + baiduApiKey: "", + devPid: "", + ...providerDefaults, + }; +} + +export function getDefaultSttSettings(): SttSettings { + return { + enabled: false, + provider: null, + providers: Object.fromEntries( + STT_PROVIDER_IDS.map((id) => [id, defaultSttProvider(id)]), + ) as Record, + }; +} + +export function normalizeSttSettings(input: unknown): SttSettings { + const defaults = getDefaultSttSettings(); + const obj = (input && typeof input === "object" ? input : {}) as Record; + const provider = STT_PROVIDER_IDS.includes(obj.provider as SttProviderId) + ? (obj.provider as SttProviderId) + : null; + const rawProviders = + obj.providers && typeof obj.providers === "object" + ? (obj.providers as Record) + : {}; + const providers = Object.fromEntries( + STT_PROVIDER_IDS.map((id) => { + const raw = + rawProviders[id] && typeof rawProviders[id] === "object" + ? (rawProviders[id] as Record) + : {}; + const base = defaults.providers[id]; + const text = (key: string) => (typeof raw[key] === "string" ? raw[key].trim() : ""); + return [ + id, + { + ...base, + configured: raw.configured === true, + websocketUrl: text("websocketUrl") || base.websocketUrl, + model: + text("model") === "paraformer-realtime-8k-v2" + ? "paraformer-realtime-v2" + : text("model") || base.model, + apiKey: text("apiKey"), + appId: text("appId"), + secretId: text("secretId"), + secretKey: text("secretKey"), + accessToken: text("accessToken"), + cluster: text("cluster"), + resourceId: text("resourceId"), + engineModelType: text("engineModelType") || base.engineModelType, + baiduAppId: text("baiduAppId"), + baiduApiKey: text("baiduApiKey"), + devPid: text("devPid"), + ...(raw.clearSecrets === true ? { clearSecrets: true } : {}), + } satisfies SttProviderSettings, + ]; + }), + ) as Record; + return { + enabled: obj.enabled === true, + provider, + providers, + ...(obj.allowIncomplete === true ? { allowIncomplete: true } : {}), + }; +} + function getKnownModelLimits( providerId: ProviderId, modelId: string | undefined, @@ -1417,6 +1523,7 @@ export function getDefaultSettings(): AppSettings { enableWebGit: false, enableWebTunnels: false, }, + stt: getDefaultSttSettings(), memory: normalizeMemorySettings({}, customProviders), customSettings: normalizeCustomSettings({}, customProviders), modelFailover: normalizeModelFailoverSettings({}, customProviders), @@ -1452,6 +1559,7 @@ export function normalizeSettings(input?: Partial | null): AppSetti agents: normalizeAgentPromptTemplates(obj.agents ?? defaults.agents), ssh: normalizeSshSettings(obj.ssh ?? defaults.ssh), remote: normalizeRemoteSettings(obj.remote ?? defaults.remote), + stt: normalizeSttSettings(obj.stt ?? defaults.stt), memory: normalizeMemorySettings(obj.memory ?? defaults.memory, customProviders), customSettings: normalizeCustomSettings( obj.customSettings ?? defaults.customSettings, diff --git a/crates/agent-ui/src/lib/settings/sync.ts b/crates/agent-ui/src/lib/settings/sync.ts index 8692d689a..ed6477c78 100644 --- a/crates/agent-ui/src/lib/settings/sync.ts +++ b/crates/agent-ui/src/lib/settings/sync.ts @@ -47,6 +47,7 @@ export type GatewaySettingsSyncProvider = Omit; +export type GatewaySttSecretUpdate = AppSettings["stt"]; export type GatewaySettingsSyncPayload = { system: AppSettings["system"]; @@ -58,6 +59,8 @@ export type GatewaySettingsSyncPayload = { AppSettings["remote"], "enableWebTerminal" | "enableWebSshTerminal" | "enableWebGit" | "enableWebTunnels" >; + /** STT 元数据与 configured 标记；所有云厂商凭据在同步出口均为空串。 */ + stt: AppSettings["stt"]; memory: AppSettings["memory"]; modelFailover: AppSettings["modelFailover"]; customSettings: GatewaySettingsSyncCustomSettings; @@ -73,6 +76,8 @@ export type GatewaySettingsSyncPayload = { // systemProxy 密码回传 sidecar（仿 providerApiKeyUpdates 的简化范式）： // system 字段本身出口必被脱敏，明文密码只经此通道回到桌面端落库。 systemProxyPasswordUpdate?: string; + /** WebUI → 桌面端的一次性 STT 凭据更新；任何公开广播前必须移除。 */ + sttSecretUpdate?: GatewaySttSecretUpdate; }; export type GatewaySettingsSyncUpdatePayload = Partial; @@ -83,6 +88,7 @@ const GATEWAY_SETTINGS_SYNC_FIELDS = [ "agents", "ssh", "remote", + "stt", "memory", "modelFailover", "customSettings", @@ -179,9 +185,34 @@ export function redactSettingsForWebStorage(settings: AppSettings): AppSettings }, customProviders: redactCustomProvidersForWebStorage(settings.customProviders), ssh: redactSshSettingsForWebStorage(settings.ssh), + stt: redactSttSettingsForWebStorage(settings.stt), }); } +export function redactSttSettingsForWebStorage(stt: AppSettings["stt"]): AppSettings["stt"] { + const { allowIncomplete: _allowIncomplete, ...publicStt } = stt; + return { + enabled: publicStt.enabled, + provider: publicStt.provider, + providers: Object.fromEntries( + Object.entries(stt.providers).map(([id, provider]) => { + const { clearSecrets: _clearSecrets, ...publicProvider } = provider; + return [ + id, + { + ...publicProvider, + apiKey: "", + secretId: "", + secretKey: "", + accessToken: "", + baiduApiKey: "", + }, + ]; + }), + ) as AppSettings["stt"]["providers"], + }; +} + function redactSystemProxyConfig( proxy: AppSettings["system"]["systemProxy"], ): AppSettings["system"]["systemProxy"] { @@ -854,6 +885,35 @@ function mergeSyncedRemoteSettings( }; } +const STT_SECRET_FIELDS = [ + "apiKey", + "secretId", + "secretKey", + "accessToken", + "baiduApiKey", +] as const satisfies readonly (keyof AppSettings["stt"]["providers"][keyof AppSettings["stt"]["providers"]])[]; + +/** + * 合并脱敏后的 STT 快照。configured 由权威端给出；空白秘密只表示“已脱敏”， + * 不能清除接收端可能持有的本地凭据。 + */ +function mergeSyncedSttSettings( + current: AppSettings["stt"], + incoming: unknown, +): AppSettings["stt"] { + const normalized = normalizeSettings({ stt: incoming as AppSettings["stt"] }).stt; + for (const [id, provider] of Object.entries(normalized.providers)) { + const currentProvider = current.providers[id as keyof typeof current.providers]; + if (!currentProvider) continue; + for (const field of STT_SECRET_FIELDS) { + if (provider.clearSecrets !== true && !provider[field].trim()) { + provider[field] = currentProvider[field]; + } + } + } + return normalized; +} + function mergeSyncedSshSettings( current: AppSettings["ssh"], incoming: unknown, @@ -1102,6 +1162,10 @@ export function buildGatewaySettingsSyncPayload( settings: AppSettings, options: { includeProviderApiKeyUpdates?: boolean } = {}, ): GatewaySettingsSyncPayload { + const stt = redactSttSettingsForWebStorage(settings.stt); + if (settings.stt.allowIncomplete === true) { + stt.allowIncomplete = true; + } const payload: GatewaySettingsSyncPayload = { system: syncableSystemSettings(settings.system), customProviders: redactCustomProvidersForGateway(settings.customProviders), @@ -1114,6 +1178,7 @@ export function buildGatewaySettingsSyncPayload( enableWebGit: settings.remote.enableWebGit, enableWebTunnels: settings.remote.enableWebTunnels, }, + stt, memory: settings.memory, modelFailover: settings.modelFailover, customSettings: syncableCustomSettings(settings.customSettings), @@ -1280,5 +1345,10 @@ export function applyGatewaySettingsSyncPayload( remote: Object.hasOwn(source, "remote") ? mergeSyncedRemoteSettings(current.remote, source.remote) : current.remote, + stt: Object.hasOwn(source, "sttSecretUpdate") + ? mergeSyncedSttSettings(current.stt, source.sttSecretUpdate) + : Object.hasOwn(source, "stt") + ? mergeSyncedSttSettings(current.stt, source.stt) + : current.stt, }); } diff --git a/crates/agent-ui/src/lib/settings/types.ts b/crates/agent-ui/src/lib/settings/types.ts index 3326f0f2a..b75ba95bb 100644 --- a/crates/agent-ui/src/lib/settings/types.ts +++ b/crates/agent-ui/src/lib/settings/types.ts @@ -495,6 +495,40 @@ export type RemoteSettings = { enableWebTunnels: boolean; }; +export type SttProviderId = + | "tencent_cloud" + | "volcengine_seed_v3" + | "aliyun_dashscope" + | "baidu_cloud"; + +export type SttProviderSettings = { + id: SttProviderId; + configured: boolean; + websocketUrl: string; + model: string; + apiKey: string; + appId: string; + secretId: string; + secretKey: string; + accessToken: string; + cluster: string; + resourceId: string; + engineModelType: string; + baiduAppId: string; + baiduApiKey: string; + devPid: string; + /** 一次性清密钥指令；保存端消费后必须移除，不得进入公开快照。 */ + clearSecrets?: boolean; +}; + +export type SttSettings = { + enabled: boolean; + provider: SttProviderId | null; + providers: Record; + /** 一次性允许仅切换语音输入开关，不因当前供应商未配置而拒绝保存。 */ + allowIncomplete?: boolean; +}; + export type AppSettings = { system: SystemSettings; customProviders: CustomProvider[]; @@ -502,6 +536,7 @@ export type AppSettings = { agents: AgentPromptTemplate[]; ssh: SshSettings; remote: RemoteSettings; + stt: SttSettings; memory: MemorySettings; customSettings: CustomSettings; modelFailover: ModelFailoverSettings; diff --git a/crates/agent-ui/src/lib/stt/audio.ts b/crates/agent-ui/src/lib/stt/audio.ts new file mode 100644 index 000000000..e26523704 --- /dev/null +++ b/crates/agent-ui/src/lib/stt/audio.ts @@ -0,0 +1,376 @@ +export const STT_SAMPLE_RATE = 16_000; +export const STT_CHANNELS = 1; +export const STT_CHUNK_MS = 100; +export const STT_SAMPLES_PER_CHUNK = (STT_SAMPLE_RATE * STT_CHUNK_MS) / 1000; +export const STT_TAIL_SILENCE_MS = 400; +export const STT_SILENCE_TIMEOUT_MS = 3_000; +export const STT_CONNECT_TIMEOUT_MS = 10_000; +export const STT_SEND_QUEUE_TIMEOUT_MS = 10_000; +export const STT_MAX_BUFFER_BYTES = STT_SAMPLE_RATE * 2 * 10; + +export type PcmChunk = { sequence: number; pcm: Int16Array; durationMs: number }; + +export class SttPcmFifo { + private readonly chunks: PcmChunk[] = []; + private bytes = 0; + constructor(private readonly maxBytes = STT_MAX_BUFFER_BYTES) {} + get sizeBytes() { + return this.bytes; + } + get length() { + return this.chunks.length; + } + push(chunk: PcmChunk) { + const chunkBytes = chunk.pcm.byteLength; + if (this.bytes + chunkBytes > this.maxBytes) return false; + this.chunks.push(chunk); + this.bytes += chunkBytes; + return true; + } + drain() { + const output = this.chunks.splice(0); + this.bytes = 0; + return output; + } + clear() { + this.chunks.splice(0); + this.bytes = 0; + } +} + +export function pcm16ToLittleEndianBytes(samples: Int16Array): Uint8Array { + const bytes = new Uint8Array(samples.length * 2); + const view = new DataView(bytes.buffer); + for (let index = 0; index < samples.length; index += 1) { + view.setInt16(index * 2, samples[index] ?? 0, true); + } + return bytes; +} + +export function appendTailSilence( + samples: number = (STT_SAMPLE_RATE * STT_TAIL_SILENCE_MS) / 1000, +) { + return new Int16Array(Math.round(samples)); +} + +export function resampleTo16k(input: Float32Array, inputRate: number): Float32Array { + if (!Number.isFinite(inputRate) || inputRate <= 0 || inputRate === STT_SAMPLE_RATE) + return input.slice(); + const outputLength = Math.max(1, Math.round((input.length * STT_SAMPLE_RATE) / inputRate)); + const output = new Float32Array(outputLength); + const ratio = inputRate / STT_SAMPLE_RATE; + for (let index = 0; index < outputLength; index += 1) { + const position = index * ratio; + const left = Math.floor(position); + const right = Math.min(left + 1, input.length - 1); + const fraction = position - left; + output[index] = (input[left] ?? 0) * (1 - fraction) + (input[right] ?? 0) * fraction; + } + return output; +} + +/** + * Linear resampler for a continuous microphone stream. Web Audio delivers + * independent blocks, but interpolation phase must carry across block + * boundaries or 44.1/48 kHz input slowly loses samples and develops clicks. + */ +export class SttStreamingResampler { + private readonly ratio: number; + private buffered = new Float32Array(0); + private bufferStart = 0; + private received = 0; + private outputIndex = 0; + + constructor( + private readonly inputRate: number, + private readonly outputRate = STT_SAMPLE_RATE, + ) { + if (!Number.isFinite(inputRate) || inputRate <= 0) { + throw new Error("麦克风采样率无效"); + } + if (!Number.isFinite(outputRate) || outputRate <= 0) { + throw new Error("语音识别采样率无效"); + } + this.ratio = inputRate / outputRate; + } + + process(input: Float32Array): Float32Array { + if (input.length === 0) return new Float32Array(0); + if (this.inputRate === this.outputRate) return input.slice(); + + const joined = new Float32Array(this.buffered.length + input.length); + joined.set(this.buffered); + joined.set(input, this.buffered.length); + this.buffered = joined; + this.received += input.length; + + const output: number[] = []; + while (this.sourcePosition() + 1 < this.received) { + const relativePosition = this.sourcePosition() - this.bufferStart; + const left = Math.floor(relativePosition); + const fraction = relativePosition - left; + output.push( + (this.buffered[left] ?? 0) * (1 - fraction) + + (this.buffered[left + 1] ?? 0) * fraction, + ); + this.outputIndex += 1; + } + this.compact(); + return Float32Array.from(output); + } + + flush(): Float32Array { + if (this.inputRate === this.outputRate || this.buffered.length === 0) { + this.reset(); + return new Float32Array(0); + } + + const output: number[] = []; + while (this.sourcePosition() < this.received - Number.EPSILON * this.received) { + const relativePosition = this.sourcePosition() - this.bufferStart; + const left = Math.floor(relativePosition); + const right = Math.min(left + 1, this.buffered.length - 1); + const fraction = relativePosition - left; + output.push( + (this.buffered[left] ?? 0) * (1 - fraction) + + (this.buffered[right] ?? 0) * fraction, + ); + this.outputIndex += 1; + } + this.reset(); + return Float32Array.from(output); + } + + reset() { + this.buffered = new Float32Array(0); + this.bufferStart = 0; + this.received = 0; + this.outputIndex = 0; + } + + private compact() { + // Keep one real source sample for interpolation with the next Web Audio block. + const keepFrom = Math.min(Math.floor(this.sourcePosition()), this.received - 1); + const consumed = Math.max(0, keepFrom - this.bufferStart); + if (consumed <= 0) return; + this.buffered = this.buffered.slice(consumed); + this.bufferStart += consumed; + } + + private sourcePosition() { + return this.outputIndex * this.ratio; + } +} + +export function floatToPcm16(input: Float32Array): Int16Array { + const output = new Int16Array(input.length); + for (let index = 0; index < input.length; index += 1) { + const sample = Math.max(-1, Math.min(1, input[index] ?? 0)); + output[index] = sample < 0 ? Math.round(sample * 0x8000) : Math.round(sample * 0x7fff); + } + return output; +} + +export function pcmRms(samples: Float32Array): number { + if (samples.length === 0) return 0; + let sum = 0; + for (const sample of samples) sum += sample * sample; + return Math.sqrt(sum / samples.length); +} + +export type SttAudioCaptureOptions = { + onChunk: (chunk: PcmChunk, hasVoice: boolean) => void; + onSilenceTimeout?: () => void; + onCaptureError?: (message: string) => void; + vadThreshold?: number; +}; + +export class SttAudioCapture { + private context: AudioContext | null = null; + private stream: MediaStream | null = null; + private source: MediaStreamAudioSourceNode | null = null; + private processor: ScriptProcessorNode | null = null; + private sink: GainNode | null = null; + private resampler: SttStreamingResampler | null = null; + private pending = new Float32Array(0); + private sequence = 0; + private lastVoiceAt = 0; + private silenceTimer: number | null = null; + private stopped = true; + private readonly threshold: number; + private baselineSamples: number[] = []; + + constructor(private readonly options: SttAudioCaptureOptions) { + this.threshold = options.vadThreshold ?? 0.012; + } + + async start() { + if (!navigator.mediaDevices?.getUserMedia) throw new Error("当前环境不支持麦克风"); + try { + const preferredConstraints: MediaStreamConstraints = { + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + video: false, + }; + try { + this.stream = await navigator.mediaDevices.getUserMedia(preferredConstraints); + } catch (cause) { + if (!isMediaConstraintError(cause)) throw cause; + // Older WebKitGTK/WebKit WebViews reject unknown audio constraints as + // a whole. Let the runtime choose its supported microphone settings; + // the AudioContext resampler still normalizes output to 16 kHz mono. + this.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + } + this.context = new AudioContext({ latencyHint: "interactive" }); + await this.context.resume(); + this.resampler = new SttStreamingResampler(this.context.sampleRate); + this.source = this.context.createMediaStreamSource(this.stream); + this.processor = this.context.createScriptProcessor(4096, 1, 1); + this.processor.onaudioprocess = (event) => this.consume(event.inputBuffer.getChannelData(0)); + for (const track of this.stream.getAudioTracks()) { + track.addEventListener("ended", this.handleTrackEnded, { once: true }); + } + // ScriptProcessor must stay in the graph to receive callbacks, but + // connecting it to destination would play the microphone through speakers. + this.sink = this.context.createGain(); + this.sink.gain.value = 0; + this.source.connect(this.processor); + this.processor.connect(this.sink); + this.sink.connect(this.context.destination); + this.stopped = false; + this.baselineSamples = []; + this.resetSilenceClock(); + this.silenceTimer = window.setInterval(() => { + if (!this.stopped && performance.now() - this.lastVoiceAt >= STT_SILENCE_TIMEOUT_MS) + this.options.onSilenceTimeout?.(); + }, 250); + } catch (cause) { + await this.releaseResources(); + throw new Error(mediaCaptureErrorMessage(cause)); + } + } + + resetSilenceClock() { + this.lastVoiceAt = performance.now(); + } + + private readonly handleTrackEnded = () => { + if (!this.stopped) this.options.onCaptureError?.("麦克风设备已断开或权限已撤销"); + }; + + private consume(input: Float32Array) { + if (this.stopped || !this.context) return; + const resampled = this.resampler?.process(input) ?? new Float32Array(0); + this.appendPending(resampled); + this.emitFullChunks(); + } + + private appendPending(samples: Float32Array) { + if (samples.length === 0) return; + const merged = new Float32Array(this.pending.length + samples.length); + merged.set(this.pending); + merged.set(samples, this.pending.length); + this.pending = merged; + } + + private emitFullChunks() { + while (this.pending.length >= STT_SAMPLES_PER_CHUNK) { + const samples = this.pending.slice(0, STT_SAMPLES_PER_CHUNK); + this.pending = this.pending.slice(STT_SAMPLES_PER_CHUNK); + const rms = pcmRms(samples); + if (this.baselineSamples.length < 10 && rms < 0.02) this.baselineSamples.push(rms); + const baseline = this.baselineSamples.length + ? this.baselineSamples.reduce((sum, value) => sum + value, 0) / this.baselineSamples.length + : 0.003; + const threshold = this.options.vadThreshold === undefined + ? Math.max(0.005, Math.min(0.009, baseline * 1.8)) + : this.threshold; + const hasVoice = rms >= threshold; + if (hasVoice) this.lastVoiceAt = performance.now(); + this.options.onChunk( + { sequence: this.sequence++, pcm: floatToPcm16(samples), durationMs: STT_CHUNK_MS }, + hasVoice, + ); + } + } + + async stop() { + if (this.stopped && !this.stream && !this.context && !this.processor && !this.source) return; + this.stopped = true; + if (this.processor) this.processor.onaudioprocess = null; + this.appendPending(this.resampler?.flush() ?? new Float32Array(0)); + if (this.pending.length > 0) { + const samples = this.pending; + this.pending = new Float32Array(0); + const hasVoice = pcmRms(samples) >= this.threshold; + this.options.onChunk( + { + sequence: this.sequence++, + pcm: floatToPcm16(samples), + durationMs: (samples.length * 1000) / STT_SAMPLE_RATE, + }, + hasVoice, + ); + } + await this.releaseResources(); + } + + private async releaseResources() { + if (this.silenceTimer !== null) window.clearInterval(this.silenceTimer); + this.silenceTimer = null; + this.processor?.disconnect(); + this.sink?.disconnect(); + this.source?.disconnect(); + this.processor = null; + this.sink = null; + this.source = null; + this.stream?.getTracks().forEach((track) => { + track.removeEventListener("ended", this.handleTrackEnded); + track.stop(); + }); + this.stream = null; + await this.context?.close().catch(() => undefined); + this.context = null; + this.resampler?.reset(); + this.resampler = null; + this.pending = new Float32Array(0); + this.baselineSamples = []; + } +} + +function mediaCaptureErrorMessage(cause: unknown): string { + const name = mediaErrorName(cause); + if (name === "NotAllowedError" || name === "SecurityError") { + return "麦克风权限已拒绝，请在系统设置中允许访问"; + } + if (name === "NotFoundError" || name === "DevicesNotFoundError") { + return "未检测到可用麦克风设备"; + } + if (name === "NotReadableError" || name === "TrackStartError") { + return "麦克风设备不可用或正被其他应用占用"; + } + if (name === "OverconstrainedError" || name === "ConstraintNotSatisfiedError") { + return "未检测到可用麦克风设备，或设备不支持录音参数"; + } + return cause instanceof Error && cause.message ? cause.message : "无法启动麦克风"; +} + +function isMediaConstraintError(cause: unknown): boolean { + const name = mediaErrorName(cause); + const message = cause instanceof Error ? cause.message.toLowerCase() : ""; + return ( + name === "OverconstrainedError" || + name === "ConstraintNotSatisfiedError" || + name === "TypeError" || + message.includes("constraint") + ); +} + +function mediaErrorName(cause: unknown): string { + if (!cause || typeof cause !== "object" || !("name" in cause)) return ""; + return typeof cause.name === "string" ? cause.name : ""; +} diff --git a/crates/agent-ui/src/lib/stt/types.ts b/crates/agent-ui/src/lib/stt/types.ts new file mode 100644 index 000000000..710e1fd52 --- /dev/null +++ b/crates/agent-ui/src/lib/stt/types.ts @@ -0,0 +1,63 @@ +import type { AppSettings, SttProviderId, SttProviderSettings } from "@liveagent/app/lib/settings"; + +export type SttUiState = + | "idle" + | "requesting-permission" + | "buffering" + | "recognizing" + | "stopping" + | "error"; +export type SttRuntimeEvent = + | { type: "ready"; sessionId: string } + | { type: "partial"; sessionId: string; text: string } + | { type: "final"; sessionId: string; text: string } + | { type: "error"; sessionId: string; code: string; message: string } + | { type: "closed"; sessionId: string }; + +export type SttTransportOpenOptions = { + sessionId: string; + provider: SttProviderId; + onEvent: (event: SttRuntimeEvent) => void; +}; + +export interface SttTransport { + requestPermission?: () => Promise; + open: (options: SttTransportOpenOptions) => Promise; + sendAudio: (sessionId: string, sequence: number, pcm: Uint8Array) => Promise; + stop: (sessionId: string) => Promise; + cancel: (sessionId: string) => Promise; + dispose?: () => void; +} + +export type SttConnectionTestResult = + | "connected" + | "connected_no_speech" + | "authentication_failed" + | "protocol_failed" + | "network_failed" + | "timeout"; + +export type SttConnectionTestResponse = { + result: SttConnectionTestResult; + message?: string; +}; + +export type SttSecretField = Extract< + keyof SttProviderSettings, + "apiKey" | "secretId" | "secretKey" | "accessToken" | "baiduApiKey" +>; + +export type SttSecretRevealMode = "value" | "field-name"; + +export interface SttSettingsService { + runtimeLabel?: string; + /** + * Desktop reveals the locally stored value on demand. WebUI deliberately + * reveals only the field label and never requests a credential value. + */ + secretRevealMode?: SttSecretRevealMode; + revealSecret?: (provider: SttProviderId, field: SttSecretField) => Promise; + get: () => Promise; + update: (settings: AppSettings["stt"]) => Promise; + test: (provider: SttProviderId) => Promise; +} diff --git a/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx b/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx index a0ebaf823..812042add 100644 --- a/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx +++ b/crates/agent-ui/src/pages/chat/ChatComposerBar.tsx @@ -5,6 +5,7 @@ import { type ProviderId, type ReasoningLevel, type SelectedModel, + type SttProviderId, } from "@liveagent/app/lib/settings"; import { ComposerAttachmentCard } from "@liveagent/ui/components/chat/ComposerAttachmentCard"; import { ComposerModelControls } from "@liveagent/ui/components/chat/ComposerModelControls"; @@ -22,6 +23,7 @@ import { Clock3, Loader2, Maximize2, + Mic, Minimize2, Paperclip, Play, @@ -36,6 +38,7 @@ import { useLocale } from "@liveagent/ui/i18n/index"; import type { GitClient } from "@liveagent/ui/lib/git/types"; import type { SharedModelOption } from "@liveagent/ui/lib/models/modelOptions"; import { cn } from "@liveagent/ui/lib/shared/utils"; +import type { SttTransport } from "@liveagent/ui/lib/stt/types"; import type { WorkspaceActivityClient } from "@liveagent/ui/lib/workspace-activity/types"; import { type MutableRefObject, @@ -56,6 +59,7 @@ import { type UploadedImagePreviewLoader, } from "../../lib/chat/uploadedImagePreview"; import type { PendingUploadedFile } from "../../lib/chat/uploadTypes"; +import { useComposerStt } from "./useComposerStt"; function useComposerUploadedImagePreview( file: PendingUploadedFile, @@ -216,6 +220,13 @@ export type ChatComposerBarProps = { isSending: boolean; isUploadingFiles: boolean; isInputDisabled: boolean; + sttProvider?: SttProviderId | null; + sttProviderConfigured?: boolean; + sttTransport?: SttTransport; + /** 当前会话身份；切换会话时取消进行中的语音识别。 */ + sttSessionKey?: string; + /** STT 失败（麦克风不可用、连接超时等）上报给宿主以 toast 形式提示。 */ + onSttError?: (message: string) => void; /** * 只读视图（如轨迹页）挂起输入区：整体 display:none 但保持挂载， * 半打的草稿与队列状态在切回聊天页时原样恢复。 @@ -288,6 +299,11 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer isSending, isUploadingFiles, isInputDisabled, + sttProvider = null, + sttProviderConfigured, + sttTransport, + sttSessionKey, + onSttError, hidden = false, inputPlaceholder, workdir, @@ -337,6 +353,16 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer } = props; const { t } = useLocale(); const [composerIsEmpty, setComposerIsEmpty] = useState(true); + const stt = useComposerStt({ + composerRef, + provider: sttProvider, + providerConfigured: sttProviderConfigured, + transport: sttTransport, + disabled: isInputDisabled, + sessionKey: sttSessionKey, + hidden, + onError: onSttError, + }); const [isComposerExpanded, setIsComposerExpanded] = useState(false); const isComposerExpandedRef = useRef(false); const glassCardRef = useRef(null); @@ -361,10 +387,11 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer DEFAULT_QUEUE_SCROLLBAR_STATE, ); const isAgentMode = isAgentExecutionMode(executionMode); - const uploadDisabled = isInputDisabled || isUploadingFiles || !isAgentMode || !workdir; - const controlsDisabled = isInputDisabled; + const uploadDisabled = + isInputDisabled || stt.active || isUploadingFiles || !isAgentMode || !workdir; + const controlsDisabled = isInputDisabled || stt.active; const hasSendableDraft = !composerIsEmpty || pendingUploadedFiles.length > 0; - const sendDisabled = isInputDisabled || isUploadingFiles || !hasSendableDraft; + const sendDisabled = isInputDisabled || stt.active || isUploadingFiles || !hasSendableDraft; const canQueueDraftWhileSending = isSending && !sendDisabled; const primaryActionTitle = canQueueDraftWhileSending ? t("chat.queue.addToQueue") @@ -872,7 +899,7 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer key={`${file.relativePath}-${file.absolutePath ?? file.fileName}`} file={file} workdir={workdir} - disabled={isInputDisabled} + disabled={controlsDisabled} removeLabel={t("chat.upload.removeFile")} previewLabel={t("chat.upload.previewImage")} closePreviewLabel={t("chat.upload.closePreview")} @@ -935,7 +962,7 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer onPasteFiles={onPasteFiles} loadHistoryPrompts={loadHistoryPrompts} placeholder={inputPlaceholder} - disabled={isInputDisabled} + disabled={isInputDisabled || stt.active} workdir={workdir} enabledSkills={enabledSkills} className={cn( @@ -988,6 +1015,35 @@ export const ChatComposerBar = memo(function ChatComposerBar(props: ChatComposer + {stt.available ? ( + + + + ) : null} + ; + connectTimer: number; + finalTimer: number | null; +}; + +export function useComposerStt(options: { + composerRef: MutableRefObject; + provider: SttProviderId | null; + providerConfigured?: boolean; + transport?: SttTransport; + disabled: boolean; + /** 当前会话或视图身份；变化时取消进行中的识别，避免写进已切换的输入框。 */ + sessionKey?: string; + /** 输入区被挂起（如轨迹页）时取消识别，避免用户无法点停止。 */ + hidden?: boolean; + /** 错误上报回调（如麦克风不可用）；由宿主决定展示方式（toast 等）。 */ + onError?: (message: string) => void; +}) { + const { + composerRef, + provider, + providerConfigured, + transport, + disabled, + sessionKey, + hidden = false, + onError, + } = options; + const [state, setState] = useState("idle"); + const [error, setError] = useState(null); + const activeRef = useRef(null); + + const cleanup = useCallback( + (preserveLastText: boolean) => { + const active = activeRef.current; + activeRef.current = null; + if (active) { + window.clearTimeout(active.connectTimer); + if (active.finalTimer !== null) window.clearTimeout(active.finalTimer); + void active.capture.stop(); + active.fifo.clear(); + } + composerRef.current?.cancelTransientText({ preserveLastText }); + }, + [composerRef], + ); + + const fail = useCallback( + (message: string, expectedSessionId?: string) => { + const active = activeRef.current; + if (expectedSessionId && active?.id !== expectedSessionId) return; + if (active) void transport?.cancel(active.id).catch(() => undefined); + const preserve = Boolean(active?.lastText.trim()); + cleanup(preserve); + setError(message); + setState("error"); + onError?.(message); + }, + [cleanup, onError, transport], + ); + + const sendChunk = useCallback( + (sequence: number, pcm: Int16Array) => { + const active = activeRef.current; + if (!active || !transport) return; + const sessionId = active.id; + active.queue = active.queue + .then(() => transport.sendAudio(sessionId, sequence, pcm16ToLittleEndianBytes(pcm))) + .catch((cause) => { + fail(errorMessageWithFallback(cause, "发送语音数据失败"), sessionId); + }); + }, + [fail, transport], + ); + + const queueChunk = useCallback( + (chunk: PcmChunk) => { + const active = activeRef.current; + if (!active) return; + active.sequence = Math.max(active.sequence, chunk.sequence + 1); + if (active.ready) { + sendChunk(chunk.sequence, chunk.pcm); + } else if (!active.fifo.push(chunk)) { + fail("云连接超时，语音缓存已达到 10 秒上限"); + } + }, + [fail, sendChunk], + ); + + const finishProvider = useCallback( + async (active: ActiveSttSession) => { + if ( + activeRef.current !== active || + !active.ready || + !active.stopping || + !active.tailQueued || + active.finishSent || + !transport + ) { + return; + } + active.finishSent = true; + let queueTimer = 0; + try { + await Promise.race([ + active.queue, + new Promise((_, reject) => { + queueTimer = window.setTimeout( + () => reject(new Error("发送语音数据超时")), + STT_SEND_QUEUE_TIMEOUT_MS, + ); + }), + ]); + } catch (cause) { + fail(errorMessageWithFallback(cause, "发送语音数据超时"), active.id); + return; + } finally { + window.clearTimeout(queueTimer); + } + if (activeRef.current !== active) return; + try { + await transport.stop(active.id); + active.finalTimer = window.setTimeout( + () => fail("识别结束超时，已保留最后转写内容"), + 5_000, + ); + } catch (cause) { + fail(errorMessageWithFallback(cause, "停止识别失败")); + } + }, + [fail, transport], + ); + + const abortActiveSession = useCallback(() => { + const active = activeRef.current; + if (!active) return; + void transport?.cancel(active.id).catch(() => undefined); + cleanup(Boolean(active.lastText.trim())); + setState("idle"); + }, [cleanup, transport]); + + const stop = useCallback(async () => { + const active = activeRef.current; + if (!active || !transport) return; + if (active.stopping) { + abortActiveSession(); + return; + } + active.stopping = true; + setState("stopping"); + + // stop() flushes the final real-audio fragment before the fixed tail. + try { + await active.capture.stop(); + } catch (cause) { + fail(errorMessageWithFallback(cause, "停止麦克风失败")); + return; + } + const tail = appendTailSilence(); + for (let offset = 0; offset < tail.length; offset += STT_SAMPLES_PER_CHUNK) { + const pcm = tail.slice(offset, offset + STT_SAMPLES_PER_CHUNK); + queueChunk({ + sequence: active.sequence++, + pcm, + durationMs: (pcm.length * 1000) / 16_000, + }); + } + if (activeRef.current === active && active.ready) { + active.tailQueued = true; + await finishProvider(active); + } else if (activeRef.current === active) { + active.tailQueued = true; + } + }, [abortActiveSession, fail, finishProvider, queueChunk, transport]); + + const onEvent = useCallback( + (event: SttRuntimeEvent) => { + const active = activeRef.current; + if (!active || event.sessionId !== active.id) return; + if (event.type === "ready") { + window.clearTimeout(active.connectTimer); + active.ready = true; + // Silence must be measured from recognition start, not mic-open / + // cloud-connect. Otherwise a slow handshake auto-stops immediately. + active.capture.resetSilenceClock(); + if (!active.stopping) setState("recognizing"); + for (const chunk of active.fifo.drain()) sendChunk(chunk.sequence, chunk.pcm); + if (active.stopping && active.tailQueued) void finishProvider(active); + } else if (event.type === "partial") { + active.lastText = event.text; + composerRef.current?.updateTransientText(event.text); + } else if (event.type === "final") { + active.lastText = event.text; + composerRef.current?.commitTransientText(event.text); + cleanup(true); + setState("idle"); + } else if (event.type === "error") { + fail(event.message || "语音识别失败"); + } else if (event.type === "closed") { + if (!active.stopping) { + fail("语音识别连接意外关闭"); + } else { + cleanup(Boolean(active.lastText.trim())); + setState("idle"); + } + } + }, + [cleanup, composerRef, fail, finishProvider, sendChunk], + ); + + const start = useCallback(async () => { + if (!transport || !provider || disabled || activeRef.current) return; + if (providerConfigured === false) { + const message = "STT供应商配置不完整"; + setError(message); + setState("error"); + onError?.(message); + return; + } + setError(null); + setState("requesting-permission"); + if (!composerRef.current?.beginTransientText()) { + setState("error"); + setError("无法锁定当前输入位置"); + onError?.("无法锁定当前输入位置"); + return; + } + + try { + await transport.requestPermission?.(); + const fifo = new SttPcmFifo(); + const capture = new SttAudioCapture({ + onChunk: (chunk) => queueChunk(chunk), + onSilenceTimeout: () => { + const current = activeRef.current; + if (!current?.ready || current.stopping) return; + void stop(); + }, + onCaptureError: (message) => fail(message), + }); + const active: ActiveSttSession = { + id: crypto.randomUUID(), + capture, + fifo, + ready: false, + stopping: false, + tailQueued: false, + finishSent: false, + lastText: "", + sequence: 0, + queue: Promise.resolve(), + connectTimer: 0, + finalTimer: null, + }; + activeRef.current = active; + + // Capture starts before the cloud session opens so the first syllable is buffered. + await capture.start(); + if (activeRef.current !== active) { + await capture.stop(); + return; + } + setState("buffering"); + active.connectTimer = window.setTimeout(() => fail("云端连接超时"), STT_CONNECT_TIMEOUT_MS); + await transport.open({ sessionId: active.id, provider, onEvent }); + } catch (cause) { + fail(errorMessageWithFallback(cause, "无法启动语音识别")); + } + }, [ + composerRef, + disabled, + fail, + onError, + onEvent, + provider, + providerConfigured, + queueChunk, + stop, + transport, + ]); + + const toggle = useCallback(() => (activeRef.current ? void stop() : void start()), [start, stop]); + + const sessionKeyRef = useRef(sessionKey); + useLayoutEffect(() => { + if (sessionKeyRef.current === sessionKey) return; + sessionKeyRef.current = sessionKey; + abortActiveSession(); + }, [abortActiveSession, sessionKey]); + + useLayoutEffect(() => { + if (hidden) abortActiveSession(); + }, [abortActiveSession, hidden]); + + useEffect( + () => () => { + const active = activeRef.current; + if (active) void transport?.cancel(active.id).catch(() => undefined); + cleanup(Boolean(active?.lastText.trim())); + transport?.dispose?.(); + }, + [cleanup, transport], + ); + + useEffect(() => { + if ((disabled || !provider) && activeRef.current) void stop(); + }, [disabled, provider, stop]); + + return { + state, + error, + toggle, + active: state !== "idle" && state !== "error", + available: Boolean(provider && transport), + }; +} diff --git a/crates/agent-ui/src/pages/settings/SettingsPage.tsx b/crates/agent-ui/src/pages/settings/SettingsPage.tsx index c7920ee01..6ce52057a 100644 --- a/crates/agent-ui/src/pages/settings/SettingsPage.tsx +++ b/crates/agent-ui/src/pages/settings/SettingsPage.tsx @@ -1,4 +1,5 @@ import { createSettingsExtension } from "@liveagent/adapters/settingsExtension"; +import type { SttProviderId } from "@liveagent/app/lib/settings"; import type { SettingsPageProps } from "@liveagent/app/pages/settings/types"; import { BookOpen, @@ -7,11 +8,12 @@ import { Cloud, Cpu, Key, + Mic, Settings2, Wrench, Zap, } from "@liveagent/ui/components/IconSet"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { SettingsSectionDefinition, UiExtensionRegistry } from "../../contracts/registry"; import { AgentsSection } from "./AgentsSection"; import { CronSection } from "./CronSection"; @@ -21,10 +23,15 @@ import { ProvidersSection } from "./ProvidersSection"; import { RemoteSection } from "./RemoteSection"; import { SettingsShell } from "./SettingsShell"; import { SshSection } from "./SshSection"; +import { SttSection } from "./SttSection"; import { SystemSettingsForm } from "./SystemSettingsForm"; import { SystemToolsSection } from "./SystemToolsSection"; const EMPTY_SERVICES = {}; +const STT_SELECTED_PROVIDER_CACHE = new WeakMap< + SettingsPageProps["sttSettingsService"], + SttProviderId +>(); export function SettingsPage(props: SettingsPageProps) { const { @@ -35,11 +42,35 @@ export function SettingsPage(props: SettingsPageProps) { initialSection = "system", initialProviderId, hiddenSections = [], + sttSettingsService, + onSttProviderChange, } = props; const [pendingProviderId, setPendingProviderId] = useState(initialProviderId); + const [sttSelectedProvider, setSttSelectedProvider] = useState( + () => + STT_SELECTED_PROVIDER_CACHE.get(sttSettingsService) ?? + settings.stt.provider ?? + "tencent_cloud", + ); + const sttSelectionChangedRef = useRef(STT_SELECTED_PROVIDER_CACHE.has(sttSettingsService)); const extension = createSettingsExtension(props); useEffect(() => setPendingProviderId(initialProviderId), [initialProviderId]); + useEffect(() => { + if (!sttSelectionChangedRef.current) { + setSttSelectedProvider(settings.stt.provider ?? "tencent_cloud"); + } + }, [settings.stt.provider]); + + const handleSttProviderChange = useCallback( + (provider: SttProviderId) => { + sttSelectionChangedRef.current = true; + STT_SELECTED_PROVIDER_CACHE.set(sttSettingsService, provider); + setSttSelectedProvider(provider); + onSttProviderChange?.(provider); + }, + [onSttProviderChange, sttSettingsService], + ); const sections = useMemo[]>( () => [ @@ -103,6 +134,23 @@ export function SettingsPage(props: SettingsPageProps) { icon: , render: () => , }, + { + id: "stt", + groupKey: "settings.groupIntelligence", + groupOrder: 20, + order: 30, + labelKey: "settings.navStt", + icon: , + render: () => ( + + ), + }, { id: "hooks", groupKey: "settings.groupAutomation", @@ -144,7 +192,16 @@ export function SettingsPage(props: SettingsPageProps) { }, ...extension.sections, ], - [extension, pendingProviderId, saveState, setSettings, settings], + [ + extension, + handleSttProviderChange, + pendingProviderId, + saveState, + setSettings, + settings, + sttSelectedProvider, + sttSettingsService, + ], ); const registry: UiExtensionRegistry = { surface: extension.surface, diff --git a/crates/agent-ui/src/pages/settings/SttSection.tsx b/crates/agent-ui/src/pages/settings/SttSection.tsx new file mode 100644 index 000000000..c00ded71d --- /dev/null +++ b/crates/agent-ui/src/pages/settings/SttSection.tsx @@ -0,0 +1,686 @@ +import type { AppSettings, SttProviderId, SttProviderSettings } from "@liveagent/app/lib/settings"; +import { + Brackets, + Check, + CheckCircle2, + Cloud, + Eye, + EyeOff, + Flame, + type IconComponent, + LoaderCircle, + Mic, + PawPrint, + Plug, + Shield, + Trash2, + XCircle, +} from "@liveagent/ui/components/IconSet"; +import { Input } from "@liveagent/ui/components/ui/input"; +import { Switch } from "@liveagent/ui/components/ui/switch"; +import { cn } from "@liveagent/ui/lib/shared/utils"; +import { errorMessageWithFallback } from "@liveagent/ui/lib/shared/value"; +import type { + SttConnectionTestResponse, + SttConnectionTestResult, + SttSecretField, + SttSettingsService, +} from "@liveagent/ui/lib/stt/types"; +import { useCallback, useMemo, useRef, useState } from "react"; + +const PROVIDERS: Array<{ + id: SttProviderId; + label: string; + vendor: string; + fields: Array; + secretFields: Array; +}> = [ + { + id: "tencent_cloud", + label: "腾讯云实时语音识别", + vendor: "Tencent Cloud", + fields: ["appId", "engineModelType", "secretId", "secretKey"], + secretFields: ["secretId", "secretKey"], + }, + { + id: "volcengine_seed_v3", + label: "火山引擎实时语音识别", + vendor: "Volcengine", + fields: ["websocketUrl", "appId", "accessToken", "resourceId"], + secretFields: ["accessToken"], + }, + { + id: "aliyun_dashscope", + label: "阿里云 DashScope", + vendor: "Alibaba Cloud", + fields: ["websocketUrl", "model", "apiKey"], + secretFields: ["apiKey"], + }, + { + id: "baidu_cloud", + label: "百度智能云实时语音识别", + vendor: "Baidu AI Cloud", + fields: ["websocketUrl", "baiduAppId", "devPid", "baiduApiKey"], + secretFields: ["baiduApiKey"], + }, +]; + +// 品牌视觉:logos 图标集没有这四家国内厂商的商标,用贴近品牌意象的 +// lucide 图形 + 品牌色近似值代替(百度=爪印、阿里云=中括号、火山=火焰)。 +const PROVIDER_BRAND: Record< + SttProviderId, + { icon: IconComponent; iconClass: string; boxClass: string } +> = { + tencent_cloud: { + icon: Cloud, + iconClass: "text-sky-600 dark:text-sky-400", + boxClass: "bg-sky-500/10", + }, + volcengine_seed_v3: { + icon: Flame, + iconClass: "text-rose-600 dark:text-rose-400", + boxClass: "bg-rose-500/10", + }, + aliyun_dashscope: { + icon: Brackets, + iconClass: "text-orange-600 dark:text-orange-400", + boxClass: "bg-orange-500/10", + }, + baidu_cloud: { + icon: PawPrint, + iconClass: "text-indigo-600 dark:text-indigo-400", + boxClass: "bg-indigo-500/10", + }, +}; + +function ProviderBrandBadge({ + provider, + className, + iconClassName, +}: { + provider: SttProviderId; + className?: string; + iconClassName?: string; +}) { + const brand = PROVIDER_BRAND[provider]; + const Icon = brand.icon; + return ( + + + + ); +} + +const FIELD_LABELS: Partial> = { + websocketUrl: "实时识别 WebSocket 地址", + model: "模型名称", + apiKey: "API Key", + secretId: "SecretId", + secretKey: "SecretKey", + accessToken: "Access Token", + resourceId: "Resource ID", + engineModelType: "引擎模型（16k_zh）", + baiduAppId: "App ID", + baiduApiKey: "API Key", + devPid: "dev_pid（识别模型编号）", +}; + +const FIELD_PLACEHOLDERS: Partial> = { + model: "paraformer-realtime-v2", + engineModelType: "16k_zh", + resourceId: "火山引擎资源 ID", + baiduAppId: "例如：124151367", + devPid: "请按已开通的实时识别模型填写", +}; + +// The value is deliberately synthetic. A password input renders it as dots +// without placing a saved credential in the page or browser state. +const SAVED_SECRET_MASK = "saved-secret-placeholder"; + +function fieldLabel(provider: SttProviderId, field: keyof SttProviderSettings) { + if (field === "appId") return provider === "tencent_cloud" ? "AppId" : "App ID"; + return FIELD_LABELS[field] ?? field; +} + +function fieldPlaceholder(provider: SttProviderId, field: keyof SttProviderSettings) { + if (field === "appId") { + if (provider === "tencent_cloud") return "腾讯云应用 AppId"; + if (provider === "volcengine_seed_v3") return "火山引擎应用 App ID"; + } + if (field === "apiKey") return "sk-..."; + if (field === "secretId") return "SecretId"; + if (field === "secretKey") return "SecretKey"; + if (field === "accessToken") return "Access Token"; + if (field === "baiduAppId") return "百度语音应用 App ID"; + if (field === "baiduApiKey") return "API Key"; + return FIELD_PLACEHOLDERS[field] ?? ""; +} + +function fieldValue(provider: SttProviderSettings, field: keyof SttProviderSettings) { + return typeof provider[field] === "string" ? (provider[field] as string) : ""; +} + +export function SttSection({ + settings, + setSettings, + service, + selectedProvider, + onSelectedProviderChange, +}: { + settings: AppSettings; + setSettings: (updater: (previous: AppSettings) => AppSettings) => void; + service: SttSettingsService; + selectedProvider: SttProviderId; + onSelectedProviderChange: (provider: SttProviderId) => void; +}) { + const displayedStt = settings.stt; + const definition = useMemo( + () => PROVIDERS.find((item) => item.id === selectedProvider) ?? PROVIDERS[0], + [selectedProvider], + ); + const [draftProviders, setDraftProviders] = useState< + Partial>> + >({}); + const provider = { + ...displayedStt.providers[definition.id], + ...draftProviders[definition.id], + }; + const [draftSecrets, setDraftSecrets] = useState>({}); + const [saving, setSaving] = useState(false); + const [testing, setTesting] = useState(false); + const [clearing, setClearing] = useState(false); + const [testResults, setTestResults] = useState< + Partial> + >({}); + const [error, setError] = useState(null); + const [visibleSecrets, setVisibleSecrets] = useState>({}); + const [revealedSecrets, setRevealedSecrets] = useState>({}); + const [revealingSecret, setRevealingSecret] = useState(null); + const revealRequestRef = useRef(0); + + const resetSecretVisibility = useCallback(() => { + revealRequestRef.current += 1; + setVisibleSecrets({}); + setRevealedSecrets({}); + setRevealingSecret(null); + }, []); + + const toggleSecretVisibility = useCallback( + async (field: SttSecretField) => { + if (visibleSecrets[field]) { + revealRequestRef.current += 1; + setVisibleSecrets((previous) => ({ ...previous, [field]: false })); + setRevealedSecrets((previous) => { + const next = { ...previous }; + delete next[field]; + return next; + }); + return; + } + + setError(null); + if (service.secretRevealMode === "field-name") { + setVisibleSecrets((previous) => ({ ...previous, [field]: true })); + return; + } + if (Object.hasOwn(draftSecrets, field) && draftSecrets[field]) { + setVisibleSecrets((previous) => ({ ...previous, [field]: true })); + return; + } + if (!service.revealSecret) { + setError("当前运行端不支持查看已保存的 STT 密钥"); + return; + } + + const requestId = ++revealRequestRef.current; + setRevealingSecret(field); + try { + const value = await service.revealSecret(definition.id, field); + if (revealRequestRef.current !== requestId) return; + setRevealedSecrets((previous) => ({ ...previous, [field]: value })); + setVisibleSecrets((previous) => ({ ...previous, [field]: true })); + } catch (cause) { + if (revealRequestRef.current !== requestId) return; + setError(errorMessageWithFallback(cause, "无法查看已保存的 STT 密钥")); + } finally { + if (revealRequestRef.current === requestId) setRevealingSecret(null); + } + }, + [definition.id, draftSecrets, service, visibleSecrets], + ); + + const updateProvider = useCallback( + (patch: Partial) => { + setTestResults((previous) => { + const next = { ...previous }; + delete next[definition.id]; + return next; + }); + setDraftProviders((previous) => ({ + ...previous, + [definition.id]: { ...previous[definition.id], ...patch }, + })); + }, + [definition.id], + ); + + const selectProvider = (id: SttProviderId) => { + setError(null); + setDraftSecrets({}); + resetSecretVisibility(); + onSelectedProviderChange(id); + }; + + const save = async (): Promise => { + setSaving(true); + setError(null); + const nextProvider = { + ...provider, + ...draftSecrets, + } as SttProviderSettings; + // save() is never a clear: leftover clearSecrets from the previous + // empty-key write would wipe newly typed credentials. + delete nextProvider.clearSecrets; + const payload = { + ...displayedStt, + provider: definition.id, + providers: { ...displayedStt.providers, [definition.id]: nextProvider }, + }; + try { + const redacted = await service.update(payload); + setSettings((previous) => ({ ...previous, stt: redacted })); + setDraftProviders((previous) => { + const next = { ...previous }; + delete next[definition.id]; + return next; + }); + setDraftSecrets({}); + resetSecretVisibility(); + return true; + } catch (cause) { + setError(errorMessageWithFallback(cause, "STT 配置保存失败")); + return false; + } finally { + setSaving(false); + } + }; + + const clearProviderSecrets = async () => { + setClearing(true); + setError(null); + setTestResults((previous) => { + const next = { ...previous }; + delete next[definition.id]; + return next; + }); + const payload = { + ...displayedStt, + provider: definition.id, + providers: { + ...displayedStt.providers, + [definition.id]: { + ...displayedStt.providers[definition.id], + clearSecrets: true, + }, + }, + }; + try { + const redacted = await service.update(payload); + // The service has already cleared the secret, but the host settings + // layer persists every local state change as well. Keep the explicit + // clear marker through that second write so an incomplete provider is + // not rejected as an accidental partial configuration. + setSettings((previous) => ({ + ...previous, + stt: { + ...redacted, + providers: { + ...redacted.providers, + [definition.id]: { + ...redacted.providers[definition.id], + clearSecrets: true, + }, + }, + }, + })); + setDraftSecrets({}); + resetSecretVisibility(); + } catch (cause) { + setError(errorMessageWithFallback(cause, "STT 密钥清空失败")); + } finally { + setClearing(false); + } + }; + + const test = async () => { + if (!(await save())) return; + setTesting(true); + setError(null); + try { + const result = await service.test(definition.id); + setTestResults((previous) => ({ ...previous, [definition.id]: result })); + } catch (cause) { + setError(errorMessageWithFallback(cause, "连接测试失败")); + } finally { + setTesting(false); + } + }; + + const resultLabel: Record = { + connected: "连接成功", + connected_no_speech: "连接成功，未检测到有效语音", + authentication_failed: "鉴权失败", + protocol_failed: "协议错误", + network_failed: "网络错误", + timeout: "连接超时", + }; + const testResult = testResults[definition.id] ?? null; + const testPassed = + testResult?.result === "connected" || testResult?.result === "connected_no_speech"; + + return ( +
+
+
+ +
+
+

语音输入

+

+ 桌面端配置会同步到 Gateway WebUI；浏览器仅接收脱敏配置，录音统一为 16 kHz 单声道 PCM。 +

+ {service.runtimeLabel ? ( +

+ 当前运行端：{service.runtimeLabel} +

+ ) : null} +
+
+
+
+
开启语音输入
+

+ 开启后，聊天输入框会显示麦克风按钮；关闭后不会启用麦克风。 +

+
+ + setSettings((previous) => ({ + ...previous, + stt: { ...previous.stt, enabled, allowIncomplete: true }, + })) + } + aria-label="开启语音输入" + /> +
+
+ {PROVIDERS.map((item) => { + const active = item.id === definition.id; + const configured = displayedStt.providers[item.id].configured; + return ( + + ); + })} +
+
+
+
+ +
+
+ {definition.label} +
+
+ {provider.configured + ? "凭据已保存，可直接使用语音输入" + : "填写凭据后保存并测试连接"} +
+
+
+ {provider.configured ? ( + + + 密钥已脱敏保存 + + ) : null} +
+
+ {definition.fields.map((field) => { + const secret = definition.secretFields.includes(field); + const secretField = secret ? (field as SttSecretField) : null; + const visible = secretField ? visibleSecrets[secretField] === true : false; + const hasDraft = secretField ? Object.hasOwn(draftSecrets, secretField) : false; + const value = !secretField + ? fieldValue(provider, field) + : visible && service.secretRevealMode === "field-name" + ? fieldLabel(definition.id, field) + : hasDraft + ? (draftSecrets[secretField] ?? "") + : visible + ? (revealedSecrets[secretField] ?? "") + : provider.configured + ? SAVED_SECRET_MASK + : ""; + const inputId = `stt-${definition.id}-${String(field)}`; + return ( +
+ +
+ { + if (secret && provider.configured && !hasDraft && !visible) { + event.currentTarget.select(); + } + }} + onClick={(event) => { + if (secret && provider.configured && !hasDraft && !visible) { + event.currentTarget.select(); + } + }} + onChange={(event) => { + if (secret) { + setTestResults((previous) => { + const next = { ...previous }; + delete next[definition.id]; + return next; + }); + setDraftSecrets((old) => ({ ...old, [field]: event.target.value })); + return; + } + updateProvider({ + [field]: event.target.value, + } as Partial); + }} + /> + {secretField ? ( + + ) : null} +
+
+ ); + })} +
+ {service.secretRevealMode === "field-name" ? ( +

+ WebUI 的查看按钮只显示字段名；已保存的密钥内容不会下发到浏览器。 +

+ ) : null} + {definition.id === "baidu_cloud" ? ( +

+ appid 必须是数字；dev_pid 不提供默认值，请按百度模型填写。 +

+ ) : null} +
+ + +
+ {testResult ? ( +
+ {testPassed ? ( + + ) : ( + + )} +
+
{resultLabel[testResult.result]}
+ {testResult.message ? ( +

+ {testResult.message} +

+ ) : null} +
+
+ ) : null} + {error ? ( +
+ +

{error}

+
+ ) : null} +
+
+ ); +} diff --git a/docs/stt-mvp-acceptance.md b/docs/stt-mvp-acceptance.md new file mode 100644 index 000000000..981739280 --- /dev/null +++ b/docs/stt-mvp-acceptance.md @@ -0,0 +1,59 @@ +# STT MVP 验收说明 + +本文记录当前语音转文字（STT）MVP 的运行边界、凭据处理方式和可复现验收方法。它描述的是自动化验证与手工验收范围，不代表所有平台或所有云厂商已经完成真实语音联调。 + +## 运行架构 + +桌面端和 WebUI 使用相同的 PCM 规范、设置字段和 Composer transient 文本契约。桌面 WebView 通过 Tauri IPC 把音频交给 Rust `SttManager`，Rust 直接连接供应商；WebUI 浏览器通过 Gateway 的 `/ws/v2/stt` Protobuf 数据面连接 Gateway 内的 Go `SttManager`。WebUI 不要求桌面 Agent 持续在线，也不经 `/ws/v2/agent` 或 `/ws/v2/terminal` 转发音频。 + +桌面端保存的 STT 配置会在连接或修改后同步到 Gateway；WebUI 中的修改也会经认证设置通道回写桌面端。公开设置快照只携带非敏感参数和 `configured` 标记，云厂商凭据使用一次性私密 sidecar 在桌面端与 Gateway 后端之间同步，并在任何 Web 广播前删除。浏览器只在用户输入时短暂接触明文，密钥不会进入 localStorage、普通设置快照、广播、日志或错误响应。Gateway 重启后可从自身存储恢复脱敏状态和 Web STT 运行时，因此不需要先打开设置页，也不要求桌面 Agent 当时在线。空白密钥更新保留旧值，带 `clearSecrets` 的明确更新会同时清空桌面端和 Gateway。 + +## 数据面与音频契约 + +`/ws/v2/stt` 是独立的 Protobuf 二进制数据面：首帧为 hello/token，随后是 `start`、`audio`、`stop` 或 `cancel` oneof。单连接只允许一个活动 session；PCM 必须是非空、偶数字节、最大 6400 字节的 16 kHz 单声道 signed 16-bit little-endian 数据。序号必须从零开始连续递增。连接关闭会取消对应云端 session，ready、partial、final、error、closed 事件均携带 session ID。 + +共享采集器按约 100 ms（1600 个采样）分块，连接建立前缓存最多 10 秒 PCM；ready 后先按序排空 FIFO，再发送实时块。停止采集后追加 400 ms（6400 个零采样）尾音，再发送供应商结束协议。Composer 在当前 selection 建立 transient marker，partial 只更新该区间，final 固化并把光标放到区间之后；失败清理 marker，可选择保留最后 partial。 + +## 供应商配置与协议隔离 + +支持的 provider 及必填字段如下： + +| Provider | 必填字段 | 协议边界 | +| --- | --- | --- | +| 腾讯云 | App ID、Secret ID、Secret Key、Engine Model Type | URL 查询参数 HMAC-SHA1 签名；按 `index` 合并结果并等待 `end` | +| 火山引擎实时语音识别 | App ID、Access Token、Resource ID | v3 端点、请求头和二进制帧 | +| 阿里云 DashScope | API Key | WebSocket `Authorization: Bearer`；`task-started` 后 ready，`finish-task` 必须得到 `task-finished` | +| 百度实时识别 | 数字 App ID、API Key/AppKey、数字 `dev_pid` | 嵌套 `START` JSON；`MID_TEXT`/`FIN_TEXT`；发送 `FINISH` 后正常关闭可完成；`3301/-3005` 表示无语音 | + +配置页面可同时保存多家 provider，但一次只选择一个当前 provider。当前 MVP 不包含自动故障转移、TTS、录音保存、翻译、热词或动态模型切换。 + +## 权限与平台边界 + +macOS 包含 `NSMicrophoneUsageDescription` 和 `com.apple.security.device.audio-input`，原生 AVFoundation 权限获得允许后才开始 WebView 采集。拒绝、受限和无设备分别返回稳定错误；Windows WebView2、Linux WebKitGTK/GStreamer 缺少媒体能力时仅禁用 STT，不影响键盘聊天。页面卸载、窗口隐藏、设备断开、连接失败和停止路径都会释放 MediaStream、AudioContext、计时器及连接 session。 + +## 自动化验收命令 + +在仓库根目录执行： + +```bash +pnpm install --frozen-lockfile +go test ./... +pnpm test:gui +pnpm test:webui +pnpm lint:ui +pnpm lint:gui +pnpm lint:webui +STT_TARGET_DIR=$(mktemp -d /tmp/liveagent-cargo.XXXXXX) +CARGO_TARGET_DIR="$STT_TARGET_DIR" cargo check --manifest-path crates/agent-gui/src-tauri/Cargo.toml --tests +CARGO_TARGET_DIR="$STT_TARGET_DIR" cargo test --manifest-path crates/agent-gui/src-tauri/Cargo.toml +pnpm build:gui +pnpm build:webui +``` + +协议生成还应在不含 AppleDouble 元数据的临时 Buf 副本中执行 `buf lint`、`buf breaking` 和 `buf generate`，再与工作区生成文件比较。仓库内的 `._*` 文件属于环境污染，不删除、不修改、不加入提交。 + +## 手工与未覆盖项目 + +没有 provider 凭据时，真实云连接测试不执行，也不会在仓库中写入假凭据。提供合法测试凭据后，应逐家验证连接测试分类、ready/partial/final/closed、慢连接首句 FIFO、手动/静音/无语音停止、复杂 Composer 光标位置以及清空凭据后的不可用状态；测试结束删除临时配置，不保存原始音频。 + +跨平台真实麦克风设备、系统权限对话框、WebView2、WebKitGTK 和五家云厂商的线上协议兼容性仍属于手工验收项。自动化 fixture 不等同于“已支持所有平台真实语音”。