diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da5b6c7..f8ec47b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: - name: Infrastructure configuration scan run: >- docker run --rm -v "$PWD:/src" aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 - config --exit-code 1 --severity HIGH,CRITICAL /src + config --ignorefile /src/.trivyignore.yaml --exit-code 1 --severity HIGH,CRITICAL /src - name: Image vulnerability scan run: | docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL data-terminal-cloud-backend:test diff --git a/.gitignore b/.gitignore index 027b940..67934f1 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ web/dist/ test/e2e/screenshots/ .design-evidence/ device-simulator-windows/story-continuity-rtc/ +/.tmp-*/ +/firmware/**/build/ +/firmware/**/managed_components/ +/firmware/**/backups/ diff --git a/.gitleaks.toml b/.gitleaks.toml index 41786bd..b927d60 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -4,8 +4,8 @@ title = "Data terminal cloud secret scanning" useDefault = true [allowlist] -description = "HTTP idempotency keys are replay identifiers, not credentials" +description = "Exact public HTTP idempotency fixtures are replay identifiers, not credentials" condition = "AND" regexTarget = "line" -regexes = ['''Idempotency-Key'''] -paths = ['''(^|/)internal/httpapi/conversation_handlers_test\.go$'''] +regexes = ['''(?:test-idempotency-key-000[12]|test-activate-idempotency-0001|turns-0123456789abcdef|3ca4e514-cbde-41fc-aa32-6f1ed460cb01)'''] +paths = ['''(^|/)internal/httpapi/conversation_handlers_test\.go$''','''(^|/)docs/设备终端接口文档\.md$'''] diff --git a/.trivyignore.yaml b/.trivyignore.yaml new file mode 100644 index 0000000..6a80053 --- /dev/null +++ b/.trivyignore.yaml @@ -0,0 +1,9 @@ +misconfigurations: + - id: KSV-0109 + paths: + - "deploy/kubernetes/base/configmap.yaml" + statement: >- + RTC_TOKEN_ISSUER and the *_TTL values are public runtime metadata, not + credentials. All RTC signing keys, provider credentials, and token key + material are mounted from Kubernetes Secrets or the production CSI + secret store. diff --git a/CHANGELOG.md b/CHANGELOG.md index e51fc94..7bc6666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ ## 未发布 +- 新增V51硬件智能体License池:License明文仅在平台加密库存与设备安全区中出现,平台按Instance ID/Product Key原子分配,设备校验摘要并回执后才允许`PROVISIONING → ACTIVE`;过期、吊销、丢失和解绑会立即阻断新会话。 +- 新增V50设备激活状态机:`FACTORY → CLAIMABLE → BOUND → PROVISIONING → ACTIVE → LOST/REVOKED`,认领、火山凭据预配、解绑、丢失恢复和注销统一写入带修订号的迁移审计;LOST/解绑会暂停火山设备凭据,恢复后重新启用。 +- 新增独立`volc_hardware_agent`运行时和`/device/v3/agent-sessions`控制面;设备每次聊天从平台取得冻结的人设、档案、近期摘要、已确认长期记忆和授权上下文,不再复用RTC房间语义。 +- 新增V49平台代理火山设备注册和`/device/v3/hardware-agent-credentials:provision`:Product Secret只从平台Secret/KMS加载,一机一密在平台AES-256-GCM加密并下发对应设备,Windows端不再接收Product Secret。 +- 硬件会话新增按设备Bot/传输/最低SDK绑定、完整能力快照、动态`session.updated`生效证明、单调事件水位、可靠最终字幕、平台`stop_session`动作、恢复扫描和SaaS运行时可见性。 +- 不具备近端与远端双路PCM录制能力的硬件会话强制降级为仅文本或不留存;通话中记忆工具扩展到硬件会话并继续从数据库反查租户、长者和活动会话边界。 +- 新增V47/V48/V49硬件运行时及凭据迁移、Windows控制面模拟接口、端到端正反向冒烟,以及带清单和SHA-256的服务器部署候选包工具。 +- 维护中心新增租户级固件与OTA升级管理:可信Ed25519公钥、浏览器私有直传、大小与SHA-256校验、发布通道、强制升级标记、兼容设备灰度批次、可靠升级命令、暂停/恢复/退役和审计下载地址。 +- 固件制品支持维护端租户级范围,同时保留原家庭端记录和设备受控下载链路;设备资产列表显示最近上报的当前固件版本。 - 聊天总结改为大模型两级内容价值判断:先判断是否值得保留,再判断是否值得形成故事;纯寒暄不再生成摘要或故事。 - 新增内容价值判断记录,保留模型、提示词版本、置信度和舍弃原因,支持任务幂等与隐私删除。 - 本地开发新增 GPU Ollama/Qwen2.5 7B 部署方式,并通过大模型管理的测试、激活和 worker 生成链路完成端到端验证。 diff --git a/README.md b/README.md index 7833aeb..82d7cdc 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ - 私有数据导出:按长者和分类生成JSON,MinIO私有存储、SHA-256校验、短期签名下载和过期对象清理。 - 主体级隐私删除:精确称呼确认、活动通话保护、立即冻结读取/上传/召回,以及PostgreSQL、Redis、对象存储、pgvector、外部索引和私有导出六子系统的可重试对账与最小删除证明。 - 设备命令可靠链路:家庭端幂等排队、设备HMAC主动拉取、重复投递计数、确认/成功/失败回执和过期终态;Web设备卡片可下发诊断命令。 -- 固件安全发布与管理页:Ed25519受信公钥、浏览器私有直传、大小与SHA-256校验、签名清单验证、确定性灰度、可靠命令、短期下载、暂停、已验证版本回滚,以及逐设备下载/安装/确认进度。 +- 家庭端及维护中心固件安全发布与OTA管理:租户级/家庭级Ed25519受信公钥、浏览器私有直传、大小与SHA-256校验、签名清单验证、兼容设备确定性灰度、可靠命令、设备与维护人员短期下载、暂停、已验证版本回滚,以及逐设备下载/安装/确认进度。 - 双轨录音媒体链路:near/far分片清单幂等、15分钟单对象直传、Outbox可靠校验、WAV PCM16容器/时长/大小/SHA-256核验、延迟删除本地密文及设备回执,以及PLAY/DOWNLOAD独立授权、授权列表/撤销、React双轨播放下载、5分钟私有访问和成功/拒绝审计。 - 联调问题闭环:按客户组织记录问题现象、严重级别、设备/通话/请求编号、责任方、根因、修复版本和脱敏回归证据;未关闭HIGH/CRITICAL形成发布阻断汇总,创建幂等、更新强ETag并全程审计。 - 认证SSE状态事件流:业务事务内生成最小事件,15分钟`Last-Event-ID`续传、活动家庭成员过滤、跨租户游标拒绝、重复事件抑制、设备上线去重/超时离线、React自动重连和2秒合并刷新,避免连续事件放大为整页请求风暴。 @@ -119,6 +119,8 @@ Web镜像在容器启动时根据`WEB_OIDC_AUTHORITY`和`WEB_OIDC_CLIENT_ID`生 真实火山RTC模式由两组独立开关启用:API设置`RTC_TOKEN_ISSUER=volcengine`、`RTC_APP_ID`和`RTC_APP_KEY`,Worker设置`VOICE_PROVIDER=volcengine`、`VOLC_ACCESS_KEY_ID`、`VOLC_SECRET_ACCESS_KEY`、区域、接口版本以及`VOLC_VOICE_CHAT_TEMPLATE_FILE`。生产模板采用`voice-chat-template-bundle-v1`,分别提供`s2s_pure`、`s2s_hybrid`和`cascade`三个完整VoiceChat `Config/AgentConfig`;平台按通话冻结模式选择模板,再覆盖AppId、RoomId、TaskId、目标设备用户、智能体用户,并追加本次已授权且脱敏的背景摘要。级联模式写入`LLMConfig.SystemMessages`,纯端到端模式写入`S2SConfig.ProviderParams.dialog.system_role`。可从[模板示例](deploy/volcengine/voice-chat-template.example.json)复制结构并按照已开通的官方能力替换每个完整`Config`对象;不要把AppKey、AK/SK、ASR/TTS Token提交到仓库。 +火山硬件对话智能体使用独立的`volc_hardware_agent`运行时。Instance ID、Product Key、默认Bot ID、`high_quality`/`low_load`传输方式和最低SDK版本是平台持有的产品级配置,创建设备时只选择运行时,不再填写Bot。设备首次调用`/device/v3/hardware-agent-credentials:provision`时,平台自动建立该产品默认绑定,并使用KMS/Secret中的Product Secret代设备调用火山DynamicRegister;火山从产品额度中分配License并返回本设备的一机一密,License本身不下发设备。设备安全保存Device Secret后进入`ACTIVE`。每次聊天再调用`/device/v3/agent-sessions`取得默认Bot和本次冻结的人设、近期摘要、长期记忆与授权上下文,并通过火山SDK或标准WebSocket动态注入。平台只有在设备证明`session.updated`已应用且Bot、传输方式、配置摘要完全匹配后才激活通话;设备不具备近/远端双路PCM能力时,本次留存自动降级为仅文本。设备必须持续上报v3事件和最终字幕,并执行事件响应中的`stop_session`。完整时序、重试和凭据边界见[设备集成说明](docs/device-integration.md#115-火山硬件对话智能体-v3-控制面)。 + 设备HMAC密钥保存在`device_credential`表中,并使用AES-256-GCM和每条记录独立随机nonce加密;数据库不保存明文。API通过`DEVICE_CREDENTIAL_MASTER_KEY`(64位十六进制、解码后32字节)解密验签,该主密钥必须由Docker/Kubernetes Secret或云KMS CSI独立托管,不能进入数据库。`DEVICE_LOCAL_SECRET`只用于把既有演示设备凭据一次性导入数据库,完成迁移后可从运行环境移除。 数据库连接串、Redis、RTC AppKey、火山AK/SK、方舟API Key、对象存储凭据、回调/告警签名等敏感配置同时支持`NAME_FILE=/run/secrets/name`。文件路径必须是容器内绝对路径、目标必须是普通文件且不超过64KiB;读取时只去掉末尾换行,不在错误中打印内容。`NAME`与`NAME_FILE`同时存在会拒绝启动,防止轮换时仍意外使用旧的明文环境变量。 diff --git a/compose.yaml b/compose.yaml index 6fb5633..0daca24 100644 --- a/compose.yaml +++ b/compose.yaml @@ -55,6 +55,9 @@ x-backend-environment: &backend-environment VOLC_RTC_REGION: ${VOLC_RTC_REGION:-cn-north-1} VOLC_RTC_API_VERSION: ${VOLC_RTC_API_VERSION:-2024-12-01} VOLC_VOICE_CHAT_TEMPLATE_FILE: ${VOLC_VOICE_CHAT_TEMPLATE_FILE:-} + VOLC_HARDWARE_AGENT_BOT_ID: ${VOLC_HARDWARE_AGENT_BOT_ID:-local-hardware-agent-bot} + VOLC_HARDWARE_AGENT_TRANSPORT: ${VOLC_HARDWARE_AGENT_TRANSPORT:-high_quality} + VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION: ${VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION:-1.0.0} PROVIDER_FAILURE_THRESHOLD: "3" PROVIDER_CIRCUIT_COOLDOWN: 1m RECOVERY_SCAN_INTERVAL: 5s @@ -83,6 +86,7 @@ x-backend-build: &backend-build VERSION: ${PLATFORM_BUILD_VERSION:-dev} COMMIT: ${PLATFORM_BUILD_COMMIT:-unknown} BUILD_TIME: ${PLATFORM_BUILD_TIME:-unknown} + GOPROXY: ${GOPROXY:-https://proxy.golang.org,direct} x-backend-dependencies: &backend-dependencies postgres: @@ -239,6 +243,14 @@ services: HTTP_ADDR: 0.0.0.0:8080 DEVICE_LOCAL_SECRET: local-simulated-device-secret-32-bytes-minimum RTC_TOKEN_SIGNING_SECRET: local-simulated-rtc-token-secret-32-bytes-minimum + VOLC_HARDWARE_PROVISIONING_ENABLED: ${VOLC_HARDWARE_PROVISIONING_ENABLED:-false} + VOLC_HARDWARE_INSTANCE_ID: ${VOLC_HARDWARE_INSTANCE_ID:-} + VOLC_HARDWARE_PRODUCT_KEY: ${VOLC_HARDWARE_PRODUCT_KEY:-} + VOLC_HARDWARE_PRODUCT_SECRET: ${VOLC_HARDWARE_PRODUCT_SECRET:-} + VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY: ${VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY:-} + VOLC_HARDWARE_REGISTER_URL: ${VOLC_HARDWARE_REGISTER_URL:-https://iot-cn-shanghai.iot.volces.com/2021-12-14/DynamicRegister?Action=DynamicRegister&Version=2021-12-14} + VOLC_HARDWARE_REGISTER_TIMEOUT: ${VOLC_HARDWARE_REGISTER_TIMEOUT:-10s} + VOLC_HARDWARE_DEVICE_NAME_PREFIX: ${VOLC_HARDWARE_DEVICE_NAME_PREFIX:-dt-} ports: - "${API_PORT:-58080}:8080" depends_on: *backend-dependencies @@ -332,6 +344,7 @@ services: VERSION: ${PLATFORM_BUILD_VERSION:-dev} COMMIT: ${PLATFORM_BUILD_COMMIT:-unknown} BUILD_TIME: ${PLATFORM_BUILD_TIME:-unknown} + NPM_CONFIG_REGISTRY: ${NPM_CONFIG_REGISTRY:-https://registry.npmjs.org} environment: WEB_OIDC_AUTHORITY: ${WEB_OIDC_AUTHORITY:-http://localhost:58180/realms/voice-local} WEB_OIDC_CLIENT_ID: ${WEB_OIDC_CLIENT_ID:-voice-platform-web} diff --git a/deploy/backend.Dockerfile b/deploy/backend.Dockerfile index 3b18955..78b706b 100644 --- a/deploy/backend.Dockerfile +++ b/deploy/backend.Dockerfile @@ -1,4 +1,5 @@ FROM golang:1.26.6-alpine AS build +ARG GOPROXY=https://proxy.golang.org,direct WORKDIR /src COPY go.mod go.sum ./ RUN go mod download diff --git a/deploy/kubernetes/base/configmap.yaml b/deploy/kubernetes/base/configmap.yaml index 9c15e24..3a5f5dd 100644 --- a/deploy/kubernetes/base/configmap.yaml +++ b/deploy/kubernetes/base/configmap.yaml @@ -50,6 +50,17 @@ data: VOLC_RTC_BASE_URL: https://rtc.volcengineapi.com VOLC_RTC_REGION: cn-north-1 VOLC_RTC_API_VERSION: "2024-12-01" + # Platform-owned product defaults. Devices receive these only through the + # signed provisioning/session control plane and cannot override them. + VOLC_HARDWARE_AGENT_BOT_ID: "replace-with-product-default-bot-id" + VOLC_HARDWARE_AGENT_TRANSPORT: "high_quality" + VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION: "1.0.0" + VOLC_HARDWARE_PROVISIONING_ENABLED: "false" + VOLC_HARDWARE_INSTANCE_ID: "" + VOLC_HARDWARE_PRODUCT_KEY: "" + VOLC_HARDWARE_REGISTER_URL: https://iot-cn-shanghai.iot.volces.com/2021-12-14/DynamicRegister?Action=DynamicRegister&Version=2021-12-14 + VOLC_HARDWARE_REGISTER_TIMEOUT: 10s + VOLC_HARDWARE_DEVICE_NAME_PREFIX: "dt-" CONTENT_GENERATOR: ark CONTENT_SAFETY_PROVIDER: managed_http CONTENT_SAFETY_ENDPOINT: https://content-safety.example.invalid/v1/classify diff --git a/deploy/kubernetes/examples/secret-provider-classes.example.yaml b/deploy/kubernetes/examples/secret-provider-classes.example.yaml index 314a6c9..7e4f96e 100644 --- a/deploy/kubernetes/examples/secret-provider-classes.example.yaml +++ b/deploy/kubernetes/examples/secret-provider-classes.example.yaml @@ -19,7 +19,7 @@ metadata: spec: provider: replace-with-installed-provider parameters: - objects: replace-with-provider-specific-rtc-app-key-cursor-share-and-device-credential-master-key + objects: replace-with-provider-specific-rtc-app-key-cursor-share-device-credential-master-key-volc-hardware-product-secret-and-volc-device-credential-master-key --- apiVersion: secrets-store.csi.x-k8s.io/v1 kind: SecretProviderClass diff --git a/deploy/kubernetes/overlays/production-csi/csi-volumes.yaml b/deploy/kubernetes/overlays/production-csi/csi-volumes.yaml index 95d9c8a..82d3af3 100644 --- a/deploy/kubernetes/overlays/production-csi/csi-volumes.yaml +++ b/deploy/kubernetes/overlays/production-csi/csi-volumes.yaml @@ -6,6 +6,13 @@ metadata: spec: template: spec: + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault volumes: - $patch: replace - name: common-secrets @@ -35,6 +42,13 @@ metadata: spec: template: spec: + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault volumes: - $patch: replace - name: common-secrets @@ -58,6 +72,13 @@ metadata: spec: template: spec: + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault volumes: - $patch: replace - name: common-secrets diff --git a/deploy/migrate.Dockerfile b/deploy/migrate.Dockerfile index 979d085..9aca042 100644 --- a/deploy/migrate.Dockerfile +++ b/deploy/migrate.Dockerfile @@ -1,3 +1,4 @@ FROM migrate/migrate:v4.18.3 -COPY migrations /migrations +COPY --chown=65532:65532 migrations /migrations +USER 65532:65532 diff --git a/deploy/server/README.md b/deploy/server/README.md index b26b515..e020da1 100644 --- a/deploy/server/README.md +++ b/deploy/server/README.md @@ -2,6 +2,8 @@ 该配置用于单机公网演示,不是生产验收配置。只向公网发布HTTP 80和HTTPS 443;PostgreSQL、Redis、MinIO、Keycloak内部端口以及API/Worker端口均不映射到宿主机。当前阿里云演示机使用Let's Encrypt短期IP证书,`certbot-ip-renew.timer`负责自动续期。单机备份覆盖PostgreSQL、Redis、MinIO对象和当前Keycloak数据目录,并执行数据库级隔离恢复校验;五分钟健康检查覆盖部署模式、公网入口、OIDC、容器、模拟服务、备份、外部告警、证书、定时任务和磁盘。跨故障域备份、真实告警接收器、真实RTC、模型、内容安全和云KMS仍按阶段8验收清单执行。 +无法直接从Git拉取或需要冻结当前工作树时,可先在开发机执行`powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\build-server-deployment-bundle.ps1 -Version 0.76.0-rc.hardware1`,再用`powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\verify-server-deployment-bundle.ps1 -Archive .\.deploy-artifacts\data-terminal-cloud-0.76.0-rc.hardware1.zip`验证。归档包含`deploy-manifest.json`和`SHA256SUMS`,不会打包`.env`、设备/供应商密钥、本地运行目录、`node_modules`或构建缓存。上传服务器后必须再次核对归档侧车`.sha256`和包内清单,再覆盖候选目录;服务器现有`deploy/server/.env`及`/etc/data-terminal-cloud`密钥目录不得从归档替换。 + 服务器至少需要Docker Engine 29与Compose 2.40。首次部署先生成权限为600的服务器专用配置: ```sh @@ -30,6 +32,15 @@ docker compose -f compose.yaml -f deploy/server/compose.override.yaml \ 需要在演示机接入真实RTC、豆包语音和方舟摘要时,把凭据文件以`0600`权限放入`/etc/data-terminal-cloud/voice-runtime`,并在`deploy/server/.env`设置`RTC_TOKEN_ISSUER=volcengine`、`VOICE_PROVIDER=volcengine`、`RTC_APP_ID`、`CONTENT_GENERATOR`和`ARK_CONTENT_MODEL`。Compose只读挂载该目录,敏感值不进入源码、镜像或环境文件。 +切换火山硬件对话智能体时,维护中心逐台只需把`voice_runtime`设为`volc_hardware_agent`,不填写设备级Bot。平台代设备动态注册时,在`deploy/server/.env`设置`VOLC_HARDWARE_PROVISIONING_ENABLED=true`、`VOLC_HARDWARE_INSTANCE_ID`、`VOLC_HARDWARE_PRODUCT_KEY`、`VOLC_HARDWARE_AGENT_BOT_ID`、`VOLC_HARDWARE_AGENT_TRANSPORT`、`VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION`、`VOLC_HARDWARE_PRODUCT_SECRET_FILE=/run/secrets/voice-runtime/volc-hardware-product-secret`和独立随机的`VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY`;将Product Secret以单行、`0600`权限写入`/etc/data-terminal-cloud/voice-runtime/volc-hardware-product-secret`。Product Secret不得进入`.env`、源码、镜像、数据库或设备。设备首次预配时,平台根据这些产品默认值自动创建火山绑定并调用DynamicRegister;火山从产品额度中分配License,平台只加密保存并下发本设备的一机一密Device Secret,不下发License明文。发布前必须验证平台代理注册、默认Bot绑定、每次会话上下文注入、`session.updated`确认后激活、`session_updated`存活上报、`stop_session`执行和最终字幕补传。 + +```sh +sudo install -d -m 0700 /etc/data-terminal-cloud/voice-runtime +sudo sh -c 'umask 077; read -r product_secret; printf "%s" "$product_secret" > /etc/data-terminal-cloud/voice-runtime/volc-hardware-product-secret' +``` + +上面的`read`从终端读取,不要把真实Product Secret写进Shell历史。启用前确认默认Bot关联的产品与`VOLC_HARDWARE_PRODUCT_KEY`完全一致;Device Name由`dt-`加平台设备UUID稳定生成。密钥文件或产品默认配置缺失、格式错误时API拒绝启动,未启用时凭据接口明确返回`HARDWARE_AGENT_PROVISIONING_UNAVAILABLE`。 + 设备HMAC密钥由`device_credential`表以AES-256-GCM密文保存。`deploy/server/.env`中的`DEVICE_CREDENTIAL_MASTER_KEY`只负责数据库密文的认证解密,不得写入数据库或设备;既有`DEVICE_LOCAL_SECRET`只用于首次部署升级时导入两台演示设备,确认数据库凭据记录完整后可移除。新增、轮换和吊销设备密钥使用后端镜像内的`/app/device-credential`维护命令,生成的设备明文密钥只显示一次。 通话内记忆工具只通过网关精确开放`/internal/v1/tools/memory-search`,请求必须携带平台为当前通话签发的短期任务令牌。`MEMORY_TOOL_SIGNING_SECRET`必须是服务器独立随机密钥,不得与游标签名、分享签名或RTC密钥复用;网关不会开放其他`/internal`路径。 diff --git a/deploy/server/bootstrap-secrets.sh b/deploy/server/bootstrap-secrets.sh index f58aea0..86a0705 100644 --- a/deploy/server/bootstrap-secrets.sh +++ b/deploy/server/bootstrap-secrets.sh @@ -34,6 +34,7 @@ umask 077 printf 'KEYCLOAK_ADMIN_PASSWORD=%s\n' "$(openssl rand -hex 24)" printf 'DEVICE_LOCAL_SECRET=%s\n' "$(openssl rand -hex 32)" printf 'DEVICE_CREDENTIAL_MASTER_KEY=%s\n' "$(openssl rand -hex 32)" + printf 'VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY=%s\n' "$(openssl rand -hex 32)" printf 'MODEL_CREDENTIAL_MASTER_KEY=%s\n' "$(openssl rand -hex 32)" printf 'RTC_TOKEN_SIGNING_SECRET=%s\n' "$(openssl rand -hex 32)" printf 'CURSOR_SIGNING_SECRET=%s\n' "$(openssl rand -hex 32)" diff --git a/deploy/server/compose.override.yaml b/deploy/server/compose.override.yaml index eac5696..7f8f620 100644 --- a/deploy/server/compose.override.yaml +++ b/deploy/server/compose.override.yaml @@ -102,6 +102,16 @@ services: RTC_APP_ID: ${RTC_APP_ID:-local-public-rtc-app} RTC_APP_KEY: "" RTC_APP_KEY_FILE: ${RTC_APP_KEY_FILE:-/run/secrets/voice-runtime/rtc-app-key} + VOLC_HARDWARE_PROVISIONING_ENABLED: ${VOLC_HARDWARE_PROVISIONING_ENABLED:-false} + VOLC_HARDWARE_INSTANCE_ID: ${VOLC_HARDWARE_INSTANCE_ID:-} + VOLC_HARDWARE_PRODUCT_KEY: ${VOLC_HARDWARE_PRODUCT_KEY:-} + VOLC_HARDWARE_AGENT_BOT_ID: ${VOLC_HARDWARE_AGENT_BOT_ID:-} + VOLC_HARDWARE_AGENT_TRANSPORT: ${VOLC_HARDWARE_AGENT_TRANSPORT:-low_load} + VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION: ${VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION:-1.0.0} + VOLC_HARDWARE_PRODUCT_SECRET: "" + VOLC_HARDWARE_PRODUCT_SECRET_FILE: ${VOLC_HARDWARE_PRODUCT_SECRET_FILE:-} + VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY: ${VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY:-} + VOLC_HARDWARE_DEVICE_NAME_PREFIX: ${VOLC_HARDWARE_DEVICE_NAME_PREFIX:-dt-} volumes: - /etc/data-terminal-cloud/voice-runtime:/run/secrets/voice-runtime:ro restart: unless-stopped diff --git a/deploy/web.Dockerfile b/deploy/web.Dockerfile index 10a69a7..1c7d710 100644 --- a/deploy/web.Dockerfile +++ b/deploy/web.Dockerfile @@ -1,4 +1,5 @@ FROM node:24-alpine AS build +ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org WORKDIR /src COPY web/package*.json ./ RUN npm ci @@ -15,6 +16,10 @@ LABEL org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.created="${BUILD_TIME}" USER root RUN apk upgrade --no-cache +# The image's IPv6 helper shells out to `apk manifest nginx` on every start. +# On small production hosts that command can block even after the package +# database is healthy. This deployment owns default.conf and does not need it. +RUN rm -f /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh COPY --chown=101:101 deploy/web.nginx.conf /etc/nginx/conf.d/default.conf COPY --chmod=755 --chown=101:101 deploy/web-runtime-config.sh /docker-entrypoint.d/40-runtime-config.sh COPY --chown=101:101 --from=build /src/dist /usr/share/nginx/html diff --git a/device-simulator-windows/README.md b/device-simulator-windows/README.md index 8a34da9..89a8863 100644 --- a/device-simulator-windows/README.md +++ b/device-simulator-windows/README.md @@ -1,8 +1,45 @@ -# Windows 语音终端模拟器 +# Windows 硬件对话智能体测试程序 -该程序使用Electron和火山引擎Web RTC SDK模拟一台真实语音终端。它采集Windows麦克风、加入平台分配的RTC房间、发布本地音频、订阅并播放豆包智能体音频,并通过设备v2接口完成激活、状态等待、心跳、Token续签和结束通话。进房后还会启动RTC实时字幕,收集长者与智能体的最终话轮并通过现有`turns:batch`设备接口可靠上传;字幕未成功启动时拒绝继续激活通话,避免生成没有聊天记录的完成通话。 +这是方案 1 的 Windows 端到端验收工具。默认页面不再使用平台代管的 RTC/StartVoiceChat,而是按火山“硬件对话智能体低负载标准协议”直接连接 `wss://ai-gateway.vei.volces.com/v1/realtime`。传统 RTC 模拟器仍保留在页面右上角的“传统 RTC 测试页”。 -创建通话返回的隐私策略允许原录音时,模拟器同时通过RTC原始音频帧回调采集长者`near`和智能体`far`轨道。结束通话后自动封装为单声道WAV PCM16、按平台限制分片、申请短期对象地址直传,并等待后台完整性与容器校验完成。家庭工作台的通话记录会标记“可听聊天语音”,获得独立PLAY授权的账号可在详情中选择“听长者语音”或“听智能体语音”。 +## 实际覆盖的流程 + +1. 使用平台设备 ID、时间戳、随机数、正文摘要和 HMAC-SHA256 调用设备 v2/v3 接口。 +2. 调用平台 `/device/v3/hardware-agent-credentials:provision`;平台自动应用产品级Instance ID、Product Key、默认Bot、传输和最低SDK配置,并使用KMS/Secret挂载的 Product Secret 代设备调用 DynamicRegister。 +3. 火山在动态注册时从产品额度中分配 License;平台只把本设备 Device Secret 下发给设备,Windows 使用 Safe Storage 加密保存后重新拉取配置版本并进入 `ACTIVE`。 +4. 创建 `/device/v3/agent-sessions`,从平台取得产品默认 Bot ID、`low_load` 传输方案、本次冻结的人设、背景、最近摘要、确认记忆和授权范围。 +5. 按火山规定的请求头签名建立 WebSocket,等待 `session.created`,提交本次 `session.update`,收到 `session.updated` 后才向平台回报 `session_started`并开放麦克风。 +6. Windows 麦克风被转换为 PCM16、16kHz、单声道、20ms 分片,通过 `input_audio_buffer.append` 实时上行;`response.audio.delta` 按火山 low-load WebSocket SDK 的 PCM16、16kHz、单声道格式实时播放。 +7. 消费服务端 VAD、用户最终识别、智能体最终字幕、Function Calling 和错误事件;双方最终话轮通过 v3 `turns:batch` 幂等上传。 +8. 智能体触发 `memory_search`、`search_memory` 或 `retrieve_memory` 时,程序使用平台签发的当前会话短期令牌调用同源记忆工具,再把结果作为 `function_call_output` 返回火山。 +9. 周期提交 `session_updated`。平台返回 `stop_session` 时立即关麦、补传字幕、关闭火山连接并回报 `session_ended`。 +10. 页面显示 13 项验收门禁,可导出不包含任何密钥的 JSON 报告。 + +## 使用前配置 + +平台维护中心只需把测试设备配置为: + +- `voice_runtime = volc_hardware_agent` + +平台部署需配置并关联同一火山产品和 Bot: + +- Instance ID +- Product Key +- 默认 Bot ID +- 传输方式 `low_load` +- 最低 SDK 版本不高于 `1.1.0-windows-standard-protocol` +- Product Secret(只进入平台密钥挂载,不进入Windows程序) +- Device Name 由平台按设备ID稳定生成,或使用设备绑定中预先登记的名称 +- 可用的硬件对话智能体 License + +Windows程序不提供 Product Secret、Device Secret 或 Bot ID 输入框。Bot ID、Instance ID、Product Key 和 Device Name 均来自平台;设备不能覆盖产品默认路由。 + +开始测试前还需按控制台中的 Bot 模型选择“智能体模型类型”: + +- 通用 LLM:使用官方 `config.LLMConfig.SystemMessages` 注入平台冻结的人设、背景与授权上下文; +- 豆包端到端语音:使用官方 `config.S2SConfig.ProviderParams.dialog` 注入 `system_role`、`speaking_style` 和 `dialog_id`。 + +两种模式都使用官方 `session.object/config/agent_config/input_audio_format/output_audio_format` 报文结构。字段依据火山[初始化会话配置](https://www.volcengine.com/docs/6348/2139862?lang=zh)与[标准协议自定义接入](https://www.volcengine.com/docs/6348/1904602?lang=zh)文档。 ## 构建 @@ -12,36 +49,70 @@ npm install npm run build ``` -输出:`release/voice-device-simulator-0.1.0-windows-x64.exe`。 +输出文件: -连接公网测试服务器时,可以直接双击EXE,也可以使用`launch-cloud-device.ps1`启动。Electron主进程优先读取启动脚本传入的临时环境变量,否则从当前Windows用户的`%LOCALAPPDATA%\DataTerminalVoiceDevice\server-device-secret`读取设备签名密钥。密钥不写入源码、安装包或浏览器存储,也不会回传到渲染进程;输入框留空并显示“已从Windows安全凭据文件加载”即表示可直接检测。 - -## 自动连续性验证 - -本地回归时可让模拟器自动启动通话、播放一段测试音频,并把双方最终字幕写入JSON结果文件。该入口只接受本机文件和回环地址,不影响正常交互使用: - -```powershell -$env:SIMULATOR_AUTO_TEST = '1' -$env:SIMULATOR_AUTOMATED_AUDIO_PATH = 'C:\path\to\question.wav' -$env:SIMULATOR_AUTO_RESULT_PATH = 'C:\path\to\result.json' -& .\release\voice-device-simulator-0.1.0-windows-x64.exe +```text +release/hardware-agent-tester-0.3.5-windows-x64.exe ``` -测试会等待长者与智能体的最终字幕,结束通话后退出;结果文件包含通话ID、最终话轮和通过/失败状态。测试音频支持WAV、MP3和OGG,大小不得超过10 MB。 +当前测试包未配置商业代码签名证书,Windows 可能显示“未知发布者”。交付前可用同目录 SHA-256 文件校验完整性;正式面向终端用户分发时应由甲方或发布主体使用其代码签名证书签名。 + +`npm run build` 会依次执行协议单测、TypeScript 检查、双页面构建和 Windows x64 便携包封装。 + +## 本地完整自动回归 + +先启动仓库根目录的 Compose 开发环境,然后运行: + +```powershell +Set-Location .\device-simulator-windows +pwsh -NoProfile -File .\test\run-hardware-e2e.ps1 +pwsh -NoProfile -File .\test\run-hardware-e2e.ps1 -ModelMode s2s +pwsh -NoProfile -File .\test\run-hardware-e2e.ps1 -ExecutablePath .\release\hardware-agent-tester-0.3.5-windows-x64.exe +``` + +第一条验证通用 LLM 报文,第二条验证端到端语音报文。回归会临时创建 `low_load` 设备绑定,启动一个校验真实火山签名和事件格式的本机 WebSocket 网关,再启动实际 Electron 窗口。测试只有在以下证据全部成立时才返回 PASS: + +- 动态上下文确认后才开放麦克风; +- PCM 上行和音频下行均发生; +- 用户与智能体各产生一条最终字幕并写入 PostgreSQL; +- 平台下发停止状态,Windows 程序通过下一次 `session_updated` 收到 `stop_session`; +- 会话、硬件会话均为 `COMPLETED`; +- SaaS `conversation.started` 和 `conversation.completed` 事件均已产生; +- 页面 13 项验收门禁全部通过。 + +测试结束会删除临时会话、事件和设备绑定,不污染日常开发数据。 ## 安全边界 -- RTC AppKey、火山AK/SK和豆包Access Token只存在本地后端的受保护文件中; -- 模拟器安装包只接收平台签发的短期RTC Token; -- 设备签名密钥只驻留Electron主进程内存,不写入浏览器存储; -- 当前测试构建只允许设备API地址指向`localhost`、`127.0.0.1`或固定测试服务器`https://121.41.46.238`,避免误连其他环境; -- 关闭软件或结束通话时停止音频采集并销毁RTC引擎。 -- 未取得`record_near`或`record_far`授权的轨道不会注册PCM回调;`upload_allowed=false`时不会采集或上传任何原录音。 +- Product Secret 只存在于平台KMS/Secret挂载和注册调用内存,不到达设备,不写入数据库明文、日志或报告。 +- 平台返回的本设备Device Secret使用 Electron `safeStorage` 加密后保存在当前 Windows 用户目录,不进入源码和安装包;License由火山产品侧分配,不下发Windows程序。 +- 平台设备密钥只驻留 Electron 主进程,渲染页面不能读取已加载的明文。 +- 记忆工具地址必须与配置的平台 API 同源,防止短期令牌被发送给第三方地址。 +- 默认只允许连接火山官方硬件网关;本机 `ws://127.0.0.1` 仅供自动回归。 +- 平台确认 `session.updated` 前不会启动麦克风;收到 `stop_session` 后不会继续发送音频。 +- 导出的报告包含设备标识、Bot、配置摘要、事件和字幕,但不包含 Product Secret、Device Secret、平台签名密钥或记忆工具令牌。 +- 自动回归通过仅在`SIMULATOR_AUTO_TEST=1`时生效的进程环境注入mock一机一密,不写入正式Windows凭据缓存。 -## 实时字幕前提 +## 真实验收与本地模拟的区别 -测试所用RTC AppID必须在火山RTC控制台开启“实时字幕”和“流式语音识别”。流式语音识别凭据来自豆包语音控制台创建的“流式语音识别”应用,包括APP ID、Access Token和Cluster ID;“音视频字幕生成”类型不能替代该应用。未开通时程序会显示`ServiceAccessDenied`并清理本次未激活通话。 +### 0.3.5 播放速度修复 -最终字幕会在程序的“实时字幕与上传”区域显示。每条最终字幕使用通话内稳定键和严格递增序号上传;网络失败时留在内存队列,并在心跳及结束通话前重试。存在未上传字幕时,程序不会结束并丢弃当前通话。 +- 根据火山 low-load WebSocket SDK 示例,将智能体下行 PCM 的源采样率从错误的 24kHz 修正为 16kHz,避免 1.5 倍速和音调升高。 +- Windows 输出设备不是 16kHz 时,由 AudioWorklet 按实际系统输出采样率连续重采样,不改变语速和音高。 -完整本地启动顺序见[本地真实RTC联调说明](../deploy/local-rtc/README.md)。 +### 0.3.4 连续音频播放修复 + +- 火山返回的 `pcm_s16le/16kHz/mono` 音频进入常驻 AudioWorklet 环形缓冲,不再按 WebSocket 分片创建独立播放器。 +- 默认预缓冲 120ms,网络瞬时抖动时自动重新缓冲,减少断音、金属音和语速漂移。 +- 连续处理跨分片的奇数字节,并在开始、结束和断流时做短淡入淡出以避免爆音。 +- 测试报告新增播放分片数、采样数和欠载次数。 + +### 0.3.3 失败会话修复 + +- 火山 `error` 事件与连接 Promise 的异常共用一个失败清理任务,同一轮仅提交一次 `session_failed`,保留首次错误。 +- 失败清理期间禁止发送文字或重复结束;延迟到达的事件不再恢复“通话中”。失败后重新检测即可开始新一轮。 +- 新通话清空上一轮验收结果、错误和计时;失败清理的关闭不再标为正常关闭。 +- `npm test` 包含执行实际渲染器代码的并发失败回归测试,不会连接线上平台或消耗火山额度。 +- 此修复不代表火山 `internal_error: StartVoiceChat failed` 的上游原因已解决。 + +本地自动回归证明 Windows 程序、平台控制面、数据库、PCM 管道和协议状态机完整可执行。最终真实火山验收仍必须使用实际 Instance/Product/Device/Bot/License 凭据,确认火山返回真实 ASR、LLM、TTS 音频;程序不会把本机模拟网关的 PASS 冒充真实供应商验收。 diff --git a/device-simulator-windows/electron/hardware-agent-client.cjs b/device-simulator-windows/electron/hardware-agent-client.cjs new file mode 100644 index 0000000..039daaf --- /dev/null +++ b/device-simulator-windows/electron/hardware-agent-client.cjs @@ -0,0 +1,252 @@ +const crypto = require("node:crypto"); +const WebSocket = require("ws"); + +const DEFAULT_GATEWAY_URL = "wss://ai-gateway.vei.volces.com/v1/realtime"; + +function requiredText(value, name, maximum = 512) { + const normalized = String(value || "").trim(); + if (!normalized || normalized.length > maximum) throw new Error(`${name}不能为空或过长`); + return normalized; +} + +function validateDeviceName(value) { + const name = requiredText(value, "火山设备名称", 128); + if (!/^[a-zA-Z0-9_\-.:@]{1,128}$/.test(name)) throw new Error("火山设备名称格式不正确"); + return name; +} + +function hmacBase64(secret, content) { + return crypto.createHmac("sha256", secret).update(content).digest("base64"); +} + +function websocketSignature({ authType = 1, deviceName, randomNumber, productKey, timestamp, instanceID }, deviceSecret) { + const content = `auth_type=${authType}&device_name=${deviceName}&random_num=${randomNumber}&product_key=${productKey}×tamp=${timestamp}&instance_id=${instanceID}`; + return hmacBase64(deviceSecret, content); +} + +function buildSessionPrompt(sessionConfig) { + const sections = [ + requiredText(sessionConfig.authorized_context, "授权上下文", 4000), + String(sessionConfig.system_prompt || "").trim(), + String(sessionConfig.persona_tone || "").trim() ? `对话语气:${String(sessionConfig.persona_tone).trim()}` : "", + `本次对话标识:${requiredText(sessionConfig.dialog_id, "对话标识", 200)}`, + ].filter(Boolean); + return sections.join("\n\n"); +} + +function buildSessionUpdate(sessionConfig, eventID, modelMode = "llm") { + const normalizedMode = String(modelMode || "llm").trim().toLowerCase(); + if (!new Set(["llm", "s2s"]).has(normalizedMode)) throw new Error("智能体模型类型必须为 llm 或 s2s"); + const dialogID = requiredText(sessionConfig.dialog_id, "对话标识", 200); + const systemPrompt = buildSessionPrompt(sessionConfig); + const config = { + ASRConfig: { TurnDetectionMode: 0 }, + }; + if (normalizedMode === "s2s") { + config.S2SConfig = { + ProviderParams: { + dialog: { + system_role: systemPrompt, + speaking_style: String(sessionConfig.persona_tone || "").trim(), + dialog_id: dialogID, + }, + }, + }; + } else { + config.LLMConfig = { SystemMessages: [systemPrompt] }; + } + return { + event_id: eventID, + type: "session.update", + session: { + object: "realtime.session", + model: "", + input_audio_format: "pcm16", + output_audio_format: "pcm16", + config, + agent_config: { EnableConversationStateCallback: false }, + }, + }; +} + +function normalizeProviderError(event) { + const code = String(event?.error?.code || event?.error?.type || "PROVIDER_ERROR"); + const message = String(event?.error?.message || "火山硬件智能体返回错误"); + return new Error(`${code}: ${message}`); +} + +class HardwareAgentClient { + constructor(options = {}) { + this.WebSocketImpl = options.WebSocketImpl || WebSocket; + this.gatewayURL = options.gatewayURL || DEFAULT_GATEWAY_URL; + this.emit = typeof options.emit === "function" ? options.emit : () => {}; + this.socket = undefined; + this.sessionCreated = undefined; + this.providerConnectionRef = ""; + this.pendingCalls = new Map(); + this.pingTimer = undefined; + this.connectionPromise = undefined; + this.sessionUpdateSent = false; + } + + async connect(input) { + if (this.socket) throw new Error("火山硬件智能体连接已经存在"); + const instanceID = requiredText(input.instanceID, "实例 ID", 200); + const productKey = requiredText(input.productKey, "Product Key", 200); + const deviceName = validateDeviceName(input.deviceName); + const deviceSecret = requiredText(input.deviceSecret, "Device Secret", 4096); + const hardwareID = requiredText(input.hardwareID, "Hardware ID", 200); + const botID = requiredText(input.botID, "Bot ID", 200); + const randomNumber = crypto.randomInt(1, 2_147_483_647); + const timestamp = Math.floor(Date.now() / 1000); + const signature = websocketSignature({ deviceName, randomNumber, productKey, timestamp, instanceID }, deviceSecret); + const target = new URL(this.gatewayURL); + target.searchParams.set("bot", botID); + target.searchParams.set("wait_for_session_update", "true"); + const headers = { + "X-Auth-Type": "1", + "X-Product-Key": productKey, + "X-Device-Name": deviceName, + "X-Random-Num": String(randomNumber), + "X-Timestamp": String(timestamp), + "X-Instance-Id": instanceID, + "X-Signature": signature, + "X-Hardware-Id": hardwareID, + }; + const socket = new this.WebSocketImpl(target.toString(), { headers, handshakeTimeout: 10_000, maxPayload: 8 * 1024 * 1024 }); + this.socket = socket; + this.connectionPromise = new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("等待火山 session.updated 超时")), 15_000); + const finish = (callback, value) => { + clearTimeout(timeout); + callback(value); + }; + socket.once("open", () => { + this.emit({ type: "transport.open" }); + // wait_for_session_update=true makes the gateway wait for the client + // configuration before it publishes the usable session. Sending only + // after session.created deadlocks until the provider-side timeout. + this.sendJSON(buildSessionUpdate(input.sessionConfig, `windows-session-update-${crypto.randomUUID()}`, input.modelMode)); + this.sessionUpdateSent = true; + }); + socket.on("message", (data) => { + let event; + try { event = JSON.parse(Buffer.isBuffer(data) ? data.toString("utf8") : String(data)); } + catch { this.emit({ type: "transport.invalid_message" }); return; } + this.handleProviderEvent(event, input.sessionConfig, input.modelMode); + // In the hardware gateway's wait_for_session_update mode, the initial + // session.created event is published only after the pre-creation + // session.update has been accepted. Some Bot versions do not emit an + // additional session.updated event for that initial configuration. + if (event.type === "session.updated" || (event.type === "session.created" && this.sessionUpdateSent)) { + const connectionRef = String(event?.session?.id || this.providerConnectionRef || event.event_id || "").trim(); + if (!connectionRef) return finish(reject, new Error("火山会话确认未提供可关联的会话编号")); + this.providerConnectionRef = connectionRef; + finish(resolve, { providerConnectionRef: connectionRef, session: event.session || {} }); + } else if (event.type === "error" && !this.providerConnectionRef) { + finish(reject, normalizeProviderError(event)); + } + }); + socket.once("unexpected-response", (_request, response) => { + finish(reject, new Error(`火山 WebSocket 鉴权失败:HTTP ${response.statusCode}`)); + }); + socket.once("error", (error) => { + this.emit({ type: "transport.error", message: error.message }); + if (!this.providerConnectionRef) finish(reject, error); + }); + socket.once("close", (code, reason) => { + this.stopPing(); + this.socket = undefined; + this.emit({ type: "transport.close", code, reason: Buffer.from(reason || "").toString("utf8") }); + if (!this.providerConnectionRef) finish(reject, new Error(`火山连接提前关闭:${code}`)); + }); + }); + try { + const result = await this.connectionPromise; + this.startPing(); + return result; + } catch (error) { + this.close(1011, "connection setup failed"); + throw error; + } + } + + handleProviderEvent(event, sessionConfig, modelMode) { + if (event.type === "session.created") { + this.sessionCreated = event.session || {}; + this.providerConnectionRef = String(event?.session?.id || event.event_id || "").trim(); + } + if (event.type === "conversation.item.created" && event?.item?.type === "function_call" && event.item.call_id) { + this.pendingCalls.set(String(event.item.call_id), String(event.item.name || "")); + } + this.emit(event); + } + + sendAudio(bytes) { + const buffer = Buffer.from(bytes); + if (!buffer.length) return; + if (buffer.length > 64 * 1024) throw new Error("单个音频分片不能超过64KiB"); + this.sendJSON({ + event_id: `windows-audio-${crypto.randomUUID()}`, + type: "input_audio_buffer.append", + audio: buffer.toString("base64"), + }); + } + + sendText(text, interruptMode = 1) { + const normalized = requiredText(text, "文本问题", 2000); + this.sendJSON({ + event_id: `windows-text-${crypto.randomUUID()}`, + type: "conversation.item.create", + item: { type: "message", role: "user", content: [{ type: "input_text", text: normalized }], interrupt_mode: interruptMode }, + }); + } + + completeFunctionCall(callID, output) { + const normalizedCallID = requiredText(callID, "函数调用编号", 200); + this.sendJSON({ + event_id: `windows-function-${crypto.randomUUID()}`, + type: "conversation.item.create", + item: { type: "function_call_output", call_id: normalizedCallID, output: JSON.stringify(output) }, + }); + this.sendJSON({ event_id: `windows-response-${crypto.randomUUID()}`, type: "response.create" }); + this.pendingCalls.delete(normalizedCallID); + } + + sendJSON(value) { + if (!this.socket || this.socket.readyState !== this.WebSocketImpl.OPEN) throw new Error("火山硬件智能体连接未就绪"); + this.socket.send(JSON.stringify(value)); + } + + startPing() { + this.stopPing(); + this.pingTimer = setInterval(() => { + try { if (this.socket?.readyState === this.WebSocketImpl.OPEN && typeof this.socket.ping === "function") this.socket.ping(); } + catch { /* close/error callbacks own recovery */ } + }, 20_000); + } + + stopPing() { + if (this.pingTimer) clearInterval(this.pingTimer); + this.pingTimer = undefined; + } + + close(code = 1000, reason = "device session ended") { + this.stopPing(); + const socket = this.socket; + this.socket = undefined; + this.providerConnectionRef = ""; + this.sessionUpdateSent = false; + this.pendingCalls.clear(); + if (!socket) return; + try { socket.close(code, reason); } catch { try { socket.terminate?.(); } catch { /* no-op */ } } + } +} + +module.exports = { + DEFAULT_GATEWAY_URL, + HardwareAgentClient, + buildSessionPrompt, + buildSessionUpdate, + websocketSignature, +}; diff --git a/device-simulator-windows/electron/hardware-agent-client.test.cjs b/device-simulator-windows/electron/hardware-agent-client.test.cjs new file mode 100644 index 0000000..ba9c897 --- /dev/null +++ b/device-simulator-windows/electron/hardware-agent-client.test.cjs @@ -0,0 +1,97 @@ +const assert = require("node:assert/strict"); +const { EventEmitter } = require("node:events"); +const test = require("node:test"); +const { + HardwareAgentClient, + buildSessionUpdate, + websocketSignature, +} = require("./hardware-agent-client.cjs"); + +test("WebSocket signatures use the documented canonical field order", () => { + const connection = websocketSignature({ authType: 1, deviceName: "device-1", randomNumber: 42, productKey: "product-1", timestamp: 1700000000, instanceID: "instance-1" }, "device-secret"); + assert.equal(connection, "6OFE1DUZzRgNy0o5IzZSmqKw3SUP4s3GxK2YHe8phfo="); +}); + +test("LLM session update uses the official Config/AgentConfig schema", () => { + const update = buildSessionUpdate({ dialog_id: "conversation-1", system_prompt: "你是陪伴助手", persona_tone: "亲切", authorized_context: "只可使用已授权背景" }, "event-1"); + assert.equal(update.type, "session.update"); + assert.equal(update.session.object, "realtime.session"); + assert.equal(update.session.input_audio_format, "pcm16"); + assert.equal(update.session.output_audio_format, "pcm16"); + assert.equal(update.session.config.ASRConfig.TurnDetectionMode, 0); + assert.match(update.session.config.LLMConfig.SystemMessages[0], /只可使用已授权背景/); + assert.match(update.session.config.LLMConfig.SystemMessages[0], /你是陪伴助手/); + assert.match(update.session.config.LLMConfig.SystemMessages[0], /亲切/); + assert.equal(update.session.agent_config.EnableConversationStateCallback, false); + assert.equal("system_prompt" in update.session, false); +}); + +test("S2S session update maps persona and dialog id to provider dialog config", () => { + const update = buildSessionUpdate({ dialog_id: "conversation-2", system_prompt: "你是陪伴助手", persona_tone: "亲切自然", authorized_context: "只使用授权信息" }, "event-2", "s2s"); + const dialog = update.session.config.S2SConfig.ProviderParams.dialog; + assert.equal(dialog.dialog_id, "conversation-2"); + assert.equal(dialog.speaking_style, "亲切自然"); + assert.match(dialog.system_role, /只使用授权信息/); + assert.match(dialog.system_role, /你是陪伴助手/); + assert.equal("LLMConfig" in update.session.config, false); +}); + +class FakeWebSocket extends EventEmitter { + static OPEN = 1; + constructor(url, options) { + super(); + this.url = url; + this.options = options; + this.readyState = 0; + this.sent = []; + FakeWebSocket.last = this; + setImmediate(() => { + this.readyState = FakeWebSocket.OPEN; + this.emit("open"); + this.emit("message", Buffer.from(JSON.stringify({ type: "session.created", event_id: "created-event", session: { id: "provider-session-1" } }))); + }); + } + send(body) { + const value = JSON.parse(body); + this.sent.push(value); + if (value.type === "session.update") { + setImmediate(() => this.emit("message", Buffer.from(JSON.stringify({ type: "session.updated", event_id: "updated-event", session: { id: "provider-session-1" } })))); + } + } + ping() {} + close(code, reason) { + this.readyState = 3; + this.emit("close", code, Buffer.from(reason)); + } +} + +test("hardware client waits for dynamic config acknowledgement before becoming ready", async () => { + const events = []; + const client = new HardwareAgentClient({ WebSocketImpl: FakeWebSocket, gatewayURL: "ws://127.0.0.1:19090/v1/realtime", emit: (event) => events.push(event) }); + const connected = await client.connect({ + instanceID: "instance-1", productKey: "product-1", deviceName: "device-1", + deviceSecret: "device-secret-with-at-least-32-characters", hardwareID: "windows-test", botID: "bot-1", + sessionConfig: { dialog_id: "conversation-1", authorized_context: "authorized" }, + }); + assert.equal(connected.providerConnectionRef, "provider-session-1"); + assert.match(FakeWebSocket.last.url, /bot=bot-1/); + assert.equal(FakeWebSocket.last.options.headers["X-Product-Key"], "product-1"); + assert.equal(FakeWebSocket.last.sent[0].type, "session.update"); + client.sendAudio(Buffer.from([1, 2, 3, 4])); + client.sendText("你好"); + assert.equal(FakeWebSocket.last.sent[1].type, "input_audio_buffer.append"); + assert.equal(FakeWebSocket.last.sent[2].type, "conversation.item.create"); + assert.ok(events.some((event) => event.type === "session.created")); + client.close(); +}); + +test("hardware client sends session.update immediately when the WebSocket opens", async () => { + const client = new HardwareAgentClient({ WebSocketImpl: FakeWebSocket, gatewayURL: "ws://127.0.0.1:19090/v1/realtime" }); + await client.connect({ + instanceID: "instance-1", productKey: "product-1", deviceName: "device-1", + deviceSecret: "device-secret-with-at-least-32-characters", hardwareID: "windows-test", botID: "bot-1", + sessionConfig: { dialog_id: "conversation-1", authorized_context: "authorized" }, + }); + assert.equal(FakeWebSocket.last.sent.filter((event) => event.type === "session.update").length, 1); + client.close(); +}); diff --git a/device-simulator-windows/electron/hardware-renderer.test.cjs b/device-simulator-windows/electron/hardware-renderer.test.cjs new file mode 100644 index 0000000..00736f6 --- /dev/null +++ b/device-simulator-windows/electron/hardware-renderer.test.cjs @@ -0,0 +1,153 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); +const fs = require("node:fs"); +const path = require("node:path"); +const vm = require("node:vm"); +const ts = require("typescript"); + +// Execute the actual renderer with a small DOM/IPC fixture; no network or audio. +const source = fs.readFileSync(path.join(__dirname, "../src/hardware.ts"), "utf8"); +const compiled = ts.transpileModule(source, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS }, +}).outputText; + +function fixture(overrides = {}) { + const elements = new Map(); + const element = () => ({ + style: {}, value: "", textContent: "", disabled: false, children: [], + append(...items) { this.children.push(...items); }, + prepend(...items) { this.children.unshift(...items); }, + replaceChildren(...items) { this.children = items; }, + querySelector() { return null; }, addEventListener() {}, + }); + const get = (id) => { + if (!elements.has(id)) elements.set(id, element()); + return elements.get(id); + }; + const events = []; + const closes = []; + const texts = []; + const created = { + conversation_id: "test-conversation", session: {}, + runtime: { transport: "low_load", config_digest: "digest", session_nonce: "nonce", provider_session_ref: "ref", bot_id: "test-bot" }, + session_config: { authorized_context: "test", prompt_bundle_version: "test-v1" }, + }; + const ipc = { + getRuntimeDefaults: async () => ({}), getHardwareRuntimeDefaults: async () => ({}), + onHardwareProviderEvent: () => () => {}, + createHardwareAgentSession: async () => created, + closeHardwareAgent: async (input) => { closes.push(input); }, + hardwareAgentEvent: async (input) => { events.push(input); return { status: "FAILED" }; }, + sendHardwareAgentText: async (input) => { texts.push(input); }, + ...overrides, + }; + const context = vm.createContext({ + exports: {}, require: () => ({}), console, URL, + document: { querySelector: () => get("app"), getElementById: get, createElement: element }, + window: { deviceSimulator: ipc, addEventListener() {}, clearInterval() {}, setTimeout, setInterval() {} }, + }); + vm.runInContext(compiled + `\nglobalThis.renderer = { state, controls, setPhase, setCheck, resetSessionState, failHardwareConversation, stopHardwareConversation, startHardwareConversation, handleProviderEvent, sendTextQuestion, buildReport };`, context); + return { r: context.renderer, events, closes, texts, created, get, ipc }; +} + +test("provider error plus connect rejection sends one failure, including late duplicates", async () => { + const f = fixture(); + const r = f.r; + f.ipc.connectHardwareAgent = async () => { + void r.handleProviderEvent({ type: "error", error: { code: "internal_error", message: "StartVoiceChat failed" } }); + throw new Error("wrapped connect rejection"); + }; + r.setPhase("ready", "ready"); + await r.startHardwareConversation(); + await r.failHardwareConversation("SECOND_FAILURE"); + await r.stopHardwareConversation(); + await r.handleProviderEvent({ type: "error", error: { message: "late error" } }); + assert.equal(f.events.length, 1); + assert.equal(f.events[0].eventType, "session_failed"); + assert.equal(f.events[0].errorCode, "VOLC_HARDWARE_AGENT_ERROR"); + assert.equal(f.closes.length, 1); + assert.equal(r.state.failure, "internal_error: StartVoiceChat failed"); + assert.equal(r.state.phase, "error"); + assert.equal(r.controls.stop.disabled, true); + assert.equal(r.controls.check.disabled, false); +}); + +test("concurrent failure callers share pending cleanup; late events cannot reactivate", async () => { + let release; + const gate = new Promise((resolve) => { release = resolve; }); + const f = fixture({ closeHardwareAgent: () => gate }); + f.r.state.platformSession = f.created; + f.r.state.failure = "first cause"; + f.r.setPhase("active", "active"); + const first = f.r.failHardwareConversation("FIRST"); + const second = f.r.failHardwareConversation("SECOND"); + assert.equal(first, second); + assert.equal(f.r.state.phase, "stopping"); + await f.r.handleProviderEvent({ type: "response.done" }); + assert.equal(f.r.state.phase, "stopping"); + release(); + await first; + assert.equal(f.events.length, 1); + assert.equal(f.events[0].errorCode, "FIRST"); + await f.r.handleProviderEvent({ type: "transport.close", code: 1011 }); + assert.equal(f.r.state.checks.get("provider_closed").status, "failed"); + f.r.controls.textQuestion.value = "should not send"; + await f.r.sendTextQuestion(); + assert.equal(f.texts.length, 0); +}); + +test("new call clears stale checks, duration and failure while retaining device preparation", async () => { + const f = fixture(); + f.r.state.platformSession = f.created; + f.r.state.failure = "old failure"; + await f.r.failHardwareConversation("OLD"); + for (const id of f.r.state.checks.keys()) f.r.setCheck(id, "passed", "old"); + f.get("duration").textContent = "48:15"; + f.r.resetSessionState(); + assert.equal(f.r.state.failure, undefined); + assert.equal(f.r.state.failureTask, undefined); + assert.equal(f.get("duration").textContent, "00:00"); + assert.equal(f.r.state.checks.get("platform_authenticated").status, "passed"); + assert.equal(f.r.state.checks.get("credential_resolved").status, "passed"); + assert.equal(f.r.state.checks.get("platform_completed").status, "pending"); + f.r.state.platformSession = { ...f.created, conversation_id: "next-conversation" }; + await f.r.failHardwareConversation("NEW"); + assert.equal(f.events.length, 2); + assert.equal(f.events[1].conversationID, "next-conversation"); + assert.equal(f.events[1].sequenceNo, 1); +}); + +test("failed platform receipt does not trigger a new terminal event on hangup", async () => { + let attempts = 0; + const f = fixture({ hardwareAgentEvent: async () => { attempts += 1; throw new Error("network unavailable"); } }); + f.r.state.platformSession = f.created; + await f.r.failHardwareConversation("FIRST"); + await f.r.stopHardwareConversation(); + assert.equal(attempts, 1); + assert.equal(f.r.state.phase, "error"); + assert.equal(f.r.buildReport().checks.find((c) => c.id === "platform_completed").status, "failed"); +}); + +test("connection rejection without a provider event still reports failure once", async () => { + const f = fixture({ connectHardwareAgent: async () => { throw new Error("connect timeout"); } }); + f.r.setPhase("ready", "ready"); + await f.r.startHardwareConversation(); + assert.equal(f.events.length, 1); + assert.equal(f.events[0].errorCode, "WINDOWS_HARDWARE_START_FAILED"); + assert.match(f.r.state.failure, /connect timeout/); + assert.equal(f.r.state.phase, "error"); +}); + +test("normal hangup still reports completed and closes normally", async () => { + const f = fixture(); + f.ipc.hardwareAgentEvent = async (input) => { f.events.push(input); return { status: "COMPLETED" }; }; + f.r.state.platformSession = f.created; + f.r.setPhase("active", "active"); + await f.r.stopHardwareConversation(); + assert.equal(f.events.length, 1); + assert.equal(f.events[0].eventType, "session_ended"); + assert.equal(f.closes[0].code, 1000); + assert.equal(f.r.state.phase, "completed"); + assert.equal(f.r.state.failureTask, undefined); + assert.equal(f.r.controls.start.disabled, false); +}); diff --git a/device-simulator-windows/electron/main.cjs b/device-simulator-windows/electron/main.cjs index cd9987c..c6ffaec 100644 --- a/device-simulator-windows/electron/main.cjs +++ b/device-simulator-windows/electron/main.cjs @@ -1,14 +1,24 @@ -const { app, BrowserWindow, ipcMain, session } = require("electron"); +const { app, BrowserWindow, dialog, ipcMain, safeStorage, session } = require("electron"); const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); const tls = require("node:tls"); +const { HardwareAgentClient } = require("./hardware-agent-client.cjs"); app.commandLine.appendSwitch("autoplay-policy", "no-user-gesture-required"); +const automatedUI = process.env.SIMULATOR_AUTO_TEST === "1" || process.env.SIMULATOR_REAL_E2E === "1"; +if (automatedUI) { + app.commandLine.appendSwitch("use-fake-device-for-media-stream"); + app.commandLine.appendSwitch("use-fake-ui-for-media-stream"); +} app.disableHardwareAcceleration(); let deviceConfig = null; let desiredConfigRevision = 1; +let mainWindow = null; +let hardwareConfig = null; +let hardwareClient = null; +let activeHardwarePlatformSession = null; function createWindow() { const window = new BrowserWindow({ @@ -26,6 +36,10 @@ function createWindow() { sandbox: true, }, }); + mainWindow = window; + window.once("closed", () => { + if (mainWindow === window) mainWindow = null; + }); const devArgument = process.argv.find((argument) => argument.startsWith("--dev-server-url=")); const devServerURL = devArgument ? devArgument.slice("--dev-server-url=".length) : ""; @@ -47,7 +61,7 @@ function createWindow() { } catch (error) { console.error("renderer diagnostics failed", error); } - if (process.env.SIMULATOR_AUTO_TEST === "1") { + if (automatedUI) { void runAutomatedSmoke(window); } }); @@ -62,9 +76,15 @@ function createWindow() { if (screenshotPath) { window.webContents.once("did-finish-load", () => { setTimeout(async () => { - const image = await window.webContents.capturePage(); - fs.writeFileSync(screenshotPath, image.toPNG()); - app.quit(); + try { + const image = await window.webContents.capturePage(); + fs.writeFileSync(screenshotPath, image.toPNG()); + } catch (error) { + console.error("simulator screenshot failed", error); + process.exitCode = 1; + } finally { + app.quit(); + } }, 1200); }); } @@ -95,24 +115,54 @@ async function runAutomatedSmoke(window) { async function collectAutomatedConversationResult(window) { const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); const deadline = Date.now() + 120_000; + let realQuestionSent = false; let snapshot = {}; while (Date.now() < deadline) { await delay(1000); const encoded = await window.webContents.executeJavaScript(`JSON.stringify({ phase: document.querySelector('#phaseBadge')?.textContent || '', conversation_id: document.querySelector('#conversationID')?.textContent || '', + model_mode: document.querySelector('#modelMode')?.value || '', subject: [...document.querySelectorAll('.transcript-turn--subject span')].map((item) => item.textContent || ''), agent: [...document.querySelectorAll('.transcript-turn--agent span')].map((item) => item.textContent || ''), turns: [...document.querySelectorAll('.transcript-turn')].map((item) => ({speaker: item.classList.contains('transcript-turn--subject') ? 'subject' : 'agent', text: item.querySelector('span')?.textContent || ''})), - events: [...document.querySelectorAll('#eventLog li span')].map((item) => item.textContent || '') + events: [...document.querySelectorAll('#eventLog li span')].map((item) => item.textContent || ''), + checks: [...document.querySelectorAll('#flowChecks .flow-check')].map((item) => ({label: item.querySelector('span')?.textContent || '', detail: item.querySelector('small')?.textContent || '', status: item.classList.contains('flow-check--passed') ? 'passed' : item.classList.contains('flow-check--failed') ? 'failed' : 'pending'})) })`); snapshot = JSON.parse(encoded); console.log(`[auto-test] conversation ${encoded}`); + if (process.env.SIMULATOR_REAL_E2E === "1" && snapshot.phase === "通话中" && !realQuestionSent) { + const question = String(process.env.SIMULATOR_REAL_E2E_QUESTION || "请用一句简短的话介绍你自己。 ").trim(); + await window.webContents.executeJavaScript(`(() => { const input = document.querySelector('#textQuestion'); if (input) input.value = ${JSON.stringify(question)}; document.querySelector('#sendTextButton')?.click(); })()`); + realQuestionSent = true; + console.log("[real-e2e] text question sent"); + } const subjectIndex = snapshot.turns?.findIndex((turn) => turn.speaker === "subject") ?? -1; const hasAgentReply = subjectIndex >= 0 && snapshot.turns.slice(subjectIndex + 1).some((turn) => turn.speaker === "agent"); if (hasAgentReply) { - await window.webContents.executeJavaScript(`document.querySelector('#stopButton')?.click()`); - snapshot.result = "PASS"; + const hardwarePage = await window.webContents.executeJavaScript(`Boolean(document.querySelector('#flowChecks'))`); + const waitForPlatformStop = hardwarePage && process.env.SIMULATOR_AUTO_WAIT_FOR_PLATFORM_STOP === "1"; + if (!waitForPlatformStop) await window.webContents.executeJavaScript(`document.querySelector('#stopButton')?.click()`); + if (hardwarePage) { + const stopDeadline = Date.now() + 30_000; + while (Date.now() < stopDeadline) { + await delay(500); + const phase = await window.webContents.executeJavaScript(`document.querySelector('#phaseBadge')?.textContent || ''`); + if (phase === "已完成" || phase === "异常") break; + } + const finalEncoded = await window.webContents.executeJavaScript(`JSON.stringify({ + phase: document.querySelector('#phaseBadge')?.textContent || '', + conversation_id: document.querySelector('#conversationID')?.textContent || '', + model_mode: document.querySelector('#modelMode')?.value || '', + turns: [...document.querySelectorAll('.transcript-turn')].map((item) => ({speaker: item.classList.contains('transcript-turn--subject') ? 'subject' : 'agent', text: item.querySelector('span')?.textContent || ''})), + events: [...document.querySelectorAll('#eventLog li span')].map((item) => item.textContent || ''), + checks: [...document.querySelectorAll('#flowChecks .flow-check')].map((item) => ({label: item.querySelector('span')?.textContent || '', detail: item.querySelector('small')?.textContent || '', status: item.classList.contains('flow-check--passed') ? 'passed' : item.classList.contains('flow-check--failed') ? 'failed' : 'pending'})) + })`); + snapshot = JSON.parse(finalEncoded); + snapshot.result = snapshot.phase === "已完成" ? "PASS" : "ERROR"; + } else { + snapshot.result = "PASS"; + } break; } if (snapshot.phase === "异常") { @@ -160,7 +210,10 @@ function loadDefaultDeviceSecret() { const localAppData = String(process.env.LOCALAPPDATA || "").trim(); const userProfile = String(process.env.USERPROFILE || "").trim(); const portableDirectory = String(process.env.PORTABLE_EXECUTABLE_DIR || "").trim(); - if (localAppData) candidates.push(path.join(localAppData, "DataTerminalVoiceDevice", "server-device-secret")); + if (localAppData) { + candidates.push(path.join(localAppData, "DataTerminalHardwareAgent", "server-device-secret")); + candidates.push(path.join(localAppData, "DataTerminalVoiceDevice", "server-device-secret")); + } if (userProfile) candidates.push(path.join(userProfile, "AppData", "Local", "DataTerminalVoiceDevice", "server-device-secret")); try { const home = app.getPath("home"); @@ -182,11 +235,134 @@ function loadDefaultDeviceSecret() { ipcMain.handle("sim:get-runtime-defaults", () => ({ apiURL: process.env.SIMULATOR_DEFAULT_API_URL || "https://121.41.46.238", - deviceID: process.env.SIMULATOR_DEFAULT_DEVICE_ID || "10000000-0000-0000-0000-000000000511", + deviceID: process.env.SIMULATOR_DEFAULT_DEVICE_ID || "10000000-0000-0000-0000-000000000512", deviceSecret: "", deviceSecretConfigured: loadDefaultDeviceSecret().length >= 32, })); +function hardwareCredentialPath() { + return path.join(app.getPath("userData"), "volc-hardware-device-credential.json"); +} + +function readSavedHardwareMetadata() { + try { + const stored = JSON.parse(fs.readFileSync(hardwareCredentialPath(), "utf8")); + if (!stored.instanceID || !stored.productKey || !stored.deviceName || !stored.encryptedDeviceSecret) return undefined; + return stored; + } catch { return undefined; } +} + +function loadSavedHardwareCredential(identity) { + if (!safeStorage.isEncryptionAvailable()) return undefined; + try { + const stored = JSON.parse(fs.readFileSync(hardwareCredentialPath(), "utf8")); + if (stored.schemaVersion !== 3 || stored.instanceID !== identity.instanceID || stored.productKey !== identity.productKey || stored.deviceName !== identity.deviceName) return undefined; + const secret = safeStorage.decryptString(Buffer.from(stored.encryptedDeviceSecret, "base64")); + if (secret.length < 16) return undefined; + return { deviceSecret: secret, hardwareID: String(stored.hardwareID || identity.hardwareID || "") }; + } catch { return undefined; } +} + +function saveHardwareCredential(identity, deviceSecret, metadata = {}) { + if (!safeStorage.isEncryptionAvailable()) return false; + const target = hardwareCredentialPath(); + fs.mkdirSync(path.dirname(target), { recursive: true }); + const payload = { + schemaVersion: 3, + instanceID: identity.instanceID, + productKey: identity.productKey, + deviceName: identity.deviceName, + hardwareID: identity.hardwareID, + platformDeviceID: String(deviceConfig?.deviceID || ""), + credentialRef: String(metadata.credentialRef || ""), + credentialVersion: Number(metadata.credentialVersion || 1), + encryptedDeviceSecret: safeStorage.encryptString(deviceSecret).toString("base64"), + updatedAt: new Date().toISOString(), + }; + const temporary = `${target}.${process.pid}.tmp`; + fs.writeFileSync(temporary, `${JSON.stringify(payload, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + fs.renameSync(temporary, target); + return true; +} + +function hardwareIdentity(input) { + const instanceID = String(input?.instanceID || "").trim(); + const productKey = String(input?.productKey || "").trim(); + const deviceName = String(input?.deviceName || "").trim(); + const hardwareID = String(input?.hardwareID || "").trim(); + if (!instanceID || instanceID.length > 200) throw new Error("实例 ID 不能为空或过长"); + if (!productKey || productKey.length > 200) throw new Error("Product Key 不能为空或过长"); + if (!/^[a-zA-Z0-9_\-.:@]{1,128}$/.test(deviceName)) throw new Error("火山设备名称格式不正确"); + if (!hardwareID || hardwareID.length > 200) throw new Error("Hardware ID 不能为空或过长"); + return { instanceID, productKey, deviceName, hardwareID }; +} + +ipcMain.handle("sim:get-hardware-runtime-defaults", () => { + const candidateMetadata = readSavedHardwareMetadata(); + const legacyMock = candidateMetadata && [candidateMetadata.instanceID, candidateMetadata.productKey, candidateMetadata.deviceName, candidateMetadata.hardwareID] + .some((value) => String(value || "").toLowerCase().includes("mock") || String(value || "").toLowerCase().includes("e2e")); + const savedMetadata = automatedUI || !legacyMock ? candidateMetadata : undefined; + const instanceID = String(process.env.SIMULATOR_VOLC_INSTANCE_ID || savedMetadata?.instanceID || "").trim(); + const productKey = String(process.env.SIMULATOR_VOLC_PRODUCT_KEY || savedMetadata?.productKey || "").trim(); + const deviceName = String(process.env.SIMULATOR_VOLC_DEVICE_NAME || savedMetadata?.deviceName || "windows-hardware-agent-test").trim(); + const hardwareID = String(process.env.SIMULATOR_VOLC_HARDWARE_ID || savedMetadata?.hardwareID || `windows-${crypto.createHash("sha256").update(app.getPath("userData")).digest("hex").slice(0, 24)}`).trim(); + const modelMode = String(process.env.SIMULATOR_VOLC_MODEL_MODE || "llm").trim().toLowerCase() === "s2s" ? "s2s" : "llm"; + const saved = instanceID && productKey ? loadSavedHardwareCredential({ instanceID, productKey, deviceName, hardwareID }) : undefined; + return { instanceID, productKey, deviceName, hardwareID, modelMode, deviceCredentialConfigured: Boolean(saved) }; +}); + +ipcMain.handle("sim:configure-hardware", async (_event, input) => { + if (!deviceConfig) throw new Error("请先连接平台"); + const requestedHardwareID = String(input?.hardwareID || "").trim(); + if (!requestedHardwareID || requestedHardwareID.length > 200) throw new Error("Hardware ID 不能为空或过长"); + const automatedTest = process.env.SIMULATOR_AUTO_TEST === "1"; + let provisioned; + if (automatedTest) { + provisioned = { + instance_id: String(process.env.SIMULATOR_VOLC_INSTANCE_ID || "").trim(), + product_key: String(process.env.SIMULATOR_VOLC_PRODUCT_KEY || "").trim(), + device_name: String(process.env.SIMULATOR_VOLC_DEVICE_NAME || "").trim(), + hardware_id: String(process.env.SIMULATOR_VOLC_HARDWARE_ID || requestedHardwareID).trim(), + device_secret: String(process.env.SIMULATOR_VOLC_DEVICE_SECRET || "").trim(), + credential_source: "isolated_test_override", + credential_version: 1, + credential_ref: "test-only", + }; + } else { + provisioned = await deviceRequest("POST", "/device/v3/hardware-agent-credentials:provision", { + hardware_id: requestedHardwareID, + }, `windows-hardware-provision-${nonce()}`); + } + const identity = hardwareIdentity({ + instanceID: provisioned?.instance_id, + productKey: provisioned?.product_key, + deviceName: provisioned?.device_name, + hardwareID: provisioned?.hardware_id, + }); + const deviceSecret = String(provisioned?.device_secret || "").trim(); + if (deviceSecret.length < 16 || deviceSecret.length > 4096) throw new Error("Device Secret 长度不正确"); + const persisted = automatedTest ? false : saveHardwareCredential(identity, deviceSecret, { + credentialRef: provisioned?.credential_ref, + credentialVersion: provisioned?.credential_version, + }); + if (!automatedTest && !persisted) throw new Error("Windows 安全存储不可用,拒绝保存 Device Secret"); + hardwareConfig = { ...identity, deviceSecret }; + // 首次预配会由平台发布产品级默认 Bot/传输配置,并可能推进配置版本。 + // 设备必须重新拉取版本,后续创建会话时不能继续上报预配前的旧 revision。 + if (!automatedTest) { + const configuration = await deviceRequest("GET", "/device/v2/config"); + desiredConfigRevision = Number(configuration?.revision || desiredConfigRevision || 1); + } + return { + ...identity, + registered: String(provisioned?.credential_source || "").includes("dynamic_registration"), + persisted, + credentialSource: String(provisioned?.credential_source || "platform_provisioning"), + credentialVersion: Number(provisioned?.credential_version || 1), + configRevision: desiredConfigRevision, + }; +}); + function nonce() { return crypto.randomBytes(16).toString("hex"); } @@ -235,7 +411,7 @@ ipcMain.handle("sim:configure", (_event, input) => configure(input)); ipcMain.handle("sim:probe", async () => { const query = new URLSearchParams({ - firmware_version: "0.3.0-windows-rtc", + firmware_version: "0.3.2-windows-hardware-agent", board_rev: "windows-x64", rtc_sdk_version: "web-4.69.0-electron", codec: "opus", @@ -280,6 +456,14 @@ ipcMain.handle("sim:probe-rtc-network", async () => { return { endpoints }; }); +ipcMain.handle("sim:probe-hardware-network", async () => { + const endpoints = await Promise.all([ + probeTLSEndpoint("ai-gateway.vei.volces.com"), + probeTLSEndpoint("iot-cn-shanghai.iot.volces.com"), + ]); + return { endpoints }; +}); + ipcMain.handle("sim:get-automation-audio", async () => { const configuredPath = String(process.env.SIMULATOR_AUTOMATED_AUDIO_PATH || "").trim(); if (!configuredPath) return undefined; @@ -309,6 +493,173 @@ ipcMain.handle("sim:create-conversation", async () => deviceRequest("POST", "/de requested_goal: "open_conversation", }, `windows-create-${nonce()}`)); +ipcMain.handle("sim:create-hardware-agent-session", async () => deviceRequest("POST", "/device/v3/agent-sessions", { + device_time: new Date().toISOString(), + monotonic_ms: Math.max(0, Math.round(performance.now())), + capabilities: { + hardware_agent: true, + sdk_version: "1.1.0-windows-standard-protocol", + transports: ["low_load"], + input_audio_format: "pcm_s16le_16000_mono", + output_audio_format: "pcm_s16le_16000_mono", + server_event_callback: false, + transcript_upload: true, + dynamic_session_config: true, + credential_rotation: true, + near_pcm_recording: false, + far_pcm_recording: false, + }, + network: { type: "windows-local" }, + requested_goal: "open_conversation", + config_revision: desiredConfigRevision, +}, `windows-hardware-create-${nonce()}`)); + +function emitHardwareEvent(event) { + if (!mainWindow || mainWindow.isDestroyed()) return; + mainWindow.webContents.send("sim:hardware-provider-event", event); +} + +function allowedHardwareGatewayURL() { + const configured = String(process.env.SIMULATOR_HARDWARE_GATEWAY_URL || "wss://ai-gateway.vei.volces.com/v1/realtime").trim(); + const parsed = new URL(configured); + const official = parsed.protocol === "wss:" && parsed.hostname === "ai-gateway.vei.volces.com"; + const local = (parsed.protocol === "ws:" || parsed.protocol === "wss:") && ["127.0.0.1", "localhost", "[::1]"].includes(parsed.hostname); + if (!official && !local) throw new Error("硬件智能体网关只允许火山官方地址或本机测试地址"); + return parsed.toString(); +} + +async function invokeHardwareMemoryTool(event) { + const client = hardwareClient; + const platformSession = activeHardwarePlatformSession; + if (!client || !platformSession) return; + const callID = String(event.call_id || ""); + const functionName = String(client.pendingCalls.get(callID) || "").toLowerCase(); + if (!["memory_search", "search_memory", "retrieve_memory"].includes(functionName)) return; + let argumentsValue = {}; + try { argumentsValue = JSON.parse(String(event.arguments || "{}")); } catch { /* invalid input is handled below */ } + const query = String(argumentsValue.query || argumentsValue.keyword || argumentsValue.text || "").trim(); + const limit = Math.max(1, Math.min(5, Number(argumentsValue.limit || 3))); + try { + if (!query) throw new Error("记忆工具缺少 query 参数"); + const baseURL = new URL(deviceConfig.apiURL); + const target = new URL(platformSession.session_config.tool_url, `${baseURL.origin}/`); + if (target.origin !== baseURL.origin) throw new Error("记忆工具地址必须与平台同源"); + const response = await fetch(target, { + method: "POST", + headers: { + Accept: "application/json", + Authorization: `Bearer ${platformSession.session_config.tool_scope_token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, limit }), + }); + const raw = await response.text(); + let body; + try { body = raw ? JSON.parse(raw) : {}; } catch { body = { message: raw }; } + if (!response.ok) throw new Error(body?.error?.message || body?.message || `HTTP ${response.status}`); + const result = body?.data ?? body; + client.completeFunctionCall(callID, result); + emitHardwareEvent({ type: "platform.memory_search.completed", call_id: callID, query, result_count: result?.items?.length || 0, degraded: Boolean(result?.degraded) }); + } catch (error) { + client.completeFunctionCall(callID, { items: [], degraded: true, error: error.message }); + emitHardwareEvent({ type: "platform.memory_search.failed", call_id: callID, message: error.message }); + } +} + +ipcMain.handle("sim:connect-hardware-agent", async (_event, input) => { + if (!deviceConfig) throw new Error("请先连接平台"); + if (!hardwareConfig) throw new Error("请先完成火山设备鉴权配置"); + if (hardwareClient) throw new Error("已有硬件智能体连接,请先结束"); + const platformSession = input?.platformSession; + if (!platformSession?.runtime?.bot_id || !platformSession?.session_config?.dialog_id) throw new Error("平台硬件会话配置不完整"); + if (platformSession.runtime.transport !== "low_load") throw new Error("Windows 标准协议测试程序只接受设备绑定为 low_load 的会话"); + const modelMode = String(input?.modelMode || "llm").trim().toLowerCase(); + if (!new Set(["llm", "s2s"]).has(modelMode)) throw new Error("智能体模型类型必须为 llm 或 s2s"); + activeHardwarePlatformSession = platformSession; + const client = new HardwareAgentClient({ + gatewayURL: allowedHardwareGatewayURL(), + emit: (event) => { + emitHardwareEvent(event); + if (event.type === "response.function_call_arguments.done") void invokeHardwareMemoryTool(event); + }, + }); + hardwareClient = client; + try { + return await client.connect({ + ...hardwareConfig, + botID: platformSession.runtime.bot_id, + sessionConfig: platformSession.session_config, + modelMode, + }); + } catch (error) { + hardwareClient = null; + activeHardwarePlatformSession = null; + throw error; + } +}); + +ipcMain.on("sim:hardware-agent-audio", (_event, bytes) => { + try { hardwareClient?.sendAudio(Buffer.from(bytes)); } + catch (error) { emitHardwareEvent({ type: "transport.audio_error", message: error.message }); } +}); + +ipcMain.handle("sim:hardware-agent-text", (_event, input) => { + if (!hardwareClient) throw new Error("火山硬件智能体连接未就绪"); + hardwareClient.sendText(input?.text, Number(input?.interruptMode || 1)); + return { accepted: true }; +}); + +ipcMain.handle("sim:close-hardware-agent", (_event, input) => { + const client = hardwareClient; + hardwareClient = null; + activeHardwarePlatformSession = null; + client?.close(Number(input?.code || 1000), String(input?.reason || "device session ended").slice(0, 100)); + return { closed: true }; +}); + +ipcMain.handle("sim:save-hardware-report", async (_event, report) => { + const result = await dialog.showSaveDialog(mainWindow, { + title: "导出硬件对话智能体测试报告", + defaultPath: `hardware-agent-test-${new Date().toISOString().replace(/[:.]/g, "-")}.json`, + filters: [{ name: "JSON 测试报告", extensions: ["json"] }], + }); + if (result.canceled || !result.filePath) return { saved: false }; + const redact = (value) => { + if (Array.isArray(value)) return value.map(redact); + if (!value || typeof value !== "object") return value; + const output = {}; + for (const [key, item] of Object.entries(value)) { + if (/secret|signature|authorization|tool_scope_token|systemmessages|system_role|system_prompt|authorized_context|speaking_style|instructions/i.test(key)) output[key] = "[REDACTED]"; + else output[key] = redact(item); + } + return output; + }; + const safeReport = redact(JSON.parse(JSON.stringify(report))); + fs.writeFileSync(result.filePath, `${JSON.stringify(safeReport, null, 2)}\n`, "utf8"); + return { saved: true, path: result.filePath }; +}); + +ipcMain.handle("sim:hardware-agent-event", async (_event, input) => deviceRequest( + "POST", + `/device/v3/agent-sessions/${encodeURIComponent(input.conversationID)}/events`, + { + event_id: input.eventID || `windows-hardware-event-${nonce()}`, + sequence_no: Number(input.sequenceNo), + event_type: input.eventType, + occurred_at: new Date().toISOString(), + provider_session_ref: input.providerSessionRef, + session_nonce: input.sessionNonce, + ...(input.eventType === "session_started" ? { + provider_connection_ref: input.providerConnectionRef, + bot_id: input.botID, + transport: input.transport, + config_digest: input.configDigest, + dynamic_config_applied: input.dynamicConfigApplied, + } : {}), + ...(input.errorCode ? { error_code: input.errorCode } : {}), + }, +)); + ipcMain.handle("sim:activate-conversation", async (_event, input) => deviceRequest( "POST", `/device/v2/conversations/${encodeURIComponent(input.conversationID)}/activate`, @@ -338,7 +689,7 @@ ipcMain.handle("sim:heartbeat", async (_event, input) => deviceRequest( conversation_status: "ACTIVE", rtc: { joined: true, uplink: true, downlink: true }, network: { type: "windows-local" }, - firmware_version: "0.3.0-windows-rtc", + firmware_version: "0.3.2-windows-hardware-agent", reported_config_revision: desiredConfigRevision, rtc_credential_expires_at: input.credentialExpiresAt, recording_queue: { pending_segments: 0, failed_segments: 0 }, @@ -356,7 +707,7 @@ ipcMain.handle("sim:submit-transcript-batch", async (_event, input) => { const digest = crypto.createHash("sha256").update(JSON.stringify(payload)).digest("hex").slice(0, 32); return deviceRequest( "POST", - `/device/v2/conversations/${encodeURIComponent(conversationID)}/turns:batch`, + `/${input?.hardwareAgent ? "device/v3/agent-sessions" : "device/v2/conversations"}/${encodeURIComponent(conversationID)}/turns:batch`, payload, `windows-turns-${digest}`, ); @@ -433,6 +784,9 @@ app.whenReady().then(() => { }); app.on("window-all-closed", () => { + hardwareClient?.close(1001, "application closed"); + hardwareClient = null; + activeHardwarePlatformSession = null; if (process.platform !== "darwin") app.quit(); }); diff --git a/device-simulator-windows/electron/pcm-playback-worklet.test.cjs b/device-simulator-windows/electron/pcm-playback-worklet.test.cjs new file mode 100644 index 0000000..3c805d5 --- /dev/null +++ b/device-simulator-windows/electron/pcm-playback-worklet.test.cjs @@ -0,0 +1,76 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); +const fs = require("node:fs"); +const path = require("node:path"); +const vm = require("node:vm"); + +const source = fs.readFileSync(path.join(__dirname, "../public/pcm-playback.worklet.js"), "utf8"); + +function fixture({ outputRate = 48000, prebufferMs = 120 } = {}) { + const messages = []; + let Processor; + class AudioWorkletProcessor { + constructor() { + this.port = { onmessage: null, postMessage: (message) => messages.push(message) }; + } + } + const context = vm.createContext({ + AudioWorkletProcessor, + Float32Array, + ArrayBuffer, + Math, + sampleRate: outputRate, + registerProcessor: (name, implementation) => { + assert.equal(name, "pcm16-playback"); + Processor = implementation; + }, + }); + vm.runInContext(source, context); + const processor = new Processor({ processorOptions: { sourceSampleRate: 16000, prebufferMs } }); + const push = (values) => { + const samples = new Float32Array(values); + processor.port.onmessage({ data: { type: "push", samples: samples.buffer } }); + }; + const process = () => { + const output = new Float32Array(128); + assert.equal(processor.process([], [[output]]), true); + return output; + }; + return { messages, processor, push, process }; +} + +test("waits for prebuffer then resamples continuous 16k PCM without fragment gaps", () => { + const f = fixture({ outputRate: 48000, prebufferMs: 10 }); + f.push(new Array(120).fill(0.25)); + assert.ok(f.process().every((sample) => sample === 0), "must remain silent below the prebuffer threshold"); + f.push(new Array(400).fill(0.25)); + const first = f.process(); + const second = f.process(); + assert.ok(first.some((sample) => sample > 0.05), "buffered playback should start with a short anti-pop fade"); + assert.ok(second.every((sample) => sample > 0.12), "playback should remain continuous across source fragments"); + assert.equal(f.messages.filter((message) => message.type === "underflow").length, 0); +}); + +test("plays 16k source at real-time duration instead of the former 1.5x speed", () => { + const f = fixture({ outputRate: 48000, prebufferMs: 1 }); + f.push(new Array(1000).fill(0.25)); + f.process(); + assert.ok(Math.abs(f.processor.readPosition - 128 / 3) < 0.001, `consumed ${f.processor.readPosition} source samples`); +}); + +test("reports network starvation and re-enters prebuffering instead of scheduling tiny buffers", () => { + const f = fixture({ outputRate: 48000, prebufferMs: 0.01 }); + f.push(new Array(8).fill(0.5)); + f.process(); + assert.equal(f.messages.filter((message) => message.type === "underflow").length, 1); + assert.ok(f.process().every((sample) => sample === 0), "starved playback should wait for a new prebuffer"); +}); + +test("drains the final response cleanly and emits a completion signal", () => { + const f = fixture({ outputRate: 24000 }); + f.push(new Array(64).fill(-0.25)); + f.processor.port.onmessage({ data: { type: "end" } }); + f.process(); + assert.equal(f.messages.filter((message) => message.type === "drained").length, 1); + assert.equal(f.messages.filter((message) => message.type === "underflow").length, 0); +}); diff --git a/device-simulator-windows/electron/preload.cjs b/device-simulator-windows/electron/preload.cjs index 453efb8..8dae624 100644 --- a/device-simulator-windows/electron/preload.cjs +++ b/device-simulator-windows/electron/preload.cjs @@ -5,8 +5,23 @@ contextBridge.exposeInMainWorld("deviceSimulator", { configure: (input) => ipcRenderer.invoke("sim:configure", input), probe: () => ipcRenderer.invoke("sim:probe"), probeRTCNetwork: () => ipcRenderer.invoke("sim:probe-rtc-network"), + probeHardwareNetwork: () => ipcRenderer.invoke("sim:probe-hardware-network"), + getHardwareRuntimeDefaults: () => ipcRenderer.invoke("sim:get-hardware-runtime-defaults"), + configureHardware: (input) => ipcRenderer.invoke("sim:configure-hardware", input), getAutomationAudio: () => ipcRenderer.invoke("sim:get-automation-audio"), createConversation: () => ipcRenderer.invoke("sim:create-conversation"), + createHardwareAgentSession: () => ipcRenderer.invoke("sim:create-hardware-agent-session"), + connectHardwareAgent: (input) => ipcRenderer.invoke("sim:connect-hardware-agent", input), + sendHardwareAgentAudio: (bytes) => ipcRenderer.send("sim:hardware-agent-audio", bytes), + sendHardwareAgentText: (input) => ipcRenderer.invoke("sim:hardware-agent-text", input), + closeHardwareAgent: (input) => ipcRenderer.invoke("sim:close-hardware-agent", input), + onHardwareProviderEvent: (callback) => { + const listener = (_event, value) => callback(value); + ipcRenderer.on("sim:hardware-provider-event", listener); + return () => ipcRenderer.removeListener("sim:hardware-provider-event", listener); + }, + saveHardwareReport: (report) => ipcRenderer.invoke("sim:save-hardware-report", report), + hardwareAgentEvent: (input) => ipcRenderer.invoke("sim:hardware-agent-event", input), activateConversation: (input) => ipcRenderer.invoke("sim:activate-conversation", input), getConversation: (conversationID) => ipcRenderer.invoke("sim:get-conversation", conversationID), heartbeat: (input) => ipcRenderer.invoke("sim:heartbeat", input), diff --git a/device-simulator-windows/index.html b/device-simulator-windows/index.html index a5e2681..77c4329 100644 --- a/device-simulator-windows/index.html +++ b/device-simulator-windows/index.html @@ -4,10 +4,10 @@ -