Compare commits
8
Commits
v0.72.0-rc.1
..
第一版
| Author | SHA1 | Date | |
|---|---|---|---|
 Codex Local
|
0c2151ace9 | ||
|
Codex Local
|
a8cdec98d1 | ||
|
Codex Local
|
78e0a19acb | ||
|
Codex Local
|
dcf0fc0d7f | ||
|
Codex Local
|
522785eda8 | ||
|
Codex Local
|
89f6641331 | ||
|
Codex Local
|
2337c6cfaa | ||
|
Codex Local
|
962dd1a063 |
@@ -85,7 +85,7 @@ jobs:
|
||||
- name: Infrastructure configuration scan
|
||||
run: >-
|
||||
docker run --rm -v "$PWD:/src" aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969
|
||||
config --exit-code 1 --severity HIGH,CRITICAL /src
|
||||
config --ignorefile /src/.trivyignore.yaml --exit-code 1 --severity HIGH,CRITICAL /src
|
||||
- name: Image vulnerability scan
|
||||
run: |
|
||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 image --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL data-terminal-cloud-backend:test
|
||||
|
||||
@@ -22,3 +22,7 @@ web/dist/
|
||||
test/e2e/screenshots/
|
||||
.design-evidence/
|
||||
device-simulator-windows/story-continuity-rtc/
|
||||
/.tmp-*/
|
||||
/firmware/**/build/
|
||||
/firmware/**/managed_components/
|
||||
/firmware/**/backups/
|
||||
|
||||
+3
-3
@@ -4,8 +4,8 @@ title = "Data terminal cloud secret scanning"
|
||||
useDefault = true
|
||||
|
||||
[allowlist]
|
||||
description = "HTTP idempotency keys are replay identifiers, not credentials"
|
||||
description = "Exact public HTTP idempotency fixtures are replay identifiers, not credentials"
|
||||
condition = "AND"
|
||||
regexTarget = "line"
|
||||
regexes = ['''Idempotency-Key''']
|
||||
paths = ['''(^|/)internal/httpapi/conversation_handlers_test\.go$''']
|
||||
regexes = ['''(?:test-idempotency-key-000[12]|test-activate-idempotency-0001|turns-0123456789abcdef|3ca4e514-cbde-41fc-aa32-6f1ed460cb01)''']
|
||||
paths = ['''(^|/)internal/httpapi/conversation_handlers_test\.go$''','''(^|/)docs/设备终端接口文档\.md$''']
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
misconfigurations:
|
||||
- id: KSV-0109
|
||||
paths:
|
||||
- "deploy/kubernetes/base/configmap.yaml"
|
||||
statement: >-
|
||||
RTC_TOKEN_ISSUER and the *_TTL values are public runtime metadata, not
|
||||
credentials. All RTC signing keys, provider credentials, and token key
|
||||
material are mounted from Kubernetes Secrets or the production CSI
|
||||
secret store.
|
||||
@@ -4,6 +4,19 @@
|
||||
|
||||
## 未发布
|
||||
|
||||
- 新增V51硬件智能体License池：License明文仅在平台加密库存与设备安全区中出现，平台按Instance ID/Product Key原子分配，设备校验摘要并回执后才允许`PROVISIONING → ACTIVE`；过期、吊销、丢失和解绑会立即阻断新会话。
|
||||
- 新增V50设备激活状态机：`FACTORY → CLAIMABLE → BOUND → PROVISIONING → ACTIVE → LOST/REVOKED`，认领、火山凭据预配、解绑、丢失恢复和注销统一写入带修订号的迁移审计；LOST/解绑会暂停火山设备凭据，恢复后重新启用。
|
||||
- 新增独立`volc_hardware_agent`运行时和`/device/v3/agent-sessions`控制面；设备每次聊天从平台取得冻结的人设、档案、近期摘要、已确认长期记忆和授权上下文，不再复用RTC房间语义。
|
||||
- 新增V49平台代理火山设备注册和`/device/v3/hardware-agent-credentials:provision`：Product Secret只从平台Secret/KMS加载，一机一密在平台AES-256-GCM加密并下发对应设备，Windows端不再接收Product Secret。
|
||||
- 硬件会话新增按设备Bot/传输/最低SDK绑定、完整能力快照、动态`session.updated`生效证明、单调事件水位、可靠最终字幕、平台`stop_session`动作、恢复扫描和SaaS运行时可见性。
|
||||
- 不具备近端与远端双路PCM录制能力的硬件会话强制降级为仅文本或不留存；通话中记忆工具扩展到硬件会话并继续从数据库反查租户、长者和活动会话边界。
|
||||
- 新增V47/V48/V49硬件运行时及凭据迁移、Windows控制面模拟接口、端到端正反向冒烟，以及带清单和SHA-256的服务器部署候选包工具。
|
||||
- 维护中心新增租户级固件与OTA升级管理：可信Ed25519公钥、浏览器私有直传、大小与SHA-256校验、发布通道、强制升级标记、兼容设备灰度批次、可靠升级命令、暂停/恢复/退役和审计下载地址。
|
||||
- 固件制品支持维护端租户级范围，同时保留原家庭端记录和设备受控下载链路；设备资产列表显示最近上报的当前固件版本。
|
||||
- 聊天总结改为大模型两级内容价值判断：先判断是否值得保留，再判断是否值得形成故事；纯寒暄不再生成摘要或故事。
|
||||
- 新增内容价值判断记录，保留模型、提示词版本、置信度和舍弃原因，支持任务幂等与隐私删除。
|
||||
- 本地开发新增 GPU Ollama/Qwen2.5 7B 部署方式，并通过大模型管理的测试、激活和 worker 生成链路完成端到端验证。
|
||||
- 修复大模型“测试连接”更新语句和审计结果枚举错误，测试通过的模型现在可以正常激活。
|
||||
- 维护端默认人设升级为老人陪伴对话规范，覆盖倾听、单问题追问、沉默与重复处理、事实边界、敏感话题和医疗安全。
|
||||
- 家庭端长者档案新增日常陪伴、回忆往事、兴趣聊天三个模板，并补充兴趣、优先话题和避免话题的创建入口。
|
||||
- Agent固定评测加强单问题追问、先回应再追问、沉默不施压、尊重重复表达和情绪优先回应检查。
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
- 私有数据导出：按长者和分类生成JSON，MinIO私有存储、SHA-256校验、短期签名下载和过期对象清理。
|
||||
- 主体级隐私删除：精确称呼确认、活动通话保护、立即冻结读取/上传/召回，以及PostgreSQL、Redis、对象存储、pgvector、外部索引和私有导出六子系统的可重试对账与最小删除证明。
|
||||
- 设备命令可靠链路：家庭端幂等排队、设备HMAC主动拉取、重复投递计数、确认/成功/失败回执和过期终态；Web设备卡片可下发诊断命令。
|
||||
- 固件安全发布与管理页：Ed25519受信公钥、浏览器私有直传、大小与SHA-256校验、签名清单验证、确定性灰度、可靠命令、短期下载、暂停、已验证版本回滚，以及逐设备下载/安装/确认进度。
|
||||
- 家庭端及维护中心固件安全发布与OTA管理：租户级/家庭级Ed25519受信公钥、浏览器私有直传、大小与SHA-256校验、签名清单验证、兼容设备确定性灰度、可靠命令、设备与维护人员短期下载、暂停、已验证版本回滚，以及逐设备下载/安装/确认进度。
|
||||
- 双轨录音媒体链路：near/far分片清单幂等、15分钟单对象直传、Outbox可靠校验、WAV PCM16容器/时长/大小/SHA-256核验、延迟删除本地密文及设备回执，以及PLAY/DOWNLOAD独立授权、授权列表/撤销、React双轨播放下载、5分钟私有访问和成功/拒绝审计。
|
||||
- 联调问题闭环：按客户组织记录问题现象、严重级别、设备/通话/请求编号、责任方、根因、修复版本和脱敏回归证据；未关闭HIGH/CRITICAL形成发布阻断汇总，创建幂等、更新强ETag并全程审计。
|
||||
- 认证SSE状态事件流：业务事务内生成最小事件，15分钟`Last-Event-ID`续传、活动家庭成员过滤、跨租户游标拒绝、重复事件抑制、设备上线去重/超时离线、React自动重连和2秒合并刷新，避免连续事件放大为整页请求风暴。
|
||||
@@ -119,6 +119,8 @@ Web镜像在容器启动时根据`WEB_OIDC_AUTHORITY`和`WEB_OIDC_CLIENT_ID`生
|
||||
|
||||
真实火山RTC模式由两组独立开关启用：API设置`RTC_TOKEN_ISSUER=volcengine`、`RTC_APP_ID`和`RTC_APP_KEY`，Worker设置`VOICE_PROVIDER=volcengine`、`VOLC_ACCESS_KEY_ID`、`VOLC_SECRET_ACCESS_KEY`、区域、接口版本以及`VOLC_VOICE_CHAT_TEMPLATE_FILE`。生产模板采用`voice-chat-template-bundle-v1`，分别提供`s2s_pure`、`s2s_hybrid`和`cascade`三个完整VoiceChat `Config/AgentConfig`；平台按通话冻结模式选择模板，再覆盖AppId、RoomId、TaskId、目标设备用户、智能体用户，并追加本次已授权且脱敏的背景摘要。级联模式写入`LLMConfig.SystemMessages`，纯端到端模式写入`S2SConfig.ProviderParams.dialog.system_role`。可从[模板示例](deploy/volcengine/voice-chat-template.example.json)复制结构并按照已开通的官方能力替换每个完整`Config`对象；不要把AppKey、AK/SK、ASR/TTS Token提交到仓库。
|
||||
|
||||
火山硬件对话智能体使用独立的`volc_hardware_agent`运行时。Instance ID、Product Key、默认Bot ID、`high_quality`/`low_load`传输方式和最低SDK版本是平台持有的产品级配置，创建设备时只选择运行时，不再填写Bot。设备首次调用`/device/v3/hardware-agent-credentials:provision`时，平台自动建立该产品默认绑定，并使用KMS/Secret中的Product Secret代设备调用火山DynamicRegister；火山从产品额度中分配License并返回本设备的一机一密，License本身不下发设备。设备安全保存Device Secret后进入`ACTIVE`。每次聊天再调用`/device/v3/agent-sessions`取得默认Bot和本次冻结的人设、近期摘要、长期记忆与授权上下文，并通过火山SDK或标准WebSocket动态注入。平台只有在设备证明`session.updated`已应用且Bot、传输方式、配置摘要完全匹配后才激活通话；设备不具备近/远端双路PCM能力时，本次留存自动降级为仅文本。设备必须持续上报v3事件和最终字幕，并执行事件响应中的`stop_session`。完整时序、重试和凭据边界见[设备集成说明](docs/device-integration.md#115-火山硬件对话智能体-v3-控制面)。
|
||||
|
||||
设备HMAC密钥保存在`device_credential`表中，并使用AES-256-GCM和每条记录独立随机nonce加密；数据库不保存明文。API通过`DEVICE_CREDENTIAL_MASTER_KEY`（64位十六进制、解码后32字节）解密验签，该主密钥必须由Docker/Kubernetes Secret或云KMS CSI独立托管，不能进入数据库。`DEVICE_LOCAL_SECRET`只用于把既有演示设备凭据一次性导入数据库，完成迁移后可从运行环境移除。
|
||||
|
||||
数据库连接串、Redis、RTC AppKey、火山AK/SK、方舟API Key、对象存储凭据、回调/告警签名等敏感配置同时支持`NAME_FILE=/run/secrets/name`。文件路径必须是容器内绝对路径、目标必须是普通文件且不超过64KiB；读取时只去掉末尾换行，不在错误中打印内容。`NAME`与`NAME_FILE`同时存在会拒绝启动，防止轮换时仍意外使用旧的明文环境变量。
|
||||
|
||||
@@ -36,6 +36,7 @@ x-backend-environment: &backend-environment
|
||||
PROVIDER_MEMORY_TIMEOUT: ${PROVIDER_MEMORY_TIMEOUT:-5s}
|
||||
DEVICE_CLOCK_SKEW: 5m
|
||||
DEVICE_CREDENTIAL_MASTER_KEY: ${DEVICE_CREDENTIAL_MASTER_KEY:-000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f}
|
||||
MODEL_CREDENTIAL_MASTER_KEY: ${MODEL_CREDENTIAL_MASTER_KEY:-1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100}
|
||||
RTC_APP_ID: local-public-rtc-app
|
||||
RTC_APP_KEY: ${RTC_APP_KEY:-}
|
||||
RTC_TOKEN_ISSUER: ${RTC_TOKEN_ISSUER:-local}
|
||||
@@ -54,6 +55,9 @@ x-backend-environment: &backend-environment
|
||||
VOLC_RTC_REGION: ${VOLC_RTC_REGION:-cn-north-1}
|
||||
VOLC_RTC_API_VERSION: ${VOLC_RTC_API_VERSION:-2024-12-01}
|
||||
VOLC_VOICE_CHAT_TEMPLATE_FILE: ${VOLC_VOICE_CHAT_TEMPLATE_FILE:-}
|
||||
VOLC_HARDWARE_AGENT_BOT_ID: ${VOLC_HARDWARE_AGENT_BOT_ID:-local-hardware-agent-bot}
|
||||
VOLC_HARDWARE_AGENT_TRANSPORT: ${VOLC_HARDWARE_AGENT_TRANSPORT:-high_quality}
|
||||
VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION: ${VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION:-1.0.0}
|
||||
PROVIDER_FAILURE_THRESHOLD: "3"
|
||||
PROVIDER_CIRCUIT_COOLDOWN: 1m
|
||||
RECOVERY_SCAN_INTERVAL: 5s
|
||||
@@ -82,6 +86,7 @@ x-backend-build: &backend-build
|
||||
VERSION: ${PLATFORM_BUILD_VERSION:-dev}
|
||||
COMMIT: ${PLATFORM_BUILD_COMMIT:-unknown}
|
||||
BUILD_TIME: ${PLATFORM_BUILD_TIME:-unknown}
|
||||
GOPROXY: ${GOPROXY:-https://proxy.golang.org,direct}
|
||||
|
||||
x-backend-dependencies: &backend-dependencies
|
||||
postgres:
|
||||
@@ -154,6 +159,26 @@ services:
|
||||
volumes:
|
||||
- ./deploy/keycloak:/opt/keycloak/data/import:ro
|
||||
|
||||
ollama:
|
||||
profiles: [local-model]
|
||||
image: ollama/ollama:latest
|
||||
ports:
|
||||
- "${OLLAMA_PORT:-11434}:11434"
|
||||
volumes:
|
||||
- ollama-data:/root/.ollama
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
healthcheck:
|
||||
test: ["CMD", "ollama", "list"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:latest
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
@@ -218,6 +243,14 @@ services:
|
||||
HTTP_ADDR: 0.0.0.0:8080
|
||||
DEVICE_LOCAL_SECRET: local-simulated-device-secret-32-bytes-minimum
|
||||
RTC_TOKEN_SIGNING_SECRET: local-simulated-rtc-token-secret-32-bytes-minimum
|
||||
VOLC_HARDWARE_PROVISIONING_ENABLED: ${VOLC_HARDWARE_PROVISIONING_ENABLED:-false}
|
||||
VOLC_HARDWARE_INSTANCE_ID: ${VOLC_HARDWARE_INSTANCE_ID:-}
|
||||
VOLC_HARDWARE_PRODUCT_KEY: ${VOLC_HARDWARE_PRODUCT_KEY:-}
|
||||
VOLC_HARDWARE_PRODUCT_SECRET: ${VOLC_HARDWARE_PRODUCT_SECRET:-}
|
||||
VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY: ${VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY:-}
|
||||
VOLC_HARDWARE_REGISTER_URL: ${VOLC_HARDWARE_REGISTER_URL:-https://iot-cn-shanghai.iot.volces.com/2021-12-14/DynamicRegister?Action=DynamicRegister&Version=2021-12-14}
|
||||
VOLC_HARDWARE_REGISTER_TIMEOUT: ${VOLC_HARDWARE_REGISTER_TIMEOUT:-10s}
|
||||
VOLC_HARDWARE_DEVICE_NAME_PREFIX: ${VOLC_HARDWARE_DEVICE_NAME_PREFIX:-dt-}
|
||||
ports:
|
||||
- "${API_PORT:-58080}:8080"
|
||||
depends_on: *backend-dependencies
|
||||
@@ -311,6 +344,7 @@ services:
|
||||
VERSION: ${PLATFORM_BUILD_VERSION:-dev}
|
||||
COMMIT: ${PLATFORM_BUILD_COMMIT:-unknown}
|
||||
BUILD_TIME: ${PLATFORM_BUILD_TIME:-unknown}
|
||||
NPM_CONFIG_REGISTRY: ${NPM_CONFIG_REGISTRY:-https://registry.npmjs.org}
|
||||
environment:
|
||||
WEB_OIDC_AUTHORITY: ${WEB_OIDC_AUTHORITY:-http://localhost:58180/realms/voice-local}
|
||||
WEB_OIDC_CLIENT_ID: ${WEB_OIDC_CLIENT_ID:-voice-platform-web}
|
||||
@@ -324,3 +358,4 @@ volumes:
|
||||
postgres-data:
|
||||
redis-data:
|
||||
minio-data:
|
||||
ollama-data:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
FROM golang:1.26.6-alpine AS build
|
||||
ARG GOPROXY=https://proxy.golang.org,direct
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -50,6 +50,17 @@ data:
|
||||
VOLC_RTC_BASE_URL: https://rtc.volcengineapi.com
|
||||
VOLC_RTC_REGION: cn-north-1
|
||||
VOLC_RTC_API_VERSION: "2024-12-01"
|
||||
# Platform-owned product defaults. Devices receive these only through the
|
||||
# signed provisioning/session control plane and cannot override them.
|
||||
VOLC_HARDWARE_AGENT_BOT_ID: "replace-with-product-default-bot-id"
|
||||
VOLC_HARDWARE_AGENT_TRANSPORT: "high_quality"
|
||||
VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION: "1.0.0"
|
||||
VOLC_HARDWARE_PROVISIONING_ENABLED: "false"
|
||||
VOLC_HARDWARE_INSTANCE_ID: ""
|
||||
VOLC_HARDWARE_PRODUCT_KEY: ""
|
||||
VOLC_HARDWARE_REGISTER_URL: https://iot-cn-shanghai.iot.volces.com/2021-12-14/DynamicRegister?Action=DynamicRegister&Version=2021-12-14
|
||||
VOLC_HARDWARE_REGISTER_TIMEOUT: 10s
|
||||
VOLC_HARDWARE_DEVICE_NAME_PREFIX: "dt-"
|
||||
CONTENT_GENERATOR: ark
|
||||
CONTENT_SAFETY_PROVIDER: managed_http
|
||||
CONTENT_SAFETY_ENDPOINT: https://content-safety.example.invalid/v1/classify
|
||||
|
||||
@@ -19,7 +19,7 @@ metadata:
|
||||
spec:
|
||||
provider: replace-with-installed-provider
|
||||
parameters:
|
||||
objects: replace-with-provider-specific-rtc-app-key-cursor-share-and-device-credential-master-key
|
||||
objects: replace-with-provider-specific-rtc-app-key-cursor-share-device-credential-master-key-volc-hardware-product-secret-and-volc-device-credential-master-key
|
||||
---
|
||||
apiVersion: secrets-store.csi.x-k8s.io/v1
|
||||
kind: SecretProviderClass
|
||||
|
||||
@@ -6,6 +6,13 @@ metadata:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
fsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumes:
|
||||
- $patch: replace
|
||||
- name: common-secrets
|
||||
@@ -35,6 +42,13 @@ metadata:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
fsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumes:
|
||||
- $patch: replace
|
||||
- name: common-secrets
|
||||
@@ -58,6 +72,13 @@ metadata:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
fsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumes:
|
||||
- $patch: replace
|
||||
- name: common-secrets
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# 本地聊天总结模型
|
||||
|
||||
本地开发环境使用 Ollama 提供 OpenAI 兼容接口，默认验证模型为 `qwen2.5:7b`。模型只监听本机 `11434` 端口，应用容器通过 `http://ollama:11434/v1` 访问。
|
||||
|
||||
## 启动与下载模型
|
||||
|
||||
```powershell
|
||||
docker compose --profile local-model up -d ollama
|
||||
docker compose --profile local-model exec ollama ollama pull qwen2.5:7b
|
||||
```
|
||||
|
||||
随后在客户门户的“大模型管理”中创建并激活配置：
|
||||
|
||||
- 服务类型：`OpenAI 兼容接口`
|
||||
- 接口基础地址：`http://ollama:11434/v1`
|
||||
- 模型 ID：`qwen2.5:7b`
|
||||
- API Key：本地 Ollama 不校验密钥，可填写长度至少 8 位的本地占位值
|
||||
- 建议温度：`0.1`
|
||||
- 建议最大输出：`1200`
|
||||
|
||||
必须先执行“测试连接”，通过后才能“设为当前模型”。配置激活后，worker 会在每个总结任务开始时读取当前模型，不需要重启。
|
||||
|
||||
## 真实模型回归测试
|
||||
|
||||
```powershell
|
||||
$env:LOCAL_MODEL_BASE_URL='http://localhost:11434/v1'
|
||||
$env:LOCAL_MODEL_ID='qwen2.5:7b'
|
||||
go test ./internal/extraction -run TestLocalModelContentValueDecisions -count=1 -v
|
||||
```
|
||||
|
||||
测试覆盖三类输入：纯寒暄应舍弃、有用近况只生成摘要、完整个人回忆生成摘要和故事候选。
|
||||
@@ -1,3 +1,4 @@
|
||||
FROM migrate/migrate:v4.18.3
|
||||
|
||||
COPY migrations /migrations
|
||||
COPY --chown=65532:65532 migrations /migrations
|
||||
USER 65532:65532
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
该配置用于单机公网演示，不是生产验收配置。只向公网发布HTTP 80和HTTPS 443；PostgreSQL、Redis、MinIO、Keycloak内部端口以及API/Worker端口均不映射到宿主机。当前阿里云演示机使用Let's Encrypt短期IP证书，`certbot-ip-renew.timer`负责自动续期。单机备份覆盖PostgreSQL、Redis、MinIO对象和当前Keycloak数据目录，并执行数据库级隔离恢复校验；五分钟健康检查覆盖部署模式、公网入口、OIDC、容器、模拟服务、备份、外部告警、证书、定时任务和磁盘。跨故障域备份、真实告警接收器、真实RTC、模型、内容安全和云KMS仍按阶段8验收清单执行。
|
||||
|
||||
无法直接从Git拉取或需要冻结当前工作树时，可先在开发机执行`powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\build-server-deployment-bundle.ps1 -Version 0.76.0-rc.hardware1`，再用`powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\verify-server-deployment-bundle.ps1 -Archive .\.deploy-artifacts\data-terminal-cloud-0.76.0-rc.hardware1.zip`验证。归档包含`deploy-manifest.json`和`SHA256SUMS`，不会打包`.env`、设备/供应商密钥、本地运行目录、`node_modules`或构建缓存。上传服务器后必须再次核对归档侧车`.sha256`和包内清单，再覆盖候选目录；服务器现有`deploy/server/.env`及`/etc/data-terminal-cloud`密钥目录不得从归档替换。
|
||||
|
||||
服务器至少需要Docker Engine 29与Compose 2.40。首次部署先生成权限为600的服务器专用配置：
|
||||
|
||||
```sh
|
||||
@@ -30,6 +32,15 @@ docker compose -f compose.yaml -f deploy/server/compose.override.yaml \
|
||||
|
||||
需要在演示机接入真实RTC、豆包语音和方舟摘要时，把凭据文件以`0600`权限放入`/etc/data-terminal-cloud/voice-runtime`，并在`deploy/server/.env`设置`RTC_TOKEN_ISSUER=volcengine`、`VOICE_PROVIDER=volcengine`、`RTC_APP_ID`、`CONTENT_GENERATOR`和`ARK_CONTENT_MODEL`。Compose只读挂载该目录，敏感值不进入源码、镜像或环境文件。
|
||||
|
||||
切换火山硬件对话智能体时，维护中心逐台只需把`voice_runtime`设为`volc_hardware_agent`，不填写设备级Bot。平台代设备动态注册时，在`deploy/server/.env`设置`VOLC_HARDWARE_PROVISIONING_ENABLED=true`、`VOLC_HARDWARE_INSTANCE_ID`、`VOLC_HARDWARE_PRODUCT_KEY`、`VOLC_HARDWARE_AGENT_BOT_ID`、`VOLC_HARDWARE_AGENT_TRANSPORT`、`VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION`、`VOLC_HARDWARE_PRODUCT_SECRET_FILE=/run/secrets/voice-runtime/volc-hardware-product-secret`和独立随机的`VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY`；将Product Secret以单行、`0600`权限写入`/etc/data-terminal-cloud/voice-runtime/volc-hardware-product-secret`。Product Secret不得进入`.env`、源码、镜像、数据库或设备。设备首次预配时，平台根据这些产品默认值自动创建火山绑定并调用DynamicRegister；火山从产品额度中分配License，平台只加密保存并下发本设备的一机一密Device Secret，不下发License明文。发布前必须验证平台代理注册、默认Bot绑定、每次会话上下文注入、`session.updated`确认后激活、`session_updated`存活上报、`stop_session`执行和最终字幕补传。
|
||||
|
||||
```sh
|
||||
sudo install -d -m 0700 /etc/data-terminal-cloud/voice-runtime
|
||||
sudo sh -c 'umask 077; read -r product_secret; printf "%s" "$product_secret" > /etc/data-terminal-cloud/voice-runtime/volc-hardware-product-secret'
|
||||
```
|
||||
|
||||
上面的`read`从终端读取，不要把真实Product Secret写进Shell历史。启用前确认默认Bot关联的产品与`VOLC_HARDWARE_PRODUCT_KEY`完全一致；Device Name由`dt-`加平台设备UUID稳定生成。密钥文件或产品默认配置缺失、格式错误时API拒绝启动，未启用时凭据接口明确返回`HARDWARE_AGENT_PROVISIONING_UNAVAILABLE`。
|
||||
|
||||
设备HMAC密钥由`device_credential`表以AES-256-GCM密文保存。`deploy/server/.env`中的`DEVICE_CREDENTIAL_MASTER_KEY`只负责数据库密文的认证解密，不得写入数据库或设备；既有`DEVICE_LOCAL_SECRET`只用于首次部署升级时导入两台演示设备，确认数据库凭据记录完整后可移除。新增、轮换和吊销设备密钥使用后端镜像内的`/app/device-credential`维护命令，生成的设备明文密钥只显示一次。
|
||||
|
||||
通话内记忆工具只通过网关精确开放`/internal/v1/tools/memory-search`，请求必须携带平台为当前通话签发的短期任务令牌。`MEMORY_TOOL_SIGNING_SECRET`必须是服务器独立随机密钥，不得与游标签名、分享签名或RTC密钥复用；网关不会开放其他`/internal`路径。
|
||||
|
||||
@@ -34,6 +34,8 @@ umask 077
|
||||
printf 'KEYCLOAK_ADMIN_PASSWORD=%s\n' "$(openssl rand -hex 24)"
|
||||
printf 'DEVICE_LOCAL_SECRET=%s\n' "$(openssl rand -hex 32)"
|
||||
printf 'DEVICE_CREDENTIAL_MASTER_KEY=%s\n' "$(openssl rand -hex 32)"
|
||||
printf 'VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY=%s\n' "$(openssl rand -hex 32)"
|
||||
printf 'MODEL_CREDENTIAL_MASTER_KEY=%s\n' "$(openssl rand -hex 32)"
|
||||
printf 'RTC_TOKEN_SIGNING_SECRET=%s\n' "$(openssl rand -hex 32)"
|
||||
printf 'CURSOR_SIGNING_SECRET=%s\n' "$(openssl rand -hex 32)"
|
||||
printf 'SHARE_TOKEN_SIGNING_SECRET=%s\n' "$(openssl rand -hex 32)"
|
||||
|
||||
@@ -21,6 +21,7 @@ x-server-backend-environment: &server-backend-environment
|
||||
SHARE_TOKEN_SIGNING_SECRET: ${SHARE_TOKEN_SIGNING_SECRET}
|
||||
MEMORY_TOOL_BASE_URL: ${PUBLIC_ORIGIN}/internal/v1/tools/memory-search
|
||||
MEMORY_TOOL_SIGNING_SECRET: ${MEMORY_TOOL_SIGNING_SECRET}
|
||||
MODEL_CREDENTIAL_MASTER_KEY: ${MODEL_CREDENTIAL_MASTER_KEY}
|
||||
EMBEDDING_PROVIDER: local
|
||||
VOICE_PROVIDER: ${VOICE_PROVIDER:-mock}
|
||||
|
||||
@@ -101,6 +102,16 @@ services:
|
||||
RTC_APP_ID: ${RTC_APP_ID:-local-public-rtc-app}
|
||||
RTC_APP_KEY: ""
|
||||
RTC_APP_KEY_FILE: ${RTC_APP_KEY_FILE:-/run/secrets/voice-runtime/rtc-app-key}
|
||||
VOLC_HARDWARE_PROVISIONING_ENABLED: ${VOLC_HARDWARE_PROVISIONING_ENABLED:-false}
|
||||
VOLC_HARDWARE_INSTANCE_ID: ${VOLC_HARDWARE_INSTANCE_ID:-}
|
||||
VOLC_HARDWARE_PRODUCT_KEY: ${VOLC_HARDWARE_PRODUCT_KEY:-}
|
||||
VOLC_HARDWARE_AGENT_BOT_ID: ${VOLC_HARDWARE_AGENT_BOT_ID:-}
|
||||
VOLC_HARDWARE_AGENT_TRANSPORT: ${VOLC_HARDWARE_AGENT_TRANSPORT:-low_load}
|
||||
VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION: ${VOLC_HARDWARE_AGENT_MINIMUM_SDK_VERSION:-1.0.0}
|
||||
VOLC_HARDWARE_PRODUCT_SECRET: ""
|
||||
VOLC_HARDWARE_PRODUCT_SECRET_FILE: ${VOLC_HARDWARE_PRODUCT_SECRET_FILE:-}
|
||||
VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY: ${VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY:-}
|
||||
VOLC_HARDWARE_DEVICE_NAME_PREFIX: ${VOLC_HARDWARE_DEVICE_NAME_PREFIX:-dt-}
|
||||
volumes:
|
||||
- /etc/data-terminal-cloud/voice-runtime:/run/secrets/voice-runtime:ro
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
FROM node:24-alpine AS build
|
||||
ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org
|
||||
WORKDIR /src
|
||||
COPY web/package*.json ./
|
||||
RUN npm ci
|
||||
@@ -15,6 +16,10 @@ LABEL org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.created="${BUILD_TIME}"
|
||||
USER root
|
||||
RUN apk upgrade --no-cache
|
||||
# The image's IPv6 helper shells out to `apk manifest nginx` on every start.
|
||||
# On small production hosts that command can block even after the package
|
||||
# database is healthy. This deployment owns default.conf and does not need it.
|
||||
RUN rm -f /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
|
||||
COPY --chown=101:101 deploy/web.nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --chmod=755 --chown=101:101 deploy/web-runtime-config.sh /docker-entrypoint.d/40-runtime-config.sh
|
||||
COPY --chown=101:101 --from=build /src/dist /usr/share/nginx/html
|
||||
|
||||
+46
-47
@@ -1,60 +1,59 @@
|
||||
# Design QA — 陪伴周记
|
||||
# 家庭陪伴门户 v0.74 设计 QA
|
||||
|
||||
- source visual truth path: `E:\Codex\story\platform\.design-evidence\selected-conversation-journal.png`
|
||||
- implementation screenshot path: `E:\Codex\story\platform\.design-evidence\implementation-home-final.png`
|
||||
- comparison image path: `E:\Codex\story\platform\.design-evidence\design-comparison-final.png`
|
||||
- responsive evidence: `E:\Codex\story\platform\.design-evidence\implementation-home-mobile.png`
|
||||
- viewport: desktop 1280 × 720 CSS px; mobile 390 × 844 CSS px
|
||||
- source pixels: 1488 × 1058
|
||||
- implementation pixels: 1265 × 720 for the desktop browser content capture; device scale factor 1
|
||||
- normalization: source was downsampled to 1265 px wide and cropped to the top 720 px; implementation remained at its native capture size. The two normalized images were placed in one 2530 × 720 comparison image.
|
||||
- state: authenticated family owner, one real seeded elder selected, completed-conversation data loaded, no dialog open
|
||||
- 功能与页面规格：`docs/customer-portal-v074-page-spec.md`
|
||||
- 设计稿源：`.design-evidence/v074/design/*.png`
|
||||
- 实际页面截图：`.design-evidence/v074/implementation/*.png`
|
||||
- 同屏对比证据：`.design-evidence/v074/comparison/*.png`
|
||||
- 统一视口：1440 × 1024 CSS px，device scale factor 1
|
||||
- 验收状态：本地所有者账号、合成数据一号家庭、2026-08-30、每页默认业务状态
|
||||
|
||||
## Full-view comparison evidence
|
||||
## 页面验收矩阵
|
||||
|
||||
The final comparison shows the same information architecture and visual hierarchy as the selected direction: warm paper background, white header and navigation, family tabs, portrait-based elder selection, a seven-day conversation/mood journal, latest-chat summary, family-attention card, and recent activity. The implementation uses real seeded data, so the number of elders, dates, durations, summary availability, and account text intentionally differ from the concept image.
|
||||
| 页面 | 核心功能 | 设计/实现对比 | 结果 |
|
||||
| --- | --- | --- | --- |
|
||||
| 首页 | 长者周记、心情线索、关怀提醒、最近动态 | `comparison/home.png` | 通过 |
|
||||
| 聊天记录 | 时间轴、AI 摘要、关键词、故事候选、下次话题、模型来源 | `comparison/conversations.png` | 通过 |
|
||||
| 故事与记忆 | 待审故事、审核入口、已确认记忆和搜索 | `comparison/stories.png` | 通过 |
|
||||
| 长者资料 | 头像、长期资料、交流节奏、兴趣和授权入口 | `comparison/subjects.png` | 通过 |
|
||||
| 日程与提醒 | 今日陪伴动态、待处理事项、近期状态汇总 | `comparison/messages.png` | 通过 |
|
||||
| 家庭管理 | 家庭切换、成员、角色、邀请、家庭资料和动态 | `comparison/members.png` | 通过 |
|
||||
| 陪伴设备 | 绑定、在线状态、固件、诊断和配置 | `comparison/devices.png` | 通过 |
|
||||
| 内容分享 | 资源级分享、接收人、权限、期限和撤销 | `comparison/sharing.png` | 通过 |
|
||||
| 数据与授权 | 数据盘点、更正、撤回、私有导出和主体删除 | `comparison/privacy.png` | 通过 |
|
||||
| 服务用量 | 聊天时长、活跃日、热力图、设备与最近通话 | `comparison/billing.png` | 通过 |
|
||||
| 大模型管理 | 模型配置、连接测试、激活、移除和密钥说明 | `comparison/models.png` | 通过 |
|
||||
| 设置与权限 | 身份、家庭角色和 15 项完整权限矩阵 | `comparison/profile.png` | 通过 |
|
||||
|
||||
## Focused region evidence
|
||||
## 必检视觉项
|
||||
|
||||
- Home journal: weekly duration columns, highlighted current day, mood markers, topic chips, summary card, and the non-diagnostic mood disclaimer were inspected in the desktop comparison.
|
||||
- Avatar flow: `.design-evidence/avatar-upload-preview.png` records the file-picker result and generated WebP preview; the browser assertion confirmed a `data:image/webp` preview with no error.
|
||||
- Permission matrix: `.design-evidence/permission-matrix.png` records all 14 capabilities across four family roles, including the two separately granted recording permissions.
|
||||
- Mobile: `.design-evidence/implementation-home-mobile.png` records the 390 px layout; browser measurement confirmed no document-level horizontal overflow.
|
||||
- 页面标题与家庭标签紧邻，标题底部到家庭上下文栏的间距为 0；内容区统一 16px 节奏。
|
||||
- 侧栏、页头、家庭 tab、卡片边框、圆角、按钮高度和表单控件使用同一套 v0.74 token。
|
||||
- 1440px 视口下 12 个页面均无页面级横向溢出，无裁切的主要按钮或正文。
|
||||
- 家庭管理重新出现在一级导航，使用可直接点击的家庭 tab，不再显示“客户组织”。
|
||||
- 日程与提醒不再使用大段空白消息头；隐私空状态压缩到 150px；故事页长者筛选保持单行。
|
||||
- 禁用按钮统一 `cursor: wait`；长者头像上传可显示本地预览。
|
||||
|
||||
## Required fidelity surfaces
|
||||
## 功能检查
|
||||
|
||||
- Fonts and typography: Chinese system sans-serif hierarchy matches the source's clear editorial treatment; headings, small metadata, labels, and long summary text remain readable and do not truncate important content.
|
||||
- Spacing and layout rhythm: sidebar/content proportions, family tabs, elder strip, two-column journal, lower cards, borders, and radii align with the source after two spacing corrections. Mobile cards stack without page overflow.
|
||||
- Colors and tokens: ivory paper, warm white cards, dusty blue interaction color, coral attention cue, muted green positive cue, and warm hairlines match the selected palette. No decorative gradient is used.
|
||||
- Image quality and asset fidelity: real supplied portrait assets are used with circular crops; uploaded avatars are center-cropped and resized to a 320 × 320 WebP. Ant Design icons supply all UI symbols.
|
||||
- Copy and content: customer-organization and platform-role jargon were removed from customer pages. “本次交流目标” became “长期陪伴方向”; pace help text states that the setting is sent to Doubao at the start of every new chat. Mood copy explicitly describes an inference from authorized summaries and not a health diagnosis.
|
||||
- 12 个一级/二级导航均能进入对应页面。
|
||||
- 新增长者头像上传与预览通过。
|
||||
- 接受家庭邀请、邀请新成员、绑定设备、新建分享、资料更正、添加大模型、故事详情 7 个非破坏性弹窗打开/关闭通过。
|
||||
- 聊天记录选中一条记录后，AI 总结区能进入真实加载/结果状态。
|
||||
- 所有采样到的禁用按钮均显示等待光标。
|
||||
- 浏览器控制台错误、非预期 HTTP 4xx/5xx、失败请求均为 0。
|
||||
|
||||
## Comparison history
|
||||
## 对比迭代
|
||||
|
||||
### Iteration 1 — blocked
|
||||
1. 修复家庭管理入口缺失，统一页面名称与客户视角的信息架构。
|
||||
2. 移除工作区旧的大空白标题层，合并家庭 tab、实时状态、刷新和页面动作。
|
||||
3. 故事页长者筛选从第二行收回同一上下文栏，并为故事卡补充“开始审核”。
|
||||
4. 日程与提醒重构为今日动态、家人待办和近期汇总，不再堆叠无意义消息。
|
||||
5. 设置页补齐家庭上下文，并集中列出 15 项角色权限。
|
||||
6. 隐私页压缩空状态；用量页日期范围校准为 2025-09 至 2026-08。
|
||||
7. 设计稿与实际页以相同 1440 × 1024 视口逐页同屏复核。
|
||||
|
||||
- P1: the authenticated home still had excessive top padding and a nested workspace card, pushing the journal well below the selected composition.
|
||||
- P2: the 390 px layout expanded to 570 px because the seven-day grid contributed a fixed min-content width.
|
||||
- Fixes: removed customer-page/workspace padding with specificity above the legacy theme, tightened the family/elder strip, contained journal cards and the weekly grid, and corrected the mobile recent-activity grid.
|
||||
- Post-fix evidence: `.design-evidence/implementation-home-v2.png` and `.design-evidence/implementation-home-mobile.png`.
|
||||
## 最终结论
|
||||
|
||||
### Iteration 2 — passed
|
||||
|
||||
- Earlier findings are resolved: family tabs now begin directly below the header, the journal occupies the intended above-the-fold area, and mobile has no document-level horizontal overflow.
|
||||
- Real-data differences (one elder rather than four, seeded durations, pending summary) are expected product constraints and do not change the design grammar.
|
||||
- No actionable P0, P1, or P2 visual issue remains.
|
||||
|
||||
## Browser and interaction verification
|
||||
|
||||
- Tested every customer section at 1280 px: home, conversations, stories, subjects, messages, members, devices, sharing, privacy, billing, and profile.
|
||||
- Confirmed every section has no horizontal overflow and no visible “客户组织” copy.
|
||||
- Confirmed all disabled buttons report `cursor: wait`.
|
||||
- Opened the new-elder dialog, uploaded a real PNG through the browser file chooser, and confirmed the WebP preview.
|
||||
- Confirmed the profile renders 14 permission rows.
|
||||
- Confirmed the final home state has no visible request error and browser logs contain no error-level entry (only Vite reconnect/debug messages from the intentional test-server restart).
|
||||
|
||||
## Follow-up polish
|
||||
|
||||
- P3: when a family contains only one elder, the portrait row naturally has more empty horizontal space than the four-person concept. This is an accurate data state and can later be enriched with a compact “添加长者” affordance if desired.
|
||||
没有遗留的 P0、P1 或 P2 视觉/交互问题。设计稿中的演示故事、已配置模型和已有分享，与本地数据库的空状态属于业务数据差异；页面结构、视觉系统和核心操作保持一致。
|
||||
|
||||
final result: passed
|
||||
|
||||
@@ -1,8 +1,45 @@
|
||||
# Windows 语音终端模拟器
|
||||
# Windows 硬件对话智能体测试程序
|
||||
|
||||
该程序使用Electron和火山引擎Web RTC SDK模拟一台真实语音终端。它采集Windows麦克风、加入平台分配的RTC房间、发布本地音频、订阅并播放豆包智能体音频，并通过设备v2接口完成激活、状态等待、心跳、Token续签和结束通话。进房后还会启动RTC实时字幕，收集长者与智能体的最终话轮并通过现有`turns:batch`设备接口可靠上传；字幕未成功启动时拒绝继续激活通话，避免生成没有聊天记录的完成通话。
|
||||
这是方案 1 的 Windows 端到端验收工具。默认页面不再使用平台代管的 RTC/StartVoiceChat，而是按火山“硬件对话智能体低负载标准协议”直接连接 `wss://ai-gateway.vei.volces.com/v1/realtime`。传统 RTC 模拟器仍保留在页面右上角的“传统 RTC 测试页”。
|
||||
|
||||
创建通话返回的隐私策略允许原录音时，模拟器同时通过RTC原始音频帧回调采集长者`near`和智能体`far`轨道。结束通话后自动封装为单声道WAV PCM16、按平台限制分片、申请短期对象地址直传，并等待后台完整性与容器校验完成。家庭工作台的通话记录会标记“可听聊天语音”，获得独立PLAY授权的账号可在详情中选择“听长者语音”或“听智能体语音”。
|
||||
## 实际覆盖的流程
|
||||
|
||||
1. 使用平台设备 ID、时间戳、随机数、正文摘要和 HMAC-SHA256 调用设备 v2/v3 接口。
|
||||
2. 调用平台 `/device/v3/hardware-agent-credentials:provision`；平台自动应用产品级Instance ID、Product Key、默认Bot、传输和最低SDK配置，并使用KMS/Secret挂载的 Product Secret 代设备调用 DynamicRegister。
|
||||
3. 火山在动态注册时从产品额度中分配 License；平台只把本设备 Device Secret 下发给设备，Windows 使用 Safe Storage 加密保存后重新拉取配置版本并进入 `ACTIVE`。
|
||||
4. 创建 `/device/v3/agent-sessions`，从平台取得产品默认 Bot ID、`low_load` 传输方案、本次冻结的人设、背景、最近摘要、确认记忆和授权范围。
|
||||
5. 按火山规定的请求头签名建立 WebSocket，等待 `session.created`，提交本次 `session.update`，收到 `session.updated` 后才向平台回报 `session_started`并开放麦克风。
|
||||
6. Windows 麦克风被转换为 PCM16、16kHz、单声道、20ms 分片，通过 `input_audio_buffer.append` 实时上行；`response.audio.delta` 按火山 low-load WebSocket SDK 的 PCM16、16kHz、单声道格式实时播放。
|
||||
7. 消费服务端 VAD、用户最终识别、智能体最终字幕、Function Calling 和错误事件；双方最终话轮通过 v3 `turns:batch` 幂等上传。
|
||||
8. 智能体触发 `memory_search`、`search_memory` 或 `retrieve_memory` 时，程序使用平台签发的当前会话短期令牌调用同源记忆工具，再把结果作为 `function_call_output` 返回火山。
|
||||
9. 周期提交 `session_updated`。平台返回 `stop_session` 时立即关麦、补传字幕、关闭火山连接并回报 `session_ended`。
|
||||
10. 页面显示 13 项验收门禁，可导出不包含任何密钥的 JSON 报告。
|
||||
|
||||
## 使用前配置
|
||||
|
||||
平台维护中心只需把测试设备配置为：
|
||||
|
||||
- `voice_runtime = volc_hardware_agent`
|
||||
|
||||
平台部署需配置并关联同一火山产品和 Bot：
|
||||
|
||||
- Instance ID
|
||||
- Product Key
|
||||
- 默认 Bot ID
|
||||
- 传输方式 `low_load`
|
||||
- 最低 SDK 版本不高于 `1.1.0-windows-standard-protocol`
|
||||
- Product Secret（只进入平台密钥挂载，不进入Windows程序）
|
||||
- Device Name 由平台按设备ID稳定生成，或使用设备绑定中预先登记的名称
|
||||
- 可用的硬件对话智能体 License
|
||||
|
||||
Windows程序不提供 Product Secret、Device Secret 或 Bot ID 输入框。Bot ID、Instance ID、Product Key 和 Device Name 均来自平台；设备不能覆盖产品默认路由。
|
||||
|
||||
开始测试前还需按控制台中的 Bot 模型选择“智能体模型类型”：
|
||||
|
||||
- 通用 LLM：使用官方 `config.LLMConfig.SystemMessages` 注入平台冻结的人设、背景与授权上下文；
|
||||
- 豆包端到端语音：使用官方 `config.S2SConfig.ProviderParams.dialog` 注入 `system_role`、`speaking_style` 和 `dialog_id`。
|
||||
|
||||
两种模式都使用官方 `session.object/config/agent_config/input_audio_format/output_audio_format` 报文结构。字段依据火山[初始化会话配置](https://www.volcengine.com/docs/6348/2139862?lang=zh)与[标准协议自定义接入](https://www.volcengine.com/docs/6348/1904602?lang=zh)文档。
|
||||
|
||||
## 构建
|
||||
|
||||
@@ -12,36 +49,70 @@ npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
输出：`release/voice-device-simulator-0.1.0-windows-x64.exe`。
|
||||
输出文件：
|
||||
|
||||
连接公网测试服务器时，可以直接双击EXE，也可以使用`launch-cloud-device.ps1`启动。Electron主进程优先读取启动脚本传入的临时环境变量，否则从当前Windows用户的`%LOCALAPPDATA%\DataTerminalVoiceDevice\server-device-secret`读取设备签名密钥。密钥不写入源码、安装包或浏览器存储，也不会回传到渲染进程；输入框留空并显示“已从Windows安全凭据文件加载”即表示可直接检测。
|
||||
|
||||
## 自动连续性验证
|
||||
|
||||
本地回归时可让模拟器自动启动通话、播放一段测试音频，并把双方最终字幕写入JSON结果文件。该入口只接受本机文件和回环地址，不影响正常交互使用：
|
||||
|
||||
```powershell
|
||||
$env:SIMULATOR_AUTO_TEST = '1'
|
||||
$env:SIMULATOR_AUTOMATED_AUDIO_PATH = 'C:\path\to\question.wav'
|
||||
$env:SIMULATOR_AUTO_RESULT_PATH = 'C:\path\to\result.json'
|
||||
& .\release\voice-device-simulator-0.1.0-windows-x64.exe
|
||||
```text
|
||||
release/hardware-agent-tester-0.3.5-windows-x64.exe
|
||||
```
|
||||
|
||||
测试会等待长者与智能体的最终字幕，结束通话后退出；结果文件包含通话ID、最终话轮和通过/失败状态。测试音频支持WAV、MP3和OGG，大小不得超过10 MB。
|
||||
当前测试包未配置商业代码签名证书，Windows 可能显示“未知发布者”。交付前可用同目录 SHA-256 文件校验完整性；正式面向终端用户分发时应由甲方或发布主体使用其代码签名证书签名。
|
||||
|
||||
`npm run build` 会依次执行协议单测、TypeScript 检查、双页面构建和 Windows x64 便携包封装。
|
||||
|
||||
## 本地完整自动回归
|
||||
|
||||
先启动仓库根目录的 Compose 开发环境，然后运行：
|
||||
|
||||
```powershell
|
||||
Set-Location .\device-simulator-windows
|
||||
pwsh -NoProfile -File .\test\run-hardware-e2e.ps1
|
||||
pwsh -NoProfile -File .\test\run-hardware-e2e.ps1 -ModelMode s2s
|
||||
pwsh -NoProfile -File .\test\run-hardware-e2e.ps1 -ExecutablePath .\release\hardware-agent-tester-0.3.5-windows-x64.exe
|
||||
```
|
||||
|
||||
第一条验证通用 LLM 报文，第二条验证端到端语音报文。回归会临时创建 `low_load` 设备绑定，启动一个校验真实火山签名和事件格式的本机 WebSocket 网关，再启动实际 Electron 窗口。测试只有在以下证据全部成立时才返回 PASS：
|
||||
|
||||
- 动态上下文确认后才开放麦克风；
|
||||
- PCM 上行和音频下行均发生；
|
||||
- 用户与智能体各产生一条最终字幕并写入 PostgreSQL；
|
||||
- 平台下发停止状态，Windows 程序通过下一次 `session_updated` 收到 `stop_session`；
|
||||
- 会话、硬件会话均为 `COMPLETED`；
|
||||
- SaaS `conversation.started` 和 `conversation.completed` 事件均已产生；
|
||||
- 页面 13 项验收门禁全部通过。
|
||||
|
||||
测试结束会删除临时会话、事件和设备绑定，不污染日常开发数据。
|
||||
|
||||
## 安全边界
|
||||
|
||||
- RTC AppKey、火山AK/SK和豆包Access Token只存在本地后端的受保护文件中；
|
||||
- 模拟器安装包只接收平台签发的短期RTC Token；
|
||||
- 设备签名密钥只驻留Electron主进程内存，不写入浏览器存储；
|
||||
- 当前测试构建只允许设备API地址指向`localhost`、`127.0.0.1`或固定测试服务器`https://121.41.46.238`，避免误连其他环境；
|
||||
- 关闭软件或结束通话时停止音频采集并销毁RTC引擎。
|
||||
- 未取得`record_near`或`record_far`授权的轨道不会注册PCM回调；`upload_allowed=false`时不会采集或上传任何原录音。
|
||||
- Product Secret 只存在于平台KMS/Secret挂载和注册调用内存，不到达设备，不写入数据库明文、日志或报告。
|
||||
- 平台返回的本设备Device Secret使用 Electron `safeStorage` 加密后保存在当前 Windows 用户目录，不进入源码和安装包；License由火山产品侧分配，不下发Windows程序。
|
||||
- 平台设备密钥只驻留 Electron 主进程，渲染页面不能读取已加载的明文。
|
||||
- 记忆工具地址必须与配置的平台 API 同源，防止短期令牌被发送给第三方地址。
|
||||
- 默认只允许连接火山官方硬件网关；本机 `ws://127.0.0.1` 仅供自动回归。
|
||||
- 平台确认 `session.updated` 前不会启动麦克风；收到 `stop_session` 后不会继续发送音频。
|
||||
- 导出的报告包含设备标识、Bot、配置摘要、事件和字幕，但不包含 Product Secret、Device Secret、平台签名密钥或记忆工具令牌。
|
||||
- 自动回归通过仅在`SIMULATOR_AUTO_TEST=1`时生效的进程环境注入mock一机一密，不写入正式Windows凭据缓存。
|
||||
|
||||
## 实时字幕前提
|
||||
## 真实验收与本地模拟的区别
|
||||
|
||||
测试所用RTC AppID必须在火山RTC控制台开启“实时字幕”和“流式语音识别”。流式语音识别凭据来自豆包语音控制台创建的“流式语音识别”应用，包括APP ID、Access Token和Cluster ID；“音视频字幕生成”类型不能替代该应用。未开通时程序会显示`ServiceAccessDenied`并清理本次未激活通话。
|
||||
### 0.3.5 播放速度修复
|
||||
|
||||
最终字幕会在程序的“实时字幕与上传”区域显示。每条最终字幕使用通话内稳定键和严格递增序号上传；网络失败时留在内存队列，并在心跳及结束通话前重试。存在未上传字幕时，程序不会结束并丢弃当前通话。
|
||||
- 根据火山 low-load WebSocket SDK 示例，将智能体下行 PCM 的源采样率从错误的 24kHz 修正为 16kHz，避免 1.5 倍速和音调升高。
|
||||
- Windows 输出设备不是 16kHz 时，由 AudioWorklet 按实际系统输出采样率连续重采样，不改变语速和音高。
|
||||
|
||||
完整本地启动顺序见[本地真实RTC联调说明](../deploy/local-rtc/README.md)。
|
||||
### 0.3.4 连续音频播放修复
|
||||
|
||||
- 火山返回的 `pcm_s16le/16kHz/mono` 音频进入常驻 AudioWorklet 环形缓冲，不再按 WebSocket 分片创建独立播放器。
|
||||
- 默认预缓冲 120ms，网络瞬时抖动时自动重新缓冲，减少断音、金属音和语速漂移。
|
||||
- 连续处理跨分片的奇数字节，并在开始、结束和断流时做短淡入淡出以避免爆音。
|
||||
- 测试报告新增播放分片数、采样数和欠载次数。
|
||||
|
||||
### 0.3.3 失败会话修复
|
||||
|
||||
- 火山 `error` 事件与连接 Promise 的异常共用一个失败清理任务，同一轮仅提交一次 `session_failed`，保留首次错误。
|
||||
- 失败清理期间禁止发送文字或重复结束；延迟到达的事件不再恢复“通话中”。失败后重新检测即可开始新一轮。
|
||||
- 新通话清空上一轮验收结果、错误和计时；失败清理的关闭不再标为正常关闭。
|
||||
- `npm test` 包含执行实际渲染器代码的并发失败回归测试，不会连接线上平台或消耗火山额度。
|
||||
- 此修复不代表火山 `internal_error: StartVoiceChat failed` 的上游原因已解决。
|
||||
|
||||
本地自动回归证明 Windows 程序、平台控制面、数据库、PCM 管道和协议状态机完整可执行。最终真实火山验收仍必须使用实际 Instance/Product/Device/Bot/License 凭据，确认火山返回真实 ASR、LLM、TTS 音频；程序不会把本机模拟网关的 PASS 冒充真实供应商验收。
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
const crypto = require("node:crypto");
|
||||
const WebSocket = require("ws");
|
||||
|
||||
const DEFAULT_GATEWAY_URL = "wss://ai-gateway.vei.volces.com/v1/realtime";
|
||||
|
||||
function requiredText(value, name, maximum = 512) {
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized || normalized.length > maximum) throw new Error(`${name}不能为空或过长`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function validateDeviceName(value) {
|
||||
const name = requiredText(value, "火山设备名称", 128);
|
||||
if (!/^[a-zA-Z0-9_\-.:@]{1,128}$/.test(name)) throw new Error("火山设备名称格式不正确");
|
||||
return name;
|
||||
}
|
||||
|
||||
function hmacBase64(secret, content) {
|
||||
return crypto.createHmac("sha256", secret).update(content).digest("base64");
|
||||
}
|
||||
|
||||
function websocketSignature({ authType = 1, deviceName, randomNumber, productKey, timestamp, instanceID }, deviceSecret) {
|
||||
const content = `auth_type=${authType}&device_name=${deviceName}&random_num=${randomNumber}&product_key=${productKey}×tamp=${timestamp}&instance_id=${instanceID}`;
|
||||
return hmacBase64(deviceSecret, content);
|
||||
}
|
||||
|
||||
function buildSessionPrompt(sessionConfig) {
|
||||
const sections = [
|
||||
requiredText(sessionConfig.authorized_context, "授权上下文", 4000),
|
||||
String(sessionConfig.system_prompt || "").trim(),
|
||||
String(sessionConfig.persona_tone || "").trim() ? `对话语气：${String(sessionConfig.persona_tone).trim()}` : "",
|
||||
`本次对话标识：${requiredText(sessionConfig.dialog_id, "对话标识", 200)}`,
|
||||
].filter(Boolean);
|
||||
return sections.join("\n\n");
|
||||
}
|
||||
|
||||
function buildSessionUpdate(sessionConfig, eventID, modelMode = "llm") {
|
||||
const normalizedMode = String(modelMode || "llm").trim().toLowerCase();
|
||||
if (!new Set(["llm", "s2s"]).has(normalizedMode)) throw new Error("智能体模型类型必须为 llm 或 s2s");
|
||||
const dialogID = requiredText(sessionConfig.dialog_id, "对话标识", 200);
|
||||
const systemPrompt = buildSessionPrompt(sessionConfig);
|
||||
const config = {
|
||||
ASRConfig: { TurnDetectionMode: 0 },
|
||||
};
|
||||
if (normalizedMode === "s2s") {
|
||||
config.S2SConfig = {
|
||||
ProviderParams: {
|
||||
dialog: {
|
||||
system_role: systemPrompt,
|
||||
speaking_style: String(sessionConfig.persona_tone || "").trim(),
|
||||
dialog_id: dialogID,
|
||||
},
|
||||
},
|
||||
};
|
||||
} else {
|
||||
config.LLMConfig = { SystemMessages: [systemPrompt] };
|
||||
}
|
||||
return {
|
||||
event_id: eventID,
|
||||
type: "session.update",
|
||||
session: {
|
||||
object: "realtime.session",
|
||||
model: "",
|
||||
input_audio_format: "pcm16",
|
||||
output_audio_format: "pcm16",
|
||||
config,
|
||||
agent_config: { EnableConversationStateCallback: false },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeProviderError(event) {
|
||||
const code = String(event?.error?.code || event?.error?.type || "PROVIDER_ERROR");
|
||||
const message = String(event?.error?.message || "火山硬件智能体返回错误");
|
||||
return new Error(`${code}: ${message}`);
|
||||
}
|
||||
|
||||
class HardwareAgentClient {
|
||||
constructor(options = {}) {
|
||||
this.WebSocketImpl = options.WebSocketImpl || WebSocket;
|
||||
this.gatewayURL = options.gatewayURL || DEFAULT_GATEWAY_URL;
|
||||
this.emit = typeof options.emit === "function" ? options.emit : () => {};
|
||||
this.socket = undefined;
|
||||
this.sessionCreated = undefined;
|
||||
this.providerConnectionRef = "";
|
||||
this.pendingCalls = new Map();
|
||||
this.pingTimer = undefined;
|
||||
this.connectionPromise = undefined;
|
||||
this.sessionUpdateSent = false;
|
||||
}
|
||||
|
||||
async connect(input) {
|
||||
if (this.socket) throw new Error("火山硬件智能体连接已经存在");
|
||||
const instanceID = requiredText(input.instanceID, "实例 ID", 200);
|
||||
const productKey = requiredText(input.productKey, "Product Key", 200);
|
||||
const deviceName = validateDeviceName(input.deviceName);
|
||||
const deviceSecret = requiredText(input.deviceSecret, "Device Secret", 4096);
|
||||
const hardwareID = requiredText(input.hardwareID, "Hardware ID", 200);
|
||||
const botID = requiredText(input.botID, "Bot ID", 200);
|
||||
const randomNumber = crypto.randomInt(1, 2_147_483_647);
|
||||
const timestamp = Math.floor(Date.now() / 1000);
|
||||
const signature = websocketSignature({ deviceName, randomNumber, productKey, timestamp, instanceID }, deviceSecret);
|
||||
const target = new URL(this.gatewayURL);
|
||||
target.searchParams.set("bot", botID);
|
||||
target.searchParams.set("wait_for_session_update", "true");
|
||||
const headers = {
|
||||
"X-Auth-Type": "1",
|
||||
"X-Product-Key": productKey,
|
||||
"X-Device-Name": deviceName,
|
||||
"X-Random-Num": String(randomNumber),
|
||||
"X-Timestamp": String(timestamp),
|
||||
"X-Instance-Id": instanceID,
|
||||
"X-Signature": signature,
|
||||
"X-Hardware-Id": hardwareID,
|
||||
};
|
||||
const socket = new this.WebSocketImpl(target.toString(), { headers, handshakeTimeout: 10_000, maxPayload: 8 * 1024 * 1024 });
|
||||
this.socket = socket;
|
||||
this.connectionPromise = new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => reject(new Error("等待火山 session.updated 超时")), 15_000);
|
||||
const finish = (callback, value) => {
|
||||
clearTimeout(timeout);
|
||||
callback(value);
|
||||
};
|
||||
socket.once("open", () => {
|
||||
this.emit({ type: "transport.open" });
|
||||
// wait_for_session_update=true makes the gateway wait for the client
|
||||
// configuration before it publishes the usable session. Sending only
|
||||
// after session.created deadlocks until the provider-side timeout.
|
||||
this.sendJSON(buildSessionUpdate(input.sessionConfig, `windows-session-update-${crypto.randomUUID()}`, input.modelMode));
|
||||
this.sessionUpdateSent = true;
|
||||
});
|
||||
socket.on("message", (data) => {
|
||||
let event;
|
||||
try { event = JSON.parse(Buffer.isBuffer(data) ? data.toString("utf8") : String(data)); }
|
||||
catch { this.emit({ type: "transport.invalid_message" }); return; }
|
||||
this.handleProviderEvent(event, input.sessionConfig, input.modelMode);
|
||||
// In the hardware gateway's wait_for_session_update mode, the initial
|
||||
// session.created event is published only after the pre-creation
|
||||
// session.update has been accepted. Some Bot versions do not emit an
|
||||
// additional session.updated event for that initial configuration.
|
||||
if (event.type === "session.updated" || (event.type === "session.created" && this.sessionUpdateSent)) {
|
||||
const connectionRef = String(event?.session?.id || this.providerConnectionRef || event.event_id || "").trim();
|
||||
if (!connectionRef) return finish(reject, new Error("火山会话确认未提供可关联的会话编号"));
|
||||
this.providerConnectionRef = connectionRef;
|
||||
finish(resolve, { providerConnectionRef: connectionRef, session: event.session || {} });
|
||||
} else if (event.type === "error" && !this.providerConnectionRef) {
|
||||
finish(reject, normalizeProviderError(event));
|
||||
}
|
||||
});
|
||||
socket.once("unexpected-response", (_request, response) => {
|
||||
finish(reject, new Error(`火山 WebSocket 鉴权失败：HTTP ${response.statusCode}`));
|
||||
});
|
||||
socket.once("error", (error) => {
|
||||
this.emit({ type: "transport.error", message: error.message });
|
||||
if (!this.providerConnectionRef) finish(reject, error);
|
||||
});
|
||||
socket.once("close", (code, reason) => {
|
||||
this.stopPing();
|
||||
this.socket = undefined;
|
||||
this.emit({ type: "transport.close", code, reason: Buffer.from(reason || "").toString("utf8") });
|
||||
if (!this.providerConnectionRef) finish(reject, new Error(`火山连接提前关闭：${code}`));
|
||||
});
|
||||
});
|
||||
try {
|
||||
const result = await this.connectionPromise;
|
||||
this.startPing();
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.close(1011, "connection setup failed");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
handleProviderEvent(event, sessionConfig, modelMode) {
|
||||
if (event.type === "session.created") {
|
||||
this.sessionCreated = event.session || {};
|
||||
this.providerConnectionRef = String(event?.session?.id || event.event_id || "").trim();
|
||||
}
|
||||
if (event.type === "conversation.item.created" && event?.item?.type === "function_call" && event.item.call_id) {
|
||||
this.pendingCalls.set(String(event.item.call_id), String(event.item.name || ""));
|
||||
}
|
||||
this.emit(event);
|
||||
}
|
||||
|
||||
sendAudio(bytes) {
|
||||
const buffer = Buffer.from(bytes);
|
||||
if (!buffer.length) return;
|
||||
if (buffer.length > 64 * 1024) throw new Error("单个音频分片不能超过64KiB");
|
||||
this.sendJSON({
|
||||
event_id: `windows-audio-${crypto.randomUUID()}`,
|
||||
type: "input_audio_buffer.append",
|
||||
audio: buffer.toString("base64"),
|
||||
});
|
||||
}
|
||||
|
||||
sendText(text, interruptMode = 1) {
|
||||
const normalized = requiredText(text, "文本问题", 2000);
|
||||
this.sendJSON({
|
||||
event_id: `windows-text-${crypto.randomUUID()}`,
|
||||
type: "conversation.item.create",
|
||||
item: { type: "message", role: "user", content: [{ type: "input_text", text: normalized }], interrupt_mode: interruptMode },
|
||||
});
|
||||
}
|
||||
|
||||
completeFunctionCall(callID, output) {
|
||||
const normalizedCallID = requiredText(callID, "函数调用编号", 200);
|
||||
this.sendJSON({
|
||||
event_id: `windows-function-${crypto.randomUUID()}`,
|
||||
type: "conversation.item.create",
|
||||
item: { type: "function_call_output", call_id: normalizedCallID, output: JSON.stringify(output) },
|
||||
});
|
||||
this.sendJSON({ event_id: `windows-response-${crypto.randomUUID()}`, type: "response.create" });
|
||||
this.pendingCalls.delete(normalizedCallID);
|
||||
}
|
||||
|
||||
sendJSON(value) {
|
||||
if (!this.socket || this.socket.readyState !== this.WebSocketImpl.OPEN) throw new Error("火山硬件智能体连接未就绪");
|
||||
this.socket.send(JSON.stringify(value));
|
||||
}
|
||||
|
||||
startPing() {
|
||||
this.stopPing();
|
||||
this.pingTimer = setInterval(() => {
|
||||
try { if (this.socket?.readyState === this.WebSocketImpl.OPEN && typeof this.socket.ping === "function") this.socket.ping(); }
|
||||
catch { /* close/error callbacks own recovery */ }
|
||||
}, 20_000);
|
||||
}
|
||||
|
||||
stopPing() {
|
||||
if (this.pingTimer) clearInterval(this.pingTimer);
|
||||
this.pingTimer = undefined;
|
||||
}
|
||||
|
||||
close(code = 1000, reason = "device session ended") {
|
||||
this.stopPing();
|
||||
const socket = this.socket;
|
||||
this.socket = undefined;
|
||||
this.providerConnectionRef = "";
|
||||
this.sessionUpdateSent = false;
|
||||
this.pendingCalls.clear();
|
||||
if (!socket) return;
|
||||
try { socket.close(code, reason); } catch { try { socket.terminate?.(); } catch { /* no-op */ } }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_GATEWAY_URL,
|
||||
HardwareAgentClient,
|
||||
buildSessionPrompt,
|
||||
buildSessionUpdate,
|
||||
websocketSignature,
|
||||
};
|
||||
@@ -0,0 +1,97 @@
|
||||
const assert = require("node:assert/strict");
|
||||
const { EventEmitter } = require("node:events");
|
||||
const test = require("node:test");
|
||||
const {
|
||||
HardwareAgentClient,
|
||||
buildSessionUpdate,
|
||||
websocketSignature,
|
||||
} = require("./hardware-agent-client.cjs");
|
||||
|
||||
test("WebSocket signatures use the documented canonical field order", () => {
|
||||
const connection = websocketSignature({ authType: 1, deviceName: "device-1", randomNumber: 42, productKey: "product-1", timestamp: 1700000000, instanceID: "instance-1" }, "device-secret");
|
||||
assert.equal(connection, "6OFE1DUZzRgNy0o5IzZSmqKw3SUP4s3GxK2YHe8phfo=");
|
||||
});
|
||||
|
||||
test("LLM session update uses the official Config/AgentConfig schema", () => {
|
||||
const update = buildSessionUpdate({ dialog_id: "conversation-1", system_prompt: "你是陪伴助手", persona_tone: "亲切", authorized_context: "只可使用已授权背景" }, "event-1");
|
||||
assert.equal(update.type, "session.update");
|
||||
assert.equal(update.session.object, "realtime.session");
|
||||
assert.equal(update.session.input_audio_format, "pcm16");
|
||||
assert.equal(update.session.output_audio_format, "pcm16");
|
||||
assert.equal(update.session.config.ASRConfig.TurnDetectionMode, 0);
|
||||
assert.match(update.session.config.LLMConfig.SystemMessages[0], /只可使用已授权背景/);
|
||||
assert.match(update.session.config.LLMConfig.SystemMessages[0], /你是陪伴助手/);
|
||||
assert.match(update.session.config.LLMConfig.SystemMessages[0], /亲切/);
|
||||
assert.equal(update.session.agent_config.EnableConversationStateCallback, false);
|
||||
assert.equal("system_prompt" in update.session, false);
|
||||
});
|
||||
|
||||
test("S2S session update maps persona and dialog id to provider dialog config", () => {
|
||||
const update = buildSessionUpdate({ dialog_id: "conversation-2", system_prompt: "你是陪伴助手", persona_tone: "亲切自然", authorized_context: "只使用授权信息" }, "event-2", "s2s");
|
||||
const dialog = update.session.config.S2SConfig.ProviderParams.dialog;
|
||||
assert.equal(dialog.dialog_id, "conversation-2");
|
||||
assert.equal(dialog.speaking_style, "亲切自然");
|
||||
assert.match(dialog.system_role, /只使用授权信息/);
|
||||
assert.match(dialog.system_role, /你是陪伴助手/);
|
||||
assert.equal("LLMConfig" in update.session.config, false);
|
||||
});
|
||||
|
||||
class FakeWebSocket extends EventEmitter {
|
||||
static OPEN = 1;
|
||||
constructor(url, options) {
|
||||
super();
|
||||
this.url = url;
|
||||
this.options = options;
|
||||
this.readyState = 0;
|
||||
this.sent = [];
|
||||
FakeWebSocket.last = this;
|
||||
setImmediate(() => {
|
||||
this.readyState = FakeWebSocket.OPEN;
|
||||
this.emit("open");
|
||||
this.emit("message", Buffer.from(JSON.stringify({ type: "session.created", event_id: "created-event", session: { id: "provider-session-1" } })));
|
||||
});
|
||||
}
|
||||
send(body) {
|
||||
const value = JSON.parse(body);
|
||||
this.sent.push(value);
|
||||
if (value.type === "session.update") {
|
||||
setImmediate(() => this.emit("message", Buffer.from(JSON.stringify({ type: "session.updated", event_id: "updated-event", session: { id: "provider-session-1" } }))));
|
||||
}
|
||||
}
|
||||
ping() {}
|
||||
close(code, reason) {
|
||||
this.readyState = 3;
|
||||
this.emit("close", code, Buffer.from(reason));
|
||||
}
|
||||
}
|
||||
|
||||
test("hardware client waits for dynamic config acknowledgement before becoming ready", async () => {
|
||||
const events = [];
|
||||
const client = new HardwareAgentClient({ WebSocketImpl: FakeWebSocket, gatewayURL: "ws://127.0.0.1:19090/v1/realtime", emit: (event) => events.push(event) });
|
||||
const connected = await client.connect({
|
||||
instanceID: "instance-1", productKey: "product-1", deviceName: "device-1",
|
||||
deviceSecret: "device-secret-with-at-least-32-characters", hardwareID: "windows-test", botID: "bot-1",
|
||||
sessionConfig: { dialog_id: "conversation-1", authorized_context: "authorized" },
|
||||
});
|
||||
assert.equal(connected.providerConnectionRef, "provider-session-1");
|
||||
assert.match(FakeWebSocket.last.url, /bot=bot-1/);
|
||||
assert.equal(FakeWebSocket.last.options.headers["X-Product-Key"], "product-1");
|
||||
assert.equal(FakeWebSocket.last.sent[0].type, "session.update");
|
||||
client.sendAudio(Buffer.from([1, 2, 3, 4]));
|
||||
client.sendText("你好");
|
||||
assert.equal(FakeWebSocket.last.sent[1].type, "input_audio_buffer.append");
|
||||
assert.equal(FakeWebSocket.last.sent[2].type, "conversation.item.create");
|
||||
assert.ok(events.some((event) => event.type === "session.created"));
|
||||
client.close();
|
||||
});
|
||||
|
||||
test("hardware client sends session.update immediately when the WebSocket opens", async () => {
|
||||
const client = new HardwareAgentClient({ WebSocketImpl: FakeWebSocket, gatewayURL: "ws://127.0.0.1:19090/v1/realtime" });
|
||||
await client.connect({
|
||||
instanceID: "instance-1", productKey: "product-1", deviceName: "device-1",
|
||||
deviceSecret: "device-secret-with-at-least-32-characters", hardwareID: "windows-test", botID: "bot-1",
|
||||
sessionConfig: { dialog_id: "conversation-1", authorized_context: "authorized" },
|
||||
});
|
||||
assert.equal(FakeWebSocket.last.sent.filter((event) => event.type === "session.update").length, 1);
|
||||
client.close();
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
const assert = require("node:assert/strict");
|
||||
const test = require("node:test");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const vm = require("node:vm");
|
||||
const ts = require("typescript");
|
||||
|
||||
// Execute the actual renderer with a small DOM/IPC fixture; no network or audio.
|
||||
const source = fs.readFileSync(path.join(__dirname, "../src/hardware.ts"), "utf8");
|
||||
const compiled = ts.transpileModule(source, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS },
|
||||
}).outputText;
|
||||
|
||||
function fixture(overrides = {}) {
|
||||
const elements = new Map();
|
||||
const element = () => ({
|
||||
style: {}, value: "", textContent: "", disabled: false, children: [],
|
||||
append(...items) { this.children.push(...items); },
|
||||
prepend(...items) { this.children.unshift(...items); },
|
||||
replaceChildren(...items) { this.children = items; },
|
||||
querySelector() { return null; }, addEventListener() {},
|
||||
});
|
||||
const get = (id) => {
|
||||
if (!elements.has(id)) elements.set(id, element());
|
||||
return elements.get(id);
|
||||
};
|
||||
const events = [];
|
||||
const closes = [];
|
||||
const texts = [];
|
||||
const created = {
|
||||
conversation_id: "test-conversation", session: {},
|
||||
runtime: { transport: "low_load", config_digest: "digest", session_nonce: "nonce", provider_session_ref: "ref", bot_id: "test-bot" },
|
||||
session_config: { authorized_context: "test", prompt_bundle_version: "test-v1" },
|
||||
};
|
||||
const ipc = {
|
||||
getRuntimeDefaults: async () => ({}), getHardwareRuntimeDefaults: async () => ({}),
|
||||
onHardwareProviderEvent: () => () => {},
|
||||
createHardwareAgentSession: async () => created,
|
||||
closeHardwareAgent: async (input) => { closes.push(input); },
|
||||
hardwareAgentEvent: async (input) => { events.push(input); return { status: "FAILED" }; },
|
||||
sendHardwareAgentText: async (input) => { texts.push(input); },
|
||||
...overrides,
|
||||
};
|
||||
const context = vm.createContext({
|
||||
exports: {}, require: () => ({}), console, URL,
|
||||
document: { querySelector: () => get("app"), getElementById: get, createElement: element },
|
||||
window: { deviceSimulator: ipc, addEventListener() {}, clearInterval() {}, setTimeout, setInterval() {} },
|
||||
});
|
||||
vm.runInContext(compiled + `\nglobalThis.renderer = { state, controls, setPhase, setCheck, resetSessionState, failHardwareConversation, stopHardwareConversation, startHardwareConversation, handleProviderEvent, sendTextQuestion, buildReport };`, context);
|
||||
return { r: context.renderer, events, closes, texts, created, get, ipc };
|
||||
}
|
||||
|
||||
test("provider error plus connect rejection sends one failure, including late duplicates", async () => {
|
||||
const f = fixture();
|
||||
const r = f.r;
|
||||
f.ipc.connectHardwareAgent = async () => {
|
||||
void r.handleProviderEvent({ type: "error", error: { code: "internal_error", message: "StartVoiceChat failed" } });
|
||||
throw new Error("wrapped connect rejection");
|
||||
};
|
||||
r.setPhase("ready", "ready");
|
||||
await r.startHardwareConversation();
|
||||
await r.failHardwareConversation("SECOND_FAILURE");
|
||||
await r.stopHardwareConversation();
|
||||
await r.handleProviderEvent({ type: "error", error: { message: "late error" } });
|
||||
assert.equal(f.events.length, 1);
|
||||
assert.equal(f.events[0].eventType, "session_failed");
|
||||
assert.equal(f.events[0].errorCode, "VOLC_HARDWARE_AGENT_ERROR");
|
||||
assert.equal(f.closes.length, 1);
|
||||
assert.equal(r.state.failure, "internal_error: StartVoiceChat failed");
|
||||
assert.equal(r.state.phase, "error");
|
||||
assert.equal(r.controls.stop.disabled, true);
|
||||
assert.equal(r.controls.check.disabled, false);
|
||||
});
|
||||
|
||||
test("concurrent failure callers share pending cleanup; late events cannot reactivate", async () => {
|
||||
let release;
|
||||
const gate = new Promise((resolve) => { release = resolve; });
|
||||
const f = fixture({ closeHardwareAgent: () => gate });
|
||||
f.r.state.platformSession = f.created;
|
||||
f.r.state.failure = "first cause";
|
||||
f.r.setPhase("active", "active");
|
||||
const first = f.r.failHardwareConversation("FIRST");
|
||||
const second = f.r.failHardwareConversation("SECOND");
|
||||
assert.equal(first, second);
|
||||
assert.equal(f.r.state.phase, "stopping");
|
||||
await f.r.handleProviderEvent({ type: "response.done" });
|
||||
assert.equal(f.r.state.phase, "stopping");
|
||||
release();
|
||||
await first;
|
||||
assert.equal(f.events.length, 1);
|
||||
assert.equal(f.events[0].errorCode, "FIRST");
|
||||
await f.r.handleProviderEvent({ type: "transport.close", code: 1011 });
|
||||
assert.equal(f.r.state.checks.get("provider_closed").status, "failed");
|
||||
f.r.controls.textQuestion.value = "should not send";
|
||||
await f.r.sendTextQuestion();
|
||||
assert.equal(f.texts.length, 0);
|
||||
});
|
||||
|
||||
test("new call clears stale checks, duration and failure while retaining device preparation", async () => {
|
||||
const f = fixture();
|
||||
f.r.state.platformSession = f.created;
|
||||
f.r.state.failure = "old failure";
|
||||
await f.r.failHardwareConversation("OLD");
|
||||
for (const id of f.r.state.checks.keys()) f.r.setCheck(id, "passed", "old");
|
||||
f.get("duration").textContent = "48:15";
|
||||
f.r.resetSessionState();
|
||||
assert.equal(f.r.state.failure, undefined);
|
||||
assert.equal(f.r.state.failureTask, undefined);
|
||||
assert.equal(f.get("duration").textContent, "00:00");
|
||||
assert.equal(f.r.state.checks.get("platform_authenticated").status, "passed");
|
||||
assert.equal(f.r.state.checks.get("credential_resolved").status, "passed");
|
||||
assert.equal(f.r.state.checks.get("platform_completed").status, "pending");
|
||||
f.r.state.platformSession = { ...f.created, conversation_id: "next-conversation" };
|
||||
await f.r.failHardwareConversation("NEW");
|
||||
assert.equal(f.events.length, 2);
|
||||
assert.equal(f.events[1].conversationID, "next-conversation");
|
||||
assert.equal(f.events[1].sequenceNo, 1);
|
||||
});
|
||||
|
||||
test("failed platform receipt does not trigger a new terminal event on hangup", async () => {
|
||||
let attempts = 0;
|
||||
const f = fixture({ hardwareAgentEvent: async () => { attempts += 1; throw new Error("network unavailable"); } });
|
||||
f.r.state.platformSession = f.created;
|
||||
await f.r.failHardwareConversation("FIRST");
|
||||
await f.r.stopHardwareConversation();
|
||||
assert.equal(attempts, 1);
|
||||
assert.equal(f.r.state.phase, "error");
|
||||
assert.equal(f.r.buildReport().checks.find((c) => c.id === "platform_completed").status, "failed");
|
||||
});
|
||||
|
||||
test("connection rejection without a provider event still reports failure once", async () => {
|
||||
const f = fixture({ connectHardwareAgent: async () => { throw new Error("connect timeout"); } });
|
||||
f.r.setPhase("ready", "ready");
|
||||
await f.r.startHardwareConversation();
|
||||
assert.equal(f.events.length, 1);
|
||||
assert.equal(f.events[0].errorCode, "WINDOWS_HARDWARE_START_FAILED");
|
||||
assert.match(f.r.state.failure, /connect timeout/);
|
||||
assert.equal(f.r.state.phase, "error");
|
||||
});
|
||||
|
||||
test("normal hangup still reports completed and closes normally", async () => {
|
||||
const f = fixture();
|
||||
f.ipc.hardwareAgentEvent = async (input) => { f.events.push(input); return { status: "COMPLETED" }; };
|
||||
f.r.state.platformSession = f.created;
|
||||
f.r.setPhase("active", "active");
|
||||
await f.r.stopHardwareConversation();
|
||||
assert.equal(f.events.length, 1);
|
||||
assert.equal(f.events[0].eventType, "session_ended");
|
||||
assert.equal(f.closes[0].code, 1000);
|
||||
assert.equal(f.r.state.phase, "completed");
|
||||
assert.equal(f.r.state.failureTask, undefined);
|
||||
assert.equal(f.r.controls.start.disabled, false);
|
||||
});
|
||||
@@ -1,14 +1,24 @@
|
||||
const { app, BrowserWindow, ipcMain, session } = require("electron");
|
||||
const { app, BrowserWindow, dialog, ipcMain, safeStorage, session } = require("electron");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const tls = require("node:tls");
|
||||
const { HardwareAgentClient } = require("./hardware-agent-client.cjs");
|
||||
|
||||
app.commandLine.appendSwitch("autoplay-policy", "no-user-gesture-required");
|
||||
const automatedUI = process.env.SIMULATOR_AUTO_TEST === "1" || process.env.SIMULATOR_REAL_E2E === "1";
|
||||
if (automatedUI) {
|
||||
app.commandLine.appendSwitch("use-fake-device-for-media-stream");
|
||||
app.commandLine.appendSwitch("use-fake-ui-for-media-stream");
|
||||
}
|
||||
app.disableHardwareAcceleration();
|
||||
|
||||
let deviceConfig = null;
|
||||
let desiredConfigRevision = 1;
|
||||
let mainWindow = null;
|
||||
let hardwareConfig = null;
|
||||
let hardwareClient = null;
|
||||
let activeHardwarePlatformSession = null;
|
||||
|
||||
function createWindow() {
|
||||
const window = new BrowserWindow({
|
||||
@@ -26,6 +36,10 @@ function createWindow() {
|
||||
sandbox: true,
|
||||
},
|
||||
});
|
||||
mainWindow = window;
|
||||
window.once("closed", () => {
|
||||
if (mainWindow === window) mainWindow = null;
|
||||
});
|
||||
|
||||
const devArgument = process.argv.find((argument) => argument.startsWith("--dev-server-url="));
|
||||
const devServerURL = devArgument ? devArgument.slice("--dev-server-url=".length) : "";
|
||||
@@ -47,7 +61,7 @@ function createWindow() {
|
||||
} catch (error) {
|
||||
console.error("renderer diagnostics failed", error);
|
||||
}
|
||||
if (process.env.SIMULATOR_AUTO_TEST === "1") {
|
||||
if (automatedUI) {
|
||||
void runAutomatedSmoke(window);
|
||||
}
|
||||
});
|
||||
@@ -62,9 +76,15 @@ function createWindow() {
|
||||
if (screenshotPath) {
|
||||
window.webContents.once("did-finish-load", () => {
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
const image = await window.webContents.capturePage();
|
||||
fs.writeFileSync(screenshotPath, image.toPNG());
|
||||
} catch (error) {
|
||||
console.error("simulator screenshot failed", error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
app.quit();
|
||||
}
|
||||
}, 1200);
|
||||
});
|
||||
}
|
||||
@@ -95,24 +115,54 @@ async function runAutomatedSmoke(window) {
|
||||
async function collectAutomatedConversationResult(window) {
|
||||
const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
const deadline = Date.now() + 120_000;
|
||||
let realQuestionSent = false;
|
||||
let snapshot = {};
|
||||
while (Date.now() < deadline) {
|
||||
await delay(1000);
|
||||
const encoded = await window.webContents.executeJavaScript(`JSON.stringify({
|
||||
phase: document.querySelector('#phaseBadge')?.textContent || '',
|
||||
conversation_id: document.querySelector('#conversationID')?.textContent || '',
|
||||
model_mode: document.querySelector('#modelMode')?.value || '',
|
||||
subject: [...document.querySelectorAll('.transcript-turn--subject span')].map((item) => item.textContent || ''),
|
||||
agent: [...document.querySelectorAll('.transcript-turn--agent span')].map((item) => item.textContent || ''),
|
||||
turns: [...document.querySelectorAll('.transcript-turn')].map((item) => ({speaker: item.classList.contains('transcript-turn--subject') ? 'subject' : 'agent', text: item.querySelector('span')?.textContent || ''})),
|
||||
events: [...document.querySelectorAll('#eventLog li span')].map((item) => item.textContent || '')
|
||||
events: [...document.querySelectorAll('#eventLog li span')].map((item) => item.textContent || ''),
|
||||
checks: [...document.querySelectorAll('#flowChecks .flow-check')].map((item) => ({label: item.querySelector('span')?.textContent || '', detail: item.querySelector('small')?.textContent || '', status: item.classList.contains('flow-check--passed') ? 'passed' : item.classList.contains('flow-check--failed') ? 'failed' : 'pending'}))
|
||||
})`);
|
||||
snapshot = JSON.parse(encoded);
|
||||
console.log(`[auto-test] conversation ${encoded}`);
|
||||
if (process.env.SIMULATOR_REAL_E2E === "1" && snapshot.phase === "通话中" && !realQuestionSent) {
|
||||
const question = String(process.env.SIMULATOR_REAL_E2E_QUESTION || "请用一句简短的话介绍你自己。 ").trim();
|
||||
await window.webContents.executeJavaScript(`(() => { const input = document.querySelector('#textQuestion'); if (input) input.value = ${JSON.stringify(question)}; document.querySelector('#sendTextButton')?.click(); })()`);
|
||||
realQuestionSent = true;
|
||||
console.log("[real-e2e] text question sent");
|
||||
}
|
||||
const subjectIndex = snapshot.turns?.findIndex((turn) => turn.speaker === "subject") ?? -1;
|
||||
const hasAgentReply = subjectIndex >= 0 && snapshot.turns.slice(subjectIndex + 1).some((turn) => turn.speaker === "agent");
|
||||
if (hasAgentReply) {
|
||||
await window.webContents.executeJavaScript(`document.querySelector('#stopButton')?.click()`);
|
||||
const hardwarePage = await window.webContents.executeJavaScript(`Boolean(document.querySelector('#flowChecks'))`);
|
||||
const waitForPlatformStop = hardwarePage && process.env.SIMULATOR_AUTO_WAIT_FOR_PLATFORM_STOP === "1";
|
||||
if (!waitForPlatformStop) await window.webContents.executeJavaScript(`document.querySelector('#stopButton')?.click()`);
|
||||
if (hardwarePage) {
|
||||
const stopDeadline = Date.now() + 30_000;
|
||||
while (Date.now() < stopDeadline) {
|
||||
await delay(500);
|
||||
const phase = await window.webContents.executeJavaScript(`document.querySelector('#phaseBadge')?.textContent || ''`);
|
||||
if (phase === "已完成" || phase === "异常") break;
|
||||
}
|
||||
const finalEncoded = await window.webContents.executeJavaScript(`JSON.stringify({
|
||||
phase: document.querySelector('#phaseBadge')?.textContent || '',
|
||||
conversation_id: document.querySelector('#conversationID')?.textContent || '',
|
||||
model_mode: document.querySelector('#modelMode')?.value || '',
|
||||
turns: [...document.querySelectorAll('.transcript-turn')].map((item) => ({speaker: item.classList.contains('transcript-turn--subject') ? 'subject' : 'agent', text: item.querySelector('span')?.textContent || ''})),
|
||||
events: [...document.querySelectorAll('#eventLog li span')].map((item) => item.textContent || ''),
|
||||
checks: [...document.querySelectorAll('#flowChecks .flow-check')].map((item) => ({label: item.querySelector('span')?.textContent || '', detail: item.querySelector('small')?.textContent || '', status: item.classList.contains('flow-check--passed') ? 'passed' : item.classList.contains('flow-check--failed') ? 'failed' : 'pending'}))
|
||||
})`);
|
||||
snapshot = JSON.parse(finalEncoded);
|
||||
snapshot.result = snapshot.phase === "已完成" ? "PASS" : "ERROR";
|
||||
} else {
|
||||
snapshot.result = "PASS";
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (snapshot.phase === "异常") {
|
||||
@@ -160,7 +210,10 @@ function loadDefaultDeviceSecret() {
|
||||
const localAppData = String(process.env.LOCALAPPDATA || "").trim();
|
||||
const userProfile = String(process.env.USERPROFILE || "").trim();
|
||||
const portableDirectory = String(process.env.PORTABLE_EXECUTABLE_DIR || "").trim();
|
||||
if (localAppData) candidates.push(path.join(localAppData, "DataTerminalVoiceDevice", "server-device-secret"));
|
||||
if (localAppData) {
|
||||
candidates.push(path.join(localAppData, "DataTerminalHardwareAgent", "server-device-secret"));
|
||||
candidates.push(path.join(localAppData, "DataTerminalVoiceDevice", "server-device-secret"));
|
||||
}
|
||||
if (userProfile) candidates.push(path.join(userProfile, "AppData", "Local", "DataTerminalVoiceDevice", "server-device-secret"));
|
||||
try {
|
||||
const home = app.getPath("home");
|
||||
@@ -182,11 +235,134 @@ function loadDefaultDeviceSecret() {
|
||||
|
||||
ipcMain.handle("sim:get-runtime-defaults", () => ({
|
||||
apiURL: process.env.SIMULATOR_DEFAULT_API_URL || "https://121.41.46.238",
|
||||
deviceID: process.env.SIMULATOR_DEFAULT_DEVICE_ID || "10000000-0000-0000-0000-000000000511",
|
||||
deviceID: process.env.SIMULATOR_DEFAULT_DEVICE_ID || "10000000-0000-0000-0000-000000000512",
|
||||
deviceSecret: "",
|
||||
deviceSecretConfigured: loadDefaultDeviceSecret().length >= 32,
|
||||
}));
|
||||
|
||||
function hardwareCredentialPath() {
|
||||
return path.join(app.getPath("userData"), "volc-hardware-device-credential.json");
|
||||
}
|
||||
|
||||
function readSavedHardwareMetadata() {
|
||||
try {
|
||||
const stored = JSON.parse(fs.readFileSync(hardwareCredentialPath(), "utf8"));
|
||||
if (!stored.instanceID || !stored.productKey || !stored.deviceName || !stored.encryptedDeviceSecret) return undefined;
|
||||
return stored;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
function loadSavedHardwareCredential(identity) {
|
||||
if (!safeStorage.isEncryptionAvailable()) return undefined;
|
||||
try {
|
||||
const stored = JSON.parse(fs.readFileSync(hardwareCredentialPath(), "utf8"));
|
||||
if (stored.schemaVersion !== 3 || stored.instanceID !== identity.instanceID || stored.productKey !== identity.productKey || stored.deviceName !== identity.deviceName) return undefined;
|
||||
const secret = safeStorage.decryptString(Buffer.from(stored.encryptedDeviceSecret, "base64"));
|
||||
if (secret.length < 16) return undefined;
|
||||
return { deviceSecret: secret, hardwareID: String(stored.hardwareID || identity.hardwareID || "") };
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
function saveHardwareCredential(identity, deviceSecret, metadata = {}) {
|
||||
if (!safeStorage.isEncryptionAvailable()) return false;
|
||||
const target = hardwareCredentialPath();
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
const payload = {
|
||||
schemaVersion: 3,
|
||||
instanceID: identity.instanceID,
|
||||
productKey: identity.productKey,
|
||||
deviceName: identity.deviceName,
|
||||
hardwareID: identity.hardwareID,
|
||||
platformDeviceID: String(deviceConfig?.deviceID || ""),
|
||||
credentialRef: String(metadata.credentialRef || ""),
|
||||
credentialVersion: Number(metadata.credentialVersion || 1),
|
||||
encryptedDeviceSecret: safeStorage.encryptString(deviceSecret).toString("base64"),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
const temporary = `${target}.${process.pid}.tmp`;
|
||||
fs.writeFileSync(temporary, `${JSON.stringify(payload, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
fs.renameSync(temporary, target);
|
||||
return true;
|
||||
}
|
||||
|
||||
function hardwareIdentity(input) {
|
||||
const instanceID = String(input?.instanceID || "").trim();
|
||||
const productKey = String(input?.productKey || "").trim();
|
||||
const deviceName = String(input?.deviceName || "").trim();
|
||||
const hardwareID = String(input?.hardwareID || "").trim();
|
||||
if (!instanceID || instanceID.length > 200) throw new Error("实例 ID 不能为空或过长");
|
||||
if (!productKey || productKey.length > 200) throw new Error("Product Key 不能为空或过长");
|
||||
if (!/^[a-zA-Z0-9_\-.:@]{1,128}$/.test(deviceName)) throw new Error("火山设备名称格式不正确");
|
||||
if (!hardwareID || hardwareID.length > 200) throw new Error("Hardware ID 不能为空或过长");
|
||||
return { instanceID, productKey, deviceName, hardwareID };
|
||||
}
|
||||
|
||||
ipcMain.handle("sim:get-hardware-runtime-defaults", () => {
|
||||
const candidateMetadata = readSavedHardwareMetadata();
|
||||
const legacyMock = candidateMetadata && [candidateMetadata.instanceID, candidateMetadata.productKey, candidateMetadata.deviceName, candidateMetadata.hardwareID]
|
||||
.some((value) => String(value || "").toLowerCase().includes("mock") || String(value || "").toLowerCase().includes("e2e"));
|
||||
const savedMetadata = automatedUI || !legacyMock ? candidateMetadata : undefined;
|
||||
const instanceID = String(process.env.SIMULATOR_VOLC_INSTANCE_ID || savedMetadata?.instanceID || "").trim();
|
||||
const productKey = String(process.env.SIMULATOR_VOLC_PRODUCT_KEY || savedMetadata?.productKey || "").trim();
|
||||
const deviceName = String(process.env.SIMULATOR_VOLC_DEVICE_NAME || savedMetadata?.deviceName || "windows-hardware-agent-test").trim();
|
||||
const hardwareID = String(process.env.SIMULATOR_VOLC_HARDWARE_ID || savedMetadata?.hardwareID || `windows-${crypto.createHash("sha256").update(app.getPath("userData")).digest("hex").slice(0, 24)}`).trim();
|
||||
const modelMode = String(process.env.SIMULATOR_VOLC_MODEL_MODE || "llm").trim().toLowerCase() === "s2s" ? "s2s" : "llm";
|
||||
const saved = instanceID && productKey ? loadSavedHardwareCredential({ instanceID, productKey, deviceName, hardwareID }) : undefined;
|
||||
return { instanceID, productKey, deviceName, hardwareID, modelMode, deviceCredentialConfigured: Boolean(saved) };
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:configure-hardware", async (_event, input) => {
|
||||
if (!deviceConfig) throw new Error("请先连接平台");
|
||||
const requestedHardwareID = String(input?.hardwareID || "").trim();
|
||||
if (!requestedHardwareID || requestedHardwareID.length > 200) throw new Error("Hardware ID 不能为空或过长");
|
||||
const automatedTest = process.env.SIMULATOR_AUTO_TEST === "1";
|
||||
let provisioned;
|
||||
if (automatedTest) {
|
||||
provisioned = {
|
||||
instance_id: String(process.env.SIMULATOR_VOLC_INSTANCE_ID || "").trim(),
|
||||
product_key: String(process.env.SIMULATOR_VOLC_PRODUCT_KEY || "").trim(),
|
||||
device_name: String(process.env.SIMULATOR_VOLC_DEVICE_NAME || "").trim(),
|
||||
hardware_id: String(process.env.SIMULATOR_VOLC_HARDWARE_ID || requestedHardwareID).trim(),
|
||||
device_secret: String(process.env.SIMULATOR_VOLC_DEVICE_SECRET || "").trim(),
|
||||
credential_source: "isolated_test_override",
|
||||
credential_version: 1,
|
||||
credential_ref: "test-only",
|
||||
};
|
||||
} else {
|
||||
provisioned = await deviceRequest("POST", "/device/v3/hardware-agent-credentials:provision", {
|
||||
hardware_id: requestedHardwareID,
|
||||
}, `windows-hardware-provision-${nonce()}`);
|
||||
}
|
||||
const identity = hardwareIdentity({
|
||||
instanceID: provisioned?.instance_id,
|
||||
productKey: provisioned?.product_key,
|
||||
deviceName: provisioned?.device_name,
|
||||
hardwareID: provisioned?.hardware_id,
|
||||
});
|
||||
const deviceSecret = String(provisioned?.device_secret || "").trim();
|
||||
if (deviceSecret.length < 16 || deviceSecret.length > 4096) throw new Error("Device Secret 长度不正确");
|
||||
const persisted = automatedTest ? false : saveHardwareCredential(identity, deviceSecret, {
|
||||
credentialRef: provisioned?.credential_ref,
|
||||
credentialVersion: provisioned?.credential_version,
|
||||
});
|
||||
if (!automatedTest && !persisted) throw new Error("Windows 安全存储不可用，拒绝保存 Device Secret");
|
||||
hardwareConfig = { ...identity, deviceSecret };
|
||||
// 首次预配会由平台发布产品级默认 Bot/传输配置，并可能推进配置版本。
|
||||
// 设备必须重新拉取版本，后续创建会话时不能继续上报预配前的旧 revision。
|
||||
if (!automatedTest) {
|
||||
const configuration = await deviceRequest("GET", "/device/v2/config");
|
||||
desiredConfigRevision = Number(configuration?.revision || desiredConfigRevision || 1);
|
||||
}
|
||||
return {
|
||||
...identity,
|
||||
registered: String(provisioned?.credential_source || "").includes("dynamic_registration"),
|
||||
persisted,
|
||||
credentialSource: String(provisioned?.credential_source || "platform_provisioning"),
|
||||
credentialVersion: Number(provisioned?.credential_version || 1),
|
||||
configRevision: desiredConfigRevision,
|
||||
};
|
||||
});
|
||||
|
||||
function nonce() {
|
||||
return crypto.randomBytes(16).toString("hex");
|
||||
}
|
||||
@@ -235,7 +411,7 @@ ipcMain.handle("sim:configure", (_event, input) => configure(input));
|
||||
|
||||
ipcMain.handle("sim:probe", async () => {
|
||||
const query = new URLSearchParams({
|
||||
firmware_version: "0.3.0-windows-rtc",
|
||||
firmware_version: "0.3.2-windows-hardware-agent",
|
||||
board_rev: "windows-x64",
|
||||
rtc_sdk_version: "web-4.69.0-electron",
|
||||
codec: "opus",
|
||||
@@ -280,6 +456,14 @@ ipcMain.handle("sim:probe-rtc-network", async () => {
|
||||
return { endpoints };
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:probe-hardware-network", async () => {
|
||||
const endpoints = await Promise.all([
|
||||
probeTLSEndpoint("ai-gateway.vei.volces.com"),
|
||||
probeTLSEndpoint("iot-cn-shanghai.iot.volces.com"),
|
||||
]);
|
||||
return { endpoints };
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:get-automation-audio", async () => {
|
||||
const configuredPath = String(process.env.SIMULATOR_AUTOMATED_AUDIO_PATH || "").trim();
|
||||
if (!configuredPath) return undefined;
|
||||
@@ -309,6 +493,173 @@ ipcMain.handle("sim:create-conversation", async () => deviceRequest("POST", "/de
|
||||
requested_goal: "open_conversation",
|
||||
}, `windows-create-${nonce()}`));
|
||||
|
||||
ipcMain.handle("sim:create-hardware-agent-session", async () => deviceRequest("POST", "/device/v3/agent-sessions", {
|
||||
device_time: new Date().toISOString(),
|
||||
monotonic_ms: Math.max(0, Math.round(performance.now())),
|
||||
capabilities: {
|
||||
hardware_agent: true,
|
||||
sdk_version: "1.1.0-windows-standard-protocol",
|
||||
transports: ["low_load"],
|
||||
input_audio_format: "pcm_s16le_16000_mono",
|
||||
output_audio_format: "pcm_s16le_16000_mono",
|
||||
server_event_callback: false,
|
||||
transcript_upload: true,
|
||||
dynamic_session_config: true,
|
||||
credential_rotation: true,
|
||||
near_pcm_recording: false,
|
||||
far_pcm_recording: false,
|
||||
},
|
||||
network: { type: "windows-local" },
|
||||
requested_goal: "open_conversation",
|
||||
config_revision: desiredConfigRevision,
|
||||
}, `windows-hardware-create-${nonce()}`));
|
||||
|
||||
function emitHardwareEvent(event) {
|
||||
if (!mainWindow || mainWindow.isDestroyed()) return;
|
||||
mainWindow.webContents.send("sim:hardware-provider-event", event);
|
||||
}
|
||||
|
||||
function allowedHardwareGatewayURL() {
|
||||
const configured = String(process.env.SIMULATOR_HARDWARE_GATEWAY_URL || "wss://ai-gateway.vei.volces.com/v1/realtime").trim();
|
||||
const parsed = new URL(configured);
|
||||
const official = parsed.protocol === "wss:" && parsed.hostname === "ai-gateway.vei.volces.com";
|
||||
const local = (parsed.protocol === "ws:" || parsed.protocol === "wss:") && ["127.0.0.1", "localhost", "[::1]"].includes(parsed.hostname);
|
||||
if (!official && !local) throw new Error("硬件智能体网关只允许火山官方地址或本机测试地址");
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
async function invokeHardwareMemoryTool(event) {
|
||||
const client = hardwareClient;
|
||||
const platformSession = activeHardwarePlatformSession;
|
||||
if (!client || !platformSession) return;
|
||||
const callID = String(event.call_id || "");
|
||||
const functionName = String(client.pendingCalls.get(callID) || "").toLowerCase();
|
||||
if (!["memory_search", "search_memory", "retrieve_memory"].includes(functionName)) return;
|
||||
let argumentsValue = {};
|
||||
try { argumentsValue = JSON.parse(String(event.arguments || "{}")); } catch { /* invalid input is handled below */ }
|
||||
const query = String(argumentsValue.query || argumentsValue.keyword || argumentsValue.text || "").trim();
|
||||
const limit = Math.max(1, Math.min(5, Number(argumentsValue.limit || 3)));
|
||||
try {
|
||||
if (!query) throw new Error("记忆工具缺少 query 参数");
|
||||
const baseURL = new URL(deviceConfig.apiURL);
|
||||
const target = new URL(platformSession.session_config.tool_url, `${baseURL.origin}/`);
|
||||
if (target.origin !== baseURL.origin) throw new Error("记忆工具地址必须与平台同源");
|
||||
const response = await fetch(target, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
Authorization: `Bearer ${platformSession.session_config.tool_scope_token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ query, limit }),
|
||||
});
|
||||
const raw = await response.text();
|
||||
let body;
|
||||
try { body = raw ? JSON.parse(raw) : {}; } catch { body = { message: raw }; }
|
||||
if (!response.ok) throw new Error(body?.error?.message || body?.message || `HTTP ${response.status}`);
|
||||
const result = body?.data ?? body;
|
||||
client.completeFunctionCall(callID, result);
|
||||
emitHardwareEvent({ type: "platform.memory_search.completed", call_id: callID, query, result_count: result?.items?.length || 0, degraded: Boolean(result?.degraded) });
|
||||
} catch (error) {
|
||||
client.completeFunctionCall(callID, { items: [], degraded: true, error: error.message });
|
||||
emitHardwareEvent({ type: "platform.memory_search.failed", call_id: callID, message: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
ipcMain.handle("sim:connect-hardware-agent", async (_event, input) => {
|
||||
if (!deviceConfig) throw new Error("请先连接平台");
|
||||
if (!hardwareConfig) throw new Error("请先完成火山设备鉴权配置");
|
||||
if (hardwareClient) throw new Error("已有硬件智能体连接，请先结束");
|
||||
const platformSession = input?.platformSession;
|
||||
if (!platformSession?.runtime?.bot_id || !platformSession?.session_config?.dialog_id) throw new Error("平台硬件会话配置不完整");
|
||||
if (platformSession.runtime.transport !== "low_load") throw new Error("Windows 标准协议测试程序只接受设备绑定为 low_load 的会话");
|
||||
const modelMode = String(input?.modelMode || "llm").trim().toLowerCase();
|
||||
if (!new Set(["llm", "s2s"]).has(modelMode)) throw new Error("智能体模型类型必须为 llm 或 s2s");
|
||||
activeHardwarePlatformSession = platformSession;
|
||||
const client = new HardwareAgentClient({
|
||||
gatewayURL: allowedHardwareGatewayURL(),
|
||||
emit: (event) => {
|
||||
emitHardwareEvent(event);
|
||||
if (event.type === "response.function_call_arguments.done") void invokeHardwareMemoryTool(event);
|
||||
},
|
||||
});
|
||||
hardwareClient = client;
|
||||
try {
|
||||
return await client.connect({
|
||||
...hardwareConfig,
|
||||
botID: platformSession.runtime.bot_id,
|
||||
sessionConfig: platformSession.session_config,
|
||||
modelMode,
|
||||
});
|
||||
} catch (error) {
|
||||
hardwareClient = null;
|
||||
activeHardwarePlatformSession = null;
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.on("sim:hardware-agent-audio", (_event, bytes) => {
|
||||
try { hardwareClient?.sendAudio(Buffer.from(bytes)); }
|
||||
catch (error) { emitHardwareEvent({ type: "transport.audio_error", message: error.message }); }
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:hardware-agent-text", (_event, input) => {
|
||||
if (!hardwareClient) throw new Error("火山硬件智能体连接未就绪");
|
||||
hardwareClient.sendText(input?.text, Number(input?.interruptMode || 1));
|
||||
return { accepted: true };
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:close-hardware-agent", (_event, input) => {
|
||||
const client = hardwareClient;
|
||||
hardwareClient = null;
|
||||
activeHardwarePlatformSession = null;
|
||||
client?.close(Number(input?.code || 1000), String(input?.reason || "device session ended").slice(0, 100));
|
||||
return { closed: true };
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:save-hardware-report", async (_event, report) => {
|
||||
const result = await dialog.showSaveDialog(mainWindow, {
|
||||
title: "导出硬件对话智能体测试报告",
|
||||
defaultPath: `hardware-agent-test-${new Date().toISOString().replace(/[:.]/g, "-")}.json`,
|
||||
filters: [{ name: "JSON 测试报告", extensions: ["json"] }],
|
||||
});
|
||||
if (result.canceled || !result.filePath) return { saved: false };
|
||||
const redact = (value) => {
|
||||
if (Array.isArray(value)) return value.map(redact);
|
||||
if (!value || typeof value !== "object") return value;
|
||||
const output = {};
|
||||
for (const [key, item] of Object.entries(value)) {
|
||||
if (/secret|signature|authorization|tool_scope_token|systemmessages|system_role|system_prompt|authorized_context|speaking_style|instructions/i.test(key)) output[key] = "[REDACTED]";
|
||||
else output[key] = redact(item);
|
||||
}
|
||||
return output;
|
||||
};
|
||||
const safeReport = redact(JSON.parse(JSON.stringify(report)));
|
||||
fs.writeFileSync(result.filePath, `${JSON.stringify(safeReport, null, 2)}\n`, "utf8");
|
||||
return { saved: true, path: result.filePath };
|
||||
});
|
||||
|
||||
ipcMain.handle("sim:hardware-agent-event", async (_event, input) => deviceRequest(
|
||||
"POST",
|
||||
`/device/v3/agent-sessions/${encodeURIComponent(input.conversationID)}/events`,
|
||||
{
|
||||
event_id: input.eventID || `windows-hardware-event-${nonce()}`,
|
||||
sequence_no: Number(input.sequenceNo),
|
||||
event_type: input.eventType,
|
||||
occurred_at: new Date().toISOString(),
|
||||
provider_session_ref: input.providerSessionRef,
|
||||
session_nonce: input.sessionNonce,
|
||||
...(input.eventType === "session_started" ? {
|
||||
provider_connection_ref: input.providerConnectionRef,
|
||||
bot_id: input.botID,
|
||||
transport: input.transport,
|
||||
config_digest: input.configDigest,
|
||||
dynamic_config_applied: input.dynamicConfigApplied,
|
||||
} : {}),
|
||||
...(input.errorCode ? { error_code: input.errorCode } : {}),
|
||||
},
|
||||
));
|
||||
|
||||
ipcMain.handle("sim:activate-conversation", async (_event, input) => deviceRequest(
|
||||
"POST",
|
||||
`/device/v2/conversations/${encodeURIComponent(input.conversationID)}/activate`,
|
||||
@@ -338,7 +689,7 @@ ipcMain.handle("sim:heartbeat", async (_event, input) => deviceRequest(
|
||||
conversation_status: "ACTIVE",
|
||||
rtc: { joined: true, uplink: true, downlink: true },
|
||||
network: { type: "windows-local" },
|
||||
firmware_version: "0.3.0-windows-rtc",
|
||||
firmware_version: "0.3.2-windows-hardware-agent",
|
||||
reported_config_revision: desiredConfigRevision,
|
||||
rtc_credential_expires_at: input.credentialExpiresAt,
|
||||
recording_queue: { pending_segments: 0, failed_segments: 0 },
|
||||
@@ -356,7 +707,7 @@ ipcMain.handle("sim:submit-transcript-batch", async (_event, input) => {
|
||||
const digest = crypto.createHash("sha256").update(JSON.stringify(payload)).digest("hex").slice(0, 32);
|
||||
return deviceRequest(
|
||||
"POST",
|
||||
`/device/v2/conversations/${encodeURIComponent(conversationID)}/turns:batch`,
|
||||
`/${input?.hardwareAgent ? "device/v3/agent-sessions" : "device/v2/conversations"}/${encodeURIComponent(conversationID)}/turns:batch`,
|
||||
payload,
|
||||
`windows-turns-${digest}`,
|
||||
);
|
||||
@@ -433,6 +784,9 @@ app.whenReady().then(() => {
|
||||
});
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
hardwareClient?.close(1001, "application closed");
|
||||
hardwareClient = null;
|
||||
activeHardwarePlatformSession = null;
|
||||
if (process.platform !== "darwin") app.quit();
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
const assert = require("node:assert/strict");
|
||||
const test = require("node:test");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const vm = require("node:vm");
|
||||
|
||||
const source = fs.readFileSync(path.join(__dirname, "../public/pcm-playback.worklet.js"), "utf8");
|
||||
|
||||
function fixture({ outputRate = 48000, prebufferMs = 120 } = {}) {
|
||||
const messages = [];
|
||||
let Processor;
|
||||
class AudioWorkletProcessor {
|
||||
constructor() {
|
||||
this.port = { onmessage: null, postMessage: (message) => messages.push(message) };
|
||||
}
|
||||
}
|
||||
const context = vm.createContext({
|
||||
AudioWorkletProcessor,
|
||||
Float32Array,
|
||||
ArrayBuffer,
|
||||
Math,
|
||||
sampleRate: outputRate,
|
||||
registerProcessor: (name, implementation) => {
|
||||
assert.equal(name, "pcm16-playback");
|
||||
Processor = implementation;
|
||||
},
|
||||
});
|
||||
vm.runInContext(source, context);
|
||||
const processor = new Processor({ processorOptions: { sourceSampleRate: 16000, prebufferMs } });
|
||||
const push = (values) => {
|
||||
const samples = new Float32Array(values);
|
||||
processor.port.onmessage({ data: { type: "push", samples: samples.buffer } });
|
||||
};
|
||||
const process = () => {
|
||||
const output = new Float32Array(128);
|
||||
assert.equal(processor.process([], [[output]]), true);
|
||||
return output;
|
||||
};
|
||||
return { messages, processor, push, process };
|
||||
}
|
||||
|
||||
test("waits for prebuffer then resamples continuous 16k PCM without fragment gaps", () => {
|
||||
const f = fixture({ outputRate: 48000, prebufferMs: 10 });
|
||||
f.push(new Array(120).fill(0.25));
|
||||
assert.ok(f.process().every((sample) => sample === 0), "must remain silent below the prebuffer threshold");
|
||||
f.push(new Array(400).fill(0.25));
|
||||
const first = f.process();
|
||||
const second = f.process();
|
||||
assert.ok(first.some((sample) => sample > 0.05), "buffered playback should start with a short anti-pop fade");
|
||||
assert.ok(second.every((sample) => sample > 0.12), "playback should remain continuous across source fragments");
|
||||
assert.equal(f.messages.filter((message) => message.type === "underflow").length, 0);
|
||||
});
|
||||
|
||||
test("plays 16k source at real-time duration instead of the former 1.5x speed", () => {
|
||||
const f = fixture({ outputRate: 48000, prebufferMs: 1 });
|
||||
f.push(new Array(1000).fill(0.25));
|
||||
f.process();
|
||||
assert.ok(Math.abs(f.processor.readPosition - 128 / 3) < 0.001, `consumed ${f.processor.readPosition} source samples`);
|
||||
});
|
||||
|
||||
test("reports network starvation and re-enters prebuffering instead of scheduling tiny buffers", () => {
|
||||
const f = fixture({ outputRate: 48000, prebufferMs: 0.01 });
|
||||
f.push(new Array(8).fill(0.5));
|
||||
f.process();
|
||||
assert.equal(f.messages.filter((message) => message.type === "underflow").length, 1);
|
||||
assert.ok(f.process().every((sample) => sample === 0), "starved playback should wait for a new prebuffer");
|
||||
});
|
||||
|
||||
test("drains the final response cleanly and emits a completion signal", () => {
|
||||
const f = fixture({ outputRate: 24000 });
|
||||
f.push(new Array(64).fill(-0.25));
|
||||
f.processor.port.onmessage({ data: { type: "end" } });
|
||||
f.process();
|
||||
assert.equal(f.messages.filter((message) => message.type === "drained").length, 1);
|
||||
assert.equal(f.messages.filter((message) => message.type === "underflow").length, 0);
|
||||
});
|
||||
@@ -5,8 +5,23 @@ contextBridge.exposeInMainWorld("deviceSimulator", {
|
||||
configure: (input) => ipcRenderer.invoke("sim:configure", input),
|
||||
probe: () => ipcRenderer.invoke("sim:probe"),
|
||||
probeRTCNetwork: () => ipcRenderer.invoke("sim:probe-rtc-network"),
|
||||
probeHardwareNetwork: () => ipcRenderer.invoke("sim:probe-hardware-network"),
|
||||
getHardwareRuntimeDefaults: () => ipcRenderer.invoke("sim:get-hardware-runtime-defaults"),
|
||||
configureHardware: (input) => ipcRenderer.invoke("sim:configure-hardware", input),
|
||||
getAutomationAudio: () => ipcRenderer.invoke("sim:get-automation-audio"),
|
||||
createConversation: () => ipcRenderer.invoke("sim:create-conversation"),
|
||||
createHardwareAgentSession: () => ipcRenderer.invoke("sim:create-hardware-agent-session"),
|
||||
connectHardwareAgent: (input) => ipcRenderer.invoke("sim:connect-hardware-agent", input),
|
||||
sendHardwareAgentAudio: (bytes) => ipcRenderer.send("sim:hardware-agent-audio", bytes),
|
||||
sendHardwareAgentText: (input) => ipcRenderer.invoke("sim:hardware-agent-text", input),
|
||||
closeHardwareAgent: (input) => ipcRenderer.invoke("sim:close-hardware-agent", input),
|
||||
onHardwareProviderEvent: (callback) => {
|
||||
const listener = (_event, value) => callback(value);
|
||||
ipcRenderer.on("sim:hardware-provider-event", listener);
|
||||
return () => ipcRenderer.removeListener("sim:hardware-provider-event", listener);
|
||||
},
|
||||
saveHardwareReport: (report) => ipcRenderer.invoke("sim:save-hardware-report", report),
|
||||
hardwareAgentEvent: (input) => ipcRenderer.invoke("sim:hardware-agent-event", input),
|
||||
activateConversation: (input) => ipcRenderer.invoke("sim:activate-conversation", input),
|
||||
getConversation: (conversationID) => ipcRenderer.invoke("sim:get-conversation", conversationID),
|
||||
heartbeat: (input) => ipcRenderer.invoke("sim:heartbeat", input),
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' data:; style-src 'self'; connect-src 'self' data: https://121.41.46.238 https://*.rtc.volcvideo.com https://*.volcvideos.com wss: http://127.0.0.1:* http://localhost:*; media-src 'self' blob:; worker-src 'self' blob: data:; img-src 'self' data:;" />
|
||||
<title>语音终端模拟器</title>
|
||||
<title>硬件对话智能体测试程序</title>
|
||||
</head>
|
||||
<body>
|
||||
<main id="app"></main>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
<script type="module" src="/src/hardware.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
@echo off
|
||||
setlocal
|
||||
set "SECRET_FILE=%~dp0.server-device-secret"
|
||||
set "DEVICE_EXE=%~dp0voice-device-simulator-0.1.0-windows-x64.exe"
|
||||
set "SECRET_FILE=%LOCALAPPDATA%\DataTerminalHardwareAgent\server-device-secret"
|
||||
set "DEVICE_EXE=%~dp0hardware-agent-tester-0.3.5-windows-x64.exe"
|
||||
if not exist "%DEVICE_EXE%" set "DEVICE_EXE=%~dp0release\hardware-agent-tester-0.3.5-windows-x64.exe"
|
||||
|
||||
if not exist "%SECRET_FILE%" (
|
||||
echo Cloud device credential is missing: %SECRET_FILE%
|
||||
@@ -9,12 +10,12 @@ if not exist "%SECRET_FILE%" (
|
||||
exit /b 1
|
||||
)
|
||||
if not exist "%DEVICE_EXE%" (
|
||||
echo Voice device simulator is missing: %DEVICE_EXE%
|
||||
echo Hardware agent tester is missing: %DEVICE_EXE%
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set /p SIMULATOR_DEFAULT_DEVICE_SECRET=<"%SECRET_FILE%"
|
||||
set "SIMULATOR_DEFAULT_API_URL=https://121.41.46.238"
|
||||
set "SIMULATOR_DEFAULT_DEVICE_ID=10000000-0000-0000-0000-000000000511"
|
||||
set "SIMULATOR_DEFAULT_DEVICE_ID=10000000-0000-0000-0000-000000000512"
|
||||
start "" "%DEVICE_EXE%"
|
||||
|
||||
@@ -1,25 +1,46 @@
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
param(
|
||||
[switch]$RealE2E,
|
||||
[string]$ResultPath = (Join-Path $env:TEMP "hardware-agent-real-e2e.json")
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$secretPath = Join-Path $env:LOCALAPPDATA "DataTerminalVoiceDevice\server-device-secret"
|
||||
$executablePath = Join-Path $PSScriptRoot "voice-device-simulator-0.1.0-windows-x64.exe"
|
||||
$secretPath = Join-Path $env:LOCALAPPDATA "DataTerminalHardwareAgent\server-device-secret"
|
||||
$executablePath = Join-Path $PSScriptRoot "hardware-agent-tester-0.3.5-windows-x64.exe"
|
||||
if (-not (Test-Path -LiteralPath $executablePath -PathType Leaf)) {
|
||||
$executablePath = Join-Path $PSScriptRoot "release\hardware-agent-tester-0.3.5-windows-x64.exe"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $secretPath -PathType Leaf)) {
|
||||
throw "Cloud device credential is missing: $secretPath"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $executablePath -PathType Leaf)) {
|
||||
throw "Voice device simulator is missing: $executablePath"
|
||||
throw "Hardware agent tester is missing: $executablePath"
|
||||
}
|
||||
|
||||
$env:SIMULATOR_DEFAULT_API_URL = "https://121.41.46.238"
|
||||
$env:SIMULATOR_DEFAULT_DEVICE_ID = "10000000-0000-0000-0000-000000000511"
|
||||
$env:SIMULATOR_DEFAULT_DEVICE_ID = "10000000-0000-0000-0000-000000000512"
|
||||
$env:SIMULATOR_DEFAULT_DEVICE_SECRET = [IO.File]::ReadAllText($secretPath).Trim()
|
||||
try {
|
||||
if ($RealE2E) {
|
||||
$env:SIMULATOR_REAL_E2E = "1"
|
||||
$env:SIMULATOR_AUTO_RESULT_PATH = $ResultPath
|
||||
$env:SIMULATOR_VOLC_MODEL_MODE = "llm"
|
||||
$env:SIMULATOR_REAL_E2E_QUESTION = "Please introduce yourself in one short sentence."
|
||||
$process = Start-Process -FilePath $executablePath -WindowStyle Hidden -PassThru -Wait
|
||||
if ($process.ExitCode -ne 0) { throw "Hardware agent tester exited with code $($process.ExitCode)" }
|
||||
if (-not (Test-Path -LiteralPath $ResultPath -PathType Leaf)) { throw "Real E2E report was not created: $ResultPath" }
|
||||
}
|
||||
else {
|
||||
Start-Process -FilePath $executablePath
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item Env:SIMULATOR_DEFAULT_API_URL -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:SIMULATOR_DEFAULT_DEVICE_ID -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:SIMULATOR_DEFAULT_DEVICE_SECRET -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:SIMULATOR_REAL_E2E -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:SIMULATOR_AUTO_RESULT_PATH -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:SIMULATOR_VOLC_MODEL_MODE -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:SIMULATOR_REAL_E2E_QUESTION -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' data:; style-src 'self'; connect-src 'self' data: https://121.41.46.238 https://*.rtc.volcvideo.com https://*.volcvideos.com wss: http://127.0.0.1:* http://localhost:*; media-src 'self' blob:; worker-src 'self' blob: data:; img-src 'self' data:;" />
|
||||
<title>传统 RTC 语音终端模拟器</title>
|
||||
</head>
|
||||
<body>
|
||||
<main id="app"></main>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
+25
-3
@@ -1,14 +1,15 @@
|
||||
{
|
||||
"name": "data-terminal-voice-device-simulator",
|
||||
"version": "0.1.0",
|
||||
"version": "0.3.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "data-terminal-voice-device-simulator",
|
||||
"version": "0.1.0",
|
||||
"version": "0.3.5",
|
||||
"dependencies": {
|
||||
"@volcengine/rtc": "4.69.0"
|
||||
"@volcengine/rtc": "4.69.0",
|
||||
"ws": "8.21.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.2.1",
|
||||
@@ -4857,6 +4858,27 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.21.3",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
|
||||
"integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/xmlbuilder": {
|
||||
"version": "15.1.1",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz",
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
{
|
||||
"name": "data-terminal-voice-device-simulator",
|
||||
"version": "0.1.0",
|
||||
"version": "0.3.5",
|
||||
"private": true,
|
||||
"description": "Windows real-audio device simulator for the data terminal cloud platform",
|
||||
"description": "Windows end-to-end tester for Volcengine hardware conversational agents",
|
||||
"main": "electron/main.cjs",
|
||||
"scripts": {
|
||||
"dev": "concurrently -k \"vite --host 127.0.0.1 --port 5177\" \"wait-on http://127.0.0.1:5177 && electron . --dev-server-url=http://127.0.0.1:5177\"",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "node --test electron/*.test.cjs",
|
||||
"build:renderer": "vite build",
|
||||
"start": "npm run build:renderer && electron .",
|
||||
"build": "npm run typecheck && npm run build:renderer && electron-builder --win portable"
|
||||
"build": "npm test && npm run typecheck && npm run build:renderer && electron-builder --win portable"
|
||||
},
|
||||
"dependencies": {
|
||||
"@volcengine/rtc": "4.69.0"
|
||||
"@volcengine/rtc": "4.69.0",
|
||||
"ws": "8.21.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.2.1",
|
||||
@@ -23,22 +25,25 @@
|
||||
"wait-on": "^8.0.4"
|
||||
},
|
||||
"build": {
|
||||
"appId": "cn.dataterminal.voice-device-simulator",
|
||||
"productName": "语音终端模拟器",
|
||||
"artifactName": "voice-device-simulator-${version}-windows-${arch}.${ext}",
|
||||
"appId": "cn.dataterminal.hardware-agent-tester",
|
||||
"productName": "硬件对话智能体测试程序",
|
||||
"artifactName": "hardware-agent-tester-${version}-windows-${arch}.${ext}",
|
||||
"directories": {
|
||||
"output": "release"
|
||||
},
|
||||
"files": [
|
||||
"dist/**/*",
|
||||
"electron/**/*",
|
||||
"!electron/*.test.cjs",
|
||||
"package.json"
|
||||
],
|
||||
"win": {
|
||||
"target": [
|
||||
{
|
||||
"target": "portable",
|
||||
"arch": ["x64"]
|
||||
"arch": [
|
||||
"x64"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
class PCM16CaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor(options) {
|
||||
super();
|
||||
this.targetRate = Number(options.processorOptions?.targetSampleRate || 16000);
|
||||
this.frameSamples = Number(options.processorOptions?.frameSamples || 320);
|
||||
this.ratio = sampleRate / this.targetRate;
|
||||
this.source = [];
|
||||
this.position = 0;
|
||||
this.output = [];
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const channel = inputs[0]?.[0];
|
||||
if (!channel?.length) return true;
|
||||
for (const sample of channel) this.source.push(sample);
|
||||
while (this.position + this.ratio <= this.source.length) {
|
||||
const start = Math.floor(this.position);
|
||||
const end = Math.max(start + 1, Math.floor(this.position + this.ratio));
|
||||
let sum = 0;
|
||||
for (let index = start; index < end && index < this.source.length; index += 1) sum += this.source[index];
|
||||
this.output.push(sum / Math.max(1, end - start));
|
||||
this.position += this.ratio;
|
||||
if (this.output.length >= this.frameSamples) this.flush();
|
||||
}
|
||||
const consumed = Math.floor(this.position);
|
||||
if (consumed > 0) {
|
||||
this.source.splice(0, consumed);
|
||||
this.position -= consumed;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
flush() {
|
||||
const frame = this.output.splice(0, this.frameSamples);
|
||||
const pcm = new Int16Array(frame.length);
|
||||
let energy = 0;
|
||||
for (let index = 0; index < frame.length; index += 1) {
|
||||
const sample = Math.max(-1, Math.min(1, frame[index]));
|
||||
energy += sample * sample;
|
||||
pcm[index] = sample < 0 ? Math.round(sample * 32768) : Math.round(sample * 32767);
|
||||
}
|
||||
const volume = Math.min(100, Math.round(Math.sqrt(energy / Math.max(1, frame.length)) * 280));
|
||||
this.port.postMessage({ type: "pcm", bytes: pcm.buffer, volume }, [pcm.buffer]);
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("pcm16-capture", PCM16CaptureProcessor);
|
||||
@@ -0,0 +1,115 @@
|
||||
class PCM16PlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor(options) {
|
||||
super();
|
||||
const config = options.processorOptions || {};
|
||||
this.sourceSampleRate = Number(config.sourceSampleRate || 16000);
|
||||
this.prebufferSamples = Math.max(1, Math.round(this.sourceSampleRate * Number(config.prebufferMs || 120) / 1000));
|
||||
this.capacity = Math.max(this.sourceSampleRate * 30, this.prebufferSamples * 4);
|
||||
this.ring = new Float32Array(this.capacity);
|
||||
this.readPosition = 0;
|
||||
this.writePosition = 0;
|
||||
this.playing = false;
|
||||
this.ended = false;
|
||||
this.drained = false;
|
||||
this.lastOutput = 0;
|
||||
this.fadeInRemaining = 0;
|
||||
this.fadeFrames = Math.max(1, Math.round(sampleRate * 0.005));
|
||||
this.underflows = 0;
|
||||
this.port.onmessage = (message) => this.onMessage(message.data || {});
|
||||
}
|
||||
|
||||
onMessage(message) {
|
||||
if (message.type === "reset") {
|
||||
this.reset();
|
||||
return;
|
||||
}
|
||||
if (message.type === "end") {
|
||||
this.ended = true;
|
||||
return;
|
||||
}
|
||||
if (message.type !== "push" || !(message.samples instanceof ArrayBuffer)) return;
|
||||
const samples = new Float32Array(message.samples);
|
||||
if (this.drained) this.reset();
|
||||
this.ended = false;
|
||||
for (let index = 0; index < samples.length; index += 1) {
|
||||
if (this.writePosition - Math.floor(this.readPosition) >= this.capacity - 1) {
|
||||
this.readPosition = this.writePosition - this.capacity + 2;
|
||||
}
|
||||
this.ring[this.writePosition % this.capacity] = samples[index];
|
||||
this.writePosition += 1;
|
||||
}
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.readPosition = 0;
|
||||
this.writePosition = 0;
|
||||
this.playing = false;
|
||||
this.ended = false;
|
||||
this.drained = false;
|
||||
this.lastOutput = 0;
|
||||
this.fadeInRemaining = 0;
|
||||
}
|
||||
|
||||
available() {
|
||||
return this.writePosition - this.readPosition;
|
||||
}
|
||||
|
||||
startWhenReady() {
|
||||
if (this.playing) return true;
|
||||
const available = this.available();
|
||||
if (available >= this.prebufferSamples || (this.ended && available > 1)) {
|
||||
this.playing = true;
|
||||
this.fadeInRemaining = this.fadeFrames;
|
||||
this.port.postMessage({ type: "started", bufferedSamples: Math.floor(available) });
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
finish(output, fromIndex, eventType) {
|
||||
const remaining = Math.min(this.fadeFrames, output.length - fromIndex);
|
||||
for (let index = 0; index < remaining; index += 1) {
|
||||
output[fromIndex + index] = this.lastOutput * (1 - (index + 1) / remaining);
|
||||
}
|
||||
output.fill(0, fromIndex + remaining);
|
||||
this.lastOutput = 0;
|
||||
this.playing = false;
|
||||
if (eventType === "drained") {
|
||||
if (!this.drained) this.port.postMessage({ type: "drained" });
|
||||
this.drained = true;
|
||||
} else {
|
||||
this.underflows += 1;
|
||||
this.port.postMessage({ type: "underflow", count: this.underflows });
|
||||
}
|
||||
}
|
||||
|
||||
process(_inputs, outputs) {
|
||||
const output = outputs[0]?.[0];
|
||||
if (!output) return true;
|
||||
output.fill(0);
|
||||
if (!this.startWhenReady()) return true;
|
||||
|
||||
const sourceStep = this.sourceSampleRate / sampleRate;
|
||||
for (let index = 0; index < output.length; index += 1) {
|
||||
const base = Math.floor(this.readPosition);
|
||||
if (base + 1 >= this.writePosition) {
|
||||
this.finish(output, index, this.ended ? "drained" : "underflow");
|
||||
break;
|
||||
}
|
||||
const fraction = this.readPosition - base;
|
||||
const first = this.ring[base % this.capacity];
|
||||
const second = this.ring[(base + 1) % this.capacity];
|
||||
let value = first + (second - first) * fraction;
|
||||
if (this.fadeInRemaining > 0) {
|
||||
value *= 1 - this.fadeInRemaining / this.fadeFrames;
|
||||
this.fadeInRemaining -= 1;
|
||||
}
|
||||
output[index] = value;
|
||||
this.lastOutput = value;
|
||||
this.readPosition += sourceStep;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("pcm16-playback", PCM16PlaybackProcessor);
|
||||
+72
-1
@@ -34,6 +34,32 @@ type CreatedConversation = {
|
||||
};
|
||||
};
|
||||
|
||||
type CreatedHardwareAgentSession = {
|
||||
conversation_id: string;
|
||||
status: "SESSION_READY";
|
||||
runtime_type: "volc_hardware_agent";
|
||||
runtime: {
|
||||
provider: "volcengine";
|
||||
bot_id: string;
|
||||
transport: "high_quality" | "low_load";
|
||||
provider_session_ref: string;
|
||||
session_nonce: string;
|
||||
config_digest: string;
|
||||
wait_for_session_update: boolean;
|
||||
};
|
||||
session_config: {
|
||||
dialog_id: string;
|
||||
system_prompt?: string;
|
||||
persona_tone?: string;
|
||||
authorized_context: string;
|
||||
prompt_bundle_version: string;
|
||||
tool_url?: string;
|
||||
tool_scope_token?: string;
|
||||
};
|
||||
privacy: CreatedConversation["privacy"];
|
||||
session: CreatedConversation["session"];
|
||||
};
|
||||
|
||||
type DeviceFinalTurn = {
|
||||
turn_key: string;
|
||||
sequence_no: number;
|
||||
@@ -50,15 +76,60 @@ interface Window {
|
||||
configure(input: SimulatorConfiguration): Promise<{ apiURL: string; deviceID: string }>;
|
||||
probe(): Promise<Record<string, unknown>>;
|
||||
probeRTCNetwork(): Promise<{ endpoints: Array<{ hostname: string; ok: boolean; error?: string }> }>;
|
||||
probeHardwareNetwork(): Promise<{ endpoints: Array<{ hostname: string; ok: boolean; error?: string }> }>;
|
||||
getHardwareRuntimeDefaults(): Promise<{
|
||||
instanceID: string;
|
||||
productKey: string;
|
||||
deviceName: string;
|
||||
hardwareID: string;
|
||||
modelMode: "llm" | "s2s";
|
||||
deviceCredentialConfigured: boolean;
|
||||
}>;
|
||||
configureHardware(input: {
|
||||
hardwareID: string;
|
||||
}): Promise<{
|
||||
instanceID: string;
|
||||
productKey: string;
|
||||
deviceName: string;
|
||||
hardwareID: string;
|
||||
registered: boolean;
|
||||
persisted: boolean;
|
||||
credentialSource: string;
|
||||
credentialVersion: number;
|
||||
}>;
|
||||
getAutomationAudio(): Promise<{ data_url: string; source_name: string } | undefined>;
|
||||
createConversation(): Promise<CreatedConversation>;
|
||||
createHardwareAgentSession(): Promise<CreatedHardwareAgentSession>;
|
||||
connectHardwareAgent(input: { platformSession: CreatedHardwareAgentSession; modelMode: "llm" | "s2s" }): Promise<{
|
||||
providerConnectionRef: string;
|
||||
session: Record<string, unknown>;
|
||||
}>;
|
||||
sendHardwareAgentAudio(bytes: ArrayBuffer): void;
|
||||
sendHardwareAgentText(input: { text: string; interruptMode?: number }): Promise<{ accepted: boolean }>;
|
||||
closeHardwareAgent(input?: { code?: number; reason?: string }): Promise<{ closed: boolean }>;
|
||||
onHardwareProviderEvent(callback: (event: Record<string, unknown>) => void): () => void;
|
||||
saveHardwareReport(report: Record<string, unknown>): Promise<{ saved: boolean; path?: string }>;
|
||||
hardwareAgentEvent(input: {
|
||||
conversationID: string;
|
||||
eventID?: string;
|
||||
sequenceNo: number;
|
||||
eventType: "session_started" | "session_updated" | "session_ended" | "session_failed";
|
||||
providerSessionRef: string;
|
||||
sessionNonce: string;
|
||||
providerConnectionRef?: string;
|
||||
botID?: string;
|
||||
transport?: "high_quality" | "low_load";
|
||||
configDigest?: string;
|
||||
dynamicConfigApplied?: boolean;
|
||||
errorCode?: string;
|
||||
}): Promise<{ conversation_id: string; status: string; action: "continue" | "stop_session"; duplicate: boolean }>;
|
||||
activateConversation(input: { conversationID: string; roomID: string; userID: string }): Promise<Record<string, unknown>>;
|
||||
getConversation(conversationID: string): Promise<{ status: string; action: string; failure_code?: string }>;
|
||||
heartbeat(input: { conversationID: string; sequenceNo: number; credentialExpiresAt: string }): Promise<{
|
||||
actions?: Array<{ type: string; rtc?: RTCGrant }>;
|
||||
desired_config_revision?: number;
|
||||
}>;
|
||||
submitTranscriptBatch(input: { conversationID: string; turns: DeviceFinalTurn[] }): Promise<{
|
||||
submitTranscriptBatch(input: { conversationID: string; turns: DeviceFinalTurn[]; hardwareAgent?: boolean }): Promise<{
|
||||
conversation_id: string;
|
||||
accepted_count: number;
|
||||
duplicate_count: number;
|
||||
|
||||
@@ -0,0 +1,778 @@
|
||||
import "./style.css";
|
||||
|
||||
type HardwarePhase = "idle" | "checking" | "ready" | "connecting" | "active" | "stopping" | "completed" | "error";
|
||||
type ReportCheck = { id: string; label: string; status: "pending" | "passed" | "failed"; detail?: string; at?: string };
|
||||
type ReportEvent = { at: string; source: "app" | "platform" | "volcengine"; type: string; detail?: Record<string, unknown> };
|
||||
|
||||
const checkDefinitions: Array<[string, string]> = [
|
||||
["platform_authenticated", "平台设备 HMAC 鉴权"],
|
||||
["credential_resolved", "火山一机一密凭据"],
|
||||
["context_frozen", "人设、背景与授权上下文冻结"],
|
||||
["provider_connected", "火山硬件智能体 WebSocket 建连"],
|
||||
["dynamic_config_applied", "session.updated 动态配置确认"],
|
||||
["platform_active", "平台确认会话 ACTIVE"],
|
||||
["microphone_streaming", "16kHz PCM 麦克风实时上行"],
|
||||
["audio_playback_buffered", "16kHz PCM 连续缓冲播放"],
|
||||
["heartbeat_ack", "平台 session_updated 存活回执"],
|
||||
["user_transcript", "用户最终字幕"],
|
||||
["agent_transcript", "智能体最终字幕"],
|
||||
["transcript_uploaded", "最终话轮可靠上传"],
|
||||
["provider_closed", "火山连接正常关闭"],
|
||||
["platform_completed", "平台确认会话 COMPLETED"],
|
||||
];
|
||||
|
||||
const state: {
|
||||
phase: HardwarePhase;
|
||||
platformSession?: CreatedHardwareAgentSession;
|
||||
providerConnectionRef?: string;
|
||||
eventSequence: number;
|
||||
transcriptSequence: number;
|
||||
pendingTurns: DeviceFinalTurn[];
|
||||
transcriptUpload?: Promise<void>;
|
||||
heartbeatTimer?: number;
|
||||
durationTimer?: number;
|
||||
startedAt?: number;
|
||||
stopping: boolean;
|
||||
intentionalProviderClose: boolean;
|
||||
microphoneStream?: MediaStream;
|
||||
captureContext?: AudioContext;
|
||||
captureNode?: AudioWorkletNode;
|
||||
playbackContext?: AudioContext;
|
||||
playbackNode?: AudioWorkletNode;
|
||||
playbackReady?: Promise<AudioWorkletNode>;
|
||||
playbackPendingByte?: number;
|
||||
playbackChunks: number;
|
||||
playbackSamples: number;
|
||||
playbackUnderflows: number;
|
||||
responseTranscripts: Map<string, string>;
|
||||
speechStartedAt?: number;
|
||||
checks: Map<string, ReportCheck>;
|
||||
events: ReportEvent[];
|
||||
turns: DeviceFinalTurn[];
|
||||
failure?: string;
|
||||
failureTask?: Promise<void>;
|
||||
} = {
|
||||
phase: "idle",
|
||||
eventSequence: 0,
|
||||
transcriptSequence: 0,
|
||||
pendingTurns: [],
|
||||
stopping: false,
|
||||
intentionalProviderClose: false,
|
||||
playbackChunks: 0,
|
||||
playbackSamples: 0,
|
||||
playbackUnderflows: 0,
|
||||
responseTranscripts: new Map(),
|
||||
checks: new Map(checkDefinitions.map(([id, label]) => [id, { id, label, status: "pending" }])),
|
||||
events: [],
|
||||
turns: [],
|
||||
};
|
||||
|
||||
const app = document.querySelector<HTMLElement>("#app")!;
|
||||
app.innerHTML = `
|
||||
<section class="shell hardware-shell">
|
||||
<header class="hero">
|
||||
<div>
|
||||
<p class="eyebrow">WINDOWS · VOLC HARDWARE AGENT · LOW LOAD</p>
|
||||
<h1>硬件对话智能体测试程序</h1>
|
||||
<p class="subtitle">真实连接平台设备 v3 控制面和火山硬件对话智能体 WebSocket，验证动态人设、PCM 双向语音、字幕、记忆工具、平台停止与结束闭环。</p>
|
||||
</div>
|
||||
<div class="hero-actions"><a class="legacy-link" href="./legacy.html">传统 RTC 测试页</a><div class="overall" id="overallStatus"><strong>尚未检测</strong></div></div>
|
||||
</header>
|
||||
|
||||
<section class="status-grid" aria-label="链路状态">
|
||||
<article class="status-card"><span>平台 v3</span><strong id="platformStatus">未检测</strong><small id="platformDetail">等待设备鉴权</small></article>
|
||||
<article class="status-card"><span>火山设备凭据</span><strong id="credentialStatus">未配置</strong><small id="credentialDetail">支持动态注册</small></article>
|
||||
<article class="status-card"><span>动态会话配置</span><strong id="configStatus">未应用</strong><small id="configDetail">麦克风保持关闭</small></article>
|
||||
<article class="status-card"><span>通话时长</span><strong id="duration">00:00</strong><small id="providerStatus">网关未连接</small></article>
|
||||
</section>
|
||||
|
||||
<section class="content-grid hardware-content-grid">
|
||||
<article class="panel setup-panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">SECURE SETUP</p><h2>平台与火山设备</h2></div><span class="safe">密钥不写入报告</span></div>
|
||||
<h3>平台设备身份</h3>
|
||||
<label>平台地址<input id="apiURL" value="https://121.41.46.238" autocomplete="off" /></label>
|
||||
<label>平台设备 ID<input id="deviceID" autocomplete="off" /></label>
|
||||
<label>平台设备签名密钥<input id="deviceSecret" type="password" autocomplete="off" /></label>
|
||||
<h3>火山硬件身份</h3>
|
||||
<div class="row"><label>实例 ID（平台下发）<input id="instanceID" readonly /></label><label>Product Key（平台下发）<input id="productKey" readonly /></label></div>
|
||||
<div class="row"><label>设备名称（平台分配）<input id="volcDeviceName" readonly /></label><label>Hardware ID<input id="hardwareID" autocomplete="off" /></label></div>
|
||||
<label>智能体模型类型<select id="modelMode"><option value="llm">通用 LLM（LLMConfig.SystemMessages）</option><option value="s2s">豆包端到端语音（S2S dialog）</option></select></label>
|
||||
<label>麦克风<select id="microphone"><option value="">默认麦克风</option></select></label>
|
||||
<button class="secondary" id="checkButton">检测、鉴权并准备设备</button>
|
||||
<p class="hint">Product Secret 只保存在平台密钥服务中。平台代设备完成动态注册，设备仅接收自己的一机一密并使用 Windows Safe Storage 加密保存；默认 Bot 由平台产品配置自动分配，设备不能覆盖。</p>
|
||||
</article>
|
||||
|
||||
<article class="panel call-panel hardware-call-panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">REAL END-TO-END FLOW</p><h2>硬件智能体通话</h2></div><span class="phase-badge" id="phaseBadge">待机</span></div>
|
||||
<div class="conversation-orb" id="conversationOrb"><div class="orb-core"><span id="orbLabel">准备检测</span></div></div>
|
||||
<div class="meters">
|
||||
<div><div class="meter-label"><span>16k PCM 上行</span><strong id="micValue">0</strong></div><div class="meter"><i id="micMeter"></i></div></div>
|
||||
<div><div class="meter-label"><span>16k PCM 下行</span><strong id="agentValue">0</strong></div><div class="meter agent"><i id="agentMeter"></i></div></div>
|
||||
</div>
|
||||
<div class="call-actions"><button class="primary" id="startButton" disabled>开始硬件智能体通话</button><button class="danger" id="stopButton" disabled>结束通话</button></div>
|
||||
<div class="text-question"><input id="textQuestion" placeholder="可选：输入文字验证智能体回复" /><button class="secondary" id="sendTextButton" disabled>发送文本问题</button></div>
|
||||
<dl class="session-info hardware-session-info">
|
||||
<div><dt>平台通话编号</dt><dd id="conversationID">—</dd></div>
|
||||
<div><dt>产品默认 Bot</dt><dd id="botID">—</dd></div>
|
||||
<div><dt>传输方案</dt><dd id="transport">low_load</dd></div>
|
||||
<div><dt>火山会话编号</dt><dd id="providerConnectionRef">—</dd></div>
|
||||
<div><dt>配置摘要</dt><dd id="configDigest">—</dd></div>
|
||||
<div><dt>冻结提示版本</dt><dd id="promptVersion">—</dd></div>
|
||||
</dl>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="panel flow-panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">ACCEPTANCE GATES</p><h2>完整流程验收</h2></div><button class="text-button" id="exportReport">导出测试报告</button></div>
|
||||
<ol id="flowChecks" class="flow-checks"></ol>
|
||||
</section>
|
||||
|
||||
<section class="panel transcript-panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">FINAL TRANSCRIPTS</p><h2>双方最终字幕与平台上传</h2></div><span class="phase-badge" id="transcriptStatus">0 条</span></div>
|
||||
<ol id="liveTranscript"><li class="transcript-empty">通话开始后显示火山返回的用户和智能体最终字幕。</li></ol>
|
||||
</section>
|
||||
|
||||
<section class="panel log-panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">AUDITABLE EVENT STREAM</p><h2>平台与火山事件</h2></div><button class="text-button" id="clearLog">清空显示</button></div>
|
||||
<ol id="eventLog"><li><time>系统</time><span>测试程序已启动，等待环境检测。</span></li></ol>
|
||||
</section>
|
||||
</section>
|
||||
`;
|
||||
|
||||
const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;
|
||||
const controls = {
|
||||
apiURL: $<HTMLInputElement>("apiURL"), deviceID: $<HTMLInputElement>("deviceID"), deviceSecret: $<HTMLInputElement>("deviceSecret"),
|
||||
instanceID: $<HTMLInputElement>("instanceID"), productKey: $<HTMLInputElement>("productKey"),
|
||||
volcDeviceName: $<HTMLInputElement>("volcDeviceName"), hardwareID: $<HTMLInputElement>("hardwareID"),
|
||||
modelMode: $<HTMLSelectElement>("modelMode"),
|
||||
microphone: $<HTMLSelectElement>("microphone"), check: $<HTMLButtonElement>("checkButton"), start: $<HTMLButtonElement>("startButton"),
|
||||
stop: $<HTMLButtonElement>("stopButton"), textQuestion: $<HTMLInputElement>("textQuestion"), sendText: $<HTMLButtonElement>("sendTextButton"),
|
||||
};
|
||||
|
||||
function setText(id: string, value: string) { $(id).textContent = value; }
|
||||
function errorMessage(error: unknown) { return error instanceof Error ? error.message : String(error); }
|
||||
|
||||
function recordEvent(source: ReportEvent["source"], type: string, detail?: Record<string, unknown>) {
|
||||
state.events.push({ at: new Date().toISOString(), source, type, detail });
|
||||
if (state.events.length > 1000) state.events.splice(0, state.events.length - 1000);
|
||||
}
|
||||
|
||||
function log(message: string, level: "info" | "success" | "error" = "info", source: ReportEvent["source"] = "app", type = "log") {
|
||||
const item = document.createElement("li");
|
||||
item.className = level;
|
||||
const time = document.createElement("time");
|
||||
time.textContent = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
||||
const content = document.createElement("span");
|
||||
content.textContent = message;
|
||||
item.append(time, content);
|
||||
$("eventLog").prepend(item);
|
||||
recordEvent(source, type, { level, message });
|
||||
}
|
||||
|
||||
function setPhase(phase: HardwarePhase, detail: string) {
|
||||
state.phase = phase;
|
||||
const labels: Record<HardwarePhase, string> = { idle: "待机", checking: "检测中", ready: "已就绪", connecting: "连接中", active: "通话中", stopping: "结束中", completed: "已完成", error: "异常" };
|
||||
setText("phaseBadge", labels[phase]);
|
||||
setText("orbLabel", detail);
|
||||
setText("overallStatus", detail);
|
||||
$("overallStatus").className = `overall phase-${phase}`;
|
||||
$("conversationOrb").className = `conversation-orb phase-${phase}`;
|
||||
controls.check.disabled = ["connecting", "active", "stopping"].includes(phase);
|
||||
controls.start.disabled = phase !== "ready" && phase !== "completed";
|
||||
controls.stop.disabled = !["active", "error"].includes(phase) || !state.platformSession || !!state.failureTask;
|
||||
controls.sendText.disabled = phase !== "active";
|
||||
}
|
||||
|
||||
function setCheck(id: string, status: ReportCheck["status"], detail?: string) {
|
||||
const check = state.checks.get(id);
|
||||
if (!check) return;
|
||||
check.status = status;
|
||||
check.detail = detail;
|
||||
check.at = new Date().toISOString();
|
||||
renderChecks();
|
||||
}
|
||||
|
||||
function resetChecks() {
|
||||
state.checks = new Map(checkDefinitions.map(([id, label]) => [id, { id, label, status: "pending" }]));
|
||||
renderChecks();
|
||||
}
|
||||
|
||||
function renderChecks() {
|
||||
const list = $("flowChecks");
|
||||
list.replaceChildren(...[...state.checks.values()].map((check) => {
|
||||
const item = document.createElement("li");
|
||||
item.className = `flow-check flow-check--${check.status}`;
|
||||
const mark = document.createElement("i");
|
||||
mark.textContent = check.status === "passed" ? "✓" : check.status === "failed" ? "!" : "·";
|
||||
const content = document.createElement("span");
|
||||
content.textContent = check.label;
|
||||
const detail = document.createElement("small");
|
||||
detail.textContent = check.detail || (check.status === "pending" ? "等待验证" : check.status === "passed" ? "已通过" : "未通过");
|
||||
item.append(mark, content, detail);
|
||||
return item;
|
||||
}));
|
||||
}
|
||||
|
||||
function updateMeter(kind: "mic" | "agent", value: number) {
|
||||
const normalized = Math.max(0, Math.min(100, Math.round(value)));
|
||||
$(`${kind}Meter`).style.width = `${normalized}%`;
|
||||
setText(`${kind}Value`, String(normalized));
|
||||
}
|
||||
|
||||
async function loadDefaults() {
|
||||
const [platform, hardware] = await Promise.all([
|
||||
window.deviceSimulator.getRuntimeDefaults(),
|
||||
window.deviceSimulator.getHardwareRuntimeDefaults(),
|
||||
]);
|
||||
controls.apiURL.value = platform.apiURL;
|
||||
controls.deviceID.value = platform.deviceID;
|
||||
if (platform.deviceSecretConfigured) controls.deviceSecret.placeholder = "已从 Windows 安全凭据文件加载";
|
||||
controls.instanceID.value = hardware.instanceID;
|
||||
controls.productKey.value = hardware.productKey;
|
||||
controls.volcDeviceName.value = hardware.deviceName;
|
||||
controls.hardwareID.value = hardware.hardwareID;
|
||||
controls.modelMode.value = hardware.modelMode;
|
||||
if (hardware.deviceCredentialConfigured) setText("credentialDetail", "本机已有平台签发的一机一密安全缓存");
|
||||
}
|
||||
|
||||
async function enumerateMicrophones() {
|
||||
const permission = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
permission.getTracks().forEach((track) => track.stop());
|
||||
const devices = (await navigator.mediaDevices.enumerateDevices()).filter((device) => device.kind === "audioinput");
|
||||
controls.microphone.replaceChildren(new Option("默认麦克风", ""));
|
||||
devices.forEach((device, index) => controls.microphone.add(new Option(device.label || `麦克风 ${index + 1}`, device.deviceId)));
|
||||
}
|
||||
|
||||
async function configureAndProbe() {
|
||||
if (state.failureTask) await state.failureTask;
|
||||
setPhase("checking", "正在鉴权和检测");
|
||||
resetChecks();
|
||||
state.events = [];
|
||||
state.turns = [];
|
||||
state.failure = undefined;
|
||||
try {
|
||||
await window.deviceSimulator.configure({ apiURL: controls.apiURL.value, deviceID: controls.deviceID.value, deviceSecret: controls.deviceSecret.value });
|
||||
await window.deviceSimulator.probe();
|
||||
setCheck("platform_authenticated", "passed", "设备 bootstrap/config HMAC 请求成功");
|
||||
setText("platformStatus", "鉴权成功");
|
||||
setText("platformDetail", controls.apiURL.value);
|
||||
const hardware = await window.deviceSimulator.configureHardware({ hardwareID: controls.hardwareID.value });
|
||||
controls.instanceID.value = hardware.instanceID;
|
||||
controls.productKey.value = hardware.productKey;
|
||||
controls.volcDeviceName.value = hardware.deviceName;
|
||||
controls.hardwareID.value = hardware.hardwareID;
|
||||
setCheck("credential_resolved", "passed", `${hardware.credentialSource}${hardware.persisted ? "，Device Secret 已写入 Windows 安全存储" : "，仅本次内存"}`);
|
||||
setText("credentialStatus", hardware.registered ? "平台注册成功" : "平台凭据可用");
|
||||
setText("credentialDetail", `${hardware.deviceName} · ${hardware.credentialSource} · 火山 License 额度已在动态注册时由服务端消费`);
|
||||
const network = await window.deviceSimulator.probeHardwareNetwork();
|
||||
const failures = network.endpoints.filter((endpoint) => !endpoint.ok);
|
||||
if (failures.length) throw new Error(`火山网络不可达：${failures.map((item) => `${item.hostname} ${item.error || "失败"}`).join("；")}`);
|
||||
await enumerateMicrophones();
|
||||
setPhase("ready", "可以开始真实通话");
|
||||
log("平台 v3、平台代理注册、火山网关和麦克风检测通过。", "success");
|
||||
} catch (error) {
|
||||
state.failure = errorMessage(error);
|
||||
setPhase("error", "环境检测失败");
|
||||
setText("platformStatus", "检测失败");
|
||||
setText("platformDetail", state.failure);
|
||||
log(`环境检测失败：${state.failure}`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
function renderPlatformSession(created: CreatedHardwareAgentSession) {
|
||||
setText("conversationID", created.conversation_id);
|
||||
setText("botID", created.runtime.bot_id);
|
||||
setText("transport", created.runtime.transport);
|
||||
setText("configDigest", created.runtime.config_digest);
|
||||
setText("promptVersion", created.session_config.prompt_bundle_version);
|
||||
setText("providerConnectionRef", "等待火山连接");
|
||||
}
|
||||
|
||||
async function startHardwareConversation() {
|
||||
if (!["ready", "completed"].includes(state.phase)) return;
|
||||
if (state.failureTask) await state.failureTask;
|
||||
setPhase("connecting", "正在冻结本次上下文");
|
||||
resetSessionState();
|
||||
try {
|
||||
const created = await window.deviceSimulator.createHardwareAgentSession();
|
||||
state.platformSession = created;
|
||||
renderPlatformSession(created);
|
||||
if (created.runtime.transport !== "low_load") throw new Error(`设备绑定传输模式为 ${created.runtime.transport}，Windows 标准协议测试要求 low_load`);
|
||||
if (!created.session_config.authorized_context || !created.runtime.config_digest || !created.runtime.session_nonce) throw new Error("平台未返回完整冻结上下文或会话凭证");
|
||||
setCheck("context_frozen", "passed", `${created.session_config.prompt_bundle_version} · ${created.runtime.config_digest.slice(0, 12)}…`);
|
||||
recordEvent("platform", "hardware.session.ready", { conversation_id: created.conversation_id, bot_id: created.runtime.bot_id, transport: created.runtime.transport });
|
||||
setText("configStatus", "等待 session.updated");
|
||||
setText("configDetail", "麦克风仍保持关闭");
|
||||
const connected = await window.deviceSimulator.connectHardwareAgent({ platformSession: created, modelMode: controls.modelMode.value as "llm" | "s2s" });
|
||||
if (state.failureTask) { await state.failureTask; return; }
|
||||
state.providerConnectionRef = connected.providerConnectionRef;
|
||||
setText("providerConnectionRef", connected.providerConnectionRef);
|
||||
setText("providerStatus", "WebSocket 已连接");
|
||||
setCheck("provider_connected", "passed", connected.providerConnectionRef);
|
||||
setCheck("dynamic_config_applied", "passed", "火山已返回 session.updated");
|
||||
setText("configStatus", "已应用");
|
||||
setText("configDetail", "允许激活麦克风");
|
||||
state.eventSequence = 1;
|
||||
const activated = await window.deviceSimulator.hardwareAgentEvent({
|
||||
conversationID: created.conversation_id, sequenceNo: state.eventSequence, eventType: "session_started",
|
||||
providerSessionRef: created.runtime.provider_session_ref, sessionNonce: created.runtime.session_nonce,
|
||||
providerConnectionRef: connected.providerConnectionRef, botID: created.runtime.bot_id, transport: created.runtime.transport,
|
||||
configDigest: created.runtime.config_digest, dynamicConfigApplied: true,
|
||||
});
|
||||
if (activated.status !== "ACTIVE" || activated.action !== "continue") throw new Error(`平台未激活硬件会话：${activated.status}/${activated.action}`);
|
||||
if (state.failureTask) { await state.failureTask; return; }
|
||||
setCheck("platform_active", "passed", "session_started 已进入 ACTIVE");
|
||||
await ensurePlaybackNode();
|
||||
await startMicrophone();
|
||||
if (state.failureTask) { await state.failureTask; await stopAudio(); return; }
|
||||
setCheck("microphone_streaming", "passed", "PCM16/16kHz/单声道/20ms 分片");
|
||||
state.startedAt = Date.now();
|
||||
setPhase("active", "请开始说话");
|
||||
startTimers(created.session.heartbeat_interval_sec || 15);
|
||||
log("硬件对话智能体已激活：动态配置确认后才开放麦克风。", "success", "platform", "hardware.session.active");
|
||||
} catch (error) {
|
||||
state.failure ||= errorMessage(error);
|
||||
log(`启动硬件智能体失败：${state.failure}`, "error");
|
||||
await failHardwareConversation("WINDOWS_HARDWARE_START_FAILED");
|
||||
}
|
||||
}
|
||||
|
||||
function resetSessionState() {
|
||||
stopTimers();
|
||||
state.platformSession = undefined;
|
||||
state.providerConnectionRef = undefined;
|
||||
state.eventSequence = 0;
|
||||
state.transcriptSequence = 0;
|
||||
state.pendingTurns = [];
|
||||
state.transcriptUpload = undefined;
|
||||
state.startedAt = undefined;
|
||||
state.stopping = false;
|
||||
state.intentionalProviderClose = false;
|
||||
state.failureTask = undefined;
|
||||
state.failure = undefined;
|
||||
state.responseTranscripts.clear();
|
||||
state.speechStartedAt = undefined;
|
||||
state.turns = [];
|
||||
state.playbackPendingByte = undefined;
|
||||
state.playbackChunks = 0;
|
||||
state.playbackSamples = 0;
|
||||
state.playbackUnderflows = 0;
|
||||
// Device preparation remains valid, but every call must earn its own checks.
|
||||
for (const [id, label] of checkDefinitions) {
|
||||
if (!["platform_authenticated", "credential_resolved"].includes(id)) {
|
||||
state.checks.set(id, { id, label, status: "pending" });
|
||||
}
|
||||
}
|
||||
renderChecks();
|
||||
setText("duration", "00:00");
|
||||
setText("providerStatus", "网关未连接");
|
||||
setText("providerConnectionRef", "—");
|
||||
setText("conversationID", "—");
|
||||
setText("configStatus", "未应用");
|
||||
setText("configDetail", "麦克风保持关闭");
|
||||
$("liveTranscript").innerHTML = '<li class="transcript-empty">通话开始后显示火山返回的用户和智能体最终字幕。</li>';
|
||||
setText("transcriptStatus", "0 条");
|
||||
updateMeter("mic", 0);
|
||||
updateMeter("agent", 0);
|
||||
}
|
||||
|
||||
async function startMicrophone() {
|
||||
const constraints: MediaTrackConstraints = { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true };
|
||||
if (controls.microphone.value) constraints.deviceId = { exact: controls.microphone.value };
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: constraints });
|
||||
const context = new AudioContext({ latencyHint: "interactive" });
|
||||
await context.audioWorklet.addModule(new URL("./pcm-capture.worklet.js", window.location.href).toString());
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const node = new AudioWorkletNode(context, "pcm16-capture", { processorOptions: { targetSampleRate: 16000, frameSamples: 320 } });
|
||||
const silent = context.createGain();
|
||||
silent.gain.value = 0;
|
||||
node.port.onmessage = (message) => {
|
||||
if (state.phase !== "active" && state.phase !== "connecting") return;
|
||||
if (message.data?.type !== "pcm" || !(message.data.bytes instanceof ArrayBuffer)) return;
|
||||
updateMeter("mic", Number(message.data.volume || 0));
|
||||
window.deviceSimulator.sendHardwareAgentAudio(message.data.bytes);
|
||||
};
|
||||
source.connect(node).connect(silent).connect(context.destination);
|
||||
await context.resume();
|
||||
state.microphoneStream = stream;
|
||||
state.captureContext = context;
|
||||
state.captureNode = node;
|
||||
}
|
||||
|
||||
async function stopAudio() {
|
||||
state.microphoneStream?.getTracks().forEach((track) => track.stop());
|
||||
state.captureNode?.disconnect();
|
||||
state.playbackNode?.port.postMessage({ type: "reset" });
|
||||
state.playbackNode?.disconnect();
|
||||
if (state.captureContext) try { await state.captureContext.close(); } catch { /* no-op */ }
|
||||
if (state.playbackContext) try { await state.playbackContext.close(); } catch { /* no-op */ }
|
||||
state.microphoneStream = undefined;
|
||||
state.captureNode = undefined;
|
||||
state.captureContext = undefined;
|
||||
state.playbackNode = undefined;
|
||||
state.playbackReady = undefined;
|
||||
state.playbackContext = undefined;
|
||||
state.playbackPendingByte = undefined;
|
||||
updateMeter("mic", 0);
|
||||
updateMeter("agent", 0);
|
||||
}
|
||||
|
||||
function ensurePlaybackNode(): Promise<AudioWorkletNode> {
|
||||
if (state.playbackNode) return Promise.resolve(state.playbackNode);
|
||||
if (state.playbackReady) return state.playbackReady;
|
||||
state.playbackReady = (async () => {
|
||||
const context = new AudioContext({ latencyHint: "interactive" });
|
||||
state.playbackContext = context;
|
||||
await context.audioWorklet.addModule(new URL("./pcm-playback.worklet.js", window.location.href).toString());
|
||||
const node = new AudioWorkletNode(context, "pcm16-playback", {
|
||||
processorOptions: { sourceSampleRate: 16000, prebufferMs: 120 },
|
||||
outputChannelCount: [1],
|
||||
});
|
||||
node.port.onmessage = (message) => {
|
||||
if (message.data?.type === "underflow") {
|
||||
state.playbackUnderflows = Number(message.data.count || state.playbackUnderflows + 1);
|
||||
recordEvent("app", "audio.playback.underflow", { count: state.playbackUnderflows });
|
||||
}
|
||||
if (message.data?.type === "drained") {
|
||||
updateMeter("agent", 0);
|
||||
setCheck("audio_playback_buffered", "passed", `${state.playbackChunks} 个分片 · ${state.playbackSamples} 个采样 · 欠载 ${state.playbackUnderflows} 次`);
|
||||
}
|
||||
};
|
||||
node.connect(context.destination);
|
||||
await context.resume();
|
||||
state.playbackNode = node;
|
||||
return node;
|
||||
})().catch(async (error) => {
|
||||
const context = state.playbackContext;
|
||||
state.playbackReady = undefined;
|
||||
state.playbackContext = undefined;
|
||||
if (context) try { await context.close(); } catch { /* no-op */ }
|
||||
throw error;
|
||||
});
|
||||
return state.playbackReady;
|
||||
}
|
||||
|
||||
async function playProviderAudio(base64: string) {
|
||||
if (!base64) return;
|
||||
const binary = atob(base64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
|
||||
const totalBytes = bytes.byteLength + (state.playbackPendingByte === undefined ? 0 : 1);
|
||||
const sampleCount = Math.floor(totalBytes / 2);
|
||||
if (sampleCount === 0) {
|
||||
if (bytes.byteLength === 1) state.playbackPendingByte = bytes[0];
|
||||
return;
|
||||
}
|
||||
const samples = new Float32Array(sampleCount);
|
||||
let energy = 0;
|
||||
let byteIndex = 0;
|
||||
let sampleIndex = 0;
|
||||
if (state.playbackPendingByte !== undefined) {
|
||||
const value = new DataView(new Uint8Array([state.playbackPendingByte, bytes[0]]).buffer).getInt16(0, true) / 32768;
|
||||
samples[sampleIndex] = value;
|
||||
energy += value * value;
|
||||
sampleIndex += 1;
|
||||
byteIndex = 1;
|
||||
state.playbackPendingByte = undefined;
|
||||
}
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
while (sampleIndex < sampleCount) {
|
||||
const value = view.getInt16(byteIndex, true) / 32768;
|
||||
samples[sampleIndex] = value;
|
||||
energy += value * value;
|
||||
sampleIndex += 1;
|
||||
byteIndex += 2;
|
||||
}
|
||||
if (byteIndex < bytes.byteLength) state.playbackPendingByte = bytes[byteIndex];
|
||||
updateMeter("agent", Math.sqrt(energy / Math.max(1, sampleCount)) * 280);
|
||||
state.playbackChunks += 1;
|
||||
state.playbackSamples += sampleCount;
|
||||
const node = await ensurePlaybackNode();
|
||||
node.port.postMessage({ type: "push", samples: samples.buffer }, [samples.buffer]);
|
||||
setCheck("audio_playback_buffered", "passed", `已接收并缓冲 ${state.playbackChunks} 个火山音频分片`);
|
||||
}
|
||||
|
||||
function sanitizeProviderEvent(event: Record<string, unknown>) {
|
||||
const redact = (value: unknown, key = ""): unknown => {
|
||||
if (/systemmessages|system_role|system_prompt|instructions|authorized_context|speaking_style/i.test(key)) return "[REDACTED]";
|
||||
if (Array.isArray(value)) return value.map((item) => redact(item));
|
||||
if (!value || typeof value !== "object") return value;
|
||||
return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([itemKey, item]) => [itemKey, redact(item, itemKey)]));
|
||||
};
|
||||
const copy = redact(event) as Record<string, unknown>;
|
||||
if (typeof copy.delta === "string" && event.type === "response.audio.delta") copy.delta = `<pcm-base64:${copy.delta.length} chars>`;
|
||||
return copy;
|
||||
}
|
||||
|
||||
async function handleProviderEvent(event: Record<string, unknown>) {
|
||||
const type = String(event.type || "unknown");
|
||||
recordEvent("volcengine", type, sanitizeProviderEvent(event));
|
||||
if (type !== "response.audio.delta") log(`火山事件：${type}`, type === "error" ? "error" : "info", "volcengine", type);
|
||||
// Teardown can still deliver queued provider events. Do not revive a failed call.
|
||||
if (state.failureTask && type !== "transport.close") return;
|
||||
switch (type) {
|
||||
case "session.created": setText("providerStatus", "会话已创建"); break;
|
||||
case "session.updated": setText("configStatus", "已确认"); break;
|
||||
case "input_audio_buffer.speech_started": state.speechStartedAt = Date.now(); setPhase("active", "检测到用户说话"); break;
|
||||
case "input_audio_buffer.speech_stopped": setPhase("active", "智能体正在思考"); break;
|
||||
case "response.created": setPhase("active", "智能体正在生成"); break;
|
||||
case "response.audio.delta": await playProviderAudio(String(event.delta || "")); break;
|
||||
case "response.audio.done": state.playbackNode?.port.postMessage({ type: "end" }); break;
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
await appendFinalTurn("subject", String(event.transcript || ""), String(event.event_id || crypto.randomUUID()), state.speechStartedAt); state.speechStartedAt = undefined; break;
|
||||
case "response.audio_transcript.delta": {
|
||||
const responseID = String(event.response_id || "unknown");
|
||||
state.responseTranscripts.set(responseID, `${state.responseTranscripts.get(responseID) || ""}${String(event.delta || "")}`);
|
||||
break;
|
||||
}
|
||||
case "response.audio_transcript.done": {
|
||||
const responseID = String(event.response_id || "unknown");
|
||||
const transcript = String(event.transcript || state.responseTranscripts.get(responseID) || "");
|
||||
state.responseTranscripts.delete(responseID);
|
||||
await appendFinalTurn("agent", transcript, responseID); break;
|
||||
}
|
||||
case "response.done": setPhase("active", "继续聆听"); break;
|
||||
case "platform.memory_search.completed": log(`平台记忆工具返回 ${Number(event.result_count || 0)} 条已授权记忆。`, "success", "platform", type); break;
|
||||
case "platform.memory_search.failed": log(`平台记忆工具降级：${String(event.message || "未知错误")}`, "error", "platform", type); break;
|
||||
case "error": {
|
||||
const providerError = event.error as Record<string, unknown> | undefined;
|
||||
state.failure ||= `${String(providerError?.code || providerError?.type || "PROVIDER_ERROR")}: ${String(providerError?.message || "火山智能体错误")}`;
|
||||
log(`火山错误：${state.failure}`, "error", "volcengine", "provider.error");
|
||||
await failHardwareConversation("VOLC_HARDWARE_AGENT_ERROR");
|
||||
break;
|
||||
}
|
||||
case "transport.close":
|
||||
setText("providerStatus", "连接已关闭");
|
||||
if (state.failureTask) setCheck("provider_closed", "failed", `失败清理关闭：WebSocket ${Number(event.code || 1000)}`);
|
||||
else if (state.intentionalProviderClose) setCheck("provider_closed", "passed", `WebSocket ${Number(event.code || 1000)}`);
|
||||
else if (["connecting", "active"].includes(state.phase)) { state.failure = `火山连接意外关闭：${String(event.code || "")}`; await failHardwareConversation("VOLC_CONNECTION_CLOSED"); }
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async function appendFinalTurn(speaker: "subject" | "agent", rawText: string, sourceID: string, startedAt?: number) {
|
||||
const text = rawText.trim();
|
||||
if (!text || !state.platformSession) return;
|
||||
state.transcriptSequence += 1;
|
||||
const now = Date.now();
|
||||
const turn: DeviceFinalTurn = {
|
||||
turn_key: `hardware-${speaker}-${sourceID}`.slice(0, 240), sequence_no: state.transcriptSequence, speaker,
|
||||
start_at: new Date(startedAt || now).toISOString(), end_at: new Date(now).toISOString(), final_text: text, interrupted: false,
|
||||
};
|
||||
state.pendingTurns.push(turn);
|
||||
state.turns.push(turn);
|
||||
renderTurn(turn);
|
||||
setCheck(speaker === "subject" ? "user_transcript" : "agent_transcript", "passed", text.slice(0, 50));
|
||||
await flushTurns();
|
||||
}
|
||||
|
||||
function renderTurn(turn: DeviceFinalTurn) {
|
||||
const list = $("liveTranscript");
|
||||
list.querySelector(".transcript-empty")?.remove();
|
||||
const item = document.createElement("li");
|
||||
item.className = `transcript-turn transcript-turn--${turn.speaker}`;
|
||||
const speaker = document.createElement("strong"); speaker.textContent = turn.speaker === "subject" ? "用户" : "智能体";
|
||||
const text = document.createElement("span"); text.textContent = turn.final_text;
|
||||
item.append(speaker, text); list.append(item);
|
||||
setText("transcriptStatus", `${state.turns.length} 条`);
|
||||
}
|
||||
|
||||
async function flushTurns() {
|
||||
if (state.transcriptUpload) return state.transcriptUpload;
|
||||
if (!state.platformSession || state.pendingTurns.length === 0) return;
|
||||
const batch = state.pendingTurns.slice(0, 500);
|
||||
state.transcriptUpload = (async () => {
|
||||
try {
|
||||
const result = await window.deviceSimulator.submitTranscriptBatch({ conversationID: state.platformSession!.conversation_id, turns: batch, hardwareAgent: true });
|
||||
state.pendingTurns.splice(0, batch.length);
|
||||
setCheck("transcript_uploaded", "passed", `新增 ${result.accepted_count}，重复 ${result.duplicate_count}`);
|
||||
log(`最终字幕已上传平台：新增 ${result.accepted_count}，重复 ${result.duplicate_count}。`, "success", "platform", "hardware.turns.uploaded");
|
||||
} catch (error) { log(`字幕上传失败，保留队列待重试：${errorMessage(error)}`, "error"); }
|
||||
finally { state.transcriptUpload = undefined; }
|
||||
})();
|
||||
await state.transcriptUpload;
|
||||
}
|
||||
|
||||
async function flushTurnsWithRetry(attempts: number) {
|
||||
for (let attempt = 1; attempt <= attempts && state.pendingTurns.length > 0; attempt += 1) {
|
||||
await flushTurns();
|
||||
if (state.pendingTurns.length && attempt < attempts) await new Promise((resolve) => window.setTimeout(resolve, attempt * 500));
|
||||
}
|
||||
if (state.pendingTurns.length) throw new Error(`仍有 ${state.pendingTurns.length} 条最终字幕未上传`);
|
||||
}
|
||||
|
||||
function startTimers(intervalSeconds: number) {
|
||||
stopTimers();
|
||||
state.durationTimer = window.setInterval(() => {
|
||||
if (!state.startedAt) return;
|
||||
const seconds = Math.floor((Date.now() - state.startedAt) / 1000);
|
||||
setText("duration", `${String(Math.floor(seconds / 60)).padStart(2, "0")}:${String(seconds % 60).padStart(2, "0")}`);
|
||||
}, 1000);
|
||||
state.heartbeatTimer = window.setInterval(() => void sendHardwareHeartbeat(), Math.max(5, intervalSeconds) * 1000);
|
||||
void sendHardwareHeartbeat();
|
||||
}
|
||||
|
||||
function stopTimers() {
|
||||
if (state.durationTimer) window.clearInterval(state.durationTimer);
|
||||
if (state.heartbeatTimer) window.clearInterval(state.heartbeatTimer);
|
||||
state.durationTimer = undefined;
|
||||
state.heartbeatTimer = undefined;
|
||||
}
|
||||
|
||||
async function sendHardwareHeartbeat() {
|
||||
const created = state.platformSession;
|
||||
if (!created || state.phase !== "active" || state.stopping) return;
|
||||
try {
|
||||
state.eventSequence += 1;
|
||||
const result = await window.deviceSimulator.hardwareAgentEvent({
|
||||
conversationID: created.conversation_id, sequenceNo: state.eventSequence, eventType: "session_updated",
|
||||
providerSessionRef: created.runtime.provider_session_ref, sessionNonce: created.runtime.session_nonce,
|
||||
});
|
||||
setCheck("heartbeat_ack", "passed", `序号 ${state.eventSequence} · ${result.action}`);
|
||||
if (result.action === "stop_session") await stopHardwareConversation("platform_stop_session");
|
||||
await flushTurns();
|
||||
} catch (error) { log(`硬件会话存活上报失败：${errorMessage(error)}`, "error"); }
|
||||
}
|
||||
|
||||
async function stopHardwareConversation(reason = "windows_user_hangup") {
|
||||
if (state.failureTask) return state.failureTask;
|
||||
if (!state.platformSession || state.stopping) return;
|
||||
state.stopping = true;
|
||||
setPhase("stopping", "正在可靠结束");
|
||||
stopTimers();
|
||||
const created = state.platformSession;
|
||||
try {
|
||||
await stopAudio();
|
||||
await flushTurnsWithRetry(3);
|
||||
state.intentionalProviderClose = true;
|
||||
await window.deviceSimulator.closeHardwareAgent({ code: 1000, reason });
|
||||
setCheck("provider_closed", "passed", "设备主动关闭 WebSocket");
|
||||
state.eventSequence += 1;
|
||||
const ended = await window.deviceSimulator.hardwareAgentEvent({
|
||||
conversationID: created.conversation_id, sequenceNo: state.eventSequence, eventType: "session_ended",
|
||||
providerSessionRef: created.runtime.provider_session_ref, sessionNonce: created.runtime.session_nonce,
|
||||
});
|
||||
if (ended.status !== "COMPLETED") throw new Error(`平台结束状态不是 COMPLETED：${ended.status}`);
|
||||
setCheck("platform_completed", "passed", "session_ended 已进入 COMPLETED");
|
||||
state.stopping = false;
|
||||
setPhase("completed", allRequiredChecksPassed() ? "完整流程通过" : "通话完成，部分验收项待覆盖");
|
||||
log("硬件对话智能体会话已正常结束，平台状态为 COMPLETED。", "success", "platform", "hardware.session.completed");
|
||||
} catch (error) {
|
||||
state.failure = errorMessage(error);
|
||||
state.stopping = false;
|
||||
setPhase("error", "结束流程失败");
|
||||
log(`结束硬件会话失败：${state.failure}`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
function failHardwareConversation(errorCode: string): Promise<void> {
|
||||
// The provider error event and connect() rejection describe the same failure.
|
||||
// Install a shared task before any asynchronous cleanup, and retain it until
|
||||
// the next call so later errors/hangup cannot submit another terminal event.
|
||||
if (state.failureTask) return state.failureTask;
|
||||
const created = state.platformSession;
|
||||
state.failureTask = Promise.resolve().then(async () => {
|
||||
try {
|
||||
await stopAudio();
|
||||
try { await window.deviceSimulator.closeHardwareAgent({ code: 1011, reason: errorCode }); } catch { /* no-op */ }
|
||||
if (created) {
|
||||
try {
|
||||
state.eventSequence += 1;
|
||||
await window.deviceSimulator.hardwareAgentEvent({
|
||||
conversationID: created.conversation_id, sequenceNo: state.eventSequence, eventType: "session_failed",
|
||||
providerSessionRef: created.runtime.provider_session_ref, sessionNonce: created.runtime.session_nonce,
|
||||
errorCode,
|
||||
});
|
||||
} catch (error) { log(`平台失败事件回报未成功：${errorMessage(error)}`, "error"); }
|
||||
}
|
||||
} finally {
|
||||
state.stopping = false;
|
||||
setPhase("error", state.failure || "硬件会话失败");
|
||||
}
|
||||
});
|
||||
stopTimers();
|
||||
state.stopping = true;
|
||||
state.intentionalProviderClose = true;
|
||||
setPhase("stopping", "正在清理失败会话");
|
||||
setCheck("platform_completed", "failed", state.failure || errorCode);
|
||||
if (!state.providerConnectionRef) {
|
||||
setText("configStatus", "未完成");
|
||||
setCheck("dynamic_config_applied", "failed", state.failure || errorCode);
|
||||
}
|
||||
return state.failureTask;
|
||||
}
|
||||
|
||||
function allRequiredChecksPassed() {
|
||||
const required = ["platform_authenticated", "credential_resolved", "context_frozen", "provider_connected", "dynamic_config_applied", "platform_active", "microphone_streaming", "audio_playback_buffered", "heartbeat_ack", "transcript_uploaded", "provider_closed", "platform_completed"];
|
||||
return required.every((id) => state.checks.get(id)?.status === "passed");
|
||||
}
|
||||
|
||||
async function sendTextQuestion() {
|
||||
if (state.phase !== "active" || state.failureTask) return;
|
||||
const text = controls.textQuestion.value.trim();
|
||||
if (!text) return;
|
||||
try {
|
||||
await window.deviceSimulator.sendHardwareAgentText({ text, interruptMode: 1 });
|
||||
// 文本输入没有 ASR final 事件，设备本地将已被火山接受的最终输入作为用户话轮上报，
|
||||
// 这样文本验收与真实语音验收都会进入同一套平台聊天记录。
|
||||
await appendFinalTurn("subject", text, `text-${crypto.randomUUID()}`);
|
||||
controls.textQuestion.value = "";
|
||||
log(`已通过硬件标准协议发送文本问题：${text}`, "success", "app", "hardware.text.sent");
|
||||
} catch (error) { log(`发送文本问题失败：${errorMessage(error)}`, "error"); }
|
||||
}
|
||||
|
||||
function buildReport(): Record<string, unknown> {
|
||||
return {
|
||||
schema_version: "hardware-agent-windows-e2e-report-v1",
|
||||
generated_at: new Date().toISOString(),
|
||||
result: state.phase === "completed" && allRequiredChecksPassed() ? "PASS" : state.failure ? "FAIL" : "INCOMPLETE",
|
||||
platform_api: controls.apiURL.value,
|
||||
platform_device_id: controls.deviceID.value,
|
||||
volc_identity: { instance_id: controls.instanceID.value, product_key: controls.productKey.value, device_name: controls.volcDeviceName.value, hardware_id: controls.hardwareID.value },
|
||||
conversation_id: state.platformSession?.conversation_id,
|
||||
bot_id: state.platformSession?.runtime.bot_id,
|
||||
model_mode: controls.modelMode.value,
|
||||
transport: state.platformSession?.runtime.transport,
|
||||
audio_playback: {
|
||||
source_format: "pcm_s16le_16000_mono",
|
||||
mode: "audio_worklet_ring_buffer",
|
||||
prebuffer_ms: 120,
|
||||
chunks: state.playbackChunks,
|
||||
samples: state.playbackSamples,
|
||||
underflows: state.playbackUnderflows,
|
||||
},
|
||||
provider_connection_ref: state.providerConnectionRef,
|
||||
config_digest: state.platformSession?.runtime.config_digest,
|
||||
prompt_bundle_version: state.platformSession?.session_config.prompt_bundle_version,
|
||||
failure: state.failure,
|
||||
checks: [...state.checks.values()],
|
||||
turns: state.turns,
|
||||
events: state.events,
|
||||
};
|
||||
}
|
||||
|
||||
async function exportReport() {
|
||||
try {
|
||||
const result = await window.deviceSimulator.saveHardwareReport(buildReport());
|
||||
if (result.saved) log(`测试报告已保存：${result.path}`, "success");
|
||||
} catch (error) { log(`导出测试报告失败：${errorMessage(error)}`, "error"); }
|
||||
}
|
||||
|
||||
const unsubscribeProvider = window.deviceSimulator.onHardwareProviderEvent((event) => void handleProviderEvent(event));
|
||||
controls.check.addEventListener("click", () => void configureAndProbe());
|
||||
controls.start.addEventListener("click", () => void startHardwareConversation());
|
||||
controls.stop.addEventListener("click", () => void stopHardwareConversation());
|
||||
controls.sendText.addEventListener("click", () => void sendTextQuestion());
|
||||
controls.textQuestion.addEventListener("keydown", (event) => { if (event.key === "Enter") void sendTextQuestion(); });
|
||||
$("exportReport").addEventListener("click", () => void exportReport());
|
||||
$("clearLog").addEventListener("click", () => $("eventLog").replaceChildren());
|
||||
window.addEventListener("beforeunload", () => {
|
||||
unsubscribeProvider();
|
||||
stopTimers();
|
||||
void stopAudio();
|
||||
void window.deviceSimulator.closeHardwareAgent({ code: 1001, reason: "window closed" });
|
||||
});
|
||||
|
||||
renderChecks();
|
||||
setPhase("idle", "尚未检测");
|
||||
void loadDefaults().catch((error) => log(`读取默认配置失败：${errorMessage(error)}`, "error"));
|
||||
@@ -16,10 +16,15 @@ button:disabled { cursor: not-allowed; opacity: .45; }
|
||||
.eyebrow { color: #c8652a; font-size: 12px; font-weight: 800; letter-spacing: .16em; margin: 0 0 9px; }
|
||||
h1 { margin: 0; font-family: Georgia, "Microsoft YaHei UI", serif; font-size: 44px; letter-spacing: -.035em; }
|
||||
h2 { margin: 0; font-size: 23px; }
|
||||
h3 { margin: 24px 0 14px; color: #315966; font-size: 14px; }
|
||||
.subtitle { max-width: 760px; color: #66808e; line-height: 1.8; margin: 12px 0 0; }
|
||||
.overall { display: flex; align-items: center; min-width: 150px; justify-content: center; padding: 13px 18px; border-radius: 999px; background: rgba(255,255,255,.82); box-shadow: 0 12px 30px rgba(36,64,69,.1); color: #647a82; }
|
||||
.overall.phase-ready, .overall.phase-active { color: #1a7d57; }
|
||||
.overall.phase-error { color: #b63d2e; }
|
||||
.overall.phase-completed { color: #1a7d57; }
|
||||
.hero-actions { display: grid; justify-items: end; gap: 10px; }
|
||||
.legacy-link { color: #617985; font-size: 12px; text-decoration: none; border-bottom: 1px solid #b9c8c7; padding-bottom: 2px; }
|
||||
.legacy-link:hover { color: #c8652a; }
|
||||
|
||||
.status-grid { display: grid; grid-template-columns: repeat(4, 1fr); gap: 14px; margin-bottom: 16px; }
|
||||
.status-card, .panel { background: rgba(255,255,255,.9); border: 1px solid rgba(23,61,69,.08); box-shadow: 0 14px 40px rgba(34,61,63,.08); }
|
||||
@@ -77,7 +82,27 @@ input:focus, select:focus { border-color: #d6783f; box-shadow: 0 0 0 3px rgba(21
|
||||
.transcript-turn strong { color: #315966; font-size: 12px; }
|
||||
.transcript-turn span { color: #263f49; font-size: 14px; line-height: 1.55; }
|
||||
|
||||
.hardware-shell { width: min(1260px, calc(100vw - 48px)); }
|
||||
.hardware-content-grid { grid-template-columns: 1.05fr 1.2fr; }
|
||||
.hardware-call-panel { min-height: 720px; }
|
||||
.hardware-session-info { grid-template-columns: 1fr 1fr; }
|
||||
.text-question { display: grid; grid-template-columns: 1fr 150px; gap: 10px; margin-bottom: 22px; }
|
||||
.text-question input { margin-top: 0; }
|
||||
.text-question .secondary { padding: 10px; }
|
||||
.flow-panel { margin-top: 16px; }
|
||||
.flow-panel .panel-heading { margin-bottom: 14px; }
|
||||
.flow-checks { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 10px; list-style: none; padding: 0; margin: 0; }
|
||||
.flow-check { display: grid; grid-template-columns: 28px 1fr; grid-template-rows: auto auto; column-gap: 10px; align-items: center; padding: 13px; border: 1px solid #e3ebe8; border-radius: 14px; background: #fafcfb; }
|
||||
.flow-check i { grid-row: 1 / span 2; display: grid; place-items: center; width: 26px; height: 26px; border-radius: 50%; font-style: normal; font-weight: 800; color: #809399; background: #e8efed; }
|
||||
.flow-check span { min-width: 0; color: #31505e; font-size: 13px; font-weight: 700; }
|
||||
.flow-check small { min-width: 0; margin-top: 4px; color: #8b9ba1; font-size: 11px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.flow-check--passed { border-color: #cce5d7; background: #f4faf6; }
|
||||
.flow-check--passed i { color: white; background: #2f8d67; }
|
||||
.flow-check--failed { border-color: #edcbc5; background: #fff7f5; }
|
||||
.flow-check--failed i { color: white; background: #b94c3d; }
|
||||
|
||||
@media (max-width: 1050px) {
|
||||
.status-grid { grid-template-columns: repeat(2, 1fr); }
|
||||
.content-grid { grid-template-columns: 1fr; }
|
||||
.flow-checks { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
const crypto = require("node:crypto");
|
||||
const { WebSocketServer } = require("ws");
|
||||
const { websocketSignature } = require("../electron/hardware-agent-client.cjs");
|
||||
|
||||
const port = Number(process.env.MOCK_HARDWARE_GATEWAY_PORT || 19090);
|
||||
const expectedSecret = process.env.MOCK_VOLC_DEVICE_SECRET || "mock-volc-device-secret-at-least-32-characters";
|
||||
const server = new WebSocketServer({ host: "127.0.0.1", port, maxPayload: 1024 * 1024 });
|
||||
|
||||
function send(socket, value) { socket.send(JSON.stringify(value)); }
|
||||
async function requestPlatformStop(conversationID) {
|
||||
const baseURL = String(process.env.MOCK_PLATFORM_API_URL || "").replace(/\/$/, "");
|
||||
const deviceID = String(process.env.MOCK_PLATFORM_DEVICE_ID || "");
|
||||
const secret = String(process.env.MOCK_PLATFORM_DEVICE_SECRET || "");
|
||||
if (!baseURL || !deviceID || !secret || !conversationID) return;
|
||||
const requestPath = `/device/v2/conversations/${encodeURIComponent(conversationID)}/stop`;
|
||||
const body = JSON.stringify({ reason: "windows_e2e_platform_stop", device_time: new Date().toISOString(), last_heartbeat_sequence: 1, rtc_left: true, recording: { near_segments: 0, far_segments: 0 } });
|
||||
const timestamp = new Date().toISOString();
|
||||
const nonce = crypto.randomBytes(16).toString("hex");
|
||||
const digest = crypto.createHash("sha256").update(body).digest("hex");
|
||||
const canonical = ["POST", requestPath, timestamp, nonce, digest].join("\n");
|
||||
const signature = crypto.createHmac("sha256", secret).update(canonical).digest("base64url");
|
||||
const response = await fetch(`${baseURL}${requestPath}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json", "Content-Type": "application/json", "Idempotency-Key": `windows-platform-stop-${crypto.randomUUID()}`,
|
||||
"X-Device-Id": deviceID, "X-Device-Timestamp": timestamp, "X-Device-Nonce": nonce,
|
||||
"X-Device-Content-SHA256": digest, "X-Device-Signature": signature,
|
||||
},
|
||||
body,
|
||||
});
|
||||
if (!response.ok) throw new Error(`platform stop HTTP ${response.status}: ${await response.text()}`);
|
||||
process.stdout.write(`${JSON.stringify({ type: "platform_stop_requested", conversation_id: conversationID })}\n`);
|
||||
}
|
||||
function tonePCMBase64() {
|
||||
const samples = 2400;
|
||||
const bytes = Buffer.alloc(samples * 2);
|
||||
for (let index = 0; index < samples; index += 1) {
|
||||
const sample = Math.round(Math.sin((index / 16000) * Math.PI * 2 * 440) * 1800);
|
||||
bytes.writeInt16LE(sample, index * 2);
|
||||
}
|
||||
return bytes.toString("base64");
|
||||
}
|
||||
|
||||
server.on("listening", () => process.stdout.write(`${JSON.stringify({ type: "listening", port })}\n`));
|
||||
server.on("connection", (socket, request) => {
|
||||
const header = (name) => String(request.headers[name.toLowerCase()] || "");
|
||||
const expectedSignature = websocketSignature({
|
||||
authType: 1,
|
||||
deviceName: header("x-device-name"),
|
||||
randomNumber: Number(header("x-random-num")),
|
||||
productKey: header("x-product-key"),
|
||||
timestamp: Number(header("x-timestamp")),
|
||||
instanceID: header("x-instance-id"),
|
||||
}, expectedSecret);
|
||||
if (!crypto.timingSafeEqual(Buffer.from(header("x-signature")), Buffer.from(expectedSignature))) {
|
||||
socket.close(1008, "invalid signature");
|
||||
return;
|
||||
}
|
||||
const target = new URL(request.url, `ws://127.0.0.1:${port}`);
|
||||
if (!target.searchParams.get("bot") || target.searchParams.get("wait_for_session_update") !== "true") {
|
||||
socket.close(1008, "invalid query");
|
||||
return;
|
||||
}
|
||||
process.stdout.write(`${JSON.stringify({ type: "authenticated", device: header("x-device-name"), bot: target.searchParams.get("bot") })}\n`);
|
||||
send(socket, { type: "session.created", event_id: "mock-created", session: { id: "mock-provider-session-windows", model: "doubao", input_audio_format: "pcm16", output_audio_format: "pcm16", turn_detection: { type: "server_vad" } } });
|
||||
let replied = false;
|
||||
let dialogID = "";
|
||||
socket.on("message", (raw) => {
|
||||
const event = JSON.parse(raw.toString("utf8"));
|
||||
if (event.type === "session.update") {
|
||||
const llmPrompt = event.session?.config?.LLMConfig?.SystemMessages?.[0];
|
||||
const s2sDialog = event.session?.config?.S2SConfig?.ProviderParams?.dialog;
|
||||
const sessionDialogID = String(s2sDialog?.dialog_id || (llmPrompt ? llmPrompt.match(/本次对话标识：([^\n]+)/)?.[1] : "") || "").trim();
|
||||
const frozenPrompt = String(llmPrompt || s2sDialog?.system_role || "").trim();
|
||||
if (event.session?.object !== "realtime.session" || event.session?.config?.ASRConfig?.TurnDetectionMode !== 0 || !sessionDialogID || !frozenPrompt) {
|
||||
send(socket, { type: "error", event_id: "mock-error", error: { code: "INVALID_SESSION_CONFIG", message: "frozen context missing" } });
|
||||
return;
|
||||
}
|
||||
dialogID = sessionDialogID;
|
||||
send(socket, { type: "session.updated", event_id: "mock-updated", session: { ...event.session, id: "mock-provider-session-windows" } });
|
||||
process.stdout.write(`${JSON.stringify({ type: "dynamic_config_applied", dialog_id: dialogID })}\n`);
|
||||
return;
|
||||
}
|
||||
if (event.type === "input_audio_buffer.append" && !replied) {
|
||||
replied = true;
|
||||
send(socket, { type: "input_audio_buffer.speech_started", event_id: "mock-speech-start", audio_start_ms: 0 });
|
||||
send(socket, { type: "input_audio_buffer.speech_stopped", event_id: "mock-speech-stop", audio_end_ms: 800 });
|
||||
send(socket, { type: "conversation.item.input_audio_transcription.completed", event_id: "mock-user-final", item_id: "mock-user-item", transcript: "你好，请介绍一下今天的安排。" });
|
||||
send(socket, { type: "response.created", event_id: "mock-response-created", response: { id: "mock-response-1", status: "in_progress" } });
|
||||
send(socket, { type: "response.audio_transcript.delta", event_id: "mock-agent-delta", response_id: "mock-response-1", delta: "您好，今天可以先聊聊您最想做的事情。" });
|
||||
send(socket, { type: "response.audio.delta", event_id: "mock-audio-delta", response_id: "mock-response-1", delta: tonePCMBase64() });
|
||||
send(socket, { type: "response.audio_transcript.done", event_id: "mock-agent-final", response_id: "mock-response-1", transcript: "您好，今天可以先聊聊您最想做的事情。" });
|
||||
send(socket, { type: "response.audio.done", event_id: "mock-audio-done", response_id: "mock-response-1" });
|
||||
send(socket, { type: "response.done", event_id: "mock-response-done", response: { id: "mock-response-1", status: "completed", output: [] } });
|
||||
process.stdout.write(`${JSON.stringify({ type: "round_completed" })}\n`);
|
||||
if (process.env.MOCK_REQUEST_PLATFORM_STOP === "1") {
|
||||
setTimeout(() => void requestPlatformStop(dialogID).catch((error) => {
|
||||
process.stderr.write(`${error.stack || error.message}\n`);
|
||||
}), 250);
|
||||
}
|
||||
}
|
||||
});
|
||||
socket.on("close", (code) => process.stdout.write(`${JSON.stringify({ type: "closed", code })}\n`));
|
||||
});
|
||||
|
||||
function shutdown() { server.close(() => process.exit(0)); }
|
||||
process.on("SIGINT", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
@@ -0,0 +1,120 @@
|
||||
param(
|
||||
[ValidateSet('llm', 's2s')]
|
||||
[string]$ModelMode = 'llm',
|
||||
[string]$ExecutablePath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$PSNativeCommandUseErrorActionPreference = $true
|
||||
|
||||
$simulatorRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..')).Path
|
||||
$projectRoot = (Resolve-Path -LiteralPath (Join-Path $simulatorRoot '..')).Path
|
||||
$devicePublicID = '10000000-0000-0000-0000-000000000511'
|
||||
$platformSecret = 'local-simulated-device-secret-32-bytes-minimum'
|
||||
$volcSecret = 'mock-volc-device-secret-at-least-32-characters'
|
||||
$mockPort = 19090
|
||||
$runID = [Guid]::NewGuid().ToString('N')
|
||||
$resultPath = Join-Path $env:TEMP "hardware-agent-e2e-$runID.json"
|
||||
$mockOutput = Join-Path $env:TEMP "hardware-agent-gateway-$runID.log"
|
||||
$mockError = Join-Path $env:TEMP "hardware-agent-gateway-$runID.err.log"
|
||||
$mockProcess = $null
|
||||
$conversationID = $null
|
||||
$deviceInternalID = $null
|
||||
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
try {
|
||||
$ready = Invoke-WebRequest -UseBasicParsing -Uri 'http://localhost:58080/health/ready' -TimeoutSec 10
|
||||
if ($ready.StatusCode -ne 200) { throw 'Local API is not ready' }
|
||||
$deviceInternalID = (docker compose exec -T postgres psql -U voice -d voice_platform -t -A -c "SELECT id FROM device WHERE public_id='$devicePublicID'::uuid" | Out-String).Trim()
|
||||
if ($deviceInternalID -notmatch '^[0-9a-fA-F-]{36}$') { throw 'Local simulator device is missing' }
|
||||
$bindingCount = (docker compose exec -T postgres psql -U voice -d voice_platform -t -A -c "SELECT count(*) FROM hardware_agent_device_binding WHERE device_id='$deviceInternalID'::uuid" | Out-String).Trim()
|
||||
if ($bindingCount -ne '0') { throw 'Simulator device already has a hardware binding; refusing to overwrite it' }
|
||||
"UPDATE device_twin SET desired_config=desired_config||'{`"voice_runtime`":`"volc_hardware_agent`",`"hardware_agent_transport`":`"low_load`",`"minimum_sdk_version`":`"1.0.0`"}'::jsonb WHERE device_id='$deviceInternalID'::uuid; INSERT INTO hardware_agent_device_binding(tenant_id,device_id,bot_id,transport,minimum_sdk_version,status) SELECT tenant_id,id,'mock-windows-hardware-bot','low_load','1.0.0','ACTIVE' FROM device WHERE id='$deviceInternalID'::uuid;" | docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q | Out-Null
|
||||
|
||||
$mockScript = Join-Path $simulatorRoot 'test\mock-hardware-gateway.cjs'
|
||||
$env:MOCK_HARDWARE_GATEWAY_PORT = [string]$mockPort
|
||||
$env:MOCK_VOLC_DEVICE_SECRET = $volcSecret
|
||||
$env:MOCK_REQUEST_PLATFORM_STOP = '1'
|
||||
$env:MOCK_PLATFORM_API_URL = 'http://localhost:58080'
|
||||
$env:MOCK_PLATFORM_DEVICE_ID = $devicePublicID
|
||||
$env:MOCK_PLATFORM_DEVICE_SECRET = $platformSecret
|
||||
$mockProcess = Start-Process -FilePath 'node' -ArgumentList @($mockScript) -PassThru -WindowStyle Hidden -RedirectStandardOutput $mockOutput -RedirectStandardError $mockError
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(10)
|
||||
do {
|
||||
Start-Sleep -Milliseconds 150
|
||||
$listening = Test-NetConnection 127.0.0.1 -Port $mockPort -InformationLevel Quiet -WarningAction SilentlyContinue
|
||||
if ($mockProcess.HasExited) { throw "Mock gateway exited: $(Get-Content -LiteralPath $mockError -Raw -ErrorAction SilentlyContinue)" }
|
||||
} until ($listening -or [DateTime]::UtcNow -gt $deadline)
|
||||
if (-not $listening) { throw 'Mock hardware gateway did not start' }
|
||||
|
||||
$env:SIMULATOR_AUTO_TEST = '1'
|
||||
$env:SIMULATOR_AUTO_WAIT_FOR_PLATFORM_STOP = '1'
|
||||
$env:SIMULATOR_AUTO_RESULT_PATH = $resultPath
|
||||
$env:SIMULATOR_DEFAULT_API_URL = 'http://localhost:58080'
|
||||
$env:SIMULATOR_DEFAULT_DEVICE_ID = $devicePublicID
|
||||
$env:SIMULATOR_DEFAULT_DEVICE_SECRET = $platformSecret
|
||||
$env:SIMULATOR_VOLC_INSTANCE_ID = 'mock-instance'
|
||||
$env:SIMULATOR_VOLC_PRODUCT_KEY = 'mock-product'
|
||||
$env:SIMULATOR_VOLC_DEVICE_NAME = 'windows-e2e-device'
|
||||
$env:SIMULATOR_VOLC_HARDWARE_ID = 'windows-e2e-hardware'
|
||||
$env:SIMULATOR_VOLC_DEVICE_SECRET = $volcSecret
|
||||
$env:SIMULATOR_VOLC_MODEL_MODE = $ModelMode
|
||||
$env:SIMULATOR_HARDWARE_GATEWAY_URL = "ws://127.0.0.1:$mockPort/v1/realtime"
|
||||
Set-Location -LiteralPath $simulatorRoot
|
||||
if ($ExecutablePath) {
|
||||
$resolvedExecutable = (Resolve-Path -LiteralPath $ExecutablePath).Path
|
||||
$electron = Start-Process -FilePath $resolvedExecutable -WorkingDirectory (Split-Path -Parent $resolvedExecutable) -PassThru -Wait
|
||||
} else {
|
||||
$electron = Start-Process -FilePath (Join-Path $simulatorRoot 'node_modules\electron\dist\electron.exe') -ArgumentList @('.') -WorkingDirectory $simulatorRoot -PassThru -Wait
|
||||
}
|
||||
if ($electron.ExitCode -ne 0) { throw "Electron E2E process failed with exit code $($electron.ExitCode)" }
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) { throw 'Electron E2E result file is missing' }
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
if ($result.result -ne 'PASS' -or $result.phase -ne '已完成') { throw "Electron E2E failed: $($result | ConvertTo-Json -Depth 8 -Compress)" }
|
||||
if ($result.model_mode -ne $ModelMode) { throw "Electron E2E used model mode '$($result.model_mode)' instead of '$ModelMode'" }
|
||||
$conversationID = [string]$result.conversation_id
|
||||
if ($conversationID -notmatch '^[0-9a-fA-F-]{36}$') { throw 'Electron E2E did not return a conversation ID' }
|
||||
$databaseState = (docker compose exec -T postgres psql -U voice -d voice_platform -t -A -F '|' -c "SELECT c.runtime_type||'|'||c.status||'|'||hs.status||'|'||hs.sdk_version||'|'||(hs.dynamic_config_applied_at IS NOT NULL)::text||'|'||count(DISTINCT ct.id)||'|'||(SELECT count(*) FROM web_event we WHERE we.resource_public_id=c.public_id AND we.event_type IN('conversation.started','conversation.completed')) FROM conversation c JOIN hardware_agent_session hs ON hs.conversation_id=c.id LEFT JOIN conversation_turn ct ON ct.conversation_id=c.id WHERE c.public_id='$conversationID'::uuid GROUP BY c.runtime_type,c.status,hs.status,hs.sdk_version,hs.dynamic_config_applied_at,c.public_id" | Out-String).Trim()
|
||||
if ($databaseState -ne 'volc_hardware_agent|COMPLETED|COMPLETED|1.1.0-windows-standard-protocol|true|2|2') { throw "Unexpected platform state: $databaseState" }
|
||||
$gatewayEvents = Get-Content -LiteralPath $mockOutput -Raw
|
||||
if ($gatewayEvents -notmatch 'dynamic_config_applied' -or $gatewayEvents -notmatch 'round_completed' -or $gatewayEvents -notmatch 'platform_stop_requested') { throw "Mock gateway did not observe config, media, and platform-stop flow: $gatewayEvents" }
|
||||
[ordered]@{
|
||||
result = 'PASS'
|
||||
conversation_id = $conversationID
|
||||
platform_state = $databaseState
|
||||
transcript_turns = $result.turns.Count
|
||||
checks_passed = @($result.checks | Where-Object status -eq 'passed').Count
|
||||
model_mode = $result.model_mode
|
||||
mock_gateway_log = $mockOutput
|
||||
} | ConvertTo-Json -Compress
|
||||
}
|
||||
finally {
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
if ($mockProcess -and -not $mockProcess.HasExited) { Stop-Process -Id $mockProcess.Id -Force -ErrorAction SilentlyContinue }
|
||||
if ($conversationID -match '^[0-9a-fA-F-]{36}$') {
|
||||
@"
|
||||
BEGIN;
|
||||
CREATE TEMP TABLE hc AS SELECT id,context_snapshot_id,consent_snapshot_id FROM conversation WHERE public_id='$conversationID'::uuid;
|
||||
DELETE FROM audit_event WHERE resource_id IN(SELECT id FROM hc);
|
||||
DELETE FROM web_event WHERE resource_public_id='$conversationID'::uuid;
|
||||
DELETE FROM idempotency_record WHERE resource_id IN(SELECT id FROM hc);
|
||||
DELETE FROM conversation_turn WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM conversation_memory_use WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM memory_tool_call WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM hardware_agent_event WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM hardware_agent_session WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM conversation_usage_budget WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM outbox_event WHERE aggregate_id IN(SELECT id FROM hc);
|
||||
DELETE FROM conversation_state_history WHERE conversation_id IN(SELECT id FROM hc);
|
||||
DELETE FROM conversation WHERE id IN(SELECT id FROM hc);
|
||||
DELETE FROM context_snapshot WHERE id IN(SELECT context_snapshot_id FROM hc);
|
||||
DELETE FROM consent_snapshot WHERE id IN(SELECT consent_snapshot_id FROM hc);
|
||||
COMMIT;
|
||||
"@ | docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q | Out-Null
|
||||
}
|
||||
if ($deviceInternalID -match '^[0-9a-fA-F-]{36}$') {
|
||||
"DELETE FROM hardware_agent_device_binding WHERE device_id='$deviceInternalID'::uuid; UPDATE device_twin SET desired_config=desired_config-'voice_runtime'-'hardware_agent_transport'-'minimum_sdk_version' WHERE device_id='$deviceInternalID'::uuid;" | docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q | Out-Null
|
||||
}
|
||||
@('MOCK_HARDWARE_GATEWAY_PORT','MOCK_VOLC_DEVICE_SECRET','MOCK_REQUEST_PLATFORM_STOP','MOCK_PLATFORM_API_URL','MOCK_PLATFORM_DEVICE_ID','MOCK_PLATFORM_DEVICE_SECRET','SIMULATOR_AUTO_TEST','SIMULATOR_AUTO_WAIT_FOR_PLATFORM_STOP','SIMULATOR_AUTO_RESULT_PATH','SIMULATOR_DEFAULT_API_URL','SIMULATOR_DEFAULT_DEVICE_ID','SIMULATOR_DEFAULT_DEVICE_SECRET','SIMULATOR_VOLC_INSTANCE_ID','SIMULATOR_VOLC_PRODUCT_KEY','SIMULATOR_VOLC_DEVICE_NAME','SIMULATOR_VOLC_HARDWARE_ID','SIMULATOR_VOLC_DEVICE_SECRET','SIMULATOR_VOLC_MODEL_MODE','SIMULATOR_HARDWARE_GATEWAY_URL') | ForEach-Object { Remove-Item "Env:$_" -ErrorAction SilentlyContinue }
|
||||
Remove-Item -LiteralPath $resultPath,$mockOutput,$mockError -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { defineConfig } from "vite";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
export default defineConfig({
|
||||
base: "./",
|
||||
@@ -6,5 +7,11 @@ export default defineConfig({
|
||||
outDir: "dist",
|
||||
emptyOutDir: true,
|
||||
sourcemap: true,
|
||||
rollupOptions: {
|
||||
input: {
|
||||
hardware: resolve(__dirname, "index.html"),
|
||||
legacy: resolve(__dirname, "legacy.html"),
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
云端语音测试设备使用说明
|
||||
云端硬件对话智能体测试程序使用说明
|
||||
|
||||
1. 将整个 ZIP 解压到普通文件夹，不要只从压缩包内直接运行。
|
||||
2. 双击“启动云端测试设备.cmd”，不要直接双击 EXE。
|
||||
3. 首次运行时允许 Windows 麦克风访问。
|
||||
4. 点击“检测平台和音频设备”，检测通过后即可开始语音聊天。
|
||||
1. 将 ZIP 完整解压到普通文件夹，不要直接在压缩包内运行。
|
||||
2. 双击“启动云端测试设备.cmd”，或运行 hardware-agent-tester-0.3.2-windows-x64.exe。
|
||||
3. 首次运行允许 Windows 麦克风访问。
|
||||
4. 填写平台设备信息和 Hardware ID；火山 Instance ID、Product Key、Device Name 由平台自动下发。
|
||||
5. Product Secret 只配置在平台密钥服务中，测试程序不会要求输入；平台代设备注册后只下发本设备的一机一密。
|
||||
6. 根据火山控制台中的 Bot 类型选择“通用 LLM”或“豆包端到端语音”。选错会导致 session.update 校验失败。
|
||||
7. 点击“检测、鉴权并准备设备”，13项验收区开始记录证据。
|
||||
8. 检测通过后点击“开始硬件智能体通话”。只有收到火山 session.updated 且平台确认 ACTIVE 后才会开放麦克风。
|
||||
9. 完成双方对话后点击结束；也可以从平台停止通话，程序会在 session_updated 回执中收到 stop_session 并自动结束。
|
||||
10. 点击“导出测试报告”，将 JSON 报告交给开发人员验收。
|
||||
|
||||
云端后台：https://121.41.46.238
|
||||
测试设备：10000000-0000-0000-0000-000000000511
|
||||
平台维护中心必须把本设备设置为 volc_hardware_agent + low_load，并绑定与火山产品关联的真实 Bot ID。公网平台尚未部署 v3 路由时会返回 404，此时不能进行真实测试。
|
||||
|
||||
网络需要允许 HTTPS 443、WSS 以及火山 RTC 所需的出站网络连接。
|
||||
网络需要允许 HTTPS 443 和 WSS 到：
|
||||
- 平台服务器
|
||||
- ai-gateway.vei.volces.com
|
||||
|
||||
如果检测平台时提示 RTC 网络检测失败，或入房超过 15 秒：
|
||||
- 关闭 VPN、系统代理和网络加速器；
|
||||
- 在 Windows 防火墙/杀毒软件中允许本程序联网；
|
||||
- 优先改用手机热点复测，以排除公司、校园或酒店网络限制；
|
||||
- 需要放行 TCP 443 到火山 RTC Web SDK 4.69.0 使用的 *.rtc.volcvideo.com 和 *.volcvideos.com。
|
||||
`iot-cn-shanghai.iot.volces.com`只需要平台服务器访问，设备无需访问动态注册地址。
|
||||
|
||||
如果手机热点、关闭VPN/代理后仍然无法入房，请在失败后点击页面底部“导出 RTC 日志”，把 Windows“下载”目录中最新生成的 RTC 日志文件交给开发人员分析。
|
||||
安全提示：Product Secret 永不到达设备。平台下发的一机一密会使用当前 Windows 用户的 Safe Storage 加密保存，导出的测试报告不包含任何密钥。
|
||||
|
||||
安全提示：本测试包包含服务器测试设备凭据，仅限受控测试使用，请勿公开上传或转发。测试完成后请删除其他电脑上的解压目录和 ZIP 文件。
|
||||
当前测试 EXE 未配置商业代码签名证书，Windows 可能提示“未知发布者”。请先用随包 SHA-256 文件核对完整性；正式外发版本应使用甲方或发布主体的代码签名证书签名。
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
| 记忆修订与停用 | 当前记忆、正文或状态、原因、强ETag | 正文变化创建新版本并立即使旧索引失效；停用后不再召回；重新启用必须重新校验授权并重新索引；`PENDING_DELETE/SUPERSEDED/DELETED`均禁止修改 | 新记忆版本或状态 |
|
||||
| 记忆永久删除 | 当前记忆、删除原因、强ETag | 事务内先进入`PENDING_DELETE`并清空活动向量，使新通话立即停止召回；该状态不可通过修改接口恢复；Outbox后台把索引标记删除、清除所有记忆版本正文并记录完成审计 | `DELETED`记忆和索引 |
|
||||
| 新通话记忆检索 | 服务端已确定的客户组织、长者和通话目标 | SQL同时限制租户、长者、当前授权、活动记忆、当前版本、普通敏感级别、有效期和活动索引；最多5条 | 冻结到`context_snapshot.memory_versions`的版本引用和最小背景摘要 |
|
||||
| 通话中记忆工具 | 供应商持有的单通话短期签名令牌、自然语言查询 | 令牌只绑定服务端既有通话和任务；租户、家庭和长者边界从数据库反查，模型不能传入或扩大；单条240字、总计800字、最多5条；Embedding失败返回空结果并标记降级，不中断通话 | `/internal/v1/tools/memory-search`、`memory_tool_call`和实际使用追踪 |
|
||||
| 通话中记忆工具 | 供应商持有的单通话短期签名令牌、自然语言查询 | 令牌只绑定服务端既有通话和RTC任务或硬件智能体会话；租户、家庭和长者边界从数据库反查，模型不能传入或扩大；单条240字、总计800字、最多5条；Embedding失败返回空结果并标记降级，不中断通话 | `/internal/v1/tools/memory-search`、`memory_tool_call`和实际使用追踪 |
|
||||
| 可选供应商原生记忆 | 当前人工确认记忆、有效授权、普通敏感级别、受控HTTP适配器 | 数据库始终是主数据；开启后用Outbox同步，修订先清理旧引用，停用、授权撤回、单条删除和主体删除均可靠删除并对账；关闭功能后通话继续走数据库检索 | `provider`索引的`PENDING/ACTIVE/FAILED/DELETED`状态 |
|
||||
| 内容质量反馈 | 故事、事实或记忆的人工确认、修改、拒绝、停用及九类标准问题 | 不复制私密正文；按生成模型和提示词版本聚合候选数、审核动作和问题类型；跨租户统一拒绝 | 维护端“内容质量”与`content_review_signal` |
|
||||
| 跨通话连续上下文 | 最近已完成通话的就绪摘要、当前授权快照、交流偏好 | 只有`story_extraction`和`personalized_memory`在新通话冻结快照中同时有效时，才按同租户同长者读取最近3次摘要；读取结果每条最多240字，供应商上下文按每条140字压缩且总计不超过1000字；撤回任一授权后不再注入 | 冻结到新通话`context_snapshot.sanitized_summary`的短期连续上下文 |
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# 聊天总结模型厂商接入
|
||||
|
||||
聊天总结任务不绑定某个模型。管理员在“大模型管理”中选择厂商、填写对应 API Key 与模型 ID、完成连接测试并激活后，Worker 会在下一条任务执行时读取当前激活配置。
|
||||
|
||||
## 已适配厂商
|
||||
|
||||
| 页面分类 | `provider` | 默认 Base URL | 需要填写 |
|
||||
|---|---|---|---|
|
||||
| 火山方舟 / 豆包 | `ark` | `https://ark.cn-beijing.volces.com/api/v3` | API Key、模型 ID 或 Endpoint ID |
|
||||
| 阿里云百炼 / 千问 | `dashscope` | `https://dashscope.aliyuncs.com/compatible-mode/v1` | 同地域 API Key、模型 ID |
|
||||
| DeepSeek | `deepseek` | `https://api.deepseek.com` | API Key、模型 ID |
|
||||
| 月之暗面 / Kimi | `moonshot` | `https://api.moonshot.cn/v1` | API Key、模型 ID |
|
||||
| 智谱 AI / GLM | `zhipu` | `https://open.bigmodel.cn/api/paas/v4` | API Key、模型 ID |
|
||||
| 百度千帆 / 文心 | `qianfan` | `https://qianfan.baidubce.com/v2` | Bearer Token、模型 ID |
|
||||
| 腾讯混元 | `hunyuan` | `https://api.hunyuan.cloud.tencent.com/v1` | API Key、模型 ID |
|
||||
| MiniMax | `minimax` | `https://api.minimaxi.com/v1` | API Key、模型 ID |
|
||||
| 百川智能 | `baichuan` | `https://api.baichuan-ai.com/v1` | API Key、模型 ID |
|
||||
| 硅基流动 | `siliconflow` | `https://api.siliconflow.cn/v1` | API Key、模型 ID |
|
||||
| 本地 Ollama | `ollama` | `http://ollama:11434/v1` | 本地模型 ID；密钥使用占位值 |
|
||||
| 自定义兼容服务 | `openai_compatible` | 管理员填写 | HTTPS Base URL、API Key、模型 ID |
|
||||
|
||||
所有云服务使用 Bearer Token 鉴权和 `/chat/completions` 请求。适配器只对官方已明确支持或项目已验证的厂商发送 `response_format`；其他厂商通过严格提示、JSON 提取和响应校验取得结构化结果，避免某些兼容接口因不认识 JSON Mode 参数而拒绝请求。带 `<think>` 思考包装的返回也会在解析前清理。
|
||||
|
||||
## 使用流程
|
||||
|
||||
1. 选择厂商，页面会预填官方 Base URL 和常用模型名。
|
||||
2. 按厂商控制台填写 API Key 与准确模型 ID。使用地域或业务空间专属域名时，可修改 Base URL。
|
||||
3. 保存后执行“测试连接”。测试内容是固定的无长者数据文本。
|
||||
4. 测试通过后设为当前模型。下一条聊天总结任务立即使用，不需要重启 Worker。
|
||||
|
||||
API Key 通过 AES-256-GCM 加密保存，读取接口仅返回末四位提示。连接测试、激活与移除均写入审计日志。
|
||||
@@ -0,0 +1,50 @@
|
||||
# 安心陪伴客户门户 v0.74 页面功能与设计基准
|
||||
|
||||
日期：2026-08-30
|
||||
设计方向：沿用已确认的“回忆时间轴”方案，以长者、聊天、心情与家庭协作为核心；暖白纸张底色、雾蓝主操作、陶土色提醒、低阴影和清晰留白。
|
||||
|
||||
## 全局信息架构
|
||||
|
||||
客户门户只保留家庭用户真正需要的 12 个一级任务。当前家庭用页首标签直接切换；家庭成员、角色和邀请恢复为独立的“家庭管理”页面。客户门户不再出现“客户组织”概念。
|
||||
|
||||
全局页面结构固定为：
|
||||
|
||||
1. 68px 顶部栏：品牌、全局搜索、提醒、账号。
|
||||
2. 228px 左侧导航：围绕长者陪伴任务排列。
|
||||
3. 统一页面首部：标题、用途说明、家庭权限状态。
|
||||
4. 统一家庭上下文栏：家庭 Tab、添加家庭、实时状态、刷新和页面主操作。
|
||||
5. 页面内容：不再使用包住整页的大卡片；只在独立对象或需要分组时使用边框容器。
|
||||
|
||||
## 逐页功能总结
|
||||
|
||||
| 页面 | 用户要解决的问题 | 主要功能 | 首屏重点 | 关键状态 |
|
||||
|---|---|---|---|---|
|
||||
| 首页 | 最近有没有陪伴、长者心情如何、哪里需要家人关注 | 长者切换、7 天聊天时长、最近聊天摘要、心情线索、关注提醒、最近动态 | 每位长者最近一次聊天和心情变化 | 无聊天、设备离线、待关注 |
|
||||
| 聊天记录 | 找到某次聊天并理解聊天内容 | 按长者/日期/状态筛选、AI 摘要、关键词、故事候选、下次话题、录音授权 | 时间轴和当前聊天 AI 总结 | 总结中、总结失败、无授权 |
|
||||
| 故事与记忆 | 审核模型整理的故事，形成可信长期记忆 | 故事候选审核、修订、确认/拒绝、记忆检索和停用 | 待审核故事与来源聊天 | 空状态、审核中、已确认 |
|
||||
| 长者资料 | 让豆包知道如何更适合地与这位长者聊天 | 头像、背景、兴趣、优先/回避话题、交流节奏、长期陪伴方向、授权 | 长者卡片和资料完整度 | 头像上传、只读、资料不完整 |
|
||||
| 日程与提醒 | 知道什么时候聊、哪些事情需要家人处理 | 陪伴计划、提醒时间、设备/故事/隐私待办、事件状态 | 今日安排和待办 | 已完成、即将开始、已错过 |
|
||||
| 家庭管理 | 管理家庭空间和可以协作的家人 | 家庭 Tab、成员、邀请、角色、所有权、家庭名称 | 家庭成员与角色 | 待接受邀请、只读、危险操作确认 |
|
||||
| 陪伴设备 | 确认设备能否正常陪伴 | 绑定、在线状态、固件、电量/音量、诊断、解除绑定 | 设备健康和所属长者 | 离线、待同步、诊断中 |
|
||||
| 内容分享 | 把指定故事/文字/录音安全分享给指定家人 | 资源选择、对象、权限、有效期、撤销 | 当前有效分享 | 无分享、即将过期、已撤销 |
|
||||
| 数据与授权 | 查看和控制长者数据如何被使用 | 五类分项授权、数据访问/更正/撤回/删除、私有导出 | 授权清单，不以按钮堆叠呈现 | 生效、处理中、已撤回 |
|
||||
| 服务用量 | 理解聊天使用情况，而不是看无意义设备统计 | 时长指标、活跃日、长者/设备筛选、热力图、最近通话 | 最近一年聊天活动 | 无数据、费用未核价 |
|
||||
| 大模型管理 | 选择哪个模型用于聊天总结 | 新增配置、连通性测试、激活、停用、参数和生成历史 | 当前激活模型及健康状态 | 未配置、测试失败、已激活 |
|
||||
| 设置与权限 | 理解当前账号能做什么 | 账号资料、家庭角色、15 项权限矩阵、敏感录音独立授权说明 | 当前身份和权限全表 | 无家庭、权限不足 |
|
||||
|
||||
## 统一视觉规格
|
||||
|
||||
- 桌面基准：1440 × 1024；内容最大宽度 1360px。
|
||||
- 页面左右 padding：桌面 28px；平板 20px；手机 14px。
|
||||
- 页面首部上下 padding：24px / 18px；与家庭上下文栏间距 16px。
|
||||
- 家庭上下文栏高度不固定，最小 54px；禁止通过空白占位制造层级。
|
||||
- 页面 section 间距 16px；对象卡片 padding 20–24px；列表行 14–16px。
|
||||
- 字号：页面标题 28px；区块标题 20px；正文 14px；辅助信息 12px。
|
||||
- 圆角：页面对象 14px、按钮/输入框 9px、状态胶囊 999px。
|
||||
- 主色 `#315D9B`；强调 `#C45F3B`；成功 `#5F8F69`；底色 `#FBFAF7`；边线 `#E6E0D8`。
|
||||
- 禁用按钮必须不可点击且鼠标为等待光标；加载按钮显示明确进行中动词。
|
||||
- 所有日期基于 Asia/Shanghai，设计数据锚点为 2026-08-30。
|
||||
|
||||
## 设计验收规则
|
||||
|
||||
每个页面必须在 1440 × 1024 同一视口完成“设计基准截图 / 实际页面截图”对照。P0–P2 的布局、padding、信息层级、文字、色彩、状态与响应式差异必须修复；最终记录写入项目根目录 `design-qa.md`。
|
||||
@@ -1,7 +1,7 @@
|
||||
# 设备 API 兼容性约定
|
||||
|
||||
> 当前设备接口主版本：`/device/v2`
|
||||
> 当前接口契约：OpenAPI 0.70.0
|
||||
> 当前接口契约：OpenAPI 0.77.0
|
||||
> 适用对象：云端 API、终端固件、模拟终端和设备联调工具
|
||||
|
||||
## 1. 兼容目标
|
||||
@@ -18,6 +18,10 @@
|
||||
|
||||
## 3. 必须升级到`/device/v3`的变更
|
||||
|
||||
当前`/device/v3/hardware-agent-credentials:provision`和`/device/v3/agent-sessions`承载火山硬件对话智能体控制面：前者由平台代设备动态注册且不下发Product Secret，后者在设备直连供应商音频前预取冻结的人设、记忆和授权配置，并回报会话事件与终稿。现有RTC/VoiceChat设备继续使用完整的`/device/v2`接口，平台不得将v3字段或状态语义反向强加给v2固件。
|
||||
|
||||
v3硬件会话必须在创建时冻结SDK版本、音频格式、可靠字幕、动态配置、凭证轮换和双路PCM录制能力，并以`session_updated`作为平台存活检查。`session_started`只有在设备证明本次`session.updated`已应用且Bot、传输、配置摘要匹配后才会激活。事件响应中的`action=stop_session`是强制控制动作：设备先断开供应商，再以新的单调序号回报结束或失败；`continue`才允许保持连接。低负载传输不具备供应商服务端状态回调，因此不得把设备事件和终稿上报能力声明为可选。硬件v3仍处于目标板卡接入期，本次新增必填能力字段不改变已经量产的v2契约。
|
||||
|
||||
以下任一变化都属于破坏性变更：删除或重命名字段、改变字段类型或业务含义、给既有请求增加必填字段、缩小已发布枚举导致旧值失效、改变签名算法或规范化规则、让旧固件收到无法安全忽略的新控制动作、改变既有状态机或重试/幂等语义。此类变更必须新增`/device/v3`，并保留v2迁移期和独立契约。
|
||||
|
||||
CI在拉取请求中以目标分支OpenAPI为基线执行破坏性变更检查。若确实需要破坏性变更，应先建立新主版本路径和迁移方案，不得通过修改或删除旧版契约绕过门禁。
|
||||
@@ -42,6 +46,7 @@ CI在拉取请求中以目标分支OpenAPI为基线执行破坏性变更检查
|
||||
|
||||
| OpenAPI版本 | 设备兼容变更 | 兼容结论 |
|
||||
|---|---|---|
|
||||
| 0.77.0 | 新增v3平台代注册端点；Windows硬件智能体客户端不再接收Product Secret，既有v2接口不变 | v2完全兼容；硬件v3客户端升级 |
|
||||
| 0.68.0 | 生产基础设施状态增加实际应用运行环境与身份服务模式证明；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 |
|
||||
| 0.67.0 | 发布候选快照增加生产基础设施强制门禁及主机部署模式、异地备份、外部告警、模拟服务状态；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 |
|
||||
| 0.66.0 | 发布候选技术验收快照增加主机健康当前性门禁；未修改设备端点、字段、状态机或终端行为 | v2完全兼容 |
|
||||
|
||||
@@ -0,0 +1,847 @@
|
||||
# 设备相关接口完整说明
|
||||
|
||||
> 版本：2026-09-05
|
||||
> 契约来源：`internal/httpapi/router.go`、实际处理器、`openapi/openapi.yaml`
|
||||
> 生产 Base URL：`https://121.41.46.238`
|
||||
> 适用对象：EchoEar/ESP32-S3 固件、Windows 模拟设备、用户 App/Web、平台运维端。
|
||||
|
||||
## 1. 文档范围
|
||||
|
||||
本文覆盖当前代码中全部直接或间接管理设备的接口：
|
||||
|
||||
| 接口面 | 方法数 | 鉴权 | 用途 |
|
||||
|---|---:|---|---|
|
||||
| 设备直连接口 `/device/v2`、`/device/v3` | 20 | 设备 HMAC | 启动、配置、命令、遥测、固件下载、会话、字幕、媒体、火山凭据 |
|
||||
| 用户设备管理 `/api/v1/devices*` | 10 | OIDC Bearer | 绑定、设备列表、在线状态、影子、命令 |
|
||||
| 运维设备管理 `/api/v1/ops/devices*` | 9 | OIDC Bearer + `X-Tenant-Id` | 登记、密钥轮换、绑定码、激活状态机 |
|
||||
| 家庭端与运维端固件接口 | 31 | OIDC Bearer | 签名密钥、版本、上传、灰度、暂停、回滚、下载 |
|
||||
| 设备诊断处置 | 2 | OIDC Bearer + `X-Tenant-Id` | 强制刷新配置、吊销设备 |
|
||||
|
||||
火山硬件智能体 WebSocket 不是平台 HTTP API，但设备聊天必须使用，故在第 10 节单独说明。
|
||||
|
||||
## 2. 通用约定
|
||||
|
||||
### 2.1 平台响应包络
|
||||
|
||||
成功：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-01J...",
|
||||
"data": {}
|
||||
}
|
||||
```
|
||||
|
||||
失败：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-01J...",
|
||||
"error": {
|
||||
"code": "VALIDATION_FAILED",
|
||||
"message": "请求参数不正确",
|
||||
"retryable": false,
|
||||
"field_errors": [{ "field": "hardware_id", "message": "不能为空" }],
|
||||
"details": {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2.2 ID、时间和分页
|
||||
|
||||
- `{id}`、`device_id`、`conversation_id`、`household_id`、`subject_id` 均为 UUID。
|
||||
- 时间均为 RFC3339 UTC，例如 `2026-09-05T06:30:00Z`。
|
||||
- `Idempotency-Key` 长度为 16—128 字符；业务重试保留原正文和原幂等键。
|
||||
- `If-Match` 使用接口返回的强 ETag，例如 `"7"`。
|
||||
- 密钥、Token、绑定码和下载地址不得写入普通日志。
|
||||
|
||||
### 2.3 设备 HMAC 请求头
|
||||
|
||||
所有 `/device/v2/*` 和 `/device/v3/*` 请求必须携带：
|
||||
|
||||
| Header | 输入说明 |
|
||||
|---|---|
|
||||
| `X-Device-Id` | 平台设备公开 UUID |
|
||||
| `X-Device-Timestamp` | RFC3339 时间，允许时钟偏差默认 ±5 分钟 |
|
||||
| `X-Device-Nonce` | 16—128 字符，一次请求只能使用一次 |
|
||||
| `X-Device-Content-SHA256` | 原始请求正文 SHA-256 小写十六进制；空正文也计算 |
|
||||
| `X-Device-Signature` | HMAC-SHA256 的 Base64URL 无填充编码 |
|
||||
|
||||
签名原文：
|
||||
|
||||
```text
|
||||
HTTP_METHOD大写\n
|
||||
完整REQUEST_URI（含查询字符串）\n
|
||||
X-Device-Timestamp\n
|
||||
X-Device-Nonce\n
|
||||
X-Device-Content-SHA256
|
||||
```
|
||||
|
||||
## 3. 设备启动、配置、命令和遥测
|
||||
|
||||
### 3.1 GET `/device/v2/bootstrap`
|
||||
|
||||
介绍：设备上电后报告固件和硬件信息，取得平台权威设备状态、绑定信息、配置版本和服务器时间。
|
||||
|
||||
输入：
|
||||
|
||||
| 位置 | 字段 | 必填 | 说明 |
|
||||
|---|---|---:|---|
|
||||
| Query | `firmware_version` | 是 | 当前固件版本，最长 80 |
|
||||
| Query | `device_time` | 是 | RFC3339 设备时间 |
|
||||
| Query | `board_rev` | 否 | 硬件版本 |
|
||||
| Query | `rtc_sdk_version` | 否 | SDK 版本 |
|
||||
| Query | `codec` | 否 | 编码，例如 `pcm_s16le` |
|
||||
| Query | `sample_rate` | 否 | 采样率，例如 `16000` |
|
||||
| Query | `network` | 否 | 网络类型，例如 `wifi` |
|
||||
|
||||
输出：`server_time`、`clock_skew_ms`、设备状态、最低版本、兼容性、`config_revision` 和绑定对象。
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
GET /device/v2/bootstrap?firmware_version=1.0.0&board_rev=echoear-v1&codec=pcm_s16le&sample_rate=16000&device_time=2026-09-05T06%3A30%3A00Z&network=wifi
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-1",
|
||||
"data": {
|
||||
"server_time": "2026-09-05T06:30:01Z",
|
||||
"clock_skew_ms": 1000,
|
||||
"device": {
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"status": "registered",
|
||||
"model": "echoear",
|
||||
"firmware_compatible": true,
|
||||
"config_revision": 7,
|
||||
"binding": { "bound": true, "household_id": "11111111-1111-1111-1111-111111111111", "subject_id": "22222222-2222-2222-2222-222222222222", "subject_name": "张大妈" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.2 GET `/device/v2/config`
|
||||
|
||||
介绍：读取云端期望配置和当前绑定。设备保存响应 `ETag`，后续用 `If-None-Match` 增量检查。
|
||||
|
||||
输入：无正文；可选 Header `If-None-Match: "7"`。
|
||||
|
||||
输出：HTTP `200` 返回 `device_id`、`revision`、`desired_config`、`reported_revision`、`binding`；配置未变化返回 `304` 且无正文。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-2",
|
||||
"data": {
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"revision": 8,
|
||||
"desired_config": { "voice_runtime": "volc_hardware_agent", "volume": { "speaker": 70 } },
|
||||
"reported_revision": 7,
|
||||
"binding": { "bound": true, "subject_id": "22222222-2222-2222-2222-222222222222" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.3 GET `/device/v2/commands`
|
||||
|
||||
介绍：拉取未过期的待执行命令，并增加投递次数。
|
||||
|
||||
输入：Query `limit`，范围 1—20，默认 10。
|
||||
|
||||
输出：`items[]`；每项包含 `id`、`type`、`payload`、`status`、`expires_at`、`delivery_attempts`、`created_at`。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-3",
|
||||
"data": {
|
||||
"items": [{
|
||||
"id": "33333333-3333-3333-3333-333333333333",
|
||||
"type": "apply_config",
|
||||
"payload": { "revision": 8 },
|
||||
"status": "DELIVERED",
|
||||
"expires_at": "2026-09-05T07:30:00Z",
|
||||
"delivery_attempts": 1,
|
||||
"created_at": "2026-09-05T06:30:00Z"
|
||||
}]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.4 POST `/device/v2/commands/{id}/ack`
|
||||
|
||||
介绍：设备回报命令已接收、成功或失败。
|
||||
|
||||
输入：`status` 为 `ACKNOWLEDGED`、`SUCCEEDED`、`FAILED`；`result_summary` 只能包含有限诊断字段，不得含 Secret/Token。
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "SUCCEEDED",
|
||||
"result_summary": { "applied_revision": 8 }
|
||||
}
|
||||
```
|
||||
|
||||
输出：更新后的命令投递对象。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-4",
|
||||
"data": { "id": "33333333-3333-3333-3333-333333333333", "type": "apply_config", "status": "SUCCEEDED", "result_summary": { "applied_revision": 8 } }
|
||||
}
|
||||
```
|
||||
|
||||
### 3.5 POST `/device/v2/telemetry:batch`
|
||||
|
||||
介绍：批量上报状态、脱敏错误或命令结果；按 `event_key` 和 `sequence_no` 去重，并返回平台动作。
|
||||
|
||||
输入：`events` 为 1—500 项；`event_type` 为 `status`、`error` 或 `command_result`，且只携带对应同名对象。
|
||||
|
||||
```json
|
||||
{
|
||||
"events": [{
|
||||
"event_key": "boot.000008",
|
||||
"sequence_no": 8,
|
||||
"occurred_at": "2026-09-05T06:31:00Z",
|
||||
"event_type": "status",
|
||||
"status": {
|
||||
"firmware_version": "1.0.0",
|
||||
"reported_config_revision": 8,
|
||||
"network": { "type": "wifi", "rssi": -51 },
|
||||
"rtc": { "joined": false, "uplink": false, "downlink": false },
|
||||
"recording_queue": { "pending_segments": 0, "failed_segments": 0 }
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
输出：接收/重复数、最新期望配置版本和动作列表。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-5",
|
||||
"data": { "accepted_count": 1, "duplicate_count": 0, "desired_config_revision": 8, "actions": [{ "type": "pull_commands" }] }
|
||||
}
|
||||
```
|
||||
|
||||
## 4. 火山硬件智能体平台控制面
|
||||
|
||||
### 4.1 POST `/device/v3/hardware-agent-credentials:provision`
|
||||
|
||||
介绍：平台使用 Product Secret 代设备调用火山动态注册，只向当前设备返回它自己的一机一密。允许状态：`BOUND`、`PROVISIONING`、`ACTIVE`。
|
||||
|
||||
输入：Header `Idempotency-Key`；正文只包含稳定 `hardware_id`。
|
||||
|
||||
```json
|
||||
{ "hardware_id": "echoear-b43a451634a8" }
|
||||
```
|
||||
|
||||
输出：`instance_id`、`product_key`、`device_name`、`hardware_id`、一次性敏感的 `device_secret`、版本和来源。响应必须 `Cache-Control: no-store`。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-6",
|
||||
"data": {
|
||||
"instance_id": "6a92945cc0c4689234aef656",
|
||||
"product_key": "6a9568047dd4b23eacab9b1a",
|
||||
"device_name": "echoear-100000000512",
|
||||
"hardware_id": "echoear-b43a451634a8",
|
||||
"device_secret": "<仅保存到设备安全区>",
|
||||
"credential_ref": "cred-01J...",
|
||||
"credential_version": 1,
|
||||
"credential_source": "platform_dynamic_registration",
|
||||
"issued_at": "2026-09-05T06:32:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 POST `/device/v3/agent-sessions`
|
||||
|
||||
介绍：每次聊天前创建一次平台会话，冻结本次 Bot、授权、人设、记忆、隐私策略和会话证明。仅 `ACTIVE` 设备可调用。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{
|
||||
"device_time": "2026-09-05T06:35:00Z",
|
||||
"monotonic_ms": 123456,
|
||||
"capabilities": {
|
||||
"hardware_agent": true,
|
||||
"sdk_version": "1.1.0-echoear",
|
||||
"transports": ["low_load"],
|
||||
"input_audio_format": "pcm_s16le_16000_mono",
|
||||
"output_audio_format": "pcm_s16le_16000_mono",
|
||||
"server_event_callback": false,
|
||||
"transcript_upload": true,
|
||||
"dynamic_session_config": true,
|
||||
"credential_rotation": true,
|
||||
"near_pcm_recording": false,
|
||||
"far_pcm_recording": false
|
||||
},
|
||||
"network": { "type": "wifi", "rssi": -52 },
|
||||
"requested_goal": "open_conversation",
|
||||
"config_revision": 8
|
||||
}
|
||||
```
|
||||
|
||||
输出：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-7",
|
||||
"data": {
|
||||
"conversation_id": "44444444-4444-4444-4444-444444444444",
|
||||
"status": "SESSION_READY",
|
||||
"runtime_type": "volc_hardware_agent",
|
||||
"runtime": {
|
||||
"provider": "volcengine",
|
||||
"bot_id": "botM4gWQ9A9y",
|
||||
"transport": "low_load",
|
||||
"provider_session_ref": "provider-ref",
|
||||
"session_nonce": "<仅内存保存>",
|
||||
"config_digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"wait_for_session_update": true
|
||||
},
|
||||
"session_config": {
|
||||
"dialog_id": "55555555-5555-5555-5555-555555555555",
|
||||
"system_prompt": "<本次人设>",
|
||||
"persona_tone": "亲切、自然",
|
||||
"authorized_context": "<本次授权背景与记忆>",
|
||||
"prompt_bundle_version": "minimal-v2"
|
||||
},
|
||||
"privacy": { "retention_mode": "transcript_only", "record_near": false, "record_far": false, "upload_allowed": false },
|
||||
"session": { "engine_mode": "s2s_hybrid", "heartbeat_interval_sec": 15, "allocation_expires_at": "2026-09-05T06:40:00Z" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4.3 POST `/device/v3/agent-sessions/{id}/events`
|
||||
|
||||
介绍：设备回流 `session_started`、`session_updated`、`session_ended`、`session_failed`。`sequence_no` 严格递增。
|
||||
|
||||
开始事件输入：
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "evt-start-1",
|
||||
"sequence_no": 1,
|
||||
"event_type": "session_started",
|
||||
"occurred_at": "2026-09-05T06:35:03Z",
|
||||
"provider_session_ref": "provider-ref",
|
||||
"session_nonce": "<创建会话返回值>",
|
||||
"provider_connection_ref": "event_LUVjX_wvR",
|
||||
"bot_id": "botM4gWQ9A9y",
|
||||
"transport": "low_load",
|
||||
"config_digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"dynamic_config_applied": true
|
||||
}
|
||||
```
|
||||
|
||||
输出：`status`、平台动作、是否重复和接收时间。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-8",
|
||||
"data": { "conversation_id": "44444444-4444-4444-4444-444444444444", "status": "ACTIVE", "action": "continue", "duplicate": false, "accepted_at": "2026-09-05T06:35:03Z" }
|
||||
}
|
||||
```
|
||||
|
||||
`action=stop_session` 时必须立即停麦、断开火山，再上报 `session_ended` 或 `session_failed`。
|
||||
|
||||
### 4.4 POST `/device/v3/agent-sessions/{id}/turns:batch`
|
||||
|
||||
介绍：可靠上传最终字幕；禁止上传增量字幕。单批 1—500 条。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{
|
||||
"turns": [{
|
||||
"turn_key": "user-final-1",
|
||||
"sequence_no": 1,
|
||||
"speaker": "subject",
|
||||
"start_at": "2026-09-05T06:35:10Z",
|
||||
"end_at": "2026-09-05T06:35:12Z",
|
||||
"final_text": "你好。",
|
||||
"interrupted": false
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-9", "data": { "conversation_id": "44444444-4444-4444-4444-444444444444", "accepted_count": 1, "duplicate_count": 0 } }
|
||||
```
|
||||
|
||||
## 5. 旧 RTC 会话兼容接口
|
||||
|
||||
新 EchoEar 使用硬件智能体 v3，不应调用本节的创建/激活接口。本节仅供存量 RTC 设备。
|
||||
|
||||
### 5.1 POST `/device/v2/conversations`
|
||||
|
||||
介绍：分配 RTC 房间和短期 Token，不启动智能体。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{
|
||||
"device_time": "2026-09-05T06:40:00Z",
|
||||
"monotonic_ms": 130000,
|
||||
"capabilities": { "rtc": true, "audio_tracks": ["near", "far"], "upload_resume": true, "rtc_credential_renewal": true },
|
||||
"network": { "type": "wifi", "rssi": -50 },
|
||||
"requested_goal": "open_conversation"
|
||||
}
|
||||
```
|
||||
|
||||
输出：`conversation_id`、`ROOM_ISSUED`、RTC 房间/用户/Token、隐私策略和心跳配置。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-10",
|
||||
"data": {
|
||||
"conversation_id": "66666666-6666-6666-6666-666666666666",
|
||||
"status": "ROOM_ISSUED",
|
||||
"rtc": { "app_id": "app", "room_id": "room", "user_id": "device-user", "agent_user_id": "agent-user", "token": "<仅内存>", "expires_at": "2026-09-05T06:45:00Z" },
|
||||
"privacy": { "retention_mode": "none", "record_near": false, "record_far": false, "upload_allowed": false },
|
||||
"session": { "engine_mode": "cascade", "heartbeat_interval_sec": 10, "allocation_expires_at": "2026-09-05T06:45:00Z" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 5.2 POST `/device/v2/conversations/{id}/activate`
|
||||
|
||||
介绍：设备确认已成功进入指定 RTC 房间后，触发平台启动旧语音任务。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{ "joined_at": "2026-09-05T06:40:02Z", "room_id": "room", "rtc_user_id": "device-user", "rtc_join_result": "success", "config_revision": 8, "connection_reference": "rtc-conn-1" }
|
||||
```
|
||||
|
||||
输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-11", "data": { "conversation_id": "66666666-6666-6666-6666-666666666666", "status": "STARTING_AGENT", "accepted_at": "2026-09-05T06:40:02Z" } }
|
||||
```
|
||||
|
||||
### 5.3 GET `/device/v2/conversations/{id}`
|
||||
|
||||
介绍：读取平台权威会话状态和设备下一步动作。
|
||||
|
||||
输入：Path `id`。
|
||||
|
||||
输出示例：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-12", "data": { "conversation_id": "66666666-6666-6666-6666-666666666666", "status": "ACTIVE", "action": "continue" } }
|
||||
```
|
||||
|
||||
失败时可能返回 `status=FAILED`、`action=leave_room`、稳定脱敏的 `failure_code`。
|
||||
|
||||
### 5.4 POST `/device/v2/conversations/{id}/heartbeat`
|
||||
|
||||
介绍：刷新会话租约、设备在线状态和配置状态；返回停止、续签或配置动作。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{
|
||||
"sequence_no": 2,
|
||||
"device_time": "2026-09-05T06:40:12Z",
|
||||
"monotonic_ms": 142000,
|
||||
"conversation_status": "ACTIVE",
|
||||
"rtc": { "joined": true, "uplink": true, "downlink": true },
|
||||
"network": { "type": "wifi", "rssi": -49 },
|
||||
"firmware_version": "1.0.0",
|
||||
"reported_config_revision": 8,
|
||||
"recording_queue": { "pending_segments": 0, "failed_segments": 0 },
|
||||
"command_results": []
|
||||
}
|
||||
```
|
||||
|
||||
输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-13", "data": { "lease_expires_at": "2026-09-05T06:40:42Z", "actions": [{ "type": "continue" }], "desired_config_revision": 8, "commands": [] } }
|
||||
```
|
||||
|
||||
### 5.5 POST `/device/v2/conversations/{id}/stop`
|
||||
|
||||
介绍：设备挂断并可靠停止旧供应商任务；支持幂等重放。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{ "reason": "user_hangup", "device_time": "2026-09-05T06:42:00Z", "last_heartbeat_sequence": 5, "rtc_left": true, "recording": { "near_segments": 0, "far_segments": 0 } }
|
||||
```
|
||||
|
||||
输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-14", "data": { "conversation_id": "66666666-6666-6666-6666-666666666666", "status": "STOPPING", "accepted_at": "2026-09-05T06:42:00Z" } }
|
||||
```
|
||||
|
||||
### 5.6 POST `/device/v2/conversations/{id}/turns:batch`
|
||||
|
||||
介绍：旧 RTC 回调缺少终稿时，由设备补传最终字幕。输入输出与 4.4 完全相同。
|
||||
|
||||
```json
|
||||
{ "turns": [{ "turn_key": "agent-final-2", "sequence_no": 2, "speaker": "agent", "start_at": "2026-09-05T06:41:00Z", "end_at": "2026-09-05T06:41:02Z", "final_text": "您好。", "interrupted": false }] }
|
||||
```
|
||||
|
||||
## 6. 设备录音上传接口
|
||||
|
||||
仅当本次 `privacy.upload_allowed=true` 且对应轨道获授权时使用。
|
||||
|
||||
### 6.1 POST `/device/v2/conversations/{id}/upload-batches`
|
||||
|
||||
介绍：声明待上传录音分片，平台返回每个对象的短期 PUT 地址；业务 API 不接收音频字节。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{
|
||||
"segments": [{
|
||||
"track": "near",
|
||||
"sequence_no": 0,
|
||||
"codec": "wav_pcm_s16le",
|
||||
"content_type": "audio/wav",
|
||||
"duration_ms": 10000,
|
||||
"expected_size": 320044,
|
||||
"expected_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
输出：批次状态和分片；创建响应中的分片包含 `upload_url`、`upload_until`。
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-15",
|
||||
"data": {
|
||||
"id": "77777777-7777-7777-7777-777777777777",
|
||||
"conversation_id": "66666666-6666-6666-6666-666666666666",
|
||||
"status": "UPLOADING",
|
||||
"segment_count": 1,
|
||||
"expected_total_size": 320044,
|
||||
"verified_count": 0,
|
||||
"rejected_count": 0,
|
||||
"verification_attempts": 0,
|
||||
"expires_at": "2026-09-05T07:02:00Z",
|
||||
"segments": [{ "id": "88888888-8888-8888-8888-888888888888", "track": "near", "sequence_no": 0, "status": "AUTHORIZED", "upload_url": "https://object.example/put?...", "upload_until": "2026-09-05T07:02:00Z" }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 6.2 POST `/device/v2/upload-batches/{id}/complete`
|
||||
|
||||
介绍：声明分片已上传，平台开始异步对象大小、摘要和容器校验。
|
||||
|
||||
输入：无正文；Path `id` 为上传批次 UUID。
|
||||
|
||||
输出：HTTP `202`，返回批次，`status` 通常为 `VERIFYING`。
|
||||
|
||||
```json
|
||||
{ "request_id": "req-16", "data": { "id": "77777777-7777-7777-7777-777777777777", "status": "VERIFYING", "segment_count": 1, "verified_count": 0, "rejected_count": 0, "segments": [] } }
|
||||
```
|
||||
|
||||
### 6.3 GET `/device/v2/upload-batches/{id}`
|
||||
|
||||
介绍：轮询后台校验结果。只有 `VERIFIED` 分片返回 `local_delete_after`。
|
||||
|
||||
输入：Path `id`。
|
||||
|
||||
输出示例：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-17", "data": { "id": "77777777-7777-7777-7777-777777777777", "status": "COMPLETED", "segment_count": 1, "verified_count": 1, "rejected_count": 0, "segments": [{ "id": "88888888-8888-8888-8888-888888888888", "status": "VERIFIED", "local_delete_after": "2026-09-05T07:10:00Z" }] } }
|
||||
```
|
||||
|
||||
### 6.4 POST `/device/v2/media-segments/{id}/local-delete-ack`
|
||||
|
||||
介绍：到达平台允许删除时间后，设备删除本地密文并回报；不会删除云端对象。
|
||||
|
||||
输入：
|
||||
|
||||
```json
|
||||
{ "status": "DELETED" }
|
||||
```
|
||||
|
||||
失败示例：`{ "status": "FAILED", "error_code": "FLASH_DELETE_FAILED" }`。
|
||||
|
||||
输出：更新后的分片对象，包含 `local_delete_status` 和 `local_delete_reported_at`。
|
||||
|
||||
## 7. 设备固件下载接口
|
||||
|
||||
### GET `/device/v2/firmware-releases/{id}/download`
|
||||
|
||||
介绍：仅向当前灰度批次中的目标设备签发 5 分钟私有下载地址。
|
||||
|
||||
输入：Path `id` 为固件版本 UUID，无正文。
|
||||
|
||||
输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-18", "data": { "download_url": "https://object.example/firmware.bin?...", "download_until": "2026-09-05T07:05:00Z" } }
|
||||
```
|
||||
|
||||
非目标设备或暂停/退役版本返回 `403 FIRMWARE_DOWNLOAD_DENIED`。
|
||||
|
||||
## 8. 用户 App/Web 设备管理接口
|
||||
|
||||
统一输入 Header：`Authorization: Bearer <OIDC access token>`。以下表中“示例输入/输出”均省略通用包络字段。
|
||||
|
||||
| 方法与路径 | 接口介绍 | 输入 | 输出 | 示例 |
|
||||
|---|---|---|---|---|
|
||||
| `POST /api/v1/device-claims` | 消费一次性绑定码 | Body：`claim_code`、`household_id`、`subject_id` | `binding_id`、`device_id`、归属、`bound_at` | 入：`{"claim_code":"CLAIM-...","household_id":"<uuid>","subject_id":"<uuid>"}`；出：`{"binding_id":"<uuid>","device_id":"<uuid>","bound_at":"2026-09-05T07:00:00Z"}` |
|
||||
| `GET /api/v1/devices` | 列出当前账号可管理设备 | 无 | `items: ManagedDevice[]` | `{"items":[{"id":"<uuid>","model":"echoear","status":"registered","desired_revision":8}]}` |
|
||||
| `GET /api/v1/devices/{id}` | 查询设备与绑定摘要 | Path：设备 UUID | `ManagedDevice` | `{"id":"<uuid>","binding_id":"<uuid>","subject_name":"张大妈","firmware_version":"1.0.0","version":3}` |
|
||||
| `GET /api/v1/devices/{id}/binding-impact` | 解绑/转移前检查阻断项 | Query：`action=unbind\|transfer`；转移时 `target_subject_id` | 影响、`allowed`、`blockers`、当前 `binding_id` | `{"action":"transfer","allowed":true,"pending_command_count":0,"active_firmware_task_count":0}` |
|
||||
| `POST /api/v1/devices/{id}/binding-change` | 执行解绑或转移 | Body：`action`、`expected_binding_id`、`reason`、`acknowledged=true`；转移时 `target_subject_id` | 旧/新绑定和变更时间 | 入：`{"action":"unbind","expected_binding_id":"<uuid>","reason":"用户解除绑定","acknowledged":true}`；出：`{"device_id":"<uuid>","action":"unbind","changed_at":"..."}` |
|
||||
| `GET /api/v1/devices/{id}/presence` | 查询实时在线状态 | Path：设备 UUID | `online`、`status`、`last_seen_at`、`source` | `{"device_id":"<uuid>","online":true,"status":"ACTIVE","last_seen_at":"...","source":"redis"}` |
|
||||
| `GET /api/v1/devices/{id}/twin` | 查询设备影子 | Path：设备 UUID | 期望/上报配置、版本、能力、同步状态；Header `ETag` | `{"device_id":"<uuid>","desired_revision":8,"desired_config":{"volume":{"speaker":70}},"sync_state":"applied"}` |
|
||||
| `PATCH /api/v1/devices/{id}/twin` | 修改非敏感期望配置 | Header：`If-Match`；Body：`desired_config`，顶层仅 `volume/interaction/upload/rtc/policy_refs` | 更新后的 `DeviceTwin` 和新 `ETag` | `{"desired_config":{"volume":{"speaker":60}}}` |
|
||||
| `GET /api/v1/devices/{id}/commands` | 查询最近 100 条命令 | Path：设备 UUID | `items: DeviceCommand[]` | `{"items":[{"id":"<uuid>","type":"reboot","status":"SUCCEEDED"}]}` |
|
||||
| `POST /api/v1/devices/{id}/commands` | 下发可靠命令 | Header：`Idempotency-Key`；Body：`type`、`payload`、`expires_in_sec` | `DeviceCommand`，首次 `202`，重放 `200` | 入：`{"type":"reboot","payload":{"reason":"maintenance"},"expires_in_sec":300}`；出：`{"id":"<uuid>","status":"QUEUED","delivery_attempts":0}` |
|
||||
|
||||
`ManagedDevice` 主要输出字段：`id`、`binding_id`、`household_id`、`subject_id`、`subject_name`、`model`、`hardware_revision`、`status`、固件/配置版本、`last_reported_at`、`bound_at`、`registered_at`、`version`。
|
||||
|
||||
## 9. 平台运维设备管理接口
|
||||
|
||||
统一输入 Header：`Authorization: Bearer <operator token>`、`X-Tenant-Id: <tenant UUID>`。只有独立运维账号可调用。
|
||||
|
||||
| 方法与路径 | 接口介绍 | 输入 | 输出 | 示例 |
|
||||
|---|---|---|---|---|
|
||||
| `GET /api/v1/ops/devices` | 查询租户全部设备 | 无 | `items: OpsDevice[]`，不返回密钥 | `{"items":[{"id":"<uuid>","model":"echoear","activation_state":"ACTIVE","credential_status":"active","voice_runtime":"volc_hardware_agent"}]}` |
|
||||
| `POST /api/v1/ops/devices` | 登记新设备并生成平台 HMAC 密钥 | Body：`model` 必填；可选硬件/最低版本/归属/运行时/初始激活状态 | `device`、只显示一次的 `credential`；可能同时返回 `claim` | 入：`{"model":"echoear","hardware_revision":"esp32-s3","voice_runtime":"volc_hardware_agent","activation_state":"FACTORY"}`；出：`{"device":{"id":"<uuid>"},"credential":{"device_id":"<uuid>","secret":"<64位hex>","key_version":1}}` |
|
||||
| `GET /api/v1/ops/device-assignment-options` | 查询可分配家庭和长者 | 无 | `items[]` 含家庭/长者 ID 和名称 | `{"items":[{"household_id":"<uuid>","household_name":"张家","subject_id":"<uuid>","subject_name":"张大妈"}]}` |
|
||||
| `PATCH /api/v1/ops/devices/{id}` | 更新设备型号、版本策略、状态、归属或运行时 | Header：`If-Match`；Body 至少一个可修改字段 | 更新后的 `OpsDevice` | `{"minimum_firmware_version":"1.1.0","status":"registered"}` |
|
||||
| `DELETE /api/v1/ops/devices/{id}` | 软删除并吊销设备/凭据/绑定 | Header：`If-Match` | 吊销后的 `OpsDevice` | `{"id":"<uuid>","status":"revoked","activation_state":"REVOKED","credential_status":"revoked"}` |
|
||||
| `POST /api/v1/ops/devices/{id}/credentials/rotate` | 轮换平台 HMAC 密钥，旧密钥立即失效 | 无正文 | 只显示一次的新 `secret`、`key_version`、`issued_at` | `{"device_id":"<uuid>","secret":"<64位hex>","key_version":2,"issued_at":"..."}` |
|
||||
| `POST /api/v1/ops/devices/{id}/claims` | 为未绑定设备生成 24 小时绑定码 | 无正文 | 只显示一次的 `claim_code` 和过期时间 | `{"device_id":"<uuid>","claim_code":"CLAIM-...","issued_at":"...","expires_at":"..."}` |
|
||||
| `POST /api/v1/ops/devices/{id}/activation-transitions` | 执行受控激活状态迁移 | Body：`target_state`、`reason`、`expected_revision` | 更新后的 `OpsDevice` | `{"target_state":"CLAIMABLE","reason":"工厂质检通过","expected_revision":1}` |
|
||||
| `GET /api/v1/ops/devices/{id}/activation-transitions` | 查询完整激活审计历史 | Path：设备 UUID | `items: DeviceActivationTransition[]` | `{"items":[{"from_state":"FACTORY","to_state":"CLAIMABLE","reason":"工厂质检通过","activation_revision":2,"created_at":"..."}]}` |
|
||||
|
||||
激活状态：`FACTORY → CLAIMABLE → BOUND → PROVISIONING → ACTIVE → LOST/REVOKED`。设备不能自行修改该状态。
|
||||
|
||||
## 10. 火山硬件智能体 WebSocket
|
||||
|
||||
### 10.1 建连
|
||||
|
||||
```text
|
||||
wss://ai-gateway.vei.volces.com/v1/realtime?bot=botM4gWQ9A9y&wait_for_session_update=true
|
||||
```
|
||||
|
||||
输入 Header：
|
||||
|
||||
| Header | 说明 |
|
||||
|---|---|
|
||||
| `X-Auth-Type` | 固定 `1` |
|
||||
| `X-Product-Key` | 平台预配返回的 Product Key |
|
||||
| `X-Device-Name` | 平台预配返回的 Device Name |
|
||||
| `X-Random-Num` | 本次随机整数 |
|
||||
| `X-Timestamp` | Unix 秒 |
|
||||
| `X-Instance-Id` | 平台预配返回的 Instance ID |
|
||||
| `X-Signature` | 使用火山 Device Secret 计算的 HMAC-SHA256 Base64 |
|
||||
| `X-Hardware-Id` | 稳定 Hardware ID |
|
||||
|
||||
签名正文顺序固定：
|
||||
|
||||
```text
|
||||
auth_type=1&device_name={name}&random_num={random}&product_key={key}×tamp={unix}&instance_id={instance}
|
||||
```
|
||||
|
||||
### 10.2 动态配置输入
|
||||
|
||||
连接打开后立即发送：
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "windows-session-update-<uuid>",
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"object": "realtime.session",
|
||||
"model": "",
|
||||
"input_audio_format": "pcm16",
|
||||
"output_audio_format": "pcm16",
|
||||
"config": {
|
||||
"ASRConfig": { "TurnDetectionMode": 0 },
|
||||
"LLMConfig": { "SystemMessages": ["<平台冻结的人设与授权上下文>"] }
|
||||
},
|
||||
"agent_config": { "EnableConversationStateCallback": false }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
输出：`session.created` 或 `session.updated`；在收到确认前不得开启麦克风。
|
||||
|
||||
### 10.3 音频和文本输入
|
||||
|
||||
上行音频：PCM16 little-endian、16kHz、单声道，建议每 20ms 320 个采样。
|
||||
|
||||
```json
|
||||
{ "event_id": "audio-<uuid>", "type": "input_audio_buffer.append", "audio": "<PCM Base64>" }
|
||||
```
|
||||
|
||||
文本测试：
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "text-<uuid>",
|
||||
"type": "conversation.item.create",
|
||||
"item": { "type": "message", "role": "user", "content": [{ "type": "input_text", "text": "你好" }], "interrupt_mode": 1 }
|
||||
}
|
||||
```
|
||||
|
||||
### 10.4 设备消费的输出事件
|
||||
|
||||
| 事件 | 输出内容 | 设备动作 |
|
||||
|---|---|---|
|
||||
| `input_audio_buffer.speech_started/stopped` | 服务端 VAD 状态 | 更新交互状态 |
|
||||
| `conversation.item.input_audio_transcription.completed` | 用户最终字幕 | 上传平台 `turns:batch` |
|
||||
| `response.audio.delta` | Base64 PCM16、16kHz、单声道 | 连续缓冲并按声卡采样率重采样播放 |
|
||||
| `response.audio.done` | 本轮音频结束 | 排空播放缓冲 |
|
||||
| `response.audio_transcript.delta/done` | 智能体增量/最终字幕 | 只把最终字幕上传平台 |
|
||||
| `response.function_call_arguments.done` | 工具名、参数和 `call_id` | 调用平台受控工具并回传结果 |
|
||||
| `response.done` | 本轮完成 | 继续监听 |
|
||||
| `error` | 供应商错误 | 脱敏后上报 `session_failed` |
|
||||
|
||||
注意：low-load WebSocket 下行是 16kHz。把它按 24kHz 播放会出现约 1.5 倍速、音调升高和明显电子感。
|
||||
|
||||
## 11. 固件管理接口
|
||||
|
||||
### 11.1 家庭/租户授权接口
|
||||
|
||||
| 方法与路径 | 介绍 | 输入 | 输出 |
|
||||
|---|---|---|---|
|
||||
| `GET /api/v1/households/{id}/firmware-signing-keys` | 查询可信 Ed25519 公钥 | Path `household id` | `FirmwareSigningKey[]` |
|
||||
| `POST /api/v1/households/{id}/firmware-signing-keys` | 登记签名公钥 | `key_id`、`public_key_base64` | `201`，创建结果 |
|
||||
| `POST /api/v1/households/{id}/firmware-signing-keys/revoke` | 吊销公钥 | `key_id` | `FirmwareSigningKey` |
|
||||
| `GET /api/v1/households/{id}/firmware-releases` | 查询固件版本 | Path `household id` | `FirmwareRelease[]` |
|
||||
| `POST /api/v1/households/{id}/firmware-releases` | 创建 `UPLOADING` 版本 | `PrepareFirmwareReleaseRequest` | 版本和短期上传地址 |
|
||||
| `GET /api/v1/firmware-releases/{id}/impact` | 发布/暂停/退役前影响评估 | Path `release id` | `FirmwareReleaseImpact` |
|
||||
| `POST /api/v1/firmware-releases/{id}/upload-session` | 重新签发上传地址 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/firmware-releases/{id}/finalize` | 校验大小、SHA-256、Ed25519 后发布 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/firmware-releases/{id}/pause` | 暂停版本和未投递命令 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/firmware-releases/{id}/resume` | 恢复版本分发 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/firmware-releases/{id}/retire` | 退役版本并保留审计 | 无正文 | `FirmwareRelease` |
|
||||
| `GET /api/v1/households/{id}/firmware-rollouts` | 查询 OTA 批次 | Path `household id` | `FirmwareRollout[]` |
|
||||
| `POST /api/v1/households/{id}/firmware-rollouts` | 启动灰度升级 | `StartFirmwareRolloutRequest` | `202 FirmwareRollout` |
|
||||
| `GET /api/v1/firmware-rollouts/{id}` | 查询逐设备升级进度 | Path `rollout id` | `FirmwareRollout` |
|
||||
| `POST /api/v1/firmware-rollouts/{id}/pause` | 暂停批次 | 无正文 | `FirmwareRollout` |
|
||||
| `POST /api/v1/firmware-rollouts/{id}/rollback` | 创建回滚批次 | `rollback_release_id` | 新的 `FirmwareRollout` |
|
||||
|
||||
### 11.2 运维固件接口
|
||||
|
||||
以下接口全部额外要求 `X-Tenant-Id`：
|
||||
|
||||
| 方法与路径 | 介绍 | 输入 | 输出 |
|
||||
|---|---|---|---|
|
||||
| `GET /api/v1/ops/firmware-signing-keys` | 查询租户可信公钥 | 无 | `FirmwareSigningKey[]` |
|
||||
| `POST /api/v1/ops/firmware-signing-keys` | 登记可信公钥 | `key_id`、`public_key_base64` | `201` |
|
||||
| `POST /api/v1/ops/firmware-signing-keys/revoke` | 吊销公钥 | `key_id` | `FirmwareSigningKey` |
|
||||
| `GET /api/v1/ops/firmware-releases` | 按 `model/status` 查询版本 | Query `model`、`status` 可选 | `FirmwareRelease[]` |
|
||||
| `POST /api/v1/ops/firmware-releases` | 创建租户级版本 | `PrepareOpsFirmwareReleaseRequest` | `201 FirmwareRelease` |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/upload-session` | 续签上传地址 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/finalize` | 正式发布 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/pause` | 暂停 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/resume` | 恢复 | 无正文 | `FirmwareRelease` |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/retire` | 退役 | 无正文 | `FirmwareRelease` |
|
||||
| `GET /api/v1/ops/firmware-releases/{id}/download` | 运维下载制品 | Path `release id` | 下载地址、摘要、签名元数据 |
|
||||
| `GET /api/v1/ops/firmware-rollouts` | 查询 OTA 批次 | 无 | `FirmwareRollout[]` |
|
||||
| `POST /api/v1/ops/firmware-rollouts` | 启动 OTA | `StartFirmwareRolloutRequest` | `202 FirmwareRollout` |
|
||||
| `GET /api/v1/ops/firmware-rollouts/{id}` | 查询逐设备进度 | Path `rollout id` | `FirmwareRollout` |
|
||||
| `POST /api/v1/ops/firmware-rollouts/{id}/pause` | 暂停 OTA | 无正文 | `FirmwareRollout` |
|
||||
|
||||
固件请求示例：
|
||||
|
||||
```json
|
||||
{ "key_id": "release-key-2026-01", "public_key_base64": "<32字节Ed25519公钥的Base64>" }
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "echoear",
|
||||
"version": "1.1.0",
|
||||
"expected_size": 2097152,
|
||||
"expected_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"signing_key_id": "release-key-2026-01",
|
||||
"signature": "<firmware-manifest-v1签名Base64>"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{ "release_id": "<uuid>", "name": "第一批灰度", "target_percentage": 10, "device_ids": ["<device-uuid>"] }
|
||||
```
|
||||
|
||||
典型版本输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-fw", "data": { "id": "<uuid>", "model": "echoear", "version": "1.1.0", "status": "READY", "sha256": "bbbb...", "created_at": "2026-09-05T07:00:00Z" } }
|
||||
```
|
||||
|
||||
典型批次输出：
|
||||
|
||||
```json
|
||||
{ "request_id": "req-rollout", "data": { "id": "<uuid>", "release_id": "<uuid>", "name": "第一批灰度", "status": "RUNNING", "target_percentage": 10, "targeted": 1, "succeeded": 0, "failed": 0 } }
|
||||
```
|
||||
|
||||
## 12. 运维诊断设备动作
|
||||
|
||||
| 方法与路径 | 介绍 | 输入 | 输出 | 示例 |
|
||||
|---|---|---|---|---|
|
||||
| `POST /api/v1/ops/diagnostics/devices/{id}/config-refresh` | 强制设备重新拉取配置 | `X-Tenant-Id`、`Idempotency-Key`、设备 UUID；Body：`reason` | `action=refresh_config`、命令 ID/状态、`queued` | 入：`{"reason":"远端配置与影子不一致"}`；出：`{"action":"refresh_config","device_id":"<uuid>","command_id":"<uuid>","command_status":"QUEUED","queued":true}` |
|
||||
| `POST /api/v1/ops/diagnostics/devices/{id}/revoke` | 紧急吊销异常设备 | `X-Tenant-Id`、`Idempotency-Key`、设备 UUID；Body：`reason` | `action=revoke_device`、设备 ID、`queued` | 入：`{"reason":"设备密钥疑似泄露"}`；出：`{"action":"revoke_device","device_id":"<uuid>","queued":false}` |
|
||||
|
||||
## 13. 主要错误码
|
||||
|
||||
| HTTP | 错误码 | 含义与处理 |
|
||||
|---:|---|---|
|
||||
| 400 | `VALIDATION_FAILED` | 字段、Header、时间或幂等键不符合契约 |
|
||||
| 401 | `DEVICE_AUTHENTICATION_FAILED` | 检查平台 Device Secret、时钟、URI、正文摘要和 Base64URL |
|
||||
| 403 | `DEVICE_ACTIVATION_REQUIRED` | 当前激活状态不允许该操作 |
|
||||
| 403 | `TRANSCRIPT_STORAGE_NOT_ALLOWED` | 当前授权不允许保存字幕 |
|
||||
| 403 | `FIRMWARE_DOWNLOAD_DENIED` | 设备不是灰度目标或版本不可下载 |
|
||||
| 404 | `RESOURCE_NOT_FOUND` | 资源不存在或不属于当前主体/设备 |
|
||||
| 409 | `IDEMPOTENCY_KEY_REUSED` | 同一幂等键对应不同正文，客户端必须换业务请求 |
|
||||
| 409 | `DEVICE_ACTIVATION_CONFLICT` | 激活状态或修订号冲突，刷新后再处理 |
|
||||
| 409 | `HARDWARE_AGENT_HARDWARE_MISMATCH` | Hardware ID 与已注册记录不一致，禁止覆盖 |
|
||||
| 409 | `HARDWARE_SESSION_PROOF_INVALID` | `session_nonce`/配置摘要/Bot/传输不匹配，丢弃本次会话重新创建 |
|
||||
| 412 | `VERSION_CONFLICT` | `If-Match` 过期，先重新 GET |
|
||||
| 426 | `FIRMWARE_UPDATE_REQUIRED` | 当前固件低于最低版本 |
|
||||
| 429 | `USAGE_QUOTA_EXCEEDED` | 额度或并发限制达到上限 |
|
||||
| 503 | `HARDWARE_AGENT_PROVISIONING_UNAVAILABLE` | 平台 Product Secret/产品配置不可用 |
|
||||
| 503 | `HARDWARE_AGENT_PROVISIONING_FAILED` | 火山动态注册失败；按 `request_id` 排障，可退避重试 |
|
||||
|
||||
## 14. 设备端推荐调用顺序
|
||||
|
||||
```text
|
||||
上电
|
||||
→ GET /device/v2/bootstrap
|
||||
→ GET /device/v2/config
|
||||
→ GET /device/v2/commands
|
||||
→ POST /device/v2/telemetry:batch
|
||||
|
||||
首次绑定后
|
||||
→ POST /device/v3/hardware-agent-credentials:provision
|
||||
→ 安全保存火山 Device Secret
|
||||
|
||||
每次聊天
|
||||
→ POST /device/v3/agent-sessions
|
||||
→ 连接火山 WebSocket
|
||||
→ session.update
|
||||
→ 收到 session.created/session.updated
|
||||
→ POST /device/v3/agent-sessions/{id}/events (session_started)
|
||||
→ 16k PCM 双向音频
|
||||
→ POST /device/v3/agent-sessions/{id}/turns:batch
|
||||
→ 周期 events (session_updated)
|
||||
→ events (session_ended/session_failed)
|
||||
```
|
||||
|
||||
## 15. 机器可读契约
|
||||
|
||||
- 完整 OpenAPI：[`../openapi/openapi.yaml`](../openapi/openapi.yaml)
|
||||
- 设备实现与安全说明：[`device-api-reference.md`](device-api-reference.md)
|
||||
- 设备接入说明：[`device-integration.md`](device-integration.md)
|
||||
- 固件灰度说明：[`firmware-rollout.md`](firmware-rollout.md)
|
||||
|
||||
若本文与 OpenAPI 或运行时代码不一致，以同一发布版本的 OpenAPI 和处理器为准，并应同步修正文档。
|
||||
@@ -0,0 +1,576 @@
|
||||
# 设备接口文档
|
||||
|
||||
> 适用范围：平台设备管理、EchoEar/ESP32-S3 设备接入、火山引擎硬件对话智能体。
|
||||
> 契约基线：当前工作区实际路由、处理器与 `openapi/openapi.yaml`，更新日期 2026-09-05。
|
||||
> 生产 API：`https://121.41.46.238`（正式域名启用后只替换 Base URL，不改变路径）。
|
||||
|
||||
## 1. 接入模型
|
||||
|
||||
设备同时使用两套彼此独立的身份：
|
||||
|
||||
| 身份/密钥 | 用途 | 生成与保存位置 | 是否下发设备 |
|
||||
|---|---|---|---|
|
||||
| 平台 `device_id` | 标识平台登记设备 | 平台生成；设备保存公开编号 | 是 |
|
||||
| 平台 Device Secret | 调用 `/device/*` 时做 HMAC 鉴权 | 平台逐设备生成；平台加密保存，设备安全区保存 | 是，只在登记/轮换时显示一次 |
|
||||
| 火山 `instance_id` / `product_key` | 标识火山实例和产品 | 平台配置 | 可随预配响应返回，非秘密 |
|
||||
| 火山 Product Secret | 调用 DynamicRegister 的产品级秘密 | 只保存在平台 KMS/Secret | **否** |
|
||||
| 火山 `device_name` / `device_secret` | 设备直连火山时使用的一机一密 | 平台代设备动态注册；平台加密保存，设备安全区保存 | 是，只返回给已认证的对应设备 |
|
||||
| Bot ID | 选择硬件对话智能体 | 平台按设备绑定 | 会话响应返回，非秘密 |
|
||||
|
||||
核心原则：设备固件不内置 Product Secret。设备先用平台 Device Secret 认证，由平台代设备调用火山动态注册，再把该设备专属的火山 Device Secret 返回给设备。
|
||||
|
||||
### 1.1 火山 License 的准确含义
|
||||
|
||||
火山硬件对话智能体的 License 是产品侧可用额度。平台调用 DynamicRegister 时，火山自动消耗当前产品的一份有效额度并返回一机一密；设备不接收一段可安装的 License 明文，也不需要向平台提交 License 安装回执。
|
||||
|
||||
因此，当前有效契约中：
|
||||
|
||||
- 没有 `/device/v3/hardware-agent-license:ack` 路由；
|
||||
- 没有设备端 `license` 字段；
|
||||
- 没有 `AVAILABLE → ALLOCATED → ACTIVE` 的设备 License 安装状态机；
|
||||
- 平台可以维护额度监控或影子账本，但不能用它替代火山的真实额度，也不能阻断已正确取得一机一密的设备。
|
||||
|
||||
火山返回额度耗尽、过期等错误时，平台统一转换为 `HARDWARE_AGENT_PROVISIONING_FAILED`，同时通过内部日志和请求编号保留供应商错误供运维定位。
|
||||
|
||||
## 2. 通用响应格式
|
||||
|
||||
成功响应：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-...",
|
||||
"data": {}
|
||||
}
|
||||
```
|
||||
|
||||
失败响应：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-...",
|
||||
"error": {
|
||||
"code": "VALIDATION_FAILED",
|
||||
"message": "请求参数不正确",
|
||||
"retryable": false,
|
||||
"field_errors": [],
|
||||
"details": {}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
排障时应记录 `request_id`、HTTP 状态码和稳定错误码，不记录任何 Secret、Token、完整人设或完整对话正文。
|
||||
|
||||
## 3. 设备 HMAC 鉴权
|
||||
|
||||
所有 `/device/v2/*` 和 `/device/v3/*` 请求必须携带：
|
||||
|
||||
| 请求头 | 说明 |
|
||||
|---|---|
|
||||
| `X-Device-Id` | 平台设备公开 UUID |
|
||||
| `X-Device-Timestamp` | RFC3339 UTC 时间，默认允许前后 5 分钟 |
|
||||
| `X-Device-Nonce` | 本次请求唯一的 16—128 字符随机值 |
|
||||
| `X-Device-Content-SHA256` | 原始 HTTP 正文的 SHA-256 小写十六进制；空正文也要计算 |
|
||||
| `X-Device-Signature` | HMAC-SHA256 结果的无填充 Base64URL |
|
||||
|
||||
签名原文必须逐字使用实际请求 URI（含查询字符串）：
|
||||
|
||||
```text
|
||||
UPPERCASE_HTTP_METHOD
|
||||
REQUEST_URI_WITH_EXACT_QUERY_STRING
|
||||
X-Device-Timestamp原值
|
||||
X-Device-Nonce原值
|
||||
X-Device-Content-SHA256原值
|
||||
```
|
||||
|
||||
伪代码：
|
||||
|
||||
```text
|
||||
bodyHash = hexLower(SHA256(rawBody))
|
||||
canonical = methodUpper + "\n" + requestURI + "\n" + timestamp + "\n" + nonce + "\n" + bodyHash
|
||||
signature = base64UrlNoPadding(HMAC_SHA256(platformDeviceSecret, canonical))
|
||||
```
|
||||
|
||||
注意事项：
|
||||
|
||||
- JSON 签名后不得重新格式化或改变字段顺序；服务端校验的是收到的原始字节。
|
||||
- 每次网络重试都生成新的 `timestamp`、`nonce` 和签名。
|
||||
- 需要幂等的业务请求保留原 `Idempotency-Key` 和原正文。
|
||||
- 时间未同步时先通过安全时间源校时；不能关闭时间窗或重放保护。
|
||||
|
||||
## 4. 设备激活状态机
|
||||
|
||||
主流程：
|
||||
|
||||
```text
|
||||
FACTORY → CLAIMABLE → BOUND → PROVISIONING → ACTIVE
|
||||
│
|
||||
├─→ LOST
|
||||
└─→ REVOKED
|
||||
```
|
||||
|
||||
| 状态 | 含义 | 允许的设备操作 |
|
||||
|---|---|---|
|
||||
| `FACTORY` | 只有出厂身份，等待质检放行 | 不允许设备 API 鉴权 |
|
||||
| `CLAIMABLE` | 可生成/消费绑定码 | 启动、配置、命令、遥测；不能通话 |
|
||||
| `BOUND` | 已绑定家庭和使用人 | 可申请火山一机一密；不能创建会话 |
|
||||
| `PROVISIONING` | 正在注册或恢复火山运行凭据 | 可重复预配；不能创建会话 |
|
||||
| `ACTIVE` | 平台身份、绑定和运行凭据就绪 | 全部设备接口 |
|
||||
| `LOST` | 设备挂失 | 设备鉴权和新会话被拒绝 |
|
||||
| `REVOKED` | 永久吊销 | 终态，不可恢复 |
|
||||
|
||||
代码允许的恢复/解绑边包括：`BOUND → CLAIMABLE`、`PROVISIONING → BOUND`、`ACTIVE → CLAIMABLE/PROVISIONING/LOST`、`LOST → CLAIMABLE/ACTIVE`；任意非 `REVOKED` 状态均可受控转为 `REVOKED`。状态变更必须由平台业务或运维接口执行，设备不能自行声明已激活。
|
||||
|
||||
每次变更增加 `activation_revision` 并保存审计历史。进入 `ACTIVE` 时，硬件智能体设备必须已经绑定且存在有效火山一机一密。
|
||||
|
||||
## 5. 首次激活完整流程
|
||||
|
||||
1. 运维调用 `POST /api/v1/ops/devices` 登记设备，取得一次性平台 Device Secret。
|
||||
2. 工厂烧录/安全注入 `device_id`、平台 Device Secret、API Base URL 和稳定 `hardware_id`；不得烧录 Product Secret。
|
||||
3. 运维将设备从 `FACTORY` 放行到 `CLAIMABLE`，并生成绑定码。
|
||||
4. 用户 App/Web 调用 `POST /api/v1/device-claims` 消费绑定码，平台把设备变为 `BOUND`。
|
||||
5. 设备使用平台 HMAC 调用 `POST /device/v3/hardware-agent-credentials:provision`。
|
||||
6. 平台把状态推进到 `PROVISIONING`，自动建立产品默认 Bot/传输/最低 SDK 绑定，再使用服务端 Product Secret 调用火山 DynamicRegister。
|
||||
7. 火山消耗产品 License 额度并返回该 `device_name` 对应的 `device_secret`。
|
||||
8. 平台加密保存火山 Device Secret，并通过 `no-store` 响应返回给同一设备，然后自动完成到 `ACTIVE` 的迁移。
|
||||
9. 设备把火山一机一密写入 NVS 加密分区/安全芯片；以后重复预配会返回已保存的同一凭据，不重复注册。
|
||||
10. 每次用户发起聊天时，设备先创建平台硬件智能体会话，注入本次人设和上下文后，再直连火山。
|
||||
|
||||
## 6. 设备端接口总览
|
||||
|
||||
### 6.1 启动、配置、命令和固件（v2 公共能力）
|
||||
|
||||
| 方法 | 路径 | 状态要求 | 幂等键 | 作用 |
|
||||
|---|---|---|---|---|
|
||||
| `GET` | `/device/v2/bootstrap` | `CLAIMABLE/BOUND/PROVISIONING/ACTIVE` | 否 | 上报启动版本并取得绑定和兼容性 |
|
||||
| `GET` | `/device/v2/config` | 同上 | 否 | 拉取期望配置；支持 `If-None-Match` |
|
||||
| `GET` | `/device/v2/commands` | 同上 | 否 | 拉取待执行命令 |
|
||||
| `POST` | `/device/v2/commands/{id}/ack` | 同上 | 否 | 上报命令阶段或终态 |
|
||||
| `POST` | `/device/v2/telemetry:batch` | 同上 | 是 | 批量上报设备遥测 |
|
||||
| `GET` | `/device/v2/firmware-releases/{id}/download` | 同上 | 否 | 下载受控固件制品 |
|
||||
|
||||
`bootstrap` 示例：
|
||||
|
||||
```http
|
||||
GET /device/v2/bootstrap?firmware_version=1.0.0&board_rev=echoear-v1&rtc_sdk_version=1.0.0&codec=pcm_s16le&sample_rate=16000&device_time=2026-09-03T08%3A00%3A00Z&network=wifi
|
||||
```
|
||||
|
||||
配置拉取建议：首次不带 `If-None-Match`；保存响应 `ETag`，后续携带该值。HTTP `304` 表示继续使用本地有效配置。
|
||||
|
||||
### 6.2 旧 RTC 通话（v2，仅兼容旧设备）
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
|---|---|---|
|
||||
| `POST` | `/device/v2/conversations` | 创建旧 RTC 通话 |
|
||||
| `POST` | `/device/v2/conversations/{id}/activate` | 入房成功后激活供应商任务 |
|
||||
| `GET` | `/device/v2/conversations/{id}` | 查询通话权威状态 |
|
||||
| `POST` | `/device/v2/conversations/{id}/heartbeat` | 通话心跳 |
|
||||
| `POST` | `/device/v2/conversations/{id}/stop` | 停止通话 |
|
||||
| `POST` | `/device/v2/conversations/{id}/turns:batch` | 补传最终字幕 |
|
||||
|
||||
新 EchoEar 固件的 `voice_runtime=volc_hardware_agent` 时不得调用这组 RTC 创建/激活接口。
|
||||
|
||||
### 6.3 录音上传（v2 公共媒体能力）
|
||||
|
||||
| 方法 | 路径 | 作用 |
|
||||
|---|---|---|
|
||||
| `POST` | `/device/v2/conversations/{id}/upload-batches` | 创建分片上传批次 |
|
||||
| `GET` | `/device/v2/upload-batches/{id}` | 查询上传校验状态 |
|
||||
| `POST` | `/device/v2/upload-batches/{id}/complete` | 通知平台分片已上传 |
|
||||
| `POST` | `/device/v2/media-segments/{id}/local-delete-ack` | 回报本地密文已删除 |
|
||||
|
||||
只有 `ACTIVE` 设备且本次隐私策略允许时才能使用；没有录音授权不得采集、缓存或上传 PCM。
|
||||
|
||||
## 7. 火山硬件智能体接口（v3）
|
||||
|
||||
### 7.1 申请火山一机一密
|
||||
|
||||
```http
|
||||
POST /device/v3/hardware-agent-credentials:provision
|
||||
Idempotency-Key: 2ee3b675-59c0-4d93-a189-6db7f8db3750
|
||||
Cache-Control: no-store
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
||||
状态要求：`BOUND`、`PROVISIONING` 或 `ACTIVE`。
|
||||
|
||||
请求：
|
||||
|
||||
```json
|
||||
{
|
||||
"hardware_id": "echoear-b43a451634a8"
|
||||
}
|
||||
```
|
||||
|
||||
`hardware_id` 必须来自稳定硬件身份（如 eFuse/Chip ID 的规范化值），设备重启、升级和恢复出厂后保持不变。
|
||||
|
||||
成功响应：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-...",
|
||||
"data": {
|
||||
"instance_id": "<instance-id>",
|
||||
"product_key": "<product-key>",
|
||||
"device_name": "echoear-...",
|
||||
"hardware_id": "echoear-b43a451634a8",
|
||||
"device_secret": "<volc-device-secret>",
|
||||
"credential_ref": "<non-secret-ref>",
|
||||
"credential_version": 1,
|
||||
"credential_source": "platform_dynamic_registration",
|
||||
"issued_at": "2026-09-03T08:00:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
安全要求：
|
||||
|
||||
- 响应带 `Cache-Control: no-store` 和 `Pragma: no-cache`。
|
||||
- `device_secret` 只写入设备安全存储，不进入普通配置文件、串口日志、崩溃转储或平台遥测。
|
||||
- `credential_source=platform_stored` 表示幂等返回平台已保存的凭据，不代表重新消耗 License。
|
||||
- `hardware_id` 与已注册记录不一致时返回冲突，不允许覆盖原凭据。
|
||||
|
||||
常见错误：
|
||||
|
||||
| HTTP | 错误码 | 处理 |
|
||||
|---|---|---|
|
||||
| `400` | `VALIDATION_FAILED` | 修正 `hardware_id` 或幂等键 |
|
||||
| `409` | `HARDWARE_AGENT_BINDING_REQUIRED` | 先在平台绑定 Bot/产品配置 |
|
||||
| `409` | `HARDWARE_AGENT_PRODUCT_MISMATCH` | 运维核对实例、产品和设备绑定 |
|
||||
| `409` | `HARDWARE_AGENT_HARDWARE_MISMATCH` | 停止自动重试，核对硬件身份 |
|
||||
| `409` | `DEVICE_ACTIVATION_REQUIRED` | 先完成用户绑定，进入 `BOUND` |
|
||||
| `503` | `HARDWARE_AGENT_PROVISIONING_UNAVAILABLE` | 平台未配置动态注册；联系运维 |
|
||||
| `503` | `HARDWARE_AGENT_PROVISIONING_FAILED` | 使用原幂等键、原正文和新签名退避重试；由 `request_id` 查火山原始错误 |
|
||||
|
||||
### 7.2 创建本次硬件智能体会话
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions
|
||||
Idempotency-Key: d7f08e7a-9e1d-455f-b8ec-a745a34d8790
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
||||
状态要求：仅 `ACTIVE`。每次聊天创建一个新会话，不复用上一通响应。
|
||||
|
||||
请求示例：
|
||||
|
||||
```json
|
||||
{
|
||||
"device_time": "2026-09-03T08:05:00Z",
|
||||
"monotonic_ms": 123456,
|
||||
"capabilities": {
|
||||
"hardware_agent": true,
|
||||
"sdk_version": "1.0.0-echoear-platform.1",
|
||||
"transports": ["low_load"],
|
||||
"input_audio_format": "pcm_s16le_16000_mono",
|
||||
"output_audio_format": "pcm_s16le_16000_mono",
|
||||
"server_event_callback": false,
|
||||
"transcript_upload": true,
|
||||
"dynamic_session_config": true,
|
||||
"credential_rotation": true,
|
||||
"near_pcm_recording": false,
|
||||
"far_pcm_recording": false
|
||||
},
|
||||
"network": {
|
||||
"type": "wifi",
|
||||
"rssi": -52
|
||||
},
|
||||
"requested_goal": "open_conversation",
|
||||
"config_revision": 1
|
||||
}
|
||||
```
|
||||
|
||||
约束：
|
||||
|
||||
- `transports` 只能包含 `low_load`、`high_quality`，至少一个。
|
||||
- `transcript_upload`、`dynamic_session_config`、`credential_rotation` 必须为 `true`。
|
||||
- EchoEar 当前使用 `low_load`；`server_event_callback=false` 时设备事件和最终字幕上传是权威回流。
|
||||
- `config_revision` 必须使用最近一次 `/device/v2/config` 返回的版本。
|
||||
|
||||
成功响应核心字段：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-...",
|
||||
"data": {
|
||||
"conversation_id": "<uuid>",
|
||||
"status": "SESSION_READY",
|
||||
"runtime_type": "volc_hardware_agent",
|
||||
"runtime": {
|
||||
"provider": "volcengine",
|
||||
"bot_id": "<bot-id>",
|
||||
"transport": "low_load",
|
||||
"provider_session_ref": "<provider-session-ref>",
|
||||
"session_nonce": "<memory-only-secret>",
|
||||
"config_digest": "<sha256-hex>",
|
||||
"wait_for_session_update": true
|
||||
},
|
||||
"session_config": {
|
||||
"dialog_id": "<uuid>",
|
||||
"system_prompt": "<本次冻结的人设提示>",
|
||||
"persona_tone": "<语气>",
|
||||
"authorized_context": "<已授权背景/近期摘要/长期记忆>",
|
||||
"prompt_bundle_version": "<version>"
|
||||
},
|
||||
"privacy": {
|
||||
"retention_mode": "transcript_only",
|
||||
"record_near": false,
|
||||
"record_far": false,
|
||||
"upload_allowed": false
|
||||
},
|
||||
"session": {
|
||||
"engine_mode": "volc_hardware_agent",
|
||||
"heartbeat_interval_sec": 10,
|
||||
"allocation_expires_at": "2026-09-03T08:10:00Z"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
设备处理顺序：
|
||||
|
||||
1. 只在运行内存保存 `session_nonce`。
|
||||
2. 使用预配取得的一机一密连接响应指定的 Bot 和传输模式。
|
||||
3. 以 `dialog_id` 作为本次业务对话标识。
|
||||
4. 在开启麦克风前通过 SDK 动态配置或 WebSocket `session.update` 注入 `system_prompt`、`persona_tone` 和 `authorized_context`。
|
||||
5. 收到火山配置已应用确认后，再上报 `session_started`。
|
||||
|
||||
`tool_scope_token` 如存在也只能保存在运行内存，并且仅用于响应指定的 `tool_url` 和本次会话。
|
||||
|
||||
### 7.3 上报会话事件
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions/{conversation_id}/events
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
||||
开始事件：
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "evt-start-0001",
|
||||
"sequence_no": 1,
|
||||
"event_type": "session_started",
|
||||
"occurred_at": "2026-09-03T08:05:03Z",
|
||||
"provider_session_ref": "<response-runtime-provider-session-ref>",
|
||||
"session_nonce": "<response-runtime-session-nonce>",
|
||||
"provider_connection_ref": "<actual-volc-connection-ref>",
|
||||
"bot_id": "<response-runtime-bot-id>",
|
||||
"transport": "low_load",
|
||||
"config_digest": "<response-runtime-config-digest>",
|
||||
"dynamic_config_applied": true
|
||||
}
|
||||
```
|
||||
|
||||
后续事件类型：
|
||||
|
||||
- `session_updated`：按 `heartbeat_interval_sec` 上报存活；`sequence_no` 严格递增。
|
||||
- `session_ended`：正常结束。
|
||||
- `session_failed`：异常结束，可附加稳定、脱敏的 `error_code`。
|
||||
|
||||
响应：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-...",
|
||||
"data": {
|
||||
"conversation_id": "<uuid>",
|
||||
"status": "ACTIVE",
|
||||
"action": "continue",
|
||||
"duplicate": false,
|
||||
"accepted_at": "2026-09-03T08:05:03Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`action=stop_session` 时，设备必须立即关闭麦克风和火山连接，然后补报 `session_ended` 或 `session_failed`。相同 `event_id` 重放时必须保持原正文一致。
|
||||
|
||||
### 7.4 批量上传最终话轮
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions/{conversation_id}/turns:batch
|
||||
Idempotency-Key: 1710b8cc-9bf2-45e8-a16d-f32c3b80cbbe
|
||||
Content-Type: application/json
|
||||
```
|
||||
|
||||
请求：
|
||||
|
||||
```json
|
||||
{
|
||||
"turns": [
|
||||
{
|
||||
"turn_key": "user-final-0001",
|
||||
"sequence_no": 1,
|
||||
"speaker": "subject",
|
||||
"start_at": "2026-09-03T08:05:10Z",
|
||||
"end_at": "2026-09-03T08:05:12Z",
|
||||
"final_text": "今天天气怎么样？",
|
||||
"interrupted": false
|
||||
},
|
||||
{
|
||||
"turn_key": "agent-final-0002",
|
||||
"sequence_no": 2,
|
||||
"speaker": "agent",
|
||||
"start_at": "2026-09-03T08:05:12Z",
|
||||
"end_at": "2026-09-03T08:05:15Z",
|
||||
"final_text": "我来帮您看看。",
|
||||
"interrupted": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
只上传最终字幕；不得上传增量识别结果。单批 1—500 条，`speaker` 只能为 `subject` 或 `agent`。同一 `turn_key`/`sequence_no` 的重试必须保持正文完全一致。
|
||||
|
||||
成功响应：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-...",
|
||||
"data": {
|
||||
"conversation_id": "<uuid>",
|
||||
"accepted_count": 2,
|
||||
"duplicate_count": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 8. App/Web 的设备接口
|
||||
|
||||
以下接口使用用户 OIDC Bearer Token，不使用设备 HMAC：
|
||||
|
||||
| 方法 | 路径 | 作用 |
|
||||
|---|---|---|
|
||||
| `POST` | `/api/v1/device-claims` | 消费绑定码并绑定设备 |
|
||||
| `GET` | `/api/v1/devices` | 列出当前用户可管理设备 |
|
||||
| `GET` | `/api/v1/devices/{id}` | 查看设备详情 |
|
||||
| `GET` | `/api/v1/devices/{id}/binding-impact` | 变更绑定前评估影响 |
|
||||
| `POST` | `/api/v1/devices/{id}/binding-change` | 解绑或改绑 |
|
||||
| `GET` | `/api/v1/devices/{id}/presence` | 查看在线状态 |
|
||||
| `GET` | `/api/v1/devices/{id}/twin` | 查看设备影子 |
|
||||
| `PATCH` | `/api/v1/devices/{id}/twin` | 修改期望配置；必须携带 `If-Match` |
|
||||
| `GET` | `/api/v1/devices/{id}/commands` | 查看命令历史 |
|
||||
| `POST` | `/api/v1/devices/{id}/commands` | 下发设备命令 |
|
||||
|
||||
绑定码由平台/运维为 `CLAIMABLE` 设备生成，App 只消费，不自行生成。解绑会影响设备激活状态和火山运行凭据，必须先调用 `binding-impact` 展示影响并要求用户确认。
|
||||
|
||||
## 9. 平台运维设备接口
|
||||
|
||||
所有接口要求 OIDC Bearer Token 和有效的 `X-Tenant-Id: <tenant-uuid>`，且仅独立运维账号可调用。
|
||||
|
||||
| 方法 | 路径 | 作用 |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/v1/ops/devices` | 设备列表 |
|
||||
| `POST` | `/api/v1/ops/devices` | 登记设备并一次性返回平台 Device Secret |
|
||||
| `GET` | `/api/v1/ops/device-assignment-options` | 可选家庭/长者列表 |
|
||||
| `PATCH` | `/api/v1/ops/devices/{id}` | 修改设备；必须携带 `If-Match` |
|
||||
| `DELETE` | `/api/v1/ops/devices/{id}` | 吊销设备；必须携带 `If-Match` |
|
||||
| `POST` | `/api/v1/ops/devices/{id}/credentials/rotate` | 轮换平台 Device Secret，只返回一次明文 |
|
||||
| `POST` | `/api/v1/ops/devices/{id}/claims` | 生成一次性绑定码 |
|
||||
| `POST` | `/api/v1/ops/devices/{id}/activation-transitions` | 人工迁移激活状态 |
|
||||
| `GET` | `/api/v1/ops/devices/{id}/activation-transitions` | 查询激活迁移历史 |
|
||||
|
||||
### 9.1 登记硬件智能体设备
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "echoear",
|
||||
"hardware_revision": "esp32-s3",
|
||||
"minimum_security_version": "1",
|
||||
"minimum_firmware_version": "1.0.0",
|
||||
"household_id": "",
|
||||
"subject_id": "",
|
||||
"voice_runtime": "volc_hardware_agent",
|
||||
"activation_state": "FACTORY"
|
||||
}
|
||||
```
|
||||
|
||||
Instance ID、Product Key、默认 Bot、传输模式和最低 SDK 由平台产品配置统一持有。设备第一次预配时自动生成绑定，运维创建设备或设备自身都不能覆盖这些字段。
|
||||
|
||||
响应中的 `data.credential.secret` 只显示一次。运维系统应立即通过受控注入工具写入设备安全区，不得复制到工单、聊天记录或普通数据库字段。
|
||||
|
||||
### 9.2 人工迁移激活状态
|
||||
|
||||
```json
|
||||
{
|
||||
"target_state": "CLAIMABLE",
|
||||
"reason": "工厂质检通过",
|
||||
"expected_revision": 1
|
||||
}
|
||||
```
|
||||
|
||||
并发控制：列表/详情返回的设备版本对应 `ETag`；修改和删除使用 `If-Match`。激活迁移单独使用 `expected_revision`。版本不一致返回 `412 VERSION_CONFLICT` 或 `409 DEVICE_ACTIVATION_CONFLICT`，刷新后再决定，不能强制覆盖。
|
||||
|
||||
## 10. 幂等、重试和本地持久化
|
||||
|
||||
| 数据 | 本地策略 |
|
||||
|---|---|
|
||||
| 平台 Device Secret | 安全区长期保存；轮换时原子替换 |
|
||||
| 火山 Device Secret | 安全区长期保存；不要每次启动都预配 |
|
||||
| `session_nonce` | 仅本次通话运行内存 |
|
||||
| `tool_scope_token` | 仅本次通话运行内存 |
|
||||
| 最终字幕待上传队列 | 授权允许时加密持久化，成功确认后删除 |
|
||||
| 增量字幕/原始音频 | 默认不长期保存 |
|
||||
|
||||
重试规则：
|
||||
|
||||
- `Idempotency-Key` 长度 16—128 字符。
|
||||
- HTTP 超时或 `retryable=true`：原业务正文 + 原幂等键 + 新 HMAC nonce/signature，指数退避。
|
||||
- `401`：先检查平台 Device Secret、时钟、请求 URI、正文摘要和 Base64URL 编码，不要盲目轮换密钥。
|
||||
- `409 IDEMPOTENCY_KEY_REUSED`：同一幂等键配了不同正文；这是客户端错误。
|
||||
- `HARDWARE_SESSION_PROOF_INVALID`：丢弃当前会话内存数据，重新创建新会话，不尝试其他设备的 nonce。
|
||||
- `action=stop_session`：优先停止音频和供应商连接，再做结束确认及本地队列收尾。
|
||||
|
||||
## 11. EchoEar 固件接入清单
|
||||
|
||||
出厂只写入：
|
||||
|
||||
- 平台 `device_id`；
|
||||
- 平台 Device Secret；
|
||||
- `hardware_id`；
|
||||
- API Base URL 和服务器信任锚；
|
||||
- 固件签名信任锚。
|
||||
|
||||
首次绑定后设备取得并安全保存：
|
||||
|
||||
- 火山 `instance_id`；
|
||||
- 火山 `product_key`；
|
||||
- 火山 `device_name`；
|
||||
- 火山 Device Secret；
|
||||
- `credential_version`。
|
||||
|
||||
绝不写入设备：
|
||||
|
||||
- 火山 Product Secret；
|
||||
- 平台数据库主密钥/KMS 密钥；
|
||||
- 其他设备的 Device Secret；
|
||||
- 固定的会话 nonce 或工具 Token；
|
||||
- 人设、长期记忆和聊天记录的永久明文副本。
|
||||
|
||||
会话侧必须做到：
|
||||
|
||||
- 每通先请求平台会话配置，再连接火山；
|
||||
- 人设、近期摘要和长期记忆以本次 `session_config` 为准；
|
||||
- 动态配置确认前不打开麦克风；
|
||||
- 设备事件和最终字幕可靠回流平台；
|
||||
- 平台下发 `stop_session` 时可在一个心跳周期内停止。
|
||||
|
||||
## 12. 当前实现边界
|
||||
|
||||
- 当前 API 已实现平台 HMAC、激活状态机、火山动态注册、v3 会话控制面、事件回流和最终字幕上传。
|
||||
- 设备音频是设备直连火山，平台不代理每一帧音频；平台通过设备事件和字幕回流维护会话权威状态。
|
||||
- Windows/EchoEar 客户端仍需按目标 SDK 完成真实音频 HAL、火山协议、断线重连和安全存储验收。
|
||||
- `/device/v3/hardware-agent-license:ack` 处理器代码虽曾存在，但没有路由、没有生产契约，也不应由设备调用。
|
||||
- 完整字段定义以 [`openapi/openapi.yaml`](../openapi/openapi.yaml) 为机器可读基准；本文件是设备研发和联调的实施说明。
|
||||
|
||||
## 13. 联调最小验收用例
|
||||
|
||||
1. 错误平台签名返回 `401`，日志不泄露设备是否存在。
|
||||
2. 同 nonce 重放被拒绝；相同业务幂等键配新 nonce 可安全重试。
|
||||
3. `CLAIMABLE` 设备不能预配，`BOUND` 设备可以预配但不能创建会话。
|
||||
4. 首次预配成功取得火山一机一密；重复预配不重复注册。
|
||||
5. Product Secret 在设备固件、NVS、日志和网络响应中均不存在。
|
||||
6. `ACTIVE` 设备创建 v3 会话并得到正确 Bot ID、人设和授权上下文。
|
||||
7. 未确认动态配置的 `session_started` 被拒绝。
|
||||
8. 正常开始、心跳、最终字幕、结束全部回流平台。
|
||||
9. 平台返回 `stop_session` 后设备关闭连接并补报结束。
|
||||
10. 火山 License 额度耗尽时预配失败，扩容后使用原幂等请求可恢复；设备无需安装 License 或 ACK。
|
||||
@@ -1,5 +1,21 @@
|
||||
# 设备接入与启动配置联调说明
|
||||
|
||||
## 设备激活状态机
|
||||
|
||||
平台从V50开始独立维护 `activation_state`，不再用通用的 `device.status` 代替激活生命周期：
|
||||
|
||||
`FACTORY → CLAIMABLE → BOUND → PROVISIONING → ACTIVE → LOST/REVOKED`
|
||||
|
||||
- `FACTORY`：只有设备身份和平台HMAC凭据，生产质检完成后由运维放行。
|
||||
- `CLAIMABLE`：可以生成一次性绑定码；设备可同步基础配置，但不能发起对话。
|
||||
- `BOUND`：已经绑定家庭和长者，可以调用火山设备凭据预配接口。
|
||||
- `PROVISIONING`：平台正在代设备调用火山 DynamicRegister 并保存一机一密；动态注册成功、有效绑定和设备凭据就绪后进入 `ACTIVE`。火山 License 是产品侧动态注册额度，不是设备安装包，也没有设备回执。
|
||||
- `ACTIVE`：允许创建和上报对话。硬件智能体会话接口只接受该状态。
|
||||
- `LOST`：设备身份认证、对话和火山运行凭据立即停用；核验归属和硬件身份后可恢复到 `ACTIVE`。
|
||||
- `REVOKED`：终态。设备平台凭据、火山凭据、绑定和未使用绑定码全部吊销，不能恢复。
|
||||
|
||||
每次迁移都会增加 `activation_revision` 并写入 `device_activation_transition`。运维人工迁移接口为 `POST /api/v1/ops/devices/{id}/activation-transitions`，请求必须携带当前 `expected_revision`；用户认领、设备预配和删除由对应业务接口自动迁移，不应由客户端伪造状态。
|
||||
|
||||
## 1. 当前接口
|
||||
|
||||
| 方法 | 路径 | 作用 |
|
||||
@@ -270,6 +286,41 @@ powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\smoke-sim-firmware
|
||||
|
||||
脚本会验证创建、首次激活、相同幂等键重放、Worker调用、两次签名回调去重和最终`ACTIVE`状态，并断言供应商仅启动一次。脚本只使用合成数据，验证结束后按本次通话编号精确清理数据库痕迹。
|
||||
|
||||
## 11.5 火山硬件对话智能体 v3 控制面
|
||||
|
||||
设备首次准备火山连接凭据时，先使用既有平台设备签名调用`POST /device/v3/hardware-agent-credentials:provision`并提交稳定`hardware_id`。平台从产品级配置解析Instance ID、Product Key、默认Bot、传输方式和最低SDK版本，自动建立当前设备的运行绑定；再从KMS或只读Secret文件加载Product Secret，以平台生成的Device Name代设备调用火山DynamicRegister。Product Secret永不到达设备。平台把返回的`device_secret`用独立的`VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY`执行AES-256-GCM加密后保存，只在设备认证且`Cache-Control: no-store`的响应中返回给同一设备。设备使用安全芯片、系统KeyStore或Windows Safe Storage保存这份一机一密，后续用于火山WebSocket签名。
|
||||
|
||||
火山硬件对话智能体 License 是产品侧可用额度。平台调用 DynamicRegister 时由火山自动消耗一份有效额度并返回对应设备的一机一密；设备不会收到 License 明文，也不需要提交安装回执。当前有效路由不包含`POST /device/v3/hardware-agent-license:ack`，预配响应也不包含`license`字段。平台可以维护额度监控或影子账本，但不得把自建池作为设备运行凭据来源。额度耗尽或到期时，应根据火山动态注册错误告警并扩容，扩容后重试原预配请求。
|
||||
|
||||
同一设备和Hardware ID重复调用时，平台解密返回既有一机一密，不重复注册；Hardware ID、产品或实例不一致时返回409，必须走受控轮换而不是覆盖。Product Secret、Device Secret明文不得进入日志、审计详情、错误响应、通话记录或测试报告。自动回归的mock密钥仅允许在隔离测试进程中注入，不得写入正式设备缓存。
|
||||
|
||||
设备期望配置中的`voice_runtime`为`volc_hardware_agent`时，不再调用`POST /device/v2/conversations`创建RTC房间。首次预配后设备要重新拉取配置版本，并读取平台发布的`minimum_sdk_version`执行本地预检；云端会以产品默认绑定中的同一版本再次门禁，过低时返回HTTP 426和`HARDWARE_AGENT_SDK_UPDATE_REQUIRED`。每次用户开始聊天必须先调用`POST /device/v3/agent-sessions`，不得复用上一次响应。创建请求必须声明SDK版本、输入/输出音频格式、传输模式、可靠字幕队列、动态会话配置、凭证轮换以及近/远端PCM录制能力。平台会在同一事务中冻结这些能力、有效授权、已发布人设、档案、最近三次已完成通话摘要和最多五条人工确认的长期记忆，并返回：
|
||||
|
||||
- `runtime.bot_id`、传输模式和仅本次有效的`session_nonce`；
|
||||
- `session_config.dialog_id`，必须作为火山会话的业务对话标识；
|
||||
- `system_prompt`、`persona_tone`和`authorized_context`，必须在开始发送音频前通过SDK动态配置或标准WebSocket的`session.update`注入；
|
||||
- `config_digest`，用于设备日志和平台诊断比对，不得当作供应商凭据；
|
||||
- 可选的`tool_url`和`tool_scope_token`，只允许配置为本次火山会话的只读记忆工具；令牌绑定本次通话和供应商会话引用，设备不得持久化或转给其他会话；
|
||||
- 本次冻结的隐私策略和会话存活间隔。
|
||||
|
||||
设备收到火山`session.created`后必须先应用本次`session.update`，收到确认后才向`POST /device/v3/agent-sessions/{id}/events`提交`session_started`。开始事件必须同时提交火山实际会话关联号、`bot_id`、传输模式、平台原样返回的`config_digest`和`dynamic_config_applied=true`；任何一项不一致都不会进入`ACTIVE`，此时不得打开麦克风。通话期间按响应中的心跳间隔提交`session_updated`；响应`action=continue`时保持会话，收到`action=stop_session`时必须立即关闭火山连接，然后提交`session_ended`或`session_failed`。这样，家庭撤回授权、运营停话和设备吊销即使发生在设备直连火山期间，也会收敛到设备执行。
|
||||
|
||||
平台不是靠轮询火山控制台才知道通话状态：v3设备事件在同一事务内更新权威通话状态，既有数据库触发器随`ACTIVE/COMPLETED/FAILED`写入最小`web_event`，已登录SaaS页面通过认证SSE收到`conversation.started/completed/failed`并刷新列表。`session_updated`只续租并更新硬件事件水位，不向家庭页面逐心跳广播，避免每10秒放大为整页查询；运维诊断仍可读取最后事件时间和序号。
|
||||
|
||||
录音能力采用保守门禁：只有创建请求同时声明`near_pcm_recording=true`和`far_pcm_recording=true`时，平台才可能依据既有授权返回`transcript_and_audio`。缺任一能力都会把本次会话强制降级为`transcript_only`（没有字幕授权时为`none`），并把`record_near`、`record_far`、`upload_allowed`全部置为`false`，设备不得以混音或供应商未知格式冒充双轨录音。
|
||||
|
||||
最终字幕写入本地加密重试队列，使用`POST /device/v3/agent-sessions/{id}/turns:batch`上传。只上传用户最终识别事件和智能体`response.audio_transcript.done`，不得长期保存或上传增量字幕；同一`turn_key`或`sequence_no`重放必须保持正文完全一致。高质量传输可额外配置火山服务端状态/字幕回调作为旁路证据；低负载传输按火山当前约束没有该服务端回调，因此设备事件与终稿上传是强制能力，创建请求必须声明`transcript_upload=true`。
|
||||
|
||||
Product Secret只保存在平台KMS/Secret挂载中；平台注册结果中的`device_secret`在平台数据库加密保存，并只下发给对应设备的安全区或供应商SDK凭据存储。普通会话响应和日志不得包含该值。当前Windows标准WebSocket测试程序覆盖平台代理注册和真实低负载协议；高质量方案的火山硬件SDK二进制和板级音频HAL仍必须由目标芯片工程集成后执行真机验收。
|
||||
|
||||
本地控制面闭环可执行：
|
||||
|
||||
```powershell
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\smoke-hardware-agent.ps1
|
||||
```
|
||||
|
||||
脚本先证明过低SDK被HTTP 426拒绝、未完成动态配置不能激活，再验证产品默认Bot绑定、每次会话上下文预取、录音能力安全降级、开始事件、最终字幕、平台停话、`stop_session`下发、设备结束确认、能力快照和数据库状态收敛，并精确清理临时会话及绑定。
|
||||
|
||||
## 12. 错误处理
|
||||
|
||||
| HTTP状态 | 错误码 | 处理建议 |
|
||||
@@ -279,6 +330,8 @@ powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\smoke-sim-firmware
|
||||
| 503 | `DEVICE_AUTHENTICATION_UNAVAILABLE` | 退避后使用新随机数重试，不关闭鉴权 |
|
||||
| 503 | `DEVICE_CONFIG_UNAVAILABLE` / `DEVICE_BOOTSTRAP_FAILED` | 退避重试并保留上一次有效配置 |
|
||||
| 503 | `RUNTIME_CONFIG_UNAVAILABLE` | 生产环境没有命中活动语音路由；停止创建通话，由运营发布或恢复目标范围配置 |
|
||||
| 426 | `HARDWARE_AGENT_SDK_UPDATE_REQUIRED` | 当前火山硬件智能体SDK低于设备绑定门禁；停止创建会话并升级固件/SDK |
|
||||
| 503 | `HARDWARE_AGENT_PROVISIONING_FAILED` | 火山动态注册失败；用原幂等键、原正文和新签名退避重试，并由运维根据`request_id`检查产品License额度、有效期和火山原始错误 |
|
||||
| 403 | `CONSENT_REQUIRED` | 不重试，等待有效授权 |
|
||||
| 403 | `TRANSCRIPT_STORAGE_NOT_ALLOWED` | 停止补传，等待留存策略或授权恢复，不得本地绕过 |
|
||||
| 409 | `DEVICE_NOT_BOUND` / `CONVERSATION_ALREADY_ACTIVE` | 修复绑定或查询已有通话 |
|
||||
@@ -288,6 +341,8 @@ powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\smoke-sim-firmware
|
||||
| 409 | `CONFIG_VERSION_CONFLICT` | 先重新获取设备配置再尝试创建新通话 |
|
||||
| 409 | `PROVIDER_EVENT_CONFLICT` | 供应商事件编号复用且正文不同，转人工排查 |
|
||||
| 409 | `CONVERSATION_STATE_CONFLICT` | 查询云端当前通话状态，不要强制覆盖状态机 |
|
||||
| 401 | `HARDWARE_SESSION_PROOF_INVALID` | 丢弃本地会话，不得尝试其他设备或其他会话的nonce；重新创建会话 |
|
||||
| 409 | `HARDWARE_AGENT_EVENT_CONFLICT` | 保留原事件和单调序号，查询当前会话状态并人工核对，不得覆盖 |
|
||||
| 409 | `TRANSCRIPT_TURN_CONFLICT` | 保留本地终稿和冲突明细，禁止覆盖云端终稿，转入人工对账 |
|
||||
| 409 | `TELEMETRY_EVENT_CONFLICT` | 保留冲突事件并停止推进本地确认点，人工核对事件键和单调序号 |
|
||||
| 404 | `TELEMETRY_CONVERSATION_NOT_FOUND` | 删除错误通话引用，不得改用任意长者编号重试 |
|
||||
|
||||
@@ -0,0 +1,857 @@
|
||||
# 设备注册到聊天记录上传接口文档
|
||||
|
||||
## 1. 设备请求平台时的请求头
|
||||
|
||||
运营后台和用户 App 调用 `/api/v1/*` 时使用：
|
||||
|
||||
```http
|
||||
Authorization: Bearer <用户访问令牌>
|
||||
```
|
||||
|
||||
设备调用 `/device/*` 时使用以下 HMAC 请求头：
|
||||
|
||||
| 请求头 | 中文含义 | 计算方式 |
|
||||
|---|---|---|
|
||||
| `X-Device-Id` | 平台设备编号 | 平台登记设备时返回的 `credential.device_id` |
|
||||
| `X-Device-Timestamp` | 请求时间 | 当前 UTC 时间，RFC3339 格式，例如 `2026-09-05T09:20:00Z` |
|
||||
| `X-Device-Nonce` | 防重放随机数 | 每个请求重新生成，长度 16–128 字符，不得重复 |
|
||||
| `X-Device-Content-SHA256` | 请求正文摘要 | `hex_lower(SHA256(rawBody))`；GET 使用空字节正文 |
|
||||
| `X-Device-Signature` | 请求签名 | `base64url_no_padding(HMAC-SHA256(platformDeviceSecret, canonical))` |
|
||||
| `Idempotency-Key` | 业务幂等键 | 仅指定接口需要；同一次请求重试必须保持不变，长度 16–128 字符 |
|
||||
| `Content-Type` | 正文类型 | 有 JSON 正文时为 `application/json` |
|
||||
|
||||
签名原文 `canonical`：
|
||||
|
||||
```text
|
||||
{大写HTTP方法}\n{请求路径和查询参数}\n{X-Device-Timestamp}\n{X-Device-Nonce}\n{X-Device-Content-SHA256}
|
||||
```
|
||||
|
||||
计算示例：
|
||||
|
||||
```text
|
||||
body = UTF8('{"hardware_id":"echoear-a1b2c3"}')
|
||||
bodySha256 = hex_lower(SHA256(body))
|
||||
timestamp = "2026-09-05T09:20:00Z"
|
||||
nonce = "d8b81b916ddd4f4584a80a82"
|
||||
|
||||
canonical =
|
||||
"POST\n" +
|
||||
"/device/v3/hardware-agent-credentials:provision\n" +
|
||||
timestamp + "\n" +
|
||||
nonce + "\n" +
|
||||
bodySha256
|
||||
|
||||
signature = base64url_no_padding(
|
||||
HMAC-SHA256(platformDeviceSecret, UTF8(canonical))
|
||||
)
|
||||
```
|
||||
|
||||
注意：
|
||||
|
||||
- 摘要必须针对实际发送的原始字节计算，签名后不能再次格式化 JSON。
|
||||
- 查询参数属于签名内容，例如 `/device/v2/commands?limit=10`。
|
||||
- GET 空正文 SHA-256 固定为 `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`。
|
||||
- `platformDeviceSecret` 是平台注册时返回的设备密钥，不是火山 `Product Secret`，也不是火山 `device_secret`。
|
||||
- 平台统一响应：成功为 `{"request_id":"...","data":...}`；失败为 `{"request_id":"...","error":...}`。
|
||||
|
||||
## 2. 完整流程
|
||||
|
||||
| 顺序 | 调用方 | 动作 | 接口/地址 | 结果 |
|
||||
|---:|---|---|---|---|
|
||||
| 1 | 运营平台 | 登记设备 | `POST /api/v1/ops/devices` | 返回平台设备 ID、平台设备密钥和绑定码 |
|
||||
| 2 | 用户 App | 绑定设备 | `POST /api/v1/device-claims` | 状态变为 `BOUND` |
|
||||
| 3 | 设备 | 获取火山运行凭据 | `POST /device/v3/hardware-agent-credentials:provision` | 平台动态注册火山设备，消耗一个 License，状态变为 `ACTIVE` |
|
||||
| 4 | 设备 | 检查固件 | `GET /device/v2/bootstrap` | 得到 `firmware_compatible` |
|
||||
| 5 | 设备 | 获取设备配置 | `GET /device/v2/config` | 得到配置版本和运行时配置 |
|
||||
| 6 | 设备 | 判断是否继续 | `firmware_compatible=false` 时停止聊天并进入 OTA 流程 | 只有兼容固件可以继续 |
|
||||
| 7 | 设备 | 创建对话会话 | `POST /device/v3/agent-sessions` | 获取 Bot ID、系统提示词、人设和授权记忆 |
|
||||
| 8 | 设备 | 连接火山 | 火山 WebSocket + `session.update` | 注入本次人设并建立 16 kHz PCM 对话 |
|
||||
| 9 | 设备 | 上报会话开始 | `POST /device/v3/agent-sessions/{id}/events` | 平台会话变为 `ACTIVE` |
|
||||
| 10 | 设备 | 上传最终聊天文本 | `POST /device/v3/agent-sessions/{id}/turns:batch` | 用户和智能体最终文本入库 |
|
||||
| 11 | 设备 | 上报会话结束 | 同一个 events 接口，`session_ended` | 平台会话变为 `COMPLETED` |
|
||||
|
||||
激活状态：
|
||||
|
||||
```text
|
||||
FACTORY → CLAIMABLE → BOUND → PROVISIONING → ACTIVE → LOST/REVOKED
|
||||
```
|
||||
|
||||
设备重置流程：
|
||||
|
||||
```text
|
||||
停止语音 → 上传剩余聊天记录 → 上报session_ended → POST /device/v3/factory-reset
|
||||
→ 平台解除绑定并暂停火山凭据 → 设备按local_actions清理 → 重启进入CLAIMABLE
|
||||
```
|
||||
|
||||
## 3. 接口
|
||||
|
||||
### 3.1 登记设备
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /api/v1/ops/devices
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---:|---|
|
||||
| `model` | string | 是 | 设备型号 |
|
||||
| `hardware_revision` | string | 否 | 硬件版本 |
|
||||
| `minimum_security_version` | string | 否 | 最低安全版本 |
|
||||
| `minimum_firmware_version` | string | 否 | 最低固件版本 |
|
||||
| `household_id` | UUID | 否 | 直接绑定的家庭编号；走绑定码时不传 |
|
||||
| `subject_id` | UUID | 否 | 直接绑定的服务对象编号；走绑定码时不传 |
|
||||
| `voice_runtime` | enum | 是 | 传 `volc_hardware_agent` |
|
||||
| `hardware_agent_bot_id` | string | 否 | 兼容字段；当前方案不传，由平台产品配置决定 |
|
||||
| `hardware_agent_transport` | enum | 否 | `low_load` 或 `high_quality`；建议由平台配置 |
|
||||
| `hardware_agent_minimum_sdk_version` | string | 否 | 最低硬件智能体 SDK 版本 |
|
||||
| `activation_state` | enum | 是 | `FACTORY`、`CLAIMABLE` 或 `BOUND`；一般传 `CLAIMABLE` |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `device.id` | UUID | 平台设备编号 |
|
||||
| `device.status` | string | 设备业务状态 |
|
||||
| `device.activation_state` | enum | 激活状态 |
|
||||
| `device.activation_revision` | integer | 激活状态版本 |
|
||||
| `device.model` | string | 设备型号 |
|
||||
| `device.hardware_revision` | string/null | 硬件版本 |
|
||||
| `device.minimum_security_version` | string/null | 最低安全版本 |
|
||||
| `device.minimum_firmware_version` | string/null | 最低固件版本 |
|
||||
| `device.voice_runtime` | enum | 语音运行时 |
|
||||
| `device.version` | integer | 设备记录版本 |
|
||||
| `credential.device_id` | UUID | 平台设备编号 |
|
||||
| `credential.secret` | string | 设备调用平台所用的 HMAC 密钥，仅明文返回一次 |
|
||||
| `credential.key_version` | integer | 平台设备密钥版本 |
|
||||
| `credential.issued_at` | datetime | 密钥签发时间 |
|
||||
| `claim.claim_code` | string/null | 一次性绑定码 |
|
||||
| `claim.issued_at` | datetime/null | 绑定码签发时间 |
|
||||
| `claim.expires_at` | datetime/null | 绑定码过期时间 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
POST /api/v1/ops/devices
|
||||
Authorization: Bearer <ops-token>
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"model": "echoear-esp32s3",
|
||||
"hardware_revision": "v1.0",
|
||||
"minimum_firmware_version": "1.0.0",
|
||||
"voice_runtime": "volc_hardware_agent",
|
||||
"activation_state": "CLAIMABLE"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-register-001",
|
||||
"data": {
|
||||
"device": {
|
||||
"id": "10000000-0000-0000-0000-000000000512",
|
||||
"status": "registered",
|
||||
"activation_state": "CLAIMABLE",
|
||||
"activation_revision": 1,
|
||||
"model": "echoear-esp32s3",
|
||||
"voice_runtime": "volc_hardware_agent",
|
||||
"version": 1
|
||||
},
|
||||
"credential": {
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"secret": "<platform-device-secret>",
|
||||
"key_version": 1,
|
||||
"issued_at": "2026-09-05T09:00:00Z"
|
||||
},
|
||||
"claim": {
|
||||
"claim_code": "ABCD-EFGH-IJKL-MNOP",
|
||||
"issued_at": "2026-09-05T09:00:00Z",
|
||||
"expires_at": "2026-09-06T09:00:00Z"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.2 用户绑定设备
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /api/v1/device-claims
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---:|---|
|
||||
| `claim_code` | string | 是 | 一次性绑定码，不能填写设备 UUID |
|
||||
| `household_id` | UUID | 是 | 家庭编号 |
|
||||
| `subject_id` | UUID | 是 | 设备服务对象编号 |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `binding_id` | UUID | 设备绑定关系编号 |
|
||||
| `device_id` | UUID | 平台设备编号 |
|
||||
| `household_id` | UUID | 家庭编号 |
|
||||
| `subject_id` | UUID | 服务对象编号 |
|
||||
| `bound_at` | datetime | 绑定时间 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
POST /api/v1/device-claims
|
||||
Authorization: Bearer <user-token>
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"claim_code": "ABCD-EFGH-IJKL-MNOP",
|
||||
"household_id": "10000000-0000-0000-0000-000000000211",
|
||||
"subject_id": "10000000-0000-0000-0000-000000000411"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-bind-001",
|
||||
"data": {
|
||||
"binding_id": "10000000-0000-0000-0000-000000000611",
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"household_id": "10000000-0000-0000-0000-000000000211",
|
||||
"subject_id": "10000000-0000-0000-0000-000000000411",
|
||||
"bound_at": "2026-09-05T09:10:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.3 占用 License 并获取火山设备凭据
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /device/v3/hardware-agent-credentials:provision
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 位置 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---|---:|---|
|
||||
| `Idempotency-Key` | Header | string | 是 | 凭据申请幂等键，16–128 字符 |
|
||||
| `hardware_id` | Body | string | 是 | 不随固件升级变化的物理硬件标识 |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `instance_id` | string | 火山硬件智能体实例编号 |
|
||||
| `product_key` | string | 火山产品标识 |
|
||||
| `device_name` | string | 火山设备名称 |
|
||||
| `hardware_id` | string | 已绑定的物理硬件标识 |
|
||||
| `device_secret` | string | 火山设备专属密钥，需要加密保存 |
|
||||
| `credential_ref` | string | 平台内部凭据引用 |
|
||||
| `credential_version` | integer | 火山设备凭据版本 |
|
||||
| `credential_source` | enum | `platform_dynamic_registration` 为新注册，`platform_stored` 为复用已有凭据 |
|
||||
| `issued_at` | datetime | 凭据签发时间 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
POST /device/v3/hardware-agent-credentials:provision
|
||||
Idempotency-Key: provision-device-00000512-v1
|
||||
Content-Type: application/json
|
||||
<设备HMAC请求头>
|
||||
|
||||
{"hardware_id":"echoear-esp32s3-a1b2c3d4"}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-provision-001",
|
||||
"data": {
|
||||
"instance_id": "6a92945cc0c4689234aef656",
|
||||
"product_key": "6a9568047dd4b23eacab9b1a",
|
||||
"device_name": "dt-10000000000000000000000000000512",
|
||||
"hardware_id": "echoear-esp32s3-a1b2c3d4",
|
||||
"device_secret": "<volc-device-secret>",
|
||||
"credential_ref": "volc-device-secret://10000000-0000-0000-0000-000000000512",
|
||||
"credential_version": 1,
|
||||
"credential_source": "platform_dynamic_registration",
|
||||
"issued_at": "2026-09-05T09:12:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.4 设备恢复出厂
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /device/v3/factory-reset
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---:|---|
|
||||
| Header `Idempotency-Key` | string | 是 | 本次重置幂等键，16–128 字符；重试保持不变 |
|
||||
| `reset_id` | UUID | 是 | 本次重置唯一编号 |
|
||||
| `reset_counter` | integer | 是 | 设备安全存储中的单调递增重置次数 |
|
||||
| `occurred_at` | datetime | 是 | 设备确认重置的 UTC 时间；不得晚于请求时间 5 分钟 |
|
||||
| `trigger` | enum | 是 | `physical_long_press`、`device_menu` 或 `service_tool` |
|
||||
| `firmware_version` | string | 是 | 当前固件版本 |
|
||||
| `pending_turn_count` | integer | 是 | 未上传聊天记录数；必须为 `0` |
|
||||
| `active_conversation_id` | UUID | 否 | 设备认为仍在运行的会话编号 |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `reset_id` | UUID | 重置编号 |
|
||||
| `device_id` | UUID | 平台设备编号 |
|
||||
| `activation_state` | enum | 重置后固定为 `CLAIMABLE` |
|
||||
| `reset_counter` | integer | 平台接受的重置次数 |
|
||||
| `binding_removed` | boolean | 是否关闭了原用户绑定 |
|
||||
| `previous_binding_id` | UUID/null | 原绑定关系编号 |
|
||||
| `provider_credential_status` | enum | 火山设备凭据状态，通常为 `SUSPENDED` |
|
||||
| `license_action` | enum | 固定为 `RETAINED`，重置不释放火山 License |
|
||||
| `ended_conversation_count` | integer | 平台强制结束的残留会话数 |
|
||||
| `cancelled_command_count` | integer | 平台取消的待执行命令数 |
|
||||
| `config_revision` | integer | 重置后的设备配置版本 |
|
||||
| `claim.code` | string | 新的一次性绑定码 |
|
||||
| `claim.issued_at` | datetime | 绑定码签发时间 |
|
||||
| `claim.expires_at` | datetime | 绑定码过期时间 |
|
||||
| `local_actions` | string[] | 设备收到成功响应后必须执行的本地清理动作 |
|
||||
| `reset_at` | datetime | 平台完成重置的时间 |
|
||||
|
||||
平台同时把旧用户侧的设备状态置为离线，并推送 `device.binding.reset` 事件。设备收到成功响应后，按 `local_actions` 清理本地火山凭据、用户上下文、会话与网络配置；设备平台身份和 HMAC 密钥不得删除。
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
POST /device/v3/factory-reset
|
||||
Idempotency-Key: factory-reset-00000512-0001
|
||||
Content-Type: application/json
|
||||
<设备HMAC请求头>
|
||||
|
||||
{
|
||||
"reset_id": "10000000-0000-0000-0000-000000000901",
|
||||
"reset_counter": 1,
|
||||
"occurred_at": "2026-09-05T09:30:00Z",
|
||||
"trigger": "physical_long_press",
|
||||
"firmware_version": "1.1.0",
|
||||
"pending_turn_count": 0
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-reset-001",
|
||||
"data": {
|
||||
"reset_id": "10000000-0000-0000-0000-000000000901",
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"activation_state": "CLAIMABLE",
|
||||
"reset_counter": 1,
|
||||
"binding_removed": true,
|
||||
"previous_binding_id": "10000000-0000-0000-0000-000000000611",
|
||||
"provider_credential_status": "SUSPENDED",
|
||||
"license_action": "RETAINED",
|
||||
"ended_conversation_count": 0,
|
||||
"cancelled_command_count": 0,
|
||||
"config_revision": 9,
|
||||
"claim": {
|
||||
"code": "QRST-UVWX-YZ23-4567",
|
||||
"issued_at": "2026-09-05T09:30:01Z",
|
||||
"expires_at": "2026-09-06T09:30:01Z"
|
||||
},
|
||||
"local_actions": [
|
||||
"clear_volc_credential",
|
||||
"clear_user_context",
|
||||
"clear_session_state",
|
||||
"clear_network_credentials",
|
||||
"reboot_to_claimable"
|
||||
],
|
||||
"reset_at": "2026-09-05T09:30:01Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
设备必须保留平台 `device_id`、平台 HMAC `secret` 和 `hardware_id`；其余用户上下文、火山凭据、会话状态和网络配置按 `local_actions` 清除。
|
||||
|
||||
### 3.5 检查固件版本
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
GET /device/v2/bootstrap
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 位置 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---|---:|---|
|
||||
| `firmware_version` | Query | string | 是 | 当前固件版本 |
|
||||
| `device_time` | Query | datetime | 是 | 设备 UTC 时间，RFC3339 |
|
||||
| `board_rev` | Query | string | 否 | 硬件版本 |
|
||||
| `rtc_sdk_version` | Query | string | 否 | 火山 SDK 版本 |
|
||||
| `codec` | Query | string | 否 | 音频格式，例如 `pcm16` |
|
||||
| `sample_rate` | Query | integer | 否 | 音频采样率，当前为 `16000` |
|
||||
| `network` | Query | string | 否 | 网络类型，例如 `wifi` |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `server_time` | datetime | 平台 UTC 时间 |
|
||||
| `clock_skew_ms` | integer | 平台时间减设备时间的毫秒差 |
|
||||
| `device.device_id` | UUID | 平台设备编号 |
|
||||
| `device.status` | string | 设备业务状态 |
|
||||
| `device.model` | string | 设备型号 |
|
||||
| `device.hardware_revision` | string/null | 硬件版本 |
|
||||
| `device.minimum_security_version` | string/null | 最低安全版本 |
|
||||
| `device.minimum_firmware_version` | string/null | 最低固件版本 |
|
||||
| `device.firmware_compatible` | boolean | 当前固件是否允许继续聊天 |
|
||||
| `device.config_revision` | integer | 当前平台配置版本 |
|
||||
| `device.binding.bound` | boolean | 是否已绑定 |
|
||||
| `device.binding.household_id` | UUID/null | 家庭编号 |
|
||||
| `device.binding.subject_id` | UUID/null | 服务对象编号 |
|
||||
| `device.binding.subject_name` | string/null | 服务对象名称 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
GET /device/v2/bootstrap?firmware_version=1.0.0&board_rev=v1.0&rtc_sdk_version=1.1.0&codec=pcm16&sample_rate=16000&network=wifi&device_time=2026-09-05T09%3A15%3A00Z
|
||||
<设备HMAC请求头>
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-bootstrap-001",
|
||||
"data": {
|
||||
"server_time": "2026-09-05T09:15:01Z",
|
||||
"clock_skew_ms": 1000,
|
||||
"device": {
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"status": "registered",
|
||||
"model": "echoear-esp32s3",
|
||||
"minimum_firmware_version": "1.0.0",
|
||||
"firmware_compatible": true,
|
||||
"config_revision": 8,
|
||||
"binding": {
|
||||
"bound": true,
|
||||
"household_id": "10000000-0000-0000-0000-000000000211",
|
||||
"subject_id": "10000000-0000-0000-0000-000000000411",
|
||||
"subject_name": "测试用户"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.6 获取设备配置
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
GET /device/v2/config
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 位置 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---|---:|---|
|
||||
| `If-None-Match` | Header | string | 否 | 本地配置 ETag；未变化时返回 `304` |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| Response Header `ETag` | string | 当前配置 ETag |
|
||||
| `device_id` | UUID | 平台设备编号 |
|
||||
| `revision` | integer | 配置版本 |
|
||||
| `desired_config` | object | 平台期望设备应用的配置 |
|
||||
| `desired_config.voice_runtime` | enum | 语音运行时 |
|
||||
| `desired_config.hardware_agent_transport` | enum | 火山传输方式 |
|
||||
| `desired_config.minimum_sdk_version` | string | 最低 SDK 版本 |
|
||||
| `reported_revision` | integer/null | 设备此前报告的配置版本 |
|
||||
| `binding` | object | 当前绑定信息，字段与 bootstrap 相同 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
GET /device/v2/config
|
||||
If-None-Match: "7"
|
||||
<设备HMAC请求头>
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-config-001",
|
||||
"data": {
|
||||
"device_id": "10000000-0000-0000-0000-000000000512",
|
||||
"revision": 8,
|
||||
"desired_config": {
|
||||
"voice_runtime": "volc_hardware_agent",
|
||||
"hardware_agent_transport": "low_load",
|
||||
"minimum_sdk_version": "1.1.0"
|
||||
},
|
||||
"reported_revision": 7,
|
||||
"binding": {
|
||||
"bound": true,
|
||||
"household_id": "10000000-0000-0000-0000-000000000211",
|
||||
"subject_id": "10000000-0000-0000-0000-000000000411"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.7 创建会话并获取系统提示词
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---:|---|
|
||||
| Header `Idempotency-Key` | string | 是 | 创建会话幂等键，16–128 字符 |
|
||||
| `device_time` | datetime | 是 | 设备 UTC 时间 |
|
||||
| `monotonic_ms` | integer | 否 | 开机后的单调时钟毫秒数 |
|
||||
| `capabilities.hardware_agent` | boolean | 是 | 必须为 `true` |
|
||||
| `capabilities.sdk_version` | string | 是 | 硬件智能体 SDK 版本 |
|
||||
| `capabilities.transports` | string[] | 是 | 支持的传输方式 |
|
||||
| `capabilities.input_audio_format` | string | 是 | 上行音频能力，当前为 `pcm_s16le_16000_mono` |
|
||||
| `capabilities.output_audio_format` | string | 是 | 下行音频能力，当前为 `pcm_s16le_16000_mono` |
|
||||
| `capabilities.server_event_callback` | boolean | 是 | 当前传 `false` |
|
||||
| `capabilities.transcript_upload` | boolean | 是 | 必须为 `true` |
|
||||
| `capabilities.dynamic_session_config` | boolean | 是 | 必须为 `true` |
|
||||
| `capabilities.credential_rotation` | boolean | 是 | 必须为 `true` |
|
||||
| `capabilities.near_pcm_recording` | boolean | 是 | 是否支持近端录音 |
|
||||
| `capabilities.far_pcm_recording` | boolean | 是 | 是否支持远端录音 |
|
||||
| `network.type` | string | 是 | 网络类型 |
|
||||
| `network.rssi` | integer | 否 | Wi-Fi RSSI |
|
||||
| `requested_goal` | string | 是 | 固定为 `open_conversation` |
|
||||
| `config_revision` | integer | 是 | `/device/v2/config` 返回的配置版本 |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `conversation_id` | UUID | 平台会话编号 |
|
||||
| `status` | string | 正常为 `SESSION_READY` |
|
||||
| `runtime_type` | string | `volc_hardware_agent` |
|
||||
| `runtime.provider` | string | 服务商，当前为 `volcengine` |
|
||||
| `runtime.bot_id` | string | 本次使用的 Bot ID |
|
||||
| `runtime.transport` | string | 本次传输方式 |
|
||||
| `runtime.provider_session_ref` | string | 火山会话引用 |
|
||||
| `runtime.session_nonce` | string | 会话证明随机值 |
|
||||
| `runtime.config_digest` | string | 本次动态配置摘要 |
|
||||
| `runtime.wait_for_session_update` | boolean | 是否必须先应用动态人设 |
|
||||
| `session_config.dialog_id` | string | 本次对话标识 |
|
||||
| `session_config.system_prompt` | string | 系统提示词/核心人设 |
|
||||
| `session_config.persona_tone` | string | 人设语气和表达风格 |
|
||||
| `session_config.authorized_context` | string | 授权的用户背景、近期摘要和长期记忆 |
|
||||
| `session_config.prompt_bundle_version` | string | 提示词版本 |
|
||||
| `session_config.tool_url` | URL/null | 工具调用地址 |
|
||||
| `session_config.tool_scope_token` | string/null | 本次会话工具令牌 |
|
||||
| `privacy.retention_mode` | string | 数据保留模式 |
|
||||
| `privacy.record_near` | boolean | 是否允许近端录音 |
|
||||
| `privacy.record_far` | boolean | 是否允许远端录音 |
|
||||
| `privacy.upload_allowed` | boolean | 是否允许上传聊天记录 |
|
||||
| `session.engine_mode` | string | 会话引擎模式 |
|
||||
| `session.heartbeat_interval_sec` | integer | 会话状态上报间隔 |
|
||||
| `session.allocation_expires_at` | datetime | 会话分配过期时间 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions
|
||||
Idempotency-Key: session-device-00000512-0001
|
||||
Content-Type: application/json
|
||||
<设备HMAC请求头>
|
||||
|
||||
{
|
||||
"device_time": "2026-09-05T09:20:00Z",
|
||||
"monotonic_ms": 812345,
|
||||
"capabilities": {
|
||||
"hardware_agent": true,
|
||||
"sdk_version": "1.1.0-echoear",
|
||||
"transports": ["low_load"],
|
||||
"input_audio_format": "pcm_s16le_16000_mono",
|
||||
"output_audio_format": "pcm_s16le_16000_mono",
|
||||
"server_event_callback": false,
|
||||
"transcript_upload": true,
|
||||
"dynamic_session_config": true,
|
||||
"credential_rotation": true,
|
||||
"near_pcm_recording": false,
|
||||
"far_pcm_recording": false
|
||||
},
|
||||
"network": {"type":"wifi","rssi":-52},
|
||||
"requested_goal": "open_conversation",
|
||||
"config_revision": 8
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-session-001",
|
||||
"data": {
|
||||
"conversation_id": "10000000-0000-0000-0000-000000000801",
|
||||
"status": "SESSION_READY",
|
||||
"runtime_type": "volc_hardware_agent",
|
||||
"runtime": {
|
||||
"provider": "volcengine",
|
||||
"bot_id": "botM4gWQ9A9y",
|
||||
"transport": "low_load",
|
||||
"provider_session_ref": "provider-session-ref",
|
||||
"session_nonce": "<session-nonce>",
|
||||
"config_digest": "<config-digest>",
|
||||
"wait_for_session_update": true
|
||||
},
|
||||
"session_config": {
|
||||
"dialog_id": "dialog-00000801",
|
||||
"system_prompt": "你是面向长者的陪伴助手……",
|
||||
"persona_tone": "率真、自然",
|
||||
"authorized_context": "经授权的背景和记忆……",
|
||||
"prompt_bundle_version": "persona-v12"
|
||||
},
|
||||
"privacy": {
|
||||
"retention_mode": "transcript_only",
|
||||
"record_near": false,
|
||||
"record_far": false,
|
||||
"upload_allowed": true
|
||||
},
|
||||
"session": {
|
||||
"engine_mode": "hardware_agent",
|
||||
"heartbeat_interval_sec": 30,
|
||||
"allocation_expires_at": "2026-09-05T09:25:00Z"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.8 连接火山并注入人设
|
||||
|
||||
地址：
|
||||
|
||||
```text
|
||||
wss://ai-gateway.vei.volces.com/v1/realtime?bot={bot_id}&wait_for_session_update=true
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 位置 | 中文含义 | 来源/计算 |
|
||||
|---|---|---|---|
|
||||
| `bot` | Query | Bot ID | 创建会话返回的 `runtime.bot_id` |
|
||||
| `X-Auth-Type` | Header | 鉴权类型 | 固定为 `1` |
|
||||
| `X-Product-Key` | Header | 火山产品标识 | provision 返回的 `product_key` |
|
||||
| `X-Device-Name` | Header | 火山设备名称 | provision 返回的 `device_name` |
|
||||
| `X-Random-Num` | Header | 随机数 | 每次连接重新生成 |
|
||||
| `X-Timestamp` | Header | Unix 秒 | 当前时间 |
|
||||
| `X-Instance-Id` | Header | 火山实例编号 | provision 返回的 `instance_id` |
|
||||
| `X-Hardware-Id` | Header | 物理硬件标识 | provision 使用的 `hardware_id` |
|
||||
| `X-Signature` | Header | 火山鉴权签名 | `base64(HMAC-SHA256(device_secret, canonical))` |
|
||||
| `session.update` | WebSocket JSON | 动态会话配置 | 注入平台返回的人设和记忆 |
|
||||
|
||||
输出：
|
||||
|
||||
| 事件 | 中文含义 |
|
||||
|---|---|
|
||||
| `session.created` / `session.updated` | 火山会话创建/动态人设生效 |
|
||||
| `conversation.item.input_audio_transcription.completed` | 用户最终识别文本 |
|
||||
| `response.audio_transcript.done` | 智能体最终回复文本 |
|
||||
| 音频数据 | 智能体返回的 PCM16 音频，按 16 kHz 单声道播放 |
|
||||
| error 事件 | 火山连接或会话失败 |
|
||||
|
||||
示例：
|
||||
|
||||
```text
|
||||
canonical = "auth_type=1&device_name={device_name}&random_num={random_num}&product_key={product_key}×tamp={timestamp}&instance_id={instance_id}"
|
||||
X-Signature = base64(HMAC-SHA256(device_secret, canonical))
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "evt-session-update-001",
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"object": "realtime.session",
|
||||
"model": "",
|
||||
"input_audio_format": "pcm16",
|
||||
"output_audio_format": "pcm16",
|
||||
"config": {
|
||||
"ASRConfig": {"TurnDetectionMode": 0},
|
||||
"LLMConfig": {
|
||||
"SystemMessages": [
|
||||
"<system_prompt>\n表达风格：<persona_tone>\n授权上下文：<authorized_context>\n会话标识：<dialog_id>"
|
||||
]
|
||||
}
|
||||
},
|
||||
"agent_config": {"EnableConversationStateCallback": false}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.9 上报会话事件
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions/{conversation_id}/events
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---:|---|
|
||||
| Path `conversation_id` | UUID | 是 | 平台会话编号 |
|
||||
| `event_id` | string | 是 | 事件唯一编号 |
|
||||
| `sequence_no` | integer | 是 | 事件顺序号，从 1 开始递增 |
|
||||
| `event_type` | enum | 是 | `session_started`、`session_updated`、`session_ended` 或 `session_failed` |
|
||||
| `occurred_at` | datetime | 是 | 事件发生时间 |
|
||||
| `provider_session_ref` | string | 是 | 火山会话引用 |
|
||||
| `session_nonce` | string | 是 | 会话证明随机值 |
|
||||
| `provider_connection_ref` | string | 启动时必填 | WebSocket 连接标识 |
|
||||
| `bot_id` | string | 启动时必填 | 实际连接的 Bot ID |
|
||||
| `transport` | string | 启动时必填 | 实际传输方式 |
|
||||
| `config_digest` | string | 启动时必填 | 已应用的人设配置摘要 |
|
||||
| `dynamic_config_applied` | boolean | 启动时必填 | 必须为 `true` |
|
||||
| `error_code` | string | 失败时填写 | 失败错误码 |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `conversation_id` | UUID | 平台会话编号 |
|
||||
| `status` | string | 会话状态，开始后为 `ACTIVE`，正常结束后为 `COMPLETED` |
|
||||
| `action` | enum | `continue` 或 `stop_session` |
|
||||
| `duplicate` | boolean | 是否为重复事件 |
|
||||
| `accepted_at` | datetime | 平台接收时间 |
|
||||
|
||||
示例：
|
||||
|
||||
开始请求：
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "device-event-start-001",
|
||||
"sequence_no": 1,
|
||||
"event_type": "session_started",
|
||||
"occurred_at": "2026-09-05T09:20:03Z",
|
||||
"provider_session_ref": "provider-session-ref",
|
||||
"session_nonce": "<session-nonce>",
|
||||
"provider_connection_ref": "ws-connection-001",
|
||||
"bot_id": "botM4gWQ9A9y",
|
||||
"transport": "low_load",
|
||||
"config_digest": "<config-digest>",
|
||||
"dynamic_config_applied": true
|
||||
}
|
||||
```
|
||||
|
||||
结束示例：
|
||||
|
||||
```json
|
||||
{
|
||||
"event_id": "device-event-end-001",
|
||||
"sequence_no": 2,
|
||||
"event_type": "session_ended",
|
||||
"occurred_at": "2026-09-05T09:25:00Z",
|
||||
"provider_session_ref": "provider-session-ref",
|
||||
"session_nonce": "<session-nonce>"
|
||||
}
|
||||
```
|
||||
|
||||
响应示例：
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-event-001",
|
||||
"data": {
|
||||
"conversation_id": "10000000-0000-0000-0000-000000000801",
|
||||
"status": "ACTIVE",
|
||||
"action": "continue",
|
||||
"duplicate": false,
|
||||
"accepted_at": "2026-09-05T09:20:03Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.10 上传聊天记录
|
||||
|
||||
地址：
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions/{conversation_id}/turns:batch
|
||||
```
|
||||
|
||||
输入：
|
||||
|
||||
| 字段 | 类型 | 必填 | 中文含义 |
|
||||
|---|---|---:|---|
|
||||
| Path `conversation_id` | UUID | 是 | 平台会话编号 |
|
||||
| Header `Idempotency-Key` | string | 是 | 本批记录幂等键，16–128 字符 |
|
||||
| `turns` | array | 是 | 最终聊天记录数组，每批 1–500 条 |
|
||||
| `turns[].turn_key` | string | 是 | 会话内唯一的轮次标识，最长 255 字符 |
|
||||
| `turns[].sequence_no` | integer | 是 | 对话顺序号，会话内唯一且大于 0 |
|
||||
| `turns[].speaker` | enum | 是 | 用户为 `subject`，智能体为 `agent` |
|
||||
| `turns[].start_at` | datetime | 是 | 本轮开始时间 |
|
||||
| `turns[].end_at` | datetime | 是 | 本轮结束时间，不早于开始时间 |
|
||||
| `turns[].final_text` | string | 是 | 最终文本，最长 20000 字符 |
|
||||
| `turns[].interrupted` | boolean | 是 | 本轮是否被打断 |
|
||||
|
||||
输出：
|
||||
|
||||
| 字段 | 类型 | 中文含义 |
|
||||
|---|---|---|
|
||||
| `conversation_id` | UUID | 平台会话编号 |
|
||||
| `accepted_count` | integer | 本次新写入的记录数 |
|
||||
| `duplicate_count` | integer | 本次幂等重复的记录数 |
|
||||
|
||||
示例：
|
||||
|
||||
```http
|
||||
POST /device/v3/agent-sessions/10000000-0000-0000-0000-000000000801/turns:batch
|
||||
Idempotency-Key: turns-conversation-00000801-0001
|
||||
Content-Type: application/json
|
||||
<设备HMAC请求头>
|
||||
|
||||
{
|
||||
"turns": [
|
||||
{
|
||||
"turn_key": "user-final-001",
|
||||
"sequence_no": 1,
|
||||
"speaker": "subject",
|
||||
"start_at": "2026-09-05T09:20:10Z",
|
||||
"end_at": "2026-09-05T09:20:12Z",
|
||||
"final_text": "今天天气怎么样？",
|
||||
"interrupted": false
|
||||
},
|
||||
{
|
||||
"turn_key": "agent-final-001",
|
||||
"sequence_no": 2,
|
||||
"speaker": "agent",
|
||||
"start_at": "2026-09-05T09:20:12Z",
|
||||
"end_at": "2026-09-05T09:20:16Z",
|
||||
"final_text": "我来帮你看看。",
|
||||
"interrupted": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"request_id": "req-turns-001",
|
||||
"data": {
|
||||
"conversation_id": "10000000-0000-0000-0000-000000000801",
|
||||
"accepted_count": 2,
|
||||
"duplicate_count": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"diagram_type": "sequence",
|
||||
"meta": {
|
||||
"title": "设备从注册到聊天结束的完整流程",
|
||||
"locale": "zh-CN",
|
||||
"output": "docs/diagrams/device-conversation-lifecycle.html",
|
||||
"viewBox": [1080, 590],
|
||||
"quality_profile": "showcase",
|
||||
"column_fit": "spread"
|
||||
},
|
||||
"participants": [
|
||||
{ "id": "ops", "type": "frontend", "label": "运维平台", "sublabel": "登记 / 放行" },
|
||||
{ "id": "app", "type": "frontend", "label": "用户 App", "sublabel": "扫码绑定" },
|
||||
{ "id": "device", "type": "external", "label": "EchoEar", "sublabel": "ESP32-S3" },
|
||||
{ "id": "saas", "type": "backend", "label": "自有 SaaS", "sublabel": "设备控制面" },
|
||||
{ "id": "volcIot", "type": "cloud", "label": "火山 IoT", "sublabel": "DynamicRegister" },
|
||||
{ "id": "volcAgent", "type": "cloud", "label": "火山智能体", "sublabel": "实时对话" }
|
||||
],
|
||||
"segments": [
|
||||
{ "from": 150, "to": 226, "label": "一、平台注册与绑定" },
|
||||
{ "from": 232, "to": 310, "label": "二、首次预配（仅一次）" },
|
||||
{ "from": 316, "to": 424, "label": "三、每次开始聊天" },
|
||||
{ "from": 430, "to": 520, "label": "四、聊天与结束" }
|
||||
],
|
||||
"messages": [
|
||||
{ "id": "register", "from": "ops", "to": "saas", "y": 166, "label": "登记设备，生成平台 Device Secret", "variant": "emphasis" },
|
||||
{ "id": "inject", "from": "saas", "to": "device", "y": 194, "label": "安全注入平台身份（无 Product Secret）", "variant": "security" },
|
||||
{ "id": "claim", "from": "app", "to": "saas", "y": 222, "label": "消费绑定码：CLAIMABLE → BOUND", "variant": "emphasis" },
|
||||
{ "id": "provision", "from": "device", "to": "saas", "y": 250, "label": "请求 hardware-agent-credentials:provision", "variant": "emphasis" },
|
||||
{ "id": "dynamicRegister", "from": "saas", "to": "volcIot", "y": 278, "label": "DynamicRegister：新设备消耗 1 个 License", "variant": "security" },
|
||||
{ "id": "active", "from": "saas", "to": "device", "y": 306, "label": "加密保存并下发一机一密 → ACTIVE", "variant": "return" },
|
||||
{ "id": "start", "from": "device", "to": "saas", "y": 334, "label": "POST /device/v3/agent-sessions", "variant": "emphasis" },
|
||||
{ "id": "config", "from": "saas", "to": "device", "y": 362, "label": "返回 Bot、人设、授权上下文与 nonce", "variant": "return" },
|
||||
{ "id": "connect", "from": "device", "to": "volcAgent", "y": 390, "label": "一机一密建连并注入 session.update", "variant": "security" },
|
||||
{ "id": "started", "from": "device", "to": "saas", "y": 418, "label": "配置确认后上报 session_started", "variant": "emphasis" },
|
||||
{ "id": "audio", "from": "device", "to": "volcAgent", "y": 446, "label": "双向音频 / 打断 / 工具调用", "variant": "default" },
|
||||
{ "id": "sync", "from": "device", "to": "saas", "y": 474, "label": "session_updated + 最终字幕回流", "variant": "dashed" },
|
||||
{ "id": "ended", "from": "device", "to": "saas", "y": 506, "label": "断开火山并上报 ended / failed → 摘要记忆", "variant": "emphasis" }
|
||||
],
|
||||
"activations": [
|
||||
{ "participant": "saas", "from": 160, "to": 228, "type": "backend" },
|
||||
{ "participant": "saas", "from": 240, "to": 312, "type": "backend" },
|
||||
{ "participant": "volcIot", "from": 268, "to": 286, "type": "cloud" },
|
||||
{ "participant": "saas", "from": 324, "to": 424, "type": "backend" },
|
||||
{ "participant": "volcAgent", "from": 380, "to": 512, "type": "cloud" },
|
||||
{ "participant": "saas", "from": 468, "to": 514, "type": "backend" }
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 126 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 122 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 164 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 161 KiB |
@@ -0,0 +1,32 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Archify visual-check · device-conversation-lifecycle.html</title>
|
||||
<style>
|
||||
*{box-sizing:border-box}body{margin:0;padding:24px;background:#e9eef5;color:#172033;font:14px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}header{max-width:1500px;margin:0 auto 18px}h1{margin:0 0 6px;font-size:20px}p{margin:0;color:#526176}.grid{max-width:1500px;margin:auto;display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:18px}figure{margin:0;padding:10px;background:white;border:1px solid #c9d4e3;border-radius:12px;box-shadow:0 10px 30px rgba(15,23,42,.08)}img{display:block;width:100%;height:auto;border:1px solid #e2e8f0}figcaption{padding:9px 4px 2px;color:#526176}@media(max-width:900px){.grid{grid-template-columns:1fr}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><h1>Archify visual-check</h1><p>device-conversation-lifecycle.html · automated containment pass · visual review pending</p></header>
|
||||
<main class="grid">
|
||||
<figure>
|
||||
<img src="device-conversation-lifecycle.visual-check.1440x900.light.png" alt="light 1440 by 900">
|
||||
<figcaption><strong>LIGHT</strong> · 1440×900 · containment pass</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="device-conversation-lifecycle.visual-check.1440x900.dark.png" alt="dark 1440 by 900">
|
||||
<figcaption><strong>DARK</strong> · 1440×900 · containment pass</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="device-conversation-lifecycle.visual-check.2048x1320.light.png" alt="light 2048 by 1320">
|
||||
<figcaption><strong>LIGHT</strong> · 2048×1320 · containment pass</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="device-conversation-lifecycle.visual-check.2048x1320.dark.png" alt="dark 2048 by 1320">
|
||||
<figcaption><strong>DARK</strong> · 2048×1320 · containment pass</figcaption>
|
||||
</figure>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,547 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"ok": true,
|
||||
"command": "visual-check",
|
||||
"status": "pass",
|
||||
"visualReview": "pending",
|
||||
"artifact": {
|
||||
"path": "E:\\Codex\\story\\platform\\docs\\diagrams\\device-conversation-lifecycle.html",
|
||||
"sha256": "7fade308f989a6d95791c6d018dfcf4c5f50aafb8f6ce6d94c907943362eeea0",
|
||||
"bytes": 715115
|
||||
},
|
||||
"state": {
|
||||
"detail": "read",
|
||||
"motion": "still"
|
||||
},
|
||||
"chrome": {
|
||||
"status": "available",
|
||||
"executable": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
|
||||
},
|
||||
"diagnostics": [],
|
||||
"containment": {
|
||||
"status": "pass",
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1376,
|
||||
"diagramWidth": 1346,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1600,
|
||||
"height": 1000,
|
||||
"theme": "light",
|
||||
"innerWidth": 1600,
|
||||
"innerHeight": 1000,
|
||||
"scrollWidth": 1600,
|
||||
"scrollHeight": 1000,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1456,
|
||||
"diagramWidth": 1426,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"theme": "light",
|
||||
"innerWidth": 1920,
|
||||
"innerHeight": 1080,
|
||||
"scrollWidth": 1920,
|
||||
"scrollHeight": 1080,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1602,
|
||||
"diagramWidth": 1572,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1920,
|
||||
"diagramWidth": 1870,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
}
|
||||
]
|
||||
},
|
||||
"readability": {
|
||||
"status": "pass",
|
||||
"minimumProjectedNodeTextPx": 6,
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1376,
|
||||
"diagramWidth": 1346,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1600,
|
||||
"height": 1000,
|
||||
"theme": "light",
|
||||
"innerWidth": 1600,
|
||||
"innerHeight": 1000,
|
||||
"scrollWidth": 1600,
|
||||
"scrollHeight": 1000,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1456,
|
||||
"diagramWidth": 1426,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"theme": "light",
|
||||
"innerWidth": 1920,
|
||||
"innerHeight": 1080,
|
||||
"scrollWidth": 1920,
|
||||
"scrollHeight": 1080,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1602,
|
||||
"diagramWidth": 1572,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1920,
|
||||
"diagramWidth": 1870,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
}
|
||||
]
|
||||
},
|
||||
"viewerChrome": {
|
||||
"status": "pass",
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1376,
|
||||
"diagramWidth": 1346,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1600,
|
||||
"height": 1000,
|
||||
"theme": "light",
|
||||
"innerWidth": 1600,
|
||||
"innerHeight": 1000,
|
||||
"scrollWidth": 1600,
|
||||
"scrollHeight": 1000,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1456,
|
||||
"diagramWidth": 1426,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"theme": "light",
|
||||
"innerWidth": 1920,
|
||||
"innerHeight": 1080,
|
||||
"scrollWidth": 1920,
|
||||
"scrollHeight": 1080,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1602,
|
||||
"diagramWidth": 1572,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1920,
|
||||
"diagramWidth": 1870,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
}
|
||||
]
|
||||
},
|
||||
"captures": {
|
||||
"status": "pass",
|
||||
"screenshots": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1376,
|
||||
"diagramWidth": 1346,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light",
|
||||
"file": "device-conversation-lifecycle.visual-check.1440x900.light.png"
|
||||
},
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "dark",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1376,
|
||||
"diagramWidth": 1346,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "dark",
|
||||
"file": "device-conversation-lifecycle.visual-check.1440x900.dark.png"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1920,
|
||||
"diagramWidth": 1870,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light",
|
||||
"file": "device-conversation-lifecycle.visual-check.2048x1320.light.png"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "dark",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1920,
|
||||
"diagramWidth": 1870,
|
||||
"viewBoxWidth": 1080,
|
||||
"minimumProjectedNodeTextPx": 7,
|
||||
"minimumProjectedNodeText": "登记 / 放行",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "dark",
|
||||
"file": "device-conversation-lifecycle.visual-check.2048x1320.dark.png"
|
||||
}
|
||||
],
|
||||
"contactSheet": "device-conversation-lifecycle.visual-check.html"
|
||||
},
|
||||
"sidecars": {
|
||||
"receipt": "device-conversation-lifecycle.visual-check.json",
|
||||
"contactSheet": "device-conversation-lifecycle.visual-check.html"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# EchoEar reboot history investigation — 2026-09-06
|
||||
|
||||
User reports loss of prior conversation context after power cycling and asks how the platform receives the conversation record.
|
||||
|
||||
## Existing path verified in source
|
||||
|
||||
- Device receives final subject transcription and agent audio transcript events, queues them outside the WebSocket receive callback, and POSTs to `/device/v3/agent-sessions/{conversation_id}/turns:batch`.
|
||||
- Backend authenticates the bound device, checks transcript retention and consent, deduplicates final turns, and writes `conversation_turn`. The web app exposes these through Chat records → Conversation detail; API: `GET /api/v1/conversations/{id}/turns`.
|
||||
- This path uploads text, not near/far audio recordings. Earlier device logs contain successful HTTP POSTs during exchanges but did not distinguish accepted transcript counts. New firmware logs accepted/duplicate counts without logging transcript content.
|
||||
- Each new connection creates a fresh provider session. Authorized platform context is supplied through `session_config.authorized_context`, which firmware includes in `LLMConfig.SystemMessages`. Record storage and cross-session context are separate steps.
|
||||
|
||||
## Defects and local changes
|
||||
|
||||
1. Device omitted periodic session renewal. The server starts a lease on `session_started`, and `session_updated` renews it. Firmware now reads the server heartbeat interval, renews outside the audio callback using the existing reusable HTTP connection, and honors `stop_session`.
|
||||
2. Recent summary selection excluded all `FAILED` conversations. It now accepts ready summaries from completed or failed conversations while preserving tenant/subject checks and existing consent gates. No raw transcript bypass or automatic long-term memory creation was added.
|
||||
3. Hardware normal completion requested extraction generation 1; the extraction processor ignores a generation below the highest stored turn sequence. Hardware completion, reported failure, and server stop-timeout recovery now schedule the latest stored sequence with a deduplicated terminal extraction key. Existing extraction consent and content safety checks remain.
|
||||
|
||||
## Verification and pending work
|
||||
|
||||
**Device update 2026-09-06 10:20:** COM6 reconnected. Final heartbeat firmware `0x2ccf40` flashed with hash verification and booted to connected standby. The real user test session `157f866d-c364-4ae8-b169-50ca07024e5a` completed normally with 3 turns (including the preceding diagnostic sentence) and 2 READY summaries. Periodic heartbeats succeeded. Native Volcano callbacks were received for input and output, but contain only device identity and do not honor the attempted per-session callback override in this low_load test. callbacks memory3 validates these as device_probe without writing transcripts; device upload remains canonical. User has been asked to power-cycle and ask which joke was just told; actual recall outcome is pending. See the newest section of `volc-native-subtitle-callback.md`.
|
||||
|
||||
**Update 2026-09-06:** Supersedes the deployment/test limitations below. The user-supplied Desktop PEM successfully authenticated root. Backend `0.78.3-rc.subtitle-memory1` is now deployed; independent PostgreSQL regression tests actually passed (not skipped), including failed-session memory eligibility and terminal extraction generation. Production has stored transcripts and READY summaries (one failed session has 50 turns). All three backend readiness checks pass. Native callback protocol smoke checks also pass; real provider callbacks and device reboot recall remain unverified because COM6 is absent. The latest firmware build is `0x2ccf40`, still **not flashed**. See `volc-native-subtitle-callback.md` for deployment commands and outstanding checks. Earlier entries below are retained as investigation history.
|
||||
|
||||
- Firmware build passed: `firmware/echoear-platform-agent/build/voice-history-heartbeat-build.log`; binary size `0x2ccd50`.
|
||||
- `go test ./internal/orchestration ./internal/httpapi ./internal/extraction` passed the available tests. New database regression cases cover failed/completed/active summary eligibility, other-subject exclusion, and extraction generation/deduplication. They were **skipped**, not validated against a database: `TEST_DATABASE_URL` is unset and Docker daemon is unavailable.
|
||||
- **Firmware is not flashed.** COM6 disappeared after the user's power cycle. COM3 is a different USB device; COM4/5 are Bluetooth ports. Do not flash them.
|
||||
- **Backend changes are not deployed.** Existing deployment key was rejected for `root@121.41.46.238`. Requested the actual SSH username; never request passwords/private key content.
|
||||
- Awaiting USB reconnection and server access. Next verify actual transcript row counts, summary READY status, subject consent state, extraction job status, new context inclusion, and a real power-cycle recall test. Do not claim that online records or memory restoration have been verified.
|
||||
- Last verified flashed app remains HTTP-reuse version `0x2cc8a0`; user confirmed continuous speech. Preserve its playback/display improvements and NVS during the next app-only flash.
|
||||
@@ -138,3 +138,26 @@ powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\smoke-firmware-rol
|
||||
A/B启动确认批次在OpenAPI 0.62.0下扩展可选回执阶段，不增加旧设备必填字段或主状态枚举。启用设备先将制品写入非活动分区并持久化启动尝试，再依次上报`staging`、`reboot_pending`和`boot_verifying`；健康通过才提交成功，健康失败或超时则恢复旧分区并提交`FAILED/rolled_back`及回退版本。云端将三类中间阶段保持为`INSTALLING`，同时保留完整`last_result`供页面展示。低强度验证只执行定向Go用例、前端类型/生产构建和一次B分区启动失败自动回A分区的纵向冒烟，不把模拟结果作为真实Bootloader验收。详细协议见[设备A/B分区固件升级与启动确认协议](device-firmware-ab-protocol.md)。
|
||||
|
||||
OpenAPI 0.63.0和V38进一步为每台固件分配保存去重的阶段事件。直接命令ACK与批量遥测补交在更新命令/分配的同一事务内写入事件，重复结果摘要通过SHA-256唯一键保持幂等；批次详情按设备返回完整时间线，页面显示云端接收时间、阶段、命令状态和稳定结果码。`cmd/firmware-conformance`可把该时间线与设备侧Bootloader/分区证据进行离线一致性校验，详细格式和15项检查见[固件真机联调一致性证据采集与校验说明](firmware-conformance-evidence.md)。
|
||||
|
||||
## 9. 维护端租户级固件管理
|
||||
|
||||
V46和OpenAPI 0.74.0在保留家庭端固件历史及设备下载协议的前提下，引入维护端租户级固件制品与OTA批次。维护端创建的`firmware_release.household_id`和`firmware_rollout.household_id`为空，`created_source=OPS`；家庭端已有记录继续使用原家庭范围，二者共享租户、型号、硬件版本和业务版本唯一性，避免同一租户发布相互冲突的活动制品。
|
||||
|
||||
维护中心“固件管理”页面支持选择客户组织、登记或吊销可信公钥、浏览器计算SHA-256后直传私有对象、恢复中断上传、完成签名校验、暂停、恢复、退役和人工下载。人工下载与设备下载都只签发短期私有地址；人工下载额外记录`ops.firmware.release.download`审计事件，设备下载继续要求设备身份、目标批次、版本状态和分配状态全部通过。
|
||||
|
||||
维护端接口使用`X-Tenant-Id`固定客户组织边界：
|
||||
|
||||
| 方法与路径 | 用途 |
|
||||
|---|---|
|
||||
| `GET/POST /api/v1/ops/firmware-signing-keys` | 查询或登记租户可信公钥 |
|
||||
| `POST /api/v1/ops/firmware-signing-keys/revoke` | 在引用保护通过后吊销公钥 |
|
||||
| `GET/POST /api/v1/ops/firmware-releases` | 查询租户固件台账或创建租户级上传记录 |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/upload-session` | 恢复未完成的私有上传 |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/finalize` | 校验对象和签名并发布版本 |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/pause` | 暂停版本分发和关联在途批次 |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/resume` | 恢复版本分发，不自动恢复旧批次 |
|
||||
| `POST /api/v1/ops/firmware-releases/{id}/retire` | 永久退役版本 |
|
||||
| `GET /api/v1/ops/firmware-releases/{id}/download` | 获取带摘要、签名和过期时间的短期人工下载地址 |
|
||||
| `GET/POST /api/v1/ops/firmware-rollouts` | 查询OTA批次或按兼容设备和灰度比例创建升级批次 |
|
||||
| `GET /api/v1/ops/firmware-rollouts/{id}` | 查询逐设备下载、安装和确认进度 |
|
||||
| `POST /api/v1/ops/firmware-rollouts/{id}/pause` | 暂停批次并取消尚未投递的升级命令 |
|
||||
|
||||
@@ -84,6 +84,10 @@ docker run --rm -v "${PWD}:/src" -w /src golang:1.26.6-alpine `
|
||||
npm --prefix web audit --audit-level=high
|
||||
docker run --rm -v "${PWD}:/src" zricethezav/gitleaks:v8.30.1 `
|
||||
detect --source=/src --config=/src/.gitleaks.toml --no-banner
|
||||
docker run --rm -v "${PWD}:/src" `
|
||||
-v data-terminal-trivy-cache:/root/.cache/trivy aquasec/trivy:0.74.0 `
|
||||
config --ignorefile /src/.trivyignore.yaml --exit-code 1 `
|
||||
--severity HIGH,CRITICAL /src
|
||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock `
|
||||
-v data-terminal-trivy-cache:/root/.cache/trivy aquasec/trivy:0.74.0 `
|
||||
image --scanners vuln --ignore-unfixed --severity HIGH,CRITICAL `
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# Viking managed memory integration
|
||||
|
||||
Configured collection: `memory`, resource `mem-3a1174a1`, project `default`, Beijing endpoint. The existing collection has emotional-companionship extraction rules `event_v1` and `profile_v1` (assistant isolation enabled). This is the managed Viking memory service, not OpenViking.
|
||||
|
||||
## Runtime
|
||||
|
||||
API and worker accept `VIKING_MEMORY_RESOURCE_ID`, `VIKING_MEMORY_API_KEY_FILE`, `VIKING_MEMORY_EVENT_TYPE` and `VIKING_MEMORY_PROFILE_TYPE`. An empty resource ID disables this integration. The API key is a server-only file; it is never returned to a device or written into a firmware image. The endpoint is HTTPS; redirects are disabled. Error logs contain status/code, never vendor response bodies or transcripts.
|
||||
|
||||
Terminal conversation extraction requests also enqueue an independent `conversation.viking_sync_requested` outbox event. The worker submits final user and assistant turns with AddSession. Frozen transcript/memory consent and current subject/memory/transcript eligibility are checked. No user-only or diagnostic assistant-only conversation is submitted. The database remains the source of full transcripts.
|
||||
|
||||
User identity is SHA-256 of the tenant and subject UUIDs, prefixed `u_`; it survives device restarts and changes of device binding. Assistant identity is the stable platform companion ID, currently shared across platform persona versions. Every conversation has a distinct `c_<conversation UUID without hyphens>_<batch>` session identifier. Batches contain up to 100 messages. A retry uses the same identifier; a late transcript revision replaces that conversation's batch. Audit digests avoid rewriting unchanged successful submissions. Initial indexing/extraction is asynchronous.
|
||||
|
||||
At hardware session creation the API retrieves the current user profile and recent events before opening the allocation transaction, with a three-second overall budget. It revalidates the subject binding and memory consent before incorporating up to 600 characters into the frozen context. Extracted memories are explicitly labeled as fallible background, not instructions or manually confirmed facts. The existing in-call memory-search endpoint also searches Viking, then falls back to local curated memories. Actual use of that tool by the hardware cloud runtime still depends on provider tool support; startup context does not depend on it. Lookup failure leaves normal conversation available.
|
||||
|
||||
There are no firmware/audio changes and no database migrations. Native subtitle callbacks remain in device-probe mode; the reliable device transcript upload supplies these sessions. Pending memory extraction is independent of streamed audio playback.
|
||||
|
||||
## Lifecycle and limits
|
||||
|
||||
Current consent withdrawal immediately gates future Viking reads/writes. Subject privacy erasure calls scoped event/profile deletion before completing the provider-index deletion subtask. Vendor extraction is asynchronous; late vendor extraction after an erasure request needs follow-up reconciliation before treating provider erasure as fully verified. Individual locally curated-memory deletion does not delete independently extracted Viking events/profiles. For the present integration, use the Viking console for editing or deleting those derived entries; a unified platform memory-management UI is not implemented. Do not describe the existing local memory list as a complete Viking list.
|
||||
|
||||
Automatic extraction begins on conversation completion/failure and final transcript delivery. A sudden power-off relies on the existing heartbeat recovery to close the conversation. Startup recalls recent events plus profile; recall of older topic-specific events during the same session depends on the in-call search tool. Full historical backfill is explicit; enabling the integration does not automatically export all past conversations.
|
||||
|
||||
## Verification
|
||||
|
||||
Deployment on 2026-09-06: API and worker `0.78.4-rc.viking1`; callbacks remain `0.78.3-rc.subtitle-memory3`. All three readiness endpoints returned 200. Current symlink points to `/opt/data-terminal-cloud/releases/0.78.4-rc.viking1`; rollback uses the previous `0.78.3-rc.subtitle-memory1` directory including its `compose.subtitle.yaml` override. The same three compose files remain required.
|
||||
|
||||
One real conversation (`157f866d-c364-4ae8-b169-50ca07024e5a`, three final messages) was explicitly queued for backfill. Viking produced two event memories, and the production service retrieved 76 characters of startup memory. Synthetic event/profile entries were deleted and subsequent queries returned zero. Device power-cycle/spoken recall is pending user verification; server retrieval alone is not proof of successful audible recall.
|
||||
|
||||
- Real API-key calls verified against the configured collection: AddSession, event search, profile search.
|
||||
- Synthetic test used a separate user namespace; no test preference is attached to the real user.
|
||||
- Tests cover vendor result scope filtering, error redaction, stable identity/session IDs, scoped deletion, SQL outbox deduplication, late transcript replacement, frozen consent, cross-tenant rejection, and subject-bound startup context.
|
||||
- A dedicated PostgreSQL/Redis environment is used for integration tests, separate from production data.
|
||||
|
||||
Official API references: https://www.volcengine.com/docs/84313/1783353 , https://www.volcengine.com/docs/84313/2100967 , https://www.volcengine.com/docs/84313/2100966 , https://www.volcengine.com/docs/84313/1783358 , https://www.volcengine.com/docs/84313/1783359 .
|
||||
@@ -0,0 +1,230 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"diagram_type": "architecture",
|
||||
"meta": {
|
||||
"title": "火山硬件对话智能体迁移后目标架构",
|
||||
"locale": "zh-CN",
|
||||
"quality_profile": "showcase",
|
||||
"viewBox": [1360, 820]
|
||||
},
|
||||
"components": [
|
||||
{
|
||||
"id": "elder",
|
||||
"type": "external",
|
||||
"label": "长者用户",
|
||||
"sublabel": "说话、打断、挂机",
|
||||
"pos": [40, 230],
|
||||
"size": [130, 72]
|
||||
},
|
||||
{
|
||||
"id": "device",
|
||||
"type": "frontend",
|
||||
"label": "陪伴硬件",
|
||||
"sublabel": "麦克风、扬声器、本地队列",
|
||||
"pos": [230, 230],
|
||||
"size": [180, 72]
|
||||
},
|
||||
{
|
||||
"id": "hardwareSdk",
|
||||
"type": "frontend",
|
||||
"label": "火山硬件智能体 SDK",
|
||||
"sublabel": "高质量 UDP/RTC 主链",
|
||||
"pos": [490, 230],
|
||||
"size": [200, 72]
|
||||
},
|
||||
{
|
||||
"id": "volcAgent",
|
||||
"type": "external",
|
||||
"label": "火山硬件对话智能体",
|
||||
"sublabel": "Bot、ASR/LLM/TTS、会话",
|
||||
"pos": [770, 230],
|
||||
"size": [220, 72]
|
||||
},
|
||||
{
|
||||
"id": "deviceRegistry",
|
||||
"type": "external",
|
||||
"label": "火山产品与设备管理",
|
||||
"sublabel": "Product、Device、License、Bot 绑定",
|
||||
"pos": [770, 50],
|
||||
"size": [220, 72]
|
||||
},
|
||||
{
|
||||
"id": "platformApi",
|
||||
"type": "backend",
|
||||
"label": "平台会话控制面",
|
||||
"sublabel": "授权、冻结快照、心跳、停止",
|
||||
"pos": [490, 470],
|
||||
"size": [200, 76]
|
||||
},
|
||||
{
|
||||
"id": "toolGateway",
|
||||
"type": "security",
|
||||
"label": "工具与记忆网关",
|
||||
"sublabel": "MCP/Function Calling、短期会话令牌",
|
||||
"pos": [1070, 230],
|
||||
"size": [220, 76]
|
||||
},
|
||||
{
|
||||
"id": "dataStore",
|
||||
"type": "database",
|
||||
"label": "业务主数据",
|
||||
"sublabel": "PostgreSQL、pgvector、Redis",
|
||||
"pos": [770, 470],
|
||||
"size": [220, 76]
|
||||
},
|
||||
{
|
||||
"id": "asyncPipeline",
|
||||
"type": "backend",
|
||||
"label": "异步内容与运营流水线",
|
||||
"sublabel": "字幕、摘要、故事、费用、审计",
|
||||
"pos": [1070, 470],
|
||||
"size": [220, 76]
|
||||
},
|
||||
{
|
||||
"id": "objectStorage",
|
||||
"type": "database",
|
||||
"label": "私有对象存储",
|
||||
"sublabel": "可选双轨录音与导出",
|
||||
"pos": [230, 650],
|
||||
"size": [180, 72]
|
||||
},
|
||||
{
|
||||
"id": "webApps",
|
||||
"type": "frontend",
|
||||
"label": "家庭端与运营端",
|
||||
"sublabel": "设备、通话、隐私、配置、诊断",
|
||||
"pos": [490, 650],
|
||||
"size": [200, 72]
|
||||
}
|
||||
],
|
||||
"boundaries": [
|
||||
{
|
||||
"kind": "region",
|
||||
"label": "设备侧",
|
||||
"wraps": ["device", "hardwareSdk"],
|
||||
"pad": 22
|
||||
},
|
||||
{
|
||||
"kind": "region",
|
||||
"label": "火山引擎托管域",
|
||||
"wraps": ["volcAgent", "deviceRegistry"],
|
||||
"pad": 22
|
||||
},
|
||||
{
|
||||
"kind": "security-group",
|
||||
"label": "自有平台可信域（业务事实唯一主库）",
|
||||
"wraps": ["platformApi", "toolGateway", "dataStore", "asyncPipeline", "objectStorage", "webApps"],
|
||||
"pad": 24
|
||||
}
|
||||
],
|
||||
"connections": [
|
||||
{
|
||||
"id": "humanVoice",
|
||||
"from": "elder",
|
||||
"to": "device",
|
||||
"label": "双向语音",
|
||||
"variant": "emphasis"
|
||||
},
|
||||
{
|
||||
"id": "audioFrames",
|
||||
"from": "device",
|
||||
"to": "hardwareSdk",
|
||||
"label": "PCM/设备事件",
|
||||
"variant": "emphasis"
|
||||
},
|
||||
{
|
||||
"id": "agentLink",
|
||||
"from": "hardwareSdk",
|
||||
"to": "volcAgent",
|
||||
"label": "SDK 直连会话",
|
||||
"variant": "emphasis",
|
||||
"labelAt": [730, 316]
|
||||
},
|
||||
{
|
||||
"id": "provisioning",
|
||||
"from": "deviceRegistry",
|
||||
"to": "hardwareSdk",
|
||||
"label": "设备凭证与 Bot 绑定",
|
||||
"variant": "security"
|
||||
},
|
||||
{
|
||||
"id": "prepareSession",
|
||||
"from": "device",
|
||||
"to": "platformApi",
|
||||
"label": "HTTPS 会话控制、字幕与状态",
|
||||
"variant": "security"
|
||||
},
|
||||
{
|
||||
"id": "toolCalls",
|
||||
"from": "volcAgent",
|
||||
"to": "toolGateway",
|
||||
"label": "MCP / FC",
|
||||
"variant": "security"
|
||||
},
|
||||
{
|
||||
"id": "toolData",
|
||||
"from": "toolGateway",
|
||||
"to": "dataStore",
|
||||
"label": "租户与长者范围查询",
|
||||
"variant": "security"
|
||||
},
|
||||
{
|
||||
"id": "conversationFacts",
|
||||
"from": "platformApi",
|
||||
"to": "dataStore",
|
||||
"label": "会话/授权快照",
|
||||
"variant": "emphasis"
|
||||
},
|
||||
{
|
||||
"id": "asyncJobs",
|
||||
"from": "dataStore",
|
||||
"to": "asyncPipeline",
|
||||
"label": "Outbox 可靠任务",
|
||||
"variant": "dashed",
|
||||
"labelAt": [1030, 560]
|
||||
},
|
||||
{
|
||||
"id": "mediaUpload",
|
||||
"from": "device",
|
||||
"to": "objectStorage",
|
||||
"label": "可选加密录音直传",
|
||||
"variant": "dashed",
|
||||
"fromSide": "bottom",
|
||||
"toSide": "top",
|
||||
"labelAt": [320, 316]
|
||||
},
|
||||
{
|
||||
"id": "webControl",
|
||||
"from": "webApps",
|
||||
"to": "platformApi",
|
||||
"label": "HTTPS/OIDC",
|
||||
"variant": "security"
|
||||
}
|
||||
],
|
||||
"cards": [
|
||||
{
|
||||
"dot": "emerald",
|
||||
"title": "保留不动",
|
||||
"items": [
|
||||
"平台继续掌握设备、授权、通话、记忆、费用和审计主数据",
|
||||
"字幕后的摘要、故事、人工确认记忆和隐私删除链路继续复用"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dot": "amber",
|
||||
"title": "替换范围",
|
||||
"items": [
|
||||
"设备不再领取平台 RTC Token，也不再等待云端 StartVoiceChat",
|
||||
"火山硬件 SDK 直接建立会话，旧 RTC 房间与 Agent Task 仅保留兼容期"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dot": "rose",
|
||||
"title": "上线硬门槛",
|
||||
"items": [
|
||||
"必须验证服务端强制停话、设备凭证吊销、字幕/用量回传和近远端录音能力",
|
||||
"任何一项无法闭环时，先限定为试点能力，不替换现网主链"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
# 火山字幕回调与重启记忆修复（2026-09-06）
|
||||
|
||||
## 设备实测更新（北京时间 10:20）
|
||||
|
||||
- **已烧录并启动**：COM6，ESP32-S3 MAC b4:3a:45:16:34:a8。最终固件 app `0x2ccf40`，日志 `build/native-subtitle-stable-{build,flash,device}.log`，烧录哈希校验成功。NVS/Wi-Fi/绑定均保留；设备已联网待机。
|
||||
- 实际用户测试会话 `157f866d-c364-4ae8-b169-50ca07024e5a`：设备保存 3 条文字（其中 1 条为前置播放测试），双方实际对话保存成功，状态 COMPLETED，2 条 READY 摘要。设备约每 15 秒续期成功。之后创建的新会话已携带非空历史上下文；实际断电后口头回忆仍需用户反馈。
|
||||
- **真实火山回调已收到，但不能据此宣称按会话入库完成。** 当前 low_load 回调只有 message/signature；解码后包含 productKey、deviceName、type、data，说话者 userId 为 input/output，没有 job_id、session_id 或 conversation_id。
|
||||
- 平台确实下发了带 conversation_id 的 URL 和 43 字符会话签名。固件 session.update 使用官方 AgentConfig.ServerMessageURLForRTS/ServerMessageSignatureForRTS 字段，但实测仍回调控制台静态 URL 和静态签名。额外测试 ServerMessageUrl/ServerMessageSignature 别名也未改变结果，该实验已从最终固件移除。
|
||||
- 不允许通过“设备最新会话”归属回调，否则旧会话延迟字幕可能写入重启后的新会话。**保持 VOLC_SUBTITLE_PRIMARY=false。设备 turns:batch 仍为唯一正式记录来源。**
|
||||
- callbacks 服务已更新为 `0.78.3-rc.subtitle-memory3`（api/worker 仍为 memory1），静态回调校验签名、设备身份及格式后返回 200，仅记无正文的验证日志，不存第二份文字。支持 Base64 JSON 和 subv 二进制封装。错误签名返回 401。公网验证前后 conversation_turn 行数不变，服务 ready。
|
||||
- 两个控制台配置值不变。其静态配置已能将字幕发送到平台，**尚未完成独立的、可靠的会话归档**。独立入库前必须解决会话身份传递，并按实测 input/output 更新和验证说话者映射；不能直接开启 primary。
|
||||
- 下一步供应商核对事项：低负载 session.update 是否支持动态回调 URL/签名？支持时需要的准确字段/生效时机是什么？不支持时能否在字幕中返回与 session.created.job_id 相同的编号？必要时评估平台管理每设备独立智能体及启动前冻结配置，但尚未创建、修改或批量克隆任何火山智能体。
|
||||
|
||||
以下为上一轮部署时的设计和验证记录；与本节冲突的“未烧录/未实测”描述已由本节更新取代。
|
||||
|
||||
## 已上线
|
||||
|
||||
- 后端：`0.78.3-rc.subtitle-memory1`，服务器 `/opt/data-terminal-cloud/current` 指向同名 release。
|
||||
- `POST /callbacks/v1/volcengine/subtitles` 支持没有 Content-Type 的火山原生请求：外层 message/signature，message 解码后为 `subv` + 四字节大端长度 + subtitle JSON。
|
||||
- 当前 `VOLC_SUBTITLE_PRIMARY=false`：回调仅验证，不存第二份文字；设备原有 turns:batch 仍是正式记录来源。
|
||||
- 有效摘要现在包含已失败的会话，仍限定租户、用户、授权及 READY 状态。硬件结束、失败、超时按实际最大 turn sequence 安排摘要提取。
|
||||
- 无数据库迁移；未发布工作区其他改动。上一个 release 可用于回退。
|
||||
|
||||
## 火山控制台填写
|
||||
|
||||
- ServerMessageUrl：`https://121.41.46.238/callbacks/v1/volcengine/subtitles`
|
||||
- ServerMessageSignature：使用本机被 Git 忽略的 `.deploy-artifacts/volc-subtitle-console-settings.txt` 中同名值。此文档不复制密钥。
|
||||
|
||||
这两项是基础配置，不能单靠基础 URL 推断归属会话。新固件上报 `native_subtitle_callback=true` 后，平台每次创建会话返回 `session_config.subtitle_callback_url`（带 conversation_id）和 `subtitle_callback_signature`（绑定该会话的 HMAC 值）。固件在 session.update 的 agent_config 中设置 `ServerMessageURLForRTS` 和 `ServerMessageSignatureForRTS`，同时设置 config.SubtitleConfig.SubtitleMode=1。火山按普通自定义 signature 原样回传它，平台进行比较。
|
||||
|
||||
已冻结的会话各自保留记录来源和说话者映射，不能用“设备当前最新会话”归档延迟回调。完成的分句全部保留，不能只保留 paragraph=true；稳定键按 userId、roundId、sequence 去重。没有供应商时间戳的回调只记录接收时间，不声称等于实际说话时间。网络乱序的分句按入库顺序分配平台序号。
|
||||
|
||||
## 尚未完成的设备验证
|
||||
|
||||
- COM6 尚未出现，COM3 的 VID:PID 为 1D6B:011E，是另一台设备；COM4/5 为蓝牙。**没有烧录新固件**。
|
||||
- 最新编译：`firmware/echoear-platform-agent/build/native-subtitle-build.log`，app 大小 `0x2ccf40`；保留此前连续播放和眼睛动画修复，新增心跳续期及字幕配置。
|
||||
- 不能把平台发出的合成 HTTP 检查当成真实火山回调。当前 low_load 模式是否发送原生服务端字幕还需用设备验证；服务端状态回调仍关闭。
|
||||
- 先烧录 app 分区，保留 NVS/Wi-Fi/绑定。确认真实用户和助手字幕均能到达，并核对实际 userId 与冻结映射。未知或有歧义的说话者禁止入库。未验证前不得启用 primary。
|
||||
- primary 只作用于新会话；固件收到 subtitle_source=provider 后停止设备文字上传，平台也拒绝该会话的设备补传，防止双份记录。切换后还需验证重连、断电回忆、分句完整性及播放连续性。
|
||||
|
||||
## 验证结果
|
||||
|
||||
- 单元测试：协议截断/错误长度、缺字段、错误签名、跨会话签名、无 Content-Type、说话者歧义等通过。
|
||||
- 独立临时 PostgreSQL（不是生产数据库）：完成/失败/活跃会话的摘要选择、隔离其他用户、末尾提取版本与去重、字幕 probe 不入库、primary 入库与重复处理、未知说话者、冲突、撤回授权测试通过。日志 `.deploy-artifacts/subtitle-db-test.log`。
|
||||
- 公网 HTTPS（正常证书校验）协议检查：错误签名 401，正确签名 probe 200；原会话记录数测试前后均为 5，无测试聊天写入。
|
||||
- API、callbacks、worker 的 `/health/ready` 均 ready，线上版本接口返回新版本。
|
||||
|
||||
## 运维命令
|
||||
|
||||
当前上线额外使用 `deploy/server/compose.subtitle.yaml` 固定三个后端服务镜像和回调环境变量。后续重启这版必须保留该文件：
|
||||
|
||||
```sh
|
||||
cd /opt/data-terminal-cloud/current
|
||||
docker compose -p data-terminal-cloud --env-file deploy/server/.env \
|
||||
-f compose.yaml -f deploy/server/compose.override.yaml \
|
||||
-f deploy/server/compose.subtitle.yaml up -d --no-deps api callbacks worker
|
||||
```
|
||||
|
||||
首次启动曾因 Windows 编译文件无 Linux 执行位被拒，已自动回退并修正为打包前 chmod 0755，随后成功上线。部署源码在本机 `.deploy-artifacts/subtitle-release`，包含明确文件清单；保留服务器基线的其他源码。
|
||||
|
||||
官方格式依据：[实时字幕（对话记录）](https://docs.volcengine.com/docs/6348/1337284)，查阅于 2026-09-06。
|
||||
@@ -11,9 +11,12 @@
|
||||
| 事件类型 | 产生条件 | 资源类型 | 状态示例 |
|
||||
|---|---|---|---|
|
||||
| `device.presence.changed` | 首次有效心跳使设备上线，或后台发现心跳超时使设备离线 | `device` | `online`、`offline` |
|
||||
| `device.binding.reset` | 设备主动恢复出厂并解除当前用户绑定 | `device` | `CLAIMABLE` |
|
||||
| `conversation.started` | 通话进入`ACTIVE` | `conversation` | `ACTIVE` |
|
||||
| `conversation.completed` | 通话进入`COMPLETED` | `conversation` | `COMPLETED` |
|
||||
| `conversation.failed` | 通话进入`FAILED` | `conversation` | `FAILED` |
|
||||
|
||||
上述通话事件由`conversation.status`统一触发，与底层运行时无关；`legacy_voicechat`和`volc_hardware_agent`都会产生同一套家庭侧SSE语义。硬件智能体的周期`session_updated`只作为平台租约和诊断证据，不逐次生成SSE事件。
|
||||
| `story.created` | 新故事候选建立 | `story` | 故事当前状态 |
|
||||
| `story.revised` | 故事版本增加 | `story` | 故事当前状态 |
|
||||
| `media.upload.completed` | 媒体上传批次完成 | `upload_batch` | `COMPLETED` |
|
||||
@@ -105,7 +108,8 @@ sequenceDiagram
|
||||
3. 设备此前不存在在线记录或已离线时，写入一次`online`事件；连续心跳只延长最后出现时间。
|
||||
4. Redis继续承担低延迟在线查询，PostgreSQL状态表承担状态转换和事件可靠性。
|
||||
5. Worker每个心跳周期扫描一次；最后出现时间超过两倍会话租约的记录切换为`offline`并只写入一次事件。
|
||||
6. 新的有效心跳可再次把离线记录切回在线并产生新的上线事件。
|
||||
6. 设备恢复出厂时，平台在同一事务内把旧用户侧在线状态改为`offline`、发送`device.binding.reset`并解除绑定。
|
||||
7. 新的有效心跳可再次把离线记录切回在线并产生新的上线事件；恢复出厂后的设备要先完成重新绑定。
|
||||
|
||||
## 7. 浏览器重连与去重
|
||||
|
||||
|
||||
+14900
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 132 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 132 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 156 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 157 KiB |
@@ -0,0 +1,32 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Archify visual-check · 火山硬件对话智能体迁移架构.html</title>
|
||||
<style>
|
||||
*{box-sizing:border-box}body{margin:0;padding:24px;background:#e9eef5;color:#172033;font:14px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}header{max-width:1500px;margin:0 auto 18px}h1{margin:0 0 6px;font-size:20px}p{margin:0;color:#526176}.grid{max-width:1500px;margin:auto;display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:18px}figure{margin:0;padding:10px;background:white;border:1px solid #c9d4e3;border-radius:12px;box-shadow:0 10px 30px rgba(15,23,42,.08)}img{display:block;width:100%;height:auto;border:1px solid #e2e8f0}figcaption{padding:9px 4px 2px;color:#526176}@media(max-width:900px){.grid{grid-template-columns:1fr}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><h1>Archify visual-check</h1><p>火山硬件对话智能体迁移架构.html · automated containment pass · visual review pending</p></header>
|
||||
<main class="grid">
|
||||
<figure>
|
||||
<img src="火山硬件对话智能体迁移架构.visual-check.1440x900.light.png" alt="light 1440 by 900">
|
||||
<figcaption><strong>LIGHT</strong> · 1440×900 · containment pass</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="火山硬件对话智能体迁移架构.visual-check.1440x900.dark.png" alt="dark 1440 by 900">
|
||||
<figcaption><strong>DARK</strong> · 1440×900 · containment pass</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="火山硬件对话智能体迁移架构.visual-check.2048x1320.light.png" alt="light 2048 by 1320">
|
||||
<figcaption><strong>LIGHT</strong> · 2048×1320 · containment pass</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="火山硬件对话智能体迁移架构.visual-check.2048x1320.dark.png" alt="dark 2048 by 1320">
|
||||
<figcaption><strong>DARK</strong> · 2048×1320 · containment pass</figcaption>
|
||||
</figure>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,547 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"ok": true,
|
||||
"command": "visual-check",
|
||||
"status": "pass",
|
||||
"visualReview": "pending",
|
||||
"artifact": {
|
||||
"path": "E:\\Codex\\story\\platform\\docs\\火山硬件对话智能体迁移架构.html",
|
||||
"sha256": "baefbdc9512b0614c236f59cc80d8a14b8fb9489a3a6281214d636ad9fb68d8c",
|
||||
"bytes": 721387
|
||||
},
|
||||
"state": {
|
||||
"detail": "read",
|
||||
"motion": "still"
|
||||
},
|
||||
"chrome": {
|
||||
"status": "available",
|
||||
"executable": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
|
||||
},
|
||||
"diagnostics": [],
|
||||
"containment": {
|
||||
"status": "pass",
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1042,
|
||||
"diagramWidth": 1012,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.697058823529412,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1600,
|
||||
"height": 1000,
|
||||
"theme": "light",
|
||||
"innerWidth": 1600,
|
||||
"innerHeight": 1000,
|
||||
"scrollWidth": 1600,
|
||||
"scrollHeight": 1000,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1082,
|
||||
"diagramWidth": 1052,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.961764705882353,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"theme": "light",
|
||||
"innerWidth": 1920,
|
||||
"innerHeight": 1080,
|
||||
"scrollWidth": 1920,
|
||||
"scrollHeight": 1080,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1216,
|
||||
"diagramWidth": 1186,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 7.848529411764706,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1619,
|
||||
"diagramWidth": 1569,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 9,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
}
|
||||
]
|
||||
},
|
||||
"readability": {
|
||||
"status": "pass",
|
||||
"minimumProjectedNodeTextPx": 6,
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1042,
|
||||
"diagramWidth": 1012,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.697058823529412,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1600,
|
||||
"height": 1000,
|
||||
"theme": "light",
|
||||
"innerWidth": 1600,
|
||||
"innerHeight": 1000,
|
||||
"scrollWidth": 1600,
|
||||
"scrollHeight": 1000,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1082,
|
||||
"diagramWidth": 1052,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.961764705882353,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"theme": "light",
|
||||
"innerWidth": 1920,
|
||||
"innerHeight": 1080,
|
||||
"scrollWidth": 1920,
|
||||
"scrollHeight": 1080,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1216,
|
||||
"diagramWidth": 1186,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 7.848529411764706,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1619,
|
||||
"diagramWidth": 1569,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 9,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
}
|
||||
]
|
||||
},
|
||||
"viewerChrome": {
|
||||
"status": "pass",
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1042,
|
||||
"diagramWidth": 1012,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.697058823529412,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1600,
|
||||
"height": 1000,
|
||||
"theme": "light",
|
||||
"innerWidth": 1600,
|
||||
"innerHeight": 1000,
|
||||
"scrollWidth": 1600,
|
||||
"scrollHeight": 1000,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1082,
|
||||
"diagramWidth": 1052,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.961764705882353,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"theme": "light",
|
||||
"innerWidth": 1920,
|
||||
"innerHeight": 1080,
|
||||
"scrollWidth": 1920,
|
||||
"scrollHeight": 1080,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1216,
|
||||
"diagramWidth": 1186,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 7.848529411764706,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1619,
|
||||
"diagramWidth": 1569,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 9,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light"
|
||||
}
|
||||
]
|
||||
},
|
||||
"captures": {
|
||||
"status": "pass",
|
||||
"screenshots": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "light",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1042,
|
||||
"diagramWidth": 1012,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.697058823529412,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light",
|
||||
"file": "火山硬件对话智能体迁移架构.visual-check.1440x900.light.png"
|
||||
},
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900,
|
||||
"theme": "dark",
|
||||
"innerWidth": 1440,
|
||||
"innerHeight": 900,
|
||||
"scrollWidth": 1440,
|
||||
"scrollHeight": 900,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1042,
|
||||
"diagramWidth": 1012,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 6.697058823529412,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 51,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "dark",
|
||||
"file": "火山硬件对话智能体迁移架构.visual-check.1440x900.dark.png"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "light",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1619,
|
||||
"diagramWidth": 1569,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 9,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "light",
|
||||
"file": "火山硬件对话智能体迁移架构.visual-check.2048x1320.light.png"
|
||||
},
|
||||
{
|
||||
"width": 2048,
|
||||
"height": 1320,
|
||||
"theme": "dark",
|
||||
"innerWidth": 2048,
|
||||
"innerHeight": 1320,
|
||||
"scrollWidth": 2048,
|
||||
"scrollHeight": 1320,
|
||||
"overflowX": false,
|
||||
"overflowY": false,
|
||||
"ok": true,
|
||||
"readerWidth": 1619,
|
||||
"diagramWidth": 1569,
|
||||
"viewBoxWidth": 1360,
|
||||
"minimumProjectedNodeTextPx": 9,
|
||||
"minimumProjectedNodeText": "说话、打断、挂机",
|
||||
"minimumProjectedNodeTextDetail": "context",
|
||||
"minimumRequiredNodeTextPx": 6,
|
||||
"readabilityOk": true,
|
||||
"hasLegend": true,
|
||||
"hasNavigationDock": true,
|
||||
"legendDockIntersectionArea": 0,
|
||||
"dockStageIntersectionArea": 0,
|
||||
"dockStageGap": 10.21875,
|
||||
"requiredDockStageGap": 10,
|
||||
"viewerChromeStageOk": true,
|
||||
"viewerChromeReserve": 41,
|
||||
"viewerChromeActive": true,
|
||||
"viewerChromeOk": true,
|
||||
"resolvedTheme": "dark",
|
||||
"file": "火山硬件对话智能体迁移架构.visual-check.2048x1320.dark.png"
|
||||
}
|
||||
],
|
||||
"contactSheet": "火山硬件对话智能体迁移架构.visual-check.html"
|
||||
},
|
||||
"sidecars": {
|
||||
"receipt": "火山硬件对话智能体迁移架构.visual-check.json",
|
||||
"contactSheet": "火山硬件对话智能体迁移架构.visual-check.html"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
# 火山硬件对话智能体迁移设计
|
||||
|
||||
> 状态：方案评审稿
|
||||
> 适用基线：`v0.70.0-rc.1`
|
||||
> 编制日期：2026-08-31
|
||||
|
||||
## 1. 结论
|
||||
|
||||
本次迁移建议采用“保留业务控制面，替换会话执行面”的方式，不对现有平台做推倒重建。
|
||||
|
||||
- 保留：设备身份与绑定、家庭和长者档案、授权、通话主记录、配置灰度、字幕存储、摘要/故事/事实候选、人工确认记忆、隐私导出删除、录音管理、费用额度、告警、诊断、固件和审计。
|
||||
- 替换：设备领取 RTC Token、设备加入平台分配的 RTC 房间、平台 Worker 调用 `StartVoiceChat/StopVoiceChat`、Agent RTC 用户和 RTC Task 对账。
|
||||
- 新增：火山产品/设备/Bot/License 映射、硬件智能体设备凭证生命周期、硬件会话适配器、动态会话配置、MCP/Function Calling 工具网关、硬件会话事件和用量归集。
|
||||
- 兼容：旧 RTC VoiceChat 链路和新硬件智能体链路至少并行一个灰度周期，按设备能力位和灰度配置选择，不按固件版本猜测。
|
||||
|
||||
推荐终端优先采用火山硬件智能体“高质量方案”（UDP/RTC）。现有终端本身已有 RTC、弱网恢复和网络切换验收基础，和该方案的目标更一致。火山官方同时提供低负载 WebSocket 方案；只有在量产板的 RAM、CPU、功耗或网络实测不满足高质量 SDK 时，才切换到低负载方案。官方对比说明高质量方案弱网和实时互动更好，低负载方案资源消耗更低。[火山官方 SDK 集成指引](https://www.volcengine.com/docs/6348/1913817?lang=zh)
|
||||
|
||||
## 2. 当前链路与目标链路
|
||||
|
||||
### 2.1 当前链路
|
||||
|
||||
```text
|
||||
设备 -> 平台创建通话 -> 平台签发 RTC Token -> 设备加入 RTC 房间
|
||||
-> 设备确认入房 -> Worker 调用 StartVoiceChat -> 豆包 Agent 入房
|
||||
-> 火山/设备回传字幕和状态 -> 平台存储与内容处理
|
||||
-> 设备或平台发起停止 -> Worker 调用 StopVoiceChat -> 通话终态
|
||||
```
|
||||
|
||||
当前关键实现：
|
||||
|
||||
- `/device/v2/conversations` 创建通话并返回 RTC Grant；
|
||||
- `/activate` 在设备入房后可靠投递 `conversation.agent_start_requested`；
|
||||
- `voiceprovider.VolcengineClient` 把平台稳定任务契约映射为 `StartVoiceChat/StopVoiceChat`；
|
||||
- RTC 房间、设备 RTC 用户、Agent RTC 用户和供应商 Task 是诊断主关联键；
|
||||
- 最终字幕进入 `conversation_turn`，继续触发摘要、故事、事实和记忆流水线。
|
||||
|
||||
### 2.2 目标链路
|
||||
|
||||
```text
|
||||
设备 -> 平台准备硬件会话并冻结授权/配置/背景
|
||||
-> 设备使用火山硬件智能体 SDK 直连 Bot
|
||||
-> SDK 收发音频并提供字幕、状态、Function Calling 事件
|
||||
-> 设备把最终字幕和状态回传平台
|
||||
-> Bot 通过 MCP/Function Calling 访问平台受控记忆/业务工具
|
||||
-> 用户、平台或授权撤回触发停止 -> 设备关闭 SDK 会话 -> 平台收敛终态
|
||||
```
|
||||
|
||||
火山硬件智能体的标准协议使用 `product_key + device_name + device_secret` 建立设备身份，连接时指定与产品关联的 `bot`。设备首次可通过 `DynamicRegister` 获取设备独享密钥；WebSocket 标准协议支持字幕事件、端侧 Function Calling、主动播报和文本提问。[火山标准协议文档](https://www.volcengine.com/docs/6348/1904602?lang=zh)
|
||||
|
||||
## 3. 架构原则
|
||||
|
||||
### 3.1 平台数据库仍是业务事实唯一主库
|
||||
|
||||
火山侧负责实时对话执行，不成为家庭、长者、授权、记忆和费用策略的主数据源。平台必须保存每次会话实际使用的：
|
||||
|
||||
- 产品、设备、Bot 和 License 的脱敏引用；
|
||||
- 平台路由版本、角色版本、声音版本、模型版本和提示词版本；
|
||||
- 发送给火山的会话配置摘要及 SHA-256；
|
||||
- 授权和背景资料冻结快照；
|
||||
- 火山会话 ID、传输方案、状态时间线、字幕来源和用量来源。
|
||||
|
||||
### 3.2 设备必须先通过平台业务门禁，再连接智能体
|
||||
|
||||
设备拥有火山凭证不等于拥有业务授权。每次对话前仍必须先调用平台准备接口，由平台检查：
|
||||
|
||||
- 设备已登记、未隔离、未吊销并已绑定长者；
|
||||
- 固件和硬件智能体 SDK 版本满足最低要求；
|
||||
- `voice_chat` 授权有效；
|
||||
- 客户组织、设备并发、时长和费用额度未超限；
|
||||
- 角色、模型、音色和 Bot 路由已发布；
|
||||
- 背景资料和记忆快照生成成功或按既定规则无记忆降级。
|
||||
|
||||
平台拒绝准备时，设备不得调用火山 SDK 建立对话。
|
||||
|
||||
### 3.3 不把长期记忆主库直接迁到火山
|
||||
|
||||
现有记忆包含人工确认、授权撤回、版本、来源、租户/长者隔离和隐私删除证明。迁移后继续使用平台 `pgvector` 和记忆工具：
|
||||
|
||||
- 会话启动时，把已授权的短背景摘要作为动态会话参数注入；
|
||||
- 通话中通过平台 MCP/Function Calling 工具查询记忆；
|
||||
- 工具令牌绑定 `conversation_id + provider_session_id + device_id + subject_id`，短期有效；
|
||||
- 模型只能提交查询内容，不能自行指定租户或长者；
|
||||
- 火山原生长期记忆只可作为可删除、可对账的派生缓存，不能成为主库。
|
||||
|
||||
火山硬件智能体产品能力包含 MCP、联网问答、长期记忆和设备管理，但这不改变本项目的业务数据权威边界。[火山硬件智能体产品页](https://www.volcengine.com/product/ConversationalAI-Embedded)
|
||||
|
||||
## 4. 功能迁移矩阵
|
||||
|
||||
| 现有功能 | 迁移方式 | 新责任边界 | 结论 |
|
||||
|---|---|---|---|
|
||||
| 设备 HMAC 身份、绑定、家庭归属 | 原样保留；新增火山设备映射 | 平台仍负责业务身份，火山凭证只负责智能体建连 | 保留并扩展 |
|
||||
| `/bootstrap`、配置、命令、遥测 | 原样保留，增加硬件智能体能力位和 SDK 信息 | 平台 | 保留 |
|
||||
| RTC 房间分配与 Token | 新运行时不再使用 | 火山硬件 SDK 自行建连 | 新链路废弃 |
|
||||
| `StartVoiceChat/StopVoiceChat` | 新运行时不再调用；旧链路继续兼容 | 设备 SDK / 火山硬件智能体 | 新链路废弃 |
|
||||
| `s2s_pure/s2s_hybrid/cascade` | 不再作为设备会话模式；改为平台内部服务档位或 Bot/模型配置 | 平台路由 + 火山 Bot | 重定义 |
|
||||
| 角色、提示词、模型、音色灰度 | 平台继续版本化；基础配置同步 Bot，会话差异通过动态参数注入 | 平台为权威，火山为执行副本 | 保留并改造 |
|
||||
| 欢迎语、主动播报 | 使用 Bot/会话配置；本地系统提示可用 `input_tts` | 火山 SDK + 设备应用 | 迁移 |
|
||||
| 背景资料冻结 | 准备会话时生成；等待 `session.updated` 后才开放麦克风 | 平台生成，设备注入 | 保留并改造 |
|
||||
| 实时打断、VAD、ASR、TTS | 使用火山硬件智能体原生能力 | 火山 | 下沉 |
|
||||
| 最终字幕 | 设备监听最终事件，写入本地加密队列，批量调用现有 `turns:batch` | 设备可靠上传，平台去重存储 | 保留并改造 |
|
||||
| 字幕服务端回调 | 若火山提供则作为第二来源，按供应商事件 ID 去重 | 火山回调 + 平台 | 待商务/技术确认 |
|
||||
| 摘要、开放话题、故事、事实候选 | 输入仍为平台最终话轮 | 平台 Worker | 原样保留 |
|
||||
| 人工确认记忆、向量检索 | 继续使用现有数据库和流程 | 平台 | 原样保留 |
|
||||
| 通话中记忆工具 | 从 VoiceChat 模板 URL 改为 MCP/Function Calling 工具网关 | 平台工具网关 | 改造 |
|
||||
| 设备音量、诊断等指令 | 云端命令链保留；安全的本地动作可同时暴露为端侧 FC | 平台命令 + 设备白名单执行器 | 扩展 |
|
||||
| 授权撤回停止 | 心跳返回 stop，设备关闭 SDK；必须补充服务端强制终止或凭证吊销 | 平台发令，火山/设备执行 | 关键改造 |
|
||||
| 双轨录音 | 需验证 SDK 是否提供近端采集和远端解码 PCM 回调 | 设备录制 + 对象存储 | 有条件迁移 |
|
||||
| 用量与费用 | 增加 License、AI 对话时长和模型用量来源；区分估算与账单 | 平台费用域 | 改造 |
|
||||
| 诊断工作台 | 房间/Task 改为产品、设备、Bot、License、会话、传输和事件 | 平台 | 改造 |
|
||||
| 配置灰度与回滚 | 增加 `hardware_agent` 运行时；按设备稳定分桶 | 平台 | 扩展 |
|
||||
| 固件发布 | SDK、License 兼容、产品信息和安全存储纳入固件门禁 | 平台 + 设备 | 扩展 |
|
||||
| 隐私导出/删除 | 平台数据链路保留；新增火山会话历史/设备数据删除与凭证吊销对账 | 平台主导 | 扩展 |
|
||||
| 发布候选验收 | 替换 RTC 三模式矩阵，新增硬件 SDK 真机矩阵 | 平台验收工具 | 改造 |
|
||||
|
||||
## 5. 终端改造
|
||||
|
||||
### 5.1 新增硬件智能体运行时模块
|
||||
|
||||
建议固件同时保留两个实现：
|
||||
|
||||
```text
|
||||
VoiceRuntime
|
||||
├── LegacyRTCRuntime 现有 RTC + 平台 StartVoiceChat
|
||||
└── VolcHardwareAgentRuntime 新硬件智能体 SDK
|
||||
```
|
||||
|
||||
平台配置返回显式 `voice_runtime`：
|
||||
|
||||
```json
|
||||
{
|
||||
"voice_runtime": "volc_hardware_agent",
|
||||
"runtime_revision": 12,
|
||||
"transport": "high_quality",
|
||||
"minimum_sdk_version": "<现场冻结版本>"
|
||||
}
|
||||
```
|
||||
|
||||
设备只在完整校验配置后切换运行时；切换失败回退上一有效配置。不要根据 `engine_mode` 或固件版本自动推断。
|
||||
|
||||
### 5.2 凭证与激活
|
||||
|
||||
设备侧需要保存：
|
||||
|
||||
- 火山 `instance_id`、`product_key`、`device_name`；
|
||||
- 动态注册换取的设备独享 `device_secret`；
|
||||
- 当前凭证版本、注册时间和最后成功鉴权时间。
|
||||
|
||||
安全要求：
|
||||
|
||||
- `product_secret`只允许保存在平台KMS或Secret挂载中，不能进入设备固件、应用、安装包或设备日志；
|
||||
- 设备使用平台设备身份调用`/device/v3/hardware-agent-credentials:provision`，由平台完成DynamicRegister并下发本设备的`device_secret`；
|
||||
- `device_secret` 使用安全芯片、TEE 或设备密钥封装后持久化；
|
||||
- 日志、遥测和崩溃转储不得输出任何 Secret 或完整签名；
|
||||
- 必须验证设备 Secret 的轮换、吊销和售后换板流程。
|
||||
|
||||
### 5.3 会话状态机
|
||||
|
||||
建议复用现有通话状态：
|
||||
|
||||
```text
|
||||
PREPARING -> SESSION_READY -> STARTING_AGENT -> ACTIVE
|
||||
-> FAILED
|
||||
ACTIVE -> STOPPING -> COMPLETED
|
||||
-> FAILED
|
||||
```
|
||||
|
||||
数据库可继续把 `SESSION_READY` 对外兼容映射为 `ROOM_ISSUED`，但新 `/device/v3` 契约应使用真实语义，不再暴露不存在的 RTC Room。
|
||||
|
||||
终端事件映射：
|
||||
|
||||
| 火山/SDK 事件 | 平台动作 |
|
||||
|---|---|
|
||||
| SDK 初始化成功 | 更新遥测，不代表会话成功 |
|
||||
| `session.created` | 保存火山会话标识，等待会话配置 |
|
||||
| `session.updated` | 调用平台 activate，允许采集和播放 |
|
||||
| 用户最终 ASR | 写入本地可靠字幕队列 |
|
||||
| Agent 最终字幕 | 写入本地可靠字幕队列 |
|
||||
| Function Call | 白名单校验后端侧执行，或转发平台工具网关 |
|
||||
| Agent 错误/断开 | 上报稳定错误码，按策略恢复或停止 |
|
||||
| 正常关闭 | 调用平台 stop 并冲刷字幕、用量和录音队列 |
|
||||
|
||||
## 6. 平台接口设计
|
||||
|
||||
现有 `/device/v2` 的创建响应把 `rtc` 定义为必填对象。为了不破坏量产固件，建议新增 `/device/v3/agent-sessions`，不要修改 v2 字段语义。
|
||||
|
||||
### 6.1 准备会话
|
||||
|
||||
`POST /device/v3/agent-sessions`
|
||||
|
||||
请求沿用设备 HMAC 和幂等键，声明：
|
||||
|
||||
- `capabilities.hardware_agent=true`；
|
||||
- SDK 版本、传输方案、音频格式；
|
||||
- 是否支持字幕可靠队列、动态会话配置、凭证轮换、近/远端 PCM 录制；
|
||||
- 网络和设备时间。
|
||||
|
||||
响应建议：
|
||||
|
||||
```json
|
||||
{
|
||||
"conversation_id": "...",
|
||||
"status": "SESSION_READY",
|
||||
"runtime": {
|
||||
"provider": "volc_hardware_agent",
|
||||
"bot_id": "<已关联 Bot>",
|
||||
"transport": "high_quality",
|
||||
"session_nonce": "<短期、单次>",
|
||||
"config_digest": "sha256:...",
|
||||
"wait_for_session_update": true
|
||||
},
|
||||
"session_config": {
|
||||
"dialog_id": "<平台生成的非个人标识>",
|
||||
"system_context": "<已授权且脱敏的本次背景>",
|
||||
"tool_scope_token": "<短期令牌>"
|
||||
},
|
||||
"privacy": {},
|
||||
"heartbeat_interval_sec": 10,
|
||||
"expires_at": "..."
|
||||
}
|
||||
```
|
||||
|
||||
响应不得返回 `product_secret` 或 `device_secret`。
|
||||
|
||||
### 6.2 激活会话
|
||||
|
||||
`POST /device/v3/agent-sessions/{conversation_id}/activate`
|
||||
|
||||
请求包含 `provider_session_id`、`connected_at`、`transport`、`bot_id`、`config_digest` 和设备侧关联号。平台核对准备记录和配置摘要后进入 `ACTIVE`。
|
||||
|
||||
### 6.3 心跳
|
||||
|
||||
继续使用严格递增序号，但把 `rtc` 改为：
|
||||
|
||||
```json
|
||||
{
|
||||
"agent_link": {
|
||||
"connected": true,
|
||||
"input_active": true,
|
||||
"output_active": true,
|
||||
"provider_session_id": "...",
|
||||
"reconnect_count": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
平台继续返回 `continue`、`stop`、`stop_after_turn`；新增 `rotate_provider_credential` 和 `refresh_session_config` 前，必须有显式设备能力位。
|
||||
|
||||
### 6.4 字幕
|
||||
|
||||
复用 `/device/v2/conversations/{id}/turns:batch` 或在 v3 提供同构接口。只上传最终字幕，不把增量字幕写入长期存储。保持 `turn_key + sequence_no` 去重和冲突检测。
|
||||
|
||||
### 6.5 停止
|
||||
|
||||
设备先请求 SDK 停止并停止采集，再调用平台 stop。平台在以下任一条件满足后收敛 `COMPLETED`：
|
||||
|
||||
- 收到可信服务端结束事件；
|
||||
- 设备报告关闭成功且心跳租约结束；
|
||||
- 强制终止接口确认成功。
|
||||
|
||||
如果只能依赖设备报告，授权撤回和设备失控时无法做到服务端 fail-close，不能直接通过生产隐私门禁。
|
||||
|
||||
## 7. 后端模块与数据模型
|
||||
|
||||
### 7.1 运行时抽象
|
||||
|
||||
现有 `voiceprovider.Client` 的 `Start/Query/Stop` 契约只适合 RTC VoiceChat。建议新增：
|
||||
|
||||
```go
|
||||
type ConversationRuntime interface {
|
||||
Prepare(context.Context, PrepareRequest) (PrepareResult, error)
|
||||
Terminate(context.Context, TerminateRequest) error
|
||||
Reconcile(context.Context, SessionRef) (RuntimeState, error)
|
||||
}
|
||||
```
|
||||
|
||||
- `LegacyVoiceChatRuntime` 封装现有 `voiceprovider.Client`；
|
||||
- `VolcHardwareAgentRuntime` 不负责创建音频会话，只负责设备/Bot 配置同步、服务端终止、事件对账和用量拉取；
|
||||
- 通话状态机、幂等、Outbox、审计和租约留在 `orchestration`。
|
||||
|
||||
### 7.2 新表建议
|
||||
|
||||
`hardware_agent_device_binding`
|
||||
|
||||
- `device_id`、`provider`、`instance_ref`、`product_ref`；
|
||||
- `provider_device_name`、`hardware_id_digest`；
|
||||
- `bot_ref`、`license_ref`、`credential_version`；
|
||||
- `status`、`registered_at`、`revoked_at`、`last_verified_at`。
|
||||
|
||||
`conversation_runtime_session`
|
||||
|
||||
- `conversation_id`、`runtime_type`、`transport`；
|
||||
- `provider_session_ref`、`provider_device_ref`、`bot_ref`；
|
||||
- `session_nonce_digest`、`config_digest`；
|
||||
- `status`、`connected_at`、`last_event_at`、`ended_at`；
|
||||
- 唯一约束：一个通话最多一个活动运行时会话；同一供应商会话只能映射一个通话。
|
||||
|
||||
`hardware_agent_config_sync`
|
||||
|
||||
- 平台版本 ID、Bot/产品引用、期望摘要、实际摘要；
|
||||
- 同步状态、供应商请求引用、最后错误码和验证时间；
|
||||
- 用于发现火山控制台手工修改造成的配置漂移。
|
||||
|
||||
### 7.3 事件归一化
|
||||
|
||||
扩展现有 `provider_event`，归一化为：
|
||||
|
||||
- `hardware_agent.session.started`；
|
||||
- `hardware_agent.session.updated`；
|
||||
- `hardware_agent.transcript.final`；
|
||||
- `hardware_agent.function_call`；
|
||||
- `hardware_agent.usage.final`；
|
||||
- `hardware_agent.session.stopped`；
|
||||
- `hardware_agent.session.failed`。
|
||||
|
||||
设备上传事件和服务端回调必须使用不同 `source_type`，最终以 `provider_event_key` 和话轮键双层去重。
|
||||
|
||||
## 8. 工具、记忆和业务动作
|
||||
|
||||
### 8.1 工具分类
|
||||
|
||||
| 类型 | 示例 | 执行位置 | 控制要求 |
|
||||
|---|---|---|---|
|
||||
| 设备本地无副作用 | 查询电量、读取音量 | 设备 | 白名单、参数校验 |
|
||||
| 设备本地可逆动作 | 调整音量、切换灯效 | 设备 | 范围限制、结果回传 |
|
||||
| 平台只读工具 | 检索已确认记忆、读取已授权日程 | 平台工具网关 | 会话短令牌、租户/长者反查 |
|
||||
| 平台有副作用工具 | 创建提醒、联系家人、修改资料 | 平台业务 API | 用户确认、幂等键、权限和审计 |
|
||||
| 高风险动作 | 删除数据、转账、医疗决策 | 不直接开放给模型 | 使用既有人工流程 |
|
||||
|
||||
### 8.2 工具网关安全
|
||||
|
||||
- 工具调用必须映射到活动通话，不能只信任模型传来的 `subject_id`；
|
||||
- 参数使用固定 JSON Schema，拒绝未知字段；
|
||||
- 每个调用有稳定 `call_id`，平台按 `conversation_id + call_id` 幂等；
|
||||
- 返回内容限制长度并做提示词注入隔离；
|
||||
- 工具超时返回结构化降级结果，不中断音频主链；
|
||||
- 全量记录工具名、结果状态、耗时和资源版本，不记录不必要的敏感正文。
|
||||
|
||||
## 9. 隐私、录音和删除
|
||||
|
||||
### 9.1 授权撤回
|
||||
|
||||
当前系统在撤回字幕、原录音、故事整理或个性化记忆授权时，会把活动通话写入停止状态并投递可靠停止任务。迁移后必须实现等价闭环：
|
||||
|
||||
1. 平台事务内写入撤回、通话停止意图、Outbox 和审计；
|
||||
2. 在线设备通过心跳/SSE 命令立即关闭 SDK 会话；
|
||||
3. 平台调用火山服务端终止或吊销接口兜底；
|
||||
4. 未确认停止前禁止新建会话和工具访问；
|
||||
5. 超时进入阻断告警和人工处置，而不是静默标记完成。
|
||||
|
||||
### 9.2 双轨录音
|
||||
|
||||
这是迁移中最可能发生范围变化的功能。PoC 必须确认：
|
||||
|
||||
- 能否取得送入 SDK 前的近端 PCM；
|
||||
- 能否取得 SDK 解码后、播放前的远端 PCM；
|
||||
- 打断、重连、丢包隐藏和采样率变化后时间轴能否对齐；
|
||||
- SDK 是否允许在本地进行受授权的录制；
|
||||
- 若使用火山云端录制，能否满足近/远轨分离、短期访问、删除和审计要求。
|
||||
|
||||
如不能取得远端 PCM，不能把混音录音冒充现有双轨录音。第一阶段应明确限定为 `transcript_only`，待产品接受范围变更或供应商能力补齐后再开放录音。
|
||||
|
||||
### 9.3 删除和凭证吊销
|
||||
|
||||
主体隐私删除增加两个子任务：
|
||||
|
||||
- 删除或匿名化火山侧与该设备/会话相关的可识别数据；
|
||||
- 吊销设备智能体凭证和 License 绑定，并保存最小删除证明。
|
||||
|
||||
供应商无删除/吊销 API 时，需要人工控制台证据和对账任务，但该方式不适合大规模量产。
|
||||
|
||||
## 10. 用量、费用与 License
|
||||
|
||||
现有七类用量可保留，但新增来源维度：
|
||||
|
||||
```text
|
||||
source = provider_callback | device_session | provider_bill
|
||||
runtime = legacy_voicechat | volc_hardware_agent
|
||||
```
|
||||
|
||||
建议新增类别：
|
||||
|
||||
- `HARDWARE_AGENT_LICENSE`：设备 License 购买、激活、到期和转移；
|
||||
- `HARDWARE_AGENT_VOICE`：AI 对话时长；
|
||||
- `HARDWARE_AGENT_MODEL`：模型推理额度；
|
||||
- 视频、声音复刻、联网问答等扩展资源按实际开通单独归类。
|
||||
|
||||
设备上报只用于实时额度预估，最终账单以火山账单/用量接口为准，并执行会话、设备、账期三级对账。火山产品页显示 License 按设备提供不同对话时长和模型额度，实际采购价格、超额规则、License 回收和换机政策必须在合同中冻结。[火山硬件智能体产品页](https://www.volcengine.com/product/ConversationalAI-Embedded)
|
||||
|
||||
## 11. 灰度迁移步骤
|
||||
|
||||
### 阶段 0：供应商能力封板
|
||||
|
||||
在编码前取得正式 SDK、API、License 和数据处理条款，完成第 13 节问题闭环。输出一份签字的能力矩阵，未确认项不能用口头承诺替代。
|
||||
|
||||
### 阶段 1：独立真机 PoC
|
||||
|
||||
- 1 个测试产品、1 个 Bot、3 台候选板；
|
||||
- 跑通高质量 SDK 初始化、动态注册、建连、打断、重连、字幕和结束；
|
||||
- 验证近/远端 PCM、强制停话、凭证吊销、License 更换；
|
||||
- 暂不连接生产家庭/长者数据。
|
||||
|
||||
### 阶段 2：平台双运行时
|
||||
|
||||
- 新增运行时抽象、表和 `/device/v3`；
|
||||
- 旧 v2/RTC 链路保持不变；
|
||||
- 新设备只接固定 Bot，先打通授权、状态、字幕、内容处理和诊断；
|
||||
- 火山配置先由人工控制台维护，平台保存摘要和漂移检查。
|
||||
|
||||
### 阶段 3：个性化与工具
|
||||
|
||||
- 接入动态会话参数和冻结背景；
|
||||
- 接入平台记忆 MCP/Function Calling；
|
||||
- 接入角色、模型、音色、Bot 的发布和回滚；
|
||||
- 接入用量、License、隐私删除和供应商对账。
|
||||
|
||||
### 阶段 4：灰度试点
|
||||
|
||||
- 内部设备 -> 友好家庭 -> 小比例真实家庭；
|
||||
- 每个设备的运行时由不可变灰度版本决定；
|
||||
- 出现阻断问题时只切回 `legacy_rtc`，不修改历史通话运行时；
|
||||
- 灰度期间两个运行时分别统计成功率、首响、打断、重连、字幕、费用和投诉。
|
||||
|
||||
### 阶段 5：主链切换与退役
|
||||
|
||||
只有新链路连续通过真机、隐私、费用和运行恢复门禁后，才把新出厂设备默认设为硬件智能体。旧 `StartVoiceChat` 代码、RTC Token 签发和模板不能立即删除，至少保留到全部在役设备完成升级或退出支持期。
|
||||
|
||||
## 12. 验收与回滚门禁
|
||||
|
||||
建议新的真机最小矩阵至少覆盖 3 台量产候选设备、6 通会话：
|
||||
|
||||
1. 正常完整通话；
|
||||
2. 用户打断和 Agent 被打断；
|
||||
3. Wi-Fi/蜂窝或批准网络切换恢复；
|
||||
4. 平台受控停止；
|
||||
5. 活动通话中授权撤回；
|
||||
6. 设备断电重启后的字幕队列恢复和资源回收。
|
||||
|
||||
每通必须核对：
|
||||
|
||||
- 平台通话、火山会话、设备和 Bot 关联唯一；
|
||||
- 音频上行、下行、首响、打断和恢复符合现场冻结标准；
|
||||
- 最终字幕完整、顺序稳定、重放不重复；
|
||||
- 背景和工具调用严格限制在当前租户、长者和通话；
|
||||
- 结束后 SDK、License 占用、平台租约和工具令牌全部释放；
|
||||
- 设备、平台、火山、字幕、用量和隐私证据引用完整。
|
||||
|
||||
建议上线 SLO 在试点前冻结，不在代码中写死。至少包含：建连成功率、首次回复时延、打断成功率、异常重连成功率、字幕完整率、停止收敛时延、重复计费率、供应商账单差异和凭证吊销生效时延。
|
||||
|
||||
回滚规则：
|
||||
|
||||
- 回滚只影响新会话；活动会话按原运行时完成或受控停止；
|
||||
- 数据库中的 `runtime_type` 和配置摘要不可改写；
|
||||
- 设备切回旧运行时前必须确认旧 RTC SDK 和配置仍在；
|
||||
- 无法强制停话、出现跨租户工具访问、删除不收敛或凭证泄露时，立即停止硬件智能体新会话灰度。
|
||||
|
||||
## 13. 必须向火山确认的问题
|
||||
|
||||
以下问题未得到书面答案前，不建议承诺完整替换日期：
|
||||
|
||||
1. 是否提供按设备或会话的服务端强制终止接口？设备离线时多久生效？
|
||||
2. `device_secret` 是否支持轮换、吊销、批量禁用和换板迁移？
|
||||
3. 产品、设备、Bot 绑定和 License 是否都有 OpenAPI，而不是只能操作控制台？
|
||||
4. 是否有服务端会话开始、结束、最终字幕、错误和用量回调？签名和重放保护是什么？
|
||||
5. 是否允许传入平台生成的 `conversation_id/dialog_id` 并在回调和账单中原样返回？
|
||||
6. 动态会话参数可以覆盖哪些 Bot 配置？何时生效？是否保证收到 `session.updated` 后再开始识别？
|
||||
7. 高质量 SDK 是否同时提供近端输入 PCM 和远端播放 PCM 回调？采样率、声道和时间戳如何定义？
|
||||
8. Function Calling/MCP 的服务端鉴权、超时、重试、幂等和最大正文限制是什么？
|
||||
9. License 的激活、到期、换机、回收、离线宽限和超额计费规则是什么？
|
||||
10. 能否按设备、Bot、会话、租户标签导出用量并与账单对账？
|
||||
11. 会话音频、字幕、日志和模型上下文分别保留多久？如何导出和删除？数据地域在哪里？
|
||||
12. SDK 支持的芯片、编译器、RTOS/Linux、RAM/Flash/CPU、AEC/AGC/ANS、网络端口和证书更新要求是什么？
|
||||
13. 高质量和低负载方案能否在同一产品/设备上切换？切换是否会改变 Bot、License、回调和计费语义？
|
||||
14. 量产并发、建连 QPS、设备数、Bot 数和回调吞吐的配额如何申请和压测？
|
||||
|
||||
## 14. 主要风险
|
||||
|
||||
| 风险 | 影响 | 控制措施 |
|
||||
|---|---|---|
|
||||
| 产品级 Secret 被固件提取 | 同产品设备可被仿冒或批量注册 | 一次性安全注册、设备独享 Secret、安全存储、吊销演练 |
|
||||
| 火山控制台配置漂移 | 平台记录与实际 Bot 行为不一致 | 平台配置摘要、定时对账、变更工单、生产禁止无审计手改 |
|
||||
| 只能由设备报告会话结束 | 授权撤回时不能服务端强制停止 | 以供应商强停/吊销能力作为上线硬门槛 |
|
||||
| 字幕只有设备单路回传 | 断电或固件故障造成内容缺口 | 本地加密可靠队列、服务端回调双源、终稿补传和差异告警 |
|
||||
| SDK 无远端 PCM | 双轨录音功能无法等价迁移 | PoC 提前验证；不满足时明确限定 `transcript_only` |
|
||||
| License 与内部设备解绑不同步 | 换机失败、成本浪费或越权使用 | License 台账、绑定状态机、每日对账和售后流程 |
|
||||
| Bot 数量或动态参数受限 | 现有多租户灰度模型无法直接复制 | 少量基础 Bot + 会话动态参数；配置版本与供应商摘要冻结 |
|
||||
| 工具调用越权或重复 | 读取他人记忆或执行重复动作 | 会话范围令牌、服务端反查、Schema、幂等、确认和审计 |
|
||||
|
||||
## 15. 评审建议
|
||||
|
||||
甲方方案评审时可以把范围分成三层：
|
||||
|
||||
- 必须等价迁移：正常语音、打断、设备绑定、授权门禁、通话状态、最终字幕、摘要/故事、人工记忆、额度、诊断和受控停止；
|
||||
- 需要供应商能力确认：服务端强停、设备凭证吊销、字幕/用量服务端回调、动态会话参数、MCP 鉴权；
|
||||
- 可能需要范围调整：双轨录音、三种旧 `engine_mode`、RTC 房间级诊断和旧 VoiceChat 模板。
|
||||
|
||||
推荐先让甲方确认“硬件对话智能体只替换语音会话执行层，现有业务平台继续作为控制面和数据主库”。确认这条边界后，再进入 SDK PoC 和接口改造，能避免后续把火山控制台误当成整个业务平台替代品。
|
||||
@@ -0,0 +1,13 @@
|
||||
# For more information about build system see
|
||||
# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
|
||||
# The following five lines of boilerplate have to be in your project's
|
||||
# CMakeLists in this exact order for cmake to work correctly
|
||||
cmake_minimum_required(VERSION 3.5)
|
||||
|
||||
# include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
|
||||
#include($ENV{ADF_PATH}/CMakeLists.txt)
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
project(VolcRTCDemo)
|
||||
|
||||
# add_compile_definitions(LWIP_DEBUG=1)
|
||||
@@ -0,0 +1,211 @@
|
||||
<h1 align="center"><img src="https://iam.volccdn.com/obj/volcengine-public/pic/volcengine-icon.png"></h1>
|
||||
<h1 align="center">ConversationalAI Embedded Kit 2.0 </h1>
|
||||
|
||||
## EchoEar 平台版固件
|
||||
|
||||
本目录是在火山官方 Embedded Kit 2.0 基础上适配的 ESP32-S3 EchoEar 固件。它使用低负载 WebSocket 方案，并将设备身份、动态注册和会话配置统一接入自有 SaaS。
|
||||
|
||||
安全边界：
|
||||
|
||||
- 固件和设备端都不包含火山 `Product Secret`；
|
||||
- SaaS 使用平台设备身份为 EchoEar 动态申请火山 `Device Secret`；
|
||||
- EchoEar 只保存自己的平台设备凭据和火山单设备凭据；
|
||||
- 每次对话前由 SaaS 下发 Bot、人设、记忆上下文和会话配置；
|
||||
- 字幕和最终轮次回传 SaaS，供会话记录与长期记忆处理。
|
||||
|
||||
当前测试硬件：ESP32-S3 EchoEar，16 MB Flash、16 MB PSRAM，硬件标识由 STA MAC 生成（当前板为 `echoear-b43a451634a8`）。
|
||||
|
||||
每次申请对话前，先通过 `/device/v2/bootstrap` 上报实际固件版本并检查 `firmware_compatible`，再获取配置和创建会话。固件版本为 `1.0.0-echoear-platform.1`；本工程基于 Embedded Kit 2.0，硬件智能体 SDK 版本独立上报为 `2.0.0`。会话能力中的 `sdk_version` 不能用固件版本代替。会话尚未创建时保持唤醒采集，不重复拆建音频管线。SPI 屏幕仅在 `on_color_trans_done` 后释放 GFX 帧缓冲，避免异步传输期间提前复用缓冲区。运行状态用眼睛动画表示，避免 80 MHz 下中文逐像素渲染长期占用 CPU：快速眨眼表示聆听，转圈表示思考，正常眼睛表示待机或回答。
|
||||
|
||||
本板构建使用 16 MB Flash，Flash 与八线 PSRAM 均为 80 MHz。通用默认配置、ESP32-S3 默认配置和当前 `sdkconfig` 必须一致；不要恢复上游的 32 MB Flash 或实验性 120 MHz PSRAM 参数。
|
||||
|
||||
唤醒词为“嗨，乐鑫”，启用 `CONFIG_SR_WN_WN9_HILEXIN`，使用 `DET_MODE_95` 高灵敏度检测，并将 AFE 线性增益设为 2.0。生成的 `build/srmodels/srmodels.bin` 必须包含 `wn9_hilexin` 模型（约291 KB），并烧录到 `model` 分区 `0x510000`；不能使用未选择任何模型时生成的4字节空文件。
|
||||
|
||||
EchoEar V1.2 的 ES7210 使用 TDM：I2S 按两路32位接收，AFE 按四路16位解释，布局为 `RMNM`（ch1/ch3 麦克风、ch0 回声参考、ch2 忽略）。不可改成两路16位 `MR`。诊断日志 `MIC_LEVEL` 仅记录各路幅度统计，不保存录音。PCM输出每帧60 ms、16 kHz、单声道16位，读取缓冲区为1920字节，避免截断半帧。唤醒事件在会话服务就绪后直接发布，不依赖已移除的 `CONFIG_LANGUAGE_WAKEUP_MODE`。
|
||||
|
||||
低负载 WebSocket 任务使用 12 KiB 栈；4 KiB 会在收到 `session.created` 后发生栈溢出并重启。握手期间暂停上传麦克风帧，待引擎进入 STARTED 后再发送。低负载接口返回的 PCM 按 24 kHz 单声道16位解码，再由播放管线转换到板载 16 kHz I2S。`session.created` 的 `job_id` 用作平台启动确认，确保后续字幕和结束事件在 ACTIVE 会话上提交。
|
||||
|
||||
联网、平台 HTTPS 激活和音频初始化在独立的 24 KiB 启动任务中执行，避免默认 4 KiB `main` 栈溢出导致循环重启；任务退出时释放栈，并记录启动阶段最小栈余量。平台激活成功后才启动音频采集。激活失败时保留设备配置、显示绑定或配置提示，并仅记录平台错误码，不输出响应正文或凭据。
|
||||
|
||||
STA 连接使用全信道扫描并按信号强度选择同名 AP，实时语音期间关闭 Wi-Fi 省电，避免多 AP 环境中误连弱信号节点造成 WebSocket 写入失败。
|
||||
|
||||
### Windows 编译与烧录
|
||||
|
||||
使用 ESP-IDF v5.5.1：
|
||||
|
||||
```powershell
|
||||
. 'E:\Codex\toolchains\esp-idf-v5.5.1\export.ps1'
|
||||
idf.py build
|
||||
idf.py -p COM6 flash
|
||||
idf.py -p COM6 monitor
|
||||
```
|
||||
|
||||
首次烧录前建议完整备份 Flash。当前板原厂备份位于 `backups/b43a451634a8/factory-2026-09-03-16mb.bin`，恢复命令：
|
||||
|
||||
```powershell
|
||||
esptool.py --chip esp32s3 --port COM6 write_flash 0x0 backups/b43a451634a8/factory-2026-09-03-16mb.bin
|
||||
```
|
||||
|
||||
### 首次激活
|
||||
|
||||
首次启动、缺少平台设备凭据、保存的 Wi-Fi 连接失败或超时，以及手动请求重新配网时，都会进入配置热点，无需清空 NVS：
|
||||
|
||||
1. 连接 `VolcConvAI-XXXXXX`，热点密码与热点名相同；
|
||||
2. 打开 `http://192.168.4.1`；
|
||||
3. 填写 2.4 GHz Wi-Fi、平台 Device ID 和平台 Device Secret；
|
||||
4. 重启后，设备通过 SaaS 完成火山动态注册并获取单设备运行凭据。
|
||||
|
||||
正常运行时短触圆屏任意位置可直接开始对话，作为唤醒词的可靠备用入口。屏幕底部区域持续按住3秒后设备重启并开启配置热点。该操作保留原来的 Wi-Fi 和设备身份，只记录重新配网请求；成功提交配网页后退出配网模式。触摸任务在联网和平台激活之前创建，因此激活失败时也可以请求重新配网。
|
||||
|
||||
如果尚未配置平台设备身份，固件不会尝试用 Product Secret 直接访问火山。
|
||||
|
||||
重新配网回归检查（需要烧录后实机验证）：
|
||||
|
||||
- 已保存可用 Wi-Fi、但缺少平台凭据：启动后直接出现热点和配网地址。
|
||||
- Wi-Fi 密码错误或 DHCP 超时：连接等待结束后进入热点，不能重启循环。
|
||||
- 正常运行时：短触屏幕启动对话；按住屏幕底部区域3秒进入配置热点。
|
||||
- 配网页提交错误 Wi-Fi 后可重试；只有取得 IP 地址后才保存配置并重启。
|
||||
- 配网页成功提交后正常启动，手动配网请求清除，原平台身份不会因进入配网而被删除。
|
||||
|
||||
本地 `idf.py build` 仅验证编译、链接和分区大小，不代替热点可见性、触摸坐标及长按时长的实机验证。
|
||||
|
||||
配网页提交使用 JSON 与后台连接任务，`POST /submit` 受理后立即返回 202，页面通过 `GET /setup-status` 查询连接、保存成功或失败状态。连接失败可以重试；保存成功后等待15秒重启，再由启动流程执行平台激活。Wi-Fi 扫描改为手动触发，连接期间不扫描。更新固件后请刷新配网页，避免继续使用已经打开的旧表单。
|
||||
|
||||
配网页交互回归：`node --test tests/setup-portal.test.cjs`（模拟设备响应，覆盖失败重试、响应丢失、断联、成功提示和输入校验）。
|
||||
|
||||
## 快速开始
|
||||
|
||||
具体操作，请参考 [官网文档](https://www.volcengine.com/docs/6348/1806625)。
|
||||
|
||||
## 运行设备端（乐鑫）
|
||||
|
||||
以下操作以 macOS 操作系统为例。
|
||||
|
||||
### 环境与硬件要求
|
||||
|
||||
- 乐鑫 ESP32-S3-Korvo-2
|
||||
- USB 数据线：两条 A 转 Micro-B 数据线，一条作为电源线，一条作为串口线。
|
||||
- PC 设备：编译和烧录。支持 Windows、Linux 或者 macOS 操作系统。（本文操作以 macOS 为例）
|
||||
|
||||
### 配置乐鑫环境
|
||||
|
||||
详见[开发环境配置文档](https://docs.espressif.com/projects/esp-idf/zh_CN/stable/esp32s3/get-started/index.html)。
|
||||
|
||||
1. 安装 CMake 和 Ninja 编译工具。
|
||||
|
||||
```bash
|
||||
brew install cmake ninja dfu-util
|
||||
```
|
||||
2. 将乐鑫 ADF 框架克隆到本地，并同步各子仓（submodule）代码。
|
||||
|
||||
> **注意**：Demo 中使用的 ADF 版本为 `eca11f20e56f9b5321b714da4305e123672d92a9`，对应 IDF 版本为 `v5.4`，请确保 ADF 版本与 IDF 版本匹配。
|
||||
>
|
||||
|
||||
```bash
|
||||
# 1. clone 乐鑫 ADF 框架
|
||||
git clone https://github.com/espressif/esp-adf.git
|
||||
# 2. 进入esp-adf目录
|
||||
cd esp-adf
|
||||
# 3. 切换到乐鑫 ADF 指定版本
|
||||
git reset --hard eca11f20e56f9b5321b714da4305e123672d92a9
|
||||
# 4. 同步各子仓代码
|
||||
git submodule update --init --recursive
|
||||
```
|
||||
3. 安装乐鑫 esp32s3 开发环境相关依赖。
|
||||
|
||||
```bash
|
||||
./install.sh esp32s3
|
||||
```
|
||||
|
||||
成功安装所有依赖后，命令行会出现如下提示：
|
||||
```bash
|
||||
All done! You can now run:
|
||||
. ./export.sh
|
||||
```
|
||||
|
||||
> 如在上述任何步骤中遇到以下错误:
|
||||
> `<urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:xxx)`
|
||||
> 可前往**访达->应用程序->Python** 文件夹，点击 `Install Certificates.command` 安装证书。更多信息，请参考 [安装 ESP-IDF 工具时出现的下载错误](https://github.com/espressif/esp-idf/issues/4775)。
|
||||
>
|
||||
4. 设置环境变量。
|
||||
|
||||
> **每次打开命令行窗口均需要运行该命令进行设置。**
|
||||
>
|
||||
|
||||
```bash
|
||||
. ./export.sh
|
||||
```
|
||||
|
||||
### 下载并配置工程
|
||||
|
||||
1. 将实时对话式 AI 硬件示例工程克隆到 乐鑫 ADF examples 目录下。
|
||||
|
||||
1. 进入 esp-adf/examples 目录。
|
||||
|
||||
```bash
|
||||
cd $ADF_PATH/examples
|
||||
```
|
||||
|
||||
2. 克隆实时对话式 AI 硬件示例工程。
|
||||
|
||||
```bash
|
||||
git clone https://github.com/volcengine/ConversationalAI-Embedded-Kit-2.0.git
|
||||
```
|
||||
2. 修复乐鑫按键问题
|
||||
|
||||
1. 进入 esp-adf 目录。
|
||||
|
||||
```bash
|
||||
cd $ADF_PATH
|
||||
```
|
||||
|
||||
2. 修复乐鑫按键问题。
|
||||
|
||||
```bash
|
||||
git apply $ADF_PATH/examples/ConversationalAI-Embedded-Kit-2.0/low_load_first/espressif/0002-fix-esp-button.patch
|
||||
```
|
||||
|
||||
### 编译固件
|
||||
|
||||
进入 `esp-adf/examples/ConversationalAI-Embedded-Kit-2.0/low_load_first/espressif` 目录下编译固件。
|
||||
|
||||
1. 进入 espressif 目录。
|
||||
```bash
|
||||
cd $ADF_PATH/examples/ConversationalAI-Embedded-Kit-2.0/low_load_first/espressif
|
||||
```
|
||||
2. 设置编译目标平台。
|
||||
```bash
|
||||
idf.py set-target esp32s3
|
||||
```
|
||||
3. 设置 实例ID、产品ID、产品秘钥、设备ID等参数。
|
||||
```bash
|
||||
idf.py menuconfig
|
||||
```
|
||||
|
||||
进入 `Example Configuration` 菜单，在 `volcano instance id` 中填入你的实例ID，在 `volcano product key` 中填入你的产品Key，在 `volcano product secret` 中填入你的产品秘钥，在 `device name` 中填入你的设备ID， 在 `bot id` 中填入你的智能体ID，并保存。
|
||||
4. 编译固件。
|
||||
```bash
|
||||
idf.py build
|
||||
```
|
||||
|
||||
### 烧录并运行示例 Demo
|
||||
|
||||
1. 打开乐鑫开发板电源开关。
|
||||
2. 烧录固件。
|
||||
|
||||
```bash
|
||||
idf.py flash
|
||||
```
|
||||
3. 运行示例 Demo 并查看串口日志输出。
|
||||
|
||||
```bash
|
||||
idf.py monitor
|
||||
```
|
||||
4. Wi-Fi 配网。
|
||||
|
||||
1. 手机找到名如 VolcConvAI-XXXXXX” 的 Wi-Fi 热点，密码同Wi-Fi名，连接上 Wi-Fi。
|
||||
2. 打开浏览器，在地址栏输入 `http://192.168.4.1`，进入 Wi-Fi 配网页面。
|
||||
3. 输入 Wi-Fi 名称和密码，点击提交。
|
||||
|
||||
> **注意**：如果需更换 Wi-Fi，请重启设备。如果设备重启后无法连接到之前保存的 Wi-Fi（例如超出了范围或旧网络已关闭），请等待 30s 进入配网模式，再重新执行上面 Wi-Fi 配网的 3 个步骤。
|
||||
>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,27 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
message(STATUS "................. ${CMAKE_SYSTEM_NAME}")
|
||||
if("${CMAKE_SYSTEM_NAME}" MATCHES "(Rtos|rtos|RTOS)+")
|
||||
set(PLATFORM_RTOS TRUE)
|
||||
add_compile_definitions(PLATFORM_RTOS)
|
||||
elseif("${CMAKE_SYSTEM_NAME}" MATCHES "(Linux|linux|LINUX)+")
|
||||
set(PLATFORM_LINUX TRUE)
|
||||
add_compile_definitions(PLATFORM_LINUX)
|
||||
elseif("${CMAKE_SYSTEM_NAME}" MATCHES "(android|Android|ANDROID)+")
|
||||
set(PLATFORM_ANDROID TRUE)
|
||||
add_compile_definitions(PLATFORM_ANDROID)
|
||||
elseif("${CMAKE_SYSTEM_NAME}" MATCHES "(Darwin|darwin|MACOSX|macosx|MACOS|macos)+")
|
||||
set(PLATFORM_MACOS TRUE)
|
||||
add_compile_definitions(PLATFORM_MACOS)
|
||||
endif()
|
||||
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/cmake/common.cmake)
|
||||
if(ESP_PLATFORM)
|
||||
message(STATUS "ESP_PLATFORM is defined, including esp.cmake")
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/cmake/espressif.cmake)
|
||||
return()
|
||||
elseif(PLATFORM_MACOS)
|
||||
message(STATUS "PLATFORM_MACOS is defined, including macos.cmake")
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/cmake/macos.cmake)
|
||||
return()
|
||||
endif()
|
||||
+734
@@ -0,0 +1,734 @@
|
||||
# `application` 代码梳理
|
||||
|
||||
> 说明：本文档由 AI 在阅读当前目录代码后生成，目标是帮助读者快速理解代码结构、模块职责与主要运行流程，作为代码导读使用。
|
||||
|
||||
## 1. 目录定位
|
||||
|
||||
`application` 不是单一的业务文件夹，而是一套面向设备侧 AI 对话场景的“应用层组件”：
|
||||
|
||||
- `framework/` 提供事件驱动框架，解决不同业务模块之间的通信问题。
|
||||
- `platform/` 提供硬件抽象层 HAL，屏蔽板级和外设细节。
|
||||
- `service/` 提供 AI 对话业务拆分后的多个服务模块。
|
||||
- `cmake/` 负责把 framework、platform、service 组装成一个 ESP-IDF 组件。
|
||||
|
||||
结合各子目录 `README_zh.md` 的设计说明，可以把这套代码理解为：
|
||||
|
||||
- `volc_conv_ai` 是底层云连接 SDK。
|
||||
- `application` 则是建立在该 SDK 之上的业务应用框架。
|
||||
|
||||
也就是说，`application` 关注的是“设备如何组织业务逻辑、如何和硬件协作、如何把 AI 对话跑起来”，而不是连接协议本身。
|
||||
|
||||
## 2. 总体分层
|
||||
|
||||
整个目录可以概括成 3 层：
|
||||
|
||||
### 2.1 Framework 层
|
||||
|
||||
职责是让不同业务模块通过事件 `pub/sub` 通信，不直接相互调用。
|
||||
|
||||
核心文件：
|
||||
|
||||
- `framework/inc/aios.h`
|
||||
- `framework/inc/aios_def.h`
|
||||
- `framework/src/aios.c`
|
||||
|
||||
这是整个应用层的“调度内核”。
|
||||
|
||||
### 2.2 Platform 层
|
||||
|
||||
职责是为 AI 对话场景提供硬件抽象：
|
||||
|
||||
- 音频采集
|
||||
- 音频播放
|
||||
- 屏幕显示
|
||||
- 按键
|
||||
- 文件系统
|
||||
- 全局硬件上下文
|
||||
|
||||
核心文件：
|
||||
|
||||
- `platform/inc/volc_hal.h`
|
||||
- `platform/inc/volc_hal_*.h`
|
||||
- `platform/src/espressif/*.c`
|
||||
|
||||
从当前实现看，`platform/src/espressif` 是主实现平台，明显以 ESP32/ESP-IDF 为目标。
|
||||
|
||||
### 2.3 Service 层
|
||||
|
||||
职责是把 AI 对话业务拆成几个相互解耦的服务：
|
||||
|
||||
- `conv_service`
|
||||
- 负责 AI 对话的建立、运行、退出。
|
||||
- `function_call_service`
|
||||
- 负责处理智能体触发的 function call。
|
||||
- `local_logic_service`
|
||||
- 负责本地预设逻辑，不依赖大模型理解。
|
||||
- `service/src/volc_manager_service.c`
|
||||
- 负责接收高层业务入口事件，并拉起对应 service。
|
||||
|
||||
这与 `service/README_zh.md` 的描述是一致的。
|
||||
|
||||
## 3. 这个目录不是完整 App Main
|
||||
|
||||
单看 `application` 目录，本身**没有完整的 `main/app_main`**。
|
||||
|
||||
它更像是一个“可复用的应用组件库”。真正的组装顺序出现在仓库里的示例，例如：
|
||||
|
||||
- `examples/high_quality_solution/espressif/main/conv_ai_embedded_kit.c`
|
||||
|
||||
该示例展示了应用启动的标准顺序：
|
||||
|
||||
1. `volc_hal_init()`
|
||||
2. `volc_hal_display_create()`
|
||||
3. 初始化网络、时间同步等系统能力
|
||||
4. `aios_init(VOLC_SERVICE_EVENT_MAX)`
|
||||
5. 初始化各 service：
|
||||
- `volc_service_manager_init()`
|
||||
- `volc_conv_service_manager_init()`
|
||||
- `function_call_service_init()`
|
||||
- `local_logic_service_init()`
|
||||
6. `aios_run()`
|
||||
7. 在 wakeup 回调中发布 `VOLC_SERVICE_AI_CONVERSATION` 事件
|
||||
|
||||
因此，`application` 目录的代码更应该被理解成“主程序可直接引用的一套模块化应用骨架”。
|
||||
|
||||
## 4. Framework：AIOS 事件框架
|
||||
|
||||
### 4.1 AIOS 是什么
|
||||
|
||||
`framework` 的 README_zh 说明得很直白：它是为了让不同业务模块通过 `sub/pub event` 进行通信。
|
||||
|
||||
`aios.c` 的实现表明，这实际上是一个：
|
||||
|
||||
- 单线程事件循环
|
||||
- 状态机驱动
|
||||
- 带优先级的 session 调度器
|
||||
- 带定时器能力的轻量运行时
|
||||
|
||||
### 4.2 核心概念
|
||||
|
||||
#### 1. `aios_session_t`
|
||||
|
||||
表示一个业务状态机实例，包含：
|
||||
|
||||
- 优先级
|
||||
- 是否启用
|
||||
- 当前状态函数
|
||||
|
||||
应用中的每个 service 基本都对应一个 session。
|
||||
|
||||
#### 2. `aios_event_t`
|
||||
|
||||
表示一个事件，包含：
|
||||
|
||||
- 事件 ID
|
||||
- 当前尚未消费它的订阅者位图
|
||||
- 事件数据
|
||||
- 数据释放回调
|
||||
|
||||
#### 3. 状态函数
|
||||
|
||||
每个 session 都由状态函数驱动，签名是：
|
||||
|
||||
```c
|
||||
aios_ret_t (*aios_state_handler)(struct aios_session *const me, aios_event_t const * const e);
|
||||
```
|
||||
|
||||
返回值可以表示：
|
||||
|
||||
- 已处理
|
||||
- 未处理
|
||||
- 状态迁移
|
||||
|
||||
### 4.3 调度方式
|
||||
|
||||
`aios_run()` 会不断调用 `aios_once()`。
|
||||
|
||||
`aios_once()` 的主逻辑是：
|
||||
|
||||
1. 处理到期定时器，定时器会转化为普通事件。
|
||||
2. 如果当前没有事件，则进入可中断睡眠。
|
||||
3. 从高优先级到低优先级寻找“当前有订阅事件待处理”的 session。
|
||||
4. 找出这个 session 最早应该消费的事件。
|
||||
5. 调用 session 的当前状态函数处理事件。
|
||||
6. 如果该事件没有剩余订阅者，则释放它。
|
||||
|
||||
这套机制的重点是：
|
||||
|
||||
- 事件是共享的，但每个订阅者只消费一次。
|
||||
- 调度按 session 优先级进行。
|
||||
- 不同 service 之间完全依赖事件通信。
|
||||
|
||||
### 4.4 为什么适合这个项目
|
||||
|
||||
这套框架特别适合当前项目，因为这里的业务是典型的“多模块协作”：
|
||||
|
||||
- 唤醒触发 AI 对话
|
||||
- AI 对话过程中收到字幕
|
||||
- 字幕再触发本地逻辑
|
||||
- function call 消息再触发本地动作
|
||||
|
||||
如果全部互相直接调用，耦合会很高；用 AIOS 后，模块只需要关心自己订阅和发布什么事件。
|
||||
|
||||
## 5. Service 层：业务模块划分
|
||||
|
||||
`service/README_zh.md` 已给出分工，源码进一步明确了具体职责。
|
||||
|
||||
### 5.1 公共事件定义
|
||||
|
||||
`service/inc/volc_service_common.h` 是业务总线的关键头文件，定义了：
|
||||
|
||||
- 事件 ID
|
||||
- session 优先级
|
||||
|
||||
主要事件包括：
|
||||
|
||||
- `VOLC_SERVICE_AI_CONVERSATION`
|
||||
- 请求进入 AI 对话
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_START`
|
||||
- 真正启动 AI 对话线程
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_QUIT`
|
||||
- 退出 AI 对话
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_OVER`
|
||||
- AI 对话线程结束后的通知
|
||||
- `VOLC_FUNCTION_CALL_EXEC`
|
||||
- 执行 function call
|
||||
- `VOLC_FUNCTION_CALL_TRIGGER`
|
||||
- function call 被智能体触发
|
||||
- `VOLC_LOCAL_LOGIC_PLAY_WELCOME`
|
||||
- 播放欢迎音
|
||||
- `VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE`
|
||||
- 本地处理字幕
|
||||
|
||||
优先级定义为：
|
||||
|
||||
- AI 对话服务优先级最低编号 `0`
|
||||
- 然后是 function call
|
||||
- 然后是 local logic
|
||||
- app manager 优先级最高编号 `3`
|
||||
|
||||
在 `aios.c` 中，高优先级 session 会先被选中执行。
|
||||
|
||||
## 6. `volc_manager_service`：业务入口管理
|
||||
|
||||
文件：
|
||||
|
||||
- `service/src/volc_manager_service.c`
|
||||
|
||||
这是最上层的“应用业务入口控制器”。
|
||||
|
||||
### 6.1 它做了什么
|
||||
|
||||
它订阅两个顶层事件：
|
||||
|
||||
- `VOLC_SERVICE_NETWORK_CONFIG`
|
||||
- `VOLC_SERVICE_AI_CONVERSATION`
|
||||
|
||||
当前代码里真正有效的是：
|
||||
|
||||
- 收到 `VOLC_SERVICE_AI_CONVERSATION`
|
||||
- 再发布 `VOLC_SERVICE_AI_CONVERSATION_START`
|
||||
|
||||
因此它的本质是：
|
||||
|
||||
- 作为“业务启动总入口”
|
||||
- 把更粗粒度的用户/系统动作转换成具体 service 执行动作
|
||||
|
||||
### 6.2 作用
|
||||
|
||||
这一层让“谁要求启动 AI 对话”和“AI 对话服务怎么起线程”分离开了。
|
||||
|
||||
## 7. `conv_service`：核心 AI 对话服务
|
||||
|
||||
这是整个目录最关键的业务模块。
|
||||
|
||||
相关文件：
|
||||
|
||||
- `service/conv_service/src/volc_conv_service.c`
|
||||
- `service/conv_service/src/volc_conv.c`
|
||||
|
||||
两者分工明显：
|
||||
|
||||
- `volc_conv_service.c`
|
||||
- service 管理器，负责事件响应与线程管理
|
||||
- `volc_conv.c`
|
||||
- AI 对话线程的真正业务实现
|
||||
|
||||
### 7.1 `volc_conv_service_manager.c` 的职责
|
||||
|
||||
`volc_conv_service_manager_init()` 会注册一个 session，并订阅：
|
||||
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_START`
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_INTERRUPT`
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_QUIT`
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_OVER`
|
||||
|
||||
收到不同事件后的行为：
|
||||
|
||||
- `START`
|
||||
- 创建 `conv_ai_service_task` 线程
|
||||
- `QUIT`
|
||||
- 调用 `conv_ai_service_task_stop()`
|
||||
- `OVER`
|
||||
- 清空线程句柄
|
||||
|
||||
也就是说，这一层负责“线程生命周期管理”，而不是 AI 对话本身。
|
||||
|
||||
### 7.2 `volc_conv.c` 的职责
|
||||
|
||||
这个文件是 AI 对话业务主实现。
|
||||
|
||||
它做了 5 件关键事情：
|
||||
|
||||
#### 1. 构造 `volc_conv_ai` 配置
|
||||
|
||||
通过 `CONV_AI_CONFIG_FORMAT` 拼出连接云端 SDK 所需的 JSON，包含：
|
||||
|
||||
- IoT 配置
|
||||
- RTC 配置
|
||||
- 编解码参数
|
||||
- RTC 参数扩展
|
||||
|
||||
然后在 `conv_ai_service_init()` 里调用：
|
||||
|
||||
- `volc_create()`
|
||||
|
||||
这说明 `application` 这一层直接把底层 SDK 当成对话引擎使用。
|
||||
|
||||
#### 2. 建立采集与播放
|
||||
|
||||
`conv_ai_service_task()` 启动时会：
|
||||
|
||||
- 创建或复用音频采集对象
|
||||
- 切换采集模式到 `VOLC_AUDIO_MODE_CAPTURE`
|
||||
- 创建或复用音频播放器
|
||||
- 启动播放器
|
||||
|
||||
其中：
|
||||
|
||||
- 采集回调 `__audio_capture_cb()`
|
||||
- 采集到音频后立即调用 `volc_send_audio_data()`
|
||||
- 音频回调 `__on_volc_audio_data()`
|
||||
- 收到智能体返回的音频后用 `volc_hal_player_play_data()` 播放
|
||||
|
||||
这就形成了一个完整的音频闭环：
|
||||
|
||||
- 麦克风采集 -> 发送到云端
|
||||
- 云端返回语音 -> 喇叭播放
|
||||
|
||||
#### 3. 处理会话状态并驱动 UI
|
||||
|
||||
`__on_volc_conversation_status()` 会根据 SDK 回调状态更新显示层：
|
||||
|
||||
- `LISTENING`
|
||||
- 状态栏显示“智能体聆听中”
|
||||
- `THINKING`
|
||||
- 主画面切到思考表情
|
||||
- 状态栏显示“智能体思考中”
|
||||
- `ANSWERING`
|
||||
- 主画面切到开心表情
|
||||
- 状态栏显示“智能体说话中”
|
||||
- `INTERRUPTED`
|
||||
- 状态栏显示“智能体被打断”
|
||||
- `ANSWER_FINISH`
|
||||
- 状态栏显示“智能体说话完成”
|
||||
|
||||
这表明 UI 与对话状态是强绑定的。
|
||||
|
||||
#### 4. 处理消息数据
|
||||
|
||||
`__on_volc_message_data()` 会解析从底层 SDK 收到的二进制消息头：
|
||||
|
||||
- `subv`
|
||||
- 字幕消息
|
||||
- `tool`
|
||||
- function call 执行消息
|
||||
- `info`
|
||||
- function call 触发提示消息
|
||||
|
||||
随后：
|
||||
|
||||
- 字幕消息交给 `__on_subtitle_message_received()`
|
||||
- `tool` 发布 `VOLC_FUNCTION_CALL_EXEC`
|
||||
- `info` 发布 `VOLC_FUNCTION_CALL_TRIGGER`
|
||||
|
||||
所以 `conv_service` 其实是“AI 中枢”，它把底层 SDK 输出的不同类别消息重新分发给其他 service。
|
||||
|
||||
#### 5. 管理对话生命周期
|
||||
|
||||
线程主体会：
|
||||
|
||||
1. 启动采集与播放
|
||||
2. 调用 `volc_start()`
|
||||
3. 进入循环等待，按秒累加 `wait_time`
|
||||
4. 超时或被要求退出时：
|
||||
- `volc_stop()`
|
||||
- 发布 `VOLC_SERVICE_AI_CONVERSATION_OVER`
|
||||
- 停止采集、停止播放、销毁播放对象
|
||||
- 重启唤醒模式采集
|
||||
- 恢复待机 UI
|
||||
|
||||
这里的设计非常关键：
|
||||
|
||||
- AI 对话期间使用 `CAPTURE` 模式持续采音
|
||||
- AI 对话结束后恢复 `WAKEUP` 模式等待下一次唤醒
|
||||
|
||||
这体现了“待机态/对话态”双模式切换。
|
||||
|
||||
## 8. `function_call_service`：函数调用桥接层
|
||||
|
||||
文件：
|
||||
|
||||
- `service/function_call_service/src/volc_function_call_service.c`
|
||||
- `service/function_call_service/src/volc_local_function_list.c`
|
||||
|
||||
### 8.1 作用
|
||||
|
||||
这个 service 专门处理智能体下发的 function call。
|
||||
|
||||
当前代码里只实现了一个本地函数：
|
||||
|
||||
- `adjust_audio_val`
|
||||
|
||||
### 8.2 两类消息
|
||||
|
||||
#### 1. 触发提示消息 `VOLC_FUNCTION_CALL_TRIGGER`
|
||||
|
||||
收到后，会判断是否是 `adjust_audio_val`，如果是就先调用：
|
||||
|
||||
- `volc_send_text_to_agent(..., "别着急，我来给你调整音量", VOLC_AGENT_TYPE_TTS, 2)`
|
||||
|
||||
这相当于先让智能体“说一句回应”，再执行工具。
|
||||
|
||||
#### 2. 执行消息 `VOLC_FUNCTION_CALL_EXEC`
|
||||
|
||||
收到真正的 `tool_calls` JSON 后：
|
||||
|
||||
- 解析函数名
|
||||
- 解析参数
|
||||
- 如果是 `adjust_audio_val`
|
||||
- 调用本地音量调整逻辑
|
||||
- 再构造带 `func` 头的结果消息
|
||||
- 调用 `volc_send_message()` 把执行结果回传给智能体
|
||||
|
||||
### 8.3 本地函数实现
|
||||
|
||||
`volc_local_function_list.c` 中实现了：
|
||||
|
||||
- 调大音量
|
||||
- 调小音量
|
||||
|
||||
它通过 HAL 获取当前播放器句柄，再调用：
|
||||
|
||||
- `volc_hal_get_audio_player_volume()`
|
||||
- `volc_hal_set_audio_player_volume()`
|
||||
|
||||
所以 function call 服务的本质是：
|
||||
|
||||
- 把模型侧工具调用，映射为设备本地硬件动作
|
||||
|
||||
## 9. `local_logic_service`：本地规则逻辑
|
||||
|
||||
文件：
|
||||
|
||||
- `service/local_logic_service/src/local_logic_service.c`
|
||||
|
||||
### 9.1 它与 function call 的区别
|
||||
|
||||
README_zh 已经说明了区别：
|
||||
|
||||
- `function_call_service`
|
||||
- 处理智能体明确发起的工具调用
|
||||
- `local_logic_service`
|
||||
- 处理设备本地预设逻辑，不依赖智能体工具协议
|
||||
|
||||
### 9.2 当前实现的两个能力
|
||||
|
||||
#### 1. 播放欢迎音
|
||||
|
||||
收到 `VOLC_LOCAL_LOGIC_PLAY_WELCOME` 事件时：
|
||||
|
||||
- 从 `/sdcard/wel.pcm` 读取 PCM 数据
|
||||
- 逐块送入播放器播放
|
||||
|
||||
#### 2. 解析字幕并触发退出逻辑
|
||||
|
||||
收到 `VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE` 事件时：
|
||||
|
||||
- 检查字幕中是否包含“退下吧”或“拜拜”
|
||||
- 如果命中，则发送一条 `TTS` 给智能体：“好的拜拜”
|
||||
- 当后续字幕中真的出现“好的拜拜”时，再发布：
|
||||
- `VOLC_SERVICE_AI_CONVERSATION_QUIT`
|
||||
|
||||
这个逻辑很有代表性，它说明：
|
||||
|
||||
- 本地逻辑可以直接基于字幕内容快速决策
|
||||
- 不必走 function call，也不必重新交给云端理解
|
||||
|
||||
## 10. Platform：硬件抽象层
|
||||
|
||||
`platform/README_zh.md` 说明了整体设计原则：
|
||||
|
||||
- 接口正交，尽量屏蔽硬件细节
|
||||
- 但为了 AI 对话场景易用性，允许一定“场景定制化”
|
||||
|
||||
核心思想是：
|
||||
|
||||
- 不追求绝对通用
|
||||
- 优先服务“AI 对话设备”这个具体场景
|
||||
|
||||
### 10.1 `volc_hal.h`：全局 HAL 上下文
|
||||
|
||||
`platform/inc/volc_hal.h` 定义了全局结构 `volc_hal_context_t`，里面集中保存：
|
||||
|
||||
- 按键句柄
|
||||
- 音频/视频采集句柄
|
||||
- 显示句柄
|
||||
- 音频/视频播放器句柄
|
||||
- 设备名
|
||||
|
||||
`volc_hal_init()` 只做必要初始化，不要求把所有外设一次性拉起。
|
||||
|
||||
### 10.2 `volc_hal.c`：ESP 平台 HAL 根初始化
|
||||
|
||||
当前 `espressif/volc_hal.c` 会完成：
|
||||
|
||||
- 分配全局 HAL context
|
||||
- 调用 `basic_board_init()` 完成板级外设探测与初始化
|
||||
- 初始化文件系统
|
||||
- 根据 MAC 生成设备名
|
||||
|
||||
其中 `basic_board.c` 又做了：
|
||||
|
||||
- 查询板型
|
||||
- 绑定播放设备、录音设备、LCD 设备
|
||||
- 配置采样率、位宽、通道数、mic layout
|
||||
- 打开 codec 设备
|
||||
|
||||
所以 `volc_hal_init()` 实际上是整个硬件层的入口。
|
||||
|
||||
## 11. 平台子模块
|
||||
|
||||
### 11.1 音频采集 `volc_hal_capture`
|
||||
|
||||
头文件：
|
||||
|
||||
- `platform/inc/volc_hal_capture.h`
|
||||
|
||||
实现：
|
||||
|
||||
- `platform/src/espressif/volc_hal_capture.c`
|
||||
|
||||
这个模块的重要特点有两个：
|
||||
|
||||
#### 1. 回调式采集
|
||||
|
||||
不是业务主动 `read()`，而是内部线程采集后异步回调一帧帧音频数据。
|
||||
|
||||
这与 README_zh 的设计说明完全一致。
|
||||
|
||||
#### 2. 支持两种音频模式
|
||||
|
||||
- `VOLC_AUDIO_MODE_WAKEUP`
|
||||
- `VOLC_AUDIO_MODE_CAPTURE`
|
||||
|
||||
当前实现里：
|
||||
|
||||
- `WAKEUP`
|
||||
- 打开带唤醒能力的 AFE/Recorder
|
||||
- `CAPTURE`
|
||||
- 打开正常语音采集
|
||||
|
||||
但底层仍然复用同一套采集模块，只是内部 pipeline 模式不同。
|
||||
|
||||
这也是 README_zh 中“唤醒和采集属于同一音频采集范畴”的具体落地。
|
||||
|
||||
### 11.2 音频播放 `volc_hal_player`
|
||||
|
||||
头文件：
|
||||
|
||||
- `platform/inc/volc_hal_player.h`
|
||||
|
||||
实现：
|
||||
|
||||
- `platform/src/espressif/volc_hal_player.c`
|
||||
|
||||
当前重点支持音频播放：
|
||||
|
||||
- 创建时初始化 feeder
|
||||
- `start()` 启动喂音流程
|
||||
- `play_data()` 将数据送入播放链路
|
||||
- 支持音量获取和设置
|
||||
|
||||
业务层基本不需要关心具体解码器或采样参数，因为这些在编译期和平台层已经固化。
|
||||
|
||||
### 11.3 显示 `volc_hal_display`
|
||||
|
||||
头文件：
|
||||
|
||||
- `platform/inc/volc_hal_display.h`
|
||||
|
||||
实现有两套：
|
||||
|
||||
- `platform/src/espressif/volc_hal_display.c`
|
||||
- `platform/src/espressif/volc_hal_display_lvgl.c`
|
||||
|
||||
README_zh 提到，显示层不是让业务任意创建区域，而是预先划分屏幕区域。接口中也确实体现了这一点。
|
||||
|
||||
预定义区域包括：
|
||||
|
||||
- `VOLC_DISPLAY_OBJ_STATUS`
|
||||
- `VOLC_DISPLAY_OBJ_MAIN`
|
||||
- `VOLC_DISPLAY_OBJ_SUBTITLE`
|
||||
|
||||
当前主实现 `volc_hal_display.c` 更偏向：
|
||||
|
||||
- 通过 `gfx/mmap_assets` 渲染 EAF 动画表情
|
||||
- 叠加状态文字
|
||||
- 根据状态切换不同动画
|
||||
|
||||
这使 UI 能直接表现智能体当前情绪和状态。
|
||||
|
||||
### 11.4 按键 `volc_hal_button`
|
||||
|
||||
头文件：
|
||||
|
||||
- `platform/inc/volc_hal_button.h`
|
||||
|
||||
实现：
|
||||
|
||||
- `platform/src/espressif/volc_hal_button.c`
|
||||
|
||||
这个模块本质是对底层 `iot_button` 的适配：
|
||||
|
||||
- 统一按钮事件枚举
|
||||
- 注册回调
|
||||
- 在回调中把底层事件转成 HAL 事件
|
||||
|
||||
### 11.5 文件 `volc_hal_file`
|
||||
|
||||
头文件：
|
||||
|
||||
- `platform/inc/volc_hal_file.h`
|
||||
|
||||
实现：
|
||||
|
||||
- `platform/src/espressif/volc_hal_file.c`
|
||||
|
||||
这层很薄，基本是 ANSI C 文件接口的简单包装：
|
||||
|
||||
- `open/read/write/seek/close`
|
||||
|
||||
当前主要被 `local_logic_service` 用来播放欢迎音频。
|
||||
|
||||
## 12. `application` 与 `volc_conv_ai` 的关系
|
||||
|
||||
这个目录与上层云端对话 SDK 的连接点，主要就在 `service/conv_service/src/volc_conv.c`。
|
||||
|
||||
对接方式是：
|
||||
|
||||
1. 用 `volc_create()` 创建对话引擎
|
||||
2. 用 `volc_start()` 启动 RTC 模式对话
|
||||
3. 采集到的音频通过 `volc_send_audio_data()` 上送
|
||||
4. 收到的音频通过 `volc_hal_player_play_data()` 播放
|
||||
5. 收到的消息按 `subv/tool/info` 分类再转给其他 service
|
||||
6. 本地业务通过：
|
||||
- `volc_send_message()`
|
||||
- `volc_send_text_to_agent()`
|
||||
继续和智能体交互
|
||||
|
||||
所以可以把 `application` 看成：
|
||||
|
||||
- 向下对接 `volc_conv_ai`
|
||||
- 向上对接硬件和本地业务
|
||||
|
||||
是一个“设备侧 AI 对话编排层”。
|
||||
|
||||
## 13. 关键运行链路
|
||||
|
||||
从实际运行角度，主链路可以总结为：
|
||||
|
||||
### 13.1 待机态
|
||||
|
||||
1. HAL 初始化
|
||||
2. 创建显示层
|
||||
3. 启动 AIOS
|
||||
4. 初始化各个 service
|
||||
5. 启动 wakeup 模式音频采集
|
||||
6. 屏幕提示“请说 hi 乐鑫,启动ai对话”
|
||||
|
||||
### 13.2 唤醒后进入对话
|
||||
|
||||
1. 唤醒回调发布 `VOLC_SERVICE_AI_CONVERSATION`
|
||||
2. `volc_manager_service` 转发为 `VOLC_SERVICE_AI_CONVERSATION_START`
|
||||
3. `conv_service_manager` 创建 `conv_ai_service_task`
|
||||
4. 采集模块切换到 `CAPTURE` 模式
|
||||
5. 启动播放器
|
||||
6. 调用 `volc_start()` 建立云端对话
|
||||
|
||||
### 13.3 对话中
|
||||
|
||||
1. 本地采集音频并上传
|
||||
2. 智能体返回音频并播放
|
||||
3. 状态回调刷新 UI
|
||||
4. 字幕消息更新显示
|
||||
5. 字幕触发本地逻辑
|
||||
6. 工具调用消息触发 function call 服务
|
||||
|
||||
### 13.4 对话结束
|
||||
|
||||
1. 超时、退出指令或其他条件触发结束
|
||||
2. `volc_stop()`
|
||||
3. 停止采集/播放
|
||||
4. 恢复 wakeup 模式采集
|
||||
5. UI 回到待机提示
|
||||
|
||||
## 14. 目录级总结
|
||||
|
||||
### 14.1 `framework/`
|
||||
|
||||
是一个轻量事件驱动状态机框架，负责 service 之间的解耦通信。
|
||||
|
||||
### 14.2 `platform/`
|
||||
|
||||
是面向 AI 对话设备场景定制的 HAL，不追求绝对通用，但强调对业务简单易用。
|
||||
|
||||
### 14.3 `service/`
|
||||
|
||||
是具体业务编排层：
|
||||
|
||||
- `volc_manager_service`
|
||||
- 业务入口与事件分发
|
||||
- `conv_service`
|
||||
- AI 对话主流程
|
||||
- `function_call_service`
|
||||
- 工具调用桥接
|
||||
- `local_logic_service`
|
||||
- 本地规则逻辑
|
||||
|
||||
### 14.4 `cmake/`
|
||||
|
||||
负责把上述模块连成一个 ESP-IDF component，并依赖底层 `volc_conv_ai` 组件。
|
||||
|
||||
## 15. 一句话总结
|
||||
|
||||
`application` 目录本质上是一套“设备侧 AI 对话应用框架”：
|
||||
|
||||
- `framework` 解决模块通信
|
||||
- `platform` 解决硬件适配
|
||||
- `service` 解决业务编排
|
||||
- `conv_service` 作为核心中枢把 `volc_conv_ai`、音频设备、显示设备、本地逻辑和 function call 串起来
|
||||
|
||||
如果后续继续深入阅读，推荐顺序如下：
|
||||
|
||||
1. `service/inc/volc_service_common.h`
|
||||
2. `framework/src/aios.c`
|
||||
3. `service/src/volc_manager_service.c`
|
||||
4. `service/conv_service/src/volc_conv_service.c`
|
||||
5. `service/conv_service/src/volc_conv.c`
|
||||
6. `service/function_call_service/src/volc_function_call_service.c`
|
||||
7. `service/local_logic_service/src/local_logic_service.c`
|
||||
8. `platform/src/espressif/volc_hal.c`
|
||||
9. `platform/src/espressif/volc_hal_capture.c`
|
||||
10. `platform/src/espressif/volc_hal_player.c`
|
||||
11. `platform/src/espressif/volc_hal_display.c`
|
||||
@@ -0,0 +1,25 @@
|
||||
set(VOLC_FRAMEWORK_SRCS "${CMAKE_CURRENT_LIST_DIR}/../framework/src/aios.c"
|
||||
CACHE INTERNAL "ConversationalAI-Embedded-Kit-2.0 framework src file")
|
||||
|
||||
set(VOLC_FRAMEWORK_INCS "${CMAKE_CURRENT_LIST_DIR}/../framework/inc"
|
||||
CACHE INTERNAL "ConversationalAI-Embedded-Kit-2.0 framework include dir")
|
||||
|
||||
set(VOLC_PLATFORM_INCS "${CMAKE_CURRENT_LIST_DIR}/../platform/inc"
|
||||
CACHE INTERNAL "ConversationalAI-Embedded-Kit-2.0 platform include dir")
|
||||
|
||||
set(VOLC_SERVICE_INCS "${CMAKE_CURRENT_LIST_DIR}/../service/inc"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/inc"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/inc"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/local_logic_service/inc"
|
||||
CACHE INTERNAL "ConversationalAI-Embedded-Kit-2.0 service include dir")
|
||||
|
||||
set(VOLC_SERVICE_SRCS "${CMAKE_CURRENT_LIST_DIR}/../service/src/volc_manager_service.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/volc_conv_service.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/volc_conv.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/src/volc_function_call_service.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/src/volc_local_function_list.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/local_logic_service/src/local_logic_service.c"
|
||||
CACHE INTERNAL "ConversationalAI-Embedded-Kit-2.0 service src file")
|
||||
|
||||
set(VOLC_UTIL_INCS "${CMAKE_CURRENT_LIST_DIR}/../utils"
|
||||
CACHE INTERNAL "ConversationalAI-Embedded-Kit-2.0 util include dir")
|
||||
@@ -0,0 +1,55 @@
|
||||
# Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# ConversationalAI SDK ESP-IDF component configuration
|
||||
# Converted from standard CMake to ESP-IDF component format
|
||||
|
||||
# Source files
|
||||
set(VOLC_ESP_SRCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal_button.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal_capture.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal_display.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal_file.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal_led.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal_player.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/basic_board.c"
|
||||
)
|
||||
set(VOLC_ESP_DISPLAY_SRCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/echoear_font.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/img_app_aiPhone.c"
|
||||
)
|
||||
set(VOLC_APP_SRCS ${VOLC_FRAMEWORK_SRCS}
|
||||
${VOLC_SERVICE_SRCS}
|
||||
${VOLC_ESP_SRCS}
|
||||
${VOLC_ESP_DISPLAY_SRCS}
|
||||
)
|
||||
|
||||
set(VOLC_LVGL_SOURCE_INCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/")
|
||||
set(VOLC_ASSETS_SOURCE_INCS "${CMAKE_CURRENT_LIST_DIR}/../../../main/")
|
||||
|
||||
|
||||
set(VOLC_APP_INCS ${VOLC_FRAMEWORK_INCS}
|
||||
${VOLC_PLATFORM_INCS}
|
||||
${VOLC_SERVICE_INCS}
|
||||
${VOLC_UTIL_INCS}
|
||||
${VOLC_LVGL_SOURCE_INCS}
|
||||
${VOLC_ASSETS_SOURCE_INCS}
|
||||
)
|
||||
|
||||
idf_component_register(
|
||||
SRCS ${VOLC_APP_SRCS}
|
||||
INCLUDE_DIRS ${VOLC_APP_INCS}
|
||||
REQUIRES volc_conv_ai
|
||||
PRIV_REQUIRES lvgl button av_processor esp_board_manager hw_init esp_mmap_assets esp_emote_gfx esp_vocat_base_control
|
||||
)
|
||||
|
||||
# Compiler flags
|
||||
target_compile_options(${COMPONENT_LIB} PRIVATE
|
||||
-w
|
||||
-Os
|
||||
-ffunction-sections
|
||||
-fdata-sections
|
||||
)
|
||||
|
||||
# Export the library for use by other components
|
||||
set(VOLC_CONV_APP_LIBRARIES ${COMPONENT_LIB} PARENT_SCOPE)
|
||||
set(VOLC_CONV_APP_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/../framework/inc PARENT_SCOPE)
|
||||
@@ -0,0 +1,6 @@
|
||||
# Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
# SPDX-License-Identifier: Apache License 2.0
|
||||
|
||||
|
||||
set(FRAMEWORK_SRCS ${CMAKE_CURRENT_LIST_DIR}/src/aios.c)
|
||||
set(FRAMEWORK_INC ${CMAKE_CURRENT_LIST_DIR}/inc)
|
||||
@@ -0,0 +1,5 @@
|
||||
# framework
|
||||
|
||||
## brief introduction
|
||||
|
||||
This directory is designed to facilitate communication between different business modules (functional modules). The main communication method is subscribing to (sub) and publishing (pub) events, and the externally provided interface files are located in the `inc` directory.
|
||||
@@ -0,0 +1,5 @@
|
||||
# framework
|
||||
|
||||
## 目录简介
|
||||
|
||||
本目录是为了方便不用业务模块（功能模块）相互间进行通信使用的，主要的方式是sub和pub event事件，对外提供的接口文件在inc。
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef AIOS_H_
|
||||
#define AIOS_H_
|
||||
|
||||
/* include ------------------------------------------------------------------ */
|
||||
#include "aios_config.h"
|
||||
#include "volc_list.h"
|
||||
#include <unistd.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* default configuration ---------------------------------------------------- */
|
||||
|
||||
#ifndef AIOS_MAX_ACTORS
|
||||
#define AIOS_MAX_ACTORS 8 // 默认最多8个Actor
|
||||
#endif
|
||||
|
||||
#ifndef AIOS_USE_ASSERT
|
||||
#define AIOS_USE_ASSERT 1 // 默认打开断言
|
||||
#endif
|
||||
|
||||
|
||||
#include "aios_def.h"
|
||||
|
||||
/* data struct -------------------------------------------------------------- */
|
||||
|
||||
// 状态返回值的定义
|
||||
typedef enum aios_ret {
|
||||
AIOS_Ret_Null = 0, // 无效值
|
||||
AIOS_Ret_Handled, // 已处理，不产生跳转
|
||||
AIOS_Ret_NotHandled, // 未处理，不产生跳转
|
||||
AIOS_Ret_Tran, // 跳转
|
||||
} aios_ret_t;
|
||||
|
||||
// 事件类
|
||||
struct aios_event;
|
||||
typedef void (*aios_event_free_handler)(const void *data);
|
||||
typedef struct aios_event {
|
||||
volc_list_head_t node;
|
||||
uint32_t sub; // 订阅者
|
||||
aios_event_id_t id; // 事件主题
|
||||
void *data; // 事件数据
|
||||
aios_event_free_handler free_handler; // 事件数据释放函数
|
||||
} aios_event_t;
|
||||
|
||||
// 数据结构 - 行为树相关 --------------------------------------------------------
|
||||
// 状态函数句柄的定义
|
||||
struct aios_session;
|
||||
typedef aios_ret_t (* aios_state_handler)(struct aios_session *const me, aios_event_t const * const e);
|
||||
|
||||
// 状态机类
|
||||
typedef struct aios_session {
|
||||
uint32_t priority : 5; // 优先级，0-31，数值越大优先级越高
|
||||
uint32_t enabled : 1; // 是否使能
|
||||
uint32_t reserve : 1; // 保留位
|
||||
volatile aios_state_handler state;
|
||||
} aios_session_t;
|
||||
|
||||
// api -------------------------------------------------------------------------
|
||||
// 对框架进行初始化，在各状态机初始化之前调用。
|
||||
void aios_init(int event_size);
|
||||
// 启动框架，放在main函数的末尾。
|
||||
void aios_run(void);
|
||||
// 停止框架的运行（不常用）
|
||||
// 停止框架后，框架会在执行完当前状态机的当前事件后，清空各状态机事件队列，清空事件池，
|
||||
// 不再执行任何功能，直至框架被再次启动。
|
||||
void aios_stop(void);
|
||||
|
||||
// 关于状态机 -----------------------------------------------
|
||||
// 状态机初始化函数
|
||||
void aios_session_init( aios_session_t * const me,
|
||||
uint8_t priority,
|
||||
void const * const parameter);
|
||||
void aios_session_start(aios_session_t * const me, aios_state_handler state_init);
|
||||
|
||||
aios_ret_t aios_tran(aios_session_t * const me, aios_state_handler state);
|
||||
|
||||
#define AIOS_TRAN(target) aios_tran((aios_session_t * )me, (aios_state_handler)target)
|
||||
#define AIOS_STATE_CAST(state) ((aios_state_handler)(state))
|
||||
|
||||
// 关于事件 -------------------------------------------------
|
||||
// 事件订阅
|
||||
void aios_event_sub(aios_session_t * const me, aios_event_id_t id);
|
||||
// 事件取消订阅
|
||||
void aios_event_unsub(aios_session_t * const me, aios_event_id_t id);
|
||||
// 事件订阅宏定义
|
||||
#define AIOS_EVENT_SUB(_evt) aios_event_sub(&(me->super), _evt)
|
||||
// 事件取消订阅宏定义
|
||||
#define AIOS_EVENT_UNSUB(_evt) aios_event_unsub(&(me->super), _evt)
|
||||
|
||||
// 发布事件（携带数据）
|
||||
int aios_event_pub(aios_event_id_t id, void *data, aios_event_free_handler free_handler);
|
||||
|
||||
// 发布周期性或延时事件
|
||||
void* aios_event_time_pub(aios_event_id_t id, uint32_t timeout_ms, uint32_t period_ms);
|
||||
|
||||
// 取消延时事件或者周期事件的发布
|
||||
void aios_event_time_cancel(void** timer);
|
||||
|
||||
/* port --------------------------------------------------------------------- */
|
||||
// void* aios_malloc(uint32_t size);
|
||||
// void aios_free(void* ptr);
|
||||
// void* aios_mutex_create(void);
|
||||
// void aios_mutex_lock(void* mutex);
|
||||
// void aios_mutex_unlock(void* mutex);
|
||||
// void aios_mutex_destroy(void** mutex);
|
||||
void aios_port_assert(uint32_t error_id);
|
||||
|
||||
/* hook --------------------------------------------------------------------- */
|
||||
|
||||
// 结束AIOS Nano的运行的时候，所调用的回调函数。
|
||||
void aios_hook_stop(void);
|
||||
|
||||
// 启动AIOS Nano的时候，所调用的回调函数
|
||||
void aios_hook_start(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef AIOS_CONFIG_H__
|
||||
#define AIOS_CONFIG_H__
|
||||
|
||||
/* AIOS General Configuration --------------------------------------- */
|
||||
#define AIOS_MCU_TYPE 32
|
||||
#define AIOS_MAX_ACTORS 4
|
||||
#define AIOS_TEST_PLATFORM 64
|
||||
#define AIOS_USE_MAGIC 0
|
||||
|
||||
/* Assert Configuration ----------------------------------------------------- */
|
||||
#define AIOS_USE_ASSERT 1
|
||||
#endif
|
||||
@@ -0,0 +1,21 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef AIOS_DEF_H__
|
||||
#define AIOS_DEF_H__
|
||||
|
||||
#include "aios_config.h"
|
||||
|
||||
#define AIOS_MAX_SLEEP_MS (1000 * 10)
|
||||
|
||||
typedef enum aios_event_id {
|
||||
// 系统事件
|
||||
Event_Null = 0,
|
||||
Event_Enter,
|
||||
Event_Exit,
|
||||
// 用户事件
|
||||
Event_User,
|
||||
} aios_event_id_t;
|
||||
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
/* Copyright (c) 2013, Ben Noordhuis <info@bnoordhuis.nl>
|
||||
*
|
||||
* Permission to use, copy, modify, and/or distribute this software for any
|
||||
* purpose with or without fee is hereby granted, provided that the above
|
||||
* copyright notice and this permission notice appear in all copies.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef UV_SRC_HEAP_H_
|
||||
#define UV_SRC_HEAP_H_
|
||||
|
||||
#include <stddef.h> /* NULL */
|
||||
|
||||
#if defined(__GNUC__)
|
||||
# define HEAP_EXPORT(declaration) __attribute__((unused)) static declaration
|
||||
#else
|
||||
# define HEAP_EXPORT(declaration) static declaration
|
||||
#endif
|
||||
|
||||
struct heap_node {
|
||||
struct heap_node* left;
|
||||
struct heap_node* right;
|
||||
struct heap_node* parent;
|
||||
};
|
||||
|
||||
/* A binary min heap. The usual properties hold: the root is the lowest
|
||||
* element in the set, the height of the tree is at most log2(nodes) and
|
||||
* it's always a complete binary tree.
|
||||
*
|
||||
* The heap function try hard to detect corrupted tree nodes at the cost
|
||||
* of a minor reduction in performance. Compile with -DNDEBUG to disable.
|
||||
*/
|
||||
struct heap {
|
||||
struct heap_node* min;
|
||||
unsigned int nelts;
|
||||
};
|
||||
|
||||
/* Return non-zero if a < b. */
|
||||
typedef int (*heap_compare_fn)(const struct heap_node* a,
|
||||
const struct heap_node* b);
|
||||
|
||||
/* Public functions. */
|
||||
HEAP_EXPORT(void heap_init(struct heap* heap));
|
||||
HEAP_EXPORT(struct heap_node* heap_min(const struct heap* heap));
|
||||
HEAP_EXPORT(void heap_insert(struct heap* heap,
|
||||
struct heap_node* newnode,
|
||||
heap_compare_fn less_than));
|
||||
HEAP_EXPORT(void heap_remove(struct heap* heap,
|
||||
struct heap_node* node,
|
||||
heap_compare_fn less_than));
|
||||
HEAP_EXPORT(void heap_dequeue(struct heap* heap, heap_compare_fn less_than));
|
||||
|
||||
/* Implementation follows. */
|
||||
|
||||
HEAP_EXPORT(void heap_init(struct heap* heap)) {
|
||||
heap->min = NULL;
|
||||
heap->nelts = 0;
|
||||
}
|
||||
|
||||
HEAP_EXPORT(void heap_node_init(struct heap_node* hnode)) {
|
||||
hnode->left = 0;
|
||||
hnode->parent = 0;
|
||||
hnode->right = 0;
|
||||
}
|
||||
|
||||
HEAP_EXPORT(int heap_node_is_empty(struct heap_node* hnode)) {
|
||||
return (hnode->left == 0 && hnode->parent == 0 && hnode->right == 0);
|
||||
}
|
||||
|
||||
HEAP_EXPORT(struct heap_node* heap_min(const struct heap* heap)) {
|
||||
return heap->min;
|
||||
}
|
||||
|
||||
/* Swap parent with child. Child moves closer to the root, parent moves away. */
|
||||
static void heap_node_swap(struct heap* heap,
|
||||
struct heap_node* parent,
|
||||
struct heap_node* child) {
|
||||
struct heap_node* sibling;
|
||||
struct heap_node t;
|
||||
|
||||
t = *parent;
|
||||
*parent = *child;
|
||||
*child = t;
|
||||
|
||||
parent->parent = child;
|
||||
if (child->left == child) {
|
||||
child->left = parent;
|
||||
sibling = child->right;
|
||||
} else {
|
||||
child->right = parent;
|
||||
sibling = child->left;
|
||||
}
|
||||
if (sibling != NULL)
|
||||
sibling->parent = child;
|
||||
|
||||
if (parent->left != NULL)
|
||||
parent->left->parent = parent;
|
||||
if (parent->right != NULL)
|
||||
parent->right->parent = parent;
|
||||
|
||||
if (child->parent == NULL)
|
||||
heap->min = child;
|
||||
else if (child->parent->left == parent)
|
||||
child->parent->left = child;
|
||||
else
|
||||
child->parent->right = child;
|
||||
}
|
||||
|
||||
HEAP_EXPORT(void heap_insert(struct heap* heap,
|
||||
struct heap_node* newnode,
|
||||
heap_compare_fn less_than)) {
|
||||
struct heap_node** parent;
|
||||
struct heap_node** child;
|
||||
unsigned int path;
|
||||
unsigned int n;
|
||||
unsigned int k;
|
||||
|
||||
newnode->left = NULL;
|
||||
newnode->right = NULL;
|
||||
newnode->parent = NULL;
|
||||
|
||||
/* Calculate the path from the root to the insertion point. This is a min
|
||||
* heap so we always insert at the left-most free node of the bottom row.
|
||||
*/
|
||||
path = 0;
|
||||
for (k = 0, n = 1 + heap->nelts; n >= 2; k += 1, n /= 2)
|
||||
path = (path << 1) | (n & 1);
|
||||
|
||||
/* Now traverse the heap using the path we calculated in the previous step. */
|
||||
parent = child = &heap->min;
|
||||
while (k > 0) {
|
||||
parent = child;
|
||||
if (path & 1)
|
||||
child = &(*child)->right;
|
||||
else
|
||||
child = &(*child)->left;
|
||||
path >>= 1;
|
||||
k -= 1;
|
||||
}
|
||||
|
||||
/* Insert the new node. */
|
||||
newnode->parent = *parent;
|
||||
*child = newnode;
|
||||
heap->nelts += 1;
|
||||
|
||||
/* Walk up the tree and check at each node if the heap property holds.
|
||||
* It's a min heap so parent < child must be true.
|
||||
*/
|
||||
while (newnode->parent != NULL && less_than(newnode, newnode->parent))
|
||||
heap_node_swap(heap, newnode->parent, newnode);
|
||||
}
|
||||
|
||||
HEAP_EXPORT(void heap_remove(struct heap* heap,
|
||||
struct heap_node* node,
|
||||
heap_compare_fn less_than)) {
|
||||
struct heap_node* smallest;
|
||||
struct heap_node** max;
|
||||
struct heap_node* child;
|
||||
unsigned int path;
|
||||
unsigned int k;
|
||||
unsigned int n;
|
||||
|
||||
if (heap->nelts == 0)
|
||||
return;
|
||||
|
||||
/* Calculate the path from the min (the root) to the max, the left-most node
|
||||
* of the bottom row.
|
||||
*/
|
||||
path = 0;
|
||||
for (k = 0, n = heap->nelts; n >= 2; k += 1, n /= 2)
|
||||
path = (path << 1) | (n & 1);
|
||||
|
||||
/* Now traverse the heap using the path we calculated in the previous step. */
|
||||
max = &heap->min;
|
||||
while (k > 0) {
|
||||
if (path & 1)
|
||||
max = &(*max)->right;
|
||||
else
|
||||
max = &(*max)->left;
|
||||
path >>= 1;
|
||||
k -= 1;
|
||||
}
|
||||
|
||||
heap->nelts -= 1;
|
||||
|
||||
/* Unlink the max node. */
|
||||
child = *max;
|
||||
*max = NULL;
|
||||
|
||||
if (child == node) {
|
||||
/* We're removing either the max or the last node in the tree. */
|
||||
if (child == heap->min) {
|
||||
heap->min = NULL;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/* Replace the to be deleted node with the max node. */
|
||||
child->left = node->left;
|
||||
child->right = node->right;
|
||||
child->parent = node->parent;
|
||||
|
||||
if (child->left != NULL) {
|
||||
child->left->parent = child;
|
||||
}
|
||||
|
||||
if (child->right != NULL) {
|
||||
child->right->parent = child;
|
||||
}
|
||||
|
||||
if (node->parent == NULL) {
|
||||
heap->min = child;
|
||||
} else if (node->parent->left == node) {
|
||||
node->parent->left = child;
|
||||
} else {
|
||||
node->parent->right = child;
|
||||
}
|
||||
|
||||
/* Walk down the subtree and check at each node if the heap property holds.
|
||||
* It's a min heap so parent < child must be true. If the parent is bigger,
|
||||
* swap it with the smallest child.
|
||||
*/
|
||||
for (;;) {
|
||||
smallest = child;
|
||||
if (child->left != NULL && less_than(child->left, smallest))
|
||||
smallest = child->left;
|
||||
if (child->right != NULL && less_than(child->right, smallest))
|
||||
smallest = child->right;
|
||||
if (smallest == child)
|
||||
break;
|
||||
heap_node_swap(heap, child, smallest);
|
||||
}
|
||||
|
||||
/* Walk up the subtree and check that each parent is less than the node
|
||||
* this is required, because `max` node is not guaranteed to be the
|
||||
* actual maximum in tree
|
||||
*/
|
||||
while (child->parent != NULL && less_than(child, child->parent))
|
||||
heap_node_swap(heap, child->parent, child);
|
||||
}
|
||||
|
||||
HEAP_EXPORT(void heap_dequeue(struct heap* heap, heap_compare_fn less_than)) {
|
||||
heap_remove(heap, heap->min, less_than);
|
||||
}
|
||||
|
||||
#undef HEAP_EXPORT
|
||||
|
||||
#endif /* UV_SRC_HEAP_H_ */
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef __CONV_AI_SRC_UTIL_VOLC_LIST_H__
|
||||
#define __CONV_AI_SRC_UTIL_VOLC_LIST_H__
|
||||
|
||||
// #include <stdio.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
typedef struct volc_list_head_t
|
||||
{
|
||||
struct volc_list_head_t *next, *prev;
|
||||
} volc_list_head_t;
|
||||
|
||||
#define __LIST_ADD(entry, before, after) \
|
||||
{ \
|
||||
volc_list_head_t *new_ = (entry), *prev = (before), *next = (after); \
|
||||
(next)->prev = (new_); \
|
||||
(new_)->next = (next); \
|
||||
(new_)->prev = (prev); \
|
||||
(prev)->next = (new_); \
|
||||
}
|
||||
#define volc_list_init(entry) \
|
||||
do \
|
||||
{ \
|
||||
(entry)->next = (entry); \
|
||||
(entry)->prev = (entry); \
|
||||
} while (0)
|
||||
#define volc_list_add(entry, base) \
|
||||
do \
|
||||
{ \
|
||||
__LIST_ADD((entry), (base), (base)->next); \
|
||||
} while (0)
|
||||
#define volc_list_add_after(entry, base) \
|
||||
do \
|
||||
{ \
|
||||
__LIST_ADD((entry), (base), (base)->next); \
|
||||
} while (0)
|
||||
#define volc_list_add_before(entry, base) \
|
||||
do \
|
||||
{ \
|
||||
__LIST_ADD((entry), (base)->prev, (base)); \
|
||||
} while (0)
|
||||
#define volc_list_add_head(entry, head) volc_list_add_after(entry, head)
|
||||
#define volc_list_add_tail(entry, head) volc_list_add_before(entry, head)
|
||||
#define volc_list_del(entry) \
|
||||
do \
|
||||
{ \
|
||||
(entry)->prev->next = (entry)->next; \
|
||||
(entry)->next->prev = (entry)->prev; \
|
||||
(entry)->next = (entry)->prev = (entry); \
|
||||
} while (0)
|
||||
#define volc_list_empty(head) ((head)->next == (head))
|
||||
#define volc_list_get_head(head) (volc_list_empty(head) ? (volc_list_head_t *)NULL : (head)->next)
|
||||
#define volc_list_get_tail(head) (volc_list_empty(head) ? (volc_list_head_t *)NULL : (head)->prev)
|
||||
#define volc_list_is_head(entry, head) ((entry)->prev == head)
|
||||
#define volc_list_is_tail(entry, head) ((entry)->next == head)
|
||||
#define volc_list_entry(ptr, type, member) ((type *)((char *)(ptr) - (unsigned long)(&((type *)0)->member)))
|
||||
#define volc_list_for_each(h, head) for (h = (head)->next; h != (head); h = h->next)
|
||||
#define volc_list_for_each_safe(h, n, head) for (h = (head)->next, n = h->next; h != (head); h = n, n = h->next)
|
||||
#define volc_list_for_each_entry(p, head, type, member) \
|
||||
for (p = volc_list_entry((head)->next, type, member); &p->member != (head); \
|
||||
p = volc_list_entry(p->member.next, type, member))
|
||||
#define volc_list_for_each_entry_safe(p, t, head, type, member) \
|
||||
for (p = volc_list_entry((head)->next, type, member), t = volc_list_entry(p->member.next, type, member); \
|
||||
&p->member != (head); p = t, t = volc_list_entry(t->member.next, type, member))
|
||||
#define volc_list_get_head_entry(head, type, member) \
|
||||
(volc_list_empty(head) ? (type *)NULL : volc_list_entry(((head)->next), type, member))
|
||||
#define volc_list_get_tail_entry(head, type, member) \
|
||||
(volc_list_empty(head) ? (type *)NULL : volc_list_entry(((head)->prev), type, member))
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif /* #define __CONV_AI_SRC_UTIL_VOLC_LIST_H__ */
|
||||
@@ -0,0 +1,646 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// include ---------------------------------------------------------------------
|
||||
#include "../inc/aios.h"
|
||||
#include <string.h>
|
||||
#include <stdbool.h>
|
||||
#include "heap-inl.h"
|
||||
|
||||
//socket
|
||||
#include <sys/socket.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <netinet/in.h>
|
||||
// time
|
||||
#include <sys/time.h>
|
||||
#include "volc_osal.h"
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* assert ------------------------------------------------------------------- */
|
||||
#if (AIOS_USE_ASSERT != 0)
|
||||
#define AIOS_ASSERT(test_) do { if (!(test_)) { \
|
||||
aios_hook_stop(); \
|
||||
aios_port_assert(__LINE__); \
|
||||
} } while (0)
|
||||
#else
|
||||
#define AIOS_ASSERT(test_) ((void)0)
|
||||
#endif
|
||||
|
||||
// aios define ------------------------------------------------------------------
|
||||
|
||||
// **aios** ---------------------------------------------------------------------
|
||||
enum {
|
||||
EosRun_OK = 0,
|
||||
EosRun_NotEnabled,
|
||||
EosRun_NoEvent,
|
||||
EosRun_NoActor,
|
||||
EosRun_NoActorSub,
|
||||
|
||||
// Timer
|
||||
EosTimer_Empty,
|
||||
EosTimer_NotTimeout,
|
||||
EosTimer_ChangeToEmpty,
|
||||
|
||||
EosRunErr_NotInitEnd = -1,
|
||||
EosRunErr_ActorNotSub = -2,
|
||||
EosRunErr_MallocFail = -3,
|
||||
EosRunErr_SubTableNull = -4,
|
||||
EosRunErr_InvalidEventData = -5,
|
||||
EosRunErr_HeapMemoryNotEnough = -6,
|
||||
EosRunErr_TimerRepeated = -7,
|
||||
};
|
||||
|
||||
typedef struct timer_item {
|
||||
struct heap_node node;
|
||||
aios_event_id_t id;
|
||||
int64_t timeout_ms;
|
||||
uint32_t period_ms;
|
||||
} aios_timer_t;
|
||||
|
||||
typedef struct aios_tag {
|
||||
uint32_t *sub_table; // event sub table
|
||||
|
||||
uint32_t session_exist;
|
||||
uint32_t session_enabled;
|
||||
aios_session_t * sessions[AIOS_MAX_ACTORS];
|
||||
|
||||
uint32_t sub_general;
|
||||
void* event_mutex;
|
||||
volc_list_head_t event_list;
|
||||
|
||||
uint32_t time;
|
||||
void* timer_mutex;
|
||||
struct heap timer_heap;
|
||||
int fd[2];
|
||||
|
||||
uint8_t enabled : 1;
|
||||
uint8_t running : 1;
|
||||
uint8_t init_end : 1;
|
||||
uint8_t reserved : 5;
|
||||
} aios_t;
|
||||
|
||||
static aios_t aios;
|
||||
|
||||
// data ------------------------------------------------------------------------
|
||||
static const aios_event_t aios_event_table[Event_User] = {
|
||||
{{ 0 }, 0, Event_Null, 0, NULL},
|
||||
{{ 0 }, 0, Event_Enter, 0, NULL},
|
||||
{{ 0 }, 0, Event_Exit, 0, NULL},
|
||||
};
|
||||
|
||||
// macro -----------------------------------------------------------------------
|
||||
#define HSM_TRIG_(state_, topic_) \
|
||||
((*(state_))(me, &aios_event_table[topic_]))
|
||||
|
||||
// static function -------------------------------------------------------------
|
||||
static void aios_session_dispath(aios_session_t * const me, aios_event_t const * const e);
|
||||
|
||||
#ifndef VOLC_OFFSET
|
||||
#define VOLC_OFFSET(type, member) ((size_t) &((type *)0)->member)
|
||||
#endif
|
||||
|
||||
#ifndef VOLC_CONTAINER_OF
|
||||
#define VOLC_CONTAINER_OF(ptr, type, member) \
|
||||
((type *)((char *)(ptr) - VOLC_OFFSET(type, member)))
|
||||
#endif
|
||||
|
||||
static int __heap_node_less(const struct heap_node* a, const struct heap_node* b) {
|
||||
aios_timer_t* item_a = VOLC_CONTAINER_OF(a, aios_timer_t, node);
|
||||
aios_timer_t* item_b = VOLC_CONTAINER_OF(b, aios_timer_t, node);
|
||||
if (item_a->timeout_ms < item_b->timeout_ms) {
|
||||
return 1;
|
||||
}
|
||||
if (item_a->timeout_ms > item_b->timeout_ms) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return item_a->id > item_b->id;
|
||||
}
|
||||
|
||||
static int __set_block_mode(int sockfd, int block) {
|
||||
int flags = fcntl(sockfd, F_GETFL);
|
||||
if (flags < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (block) {
|
||||
if ((flags & O_NONBLOCK) != 0) {
|
||||
flags ^= O_NONBLOCK;
|
||||
}
|
||||
} else {
|
||||
flags |= O_NONBLOCK;
|
||||
}
|
||||
if (fcntl(sockfd, F_SETFL, flags) < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __pipe(int fds[2]) {
|
||||
int ret = 0;
|
||||
fds[0] = -1;
|
||||
fds[1] = -1;
|
||||
int err = 0,opt = 0,write_fd = -1,read_fd = -1;
|
||||
read_fd = socket(AF_INET,SOCK_DGRAM,IPPROTO_UDP);
|
||||
if (read_fd < 0) {
|
||||
ret = -1;
|
||||
goto err_out_label;
|
||||
}
|
||||
|
||||
struct sockaddr_in servaddr;
|
||||
uint32_t addr_len = sizeof(servaddr);
|
||||
memset(&servaddr, 0, sizeof(servaddr));
|
||||
servaddr.sin_family = AF_INET;
|
||||
servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
servaddr.sin_port = 0;
|
||||
|
||||
if (bind(read_fd, (struct sockaddr*)&servaddr, sizeof(servaddr)) < 0) {
|
||||
ret = -1;
|
||||
goto err_out_label;
|
||||
}
|
||||
|
||||
// get port bind by system
|
||||
if (getsockname(read_fd, (struct sockaddr*)&servaddr, &addr_len) < 0) {
|
||||
ret = -1;
|
||||
goto err_out_label;
|
||||
}
|
||||
|
||||
if(servaddr.sin_addr.s_addr == htonl(INADDR_ANY))
|
||||
servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
|
||||
write_fd = socket(AF_INET,SOCK_DGRAM,IPPROTO_UDP);
|
||||
int conn_ret = connect(write_fd, (struct sockaddr*)&servaddr, addr_len);
|
||||
|
||||
__set_block_mode(write_fd, 0);
|
||||
__set_block_mode(read_fd, 0);
|
||||
|
||||
fds[0] = read_fd;
|
||||
fds[1] = write_fd;
|
||||
err_out_label:
|
||||
if(ret < 0) {
|
||||
if (read_fd > 0) {
|
||||
close(read_fd);
|
||||
}
|
||||
if (write_fd > 0) {
|
||||
close(write_fd);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int __interruptable_init(void) {
|
||||
if (__pipe(aios.fd) < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __interruptable_deinit(void) {
|
||||
close(aios.fd[0]);
|
||||
close(aios.fd[1]);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __interruptable_sleep(int64_t sleep_ms) {
|
||||
int ret = 0;
|
||||
char buf[64] = { 0 };
|
||||
fd_set readfds = { 0 };
|
||||
struct timeval tv = { 0 };
|
||||
if (sleep_ms < 0) {
|
||||
sleep_ms = AIOS_MAX_SLEEP_MS;
|
||||
}
|
||||
tv.tv_sec = sleep_ms / 1000;
|
||||
tv.tv_usec = (sleep_ms % 1000) * 1000;
|
||||
FD_ZERO(&readfds);
|
||||
FD_SET(aios.fd[0], &readfds);
|
||||
ret = select(aios.fd[0] + 1, &readfds, NULL, NULL, &tv);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (FD_ISSET(aios.fd[0], &readfds)) {
|
||||
if ((ret = read(aios.fd[0], buf, sizeof(buf))) < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __interruptable_break(void) {
|
||||
char c[1] = { 0x5A};
|
||||
return (write(aios.fd[1], c, sizeof(c)) ? 0 : -1);
|
||||
}
|
||||
|
||||
static int64_t aios_get_time(void)
|
||||
{
|
||||
struct timeval time_crt;
|
||||
gettimeofday(&time_crt, (void *)0);
|
||||
int64_t time_crt_ms = time_crt.tv_sec * 1000 + time_crt.tv_usec / 1000;
|
||||
|
||||
return time_crt_ms;
|
||||
}
|
||||
|
||||
void aios_init(int event_size)
|
||||
{
|
||||
aios.enabled = true;
|
||||
aios.running = false;
|
||||
aios.session_exist = 0;
|
||||
aios.session_enabled = 0;
|
||||
aios.sub_table = NULL;
|
||||
|
||||
heap_init(&aios.timer_heap);
|
||||
if (__interruptable_init() < 0) {
|
||||
printf("aios_init: __interruptable_init fail\n");
|
||||
return;
|
||||
}
|
||||
|
||||
aios.sub_general = 0;
|
||||
volc_list_init(&aios.event_list);
|
||||
aios.event_mutex = volc_osal_mutex_create();
|
||||
if (NULL == aios.event_mutex) {
|
||||
printf("aios_init: event_mutex create fail\n");
|
||||
return;
|
||||
}
|
||||
// printf("event: %p\n", aios.event_mutex);
|
||||
|
||||
aios.init_end = 1;
|
||||
aios.time = 0;
|
||||
aios.timer_mutex = volc_osal_mutex_create();
|
||||
if (NULL == aios.timer_mutex) {
|
||||
printf("aios_init: timer_mutex create fail\n");
|
||||
return;
|
||||
}
|
||||
// printf("timer: %p\n", aios.timer_mutex);
|
||||
|
||||
aios.sub_table = (uint32_t *)volc_osal_malloc(sizeof(uint32_t) * event_size);
|
||||
if (NULL == aios.sub_table) {
|
||||
printf("aios_init: sub_table create fail\n");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void aios_deinit(void)
|
||||
{
|
||||
volc_osal_mutex_destroy(&aios.event_mutex);
|
||||
volc_osal_mutex_destroy(&aios.timer_mutex);
|
||||
__interruptable_deinit();
|
||||
}
|
||||
|
||||
void aios_sub_init(uint32_t *flag_sub, aios_event_id_t topic_max)
|
||||
{
|
||||
aios.sub_table = flag_sub;
|
||||
for (int i = 0; i < topic_max; i ++) {
|
||||
aios.sub_table[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
int64_t aios_evttimer(void)
|
||||
{
|
||||
int64_t system_time = aios_get_time();
|
||||
volc_osal_mutex_lock(aios.timer_mutex);
|
||||
if (aios.timer_heap.nelts == 0) {
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
return -1;
|
||||
}
|
||||
aios_timer_t* item = VOLC_CONTAINER_OF(heap_min(&aios.timer_heap), aios_timer_t, node);
|
||||
if (item->timeout_ms > system_time) {
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
return item->timeout_ms - system_time;
|
||||
}
|
||||
while (item->timeout_ms <= system_time) {
|
||||
heap_remove(&aios.timer_heap, &item->node, __heap_node_less);
|
||||
aios_event_pub(item->id, NULL, NULL);
|
||||
if (item->period_ms == 0) {
|
||||
continue;
|
||||
}
|
||||
item->timeout_ms += item->period_ms;
|
||||
heap_insert(&aios.timer_heap, &item->node, __heap_node_less);
|
||||
item = VOLC_CONTAINER_OF(heap_min(&aios.timer_heap), aios_timer_t, node);
|
||||
}
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void *aios_event_get_block(uint8_t priority);
|
||||
void aios_event_gc(void *data);
|
||||
|
||||
int aios_once(void)
|
||||
{
|
||||
int64_t timeout_ms = -1;
|
||||
if (aios.init_end == 0) {
|
||||
return (int)EosRunErr_NotInitEnd;
|
||||
}
|
||||
|
||||
if (aios.sub_table == NULL) {
|
||||
return (int)EosRunErr_SubTableNull;
|
||||
}
|
||||
|
||||
if (aios.enabled == false) {
|
||||
return (int)EosRun_NotEnabled;
|
||||
}
|
||||
|
||||
// 检查是否有状态机的注册
|
||||
if (aios.session_exist == 0 || aios.session_enabled == 0) {
|
||||
return (int)EosRun_NoActor;
|
||||
}
|
||||
|
||||
timeout_ms = aios_evttimer();
|
||||
|
||||
if (volc_list_empty(&aios.event_list)) {
|
||||
__interruptable_sleep(timeout_ms);
|
||||
return (int)EosRun_NoEvent;
|
||||
}
|
||||
|
||||
// 寻找到优先级最高，且有事件需要处理的Actor
|
||||
aios_session_t *session = NULL;
|
||||
uint8_t priority = AIOS_MAX_ACTORS;
|
||||
for (int i = (int)(AIOS_MAX_ACTORS - 1); i >= 0; i --) {
|
||||
if ((aios.session_exist & (1 << i)) == 0)
|
||||
continue;
|
||||
if ((aios.sub_general & (1 << i)) == 0)
|
||||
continue;
|
||||
session = aios.sessions[i];
|
||||
priority = i;
|
||||
break;
|
||||
}
|
||||
// 如果没有找到，返回
|
||||
if (priority == AIOS_MAX_ACTORS) {
|
||||
__interruptable_sleep(AIOS_MAX_SLEEP_MS);
|
||||
return (int)EosRun_NoActorSub;
|
||||
}
|
||||
|
||||
// 寻找当前Actor的最老的事件
|
||||
volc_osal_mutex_lock(aios.event_mutex);
|
||||
aios_event_t * e = aios_event_get_block(priority);
|
||||
AIOS_ASSERT(e != NULL);
|
||||
|
||||
volc_osal_mutex_unlock(aios.event_mutex);
|
||||
|
||||
// 对事件进行执行
|
||||
if ((aios.sub_table[e->id] & (1 << session->priority)) != 0)
|
||||
{
|
||||
// 执行状态的转换
|
||||
aios_session_dispath(session, e);
|
||||
}
|
||||
else {
|
||||
__interruptable_sleep(AIOS_MAX_SLEEP_MS);
|
||||
return (int)EosRunErr_ActorNotSub;
|
||||
}
|
||||
// 销毁过期事件与其携带的参数
|
||||
volc_osal_mutex_lock(aios.event_mutex);
|
||||
aios_event_gc(e);
|
||||
volc_osal_mutex_unlock(aios.event_mutex);
|
||||
|
||||
volc_osal_mutex_lock(aios.timer_mutex);
|
||||
if (aios.timer_heap.nelts == 0) {
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
// __interruptable_sleep(AIOS_MAX_SLEEP_MS);
|
||||
} else {
|
||||
aios_timer_t* item = VOLC_CONTAINER_OF(heap_min(&aios.timer_heap), aios_timer_t, node);
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
int sleep_ms = item->timeout_ms - aios_get_time();
|
||||
__interruptable_sleep(sleep_ms);
|
||||
}
|
||||
return (int)EosRun_OK;
|
||||
}
|
||||
|
||||
void aios_run(void)
|
||||
{
|
||||
aios_hook_start();
|
||||
|
||||
AIOS_ASSERT(aios.enabled == true);
|
||||
AIOS_ASSERT(aios.sub_table != 0);
|
||||
|
||||
aios.running = true;
|
||||
|
||||
while (aios.enabled) {
|
||||
int ret = aios_once();
|
||||
AIOS_ASSERT(ret >= 0);
|
||||
|
||||
if (ret == EosRun_NotEnabled) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
void aios_hook_start(void)
|
||||
{
|
||||
}
|
||||
|
||||
void aios_hook_stop(void)
|
||||
{
|
||||
}
|
||||
|
||||
void aios_port_assert(uint32_t error_id)
|
||||
{
|
||||
printf("------------------------------------\n");
|
||||
printf("ASSERT >>> Module: EventOS Nano, ErrorId: %d.\n", (int)error_id);
|
||||
printf("------------------------------------\n");
|
||||
|
||||
while (1) {
|
||||
usleep(100000);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void aios_stop(void)
|
||||
{
|
||||
aios.enabled = false;
|
||||
// TODO: release resource
|
||||
aios_hook_stop();
|
||||
}
|
||||
|
||||
// state machine ---------------------------------------------------------------
|
||||
void aios_session_init( aios_session_t * const me,
|
||||
uint8_t priority,
|
||||
void const * const parameter)
|
||||
{
|
||||
// 框架需要先启动起来
|
||||
AIOS_ASSERT(aios.enabled == true);
|
||||
AIOS_ASSERT(aios.running == false);
|
||||
AIOS_ASSERT(aios.sub_table != NULL);
|
||||
// 参数检查
|
||||
AIOS_ASSERT(me != (aios_session_t *)0);
|
||||
AIOS_ASSERT(priority < AIOS_MAX_ACTORS);
|
||||
|
||||
// 防止二次启动
|
||||
if (me->enabled == true)
|
||||
return;
|
||||
|
||||
// 检查优先级的重复注册
|
||||
AIOS_ASSERT((aios.session_exist & (1 << priority)) == 0);
|
||||
|
||||
// 注册到框架里
|
||||
aios.session_exist |= (1 << priority);
|
||||
aios.sessions[priority] = me;
|
||||
// 状态机
|
||||
me->priority = priority;
|
||||
me->state = NULL;
|
||||
}
|
||||
|
||||
void aios_session_start(aios_session_t * const me, aios_state_handler state_init)
|
||||
{
|
||||
aios_state_handler t;
|
||||
|
||||
me->state = state_init;
|
||||
me->enabled = true;
|
||||
aios.session_enabled |= (1 << me->priority);
|
||||
|
||||
// 进入初始状态，执行TRAN动作。这也意味着，进入初始状态，必须无条件执行Tran动作。
|
||||
t = me->state;
|
||||
aios_ret_t ret = t(me, &aios_event_table[Event_Null]);
|
||||
AIOS_ASSERT(ret == AIOS_Ret_Tran);
|
||||
ret = me->state(me, &aios_event_table[Event_Enter]);
|
||||
AIOS_ASSERT(ret != AIOS_Ret_Tran);
|
||||
}
|
||||
|
||||
// event -----------------------------------------------------------------------
|
||||
int aios_event_pub(aios_event_id_t id, void *data, aios_event_free_handler free_handler)
|
||||
{
|
||||
if (aios.init_end == 0) {
|
||||
return EosRunErr_NotInitEnd;
|
||||
}
|
||||
|
||||
if (aios.sub_table == NULL) {
|
||||
return EosRunErr_SubTableNull;
|
||||
}
|
||||
|
||||
// 保证框架已经运行
|
||||
if (aios.enabled == 0) {
|
||||
return EosRun_NotEnabled;
|
||||
}
|
||||
|
||||
if (aios.session_exist == 0) {
|
||||
return EosRun_NoActor;
|
||||
}
|
||||
|
||||
// 没有状态机使能，返回
|
||||
if (aios.session_enabled == 0) {
|
||||
return EosRun_NotEnabled;
|
||||
}
|
||||
// 没有状态机订阅，返回
|
||||
if (aios.sub_table[id] == 0) {
|
||||
return EosRun_NoActorSub;
|
||||
}
|
||||
|
||||
// 申请事件空间
|
||||
aios_event_t *e = (aios_event_t *)volc_osal_malloc(sizeof(aios_event_t));
|
||||
if (e == NULL) {
|
||||
return EosRunErr_MallocFail;
|
||||
}
|
||||
e->id = id;
|
||||
e->sub = aios.sub_table[e->id];
|
||||
aios.sub_general |= e->sub;
|
||||
e->data = data;
|
||||
e->free_handler = free_handler;
|
||||
volc_osal_mutex_lock(aios.event_mutex);
|
||||
volc_list_add_tail(&e->node, &aios.event_list);
|
||||
volc_osal_mutex_unlock(aios.event_mutex);
|
||||
__interruptable_break();
|
||||
|
||||
return EosRun_OK;
|
||||
}
|
||||
|
||||
void aios_event_sub(aios_session_t * const me, aios_event_id_t id)
|
||||
{
|
||||
aios.sub_table[id] |= (1 << me->priority);
|
||||
}
|
||||
|
||||
void aios_event_unsub(aios_session_t * const me, aios_event_id_t id)
|
||||
{
|
||||
aios.sub_table[id] &= ~(1 << me->priority);
|
||||
}
|
||||
|
||||
void* aios_event_time_pub(aios_event_id_t id, uint32_t time_ms, uint32_t period_ms)
|
||||
{
|
||||
aios_timer_t* item = (aios_timer_t *)volc_osal_malloc(sizeof(aios_timer_t));
|
||||
AIOS_ASSERT(item != NULL);
|
||||
memset(item, 0, sizeof(aios_timer_t));
|
||||
item->id = id;
|
||||
item->timeout_ms = aios_get_time() + time_ms;
|
||||
item->period_ms = period_ms;
|
||||
volc_osal_mutex_lock(aios.timer_mutex);
|
||||
heap_insert(&aios.timer_heap, &item->node, __heap_node_less);
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
return (void *)item;
|
||||
}
|
||||
|
||||
void aios_event_time_cancel(void** timer)
|
||||
{
|
||||
if (timer == NULL || NULL == *timer) {
|
||||
return;
|
||||
}
|
||||
aios_timer_t* item = (aios_timer_t *)*timer;
|
||||
volc_osal_mutex_lock(aios.timer_mutex);
|
||||
heap_remove(&aios.timer_heap, &item->node, __heap_node_less);
|
||||
volc_osal_mutex_unlock(aios.timer_mutex);
|
||||
volc_osal_free(item);
|
||||
*timer = NULL;
|
||||
}
|
||||
|
||||
// state tran ------------------------------------------------------------------
|
||||
aios_ret_t aios_tran(aios_session_t * const me, aios_state_handler state)
|
||||
{
|
||||
me->state = state;
|
||||
|
||||
return AIOS_Ret_Tran;
|
||||
}
|
||||
|
||||
// static function -------------------------------------------------------------
|
||||
static void aios_session_dispath(aios_session_t * const me, aios_event_t const * const e)
|
||||
{
|
||||
aios_ret_t r;
|
||||
|
||||
AIOS_ASSERT(e != (aios_event_t *)0);
|
||||
|
||||
aios_state_handler s = me->state;
|
||||
aios_state_handler t;
|
||||
|
||||
r = s(me, e);
|
||||
if (r == AIOS_Ret_Tran) {
|
||||
t = me->state;
|
||||
r = s(me, &aios_event_table[Event_Exit]);
|
||||
AIOS_ASSERT(r == AIOS_Ret_Handled || r == AIOS_Ret_NotHandled);
|
||||
r = t(me, &aios_event_table[Event_Enter]);
|
||||
AIOS_ASSERT(r == AIOS_Ret_Handled || r == AIOS_Ret_NotHandled);
|
||||
me->state = t;
|
||||
}
|
||||
else {
|
||||
me->state = s;
|
||||
}
|
||||
}
|
||||
|
||||
/* heap library ------------------------------------------------------------- */
|
||||
void aios_event_gc(void *data)
|
||||
{
|
||||
aios_event_t * e = (aios_event_t *)data;
|
||||
if (e->sub == 0) {
|
||||
volc_list_del(&e->node);
|
||||
if (e->free_handler != NULL) {
|
||||
e->free_handler(e->data);
|
||||
}
|
||||
volc_osal_free(e);
|
||||
}
|
||||
aios.sub_general = 0;
|
||||
volc_list_for_each_entry(e, &aios.event_list, aios_event_t, node) {
|
||||
aios.sub_general |= e->sub;
|
||||
}
|
||||
}
|
||||
|
||||
void *aios_event_get_block(uint8_t priority)
|
||||
{
|
||||
aios_event_t * e = NULL;
|
||||
volc_list_for_each_entry(e, &aios.event_list, aios_event_t, node) {
|
||||
if ((e->sub & (1 << priority)) == 0) {
|
||||
continue;
|
||||
}
|
||||
e->sub &=~ (1 << priority);
|
||||
break;
|
||||
}
|
||||
|
||||
return (void *)e;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,49 @@
|
||||
# Platform
|
||||
|
||||
## brief introduction
|
||||
|
||||
This directory serves as the software abstraction layer for the Hardware Conversational AI Agent Development Kit, built around the business characteristic of AI conversation. Its purpose is to shield hardware details, enabling users to focus on software development and business implementation.
|
||||
Among this directory:
|
||||
|
||||
- inc: Contains header files
|
||||
- src: Contains specific implementations for different hardware platforms
|
||||
|
||||
## principles
|
||||
|
||||
As a Hardware Abstraction Layer (HAL), we typically expect capabilities to be atomized, interfaces to be orthogonal, and high flexibility to be maintained. However, excessive pursuit of atomization and flexibility often reduces interface usability and impairs business development efficiency. These two aspects are often mutually exclusive. // Nevertheless, this development platform focuses on AI conversation scenarios; therefore, given the relatively fixed business scenarios, we can make trade-offs with greater engineering practical value. // Our core design principles are as follows:
|
||||
|
||||
1. Orthogonal interfaces to reduce coupling and shield hardware details.
|
||||
2. Interface design takes certain business characteristics into account:
|
||||
some implementations and configuration parameters are implicitly implemented according to business requirements,and not all software configurable items are retained.This prevents upper-layer business development from being bogged down in endless detailed configuration and parameter tuning. For example: we enable the Acoustic Echo Cancellation (AEC) module by default for audio capture,instead of requiring upper-layer business to configure aec_enabled in the config file;thus, many interface designs at this layer do not make complete sense.For detailed explanations and trade-offs, refer to the implementation details.Interface design takes certain business characteristics into account: ails.
|
||||
|
||||
## implementation
|
||||
|
||||
### volc_hal
|
||||
|
||||
volc_hal_init() Do the necessary actions after the hardware is powered on, it is not necessary to initialize all the hardware modules, but it is necessary to set the device name.
|
||||
|
||||
volc_get_global_hal_context() ithis function could help you get the hal context everywhere. Do not Modify the return value
|
||||
|
||||
### volc_hal_player
|
||||
|
||||
We do not specify specific media configurations (e.g., encoding format, sampling rate, or resolution).This is because we expect all the aforementioned parameters to be uniquely determined and immutable across all scenarios.Therefore, these values must be fixed at compile time; if your device supports multiple parameters,macros should be used to select the desired ones at compile time.This design adheres to Principle 2.
|
||||
|
||||
for volc_hal_player_play_data()，We have not specified whether this function is sync or async. However, it is strongly recommended to implement this function as non-blocking with a certain internal buffer, ensuring that the caller of the function does not need to overly consider the function's call frequency. For example, the function can be called every 1 second to play 1.2 seconds of audio or video data.
|
||||
|
||||
### volc_hal_capture
|
||||
|
||||
We do not specify specific media configurations (e.g., encoding format, sampling rate, or resolution).This is because we expect all the aforementioned parameters to be uniquely determined and immutable across all scenarios.Therefore, these values must be fixed at compile time; if your device supports multiple parameters,macros should be used to select the desired ones at compile time.This design adheres to Principle 2.
|
||||
|
||||
To ensure a uniform rate for media capture data, we adopt the callback function approach: frame-by-frame audio and video data is delivered asynchronously via callbacks, instead of providing an active capture interface. Therefore, during the function implementation, a capture thread needs to be created to asynchronously callback the data out.
|
||||
|
||||
We consider both voice wake-up and voice data capture to fall under the category of audio capture, differing only in mode (internal pipeline). For this reason, we do not redesign a separate module for voice wake-up; instead, we add an additional mode specifically for voice wake-up.
|
||||
|
||||
Finally, note that acoustic echo cancellation (AEC) must be implemented for audio capture.
|
||||
|
||||
### volc_hal_display
|
||||
|
||||
Typically, a set of screen display interfaces allows upper-layer callers to independently create and destroy regions for display purposes.However, to minimize the development effort of upper-layer business logic,we do not allow the business side to create and manage different screen regions on their own.Instead, different screen regions are divided and created during the execution of volc_hal_display_create().The business layer can only obtain different render layer objects,pass data to them, and perform rendering operations. For example, we divide the screen into a status bar(VOLC_DISPLAY_OBJ_STATUS), main screen(VOLC_DISPLAY_OBJ_MAIN), and subtitle display bar(VOLC_DISPLAY_OBJ_SUBTITLE).You may define your own divisions, as all these codes are open-source and modifiable.
|
||||
|
||||
### volc_hal_file
|
||||
|
||||
File systems may have different implementations on different devices,but we generally adhere to the ANSI C style.For devices without a file system, these interfaces may be left unimplemented as this is not mandatory.
|
||||
@@ -0,0 +1,45 @@
|
||||
# Platform
|
||||
|
||||
## 目录简介
|
||||
|
||||
本目录是硬件对话智能体开发套件基于ai对话这一业务特性，进行的软件抽象层，目的是屏蔽硬件细节，使用户专注软件开发与业务实现。其中inc是header，src是不同硬件平台的具体实现。
|
||||
|
||||
## 设计原则
|
||||
|
||||
作为硬件抽象层往往期待，能力原子化，接口正交，且保持高度的灵活性。但是过分追求原子化和灵活性，往往会降低接口易用性，降低业务开发的效率。这两者往往不可兼得。不过本开发平台专注于ai对话场景，因此在业务场景相对固定的情况下，还是能做成一些更有工程实践价值的trade off的。我们设计的基本原则如下:
|
||||
|
||||
1 接口正交，减少耦合且屏蔽硬件细节
|
||||
|
||||
2 接口设计考虑一定的业务特点，某些实现和配置参数会按照业务要求隐式实现，不会保留所有的软件可配置项，避免上层业务开发陷入无穷无尽的细节配置和调参上。这里举一个例子： 我们默认音频采集都会打开回声消除（aec）模块，而不是让上层业务去config里配置 aec_enabled；因此，这一层不少的接口设计并不百分百的make sense，详细的说明和取舍见实现细节。
|
||||
|
||||
## 部分实现思路
|
||||
|
||||
### volc_hal
|
||||
|
||||
volc_hal_init() 只实现硬件上电后必要的初始化动作，例如设置gpid，联网等；音视频采播等模块可以在其他子模块中init
|
||||
|
||||
volc_get_global_hal_context() 方便拿到全局唯一的硬件context，但不要修改该返回值
|
||||
|
||||
### volc_hal_player
|
||||
|
||||
请注意该模块的接口设计上，我们没有给定一些具体的媒体配置，例如编码格式，采样率或者分辨率；这是因为我们期待上述的所有参数在全场景都是唯一确定且不能改变的。所以这些值需要在编译时就确定好，如果你的设备支持多种参数，那么也应该在编译时用宏做以选择。这一设计遵循原则2；
|
||||
|
||||
对于volc_hal_player_play_data()接口，我们并未规定该函数是同步的还是异步的，但是还是强烈建议你讲该函数实现为非阻塞的，且内部有一定缓存，确保函数的调用者不用过分考虑函数的调用频次；例如，可以1s中调用该函数play 1.2s的audio或video数据
|
||||
|
||||
### volc_hal_capture
|
||||
|
||||
请注意该模块的接口设计上，我们没有给定一些具体的媒体配置，例如编码格式，采样率或者分辨率；这是因为我们期待上述的所有参数在全场景都是唯一确定且不能改变的。所以这些值需要在编译时就确定好，如果你的设备支持多种参数，那么也应该在编译时用宏做以选择。这一设计遵循原则2；
|
||||
|
||||
为了确保媒体采集数据的速率均匀，我们采取回调函数的方式，异步的回调出一帧一帧的音视频数据，而非主动给一个capture接口。所以需要在函数实现的过程中，创建一个采集线程，异步的将数据回调出来。
|
||||
|
||||
请注意，我们认为语音唤醒和语音数据采集都属于音频采集的范畴，只是模式（内部pipeline）不同，所以我们并未重新设计一套语音唤醒的模块，而是多设计了一个语音唤醒的mode
|
||||
|
||||
最后，需要注意，音频的采集需要做aec和ns等模块，这一设计遵循原则2；
|
||||
|
||||
### volc_hal_display
|
||||
|
||||
通常情况下一套屏幕显示的接口，会允许上层调用者自行创建和销毁一块用于显示的区域；但是根据原则2，为了最简化上层业务的开发，我们不允许业务侧去自行创建和管理屏幕的不同区域；而是在volc\_hal\_display\_create就将不同的屏幕区域划分且创建好。然后业务层只能拿到不同的渲染层obj，传递数据并进行渲染。例如，我们将屏幕分为状态栏、主屏幕和字幕显示栏。你也可以有自己的划分，因为这些代码都是开源可修改的
|
||||
|
||||
### volc_hal_file
|
||||
|
||||
不同的设备上文件系统可能有不同的实现，但是我们总体上遵循ANSI C style。对于没有文件系统的设备，这些接口可以不做实现，这并不是必须的
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef __VOLC_HAL_H__
|
||||
#define __VOLC_HAL_H__
|
||||
|
||||
#include "volc_hal_button.h"
|
||||
#include "volc_hal_capture.h"
|
||||
#include "volc_hal_display.h"
|
||||
#include "volc_hal_player.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define VOLC_DEVICE_NAME_LENGTH 32
|
||||
typedef enum {
|
||||
VOLC_HAL_CAPTURE_AUDIO = 0, // audio capture
|
||||
VOLC_HAL_CAPTURE_VIDEO, // video capture
|
||||
VOLC_HAL_CAPTURE_MAX, // Maximum capture count
|
||||
} volc_hal_capture_e;
|
||||
|
||||
typedef enum {
|
||||
VOLC_HAL_PLAYER_AUDIO = 0, // audio player
|
||||
VOLC_HAL_PLAYER_VIDEO, // video player
|
||||
VOLC_HAL_PLAYER_MAX, // Maximum player count
|
||||
} volc_hal_player_e;
|
||||
|
||||
/**
|
||||
* @brief All hardware resources handle and context
|
||||
* @details generally,this structure should be singleton
|
||||
* you can get the Instance by volc_get_global_hal_context()
|
||||
* And, you just set the value when the hardware submodule be inited
|
||||
* the function volc_hal_init() just do some necessary initialization actions
|
||||
* such as, malloc the volc_hal_context_t mem and set the device_name
|
||||
*/
|
||||
typedef struct {
|
||||
/**
|
||||
* @brief the button handle pointer
|
||||
* @details It should be set value when the volc_hal_button_create() is called
|
||||
*/
|
||||
volc_hal_button_t button_handle;
|
||||
/**
|
||||
* @brief the capture pointer array, include audio_capture and video_capture
|
||||
* @details It should be set value when the volc_hal_capture_create() is called
|
||||
*/
|
||||
volc_hal_capture_t capture_handle[VOLC_HAL_CAPTURE_MAX];
|
||||
/**
|
||||
* @brief the display pointer
|
||||
* @details It should be set value when the volc_hal_display_create() is called
|
||||
*/
|
||||
volc_hal_display_t display_handle;
|
||||
/**
|
||||
* @brief the player pointer array, include audio_player and video_player
|
||||
* @details It should be set value when the volc_hal_player_create() is called
|
||||
*/
|
||||
volc_hal_player_t player_handle[VOLC_HAL_PLAYER_MAX];
|
||||
/**
|
||||
* @brief the device name, in most cases, it is a mac address
|
||||
* @details It should be set a value when the volc_hal_init() is called
|
||||
*/
|
||||
char device_name[VOLC_DEVICE_NAME_LENGTH];
|
||||
} volc_hal_context_t;
|
||||
|
||||
/**
|
||||
* @brief Initialize the HAL module
|
||||
* @details Do the necessary actions after the hardware is powered on
|
||||
* it is not necessary to initialize all the hardware modules
|
||||
* but it is necessary to set the device name
|
||||
*
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_init(void);
|
||||
|
||||
/**
|
||||
* @brief return global hal context
|
||||
* @details you can get the singleton Instance volc_hal_context_t by volc_get_global_hal_context()
|
||||
* this function could help you get the hal context everywhere
|
||||
* please be carefully, Do not Modify the return value
|
||||
*/
|
||||
volc_hal_context_t* volc_get_global_hal_context();
|
||||
|
||||
/**
|
||||
* @brief Deinitialize the HAL module
|
||||
*/
|
||||
void volc_hal_fini(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef __VOLC_HAL_BUTTON_H__
|
||||
#define __VOLC_HAL_BUTTON_H__
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef void* volc_hal_button_t;
|
||||
|
||||
typedef enum {
|
||||
VOLC_BUTTON_PRESS_DOWN = 0, // Button press down event
|
||||
VOLC_BUTTON_PRESS_UP, // Button press up event
|
||||
VOLC_BUTTON_PRESS_REPEAT, // Button press repeat event
|
||||
VOLC_BUTTON_PRESS_REPEAT_DONE, // Button press repeat done event
|
||||
VOLC_BUTTON_SINGLE_CLICK, // Button single click event
|
||||
VOLC_BUTTON_DOUBLE_CLICK, // Button double click event
|
||||
VOLC_BUTTON_MULTIPLE_CLICK, // Button multiple click event
|
||||
VOLC_BUTTON_LONG_PRESS_START, // Button long press start event
|
||||
VOLC_BUTTON_LONG_PRESS_HOLD, // Button long press hold event
|
||||
VOLC_BUTTON_LONG_PRESS_UP, // Button long press up event
|
||||
VOLC_BUTTON_PRESS_END, // Button press end event
|
||||
VOLC_BUTTON_EVENT_MAX, // Maximum number of button events
|
||||
} volc_hal_button_event_e;
|
||||
|
||||
typedef void (*volc_hal_button_event_cb_t)(volc_hal_button_t button, volc_hal_button_event_e event, void* user_data);
|
||||
|
||||
typedef struct volc_hal_button_config {
|
||||
int gpio_num; // GPIO number of the button
|
||||
int active_level; // Active level of the button (0 or 1)
|
||||
int long_press_ms; // Long press threshold in milliseconds
|
||||
int short_press_ms; // Short press threshold in milliseconds
|
||||
int enable_power_save; // Whether to enable power save mode (0 or 1)
|
||||
void* user_data; // User data pointer
|
||||
volc_hal_button_event_cb_t event_cb; // Button event callback function
|
||||
} volc_hal_button_config_t;
|
||||
|
||||
/**
|
||||
* @brief Create a button instance
|
||||
*
|
||||
* @param config Button configuration
|
||||
* @return volc_hal_button_t Button instance pointer
|
||||
*/
|
||||
volc_hal_button_t volc_hal_button_create(volc_hal_button_config_t* config);
|
||||
|
||||
/**
|
||||
* @brief Destroy a button instance
|
||||
*
|
||||
* @param button Button instance pointer
|
||||
*/
|
||||
void volc_hal_button_destroy(volc_hal_button_t button);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef __VOLC_HAL_CAPTURE_H__
|
||||
#define __VOLC_HAL_CAPTURE_H__
|
||||
|
||||
#include "volc_hal_type.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
/*
|
||||
* @brief Capture instance pointer
|
||||
*/
|
||||
typedef void* volc_hal_capture_t;
|
||||
|
||||
typedef struct {
|
||||
volc_audio_data_type_e data_type; // Audio data type
|
||||
void* user_data; // User data pointer
|
||||
} volc_hal_frame_info_t;
|
||||
|
||||
typedef enum {
|
||||
VOLC_AUDIO_MODE_WAKEUP = 0,
|
||||
VOLC_AUDIO_MODE_CAPTURE,
|
||||
} volc_hal_audio_capture_mode_e;
|
||||
|
||||
typedef void (*volc_hal_capture_data_cb_t)(volc_hal_capture_t capture, const void* data, int len, volc_hal_frame_info_t* frame_info);
|
||||
typedef void (*volc_hal_audio_wakeup_cb_t)(void *event, void *user_data);
|
||||
|
||||
typedef struct volc_hal_capture_config {
|
||||
volc_media_type_e media_type; // Media type
|
||||
volc_hal_capture_data_cb_t data_cb; // Data callback function
|
||||
volc_hal_audio_wakeup_cb_t audio_wakeup_cb; // audio wakeup cb
|
||||
void* user_data; // User data pointer
|
||||
} volc_hal_capture_config_t;
|
||||
|
||||
/**
|
||||
* @brief Create a capture instance
|
||||
* @details To ensure a uniform rate for media capture data, we adopt the callback function approach:
|
||||
* frame-by-frame audio and video data is delivered asynchronously via callbacks,
|
||||
* instead of providing an active capture interface.
|
||||
* Therefore, during the function implementation, a capture thread needs to be created
|
||||
* to asynchronously callback the data out.
|
||||
* @note We do not specify specific media configurations (e.g., encoding format, sampling rate, or resolution).
|
||||
* This is because we expect all the aforementioned parameters to be uniquely determined and immutable across all scenarios.
|
||||
* Therefore, these values must be fixed at compile time; if your device supports multiple parameters,
|
||||
* macros should be used to select the desired ones at compile time.
|
||||
* We consider both voice wake-up and voice data capture to fall under the category of audio capture,
|
||||
* differing only in mode (internal pipeline).
|
||||
* For this reason, we do not redesign a separate module for voice wake-up;
|
||||
* instead, we add an additional mode specifically for voice wake-up.
|
||||
* Finally, note that acoustic echo cancellation (AEC) must be implemented for audio capture.
|
||||
*
|
||||
* @param config Capture configuration pointer
|
||||
* @return volc_hal_capture_t Capture instance pointer
|
||||
*/
|
||||
volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t* config);
|
||||
|
||||
/**
|
||||
* @brief Destroy a capture instance
|
||||
*
|
||||
* @param capture Capture instance pointer
|
||||
*/
|
||||
void volc_hal_capture_destroy(volc_hal_capture_t capture);
|
||||
|
||||
/**
|
||||
* @brief Start a capture instance
|
||||
*
|
||||
* @param capture Capture instance pointer
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_capture_start(volc_hal_capture_t capture,volc_hal_audio_capture_mode_e mode);
|
||||
|
||||
/**
|
||||
* @brief Stop a capture instance
|
||||
* @details you should start your capture thread within this function.
|
||||
*
|
||||
* @param capture Capture instance pointer
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_capture_stop(volc_hal_capture_t capture);
|
||||
|
||||
/**
|
||||
* @brief Update audio capture configuration
|
||||
* @note this function could help you update the capture config
|
||||
* Please check if it is thread-safe or not;
|
||||
*
|
||||
* @param capture Capture instance pointer
|
||||
* @param config Capture configuration pointer
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_capture_update_config(volc_hal_capture_t capture, volc_hal_capture_config_t* config);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef __VOLC_HAL_DISPLAY_H__
|
||||
#define __VOLC_HAL_DISPLAY_H__
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief Display instance pointer
|
||||
*/
|
||||
typedef void* volc_hal_display_t;
|
||||
|
||||
/** This enumeration exists to map to distinct regions of the screen.
|
||||
* Note: Typically, a set of screen display interfaces allows upper-layer callers
|
||||
* to independently create and destroy regions for display purposes.
|
||||
* However, to minimize the development effort of upper-layer business logic,
|
||||
* we do not allow the business side to create and manage different screen regions on their own.
|
||||
* Instead, different screen regions are divided and created during the execution of volc_hal_display_create().
|
||||
* The business layer can only obtain different render layer objects,
|
||||
* pass data to them, and perform rendering operations.
|
||||
* For example, we divide the screen into a status bar(VOLC_DISPLAY_OBJ_STATUS),
|
||||
* main screen(VOLC_DISPLAY_OBJ_MAIN), and subtitle display bar(VOLC_DISPLAY_OBJ_SUBTITLE).
|
||||
* You may define your own divisions, as all these codes are open-source and modifiable.
|
||||
*/
|
||||
typedef enum {
|
||||
VOLC_DISPLAY_OBJ_STATUS = 0, // Status display object
|
||||
VOLC_DISPLAY_OBJ_MAIN,
|
||||
VOLC_DISPLAY_OBJ_SUBTITLE, // Subtitle display object
|
||||
// VOLC_DISPLAY_OBJ_EMOJI, // Emoji display object
|
||||
VOLC_DISPLAY_OBJ_MAX, // Maximum display object count
|
||||
} volc_hal_display_obj_e;
|
||||
|
||||
typedef enum {
|
||||
VOLC_DISPLAY_TEXT = 0, // Text display type
|
||||
VOLC_DISPLAY_IMAGE, // Image display type
|
||||
VOLC_DISPLAY_EMOJI // Emoji display type
|
||||
} volc_hal_display_type_e;
|
||||
|
||||
typedef struct volc_hal_display_config {
|
||||
int width; // Width of the display
|
||||
int height; // Height of the display
|
||||
} volc_hal_display_config_t;
|
||||
|
||||
/**
|
||||
* @brief Create a Display instance
|
||||
*
|
||||
* @param config Display configuration
|
||||
* @return volc_hal_display_t Display instance pointer
|
||||
*/
|
||||
volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config);
|
||||
|
||||
/**
|
||||
* @brief set brightness
|
||||
*
|
||||
* @param display Display instance pointer
|
||||
* @param brightness the brightness value [0,100]
|
||||
* @return volc_hal_display_t Display instance pointer
|
||||
*/
|
||||
int volc_hal_display_set_brightness(volc_hal_display_t display, int brightness);
|
||||
|
||||
/**
|
||||
* @brief Destroy a Display instance
|
||||
*
|
||||
* @param display Display instance pointer
|
||||
*/
|
||||
void volc_hal_display_destroy(volc_hal_display_t display);
|
||||
|
||||
/**
|
||||
* @brief Set the display emotion
|
||||
*
|
||||
* @param display Display instance pointer
|
||||
* @param type Display type (VOLC_DISPLAY_TEXT, VOLC_DISPLAY_IMAGE, or VOLC_DISPLAY_EMOJI)
|
||||
* @param content Content to be displayed (string for text, image pointer for image, emoji string for emoji)
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_display_set_content(volc_hal_display_t display, volc_hal_display_obj_e obj, volc_hal_display_type_e type, const void* content);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#ifndef __VOLC_HAL_FILE_H__
|
||||
#define __VOLC_HAL_FILE_H__
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* File systems may have different implementations on different devices,
|
||||
* but we generally adhere to the ANSI C style.
|
||||
* For devices without a file system, these interfaces may be left unimplemented
|
||||
* as this is not mandatory.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief file instance pointer
|
||||
*/
|
||||
typedef void* volc_hal_file_t;
|
||||
|
||||
/**
|
||||
* @brief init the file system
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_file_system_init();
|
||||
|
||||
/**
|
||||
* @brief open a file instance
|
||||
* @param file_name the file name
|
||||
* @param mode openmode, just follow ANSI C style(fopen)
|
||||
* @return volc_hal_file_t file instance pointer
|
||||
*/
|
||||
volc_hal_file_t volc_hal_file_open(char* file_name, char* mode);
|
||||
|
||||
/**
|
||||
* @brief read the data from the file
|
||||
* @param file the file instance pointer
|
||||
* @param data the data pointer
|
||||
* @param size the data size
|
||||
* @return return the real read data size
|
||||
*/
|
||||
uint64_t volc_hal_file_read(volc_hal_file_t file, void *data, uint64_t size);
|
||||
|
||||
/**
|
||||
* @brief write the data from the file
|
||||
* @param file the file instance pointer
|
||||
* @param data the data pointer
|
||||
* @param size the data size
|
||||
* @return return the real writed data size
|
||||
*/
|
||||
uint64_t volc_hal_file_write(volc_hal_file_t file, void *data, uint64_t size);
|
||||
|
||||
/**
|
||||
* @brief get the file size
|
||||
* @param file the file instance pointer
|
||||
* @return return the file size
|
||||
*/
|
||||
uint64_t volc_hal_file_get_size(volc_hal_file_t file);
|
||||
|
||||
/**
|
||||
* @brief get the file size
|
||||
* @param file the file instance pointer
|
||||
* @param offset the file offset (start is 0)
|
||||
* @return int 0 if success, otherwise error code
|
||||
*/
|
||||
int volc_hal_file_seek(volc_hal_file_t file,uint64_t offset);
|
||||
|
||||
/**
|
||||
* @brief close a file instance
|
||||
* @param mode openmode, just follow ANSI C style(fopen)
|
||||
* @return volc_hal_file_t file instance pointer
|
||||
*/
|
||||
int volc_hal_file_close(volc_hal_file_t file);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user