feat: enable S2S with native Viking memory

This commit is contained in:
Codex Local
2026-09-06 17:27:49 +08:00
parent 0c2151ace9
commit d60f5d9a9b
15 changed files with 223 additions and 42 deletions
+5
View File
@@ -30,6 +30,11 @@ x-backend-environment: &backend-environment
MEMORY_TOOL_TOKEN_TTL: 4h MEMORY_TOOL_TOKEN_TTL: 4h
MEMORY_TOOL_RATE_RPS: "2" MEMORY_TOOL_RATE_RPS: "2"
MEMORY_TOOL_RATE_BURST: "5" MEMORY_TOOL_RATE_BURST: "5"
VIKING_MEMORY_RESOURCE_ID: ${VIKING_MEMORY_RESOURCE_ID:-}
VIKING_MEMORY_COLLECTION_NAME: ${VIKING_MEMORY_COLLECTION_NAME:-}
VIKING_MEMORY_API_KEY: ${VIKING_MEMORY_API_KEY:-}
VIKING_MEMORY_EVENT_TYPE: ${VIKING_MEMORY_EVENT_TYPE:-event_v1}
VIKING_MEMORY_PROFILE_TYPE: ${VIKING_MEMORY_PROFILE_TYPE:-profile_v1}
PROVIDER_MEMORY_MODE: ${PROVIDER_MEMORY_MODE:-disabled} PROVIDER_MEMORY_MODE: ${PROVIDER_MEMORY_MODE:-disabled}
PROVIDER_MEMORY_BASE_URL: ${PROVIDER_MEMORY_BASE_URL:-} PROVIDER_MEMORY_BASE_URL: ${PROVIDER_MEMORY_BASE_URL:-}
PROVIDER_MEMORY_API_KEY: ${PROVIDER_MEMORY_API_KEY:-} PROVIDER_MEMORY_API_KEY: ${PROVIDER_MEMORY_API_KEY:-}
+8
View File
@@ -22,6 +22,10 @@ x-server-backend-environment: &server-backend-environment
MEMORY_TOOL_BASE_URL: ${PUBLIC_ORIGIN}/internal/v1/tools/memory-search MEMORY_TOOL_BASE_URL: ${PUBLIC_ORIGIN}/internal/v1/tools/memory-search
MEMORY_TOOL_SIGNING_SECRET: ${MEMORY_TOOL_SIGNING_SECRET} MEMORY_TOOL_SIGNING_SECRET: ${MEMORY_TOOL_SIGNING_SECRET}
MODEL_CREDENTIAL_MASTER_KEY: ${MODEL_CREDENTIAL_MASTER_KEY} MODEL_CREDENTIAL_MASTER_KEY: ${MODEL_CREDENTIAL_MASTER_KEY}
VIKING_MEMORY_RESOURCE_ID: ${VIKING_MEMORY_RESOURCE_ID:-}
VIKING_MEMORY_COLLECTION_NAME: ${VIKING_MEMORY_COLLECTION_NAME:-}
VIKING_MEMORY_EVENT_TYPE: ${VIKING_MEMORY_EVENT_TYPE:-event_v1}
VIKING_MEMORY_PROFILE_TYPE: ${VIKING_MEMORY_PROFILE_TYPE:-profile_v1}
EMBEDDING_PROVIDER: local EMBEDDING_PROVIDER: local
VOICE_PROVIDER: ${VOICE_PROVIDER:-mock} VOICE_PROVIDER: ${VOICE_PROVIDER:-mock}
@@ -112,6 +116,8 @@ services:
VOLC_HARDWARE_PRODUCT_SECRET_FILE: ${VOLC_HARDWARE_PRODUCT_SECRET_FILE:-} VOLC_HARDWARE_PRODUCT_SECRET_FILE: ${VOLC_HARDWARE_PRODUCT_SECRET_FILE:-}
VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY: ${VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY:-} VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY: ${VOLC_HARDWARE_DEVICE_CREDENTIAL_MASTER_KEY:-}
VOLC_HARDWARE_DEVICE_NAME_PREFIX: ${VOLC_HARDWARE_DEVICE_NAME_PREFIX:-dt-} VOLC_HARDWARE_DEVICE_NAME_PREFIX: ${VOLC_HARDWARE_DEVICE_NAME_PREFIX:-dt-}
VIKING_MEMORY_API_KEY: ""
VIKING_MEMORY_API_KEY_FILE: ${VIKING_MEMORY_API_KEY_FILE:-/run/secrets/voice-runtime/viking_memory_api_key}
volumes: volumes:
- /etc/data-terminal-cloud/voice-runtime:/run/secrets/voice-runtime:ro - /etc/data-terminal-cloud/voice-runtime:/run/secrets/voice-runtime:ro
restart: unless-stopped restart: unless-stopped
@@ -153,6 +159,8 @@ services:
ARK_API_KEY_FILE: ${ARK_API_KEY_FILE:-/run/secrets/voice-runtime/ark-api-key} ARK_API_KEY_FILE: ${ARK_API_KEY_FILE:-/run/secrets/voice-runtime/ark-api-key}
ARK_CONTENT_MODEL: ${ARK_CONTENT_MODEL:-} ARK_CONTENT_MODEL: ${ARK_CONTENT_MODEL:-}
CONTENT_SAFETY_PROVIDER: local CONTENT_SAFETY_PROVIDER: local
VIKING_MEMORY_API_KEY: ""
VIKING_MEMORY_API_KEY_FILE: ${VIKING_MEMORY_API_KEY_FILE:-/run/secrets/voice-runtime/viking_memory_api_key}
volumes: volumes:
- /etc/data-terminal-cloud/voice-runtime:/run/secrets/voice-runtime:ro - /etc/data-terminal-cloud/voice-runtime:/run/secrets/voice-runtime:ro
restart: unless-stopped restart: unless-stopped
@@ -34,8 +34,8 @@ function buildSessionPrompt(sessionConfig) {
return sections.join("\n\n"); return sections.join("\n\n");
} }
function buildSessionUpdate(sessionConfig, eventID, modelMode = "llm") { function buildSessionUpdate(sessionConfig, eventID, modelMode) {
const normalizedMode = String(modelMode || "llm").trim().toLowerCase(); const normalizedMode = String(modelMode || sessionConfig.model_mode || "llm").trim().toLowerCase();
if (!new Set(["llm", "s2s"]).has(normalizedMode)) throw new Error("智能体模型类型必须为 llm 或 s2s"); if (!new Set(["llm", "s2s"]).has(normalizedMode)) throw new Error("智能体模型类型必须为 llm 或 s2s");
const dialogID = requiredText(sessionConfig.dialog_id, "对话标识", 200); const dialogID = requiredText(sessionConfig.dialog_id, "对话标识", 200);
const systemPrompt = buildSessionPrompt(sessionConfig); const systemPrompt = buildSessionPrompt(sessionConfig);
@@ -52,8 +52,32 @@ function buildSessionUpdate(sessionConfig, eventID, modelMode = "llm") {
}, },
}, },
}; };
} else { }
config.LLMConfig = { SystemMessages: [systemPrompt] }; config.LLMConfig = { SystemMessages: [systemPrompt] };
const memory = sessionConfig.memory;
if (memory && typeof memory === "object") {
const collectionName = requiredText(memory.collection_name, "Viking 记忆库名称", 200);
const userID = requiredText(memory.user_id, "Viking 用户标识", 200);
const memoryTypes = Array.isArray(memory.memory_types)
? memory.memory_types.map((value) => requiredText(value, "Viking 记忆类型", 100))
: [];
if (!memoryTypes.length) throw new Error("Viking 记忆类型不能为空");
const filter = { user_id: [userID], memory_type: memoryTypes };
if (String(memory.assistant_id || "").trim()) {
filter.assistant_id = [requiredText(memory.assistant_id, "Viking 助手标识", 200)];
}
config.MemoryConfig = {
Enable: true,
Provider: "volc",
ProviderParams: {
collection_name: collectionName,
filter,
limit: Number.isInteger(memory.limit) && memory.limit > 0 ? Math.min(memory.limit, 5) : 5,
...(String(memory.transition_words || "").trim()
? { transition_words: String(memory.transition_words).trim() }
: {}),
},
};
} }
return { return {
event_id: eventID, event_id: eventID,
@@ -33,7 +33,25 @@ test("S2S session update maps persona and dialog id to provider dialog config",
assert.equal(dialog.speaking_style, "亲切自然"); assert.equal(dialog.speaking_style, "亲切自然");
assert.match(dialog.system_role, /只使用授权信息/); assert.match(dialog.system_role, /只使用授权信息/);
assert.match(dialog.system_role, /你是陪伴助手/); assert.match(dialog.system_role, /你是陪伴助手/);
assert.equal("LLMConfig" in update.session.config, false); assert.match(update.session.config.LLMConfig.SystemMessages[0], /你是陪伴助手/);
});
test("session update maps the bound user to native Viking memory without credentials", () => {
const update = buildSessionUpdate({
dialog_id: "conversation-memory", model_mode: "s2s", authorized_context: "authorized",
memory: {
collection_name: "memory", user_id: "u_subject_hash", assistant_id: "platform_companion_v1",
memory_types: ["event_v1", "profile_v1"], limit: 5, transition_words: "自然承接历史记忆",
},
}, "event-memory");
const memory = update.session.config.MemoryConfig;
assert.equal(memory.Enable, true);
assert.equal(memory.Provider, "volc");
assert.equal(memory.ProviderParams.collection_name, "memory");
assert.deepEqual(memory.ProviderParams.filter.user_id, ["u_subject_hash"]);
assert.deepEqual(memory.ProviderParams.filter.assistant_id, ["platform_companion_v1"]);
assert.deepEqual(memory.ProviderParams.filter.memory_type, ["event_v1", "profile_v1"]);
assert.equal("token" in memory.ProviderParams, false);
}); });
class FakeWebSocket extends EventEmitter { class FakeWebSocket extends EventEmitter {
+8 -6
View File
@@ -4,31 +4,33 @@ Configured collection: `memory`, resource `mem-3a1174a1`, project `default`, Bei
## Runtime ## Runtime
API and worker accept `VIKING_MEMORY_RESOURCE_ID`, `VIKING_MEMORY_API_KEY_FILE`, `VIKING_MEMORY_EVENT_TYPE` and `VIKING_MEMORY_PROFILE_TYPE`. An empty resource ID disables this integration. The API key is a server-only file; it is never returned to a device or written into a firmware image. The endpoint is HTTPS; redirects are disabled. Error logs contain status/code, never vendor response bodies or transcripts. API and worker accept `VIKING_MEMORY_RESOURCE_ID`, `VIKING_MEMORY_COLLECTION_NAME`, `VIKING_MEMORY_API_KEY_FILE`, `VIKING_MEMORY_EVENT_TYPE` and `VIKING_MEMORY_PROFILE_TYPE`. An empty resource ID disables the integration. An empty collection name keeps the compatibility startup-prefetch path. The API key is a server-only file; it is never returned to a device or written into a firmware image. The endpoint is HTTPS; redirects are disabled. Error logs contain status/code, never vendor response bodies or transcripts.
Terminal conversation extraction requests also enqueue an independent `conversation.viking_sync_requested` outbox event. The worker submits final user and assistant turns with AddSession. Frozen transcript/memory consent and current subject/memory/transcript eligibility are checked. No user-only or diagnostic assistant-only conversation is submitted. The database remains the source of full transcripts. Terminal conversation extraction requests also enqueue an independent `conversation.viking_sync_requested` outbox event. The worker submits final user and assistant turns with AddSession. Frozen transcript/memory consent and current subject/memory/transcript eligibility are checked. No user-only or diagnostic assistant-only conversation is submitted. The database remains the source of full transcripts.
User identity is SHA-256 of the tenant and subject UUIDs, prefixed `u_`; it survives device restarts and changes of device binding. Assistant identity is the stable platform companion ID, currently shared across platform persona versions. Every conversation has a distinct `c_<conversation UUID without hyphens>_<batch>` session identifier. Batches contain up to 100 messages. A retry uses the same identifier; a late transcript revision replaces that conversation's batch. Audit digests avoid rewriting unchanged successful submissions. Initial indexing/extraction is asynchronous. User identity is SHA-256 of the tenant and subject UUIDs, prefixed `u_`; it survives device restarts and changes of device binding. Assistant identity is the stable platform companion ID, currently shared across platform persona versions. Every conversation has a distinct `c_<conversation UUID without hyphens>_<batch>` session identifier. Batches contain up to 100 messages. A retry uses the same identifier; a late transcript revision replaces that conversation's batch. Audit digests avoid rewriting unchanged successful submissions. Initial indexing/extraction is asynchronous.
At hardware session creation the API retrieves the current user profile and recent events before opening the allocation transaction, with a three-second overall budget. It revalidates the subject binding and memory consent before incorporating up to 600 characters into the frozen context. Extracted memories are explicitly labeled as fallible background, not instructions or manually confirmed facts. The existing in-call memory-search endpoint also searches Viking, then falls back to local curated memories. Actual use of that tool by the hardware cloud runtime still depends on provider tool support; startup context does not depend on it. Lookup failure leaves normal conversation available. At hardware session creation the API resolves the currently bound subject and revalidates memory consent. It returns a per-user, non-secret native memory description containing the collection name, hashed user ID, stable assistant ID, extraction-rule names, limit and transition words. Firmware maps this description to Volcano `Config.MemoryConfig`; Volcano therefore retrieves topic-relevant memories for every turn. A different subject binding produces a different hashed user ID. The Viking API key stays on the platform worker and is used only for writes, platform search and erasure.
There are no firmware/audio changes and no database migrations. Native subtitle callbacks remain in device-probe mode; the reliable device transcript upload supplies these sessions. Pending memory extraction is independent of streamed audio playback. The dynamic device update also enables `S2SConfig` while retaining `LLMConfig` for hybrid orchestration. S2S App ID and token live in the Volcano bot configuration and in root-owned server recovery files, never in device session responses or firmware. The device supplies only per-session dialog/persona fields. Existing in-call memory search remains as a compatibility path for deployments without a configured native collection.
There are no database migrations. Native subtitle callbacks remain in device-probe mode; the reliable device transcript upload supplies these sessions. Pending memory extraction is independent of streamed audio playback.
## Lifecycle and limits ## Lifecycle and limits
Current consent withdrawal immediately gates future Viking reads/writes. Subject privacy erasure calls scoped event/profile deletion before completing the provider-index deletion subtask. Vendor extraction is asynchronous; late vendor extraction after an erasure request needs follow-up reconciliation before treating provider erasure as fully verified. Individual locally curated-memory deletion does not delete independently extracted Viking events/profiles. For the present integration, use the Viking console for editing or deleting those derived entries; a unified platform memory-management UI is not implemented. Do not describe the existing local memory list as a complete Viking list. Current consent withdrawal immediately gates future Viking reads/writes. Subject privacy erasure calls scoped event/profile deletion before completing the provider-index deletion subtask. Vendor extraction is asynchronous; late vendor extraction after an erasure request needs follow-up reconciliation before treating provider erasure as fully verified. Individual locally curated-memory deletion does not delete independently extracted Viking events/profiles. For the present integration, use the Viking console for editing or deleting those derived entries; a unified platform memory-management UI is not implemented. Do not describe the existing local memory list as a complete Viking list.
Automatic extraction begins on conversation completion/failure and final transcript delivery. A sudden power-off relies on the existing heartbeat recovery to close the conversation. Startup recalls recent events plus profile; recall of older topic-specific events during the same session depends on the in-call search tool. Full historical backfill is explicit; enabling the integration does not automatically export all past conversations. Automatic extraction begins on conversation completion/failure and final transcript delivery. A sudden power-off relies on the existing heartbeat recovery to close the conversation. Native retrieval is available again after every restart because identity comes from the platform binding, not device RAM. Full historical backfill is explicit; enabling the integration does not automatically export all past conversations.
## Verification ## Verification
Deployment on 2026-09-06: API and worker `0.78.4-rc.viking1`; callbacks remain `0.78.3-rc.subtitle-memory3`. All three readiness endpoints returned 200. Current symlink points to `/opt/data-terminal-cloud/releases/0.78.4-rc.viking1`; rollback uses the previous `0.78.3-rc.subtitle-memory1` directory including its `compose.subtitle.yaml` override. The same three compose files remain required. Deployment on 2026-09-06: API and worker `0.79.0-rc.s2s-memory1`; callbacks remain `0.78.3-rc.subtitle-memory3`. Readiness returned 200. Current symlink points to `/opt/data-terminal-cloud/releases/0.79.0-rc.s2s-memory1`; rollback uses `0.78.4-rc.viking1`. The Volcano bot uses S2S hybrid mode, and `VoiceChatRoleForRTC` has both required Viking policies. The pre-update bot configuration is stored in the protected first-version backup.
One real conversation (`157f866d-c364-4ae8-b169-50ca07024e5a`, three final messages) was explicitly queued for backfill. Viking produced two event memories, and the production service retrieved 76 characters of startup memory. Synthetic event/profile entries were deleted and subsequent queries returned zero. Device power-cycle/spoken recall is pending user verification; server retrieval alone is not proof of successful audible recall. One real conversation (`157f866d-c364-4ae8-b169-50ca07024e5a`, three final messages) was explicitly queued for backfill. Viking produced two event memories, and the production service retrieved 76 characters of startup memory. Synthetic event/profile entries were deleted and subsequent queries returned zero. Device power-cycle/spoken recall is pending user verification; server retrieval alone is not proof of successful audible recall.
- Real API-key calls verified against the configured collection: AddSession, event search, profile search. - Real API-key calls verified against the configured collection: AddSession, event search, profile search.
- Synthetic test used a separate user namespace; no test preference is attached to the real user. - Synthetic test used a separate user namespace; no test preference is attached to the real user.
- Tests cover vendor result scope filtering, error redaction, stable identity/session IDs, scoped deletion, SQL outbox deduplication, late transcript replacement, frozen consent, cross-tenant rejection, and subject-bound startup context. - Tests cover vendor result scope filtering, error redaction, stable identity/session IDs, native per-user filter mapping, scoped deletion, SQL outbox deduplication, late transcript replacement, frozen consent, cross-tenant rejection, and subject-bound startup context.
- A dedicated PostgreSQL/Redis environment is used for integration tests, separate from production data. - A dedicated PostgreSQL/Redis environment is used for integration tests, separate from production data.
Official API references: https://www.volcengine.com/docs/84313/1783353 , https://www.volcengine.com/docs/84313/2100967 , https://www.volcengine.com/docs/84313/2100966 , https://www.volcengine.com/docs/84313/1783358 , https://www.volcengine.com/docs/84313/1783359 . Official API references: https://www.volcengine.com/docs/84313/1783353 , https://www.volcengine.com/docs/84313/2100967 , https://www.volcengine.com/docs/84313/2100966 , https://www.volcengine.com/docs/84313/1783358 , https://www.volcengine.com/docs/84313/1783359 .
+2 -2
View File
@@ -10,12 +10,12 @@
- 固件和设备端都不包含火山 `Product Secret`; - 固件和设备端都不包含火山 `Product Secret`;
- SaaS 使用平台设备身份为 EchoEar 动态申请火山 `Device Secret`; - SaaS 使用平台设备身份为 EchoEar 动态申请火山 `Device Secret`;
- EchoEar 只保存自己的平台设备凭据和火山单设备凭据; - EchoEar 只保存自己的平台设备凭据和火山单设备凭据;
- 每次对话前由 SaaS 下发 Bot、人设、记忆上下文和会话配置; - 每次对话前由 SaaS 下发 Bot、人设、绑定用户的非密钥 Viking 查询范围和会话配置;
- 字幕和最终轮次回传 SaaS,供会话记录与长期记忆处理。 - 字幕和最终轮次回传 SaaS,供会话记录与长期记忆处理。
当前测试硬件:ESP32-S3 EchoEar,16 MB Flash、16 MB PSRAM,硬件标识由 STA MAC 生成(当前板为 `echoear-b43a451634a8`)。 当前测试硬件:ESP32-S3 EchoEar,16 MB Flash、16 MB PSRAM,硬件标识由 STA MAC 生成(当前板为 `echoear-b43a451634a8`)。
每次申请对话前,先通过 `/device/v2/bootstrap` 上报实际固件版本并检查 `firmware_compatible`,再获取配置和创建会话。固件版本为 `1.0.0-echoear-platform.1`;本工程基于 Embedded Kit 2.0,硬件智能体 SDK 版本独立上报为 `2.0.0`。会话能力中的 `sdk_version` 不能用固件版本代替。会话尚未创建时保持唤醒采集,不重复拆建音频管线。SPI 屏幕仅在 `on_color_trans_done` 后释放 GFX 帧缓冲,避免异步传输期间提前复用缓冲区。运行状态用眼睛动画表示,避免 80 MHz 下中文逐像素渲染长期占用 CPU:快速眨眼表示聆听,转圈表示思考,正常眼睛表示待机或回答。 每次申请对话前,先通过 `/device/v2/bootstrap` 上报实际固件版本并检查 `firmware_compatible`,再获取配置和创建会话。固件版本为 `1.1.0-echoear-s2s-memory.1`;本工程基于 Embedded Kit 2.0,硬件智能体 SDK 版本独立上报为 `2.0.0`。会话能力中的 `sdk_version` 不能用固件版本代替。动态 `session.update` 同时配置端到端语音、混合编排 LLM,以及按当前绑定用户隔离的 Viking `MemoryConfig`;端到端语音凭据和 Viking API Key 均不进入固件。会话尚未创建时保持唤醒采集,不重复拆建音频管线。SPI 屏幕仅在 `on_color_trans_done` 后释放 GFX 帧缓冲,避免异步传输期间提前复用缓冲区。运行状态用眼睛动画表示,避免 80 MHz 下中文逐像素渲染长期占用 CPU:快速眨眼表示聆听,转圈表示思考,正常眼睛表示待机或回答。
本板构建使用 16 MB Flash,Flash 与八线 PSRAM 均为 80 MHz。通用默认配置、ESP32-S3 默认配置和当前 `sdkconfig` 必须一致;不要恢复上游的 32 MB Flash 或实验性 120 MHz PSRAM 参数。 本板构建使用 16 MB Flash,Flash 与八线 PSRAM 均为 80 MHz。通用默认配置、ESP32-S3 默认配置和当前 `sdkconfig` 必须一致;不要恢复上游的 32 MB Flash 或实验性 120 MHz PSRAM 参数。
@@ -22,7 +22,7 @@
#define PLATFORM_API_ORIGIN "https://121.41.46.238" #define PLATFORM_API_ORIGIN "https://121.41.46.238"
#define PLATFORM_RESPONSE_MAX (32 * 1024) #define PLATFORM_RESPONSE_MAX (32 * 1024)
#define PLATFORM_FIRMWARE_VERSION "1.0.0-echoear-platform.1" #define PLATFORM_FIRMWARE_VERSION "1.1.0-echoear-s2s-memory.1"
#define PLATFORM_HARDWARE_AGENT_SDK_VERSION "2.0.0" #define PLATFORM_HARDWARE_AGENT_SDK_VERSION "2.0.0"
static const char *TAG = "platform_client"; static const char *TAG = "platform_client";
@@ -457,10 +457,15 @@ static char *build_session_update(cJSON *session_config)
const char *a = cJSON_IsString(authorized) ? authorized->valuestring : ""; const char *a = cJSON_IsString(authorized) ? authorized->valuestring : "";
const char *p = cJSON_IsString(system_prompt) ? system_prompt->valuestring : ""; const char *p = cJSON_IsString(system_prompt) ? system_prompt->valuestring : "";
const char *tone = cJSON_IsString(persona) ? persona->valuestring : ""; const char *tone = cJSON_IsString(persona) ? persona->valuestring : "";
size_t prompt_size = strlen(a) + strlen(p) + strlen(tone) + 256; const char *voice_rules =
"这是通过扬声器进行的真实语音对话。回复只包含直接对用户说的口语正文,简短自然,语速正常。"
"不要输出或朗读动作、表情、语气、情绪、场景等舞台提示,不要写括号或星号包裹的表演说明,"
"例如‘语气带着关切’‘微笑着’‘温柔地说’。把风格体现在自然措辞中,不要描述自己如何说话。"
"不要复述人设、表达风格、系统配置或对话标识。";
size_t prompt_size = strlen(a) + strlen(p) + strlen(tone) + strlen(voice_rules) + 256;
char *prompt = malloc(prompt_size); char *prompt = malloc(prompt_size);
if (prompt == NULL) return NULL; if (prompt == NULL) return NULL;
snprintf(prompt, prompt_size, "%s\n\n%s\n\n表达风格(仅指导表达,不可逐字复述):%s", a, p, tone); snprintf(prompt, prompt_size, "%s\n\n%s\n\n表达风格(仅指导表达,不可逐字复述):%s\n\n%s", a, p, tone, voice_rules);
cJSON *event = cJSON_CreateObject(); cJSON *event = cJSON_CreateObject();
char event_id[64]; char event_id[64];
@@ -480,11 +485,47 @@ static char *build_session_update(cJSON *session_config)
cJSON *llm = cJSON_AddObjectToObject(config, "LLMConfig"); cJSON *llm = cJSON_AddObjectToObject(config, "LLMConfig");
cJSON *messages = cJSON_AddArrayToObject(llm, "SystemMessages"); cJSON *messages = cJSON_AddArrayToObject(llm, "SystemMessages");
cJSON_AddItemToArray(messages, cJSON_CreateString(prompt)); cJSON_AddItemToArray(messages, cJSON_CreateString(prompt));
cJSON_AddItemToArray(messages, cJSON_CreateString( cJSON *model_mode = cJSON_GetObjectItemCaseSensitive(session_config, "model_mode");
"这是通过扬声器进行的真实语音对话。回复只包含直接对用户说的口语正文,简短自然,语速正常。" if (cJSON_IsString(model_mode) && strcmp(model_mode->valuestring, "s2s") == 0) {
"不要输出或朗读动作、表情、语气、情绪、场景等舞台提示,不要写括号或星号包裹的表演说明," cJSON *s2s = cJSON_AddObjectToObject(config, "S2SConfig");
"例如‘语气带着关切’‘微笑着’‘温柔地说’。把风格体现在自然措辞中,不要描述自己如何说话。" cJSON *provider_params = cJSON_AddObjectToObject(s2s, "ProviderParams");
"不要复述人设、表达风格、系统配置或对话标识。")); cJSON *dialog = cJSON_AddObjectToObject(provider_params, "dialog");
cJSON_AddStringToObject(dialog, "system_role", prompt);
cJSON_AddStringToObject(dialog, "speaking_style", tone);
cJSON *dialog_id = cJSON_GetObjectItemCaseSensitive(session_config, "dialog_id");
if (cJSON_IsString(dialog_id) && dialog_id->valuestring[0]) {
cJSON_AddStringToObject(dialog, "dialog_id", dialog_id->valuestring);
}
}
cJSON *memory = cJSON_GetObjectItemCaseSensitive(session_config, "memory");
if (cJSON_IsObject(memory)) {
cJSON *collection = cJSON_GetObjectItemCaseSensitive(memory, "collection_name");
cJSON *user_id = cJSON_GetObjectItemCaseSensitive(memory, "user_id");
cJSON *assistant_id = cJSON_GetObjectItemCaseSensitive(memory, "assistant_id");
cJSON *memory_types = cJSON_GetObjectItemCaseSensitive(memory, "memory_types");
cJSON *limit = cJSON_GetObjectItemCaseSensitive(memory, "limit");
cJSON *transition = cJSON_GetObjectItemCaseSensitive(memory, "transition_words");
if (cJSON_IsString(collection) && collection->valuestring[0] &&
cJSON_IsString(user_id) && user_id->valuestring[0] && cJSON_IsArray(memory_types)) {
cJSON *memory_config = cJSON_AddObjectToObject(config, "MemoryConfig");
cJSON_AddBoolToObject(memory_config, "Enable", true);
cJSON_AddStringToObject(memory_config, "Provider", "volc");
cJSON *memory_params = cJSON_AddObjectToObject(memory_config, "ProviderParams");
cJSON_AddStringToObject(memory_params, "collection_name", collection->valuestring);
cJSON_AddNumberToObject(memory_params, "limit", cJSON_IsNumber(limit) && limit->valueint > 0 ? limit->valueint : 5);
cJSON *filter = cJSON_AddObjectToObject(memory_params, "filter");
cJSON *users = cJSON_AddArrayToObject(filter, "user_id");
cJSON_AddItemToArray(users, cJSON_CreateString(user_id->valuestring));
if (cJSON_IsString(assistant_id) && assistant_id->valuestring[0]) {
cJSON *assistants = cJSON_AddArrayToObject(filter, "assistant_id");
cJSON_AddItemToArray(assistants, cJSON_CreateString(assistant_id->valuestring));
}
cJSON_AddItemToObject(filter, "memory_type", cJSON_Duplicate(memory_types, true));
if (cJSON_IsString(transition) && transition->valuestring[0]) {
cJSON_AddStringToObject(memory_params, "transition_words", transition->valuestring);
}
}
}
cJSON *agent = cJSON_AddObjectToObject(session, "agent_config"); cJSON *agent = cJSON_AddObjectToObject(session, "agent_config");
cJSON_AddBoolToObject(agent, "EnableConversationStateCallback", false); cJSON_AddBoolToObject(agent, "EnableConversationStateCallback", false);
cJSON *callback_url = cJSON_GetObjectItemCaseSensitive(session_config, "subtitle_callback_url"); cJSON *callback_url = cJSON_GetObjectItemCaseSensitive(session_config, "subtitle_callback_url");
@@ -2948,6 +2948,7 @@ CONFIG_ESP_PAINTER_BASIC_FONT_12=y
# #
# Audio Codec Device Configuration # Audio Codec Device Configuration
# #
# CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE is not set
CONFIG_CODEC_ES8311_SUPPORT=y CONFIG_CODEC_ES8311_SUPPORT=y
CONFIG_CODEC_ES7210_SUPPORT=y CONFIG_CODEC_ES7210_SUPPORT=y
CONFIG_CODEC_ES7243_SUPPORT=y CONFIG_CODEC_ES7243_SUPPORT=y
+1 -1
View File
@@ -86,7 +86,7 @@ func Run(ctx context.Context, serviceName string) error {
var vikingMemory *vikingmemory.Service var vikingMemory *vikingmemory.Service
if serviceName == "api" || serviceName == "worker" { if serviceName == "api" || serviceName == "worker" {
if cfg.VikingResourceID != "" { if cfg.VikingResourceID != "" {
client, vikingErr := vikingmemory.NewClient(vikingmemory.Options{ResourceID: cfg.VikingResourceID, APIKey: cfg.VikingAPIKey, EventType: cfg.VikingEventType, ProfileType: cfg.VikingProfileType, Timeout: 8 * time.Second}) client, vikingErr := vikingmemory.NewClient(vikingmemory.Options{ResourceID: cfg.VikingResourceID, CollectionName: cfg.VikingCollectionName, APIKey: cfg.VikingAPIKey, EventType: cfg.VikingEventType, ProfileType: cfg.VikingProfileType, Timeout: 8 * time.Second})
if vikingErr != nil { if vikingErr != nil {
return fmt.Errorf("configure Viking memory: %w", vikingErr) return fmt.Errorf("configure Viking memory: %w", vikingErr)
} }
+2
View File
@@ -24,6 +24,7 @@ const (
type Config struct { type Config struct {
VikingResourceID string VikingResourceID string
VikingCollectionName string
VikingAPIKey string VikingAPIKey string
VikingEventType string VikingEventType string
VikingProfileType string VikingProfileType string
@@ -414,6 +415,7 @@ func Load(serviceName string) (Config, error) {
MemoryToolSigningSecret: memoryToolSigningSecret, MemoryToolSigningSecret: memoryToolSigningSecret,
MemoryToolTokenTTL: memoryToolTokenTTL, MemoryToolTokenTTL: memoryToolTokenTTL,
VikingResourceID: strings.TrimSpace(os.Getenv("VIKING_MEMORY_RESOURCE_ID")), VikingResourceID: strings.TrimSpace(os.Getenv("VIKING_MEMORY_RESOURCE_ID")),
VikingCollectionName: strings.TrimSpace(os.Getenv("VIKING_MEMORY_COLLECTION_NAME")),
VikingAPIKey: secretValues["VIKING_MEMORY_API_KEY"], VikingAPIKey: secretValues["VIKING_MEMORY_API_KEY"],
VikingEventType: stringEnv("VIKING_MEMORY_EVENT_TYPE", "event_v1"), VikingEventType: stringEnv("VIKING_MEMORY_EVENT_TYPE", "event_v1"),
VikingProfileType: stringEnv("VIKING_MEMORY_PROFILE_TYPE", "profile_v1"), VikingProfileType: stringEnv("VIKING_MEMORY_PROFILE_TYPE", "profile_v1"),
+19 -4
View File
@@ -42,13 +42,28 @@ func (s *Service) CreateHardwareAgent(ctx context.Context, identity deviceauth.I
return HardwareAgentCreateResponse{}, false, err return HardwareAgentCreateResponse{}, false, err
} }
var vikingSubject, vikingContext string var vikingSubject, vikingContext string
var vikingNativeMemory *HardwareAgentMemoryConfig
if s.viking != nil { if s.viking != nil {
readContext, cancel := context.WithTimeout(ctx, 3*time.Second) readContext, cancel := context.WithTimeout(ctx, 3*time.Second)
var readErr error native, readErr := s.viking.NativeAccessForDevice(readContext, identity.TenantID, identity.ID)
vikingSubject, vikingContext, readErr = s.viking.PrepareForDevice(readContext, identity.TenantID, identity.ID)
cancel() cancel()
if readErr != nil { if readErr != nil {
slog.WarnContext(ctx, "Viking startup memory unavailable", "error", readErr) slog.WarnContext(ctx, "Viking native memory configuration unavailable", "error", readErr)
} else if native != nil {
vikingSubject = native.SubjectID
vikingNativeMemory = &HardwareAgentMemoryConfig{
CollectionName: native.CollectionName, UserID: native.UserID, AssistantID: native.AssistantID,
MemoryTypes: native.MemoryTypes, Limit: native.Limit, TransitionWords: native.TransitionWords,
}
} else {
// Older deployments without a native collection name keep the
// platform-side startup retrieval path as a compatibility fallback.
fallbackContext, fallbackCancel := context.WithTimeout(ctx, 3*time.Second)
vikingSubject, vikingContext, readErr = s.viking.PrepareForDevice(fallbackContext, identity.TenantID, identity.ID)
fallbackCancel()
if readErr != nil {
slog.WarnContext(ctx, "Viking startup memory unavailable", "error", readErr)
}
} }
} }
result, err := s.executor.Execute(ctx, request, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) { result, err := s.executor.Execute(ctx, request, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
@@ -61,7 +76,7 @@ func (s *Service) CreateHardwareAgent(ctx context.Context, identity deviceauth.I
TokenExpires: now.Add(s.options.TokenTTL), RTCAppID: s.options.RTCAppID, TokenExpires: now.Add(s.options.TokenTTL), RTCAppID: s.options.RTCAppID,
VoiceProvider: s.options.VoiceProvider, EngineMode: s.options.EngineMode, VoiceProvider: s.options.VoiceProvider, EngineMode: s.options.EngineMode,
HeartbeatInterval: s.options.HeartbeatInterval, MemoryQueryVector: memoryVector, HeartbeatInterval: s.options.HeartbeatInterval, MemoryQueryVector: memoryVector,
VikingSubjectID: vikingSubject, VikingContext: vikingContext, VikingSubjectID: vikingSubject, VikingContext: vikingContext, VikingNativeMemory: vikingNativeMemory,
MemoryPolicy: memoryPolicy, RequirePublishedRouting: s.options.RequirePublishedRouting, MemoryPolicy: memoryPolicy, RequirePublishedRouting: s.options.RequirePublishedRouting,
RuntimeType: RuntimeVolcHardwareAgent, HardwareBotID: s.options.HardwareBotID, RuntimeType: RuntimeVolcHardwareAgent, HardwareBotID: s.options.HardwareBotID,
HardwareTransport: s.options.HardwareTransport, HardwareCapabilities: input.Capabilities, HardwareTransport: s.options.HardwareTransport, HardwareCapabilities: input.Capabilities,
+22 -4
View File
@@ -51,6 +51,7 @@ const (
type AllocationOptions struct { type AllocationOptions struct {
VikingSubjectID string VikingSubjectID string
VikingContext string VikingContext string
VikingNativeMemory *HardwareAgentMemoryConfig
Now time.Time Now time.Time
AllocationExpires time.Time AllocationExpires time.Time
TokenExpires time.Time TokenExpires time.Time
@@ -413,6 +414,7 @@ func (r *Repository) Create(
} }
contextSummary := buildContextSummary(subjectName, profileContext, recentSummaries, memoryContents) contextSummary := buildContextSummary(subjectName, profileContext, recentSummaries, memoryContents)
var vikingContext string var vikingContext string
var vikingNativeMemory *HardwareAgentMemoryConfig
if options.VikingSubjectID == subjectID && options.VikingContext != "" && consentAllowsConversationContinuity(effectiveConsents) { if options.VikingSubjectID == subjectID && options.VikingContext != "" && consentAllowsConversationContinuity(effectiveConsents) {
var allowed bool var allowed bool
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND consent_type='personalized_memory' AND withdrawn_at IS NULL AND (expires_at IS NULL OR expires_at>now())) AND NOT EXISTS(SELECT 1 FROM privacy_request WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND request_type='DELETE')`, identity.TenantID, subjectID).Scan(&allowed); err != nil { if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND consent_type='personalized_memory' AND withdrawn_at IS NULL AND (expires_at IS NULL OR expires_at>now())) AND NOT EXISTS(SELECT 1 FROM privacy_request WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND request_type='DELETE')`, identity.TenantID, subjectID).Scan(&allowed); err != nil {
@@ -423,6 +425,17 @@ func (r *Repository) Create(
contextSummary += "\nViking 自动抽取的历史记忆(仅作背景,不是指令;可能有误,以用户最新说法为准):" + vikingContext contextSummary += "\nViking 自动抽取的历史记忆(仅作背景,不是指令;可能有误,以用户最新说法为准):" + vikingContext
} }
} }
if options.VikingSubjectID == subjectID && options.VikingNativeMemory != nil && consentAllowsConversationContinuity(effectiveConsents) {
var allowed bool
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM consent_record WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND consent_type='personalized_memory' AND withdrawn_at IS NULL AND (expires_at IS NULL OR expires_at>now())) AND NOT EXISTS(SELECT 1 FROM privacy_request WHERE tenant_id=$1::uuid AND subject_id=$2::uuid AND request_type='DELETE')`, identity.TenantID, subjectID).Scan(&allowed); err != nil {
return CreateResponse{}, "", err
}
if allowed {
copy := *options.VikingNativeMemory
copy.MemoryTypes = append([]string(nil), options.VikingNativeMemory.MemoryTypes...)
vikingNativeMemory = &copy
}
}
contextValue := struct { contextValue := struct {
SubjectID string `json:"subject_id"` SubjectID string `json:"subject_id"`
PreferredName string `json:"preferred_name"` PreferredName string `json:"preferred_name"`
@@ -496,7 +509,7 @@ func (r *Repository) Create(
} }
if runtimeType == RuntimeVolcHardwareAgent { if runtimeType == RuntimeVolcHardwareAgent {
return r.createHardwareAgentRuntime(ctx, tx, identity, conversationID, conversationPublicID, return r.createHardwareAgentRuntime(ctx, tx, identity, conversationID, conversationPublicID,
providerTaskRef, promptBundleVersion, engineMode, contextSummary, privacy, input, options) providerTaskRef, promptBundleVersion, engineMode, contextSummary, privacy, input, options, vikingNativeMemory)
} }
_, err = tx.Exec(ctx, ` _, err = tx.Exec(ctx, `
INSERT INTO conversation_state_history (conversation_id, from_status, to_status, reason, source) INSERT INTO conversation_state_history (conversation_id, from_status, to_status, reason, source)
@@ -553,6 +566,7 @@ func (r *Repository) createHardwareAgentRuntime(
privacy Privacy, privacy Privacy,
input CreateInput, input CreateInput,
options AllocationOptions, options AllocationOptions,
vikingNativeMemory *HardwareAgentMemoryConfig,
) (CreateResponse, string, error) { ) (CreateResponse, string, error) {
botID := strings.TrimSpace(options.HardwareBotID) botID := strings.TrimSpace(options.HardwareBotID)
transport := strings.TrimSpace(options.HardwareTransport) transport := strings.TrimSpace(options.HardwareTransport)
@@ -610,11 +624,15 @@ func (r *Repository) createHardwareAgentRuntime(
} }
sessionNonce := base64.RawURLEncoding.EncodeToString(nonceBytes) sessionNonce := base64.RawURLEncoding.EncodeToString(nonceBytes)
nonceDigest := sha256.Sum256([]byte(sessionNonce)) nonceDigest := sha256.Sum256([]byte(sessionNonce))
configValue := map[string]string{ configValue := map[string]any{
"bot_id": botID, "transport": transport, "system_prompt": systemPrompt, "bot_id": botID, "transport": transport, "system_prompt": systemPrompt,
"persona_tone": personaTone, "authorized_context": contextSummary, "persona_tone": personaTone, "authorized_context": contextSummary,
"prompt_bundle_version": promptBundleVersion, "prompt_bundle_version": promptBundleVersion,
"tool_url": options.HardwareMemoryToolURL, "tool_url": options.HardwareMemoryToolURL,
"model_mode": "s2s",
}
if vikingNativeMemory != nil {
configValue["memory"] = vikingNativeMemory
} }
configJSON, _ := json.Marshal(configValue) configJSON, _ := json.Marshal(configValue)
callbackURL, callbackSignature, subtitleSource := "", "", "device" callbackURL, callbackSignature, subtitleSource := "", "", "device"
@@ -668,10 +686,10 @@ func (r *Repository) createHardwareAgentRuntime(
Runtime: HardwareAgentRuntime{Provider: "volcengine", BotID: botID, Transport: transport, Runtime: HardwareAgentRuntime{Provider: "volcengine", BotID: botID, Transport: transport,
ProviderSessionRef: providerSessionRef, SessionNonce: sessionNonce, ProviderSessionRef: providerSessionRef, SessionNonce: sessionNonce,
ConfigDigest: hex.EncodeToString(configDigest[:]), WaitForSessionUpdate: true}, ConfigDigest: hex.EncodeToString(configDigest[:]), WaitForSessionUpdate: true},
SessionConfig: HardwareAgentSessionConfig{DialogID: conversationPublicID, SystemPrompt: systemPrompt, SessionConfig: HardwareAgentSessionConfig{DialogID: conversationPublicID, ModelMode: "s2s", SystemPrompt: systemPrompt,
PersonaTone: personaTone, AuthorizedContext: contextSummary, PromptBundleVersion: promptBundleVersion, PersonaTone: personaTone, AuthorizedContext: contextSummary, PromptBundleVersion: promptBundleVersion,
ToolURL: options.HardwareMemoryToolURL, SubtitleCallbackURL: callbackURL, ToolURL: options.HardwareMemoryToolURL, SubtitleCallbackURL: callbackURL,
SubtitleCallbackSignature: callbackSignature, SubtitleSource: subtitleSource}, SubtitleCallbackSignature: callbackSignature, SubtitleSource: subtitleSource, Memory: vikingNativeMemory},
Privacy: privacy, Privacy: privacy,
Session: Session{EngineMode: engineMode, HeartbeatIntervalSecond: options.HeartbeatInterval, Session: Session{EngineMode: engineMode, HeartbeatIntervalSecond: options.HeartbeatInterval,
+21 -10
View File
@@ -112,16 +112,27 @@ type HardwareAgentRuntime struct {
} }
type HardwareAgentSessionConfig struct { type HardwareAgentSessionConfig struct {
DialogID string `json:"dialog_id"` DialogID string `json:"dialog_id"`
SystemPrompt string `json:"system_prompt,omitempty"` ModelMode string `json:"model_mode"`
PersonaTone string `json:"persona_tone,omitempty"` SystemPrompt string `json:"system_prompt,omitempty"`
AuthorizedContext string `json:"authorized_context"` PersonaTone string `json:"persona_tone,omitempty"`
PromptBundleVersion string `json:"prompt_bundle_version"` AuthorizedContext string `json:"authorized_context"`
ToolURL string `json:"tool_url,omitempty"` PromptBundleVersion string `json:"prompt_bundle_version"`
ToolScopeToken string `json:"tool_scope_token,omitempty"` ToolURL string `json:"tool_url,omitempty"`
SubtitleCallbackURL string `json:"subtitle_callback_url,omitempty"` ToolScopeToken string `json:"tool_scope_token,omitempty"`
SubtitleCallbackSignature string `json:"subtitle_callback_signature,omitempty"` SubtitleCallbackURL string `json:"subtitle_callback_url,omitempty"`
SubtitleSource string `json:"subtitle_source,omitempty"` SubtitleCallbackSignature string `json:"subtitle_callback_signature,omitempty"`
SubtitleSource string `json:"subtitle_source,omitempty"`
Memory *HardwareAgentMemoryConfig `json:"memory,omitempty"`
}
type HardwareAgentMemoryConfig struct {
CollectionName string `json:"collection_name"`
UserID string `json:"user_id"`
AssistantID string `json:"assistant_id,omitempty"`
MemoryTypes []string `json:"memory_types"`
Limit int `json:"limit"`
TransitionWords string `json:"transition_words,omitempty"`
} }
type HardwareAgentCreateResponse struct { type HardwareAgentCreateResponse struct {
+2 -2
View File
@@ -21,8 +21,8 @@ import (
const AssistantID = "platform_companion_v1" const AssistantID = "platform_companion_v1"
type Options struct { type Options struct {
BaseURL, ResourceID, APIKey, EventType, ProfileType string BaseURL, ResourceID, CollectionName, APIKey, EventType, ProfileType string
Timeout time.Duration Timeout time.Duration
} }
type Client struct { type Client struct {
options Options options Options
+36
View File
@@ -27,6 +27,15 @@ type Service struct {
client *Client client *Client
} }
// NativeAccess contains only non-secret values that Volcano needs to query
// this user's memory. The Viking API key is never returned to the device.
type NativeAccess struct {
SubjectID, UserID, CollectionName, AssistantID string
MemoryTypes []string
Limit int
TransitionWords string
}
func NewService(pool *pgxpool.Pool, client *Client) *Service { func NewService(pool *pgxpool.Pool, client *Client) *Service {
return &Service{pool: pool, client: client} return &Service{pool: pool, client: client}
} }
@@ -71,6 +80,33 @@ func (s *Service) PrepareForDevice(ctx context.Context, tenant, device string) (
} }
return subject, strings.Join(parts, "\n"), nil return subject, strings.Join(parts, "\n"), nil
} }
func (s *Service) NativeAccessForDevice(ctx context.Context, tenant, device string) (*NativeAccess, error) {
if strings.TrimSpace(s.client.options.CollectionName) == "" {
return nil, nil
}
var subject string
err := s.pool.QueryRow(ctx, `SELECT b.subject_id::text FROM device_binding b JOIN device d ON d.id=b.device_id AND d.tenant_id=b.tenant_id WHERE b.device_id=$1::uuid AND b.tenant_id=$2::uuid AND b.unbound_at IS NULL AND d.status='registered'`, device, tenant).Scan(&subject)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
ok, err := s.Eligible(ctx, tenant, subject)
if err != nil || !ok {
return nil, err
}
memoryTypes := []string{s.client.options.EventType}
if s.client.options.ProfileType != "" && s.client.options.ProfileType != s.client.options.EventType {
memoryTypes = append(memoryTypes, s.client.options.ProfileType)
}
return &NativeAccess{
SubjectID: subject, UserID: UserID(tenant, subject), CollectionName: s.client.options.CollectionName,
AssistantID: AssistantID, MemoryTypes: memoryTypes, Limit: 5,
TransitionWords: "可参考的用户历史记忆如下,请自然承接;若与用户最新说法冲突,以最新说法为准:",
}, nil
}
func (s *Service) Enqueue(ctx context.Context, event outbox.Event) error { func (s *Service) Enqueue(ctx context.Context, event outbox.Event) error {
tx, err := s.pool.Begin(ctx) tx, err := s.pool.Begin(ctx)
if err != nil { if err != nil {