feat: expand companion platform, device controls, and memoir workflows
ci / verify (push) Canceled after 0s
ci / verify (push) Canceled after 0s
Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
This commit is contained in:
Codex Local
@@ -0,0 +1,396 @@
|
||||
// Direct chat: fetch parameters, talk to Volcano, report end. Subtitles come from Volcano.
|
||||
#include "call_client.h"
|
||||
#include "firmware_version.h"
|
||||
#include "../components/volc_conv_ai/src/base/volc_device_manager.h"
|
||||
#include "ble_cloud.h"
|
||||
#include "cJSON.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_timer.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/semphr.h"
|
||||
#include "freertos/task.h"
|
||||
#include "mbedtls/platform_util.h"
|
||||
#include "mbedtls/sha256.h"
|
||||
#include "nvs.h"
|
||||
#include <stdatomic.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
#define AUDIO_PROFILE "echoear-pcm16-16k-v1"
|
||||
#define FIRMWARE DEVICE_FIRMWARE_VERSION
|
||||
#define QUEUE_SIZE 12
|
||||
typedef struct {
|
||||
unsigned version;
|
||||
int64_t binding, config;
|
||||
platform_volc_credential_t credential;
|
||||
char activation[37], attempt[37], result_id[37], rtc_app[129];
|
||||
int pending; // 1=contact possible; 2=successful registration saved; 3=unknown
|
||||
} voice_store_t;
|
||||
typedef struct {
|
||||
unsigned version;
|
||||
char session[37], boot[37], end_id[37], reason[24];
|
||||
bool finished, failed, sdk_stopped;
|
||||
} call_store_t;
|
||||
static voice_store_t voice;
|
||||
static call_store_t call;
|
||||
static atomic_bool audio_allowed;
|
||||
static atomic_uint lease_deadline;
|
||||
static char *params;
|
||||
static char bot[201];
|
||||
static bool boot_loaded, retry_started, engine_ready, sdk_initialized, sdk_stopped;
|
||||
static bool maintenance;
|
||||
static atomic_bool standby_mode;
|
||||
static int64_t renew_due;
|
||||
static StaticSemaphore_t state_mutex_storage;
|
||||
static SemaphoreHandle_t state_mutex;
|
||||
static const char *TAG = "call_client";
|
||||
static char start_error[80];
|
||||
const char *call_client_start_error(void) { return start_error; }
|
||||
static void retry_task(void *unused);
|
||||
static void finish_pending(void);
|
||||
|
||||
static const char *str(const cJSON *j, const char *name) {
|
||||
const cJSON *v = cJSON_GetObjectItemCaseSensitive(j, name);
|
||||
return cJSON_IsString(v) ? v->valuestring : "";
|
||||
}
|
||||
static int64_t num(const cJSON *j, const char *name) {
|
||||
const cJSON *v = cJSON_GetObjectItemCaseSensitive(j, name);
|
||||
return cJSON_IsNumber(v) ? (int64_t)v->valuedouble : -1;
|
||||
}
|
||||
static int64_t wall(void) {
|
||||
struct timeval t;
|
||||
gettimeofday(&t, NULL);
|
||||
return (int64_t)t.tv_sec * 1000 + t.tv_usec / 1000;
|
||||
}
|
||||
static int64_t mono(void) { return esp_timer_get_time() / 1000; }
|
||||
static esp_err_t persist(const char *key, const void *data, size_t len) {
|
||||
nvs_handle_t n;
|
||||
esp_err_t e = nvs_open("voice", NVS_READWRITE, &n);
|
||||
if (e != ESP_OK)
|
||||
return e;
|
||||
e = data ? nvs_set_blob(n, key, data, len) : nvs_erase_key(n, key);
|
||||
if (e == ESP_ERR_NVS_NOT_FOUND)
|
||||
e = ESP_OK;
|
||||
if (e == ESP_OK)
|
||||
e = nvs_commit(n);
|
||||
nvs_close(n);
|
||||
return e;
|
||||
}
|
||||
static void restore(const char *key, void *data, size_t len) {
|
||||
nvs_handle_t n;
|
||||
if (nvs_open("voice", NVS_READONLY, &n) != ESP_OK)
|
||||
return;
|
||||
size_t size = len;
|
||||
if (nvs_get_blob(n, key, data, &size) != ESP_OK || size != len)
|
||||
memset(data, 0, len);
|
||||
nvs_close(n);
|
||||
}
|
||||
// Save the five fields from ble-bind before reporting BLE success. Boot still
|
||||
// obtains the current binding/config metadata for SDK initialization and ACK.
|
||||
esp_err_t call_client_save_bind_voice(const cJSON *data) {
|
||||
voice_store_t saved = {.version=1};
|
||||
const char *instance=str(data,"instance_id"), *product=str(data,"product_key"),
|
||||
*name=str(data,"device_name"), *secret=str(data,"device_secret"), *app=str(data,"RTCAppID");
|
||||
if (!instance[0] || strlen(instance)>=sizeof(saved.credential.instance_id) ||
|
||||
!product[0] || strlen(product)>=sizeof(saved.credential.product_key) ||
|
||||
!name[0] || strlen(name)>=sizeof(saved.credential.device_name) ||
|
||||
strlen(secret)<16 || strlen(secret)>=sizeof(saved.credential.device_secret) ||
|
||||
!app[0] || strlen(app)>=sizeof(saved.rtc_app)) return ESP_ERR_INVALID_RESPONSE;
|
||||
snprintf(saved.credential.instance_id,sizeof(saved.credential.instance_id),"%s",instance);
|
||||
snprintf(saved.credential.product_key,sizeof(saved.credential.product_key),"%s",product);
|
||||
snprintf(saved.credential.device_name,sizeof(saved.credential.device_name),"%s",name);
|
||||
snprintf(saved.credential.device_secret,sizeof(saved.credential.device_secret),"%s",secret);
|
||||
snprintf(saved.rtc_app,sizeof(saved.rtc_app),"%s",app);
|
||||
esp_err_t err=persist("credential",&saved,sizeof(saved));
|
||||
mbedtls_platform_zeroize(&saved,sizeof(saved));
|
||||
return err;
|
||||
}
|
||||
static esp_err_t post(const char *path, cJSON *body, const char *rid, cJSON **out) {
|
||||
char *raw = cJSON_PrintUnformatted(body), *response = NULL;
|
||||
if (!raw)
|
||||
return ESP_ERR_NO_MEM;
|
||||
esp_err_t e = platform_request("POST", path, raw, rid, &response);
|
||||
// An idle keep-alive connection may have been closed by the gateway. Chat
|
||||
// requests are keyed by session_id, so retry once on a transport failure.
|
||||
if (e != ESP_OK && !platform_last_error_code()[0] && strstr(path,"/device/chat/")) {
|
||||
if (response) { mbedtls_platform_zeroize(response,strlen(response)); free(response); response=NULL; }
|
||||
e = platform_request("POST",path,raw,rid,&response);
|
||||
}
|
||||
mbedtls_platform_zeroize(raw,strlen(raw));
|
||||
free(raw);
|
||||
if (e == ESP_OK) {
|
||||
cJSON *root = cJSON_Parse(response), *data = cJSON_GetObjectItemCaseSensitive(root, "data");
|
||||
if (!cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(root, "ok")) || !cJSON_IsObject(data))
|
||||
e = ESP_ERR_INVALID_RESPONSE;
|
||||
else if (out) {
|
||||
*out = cJSON_Duplicate(data, true);
|
||||
cJSON *server_time = cJSON_GetObjectItemCaseSensitive(root,"server_time_ms");
|
||||
if (*out && cJSON_IsNumber(server_time))
|
||||
cJSON_AddNumberToObject(*out,"_server_time_ms",server_time->valuedouble);
|
||||
}
|
||||
cJSON_Delete(root);
|
||||
}
|
||||
if (response) {
|
||||
mbedtls_platform_zeroize(response, strlen(response));
|
||||
free(response);
|
||||
}
|
||||
return e;
|
||||
}
|
||||
const platform_volc_credential_t *call_client_credential(void) { return &voice.credential; }
|
||||
const char *call_client_rtc_app_id(void) { return voice.rtc_app; }
|
||||
|
||||
static bool saved_voice_valid(void) {
|
||||
return voice.version == 1 &&
|
||||
memchr(voice.credential.instance_id, 0, sizeof(voice.credential.instance_id)) &&
|
||||
memchr(voice.credential.product_key, 0, sizeof(voice.credential.product_key)) &&
|
||||
memchr(voice.credential.device_name, 0, sizeof(voice.credential.device_name)) &&
|
||||
memchr(voice.credential.device_secret, 0, sizeof(voice.credential.device_secret)) &&
|
||||
memchr(voice.rtc_app, 0, sizeof(voice.rtc_app)) &&
|
||||
voice.credential.instance_id[0] && voice.credential.product_key[0] &&
|
||||
voice.credential.device_name[0] && strlen(voice.credential.device_secret) >= 16 &&
|
||||
voice.rtc_app[0];
|
||||
}
|
||||
|
||||
esp_err_t call_client_boot(void) {
|
||||
start_error[0] = 0;
|
||||
if (!state_mutex) state_mutex = xSemaphoreCreateMutexStatic(&state_mutex_storage);
|
||||
if (!state_mutex) return ESP_ERR_NO_MEM;
|
||||
atomic_store(&audio_allowed, false);
|
||||
if (!boot_loaded) {
|
||||
restore("credential", &voice, sizeof(voice));
|
||||
restore("direct_call", &call, sizeof(call));
|
||||
if (call.version != 2 || !memchr(call.session,0,sizeof(call.session)) ||
|
||||
!memchr(call.end_id,0,sizeof(call.end_id))) memset(&call,0,sizeof(call));
|
||||
if (call.session[0]) {
|
||||
call.finished = true; call.failed = true; call.sdk_stopped = false;
|
||||
snprintf(call.reason,sizeof(call.reason),"reboot");
|
||||
}
|
||||
boot_loaded = true;
|
||||
}
|
||||
if (ble_cloud_sync_time() != ESP_OK) return ESP_FAIL;
|
||||
if (call.finished) {
|
||||
finish_pending();
|
||||
if (call.session[0]) return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
// A retained credential is not proof of a current household/elder binding.
|
||||
cJSON *j=cJSON_CreateObject(), *data=NULL;
|
||||
esp_err_t e=post("/api/v1/device/ble-state",j,NULL,&data);
|
||||
cJSON_Delete(j);
|
||||
if (e!=ESP_OK) { cJSON_Delete(data); return e; }
|
||||
bool bound=cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(data,"bound"));
|
||||
cJSON_Delete(data); data=NULL;
|
||||
if (!bound) { snprintf(start_error,sizeof(start_error),"DEVICE_UNBOUND"); return ESP_ERR_INVALID_STATE; }
|
||||
// Normal boot uses the five fields saved from ble-bind. Recovery only.
|
||||
if (!saved_voice_valid()) {
|
||||
j=cJSON_CreateObject();
|
||||
e=post("/api/v1/device/voice-config",j,NULL,&data); cJSON_Delete(j);
|
||||
if (e==ESP_OK && !strcmp(str(data,"voice_status"),"ready")) {
|
||||
e=call_client_save_bind_voice(cJSON_GetObjectItemCaseSensitive(data,"voice"));
|
||||
} else if(e==ESP_OK) {
|
||||
snprintf(start_error,sizeof(start_error),"%s",
|
||||
!strcmp(str(data,"voice_status"),"preparing") ? "VOICE_PREPARING" : "VOICE_RECOVERY_REQUIRED");
|
||||
e=ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
cJSON *secret=cJSON_GetObjectItemCaseSensitive(cJSON_GetObjectItemCaseSensitive(data,"voice"),"device_secret");
|
||||
if(cJSON_IsString(secret))mbedtls_platform_zeroize(secret->valuestring,strlen(secret->valuestring));
|
||||
cJSON_Delete(data);
|
||||
if(e!=ESP_OK)return e;
|
||||
}
|
||||
if (!retry_started) {
|
||||
if(xTaskCreate(retry_task,"call_sync",8192,NULL,4,NULL)!=pdPASS)return ESP_ERR_NO_MEM;
|
||||
retry_started=true;
|
||||
}
|
||||
ESP_LOGI(TAG,"Direct chat ready; firmware=%s credentials=platform-issued",FIRMWARE);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
void call_client_engine_ready(bool initialized) {
|
||||
sdk_initialized=initialized;
|
||||
engine_ready=initialized; // Local SDK initialization, no platform activation ACK.
|
||||
}
|
||||
bool call_client_audio_allowed(void) {
|
||||
if (!atomic_load(&audio_allowed)) return false;
|
||||
if ((int32_t)(atomic_load(&lease_deadline) - (uint32_t)mono()) <= 0) {
|
||||
atomic_store(&audio_allowed, false);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool call_client_is_engine_ready(void) { return engine_ready; }
|
||||
const char *call_client_bot_id(void) { return bot; }
|
||||
const char *call_client_params(void) { return params ? params : "{}"; }
|
||||
// No device transcript queue or event upload: provider callbacks own records.
|
||||
void call_client_observe(const char *type) { (void)type; }
|
||||
int volc_platform_recording_prepare(const char *app, const char *room, const char *user) {
|
||||
if (!app || !room || !user || strlen(app)>128 || strlen(room)>128 || strlen(user)>128 || !state_mutex) return ESP_ERR_INVALID_ARG;
|
||||
char session[37]={0};
|
||||
xSemaphoreTake(state_mutex,portMAX_DELAY);
|
||||
if (call.session[0] && !call.finished && call_client_audio_allowed()) snprintf(session,sizeof(session),"%s",call.session);
|
||||
xSemaphoreGive(state_mutex);
|
||||
if (!session[0]) return ESP_ERR_INVALID_STATE;
|
||||
cJSON *body=cJSON_CreateObject(), *data=NULL;
|
||||
if (!body) return ESP_ERR_NO_MEM;
|
||||
cJSON_AddStringToObject(body,"session_id",session);
|
||||
cJSON_AddStringToObject(body,"device_id",ble_factory_id());
|
||||
cJSON_AddStringToObject(body,"rtc_app_id",app);
|
||||
cJSON_AddStringToObject(body,"room_id",room);
|
||||
cJSON_AddStringToObject(body,"rtc_user_id",user);
|
||||
esp_err_t result=post("/api/v1/device/chat/recording",body,NULL,&data);
|
||||
ESP_LOGI(TAG,"Cloud conversation recording: %s",result==ESP_OK ? str(data,"recording") : "unavailable");
|
||||
cJSON_Delete(data);cJSON_Delete(body);
|
||||
return result;
|
||||
}
|
||||
esp_err_t call_client_turn(const char *speaker, const char *id, const char *text) {
|
||||
(void)speaker; (void)id; (void)text; return ESP_OK;
|
||||
}
|
||||
static esp_err_t fetch_parameters(bool initial) {
|
||||
int64_t requested_at = mono();
|
||||
cJSON *j = cJSON_CreateObject(), *data = NULL;
|
||||
cJSON_AddStringToObject(j, "session_id", call.session);
|
||||
cJSON_AddStringToObject(j, "device_id", ble_factory_id());
|
||||
cJSON_AddStringToObject(j, "boot_id", call.boot);
|
||||
esp_err_t e = post("/api/v1/device/chat/start", j, NULL, &data);
|
||||
cJSON_Delete(j);
|
||||
// A slow reply cannot reopen an expired local authorization.
|
||||
if (!initial && !call_client_audio_allowed()) e = ESP_ERR_TIMEOUT;
|
||||
if (e == ESP_OK) {
|
||||
// Compare server timestamps with each other: boot-time clock skew must
|
||||
// not reject a valid 45-second lease. Subtract the full request duration.
|
||||
int64_t server_now = num(data,"_server_time_ms");
|
||||
int64_t lease = num(data,"lease_expires_at_ms") - (server_now > 0 ? server_now : wall());
|
||||
if (server_now > 0) lease -= mono() - requested_at;
|
||||
cJSON *p = cJSON_GetObjectItemCaseSensitive(data, "params");
|
||||
if (strcmp(str(data,"session_id"),call.session) || !cJSON_IsObject(p) ||
|
||||
lease <= 0 || lease > 45000 || num(data,"expires_at_ms") <= (server_now > 0 ? server_now : wall()) ||
|
||||
!str(data,"bot_id")[0]) e = ESP_ERR_INVALID_RESPONSE;
|
||||
else {
|
||||
if (initial) {
|
||||
char *encoded = cJSON_PrintUnformatted(p);
|
||||
if (!encoded || strlen(encoded) > 16384) {
|
||||
free(encoded); e = ESP_ERR_NO_MEM;
|
||||
} else {
|
||||
free(params); params = encoded;
|
||||
snprintf(bot, sizeof(bot), "%s", str(data,"bot_id"));
|
||||
}
|
||||
}
|
||||
if (e == ESP_OK) {
|
||||
atomic_store(&lease_deadline, (uint32_t)(mono() + lease));
|
||||
atomic_store(&audio_allowed, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
cJSON_Delete(data);
|
||||
renew_due = mono() + 10000;
|
||||
if (e != ESP_OK && initial) atomic_store(&audio_allowed, false);
|
||||
return e;
|
||||
}
|
||||
static void finish_pending(void) {
|
||||
if (!call.session[0] || !call.finished) return;
|
||||
atomic_store(&audio_allowed, false);
|
||||
cJSON *j = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(j,"session_id",call.session);
|
||||
cJSON_AddStringToObject(j,"device_id",ble_factory_id());
|
||||
cJSON_AddStringToObject(j,"reason",call.reason[0] ? call.reason : "disconnected");
|
||||
cJSON_AddBoolToObject(j,"sdk_stopped",call.sdk_stopped);
|
||||
esp_err_t e = post("/api/v1/device/chat/end",j,call.end_id,NULL);
|
||||
cJSON_Delete(j);
|
||||
if (e == ESP_OK && persist("direct_call",NULL,0) == ESP_OK) memset(&call,0,sizeof(call));
|
||||
if (params) {
|
||||
mbedtls_platform_zeroize(params,strlen(params)); free(params); params=NULL;
|
||||
}
|
||||
ESP_LOGI(TAG,"chat end acknowledged=%d",e == ESP_OK);
|
||||
}
|
||||
esp_err_t call_client_begin(void) {
|
||||
start_error[0] = '\0';
|
||||
if (!state_mutex) return ESP_ERR_INVALID_STATE;
|
||||
if (!engine_ready && sdk_initialized) call_client_engine_ready(true);
|
||||
if (!engine_ready) return ESP_ERR_INVALID_STATE;
|
||||
xSemaphoreTake(state_mutex,portMAX_DELAY);
|
||||
if (call.finished) finish_pending();
|
||||
if (maintenance || atomic_load(&standby_mode) || call.session[0]) { xSemaphoreGive(state_mutex); return ESP_ERR_INVALID_STATE; }
|
||||
memset(&call,0,sizeof(call)); call.version=2;
|
||||
ble_random_uuid(call.session); ble_random_uuid(call.end_id);
|
||||
snprintf(call.boot,sizeof(call.boot),"%s",ble_cloud_boot_id());
|
||||
sdk_stopped=false;
|
||||
esp_err_t e = persist("direct_call",&call,sizeof(call));
|
||||
if (e == ESP_OK) e=fetch_parameters(true);
|
||||
if (e != ESP_OK) {
|
||||
snprintf(start_error,sizeof(start_error),"%s",platform_last_error_code());
|
||||
call.finished=true; call.failed=true;
|
||||
snprintf(call.reason,sizeof(call.reason),"start_failed");
|
||||
persist("direct_call",&call,sizeof(call)); finish_pending();
|
||||
}
|
||||
xSemaphoreGive(state_mutex); return e;
|
||||
}
|
||||
int call_client_poll(void) {
|
||||
if (!call_client_audio_allowed()) return 1;
|
||||
xSemaphoreTake(state_mutex,portMAX_DELAY);
|
||||
if (mono() >= renew_due) {
|
||||
esp_err_t e=fetch_parameters(false);
|
||||
// A known refusal is terminal. Transient transport failures get only
|
||||
// the remaining previously granted time, never a new local lease.
|
||||
if (e != ESP_OK) {
|
||||
const char *code=platform_last_error_code();
|
||||
if (code && (!strcmp(code,"CHAT_ENDED") || !strcmp(code,"DEVICE_BINDING_CHANGED") ||
|
||||
!strcmp(code,"VOICE_CHAT_CONSENT_REQUIRED") || !strcmp(code,"DEVICE_NOT_READY") ||
|
||||
!strcmp(code,"MEMORY_CONSENT_REVOKED") || !strcmp(code,"DEVICE_UNBOUND") ||
|
||||
!strcmp(code,"VOICE_AGENT_NOT_READY"))) atomic_store(&audio_allowed,false);
|
||||
}
|
||||
}
|
||||
int stopped=!call_client_audio_allowed();
|
||||
xSemaphoreGive(state_mutex); return stopped;
|
||||
}
|
||||
void call_client_stop_result(bool started,int result) { sdk_stopped=started && result==0; }
|
||||
void call_client_finish(bool failed) {
|
||||
atomic_store(&audio_allowed,false);
|
||||
if (!state_mutex) return;
|
||||
xSemaphoreTake(state_mutex,portMAX_DELAY);
|
||||
if (call.session[0] && !call.finished) {
|
||||
call.finished=true; call.failed=failed; call.sdk_stopped=sdk_stopped;
|
||||
snprintf(call.reason,sizeof(call.reason),"%s",failed ? "disconnected" : "hangup");
|
||||
persist("direct_call",&call,sizeof(call));
|
||||
}
|
||||
finish_pending();
|
||||
xSemaphoreGive(state_mutex);
|
||||
}
|
||||
static void retry_task(void *unused) {
|
||||
(void)unused;
|
||||
for (;;) {
|
||||
vTaskDelay(pdMS_TO_TICKS(10000));
|
||||
if (xSemaphoreTake(state_mutex,0)==pdTRUE) {
|
||||
if (call.finished) finish_pending();
|
||||
xSemaphoreGive(state_mutex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool call_client_busy(void) {
|
||||
if (!state_mutex || xSemaphoreTake(state_mutex,0)!=pdTRUE) return true;
|
||||
bool busy=maintenance || call.session[0]!=0;
|
||||
xSemaphoreGive(state_mutex);
|
||||
return busy;
|
||||
}
|
||||
bool call_client_standby(void) { return atomic_load(&standby_mode); }
|
||||
esp_err_t call_client_set_standby(bool standby) {
|
||||
if (!state_mutex) { atomic_store(&standby_mode,standby); return ESP_OK; }
|
||||
xSemaphoreTake(state_mutex,portMAX_DELAY);
|
||||
if (call.session[0] || !maintenance) { xSemaphoreGive(state_mutex); return ESP_ERR_INVALID_STATE; }
|
||||
atomic_store(&standby_mode,standby);
|
||||
xSemaphoreGive(state_mutex);
|
||||
return ESP_OK;
|
||||
}
|
||||
bool call_client_enter_maintenance(void) {
|
||||
if (!state_mutex || xSemaphoreTake(state_mutex,0)!=pdTRUE) return false;
|
||||
bool allowed=!maintenance && !call.session[0];
|
||||
if (allowed) maintenance=true;
|
||||
xSemaphoreGive(state_mutex);
|
||||
return allowed;
|
||||
}
|
||||
void call_client_leave_maintenance(void) {
|
||||
if (!state_mutex) return;
|
||||
xSemaphoreTake(state_mutex,portMAX_DELAY);
|
||||
maintenance=false;
|
||||
xSemaphoreGive(state_mutex);
|
||||
}
|
||||
Reference in New Issue
Block a user