// Direct chat: fetch parameters, talk to Volcano, report end. Subtitles come from Volcano. #include "call_client.h" #include "firmware_version.h" #include "../components/volc_conv_ai/src/base/volc_device_manager.h" #include "ble_cloud.h" #include "cJSON.h" #include "esp_log.h" #include "esp_timer.h" #include "freertos/FreeRTOS.h" #include "freertos/semphr.h" #include "freertos/task.h" #include "mbedtls/platform_util.h" #include "mbedtls/sha256.h" #include "nvs.h" #include #include #include #include #include #define AUDIO_PROFILE "echoear-pcm16-16k-v1" #define FIRMWARE DEVICE_FIRMWARE_VERSION #define QUEUE_SIZE 12 typedef struct { unsigned version; int64_t binding, config; platform_volc_credential_t credential; char activation[37], attempt[37], result_id[37], rtc_app[129]; int pending; // 1=contact possible; 2=successful registration saved; 3=unknown } voice_store_t; typedef struct { unsigned version; char session[37], boot[37], end_id[37], reason[24]; bool finished, failed, sdk_stopped; } call_store_t; static voice_store_t voice; static call_store_t call; static atomic_bool audio_allowed; static atomic_uint lease_deadline; static char *params; static char bot[201]; static bool boot_loaded, retry_started, engine_ready, sdk_initialized, sdk_stopped; static bool maintenance; static atomic_bool standby_mode; static int64_t renew_due; static StaticSemaphore_t state_mutex_storage; static SemaphoreHandle_t state_mutex; static const char *TAG = "call_client"; static char start_error[80]; const char *call_client_start_error(void) { return start_error; } static void retry_task(void *unused); static void finish_pending(void); static const char *str(const cJSON *j, const char *name) { const cJSON *v = cJSON_GetObjectItemCaseSensitive(j, name); return cJSON_IsString(v) ? v->valuestring : ""; } static int64_t num(const cJSON *j, const char *name) { const cJSON *v = cJSON_GetObjectItemCaseSensitive(j, name); return cJSON_IsNumber(v) ? (int64_t)v->valuedouble : -1; } static int64_t wall(void) { struct timeval t; gettimeofday(&t, NULL); return (int64_t)t.tv_sec * 1000 + t.tv_usec / 1000; } static int64_t mono(void) { return esp_timer_get_time() / 1000; } static esp_err_t persist(const char *key, const void *data, size_t len) { nvs_handle_t n; esp_err_t e = nvs_open("voice", NVS_READWRITE, &n); if (e != ESP_OK) return e; e = data ? nvs_set_blob(n, key, data, len) : nvs_erase_key(n, key); if (e == ESP_ERR_NVS_NOT_FOUND) e = ESP_OK; if (e == ESP_OK) e = nvs_commit(n); nvs_close(n); return e; } static void restore(const char *key, void *data, size_t len) { nvs_handle_t n; if (nvs_open("voice", NVS_READONLY, &n) != ESP_OK) return; size_t size = len; if (nvs_get_blob(n, key, data, &size) != ESP_OK || size != len) memset(data, 0, len); nvs_close(n); } // Save the five fields from ble-bind before reporting BLE success. Boot still // obtains the current binding/config metadata for SDK initialization and ACK. esp_err_t call_client_save_bind_voice(const cJSON *data) { voice_store_t saved = {.version=1}; const char *instance=str(data,"instance_id"), *product=str(data,"product_key"), *name=str(data,"device_name"), *secret=str(data,"device_secret"), *app=str(data,"RTCAppID"); if (!instance[0] || strlen(instance)>=sizeof(saved.credential.instance_id) || !product[0] || strlen(product)>=sizeof(saved.credential.product_key) || !name[0] || strlen(name)>=sizeof(saved.credential.device_name) || strlen(secret)<16 || strlen(secret)>=sizeof(saved.credential.device_secret) || !app[0] || strlen(app)>=sizeof(saved.rtc_app)) return ESP_ERR_INVALID_RESPONSE; snprintf(saved.credential.instance_id,sizeof(saved.credential.instance_id),"%s",instance); snprintf(saved.credential.product_key,sizeof(saved.credential.product_key),"%s",product); snprintf(saved.credential.device_name,sizeof(saved.credential.device_name),"%s",name); snprintf(saved.credential.device_secret,sizeof(saved.credential.device_secret),"%s",secret); snprintf(saved.rtc_app,sizeof(saved.rtc_app),"%s",app); esp_err_t err=persist("credential",&saved,sizeof(saved)); mbedtls_platform_zeroize(&saved,sizeof(saved)); return err; } static esp_err_t post(const char *path, cJSON *body, const char *rid, cJSON **out) { char *raw = cJSON_PrintUnformatted(body), *response = NULL; if (!raw) return ESP_ERR_NO_MEM; esp_err_t e = platform_request("POST", path, raw, rid, &response); // An idle keep-alive connection may have been closed by the gateway. Chat // requests are keyed by session_id, so retry once on a transport failure. if (e != ESP_OK && !platform_last_error_code()[0] && strstr(path,"/device/chat/")) { if (response) { mbedtls_platform_zeroize(response,strlen(response)); free(response); response=NULL; } e = platform_request("POST",path,raw,rid,&response); } mbedtls_platform_zeroize(raw,strlen(raw)); free(raw); if (e == ESP_OK) { cJSON *root = cJSON_Parse(response), *data = cJSON_GetObjectItemCaseSensitive(root, "data"); if (!cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(root, "ok")) || !cJSON_IsObject(data)) e = ESP_ERR_INVALID_RESPONSE; else if (out) { *out = cJSON_Duplicate(data, true); cJSON *server_time = cJSON_GetObjectItemCaseSensitive(root,"server_time_ms"); if (*out && cJSON_IsNumber(server_time)) cJSON_AddNumberToObject(*out,"_server_time_ms",server_time->valuedouble); } cJSON_Delete(root); } if (response) { mbedtls_platform_zeroize(response, strlen(response)); free(response); } return e; } const platform_volc_credential_t *call_client_credential(void) { return &voice.credential; } const char *call_client_rtc_app_id(void) { return voice.rtc_app; } static bool saved_voice_valid(void) { return voice.version == 1 && memchr(voice.credential.instance_id, 0, sizeof(voice.credential.instance_id)) && memchr(voice.credential.product_key, 0, sizeof(voice.credential.product_key)) && memchr(voice.credential.device_name, 0, sizeof(voice.credential.device_name)) && memchr(voice.credential.device_secret, 0, sizeof(voice.credential.device_secret)) && memchr(voice.rtc_app, 0, sizeof(voice.rtc_app)) && voice.credential.instance_id[0] && voice.credential.product_key[0] && voice.credential.device_name[0] && strlen(voice.credential.device_secret) >= 16 && voice.rtc_app[0]; } esp_err_t call_client_boot(void) { start_error[0] = 0; if (!state_mutex) state_mutex = xSemaphoreCreateMutexStatic(&state_mutex_storage); if (!state_mutex) return ESP_ERR_NO_MEM; atomic_store(&audio_allowed, false); if (!boot_loaded) { restore("credential", &voice, sizeof(voice)); restore("direct_call", &call, sizeof(call)); if (call.version != 2 || !memchr(call.session,0,sizeof(call.session)) || !memchr(call.end_id,0,sizeof(call.end_id))) memset(&call,0,sizeof(call)); if (call.session[0]) { call.finished = true; call.failed = true; call.sdk_stopped = false; snprintf(call.reason,sizeof(call.reason),"reboot"); } boot_loaded = true; } if (ble_cloud_sync_time() != ESP_OK) return ESP_FAIL; if (call.finished) { finish_pending(); if (call.session[0]) return ESP_ERR_INVALID_STATE; } // A retained credential is not proof of a current household/elder binding. cJSON *j=cJSON_CreateObject(), *data=NULL; esp_err_t e=post("/api/v1/device/ble-state",j,NULL,&data); cJSON_Delete(j); if (e!=ESP_OK) { cJSON_Delete(data); return e; } bool bound=cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(data,"bound")); cJSON_Delete(data); data=NULL; if (!bound) { snprintf(start_error,sizeof(start_error),"DEVICE_UNBOUND"); return ESP_ERR_INVALID_STATE; } // Normal boot uses the five fields saved from ble-bind. Recovery only. if (!saved_voice_valid()) { j=cJSON_CreateObject(); e=post("/api/v1/device/voice-config",j,NULL,&data); cJSON_Delete(j); if (e==ESP_OK && !strcmp(str(data,"voice_status"),"ready")) { e=call_client_save_bind_voice(cJSON_GetObjectItemCaseSensitive(data,"voice")); } else if(e==ESP_OK) { snprintf(start_error,sizeof(start_error),"%s", !strcmp(str(data,"voice_status"),"preparing") ? "VOICE_PREPARING" : "VOICE_RECOVERY_REQUIRED"); e=ESP_ERR_INVALID_STATE; } cJSON *secret=cJSON_GetObjectItemCaseSensitive(cJSON_GetObjectItemCaseSensitive(data,"voice"),"device_secret"); if(cJSON_IsString(secret))mbedtls_platform_zeroize(secret->valuestring,strlen(secret->valuestring)); cJSON_Delete(data); if(e!=ESP_OK)return e; } if (!retry_started) { if(xTaskCreate(retry_task,"call_sync",8192,NULL,4,NULL)!=pdPASS)return ESP_ERR_NO_MEM; retry_started=true; } ESP_LOGI(TAG,"Direct chat ready; firmware=%s credentials=platform-issued",FIRMWARE); return ESP_OK; } void call_client_engine_ready(bool initialized) { sdk_initialized=initialized; engine_ready=initialized; // Local SDK initialization, no platform activation ACK. } bool call_client_audio_allowed(void) { if (!atomic_load(&audio_allowed)) return false; if ((int32_t)(atomic_load(&lease_deadline) - (uint32_t)mono()) <= 0) { atomic_store(&audio_allowed, false); return false; } return true; } bool call_client_is_engine_ready(void) { return engine_ready; } const char *call_client_bot_id(void) { return bot; } const char *call_client_params(void) { return params ? params : "{}"; } // No device transcript queue or event upload: provider callbacks own records. void call_client_observe(const char *type) { (void)type; } int volc_platform_recording_prepare(const char *app, const char *room, const char *user) { if (!app || !room || !user || strlen(app)>128 || strlen(room)>128 || strlen(user)>128 || !state_mutex) return ESP_ERR_INVALID_ARG; char session[37]={0}; xSemaphoreTake(state_mutex,portMAX_DELAY); if (call.session[0] && !call.finished && call_client_audio_allowed()) snprintf(session,sizeof(session),"%s",call.session); xSemaphoreGive(state_mutex); if (!session[0]) return ESP_ERR_INVALID_STATE; cJSON *body=cJSON_CreateObject(), *data=NULL; if (!body) return ESP_ERR_NO_MEM; cJSON_AddStringToObject(body,"session_id",session); cJSON_AddStringToObject(body,"device_id",ble_factory_id()); cJSON_AddStringToObject(body,"rtc_app_id",app); cJSON_AddStringToObject(body,"room_id",room); cJSON_AddStringToObject(body,"rtc_user_id",user); esp_err_t result=post("/api/v1/device/chat/recording",body,NULL,&data); ESP_LOGI(TAG,"Cloud conversation recording: %s",result==ESP_OK ? str(data,"recording") : "unavailable"); cJSON_Delete(data);cJSON_Delete(body); return result; } esp_err_t call_client_turn(const char *speaker, const char *id, const char *text) { (void)speaker; (void)id; (void)text; return ESP_OK; } static esp_err_t fetch_parameters(bool initial) { int64_t requested_at = mono(); cJSON *j = cJSON_CreateObject(), *data = NULL; cJSON_AddStringToObject(j, "session_id", call.session); cJSON_AddStringToObject(j, "device_id", ble_factory_id()); cJSON_AddStringToObject(j, "boot_id", call.boot); esp_err_t e = post("/api/v1/device/chat/start", j, NULL, &data); cJSON_Delete(j); // A slow reply cannot reopen an expired local authorization. if (!initial && !call_client_audio_allowed()) e = ESP_ERR_TIMEOUT; if (e == ESP_OK) { // Compare server timestamps with each other: boot-time clock skew must // not reject a valid 45-second lease. Subtract the full request duration. int64_t server_now = num(data,"_server_time_ms"); int64_t lease = num(data,"lease_expires_at_ms") - (server_now > 0 ? server_now : wall()); if (server_now > 0) lease -= mono() - requested_at; cJSON *p = cJSON_GetObjectItemCaseSensitive(data, "params"); if (strcmp(str(data,"session_id"),call.session) || !cJSON_IsObject(p) || lease <= 0 || lease > 45000 || num(data,"expires_at_ms") <= (server_now > 0 ? server_now : wall()) || !str(data,"bot_id")[0]) e = ESP_ERR_INVALID_RESPONSE; else { if (initial) { char *encoded = cJSON_PrintUnformatted(p); if (!encoded || strlen(encoded) > 16384) { free(encoded); e = ESP_ERR_NO_MEM; } else { free(params); params = encoded; snprintf(bot, sizeof(bot), "%s", str(data,"bot_id")); } } if (e == ESP_OK) { atomic_store(&lease_deadline, (uint32_t)(mono() + lease)); atomic_store(&audio_allowed, true); } } } cJSON_Delete(data); renew_due = mono() + 10000; if (e != ESP_OK && initial) atomic_store(&audio_allowed, false); return e; } static void finish_pending(void) { if (!call.session[0] || !call.finished) return; atomic_store(&audio_allowed, false); cJSON *j = cJSON_CreateObject(); cJSON_AddStringToObject(j,"session_id",call.session); cJSON_AddStringToObject(j,"device_id",ble_factory_id()); cJSON_AddStringToObject(j,"reason",call.reason[0] ? call.reason : "disconnected"); cJSON_AddBoolToObject(j,"sdk_stopped",call.sdk_stopped); esp_err_t e = post("/api/v1/device/chat/end",j,call.end_id,NULL); cJSON_Delete(j); if (e == ESP_OK && persist("direct_call",NULL,0) == ESP_OK) memset(&call,0,sizeof(call)); if (params) { mbedtls_platform_zeroize(params,strlen(params)); free(params); params=NULL; } ESP_LOGI(TAG,"chat end acknowledged=%d",e == ESP_OK); } esp_err_t call_client_begin(void) { start_error[0] = '\0'; if (!state_mutex) return ESP_ERR_INVALID_STATE; if (!engine_ready && sdk_initialized) call_client_engine_ready(true); if (!engine_ready) return ESP_ERR_INVALID_STATE; xSemaphoreTake(state_mutex,portMAX_DELAY); if (call.finished) finish_pending(); if (maintenance || atomic_load(&standby_mode) || call.session[0]) { xSemaphoreGive(state_mutex); return ESP_ERR_INVALID_STATE; } memset(&call,0,sizeof(call)); call.version=2; ble_random_uuid(call.session); ble_random_uuid(call.end_id); snprintf(call.boot,sizeof(call.boot),"%s",ble_cloud_boot_id()); sdk_stopped=false; esp_err_t e = persist("direct_call",&call,sizeof(call)); if (e == ESP_OK) e=fetch_parameters(true); if (e != ESP_OK) { snprintf(start_error,sizeof(start_error),"%s",platform_last_error_code()); call.finished=true; call.failed=true; snprintf(call.reason,sizeof(call.reason),"start_failed"); persist("direct_call",&call,sizeof(call)); finish_pending(); } xSemaphoreGive(state_mutex); return e; } int call_client_poll(void) { if (!call_client_audio_allowed()) return 1; xSemaphoreTake(state_mutex,portMAX_DELAY); if (mono() >= renew_due) { esp_err_t e=fetch_parameters(false); // A known refusal is terminal. Transient transport failures get only // the remaining previously granted time, never a new local lease. if (e != ESP_OK) { const char *code=platform_last_error_code(); if (code && (!strcmp(code,"CHAT_ENDED") || !strcmp(code,"DEVICE_BINDING_CHANGED") || !strcmp(code,"VOICE_CHAT_CONSENT_REQUIRED") || !strcmp(code,"DEVICE_NOT_READY") || !strcmp(code,"MEMORY_CONSENT_REVOKED") || !strcmp(code,"DEVICE_UNBOUND") || !strcmp(code,"VOICE_AGENT_NOT_READY"))) atomic_store(&audio_allowed,false); } } int stopped=!call_client_audio_allowed(); xSemaphoreGive(state_mutex); return stopped; } void call_client_stop_result(bool started,int result) { sdk_stopped=started && result==0; } void call_client_finish(bool failed) { atomic_store(&audio_allowed,false); if (!state_mutex) return; xSemaphoreTake(state_mutex,portMAX_DELAY); if (call.session[0] && !call.finished) { call.finished=true; call.failed=failed; call.sdk_stopped=sdk_stopped; snprintf(call.reason,sizeof(call.reason),"%s",failed ? "disconnected" : "hangup"); persist("direct_call",&call,sizeof(call)); } finish_pending(); xSemaphoreGive(state_mutex); } static void retry_task(void *unused) { (void)unused; for (;;) { vTaskDelay(pdMS_TO_TICKS(10000)); if (xSemaphoreTake(state_mutex,0)==pdTRUE) { if (call.finished) finish_pending(); xSemaphoreGive(state_mutex); } } } bool call_client_busy(void) { if (!state_mutex || xSemaphoreTake(state_mutex,0)!=pdTRUE) return true; bool busy=maintenance || call.session[0]!=0; xSemaphoreGive(state_mutex); return busy; } bool call_client_standby(void) { return atomic_load(&standby_mode); } esp_err_t call_client_set_standby(bool standby) { if (!state_mutex) { atomic_store(&standby_mode,standby); return ESP_OK; } xSemaphoreTake(state_mutex,portMAX_DELAY); if (call.session[0] || !maintenance) { xSemaphoreGive(state_mutex); return ESP_ERR_INVALID_STATE; } atomic_store(&standby_mode,standby); xSemaphoreGive(state_mutex); return ESP_OK; } bool call_client_enter_maintenance(void) { if (!state_mutex || xSemaphoreTake(state_mutex,0)!=pdTRUE) return false; bool allowed=!maintenance && !call.session[0]; if (allowed) maintenance=true; xSemaphoreGive(state_mutex); return allowed; } void call_client_leave_maintenance(void) { if (!state_mutex) return; xSemaphoreTake(state_mutex,portMAX_DELAY); maintenance=false; xSemaphoreGive(state_mutex); }