Files
story/firmware/echoear-platform-agent/main/conv_ai_embedded_kit.c
T
Codex Local bf1233d5df
ci / verify (push) Canceled after 0s
feat: expand companion platform, device controls, and memoir workflows
Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
2026-10-08 10:06:44 +08:00

534 lines
21 KiB
C

// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>
#include <time.h>
#include <inttypes.h>
#include <stdatomic.h>
#include "esp_board_manager.h"
#include "dev_lcd_touch_i2c.h"
#include "ble_setup.h"
#include "device_reset.h"
#include "ble_cloud.h"
#include "esp_event.h"
#include "esp_log.h"
#include "esp_system.h"
#include "nvs_flash.h"
#include "esp_netif.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_heap_task_info.h"
#include "esp_random.h"
#include "esp_sntp.h"
#include "freertos/semphr.h"
#include "esp_err.h"
#include "sdkconfig.h"
// #include "periph_wifi.h"
// #include "fatfs_stream.h"
#include "cJSON.h"
#include "network.h"
#include "volc_manager_service.h"
#include "volc_service_common.h"
#include "volc_conv_service.h"
#include "local_logic_service.h"
#include "volc_function_call_service.h"
#include "volc_hal.h"
#include "volc_hal_display.h"
#include "volc_hal_capture.h"
#include "volc_lvgl_source.h"
// #include "board.h"
#include "esp_mac.h"
#include "esp_gmf_afe.h"
#include "mmap_generate_eaf.h"
#include "platform_client.h"
#include "call_client.h"
#include "device_control.h"
#include "device_power.h"
#include "device_ota.h"
#include "freertos/idf_additions.h"
#include "hal/usb_serial_jtag_ll.h"
#include "lwip/sockets.h"
#define STATS_TASK_PRIO 5
static const char *TAG = "VolcConvAI";
static volatile bool is_interrupt = false;
static volatile bool conversation_service_ready = false;
extern void conv_ai_service_diagnostic_say(void);
extern void conv_ai_service_diagnostic_tts(void);
extern void volc_rtc_diagnostic_snapshot(void);
/* RTC_PRIORITY_DIAGNOSTIC_BEGIN: the production scheduling floor has no USB
* override. Retired experiment commands acknowledge rejection explicitly. */
static void diagnostic_rtc_priority(bool high)
{
ESP_LOGI(TAG, "RTC_PRIORITY requested_high=%d accepted=0 reason=fixed_floor floor=6", high);
}
/* RTC_PRIORITY_DIAGNOSTIC_END */
/* A local USB diagnostic command lets service tests run without repeated taps.
* It does not run automatically or accept commands over the network. */
static void usb_diagnostic_task(void *arg)
{
char command[32];
size_t length = 0;
while (true) {
while (usb_serial_jtag_ll_rxfifo_data_available()) {
uint8_t ch;
if (!usb_serial_jtag_ll_read_rxfifo(&ch, 1)) break;
if (ch == '\n' || ch == '\r') {
command[length] = 0;
if (conversation_service_ready && !strcmp(command, "diag-say")) {
ESP_LOGI(TAG, "USB diagnostic: requesting fixed spoken sentence");
conv_ai_service_diagnostic_say();
}
if (conversation_service_ready && !strcmp(command, "diag-tts")) {
ESP_LOGI(TAG, "USB diagnostic: requesting fixed TTS passage with microphone uplink suppressed");
conv_ai_service_diagnostic_tts();
}
if (!strcmp(command, "diag-rtc-high")) diagnostic_rtc_priority(true);
if (!strcmp(command, "diag-rtc-normal")) diagnostic_rtc_priority(false);
if (conversation_service_ready && !strcmp(command, "diag-stop")) {
ESP_LOGI(TAG, "USB diagnostic: ending test conversation");
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_QUIT, NULL, NULL);
}
if (!strcmp(command, "diag-state")) {
unsigned sockets=0;
for(int fd=LWIP_SOCKET_OFFSET;fd<FD_SETSIZE;fd++){
int type;socklen_t length=sizeof(type);
if(getsockopt(fd,SOL_SOCKET,SO_TYPE,&type,&length)==0)sockets++;
}
ESP_LOGI(TAG,"Network diagnostic: sockets=%u limit=%u",sockets,(unsigned)CONFIG_LWIP_MAX_SOCKETS);
ESP_LOGI(TAG,"RTC diagnostic: internal_free=%u largest=%u",
(unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT),
(unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT));
volc_rtc_diagnostic_snapshot();
}
length = 0;
} else if (length < sizeof(command) - 1) {
command[length++] = ch;
} else {
length = 0;
}
}
vTaskDelay(pdMS_TO_TICKS(20));
}
}
void session_init(){
volc_service_manager_init();
volc_conv_service_manager_init();
function_call_service_init();
local_logic_service_init();
}
void Task(void* data){
aios_init(VOLC_SERVICE_EVENT_MAX); // AIOS初始化
session_init();
if(call_client_is_engine_ready())device_ota_boot_ready();
else ESP_LOGE(TAG,"Voice engine failed startup; OTA image remains unconfirmed");
conversation_service_ready = true;
xTaskCreate(usb_diagnostic_task, "usb_diagnostic", 3072, NULL, 3, NULL);
aios_run(); // AIOS启动
return;
}
static void sys_monitor_task(void *pvParameters)
{
static char run_info[1024] = {0};
while (1)
{
// vTaskGetRunTimeStats(run_info);
vTaskList(run_info);
printf("Task Runtime Stats:\n%s", run_info);
printf("--------------------------------\n");
// ESP_LOGI(TAG, "Task Runtime Stats:\n%s", run_info);
// ESP_LOGI(TAG, "--------------------------------\n");
vTaskDelay(pdMS_TO_TICKS(5000));
}
}
void initialize_sntp(void)
{
ESP_LOGI(TAG, "Initializing SNTP");
// 设置SNTP操作模式为轮询(客户端)
esp_sntp_setoperatingmode(SNTP_OPMODE_POLL);
// 设置NTP服务器(可以设置多个备用)
esp_sntp_setservername(0, "pool.ntp.org"); // 主服务器
esp_sntp_setservername(1, "cn.pool.ntp.org"); // 备用服务器:中国区
esp_sntp_setservername(2, "ntp1.aliyun.com"); // 备用服务器:阿里云
// 初始化SNTP服务
esp_sntp_init();
// 设置时区(例如北京时间CST-8)
setenv("TZ", "CST-8", 1);
tzset();
}
void wait_for_time_sync(void)
{
ESP_LOGI(TAG, "Waiting for time synchronization...");
// 检查时间是否已同步(1970年之后)
time_t now = 0;
struct tm timeinfo = {0};
int retry_count = 5; // 最大重试次数
while (timeinfo.tm_year < (2024 - 1900) && retry_count > 0)
{
ESP_LOGI(TAG, "Time not set yet, retrying... (%d)", retry_count);
vTaskDelay(1000 / portTICK_PERIOD_MS);
time(&now);
localtime_r(&now, &timeinfo);
retry_count--;
}
if (retry_count > 0)
{
// print_current_time();
}
else
{
ESP_LOGE(TAG, "Failed to synchronize time within timeout period");
}
}
static void rec_engine_cb(void *event, void *user_data)
{
esp_gmf_afe_evt_t * e = (esp_gmf_afe_evt_t *)event;
if (ESP_GMF_AFE_EVT_WAKEUP_START == e->type) {
if (conversation_service_ready && !call_client_standby()) {
ESP_LOGI(TAG, "Wake word detected; requesting conversation");
volc_hal_context_t *ctx = volc_get_global_hal_context();
if (ctx && ctx->display_handle) {
int index = MMAP_EAF_EMOTION_BLINK_FAST_284_126_EAF;
volc_hal_display_set_content(ctx->display_handle, VOLC_DISPLAY_OBJ_MAIN,
VOLC_DISPLAY_IMAGE, &index);
}
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION,NULL,NULL);
}
}
return;
}
/* SCREEN_CALL_TAP_BEGIN: debounce only touch bounce, never impose the old
* five-second start-only guard on a deliberate hangup. */
static void screen_call_tap(TickType_t now)
{
static TickType_t last_tap;
static bool tapped;
if (tapped && (TickType_t)(now - last_tap) < pdMS_TO_TICKS(300)) return;
if (call_client_standby()) {
if (device_control_request_screen_wake()) {
last_tap = now;
tapped = true;
ESP_LOGI(TAG, "Screen tapped; requesting wake without starting a conversation");
}
return;
}
if (aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_TOGGLE, NULL, NULL) == 0) {
last_tap = now;
tapped = true;
ESP_LOGI(TAG, "Screen tapped; toggling conversation");
}
}
/* SCREEN_CALL_TAP_END */
static void wifi_setup_touch_task(void *arg)
{
dev_lcd_touch_i2c_handles_t *touch = arg;
TickType_t pressed_at = 0;
TickType_t last_setup_tap = 0;
uint16_t pressed_x = 0;
uint16_t pressed_y = 0;
bool holding = false;
bool setup_triggered = false;
for (;;) {
uint16_t x = 0, y = 0;
uint8_t count = 0;
bool pressed = esp_lcd_touch_read_data(touch->touch_handle) == ESP_OK &&
esp_lcd_touch_get_coordinates(touch->touch_handle, &x, &y, NULL, &count, 1) &&
count > 0;
if (!pressed) {
if (holding && !setup_triggered) {
TickType_t held_for = xTaskGetTickCount() - pressed_at;
ESP_LOGI(TAG, "Touch released at (%u,%u) after %u ms",
pressed_x, pressed_y, (unsigned)(held_for * portTICK_PERIOD_MS));
if (held_for >= pdMS_TO_TICKS(50) && held_for < pdMS_TO_TICKS(1500) && ble_setup_confirm()) {
holding = false;
vTaskDelay(pdMS_TO_TICKS(50));
continue;
}
// A double tap is discoverable and cannot be confused with the
// physical power-key long press. Never interrupt a voice call.
if (!conversation_service_ready && ble_cloud_configured() &&
held_for >= pdMS_TO_TICKS(50) && held_for < pdMS_TO_TICKS(600)) {
TickType_t now = xTaskGetTickCount();
if (last_setup_tap && now-last_setup_tap < pdMS_TO_TICKS(800)) {
nvs_handle_t setup_nvs;
if (nvs_open("wifi",NVS_READWRITE,&setup_nvs)==ESP_OK) {
esp_err_t err=nvs_set_u8(setup_nvs,"setup_requested",1);
if(err==ESP_OK)err=nvs_commit(setup_nvs);
nvs_close(setup_nvs);
if(err==ESP_OK)esp_restart();
}
}
last_setup_tap=now;
}
if (held_for >= pdMS_TO_TICKS(50) &&
held_for < pdMS_TO_TICKS(1500) && conversation_service_ready) {
screen_call_tap(xTaskGetTickCount());
}
}
holding = false;
setup_triggered = false;
} else if (!holding) {
holding = true;
pressed_at = xTaskGetTickCount();
pressed_x = x;
pressed_y = y;
} else if (!setup_triggered && pressed_y >= 280 && pressed_y <= 335 &&
pressed_x >= 60 && pressed_x <= 300 &&
(TickType_t)(xTaskGetTickCount() - pressed_at) >= pdMS_TO_TICKS(3000)) {
nvs_handle_t nvs;
esp_err_t ret = nvs_open("wifi", NVS_READWRITE, &nvs);
if (ret == ESP_OK) {
ret = nvs_set_u8(nvs, "setup_requested", 1);
if (ret == ESP_OK) ret = nvs_commit(nvs);
nvs_close(nvs);
}
if (ret == ESP_OK) {
ESP_LOGI(TAG, "Restarting into setup; device identity and Wi-Fi preserved");
setup_triggered = true;
esp_restart();
}
ESP_LOGE(TAG, "Cannot request setup: %s", esp_err_to_name(ret));
holding = false;
}
vTaskDelay(pdMS_TO_TICKS(50));
}
}
static void wifi_setup_button_init(void)
{
dev_lcd_touch_i2c_handles_t *touch = NULL;
esp_err_t ret = esp_board_manager_get_device_handle("lcd_touch", (void **)&touch);
if (ret != ESP_OK || !touch || !touch->touch_handle) {
ESP_LOGE(TAG, "Wi-Fi setup touch input unavailable");
return;
}
if (xTaskCreate(wifi_setup_touch_task, "wifi_setup", 4096, touch, 5, NULL) != pdPASS) {
ESP_LOGE(TAG, "Cannot start Wi-Fi setup touch task");
}
}
void hal_level_init()
{
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
volc_hal_capture_config_t config = {0};
config.media_type = VOLC_MEDIA_TYPE_AUDIO;
config.data_cb = NULL;
config.user_data = g_hal_context;
config.audio_wakeup_cb = (volc_hal_audio_wakeup_cb_t)rec_engine_cb;
volc_hal_capture_t capture = volc_hal_capture_create(&config);
volc_hal_capture_start(capture,VOLC_AUDIO_MODE_WAKEUP);
// Allow other core to finish initialization
// vTaskDelay(pdMS_TO_TICKS(2000));
}
static void application_network_start(volc_hal_display_t global_display);
static void application_network_boot_task(void *arg)
{
application_network_start((volc_hal_display_t)arg);
ESP_LOGI(TAG, "Network startup finished; stack headroom: %u bytes",
(unsigned)uxTaskGetStackHighWaterMark(NULL));
vTaskDelete(NULL);
}
static void application_boot(void)
{
// xTaskCreate(&sys_monitor_task, "sys_monitor_task", 4096, NULL, 5, NULL);
volc_hal_init();
volc_hal_display_config_t display_config = {0};
volc_hal_display_create(&display_config);
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
if(g_hal_context == NULL){
return;
}
volc_hal_display_t global_display = g_hal_context->display_handle;
/* Initialize the default event loop */
ESP_ERROR_CHECK(esp_event_loop_create_default());
esp_log_level_set("*", ESP_LOG_WARN);
esp_log_level_set("*", ESP_LOG_INFO);
// /* Initialize NVS flash for WiFi configuration */
esp_err_t ret = nvs_flash_init();
if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND)
{
ESP_LOGE(TAG, "NVS requires recovery; preserving Wi-Fi and device credentials");
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE,
VOLC_DISPLAY_TEXT, "Storage recovery required");
return;
}
ESP_ERROR_CHECK(ret);
if (device_reset_boot_cleanup()!=ESP_OK) {
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,"Reset recovery required");
return;
}
device_ota_boot_guard();
ESP_ERROR_CHECK(platform_client_init());
ESP_ERROR_CHECK(esp_netif_init());
wifi_setup_button_init();
if (ble_setup_start_if_needed()) return;
// TLS/audio startup needs a large stack, but BLE provisioning does not.
// Allocate it only for normal operation, after choosing the boot mode.
if (xTaskCreate(application_network_boot_task, "network_boot", 24 * 1024,
global_display, 5, NULL) != pdPASS) {
ESP_LOGE(TAG, "Cannot allocate network startup task");
}
}
static void application_network_start(volc_hal_display_t global_display)
{
esp_err_t ret;
if(device_power_boot()!=ESP_OK){ESP_LOGE(TAG,"Standby state restore failed; preserving microphone gate");return;}
uint8_t station_mac[6] = {0};
char setup_status[96] = "等待配网";
if (esp_read_mac(station_mac, ESP_MAC_WIFI_STA) == ESP_OK) {
snprintf(setup_status, sizeof(setup_status),
"WiFi: VolcConvAI-%02X%02X%02X",
station_mac[3], station_mac[4], station_mac[5]);
}
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,
VOLC_DISPLAY_TEXT, setup_status);
bool connected = configure_network();
if (connected == false)
{
if(network_last_error()!=ESP_OK){
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"网络初始化失败,请重启");
return;
}
if (ble_cloud_configured()) {
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"旧 Wi-Fi 不可用,正在开启蓝牙配网");
esp_restart();
return;
}
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,
VOLC_DISPLAY_TEXT, setup_status);
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE,
VOLC_DISPLAY_TEXT, "192.168.4.1 密码同热点名");
ESP_LOGI(TAG, "Waiting for setup through the Wi-Fi portal");
return;
}
initialize_sntp();
wait_for_time_sync();
if (ble_cloud_configured()) {
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"正在同步语音服务");
ret=call_client_boot();
while(ret!=ESP_OK && !strcmp(call_client_start_error(),"VOICE_PREPARING")) {
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"平台正在准备语音服务");
vTaskDelay(pdMS_TO_TICKS(5000));
ret=call_client_boot();
}
if(ret!=ESP_OK){
const char *message="语音服务未就绪,请稍后重启";
if(!strcmp(platform_last_error_code(),"PROVIDER_PROFILE_UNVERIFIED"))message="语音配置待验证";
if(!strcmp(platform_last_error_code(),"DEVICE_UNBOUND") || !strcmp(call_client_start_error(),"DEVICE_UNBOUND")) {
message="请先在小程序绑定设备";
// Platform unbind invalidates cached credentials. Restart once
// into BLE setup; keep factory identity and Wi-Fi unchanged.
nvs_handle_t setup_nvs;
if(nvs_open("wifi",NVS_READWRITE,&setup_nvs)==ESP_OK) {
esp_err_t saved=nvs_set_u8(setup_nvs,"setup_requested",1);
if(saved==ESP_OK)saved=nvs_commit(setup_nvs);
nvs_close(setup_nvs);
if(saved==ESP_OK) {
ESP_LOGI(TAG,"Device unbound; entering BLE setup");
esp_restart();
}
}
}
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,message);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,"双击屏幕重新配网");
return;
}
hal_level_init();
if(xTaskCreateWithCaps(&Task,"aios_task",12288,NULL,5,NULL,MALLOC_CAP_SPIRAM|MALLOC_CAP_8BIT)!=pdPASS)
ESP_LOGE(TAG,"Cannot allocate AI service task");
if(device_control_start()!=ESP_OK)ESP_LOGE(TAG,"Device control worker unavailable");
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"轻触屏幕开始聊天");return;
}
if (!platform_identity_is_configured()) {
ESP_LOGE(TAG, "Platform identity is missing; erase Wi-Fi NVS or use the setup portal");
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT,
"请通过配网页填写平台设备凭据");
return;
}
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT,
"网络已连接,正在激活");
ret = platform_provision_volc_credential();
ESP_LOGI(TAG, "Startup stack headroom after activation: %u bytes", (unsigned)uxTaskGetStackHighWaterMark(NULL));
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Platform provider provisioning failed: %s", esp_err_to_name(ret));
const char *code = platform_last_error_code();
const char *message = "平台激活失败,请检查配置";
if (strcmp(code, "DEVICE_ACTIVATION_REQUIRED") == 0) message = "请先在平台绑定家庭和长者";
else if (strcmp(code, "HARDWARE_AGENT_BINDING_REQUIRED") == 0) message = "请在平台配置硬件智能体";
else if (strcmp(code, "HARDWARE_AGENT_PRODUCT_MISMATCH") == 0) message = "平台产品配置不一致";
else if (strcmp(code, "HARDWARE_AGENT_HARDWARE_MISMATCH") == 0) message = "设备凭据与本机不匹配";
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, message);
return;
}
ret = platform_recover_pending_conversation();
if (ret != ESP_OK) {
ESP_LOGW(TAG, "Cannot close the conversation interrupted by the previous restart: %s",
esp_err_to_name(ret));
}
hal_level_init();
ESP_LOGI(TAG, "Startup stack headroom after audio: %u bytes", (unsigned)uxTaskGetStackHighWaterMark(NULL));
if(xTaskCreateWithCaps(&Task,"aios_task",12288,NULL,5,NULL,MALLOC_CAP_SPIRAM|MALLOC_CAP_8BIT)!=pdPASS)
ESP_LOGE(TAG,"Cannot allocate AI service task");
int index = MMAP_EAF_HAPPY_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"轻触屏幕或说嗨,乐鑫");
}
/* Board/setup startup keeps internal RAM available for BLE and LCD DMA.
* Network signing/TLS gets its separate larger stack only in voice mode. */
static void application_boot_task(void *arg)
{
application_boot();
ESP_LOGI(TAG, "Startup task finished; stack headroom: %u bytes",
(unsigned)uxTaskGetStackHighWaterMark(NULL));
vTaskDelete(NULL);
}
void app_main(void)
{
if (xTaskCreate(application_boot_task, "platform_boot", 8 * 1024,
NULL, 5, NULL) != pdPASS) {
ESP_LOGE(TAG, "Cannot allocate platform startup task");
}
}