story/firmware/echoear-platform-agent/main/conv_ai_embedded_kit.c

// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0

#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>
#include <time.h>
#include <inttypes.h>
#include <stdatomic.h>
#include "esp_board_manager.h"
#include "dev_lcd_touch_i2c.h"
#include "ble_setup.h"
#include "device_reset.h"
#include "ble_cloud.h"

#include "esp_event.h"
#include "esp_log.h"
#include "esp_system.h"
#include "nvs_flash.h"
#include "esp_netif.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_heap_task_info.h"
#include "esp_random.h"
#include "esp_sntp.h"

#include "freertos/semphr.h"
#include "esp_err.h"
#include "sdkconfig.h"

// #include "periph_wifi.h"
// #include "fatfs_stream.h"

#include "cJSON.h"
#include "network.h"
#include "volc_manager_service.h"
#include "volc_service_common.h"
#include "volc_conv_service.h"
#include "local_logic_service.h"
#include "volc_function_call_service.h"

#include "volc_hal.h"
#include "volc_hal_display.h"
#include "volc_hal_capture.h"
#include "volc_lvgl_source.h"
// #include "board.h"
#include "esp_mac.h"
#include "esp_gmf_afe.h"
#include "mmap_generate_eaf.h"
#include "platform_client.h"
#include "call_client.h"
#include "device_control.h"
#include "device_power.h"
#include "device_ota.h"
#include "freertos/idf_additions.h"
#include "hal/usb_serial_jtag_ll.h"
#include "lwip/sockets.h"
#define STATS_TASK_PRIO 5

static const char *TAG = "VolcConvAI";
static volatile bool is_interrupt = false;
static volatile bool conversation_service_ready = false;
extern void conv_ai_service_diagnostic_say(void);
extern void conv_ai_service_diagnostic_tts(void);
extern void volc_rtc_diagnostic_snapshot(void);

/* RTC_PRIORITY_DIAGNOSTIC_BEGIN: the production scheduling floor has no USB
 * override. Retired experiment commands acknowledge rejection explicitly. */
static void diagnostic_rtc_priority(bool high)
{
    ESP_LOGI(TAG, "RTC_PRIORITY requested_high=%d accepted=0 reason=fixed_floor floor=6", high);
}
/* RTC_PRIORITY_DIAGNOSTIC_END */

/* A local USB diagnostic command lets service tests run without repeated taps.
 * It does not run automatically or accept commands over the network. */
static void usb_diagnostic_task(void *arg)
{
    char command[32];
    size_t length = 0;
    while (true) {
        while (usb_serial_jtag_ll_rxfifo_data_available()) {
            uint8_t ch;
            if (!usb_serial_jtag_ll_read_rxfifo(&ch, 1)) break;
            if (ch == '\n' || ch == '\r') {
                command[length] = 0;
                if (conversation_service_ready && !strcmp(command, "diag-say")) {
                    ESP_LOGI(TAG, "USB diagnostic: requesting fixed spoken sentence");
                    conv_ai_service_diagnostic_say();
                }
                if (conversation_service_ready && !strcmp(command, "diag-tts")) {
                    ESP_LOGI(TAG, "USB diagnostic: requesting fixed TTS passage with microphone uplink suppressed");
                    conv_ai_service_diagnostic_tts();
                }
                if (!strcmp(command, "diag-rtc-high")) diagnostic_rtc_priority(true);
                if (!strcmp(command, "diag-rtc-normal")) diagnostic_rtc_priority(false);
                if (conversation_service_ready && !strcmp(command, "diag-stop")) {
                    ESP_LOGI(TAG, "USB diagnostic: ending test conversation");
                    aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_QUIT, NULL, NULL);
                }
                if (!strcmp(command, "diag-state")) {
                    unsigned sockets=0;
                    for(int fd=LWIP_SOCKET_OFFSET;fd<FD_SETSIZE;fd++){
                        int type;socklen_t length=sizeof(type);
                        if(getsockopt(fd,SOL_SOCKET,SO_TYPE,&type,&length)==0)sockets++;
                    }
                    ESP_LOGI(TAG,"Network diagnostic: sockets=%u limit=%u",sockets,(unsigned)CONFIG_LWIP_MAX_SOCKETS);
                    ESP_LOGI(TAG,"RTC diagnostic: internal_free=%u largest=%u",
                        (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT),
                        (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT));
                    volc_rtc_diagnostic_snapshot();
                }
                length = 0;
            } else if (length < sizeof(command) - 1) {
                command[length++] = ch;
            } else {
                length = 0;
            }
        }
        vTaskDelay(pdMS_TO_TICKS(20));
    }
}

void session_init(){
    volc_service_manager_init();
    volc_conv_service_manager_init();
    function_call_service_init();
    local_logic_service_init();
}

void Task(void* data){
    aios_init(VOLC_SERVICE_EVENT_MAX);               // AIOS初始化
    session_init();
    if(call_client_is_engine_ready())device_ota_boot_ready();
    else ESP_LOGE(TAG,"Voice engine failed startup; OTA image remains unconfirmed");
    conversation_service_ready = true;
    xTaskCreate(usb_diagnostic_task, "usb_diagnostic", 3072, NULL, 3, NULL);
    aios_run();                                      // AIOS启动
    return;
}

static void sys_monitor_task(void *pvParameters)
{
    static char run_info[1024] = {0};
    while (1)
    {
        // vTaskGetRunTimeStats(run_info);
        vTaskList(run_info);
        printf("Task Runtime Stats:\n%s", run_info);
        printf("--------------------------------\n");
        // ESP_LOGI(TAG, "Task Runtime Stats:\n%s", run_info);
        // ESP_LOGI(TAG, "--------------------------------\n");
        vTaskDelay(pdMS_TO_TICKS(5000));
    }
}

void initialize_sntp(void)
{
    ESP_LOGI(TAG, "Initializing SNTP");

    // 设置SNTP操作模式为轮询（客户端）
    esp_sntp_setoperatingmode(SNTP_OPMODE_POLL);

    // 设置NTP服务器（可以设置多个备用）
    esp_sntp_setservername(0, "pool.ntp.org");    // 主服务器
    esp_sntp_setservername(1, "cn.pool.ntp.org"); // 备用服务器：中国区
    esp_sntp_setservername(2, "ntp1.aliyun.com"); // 备用服务器：阿里云

    // 初始化SNTP服务
    esp_sntp_init();

    // 设置时区（例如北京时间CST-8）
    setenv("TZ", "CST-8", 1);
    tzset();
}

void wait_for_time_sync(void)
{
    ESP_LOGI(TAG, "Waiting for time synchronization...");

    // 检查时间是否已同步（1970年之后）
    time_t now = 0;
    struct tm timeinfo = {0};
    int retry_count = 5; // 最大重试次数

    while (timeinfo.tm_year < (2024 - 1900) && retry_count > 0)
    {
        ESP_LOGI(TAG, "Time not set yet, retrying... (%d)", retry_count);
        vTaskDelay(1000 / portTICK_PERIOD_MS);
        time(&now);
        localtime_r(&now, &timeinfo);
        retry_count--;
    }

    if (retry_count > 0)
    {
        // print_current_time();
    }
    else
    {
        ESP_LOGE(TAG, "Failed to synchronize time within timeout period");
    }
}

static void rec_engine_cb(void *event, void *user_data)
{
     esp_gmf_afe_evt_t * e =  (esp_gmf_afe_evt_t *)event;

    if (ESP_GMF_AFE_EVT_WAKEUP_START == e->type) {
        if (conversation_service_ready && !call_client_standby()) {
            ESP_LOGI(TAG, "Wake word detected; requesting conversation");
            volc_hal_context_t *ctx = volc_get_global_hal_context();
            if (ctx && ctx->display_handle) {
                int index = MMAP_EAF_EMOTION_BLINK_FAST_284_126_EAF;
                volc_hal_display_set_content(ctx->display_handle, VOLC_DISPLAY_OBJ_MAIN,
                                             VOLC_DISPLAY_IMAGE, &index);
            }
            aios_event_pub(VOLC_SERVICE_AI_CONVERSATION,NULL,NULL);
        }
    }
    return;
}

/* SCREEN_CALL_TAP_BEGIN: debounce only touch bounce, never impose the old
 * five-second start-only guard on a deliberate hangup. */
static void screen_call_tap(TickType_t now)
{
    static TickType_t last_tap;
    static bool tapped;
    if (tapped && (TickType_t)(now - last_tap) < pdMS_TO_TICKS(300)) return;
    if (call_client_standby()) {
        if (device_control_request_screen_wake()) {
            last_tap = now;
            tapped = true;
            ESP_LOGI(TAG, "Screen tapped; requesting wake without starting a conversation");
        }
        return;
    }
    if (aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_TOGGLE, NULL, NULL) == 0) {
        last_tap = now;
        tapped = true;
        ESP_LOGI(TAG, "Screen tapped; toggling conversation");
    }
}
/* SCREEN_CALL_TAP_END */

static void wifi_setup_touch_task(void *arg)
{
    dev_lcd_touch_i2c_handles_t *touch = arg;
    TickType_t pressed_at = 0;
    TickType_t last_setup_tap = 0;
    uint16_t pressed_x = 0;
    uint16_t pressed_y = 0;
    bool holding = false;
    bool setup_triggered = false;
    for (;;) {
        uint16_t x = 0, y = 0;
        uint8_t count = 0;
        bool pressed = esp_lcd_touch_read_data(touch->touch_handle) == ESP_OK &&
            esp_lcd_touch_get_coordinates(touch->touch_handle, &x, &y, NULL, &count, 1) &&
            count > 0;
        if (!pressed) {
            if (holding && !setup_triggered) {
                TickType_t held_for = xTaskGetTickCount() - pressed_at;
                ESP_LOGI(TAG, "Touch released at (%u,%u) after %u ms",
                         pressed_x, pressed_y, (unsigned)(held_for * portTICK_PERIOD_MS));
                if (held_for >= pdMS_TO_TICKS(50) && held_for < pdMS_TO_TICKS(1500) && ble_setup_confirm()) {
                    holding = false;
                    vTaskDelay(pdMS_TO_TICKS(50));
                    continue;
                }
                // A double tap is discoverable and cannot be confused with the
                // physical power-key long press. Never interrupt a voice call.
                if (!conversation_service_ready && ble_cloud_configured() &&
                    held_for >= pdMS_TO_TICKS(50) && held_for < pdMS_TO_TICKS(600)) {
                    TickType_t now = xTaskGetTickCount();
                    if (last_setup_tap && now-last_setup_tap < pdMS_TO_TICKS(800)) {
                        nvs_handle_t setup_nvs;
                        if (nvs_open("wifi",NVS_READWRITE,&setup_nvs)==ESP_OK) {
                            esp_err_t err=nvs_set_u8(setup_nvs,"setup_requested",1);
                            if(err==ESP_OK)err=nvs_commit(setup_nvs);
                            nvs_close(setup_nvs);
                            if(err==ESP_OK)esp_restart();
                        }
                    }
                    last_setup_tap=now;
                }
                if (held_for >= pdMS_TO_TICKS(50) &&
                    held_for < pdMS_TO_TICKS(1500) && conversation_service_ready) {
                    screen_call_tap(xTaskGetTickCount());
                }
            }
            holding = false;
            setup_triggered = false;
        } else if (!holding) {
            holding = true;
            pressed_at = xTaskGetTickCount();
            pressed_x = x;
            pressed_y = y;
        } else if (!setup_triggered && pressed_y >= 280 && pressed_y <= 335 &&
                   pressed_x >= 60 && pressed_x <= 300 &&
                   (TickType_t)(xTaskGetTickCount() - pressed_at) >= pdMS_TO_TICKS(3000)) {
            nvs_handle_t nvs;
            esp_err_t ret = nvs_open("wifi", NVS_READWRITE, &nvs);
            if (ret == ESP_OK) {
                ret = nvs_set_u8(nvs, "setup_requested", 1);
                if (ret == ESP_OK) ret = nvs_commit(nvs);
                nvs_close(nvs);
            }
            if (ret == ESP_OK) {
                ESP_LOGI(TAG, "Restarting into setup; device identity and Wi-Fi preserved");
                setup_triggered = true;
                esp_restart();
            }
            ESP_LOGE(TAG, "Cannot request setup: %s", esp_err_to_name(ret));
            holding = false;
        }
        vTaskDelay(pdMS_TO_TICKS(50));
    }
}

static void wifi_setup_button_init(void)
{
    dev_lcd_touch_i2c_handles_t *touch = NULL;
    esp_err_t ret = esp_board_manager_get_device_handle("lcd_touch", (void **)&touch);
    if (ret != ESP_OK || !touch || !touch->touch_handle) {
        ESP_LOGE(TAG, "Wi-Fi setup touch input unavailable");
        return;
    }
    if (xTaskCreate(wifi_setup_touch_task, "wifi_setup", 4096, touch, 5, NULL) != pdPASS) {
        ESP_LOGE(TAG, "Cannot start Wi-Fi setup touch task");
    }
}

void  hal_level_init()
{

    volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
    volc_hal_capture_config_t config = {0};
    config.media_type = VOLC_MEDIA_TYPE_AUDIO;
    config.data_cb = NULL;
    config.user_data = g_hal_context;
    config.audio_wakeup_cb = (volc_hal_audio_wakeup_cb_t)rec_engine_cb;
    volc_hal_capture_t capture = volc_hal_capture_create(&config);

    volc_hal_capture_start(capture,VOLC_AUDIO_MODE_WAKEUP);


    // Allow other core to finish initialization
    // vTaskDelay(pdMS_TO_TICKS(2000));
}

static void application_network_start(volc_hal_display_t global_display);

static void application_network_boot_task(void *arg)
{
    application_network_start((volc_hal_display_t)arg);
    ESP_LOGI(TAG, "Network startup finished; stack headroom: %u bytes",
             (unsigned)uxTaskGetStackHighWaterMark(NULL));
    vTaskDelete(NULL);
}

static void application_boot(void)
{
    // xTaskCreate(&sys_monitor_task, "sys_monitor_task", 4096, NULL, 5, NULL);
    volc_hal_init();
    volc_hal_display_config_t display_config = {0};
    volc_hal_display_create(&display_config);

    volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
    if(g_hal_context == NULL){
        return;
    }
    volc_hal_display_t global_display = g_hal_context->display_handle;
   
    /* Initialize the default event loop */
    ESP_ERROR_CHECK(esp_event_loop_create_default());
    esp_log_level_set("*", ESP_LOG_WARN);
    esp_log_level_set("*", ESP_LOG_INFO);

    // /* Initialize NVS flash for WiFi configuration */
    esp_err_t ret = nvs_flash_init();
    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND)
    {
        ESP_LOGE(TAG, "NVS requires recovery; preserving Wi-Fi and device credentials");
        volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE,
                                     VOLC_DISPLAY_TEXT, "Storage recovery required");
        return;
    }
    ESP_ERROR_CHECK(ret);
    if (device_reset_boot_cleanup()!=ESP_OK) {
        volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,"Reset recovery required");
        return;
    }
    device_ota_boot_guard();
    ESP_ERROR_CHECK(platform_client_init());
    ESP_ERROR_CHECK(esp_netif_init());
    wifi_setup_button_init();
    if (ble_setup_start_if_needed()) return;
    // TLS/audio startup needs a large stack, but BLE provisioning does not.
    // Allocate it only for normal operation, after choosing the boot mode.
    if (xTaskCreate(application_network_boot_task, "network_boot", 24 * 1024,
                    global_display, 5, NULL) != pdPASS) {
        ESP_LOGE(TAG, "Cannot allocate network startup task");
    }
}

static void application_network_start(volc_hal_display_t global_display)
{
    esp_err_t ret;
    if(device_power_boot()!=ESP_OK){ESP_LOGE(TAG,"Standby state restore failed; preserving microphone gate");return;}

    uint8_t station_mac[6] = {0};
    char setup_status[96] = "等待配网";
    if (esp_read_mac(station_mac, ESP_MAC_WIFI_STA) == ESP_OK) {
        snprintf(setup_status, sizeof(setup_status),
                 "WiFi: VolcConvAI-%02X%02X%02X",
                 station_mac[3], station_mac[4], station_mac[5]);
    }
    volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,
                                 VOLC_DISPLAY_TEXT, setup_status);

    bool connected = configure_network();
    if (connected == false)
    {
        if(network_last_error()!=ESP_OK){
            volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"网络初始化失败，请重启");
            return;
        }
        if (ble_cloud_configured()) {
            volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"旧 Wi-Fi 不可用，正在开启蓝牙配网");
            esp_restart();
            return;
        }
        volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,
                                     VOLC_DISPLAY_TEXT, setup_status);
        volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE,
                                     VOLC_DISPLAY_TEXT, "192.168.4.1 密码同热点名");
        ESP_LOGI(TAG, "Waiting for setup through the Wi-Fi portal");
        return;
    }
    initialize_sntp();
    wait_for_time_sync();
    if (ble_cloud_configured()) {
        volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"正在同步语音服务");
        ret=call_client_boot();
        while(ret!=ESP_OK && !strcmp(call_client_start_error(),"VOICE_PREPARING")) {
            volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"平台正在准备语音服务");
            vTaskDelay(pdMS_TO_TICKS(5000));
            ret=call_client_boot();
        }
        if(ret!=ESP_OK){
            const char *message="语音服务未就绪，请稍后重启";
            if(!strcmp(platform_last_error_code(),"PROVIDER_PROFILE_UNVERIFIED"))message="语音配置待验证";
            if(!strcmp(platform_last_error_code(),"DEVICE_UNBOUND") || !strcmp(call_client_start_error(),"DEVICE_UNBOUND")) {
                message="请先在小程序绑定设备";
                // Platform unbind invalidates cached credentials. Restart once
                // into BLE setup; keep factory identity and Wi-Fi unchanged.
                nvs_handle_t setup_nvs;
                if(nvs_open("wifi",NVS_READWRITE,&setup_nvs)==ESP_OK) {
                    esp_err_t saved=nvs_set_u8(setup_nvs,"setup_requested",1);
                    if(saved==ESP_OK)saved=nvs_commit(setup_nvs);
                    nvs_close(setup_nvs);
                    if(saved==ESP_OK) {
                        ESP_LOGI(TAG,"Device unbound; entering BLE setup");
                        esp_restart();
                    }
                }
            }
            volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,message);
            volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,"双击屏幕重新配网");
            return;
        }
        hal_level_init();
        if(xTaskCreateWithCaps(&Task,"aios_task",12288,NULL,5,NULL,MALLOC_CAP_SPIRAM|MALLOC_CAP_8BIT)!=pdPASS)
            ESP_LOGE(TAG,"Cannot allocate AI service task");
        if(device_control_start()!=ESP_OK)ESP_LOGE(TAG,"Device control worker unavailable");
        volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"轻触屏幕开始聊天");return;
    }
    if (!platform_identity_is_configured()) {
        ESP_LOGE(TAG, "Platform identity is missing; erase Wi-Fi NVS or use the setup portal");
        volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT,
                                     "请通过配网页填写平台设备凭据");
        return;
    }
    volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT,
                                 "网络已连接，正在激活");
    ret = platform_provision_volc_credential();
    ESP_LOGI(TAG, "Startup stack headroom after activation: %u bytes", (unsigned)uxTaskGetStackHighWaterMark(NULL));
    if (ret != ESP_OK) {
        ESP_LOGE(TAG, "Platform provider provisioning failed: %s", esp_err_to_name(ret));
        const char *code = platform_last_error_code();
        const char *message = "平台激活失败，请检查配置";
        if (strcmp(code, "DEVICE_ACTIVATION_REQUIRED") == 0) message = "请先在平台绑定家庭和长者";
        else if (strcmp(code, "HARDWARE_AGENT_BINDING_REQUIRED") == 0) message = "请在平台配置硬件智能体";
        else if (strcmp(code, "HARDWARE_AGENT_PRODUCT_MISMATCH") == 0) message = "平台产品配置不一致";
        else if (strcmp(code, "HARDWARE_AGENT_HARDWARE_MISMATCH") == 0) message = "设备凭据与本机不匹配";
        volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, message);
        return;
    }
    ret = platform_recover_pending_conversation();
    if (ret != ESP_OK) {
        ESP_LOGW(TAG, "Cannot close the conversation interrupted by the previous restart: %s",
                 esp_err_to_name(ret));
    }
    hal_level_init();
    ESP_LOGI(TAG, "Startup stack headroom after audio: %u bytes", (unsigned)uxTaskGetStackHighWaterMark(NULL));
    if(xTaskCreateWithCaps(&Task,"aios_task",12288,NULL,5,NULL,MALLOC_CAP_SPIRAM|MALLOC_CAP_8BIT)!=pdPASS)
        ESP_LOGE(TAG,"Cannot allocate AI service task");
    int index = MMAP_EAF_HAPPY_EAF;
    volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
    volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"轻触屏幕或说嗨，乐鑫");
}

/* Board/setup startup keeps internal RAM available for BLE and LCD DMA.
 * Network signing/TLS gets its separate larger stack only in voice mode. */
static void application_boot_task(void *arg)
{
    application_boot();
    ESP_LOGI(TAG, "Startup task finished; stack headroom: %u bytes",
             (unsigned)uxTaskGetStackHighWaterMark(NULL));
    vTaskDelete(NULL);
}

void app_main(void)
{
    if (xTaskCreate(application_boot_task, "platform_boot", 8 * 1024,
                    NULL, 5, NULL) != pdPASS) {
        ESP_LOGE(TAG, "Cannot allocate platform startup task");
    }
}