feat: expand companion platform, device controls, and memoir workflows
ci / verify (push) Canceled after 0s

Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
This commit is contained in:
Codex Local
2026-10-08 10:06:44 +08:00
parent 98fb2965d0
commit bf1233d5df
861 changed files with 94614 additions and 3209 deletions
@@ -16,6 +16,8 @@ set(VOLC_SERVICE_INCS "${CMAKE_CURRENT_LIST_DIR}/../service/inc"
set(VOLC_SERVICE_SRCS "${CMAKE_CURRENT_LIST_DIR}/../service/src/volc_manager_service.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/volc_conv_service.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/volc_conv.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/call_display_state.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/call_voice_end.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/src/volc_function_call_service.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/src/volc_local_function_list.c"
"${CMAKE_CURRENT_LIST_DIR}/../service/local_logic_service/src/local_logic_service.c"
@@ -15,6 +15,7 @@ set(VOLC_ESP_SRCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal.
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/basic_board.c"
)
set(VOLC_ESP_DISPLAY_SRCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/echoear_font.c"
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/echoear_call_status_font.c"
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/img_app_aiPhone.c"
)
set(VOLC_APP_SRCS ${VOLC_FRAMEWORK_SRCS}
@@ -39,7 +40,7 @@ idf_component_register(
SRCS ${VOLC_APP_SRCS}
INCLUDE_DIRS ${VOLC_APP_INCS}
REQUIRES volc_conv_ai
PRIV_REQUIRES lvgl button av_processor esp_board_manager hw_init esp_mmap_assets esp_emote_gfx esp_vocat_base_control
PRIV_REQUIRES nvs_flash lvgl button av_processor esp_board_manager hw_init esp_mmap_assets esp_emote_gfx esp_vocat_base_control
)
# Compiler flags
@@ -60,6 +60,7 @@ volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config);
* @return volc_hal_display_t Display instance pointer
*/
int volc_hal_display_set_brightness(volc_hal_display_t display, int brightness);
int volc_hal_display_get_brightness(volc_hal_display_t display);
/**
* @brief Destroy a Display instance
@@ -82,4 +83,4 @@ int volc_hal_display_set_content(volc_hal_display_t display, volc_hal_display_ob
}
#endif
#endif
#endif
@@ -48,6 +48,10 @@ int volc_hal_set_audio_player_volume(volc_hal_player_t player, int volume);
* @return the volume
*/
int volc_hal_get_audio_player_volume(volc_hal_player_t player);
// Board codec exists while idle; cloud volume must not depend on a temporary
// conversation player, which is created and destroyed for each call.
int volc_hal_set_device_volume(int volume);
int volc_hal_get_device_volume(void);
/**
* @brief Destroy a player instance
@@ -90,6 +94,10 @@ int volc_hal_player_play_data(volc_hal_player_t player, const void* data, int le
void volc_hal_player_discard_pending(volc_hal_player_t player);
/* Allow a fully received short response to start without filling the prebuffer. */
void volc_hal_player_finish_segment(volc_hal_player_t player);
/* Conversation task only, while it owns a live player. Includes feeder tail. */
bool volc_hal_player_voice_quiet(volc_hal_player_t player, unsigned quiet_ms);
/* Set before start. NULL leaves non-call audio unrestricted. */
void volc_hal_player_set_authorization_gate(volc_hal_player_t player, bool (*allowed)(void));
#ifdef __cplusplus
}
@@ -0,0 +1,166 @@
/* Generated by tools/generate_call_status_font.py; do not hand edit.
* Source SHA256: abfffdf0cd809cf2e3cd9425988b8945786fb5166bf4c5e91d2b6e1fb3839f74
* 2x nearest-neighbor glyphs; packed 1bpp, GFX 1/256-pixel advance.
* The original source font's leading notice is preserved below.
*/
/*******************************************************************************
* Size: 16 px
* Bpp: 1
* Opts: --bpp 1 --size 16 --no-compress --stride 1 --align 1 --font 方正兰亭纤黑.TTF --range 32-126,19968-40869,12288-12544,65280-65312 --format lvgl -o zh_font.c
******************************************************************************/
#ifdef LVGL_VERSION_MAJOR
#error "Build this font in its own GFX-only translation unit"
#endif
#include "widget/gfx_font_lvgl.h"
#if LVGL_VERSION_MAJOR != 8 || LV_FONT_FMT_TXT_LARGE != 1
#error "Unexpected GFX font compatibility ABI"
#endif
_Static_assert(sizeof(lv_font_fmt_txt_glyph_dsc_t) == 16, "GFX LARGE glyph ABI required");
static const uint8_t status_bitmap[] = {
/* U+003F ? */
15,240,15,240,48,12,48,12,192,15,192,15,192,3,192,3,0,15,0,15,0,12,0,12,
0,240,0,240,3,192,3,192,3,0,3,0,3,0,3,0,0,0,0,0,0,0,0,0,
3,0,3,0,
/* U+4E2D 中 */
0,12,0,0,3,0,0,0,192,0,0,48,0,0,12,0,0,3,0,15,255,255,255,255,
255,255,192,12,0,240,3,0,60,0,192,15,0,48,3,192,12,0,240,3,0,60,0,192,
15,0,48,3,255,255,255,255,255,255,252,0,192,15,0,48,3,0,12,0,0,3,0,0,
0,192,0,0,48,0,0,12,0,0,3,0,0,0,192,0,0,48,0,0,12,0,0,3,
0,0,
/* U+5728 在 */
0,12,0,0,0,48,0,0,3,0,0,0,12,0,0,63,255,255,252,255,255,255,240,3,
0,0,0,12,0,0,0,192,48,0,3,0,192,0,48,3,0,0,192,12,0,3,0,48,
0,12,0,192,0,243,255,255,3,207,255,252,51,0,48,0,204,0,192,0,48,3,0,0,
192,12,0,3,0,48,0,12,0,192,0,48,3,0,0,192,12,0,3,0,48,0,12,0,
192,0,63,255,255,192,255,255,255,3,0,0,0,12,0,0,0,
/* U+5931 失 */
0,3,0,0,0,12,0,0,48,48,0,0,192,192,0,3,3,0,0,12,12,0,0,255,
255,255,3,255,255,252,60,3,0,0,240,12,0,3,0,48,0,12,0,192,0,0,3,0,
0,0,12,0,15,255,255,255,255,255,255,255,0,3,0,0,0,12,0,0,0,204,0,0,
3,48,0,0,12,192,0,0,51,0,0,3,3,0,0,12,12,0,3,192,15,0,15,0,
60,3,192,0,15,207,0,0,63,192,0,0,3,0,0,0,0,
/* U+5F85 待 */
0,0,48,0,0,0,192,0,48,3,0,0,192,12,0,15,0,48,0,60,0,192,0,192,
255,255,3,3,255,252,48,0,48,0,192,0,192,12,60,3,0,48,240,12,0,3,63,255,
252,12,255,255,240,192,0,0,3,0,0,0,60,0,3,0,240,0,12,12,195,255,255,243,
15,255,255,12,0,3,0,48,0,12,0,192,192,48,3,3,0,192,12,3,3,0,48,12,
12,0,192,48,48,3,0,192,192,12,0,3,0,48,0,12,0,192,15,240,3,0,63,192,
/* U+63A5 接 */
12,0,48,0,192,3,0,12,0,48,0,192,3,0,12,63,255,240,195,255,255,255,12,3,
207,240,192,60,12,3,3,0,192,48,48,12,255,255,240,207,255,255,12,0,0,0,192,0,
0,63,3,0,3,240,48,0,204,255,255,252,207,255,255,12,12,3,0,192,192,48,12,12,
3,0,192,192,48,12,15,12,0,192,240,192,12,0,252,0,192,15,192,12,15,195,192,192,
252,60,240,240,0,63,15,0,3,
/* U+65AD 断 */
192,192,0,12,12,0,0,192,192,255,252,12,15,255,204,204,192,12,204,204,0,204,204,192,
12,204,204,0,192,192,192,12,12,12,0,192,192,192,12,12,12,0,207,252,255,252,255,207,
255,195,192,192,204,60,12,12,195,240,192,204,63,12,12,204,204,192,204,204,204,12,240,192,
192,207,12,12,12,192,192,192,204,12,12,12,192,3,0,204,0,48,12,255,255,0,207,255,
240,12,0,15,0,192,0,240,12,
/* U+673A 机 */
12,0,0,0,48,0,0,0,192,63,240,3,0,255,192,12,3,3,0,48,12,12,15,252,
48,48,63,240,192,192,12,3,3,0,48,12,12,0,192,48,48,3,0,192,192,63,3,3,
0,252,12,12,3,204,48,48,15,48,192,192,204,3,3,3,48,12,12,12,192,48,48,51,
0,192,192,12,3,3,0,48,12,12,0,192,192,48,195,3,0,195,12,12,3,12,48,48,
12,48,195,0,48,195,12,0,195,12,192,3,240,51,0,15,192,
/* U+675F 束 */
0,3,0,0,0,12,0,15,255,255,255,255,255,255,255,0,3,0,0,0,12,0,0,0,
48,0,0,0,192,0,15,255,255,192,63,255,255,3,0,48,12,12,0,192,48,48,3,0,
192,192,12,3,3,0,48,12,12,0,192,48,15,255,255,192,63,255,255,0,0,252,0,0,
3,240,0,0,51,48,0,0,204,192,0,12,48,192,0,48,195,0,3,3,3,192,12,12,
15,15,192,48,3,255,0,192,15,0,3,0,0,0,12,0,0,
/* U+6B63 正 */
63,255,255,252,255,255,255,240,0,12,0,0,0,48,0,0,0,192,0,0,3,0,0,0,
12,0,0,0,48,0,3,0,192,0,12,3,0,0,48,12,0,0,192,48,0,3,0,255,
240,12,3,255,192,48,12,0,0,192,48,0,3,0,192,0,12,3,0,0,48,12,0,0,
192,48,0,3,0,192,0,12,3,0,0,48,12,0,0,192,48,0,255,255,255,255,255,255,
255,240,
/* U+7ED3 结 */
3,0,12,0,12,0,48,0,48,0,192,0,192,3,0,12,15,255,252,48,63,255,240,192,
0,192,3,0,3,0,48,192,12,0,195,0,48,15,252,0,192,63,240,3,0,3,15,255,
240,12,63,255,192,192,0,0,3,0,0,0,12,0,0,0,48,0,0,3,60,63,255,12,
240,255,252,60,3,0,48,240,12,0,192,0,48,3,0,0,192,12,3,243,0,48,15,204,
0,195,192,63,255,15,0,255,252,0,3,0,48,0,12,0,192,
/* U+8BDD 话 */
48,0,0,0,192,0,0,3,3,255,255,12,15,255,252,12,0,48,0,48,0,192,0,0,
3,0,0,0,12,0,0,255,255,252,3,255,255,255,192,3,0,63,0,12,0,12,0,48,
0,48,0,192,0,192,3,0,3,0,12,0,12,15,255,240,48,63,255,192,192,192,3,3,
3,0,12,12,12,0,48,48,48,0,192,192,192,3,3,3,0,12,12,204,0,48,51,48,
0,192,240,255,255,3,195,255,252,0,12,0,48,0,48,0,192,
/* U+8D25 败 */
0,0,12,0,0,0,12,0,63,252,12,0,63,252,12,0,48,12,48,0,48,12,48,0,
51,12,63,255,51,12,63,255,51,12,48,48,51,12,48,48,51,12,240,48,51,12,240,48,
51,15,48,48,51,15,48,48,51,12,48,48,51,12,48,48,51,12,48,192,51,12,48,192,
51,12,12,192,51,12,12,192,51,12,12,192,51,12,12,192,12,192,3,0,12,192,3,0,
12,60,12,192,12,60,12,192,60,0,48,48,60,0,48,48,240,3,192,15,240,3,192,15,
0,3,0,0,0,3,0,0,
/* U+8FDE 连 */
48,0,192,0,192,3,0,0,192,12,0,3,0,48,0,12,63,255,252,48,255,255,240,0,
48,0,0,0,192,0,0,3,12,0,0,12,48,0,0,192,192,0,3,3,0,252,12,12,
3,240,48,48,0,195,255,255,195,15,255,255,12,0,12,0,48,0,48,0,192,0,192,3,
0,3,0,12,255,255,252,51,255,255,240,192,0,192,3,0,3,0,12,0,12,0,48,0,
48,3,60,0,0,12,240,0,0,192,255,255,255,3,255,255,240,
/* U+901A 通 */
60,63,255,240,240,255,255,192,240,48,12,3,192,192,48,3,0,255,0,12,3,252,0,3,
255,255,0,15,255,252,252,48,48,51,240,192,192,192,195,255,255,3,15,255,252,12,48,48,
48,48,192,192,192,195,3,3,3,12,12,12,12,63,255,240,48,255,255,192,195,3,3,3,
12,12,12,12,48,48,48,48,192,192,195,243,0,63,15,204,0,252,192,255,255,255,3,255,
255,240,
};
static const lv_font_fmt_txt_glyph_dsc_t status_glyphs[] = {
{0}, /* glyph zero is reserved */
{0,4960,16,26,2,-2}, /* ? */
{52,8192,26,30,2,-4}, /* 中 */
{150,8192,30,30,0,-4}, /* 在 */
{263,8192,30,30,2,-4}, /* 失 */
{376,8192,30,32,2,-4}, /* 待 */
{496,8192,28,30,2,-4}, /* 接 */
{601,8192,28,30,2,-4}, /* 断 */
{706,8192,30,30,2,-4}, /* 机 */
{819,8192,30,30,0,-4}, /* 束 */
{932,8192,30,26,0,-2}, /* 正 */
{1030,8192,30,30,0,-4}, /* 结 */
{1143,8192,30,30,0,-4}, /* 话 */
{1256,8192,32,32,0,-6}, /* 败 */
{1384,8192,30,30,0,-4}, /* 连 */
{1497,8192,30,26,0,-2}, /* 通 */
};
#define STATUS_CMAP(cp, id) { .range_start=(cp), .range_length=1, .glyph_id_start=(id), .unicode_list=NULL, .glyph_id_ofs_list=NULL, .list_length=0, .type=LV_FONT_FMT_TXT_CMAP_FORMAT0_TINY }
static const lv_font_fmt_txt_cmap_t status_cmaps[] = {
STATUS_CMAP(0x003F, 1),
STATUS_CMAP(0x4E2D, 2),
STATUS_CMAP(0x5728, 3),
STATUS_CMAP(0x5931, 4),
STATUS_CMAP(0x5F85, 5),
STATUS_CMAP(0x63A5, 6),
STATUS_CMAP(0x65AD, 7),
STATUS_CMAP(0x673A, 8),
STATUS_CMAP(0x675F, 9),
STATUS_CMAP(0x6B63, 10),
STATUS_CMAP(0x7ED3, 11),
STATUS_CMAP(0x8BDD, 12),
STATUS_CMAP(0x8D25, 13),
STATUS_CMAP(0x8FDE, 14),
STATUS_CMAP(0x901A, 15),
};
#undef STATUS_CMAP
static const lv_font_fmt_txt_dsc_t status_font_dsc = {
.glyph_bitmap=status_bitmap, .glyph_dsc=status_glyphs, .cmaps=status_cmaps,
.kern_dsc=NULL, .kern_scale=0, .cmap_num=15, .bpp=1,
.kern_classes=0, .bitmap_format=0, .cache=NULL
};
static const lv_font_t status_font = {
.get_glyph_dsc=lv_font_get_glyph_dsc_fmt_txt, .get_glyph_bitmap=lv_font_get_bitmap_fmt_txt,
.line_height=38, .base_line=10, .subpx=LV_FONT_SUBPX_NONE, .static_bitmap=1,
.underline_position=-6, .underline_thickness=2, .dsc=&status_font_dsc, .fallback=NULL, .user_data=NULL
};
const void *echoear_call_status_font(void) { return &status_font; }
@@ -0,0 +1,14 @@
/* Generated by tools/generate_call_status_font.py; do not hand edit.
* Source SHA256: abfffdf0cd809cf2e3cd9425988b8945786fb5166bf4c5e91d2b6e1fb3839f74
* 2x nearest-neighbor glyphs; packed 1bpp, GFX 1/256-pixel advance.
* The original source font's leading notice is preserved below.
*/
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
/* GFX compatibility font; deliberately exposes no LVGL types. */
const void *echoear_call_status_font(void);
#ifdef __cplusplus
}
#endif
@@ -2,6 +2,10 @@
// SPDX-License-Identifier: Apache-2.0
#include <stddef.h>
#include <stdatomic.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "freertos/semphr.h"
#include "volc_hal.h"
#include "volc_hal_capture.h"
@@ -39,8 +43,10 @@ typedef struct {
volc_hal_capture_data_cb_t data_cb; // Data callback function
volc_hal_audio_wakeup_cb_t audio_wakeup_cb; // audio wakeup cb
void* user_data; // User data pointer
volc_osal_tid_t capture_thread;
volatile bool is_started;
TaskHandle_t capture_thread;
SemaphoreHandle_t capture_stopped;
uint8_t *read_buf;
atomic_bool is_started;
} volc_hal_capture_impl_t;
extern basic_board_periph_t global_periph;
@@ -61,13 +67,12 @@ static void __volc_audio_capture_task(void *arg)
volc_hal_capture_impl_t *impl = (volc_hal_capture_impl_t *)arg;
LOGI("capture task started...");
int read_size = __volc_capture_get_default_read_size(impl);
uint8_t *read_buf = volc_osal_malloc(read_size);
// mem_assert(read_buf);
uint8_t *read_buf = impl->read_buf;
while(impl->is_started) {
int ret = audio_recorder_read_data(read_buf, read_size);
if (ret > 0 && impl->data_cb && impl->audio_capture_config.state == CAPTURE) {
if (impl->is_started && ret > 0 && impl->data_cb && impl->audio_capture_config.state == CAPTURE) {
#ifdef CONFIG_VOLC_AUDIO_G711A
volc_hal_frame_info_t frame_info = {
.data_type = VOLC_AUDIO_DATA_TYPE_G711A,
@@ -83,13 +88,10 @@ static void __volc_audio_capture_task(void *arg)
#endif
}
}
// audio_recorder_pause();
if (impl->capture_thread) {
volc_osal_thread_destroy(impl->capture_thread);
impl->capture_thread = NULL;
}
volc_osal_free(read_buf);
volc_osal_thread_exit(NULL);
// Stop owns deletion with the original WithCaps handle. No old reader may
// survive into the next recorder, or erase a newly created task's handle.
xSemaphoreGive(impl->capture_stopped);
for (;;) vTaskSuspend(NULL);
}
volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t *config)
@@ -100,6 +102,9 @@ volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t *config)
return NULL;
}
volc_hal_capture_impl_t *capture = (volc_hal_capture_impl_t *)volc_osal_calloc(1, sizeof(volc_hal_capture_impl_t));
if (!capture) return NULL;
capture->capture_stopped = xSemaphoreCreateBinary();
if (!capture->capture_stopped) { volc_osal_free(capture); return NULL; }
// config
capture->is_started = false;
capture->media_type = config->media_type;
@@ -132,53 +137,37 @@ volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t *config)
return (volc_hal_capture_t)capture;
}
static void __start_audio_capture(volc_hal_capture_impl_t *capture)
static int __start_audio(volc_hal_capture_impl_t *capture, bool wakeup)
{
// audio_pipeline_unlink(impl->audio_capture_config.audio_pipeline);
av_processor_afe_config_t afe_config = DEFAULT_AV_PROCESSOR_AFE_CONFIG();
afe_config.ai_mode_wakeup = false;
afe_config.ai_mode_wakeup = wakeup;
audio_recorder_config_t recorder_config = DEFAULT_AUDIO_RECORDER_CONFIG();
memcpy((void *)&recorder_config.encoder_cfg, &capture->audio_capture_config.recorder_cfg, sizeof(av_processor_encoder_config_t));
memcpy(&recorder_config.afe_config, &afe_config, sizeof(av_processor_afe_config_t));
recorder_config.recorder_event_cb = capture->audio_wakeup_cb;
recorder_config.recorder_ctx = NULL;
audio_recorder_open(&recorder_config);
volc_osal_thread_param_t param = {0};
snprintf(param.name, sizeof(param.name), "%s", "volc_capture_task");
param.stack_size = 4 * 1024;
param.priority = 5;
param.stack_in_ext = 1;
capture->read_buf = volc_osal_malloc(__volc_capture_get_default_read_size(capture));
if (!capture->read_buf) return -1;
if (audio_recorder_open(&recorder_config) != ESP_OK) {
volc_osal_free(capture->read_buf); capture->read_buf = NULL;
return -1;
}
capture->is_started = true;
capture->audio_capture_config.state = CAPTURE;
esp_err_t ret = volc_osal_thread_create(&capture->capture_thread, &param, __volc_audio_capture_task, capture);
LOGI("capture task start... mode: CAPTURE");
return;
}
static void __start_audio_wake(volc_hal_capture_impl_t *capture)
{
av_processor_afe_config_t afe_config = DEFAULT_AV_PROCESSOR_AFE_CONFIG();
afe_config.ai_mode_wakeup = true;
audio_recorder_config_t recorder_config = DEFAULT_AUDIO_RECORDER_CONFIG();
memcpy((void *)&recorder_config.encoder_cfg, &capture->audio_capture_config.recorder_cfg, sizeof(av_processor_encoder_config_t));
memcpy(&recorder_config.afe_config, &afe_config, sizeof(av_processor_afe_config_t));
recorder_config.recorder_event_cb = capture->audio_wakeup_cb;
recorder_config.recorder_ctx = NULL;
audio_recorder_open(&recorder_config);
volc_osal_thread_param_t param = {0};
snprintf(param.name, sizeof(param.name), "%s", "volc_capture_task");
param.stack_size = 4 * 1024;
param.priority = 5;
param.stack_in_ext = 1;
capture->is_started = true;
capture->audio_capture_config.state = WAKEUP;
esp_err_t ret = volc_osal_thread_create(&capture->capture_thread, &param, __volc_audio_capture_task, capture);
LOGI("capture task start... mode: WAKEUP");
return;
capture->audio_capture_config.state = wakeup ? WAKEUP : CAPTURE;
// RTC Opus encoding needs the larger stack only during conversation.
if (xTaskCreatePinnedToCoreWithCaps(__volc_audio_capture_task, "volc_capture_task",
wakeup ? 4*1024 : 48*1024, capture, 5, &capture->capture_thread,
1, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT) != pdPASS) {
capture->is_started = false;
capture->capture_thread = NULL;
audio_recorder_close();
capture->audio_capture_config.state = INITED;
volc_osal_free(capture->read_buf); capture->read_buf = NULL;
return -1;
}
LOGI("capture task start... mode: %s", wakeup ? "WAKEUP" : "CAPTURE");
return 0;
}
int volc_hal_capture_start(volc_hal_capture_t capture,volc_hal_audio_capture_mode_e mode)
@@ -191,13 +180,11 @@ int volc_hal_capture_start(volc_hal_capture_t capture,volc_hal_audio_capture_mod
switch (mode)
{
case VOLC_AUDIO_MODE_WAKEUP:
__start_audio_wake(impl);
break;
return __start_audio(impl, true);
case VOLC_AUDIO_MODE_CAPTURE:
__start_audio_capture(impl);
break;
return __start_audio(impl, false);
default:
break;
return -1;
}
}
return 0;
@@ -217,6 +204,12 @@ int volc_hal_capture_stop(volc_hal_capture_t capture)
// audio_recorder_pause();
// audio_recorder_resume();
audio_recorder_close();
if (impl->capture_thread) {
xSemaphoreTake(impl->capture_stopped, portMAX_DELAY);
vTaskDeleteWithCaps(impl->capture_thread);
impl->capture_thread = NULL;
}
volc_osal_free(impl->read_buf); impl->read_buf = NULL;
impl->audio_capture_config.state = INITED;
break;
case VOLC_MEDIA_TYPE_VIDEO:
@@ -252,6 +245,7 @@ void volc_hal_capture_destroy(volc_hal_capture_t capture)
default:
break;
}
vSemaphoreDelete(impl->capture_stopped);
volc_osal_free(impl);
}
@@ -3,7 +3,7 @@
#include "volc_hal.h"
#include "volc_hal_display.h"
#include "volc_lvgl_source.h"
#include "echoear_call_status_font.h"
#include "hw_init.h"
#include "esp_lcd_panel_ops.h"
#include "esp_mmap_assets.h"
@@ -17,6 +17,7 @@
#include <stddef.h>
#include "volc_osal.h"
#include <string.h>
#include <stdatomic.h>
#include "basic_board.h"
#include "mmap_generate_eaf.h"
@@ -31,6 +32,8 @@ typedef struct volc_hal_display_impl {
gfx_obj_t *s_label;
gfx_obj_t *s_subtitle;
gfx_obj_t *s_setup_hint;
atomic_int requested_status;
int rendered_status;
esp_lcd_panel_io_handle_t display_io_handle;
SemaphoreHandle_t flush_done;
int s_current_emote;
@@ -40,6 +43,10 @@ typedef struct volc_hal_display_impl {
static volc_hal_display_impl_t* global_display_impl = NULL;
extern basic_board_periph_t global_periph;
static const char *const call_status_texts[] = {
"", "待机", "正在连接", "通话中", "正在结束", "通话结束", "连接失败", "连接中断"
};
static esp_lv_adapter_rotation_t __get_configured_rotation(void)
{
#if CONFIG_EXAMPLE_DISPLAY_ROTATION_0
@@ -157,6 +164,43 @@ static esp_err_t __create_animation(void)
return ESP_OK;
}
/* Keep status outside the eyes, in the safe area of the round panel. GFX
* caches the glyph mask; update it only when the status actually changes. */
static esp_err_t __create_status_label(void)
{
gfx_emote_lock(global_display_impl->s_gfx);
gfx_obj_t *label = gfx_label_create(global_display_impl->s_gfx);
if (!label) {
gfx_emote_unlock(global_display_impl->s_gfx);
return ESP_ERR_NO_MEM;
}
global_display_impl->s_label = label;
gfx_obj_set_size(label, 240, 42);
gfx_obj_align(label, GFX_ALIGN_TOP_MID, 0, 58);
gfx_label_set_font(label, (gfx_font_t)echoear_call_status_font());
gfx_label_set_text_align(label, GFX_TEXT_ALIGN_CENTER);
gfx_label_set_long_mode(label, GFX_LABEL_LONG_CLIP);
gfx_label_set_color(label, GFX_COLOR_HEX(0xFFFFFF));
gfx_label_set_bg_color(label, GFX_COLOR_HEX(0x000000));
gfx_label_set_bg_enable(label, true);
gfx_obj_set_visible(label, false);
esp_err_t ret = gfx_label_set_text(label, "");
gfx_emote_unlock(global_display_impl->s_gfx);
return ret;
}
static void __render_status(volc_hal_display_impl_t *display)
{
int requested = atomic_load(&display->requested_status);
if (requested == display->rendered_status) return;
gfx_emote_lock(display->s_gfx);
if (gfx_label_set_text(display->s_label, call_status_texts[requested]) == ESP_OK) {
gfx_obj_set_visible(display->s_label, requested != 0);
display->rendered_status = requested;
}
gfx_emote_unlock(display->s_gfx);
}
static void __emote_task(void *arg)
{
/* Start animation playback */
@@ -166,6 +210,7 @@ static void __emote_task(void *arg)
/* The display task owns animation playback for the lifetime of the app. */
while (1) {
__render_status(global_display_impl);
vTaskDelay(pdMS_TO_TICKS(100));
}
}
@@ -267,7 +312,10 @@ volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config)
* animation instead; application status text explains the setup state.
*/
global_display_impl->s_current_emote = MMAP_EAF_HAPPY_EAF;
atomic_init(&global_display_impl->requested_status, 0);
global_display_impl->rendered_status = -1;
__create_animation();
ESP_ERROR_CHECK(__create_status_label());
volc_osal_thread_param_t param = {0};
snprintf(param.name, sizeof(param.name), "%s", "volc_capture_task");
param.stack_size = 8 * 1024;
@@ -282,36 +330,31 @@ volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config)
return (volc_hal_display_t)global_display_impl;
}
static int display_backlight(volc_hal_display_t display, periph_ledc_handle_t **handle, uint32_t *maximum)
{
if(!display || esp_board_manager_get_device_handle("lcd_brightness",(void **)handle)!=ESP_OK || !*handle)return -1;
dev_ledc_ctrl_config_t *device=NULL;periph_ledc_config_t *config=NULL;
if(esp_board_manager_get_device_config("lcd_brightness",(void *)&device)!=ESP_OK || !device)return -1;
if(esp_board_manager_get_periph_config(device->ledc_name,(void **)&config)!=ESP_OK || !config)return -1;
if(config->duty_resolution<1 || config->duty_resolution>20)return -1;
*maximum=(1U<<(uint32_t)config->duty_resolution)-1;
return 0;
}
int volc_hal_display_set_brightness(volc_hal_display_t display, int brightness)
{
if(display == NULL) return -1;
if (brightness > 100) {
brightness = 100;
}
if (brightness < 0) {
brightness = 0;
}
periph_ledc_handle_t *ledc_handle = NULL;
esp_board_manager_get_device_handle("lcd_brightness", (void **)&ledc_handle);
dev_ledc_ctrl_config_t *dev_ledc_cfg = NULL;
esp_err_t config_ret = esp_board_manager_get_device_config("lcd_brightness", (void*)&dev_ledc_cfg);
if (config_ret != ESP_OK) {
return -1;
}
// ESP_LOGI(TAG, "dev_ledc_cfg.ledc_name: %s, name: %s, type: %s", dev_ledc_cfg->ledc_name, dev_ledc_cfg->name, dev_ledc_cfg->type);
periph_ledc_config_t *ledc_config = NULL;
esp_board_manager_get_periph_config(dev_ledc_cfg->ledc_name, (void**)&ledc_config);
uint32_t duty = (brightness * ((1 << (uint32_t)ledc_config->duty_resolution) - 1)) / 100;
// ESP_LOGI(TAG, "duty_cycle: %" PRIu32 ", speed_mode: %d, channel: %d, duty_resolution: %d", duty, ledc_handle->speed_mode, ledc_handle->channel, ledc_config->duty_resolution);
// ESP_BOARD_RETURN_ON_ERROR(ledc_set_duty(ledc_handle->speed_mode, ledc_handle->channel, duty), TAG, "LEDC set duty failed");
ledc_set_duty(ledc_handle->speed_mode, ledc_handle->channel, duty);
// ESP_BOARD_RETURN_ON_ERROR(ledc_update_duty(ledc_handle->speed_mode, ledc_handle->channel), TAG, "LEDC update duty failed");
ledc_update_duty(ledc_handle->speed_mode, ledc_handle->channel);
return 0;
// return bsp_display_brightness_set(brightness);
periph_ledc_handle_t *handle=NULL;uint32_t maximum=0;
if(display_backlight(display,&handle,&maximum))return -1;
if(brightness<0)brightness=0;if(brightness>100)brightness=100;
if(ledc_set_duty(handle->speed_mode,handle->channel,brightness*maximum/100)!=ESP_OK)return -1;
return ledc_update_duty(handle->speed_mode,handle->channel)==ESP_OK?0:-1;
}
int volc_hal_display_get_brightness(volc_hal_display_t display)
{
periph_ledc_handle_t *handle=NULL;uint32_t maximum=0;
if(display_backlight(display,&handle,&maximum))return -1;
uint32_t duty=ledc_get_duty(handle->speed_mode,handle->channel);
if(duty>maximum)return -1;
return (duty*100+maximum/2)/maximum;
}
void volc_hal_display_destroy(volc_hal_display_t display)
@@ -339,7 +382,17 @@ int volc_hal_display_set_content(volc_hal_display_t display, volc_hal_display_ob
volc_hal_display_impl_t* global_display_impl = (volc_hal_display_impl_t*)(g_hal_context->display_handle);
if(obj == VOLC_DISPLAY_OBJ_STATUS && type == VOLC_DISPLAY_TEXT){
ESP_LOGI("DISPLAY", "Status: %s", (const char *)content);
if (!content) return -1;
int requested = 0;
for (int i = 1; i < sizeof(call_status_texts) / sizeof(call_status_texts[0]); ++i) {
if (strcmp(content, call_status_texts[i]) == 0) { requested = i; break; }
}
if (strcmp(content, "轻触屏幕开始聊天") == 0 ||
strcmp(content, "轻触屏幕或说嗨,乐鑫") == 0) requested = 1;
/* RTC callbacks never wait on the renderer's full-frame lock. Other
* setup text keeps its previous behavior, with no subtitle rendering. */
atomic_store(&global_display_impl->requested_status, requested);
return 0;
}
if(obj == VOLC_DISPLAY_OBJ_SUBTITLE && type == VOLC_DISPLAY_TEXT){
ESP_LOGI("DISPLAY", "Subtitle: %s", (const char *)content);
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
#include <stddef.h>
#include <stdatomic.h>
#include "volc_hal.h"
#include "volc_hal_player.h"
@@ -38,17 +39,33 @@ typedef struct {
audio_player_config_t audio_player_config;
video_player_config_t video_player_config;
};
volatile bool is_started;
atomic_bool is_started;
atomic_int feeder_stop_phase;
StreamBufferHandle_t pcm_stream;
StaticStreamBuffer_t stream_state;
uint8_t *stream_storage;
SemaphoreHandle_t worker_stopped;
volatile bool discard_pending;
volatile bool segment_complete;
bool (*authorization_gate)(void);
atomic_bool write_in_progress;
atomic_uint last_voice_write_ms;
} volc_player_impl_t;
extern basic_board_periph_t global_periph;
static int stop_feeder_once(volc_player_impl_t *impl)
{
int expected = 0;
if (atomic_compare_exchange_strong(&impl->feeder_stop_phase, &expected, 1)) {
int ret = audio_feeder_stop();
atomic_store(&impl->feeder_stop_phase, 2);
return ret;
}
while (atomic_load(&impl->feeder_stop_phase) == 1) vTaskDelay(1);
return 0;
}
static void pcm_playback_task(void *arg)
{
volc_player_impl_t *impl = arg;
@@ -56,7 +73,22 @@ static void pcm_playback_task(void *arg)
bool primed = false;
uint64_t first_byte_ms = 0;
uint64_t last_feed_ms = 0;
bool authorization_closed = false;
uint64_t report_ms=volc_osal_get_time_ms();
unsigned fed_bytes=0,max_write_ms=0,write_errors=0;
while (impl->is_started) {
uint64_t tick_ms=volc_osal_get_time_ms();
if(tick_ms-report_ms>=1000){
LOGI("PLAYBACK stats span_ms=%u fed_ms=%u queued_ms=%u max_write_ms=%u errors=%u primed=%d",(unsigned)(tick_ms-report_ms),fed_bytes/32,(unsigned)xStreamBufferBytesAvailable(impl->pcm_stream)/32,max_write_ms,write_errors,primed);
report_ms=tick_ms;fed_bytes=max_write_ms=write_errors=0;
}
// Runs in the audio task even while the control task is blocked on HTTP.
if (authorization_closed || (impl->authorization_gate && !impl->authorization_gate())) {
if (!authorization_closed) { stop_feeder_once(impl); authorization_closed = true; }
while (xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), 0)) {}
vTaskDelay(pdMS_TO_TICKS(10));
continue;
}
if (impl->discard_pending) {
while (xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), 0)) {}
impl->discard_pending = false;
@@ -80,8 +112,10 @@ static void pcm_playback_task(void *arg)
LOGI("PLAYBACK start source=16000Hz mono PCM16 buffered=%u wait_ms=%u complete=%d",
(unsigned)available, (unsigned)(volc_osal_get_time_ms() - first_byte_ms), impl->segment_complete);
}
size_t count = xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), pdMS_TO_TICKS(30));
atomic_store(&impl->write_in_progress, true);
size_t count = xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), 0);
if (!count) {
atomic_store(&impl->write_in_progress, false);
/* An empty ingress queue does not mean the speaker has run dry:
* the downstream feeder may still hold 256 ms of PCM. Keep feeding
* arriving chunks immediately instead of inserting another prefill
@@ -91,13 +125,35 @@ static void pcm_playback_task(void *arg)
first_byte_ms = 0;
LOGI("PLAYBACK idle after %u ms without PCM", PCM_REPRIME_IDLE_MS);
}
vTaskDelay(pdMS_TO_TICKS(10));
continue;
}
if (audio_feeder_feed_data(pcm, count) == ESP_OK) {
if (impl->authorization_gate && !impl->authorization_gate()) {
atomic_store(&impl->write_in_progress, false);
continue;
}
uint64_t magnitude = 0;
for (size_t i = 0; i + 1 < count; i += 2) {
int16_t sample;
memcpy(&sample, pcm + i, sizeof(sample));
magnitude += sample < 0 ? -(int)sample : sample;
}
bool voiced = count >= 2 && magnitude / (count / 2) >= 64;
if (voiced) atomic_store(&impl->last_voice_write_ms, (uint32_t)volc_osal_get_time_ms());
// A full 1024-byte chunk represents 32 ms at 16 kHz PCM16. Allow
// the sink to consume it without dropping audio at a 20 ms timeout.
uint64_t write_ms=volc_osal_get_time_ms();
if (audio_feeder_feed_data_timeout(pcm, count, 50) == ESP_OK) {
fed_bytes+=count;
last_feed_ms = volc_osal_get_time_ms();
} else if (impl->is_started) {
write_errors++;
LOGE("PLAYBACK feeder write failed");
}
unsigned elapsed_write=(unsigned)(volc_osal_get_time_ms()-write_ms);
if (voiced) atomic_store(&impl->last_voice_write_ms, (uint32_t)volc_osal_get_time_ms());
atomic_store(&impl->write_in_progress, false);
if(elapsed_write>max_write_ms)max_write_ms=elapsed_write;
}
xSemaphoreGive(impl->worker_stopped);
vTaskDelete(NULL);
@@ -115,10 +171,7 @@ volc_hal_player_t volc_hal_player_create(volc_hal_player_config_t* config)
impl->media_type = config->media_type;
switch (impl->media_type) {
case VOLC_MEDIA_TYPE_AUDIO:
if(global_periph.play_dev){
esp_codec_dev_set_out_vol(global_periph.play_dev, DEFAULT_AUDIO_VOL);
}
impl->audio_player_config.volume = DEFAULT_AUDIO_VOL;
impl->audio_player_config.volume = volc_hal_get_device_volume();
#if (CONFIG_VOLC_AUDIO_G711A)
impl->audio_player_config.feeder_cfg.format = AV_PROCESSOR_FORMAT_ID_G711A;
impl->audio_player_config.feeder_cfg.params.g711.audio_info.sample_rate = 8000;
@@ -168,14 +221,24 @@ volc_hal_player_t volc_hal_player_create(volc_hal_player_config_t* config)
return (volc_hal_player_t)impl;
}
int volc_hal_set_device_volume(int volume){
if(volume<0 || volume>100 || !global_periph.play_dev)return -1;
return esp_codec_dev_set_out_vol(global_periph.play_dev,volume);
}
int volc_hal_get_device_volume(void){
int volume=-1;
if(!global_periph.play_dev || esp_codec_dev_get_out_vol(global_periph.play_dev,&volume)!=ESP_OK)return -1;
return volume;
}
int volc_hal_set_audio_player_volume(volc_hal_player_t player, int volume){
volc_player_impl_t* impl = (volc_player_impl_t*)player;
if (impl == NULL) {
return -1;
}
if(impl->media_type == VOLC_MEDIA_TYPE_AUDIO){
impl->audio_player_config.volume = volume;
return esp_codec_dev_set_out_vol(global_periph.play_dev, volume);
int result=volc_hal_set_device_volume(volume);
if (result==ESP_OK) impl->audio_player_config.volume = volume;
return result;
} else if(impl->media_type == VOLC_MEDIA_TYPE_VIDEO){
LOGE("media type %d not supported set volume", impl->media_type);
return -1;
@@ -187,7 +250,7 @@ int volc_hal_get_audio_player_volume(volc_hal_player_t player){
if (impl == NULL) {
return -1;
}
return impl->audio_player_config.volume;
return volc_hal_get_device_volume();
}
void volc_hal_player_destroy(volc_hal_player_t player)
@@ -231,6 +294,9 @@ int volc_hal_player_start(volc_hal_player_t player)
case VOLC_MEDIA_TYPE_AUDIO:
if (audio_feeder_run() != ESP_OK) return -1;
xStreamBufferReset(impl->pcm_stream);
atomic_store(&impl->feeder_stop_phase, 0);
atomic_store(&impl->write_in_progress, false);
atomic_store(&impl->last_voice_write_ms, (uint32_t)volc_osal_get_time_ms());
impl->is_started = true;
if (xTaskCreatePinnedToCore(pcm_playback_task, "pcm_playback", 4096, impl, 5, NULL, 1) != pdPASS) {
impl->is_started = false;
@@ -260,7 +326,7 @@ int volc_hal_player_stop(volc_hal_player_t player)
case VOLC_MEDIA_TYPE_AUDIO:
impl->is_started = false;
/* Stop the feeder first to release any in-flight blocking write. */
int ret = audio_feeder_stop();
int ret = stop_feeder_once(impl);
xSemaphoreTake(impl->worker_stopped, portMAX_DELAY);
return ret;
break;
@@ -313,3 +379,19 @@ void volc_hal_player_finish_segment(volc_hal_player_t player)
volc_player_impl_t *impl = player;
if (impl && impl->is_started) impl->segment_complete = true;
}
bool volc_hal_player_voice_quiet(volc_hal_player_t player, unsigned quiet_ms)
{
volc_player_impl_t *impl = player;
if (!impl || !impl->is_started || !impl->pcm_stream) return false;
uint32_t now = (uint32_t)volc_osal_get_time_ms();
return !atomic_load(&impl->write_in_progress) &&
xStreamBufferBytesAvailable(impl->pcm_stream) == 0 &&
now - atomic_load(&impl->last_voice_write_ms) >= quiet_ms;
}
void volc_hal_player_set_authorization_gate(volc_hal_player_t player, bool (*allowed)(void))
{
volc_player_impl_t *impl = player;
if (impl && !impl->is_started) impl->authorization_gate = allowed;
}
@@ -0,0 +1,40 @@
#ifndef CALL_DISPLAY_STATE_H
#define CALL_DISPLAY_STATE_H
#include <stdbool.h>
#include <stdint.h>
typedef enum {
CALL_DISPLAY_IDLE,
CALL_DISPLAY_CONNECTING,
CALL_DISPLAY_TALKING,
CALL_DISPLAY_ENDING,
CALL_DISPLAY_ENDED,
CALL_DISPLAY_FAILED,
CALL_DISPLAY_INTERRUPTED,
} call_display_phase_t;
typedef enum {
CALL_DISPLAY_STOP_NORMAL,
CALL_DISPLAY_STOP_FAILED,
CALL_DISPLAY_STOP_INTERRUPTED,
} call_display_stop_t;
// Owned by the conversation task. SDK callbacks only latch atomic observations;
// they never render text or mutate this lifecycle while the task is stopping.
typedef struct {
call_display_phase_t phase;
unsigned generation;
bool connected;
uint64_t terminal_until_ms;
} call_display_state_t;
unsigned call_display_begin(call_display_state_t *state);
void call_display_connected(call_display_state_t *state, unsigned generation);
void call_display_stopping(call_display_state_t *state, unsigned generation);
void call_display_finished(call_display_state_t *state, unsigned generation,
call_display_stop_t reason, bool audio_stopped, uint64_t now_ms);
bool call_display_idle_if_due(call_display_state_t *state, unsigned generation, uint64_t now_ms);
const char *call_display_text(const call_display_state_t *state);
#endif
@@ -0,0 +1,19 @@
#pragma once
#include <stdbool.h>
#include <stdint.h>
/* Owned by the conversation task. SDK callbacks only publish atomic counters. */
typedef struct {
bool pending;
bool reply_complete;
uint32_t requested_ms;
unsigned finish_baseline;
} call_voice_end_t;
bool call_voice_end_command(const char *text);
void call_voice_end_reset(call_voice_end_t *state);
void call_voice_end_user(call_voice_end_t *state, const char *text,
uint32_t now_ms, unsigned finish_serial);
bool call_voice_end_due(const call_voice_end_t *state, uint32_t now_ms,
unsigned finish_serial, bool playback_quiet,
bool user_speaking);
@@ -12,6 +12,7 @@ typedef struct {
} volc_conv_service_t;
void conv_ai_service_task(void *pvParameters);
void conv_ai_service_prepare_task(void);
void conv_ai_service_task_stop();
void conv_ai_service_init();
volc_conv_service_t get_conv_ai_service();
@@ -0,0 +1,57 @@
#include "call_display_state.h"
unsigned call_display_begin(call_display_state_t *state)
{
++state->generation;
state->phase = CALL_DISPLAY_CONNECTING;
state->connected = false;
state->terminal_until_ms = 0;
return state->generation;
}
void call_display_connected(call_display_state_t *state, unsigned generation)
{
if (state->generation == generation && state->phase == CALL_DISPLAY_CONNECTING) {
state->connected = true;
state->phase = CALL_DISPLAY_TALKING;
}
}
void call_display_stopping(call_display_state_t *state, unsigned generation)
{
if (state->generation == generation &&
(state->phase == CALL_DISPLAY_CONNECTING || state->phase == CALL_DISPLAY_TALKING)) {
state->phase = CALL_DISPLAY_ENDING;
}
}
void call_display_finished(call_display_state_t *state, unsigned generation,
call_display_stop_t reason, bool audio_stopped, uint64_t now_ms)
{
if (state->generation != generation || state->phase != CALL_DISPLAY_ENDING || !audio_stopped) return;
state->phase = reason == CALL_DISPLAY_STOP_NORMAL ? CALL_DISPLAY_ENDED :
state->connected ? CALL_DISPLAY_INTERRUPTED : CALL_DISPLAY_FAILED;
state->terminal_until_ms = now_ms + 2000;
}
bool call_display_idle_if_due(call_display_state_t *state, unsigned generation, uint64_t now_ms)
{
if (state->generation != generation ||
(state->phase != CALL_DISPLAY_ENDED && state->phase != CALL_DISPLAY_FAILED &&
state->phase != CALL_DISPLAY_INTERRUPTED) || now_ms < state->terminal_until_ms) return false;
state->phase = CALL_DISPLAY_IDLE;
return true;
}
const char *call_display_text(const call_display_state_t *state)
{
switch (state->phase) {
case CALL_DISPLAY_CONNECTING: return "正在连接";
case CALL_DISPLAY_TALKING: return "通话中";
case CALL_DISPLAY_ENDING: return "正在结束";
case CALL_DISPLAY_ENDED: return "通话结束";
case CALL_DISPLAY_FAILED: return "连接失败";
case CALL_DISPLAY_INTERRUPTED: return "连接中断";
default: return "待机";
}
}
@@ -0,0 +1,98 @@
#include "call_voice_end.h"
#include <stddef.h>
#include <string.h>
/* Match a whole short request, never a substring in a story, quote, question
* or negated instruction. This intentionally is a bounded command vocabulary. */
bool call_voice_end_command(const char *text)
{
if (!text) return false;
char normalized[256];
size_t n = 0;
static const char *punctuation[] = {",", "。", "!", "、"};
for (const char *p = text; *p;) {
bool skipped = false;
for (size_t i = 0; i < sizeof(punctuation) / sizeof(*punctuation); ++i) {
size_t len = strlen(punctuation[i]);
if (!strncmp(p, punctuation[i], len)) { p += len; skipped = true; break; }
}
if (skipped) continue;
if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n' ||
*p == ',' || *p == '.' || *p == '!') { ++p; continue; }
/* Preserve question marks, quotation marks and all other words. */
if (n + 1 >= sizeof(normalized)) return false;
normalized[n++] = *p++;
}
normalized[n] = 0;
static const char *prefixes[] = {
"好了", "好啦", "好的", "嗯", "那就", "那", "我们", "咱们",
"今天", "现在", "就先", "就", "先", "请", "麻烦你", "谢谢你", "谢谢"
};
const char *p = normalized;
for (unsigned pass = 0; pass < 8; ++pass) {
bool found = false;
for (size_t i = 0; i < sizeof(prefixes) / sizeof(*prefixes); ++i) {
size_t len = strlen(prefixes[i]);
if (!strncmp(p, prefixes[i], len)) { p += len; found = true; break; }
}
if (!found) break;
}
static const char *commands[] = {
"拜拜", "再见", "下次再聊", "下回再聊", "回头再聊",
"聊到这里", "聊到这儿", "聊到这", "不聊了", "我不聊了",
"结束聊天", "结束对话", "结束通话", "结束这次聊天", "结束本次通话",
"挂断电话", "挂断通话", "挂断", "挂了", "我挂了", "我先挂了",
"我先走了", "我要休息了", "我去休息了"
};
static const char *tails[] = {
"吧", "把", "啦", "了", "啊", "呀", "哦", "哈", "谢谢你", "谢谢",
"拜拜", "再见", "下次再聊", "下回再聊", "回头再聊"
};
for (size_t i = 0; i < sizeof(commands) / sizeof(*commands); ++i) {
size_t len = strlen(commands[i]);
if (strncmp(p, commands[i], len)) continue;
const char *tail = p + len;
for (unsigned pass = 0; *tail && pass < 8; ++pass) {
bool found = false;
for (size_t j = 0; j < sizeof(tails) / sizeof(*tails); ++j) {
size_t tlen = strlen(tails[j]);
if (!strncmp(tail, tails[j], tlen)) { tail += tlen; found = true; break; }
}
if (!found) break;
}
if (!*tail) return true;
}
return false;
}
void call_voice_end_reset(call_voice_end_t *state)
{
memset(state, 0, sizeof(*state));
}
void call_voice_end_user(call_voice_end_t *state, const char *text,
uint32_t now_ms, unsigned finish_serial)
{
if (!call_voice_end_command(text)) {
/* A later complete user utterance cancels the pending goodbye. */
call_voice_end_reset(state);
} else if (!state->pending) {
state->pending = true;
state->reply_complete = false;
state->requested_ms = now_ms;
state->finish_baseline = finish_serial;
}
/* Repeated subtitles must not extend the stop deadline. */
}
bool call_voice_end_due(const call_voice_end_t *state, uint32_t now_ms,
unsigned finish_serial, bool playback_quiet,
bool user_speaking)
{
if (!state->pending || user_speaking) return false;
uint32_t elapsed = now_ms - state->requested_ms;
/* Missing response events/audio must not leave an ended chat open forever. */
if (elapsed >= 12000u) return true;
return elapsed >= 1500u && playback_quiet &&
(state->reply_complete || finish_serial != state->finish_baseline);
}
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
#include <stdio.h>
#include <string.h>
#include <stdatomic.h>
#include "aios.h"
#include "cJSON.h"
@@ -10,6 +11,9 @@
#include "util/volc_log.h"
#include "volc_service_common.h"
#include "volc_conv.h"
#include "call_display_state.h"
#include "call_voice_end.h"
#include "esp_gmf_afe.h"
#include "volc_hal.h"
#include "volc_hal_capture.h"
#include "volc_hal_display.h"
@@ -20,12 +24,18 @@
#include "mmap_generate_eaf.h"
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/task.h"
#include "freertos/idf_additions.h"
extern int8_t get_rssi(void);
#if __cplusplus
extern "C" {
#endif
/* Opus RTP uses a 48 kHz clock in BOTH directions (RFC 7587). We submit
* externally encoded 20 ms packets, so the SDK must timestamp them with
* 960 ticks. The separate PCM encoder/decoder still uses 16 kHz / 320 samples;
* using that PCM rate for RTP compresses the transmitted audio timeline. */
#define CONV_AI_CONFIG_FORMAT "{\
\"ver\": 1,\
\"iot\": {\
@@ -50,32 +60,150 @@ extern "C" {
\"{\\\"debug\\\":{\\\"log_to_console\\\":1}}\",\
\"{\\\"rtc\\\":{\\\"access\\\":{\\\"concurrent_requests\\\":1}}}\",\
\"{\\\"rtc\\\":{\\\"ice\\\":{\\\"concurrent_agents\\\":1}}}\",\
\"{\\\"rtc\\\":{\\\"thread\\\":{\\\"stack_in_ext\\\":1}}}\",\
\"{\\\"rtc\\\":{\\\"thread\\\":{\\\"stack_in_ext\\\":1,\\\"stack_size\\\":49152}}}\",\
\"{\\\"rtc\\\":{\\\"network\\\":{\\\"enable_audio_jitter2\\\":0}}}\",\
\"{\\\"rtc\\\":{\\\"report\\\":{\\\"enable\\\":0}}}\",\
\"{\\\"audio\\\":{\\\"codec\\\":{\\\"pcma\\\":{\\\"s_samples_per_frame\\\":480}}}}\"\
\"{\\\"audio\\\":{\\\"codec\\\":{\\\"opus\\\":{\\\"s_sample_rate\\\":48000,\\\"s_samples_per_frame\\\":960,\\\"r_sample_rate\\\":48000,\\\"r_samples_per_frame\\\":960}}}}\"\
]\
}\
}"
#define CONV_SERVICE_TIMEOUT 60
/* CALL_IDLE_BEGIN: callbacks only update atomic activity, never stop the SDK.
* Continuous microphone/silence packets and lease renewals are not activity.
* Unsigned monotonic milliseconds also handle the 32-bit clock wrap. */
static atomic_uint last_call_activity_ms;
static atomic_bool local_speech_active;
static void call_idle_activity(void)
{
atomic_store(&last_call_activity_ms, (uint32_t)volc_osal_get_time_ms());
}
static bool call_idle_expired(void)
{
uint32_t now = (uint32_t)volc_osal_get_time_ms();
return (uint32_t)(now - atomic_load(&last_call_activity_ms)) >= CONV_SERVICE_TIMEOUT * 1000u;
}
static void call_idle_capture_event(void *event, void *user_data)
{
(void)user_data;
const esp_gmf_afe_evt_t *vad = event;
if (!vad) return;
if (vad->type == ESP_GMF_AFE_EVT_VAD_START || vad->type == ESP_GMF_AFE_EVT_VAD_END) {
atomic_store(&local_speech_active, vad->type == ESP_GMF_AFE_EVT_VAD_START);
call_idle_activity();
}
}
static bool call_idle_pcm_active(const void *data, size_t bytes)
{
if (!data || bytes < sizeof(int16_t) || (bytes & 1)) return false;
const int16_t *pcm = data;
uint64_t magnitude = 0;
size_t samples = bytes / sizeof(*pcm);
for (size_t i = 0; i < samples; ++i) {
magnitude += pcm[i] < 0 ? -(int)pcm[i] : pcm[i];
}
// RTC sends decoded silence continuously, even when the agent is quiet.
// Measure synthesized audio before the user's speaker volume is applied.
return magnitude / samples >= 64;
}
/* CALL_IDLE_END */
static char config_buf[1024] = {0};
static volc_conv_service_t conv_service = {0};
static volatile bool is_stop = false;
static volatile bool provider_disconnected = false;
static atomic_bool is_stop = true;
static atomic_bool stop_requested = false;
static atomic_bool provider_disconnected = false;
static atomic_bool display_connected = false;
static atomic_bool user_stop_requested = false;
static call_display_state_t call_display;
static volc_hal_display_t global_display = NULL;
static bool session_started_reported = false;
static QueueHandle_t platform_report_queue;
static volatile bool diagnostic_say_pending;
static call_voice_end_t voice_end;
static atomic_uint voice_finish_serial;
static atomic_uint last_voice_received_ms;
/* VOICE_DIAGNOSTIC_BEGIN: local USB requests only. TTS keeps microphone uplink
* suppressed until this call ends or an explicit diag-say restores it. */
enum { DIAGNOSTIC_NONE, DIAGNOSTIC_SAY, DIAGNOSTIC_TTS, DIAGNOSTIC_PENDING_MASK = 3,
DIAGNOSTIC_MUTE = 4, DIAGNOSTIC_CLOSING = 8, DIAGNOSTIC_ACTIVE = 16 };
static atomic_uint diagnostic_state;
extern bool platform_runtime_rtc(void);
bool conv_ai_service_diagnostic_active(void)
{
return (atomic_load(&diagnostic_state) & DIAGNOSTIC_ACTIVE) != 0;
}
void conv_ai_service_diagnostic_reset(void)
{
atomic_fetch_and(&diagnostic_state, DIAGNOSTIC_CLOSING);
}
static void diagnostic_close(void)
{
atomic_store(&diagnostic_state, DIAGNOSTIC_CLOSING);
}
static void diagnostic_prepare(void)
{
atomic_fetch_and(&diagnostic_state, ~DIAGNOSTIC_CLOSING);
}
static bool diagnostic_uplink_suppressed(void)
{
return (atomic_load(&diagnostic_state) & DIAGNOSTIC_MUTE) != 0;
}
static unsigned diagnostic_take_pending(void)
{
return atomic_fetch_and(&diagnostic_state, ~DIAGNOSTIC_PENDING_MASK) & DIAGNOSTIC_PENDING_MASK;
}
static void diagnostic_request(int mode)
{
unsigned state = atomic_load(&diagnostic_state);
do {
if ((state & DIAGNOSTIC_CLOSING) && xTaskGetHandle("conv_ai")) {
LOGI("PLAYBACK diagnostic request rejected: call closing");
return;
}
} while (!atomic_compare_exchange_weak(&diagnostic_state, &state,
(unsigned)mode | DIAGNOSTIC_ACTIVE |
(mode == DIAGNOSTIC_TTS ? DIAGNOSTIC_MUTE : 0)));
/* Repeating the fixed passage reuses the existing room. */
if (!xTaskGetHandle("conv_ai") && aios_event_pub(VOLC_SERVICE_AI_CONVERSATION, NULL, NULL) != 0) {
conv_ai_service_diagnostic_reset();
}
}
void conv_ai_service_diagnostic_say(void)
{
diagnostic_say_pending = true;
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION, NULL, NULL);
diagnostic_request(DIAGNOSTIC_SAY);
}
void conv_ai_service_diagnostic_tts(void)
{
diagnostic_request(DIAGNOSTIC_TTS);
}
static void diagnostic_send_tts(void)
{
static const char *passage = "这是连续语音播放测试。请您听一听,这段声音是否清楚、连贯,中间有没有突然停顿。我们用自然的语速,把这段话平稳地读完。";
int sent = platform_runtime_rtc()
? volc_send_text_to_agent(conv_service.engine, passage, VOLC_AGENT_TYPE_TTS, 1) : -2;
LOGI("PLAYBACK diagnostic TTS requested ret=%d mic_suppressed=1", sent);
if (sent != 0) conv_ai_service_diagnostic_reset();
conv_service.wait_time = 0;
call_idle_activity();
}
/* VOICE_DIAGNOSTIC_END */
/* Implemented by the platform client in the top-level firmware component. */
extern const char *platform_runtime_instance_id(void);
extern const char *platform_runtime_product_key(void);
@@ -83,12 +211,39 @@ extern const char *platform_runtime_device_name(void);
extern const char *platform_runtime_device_secret(void);
extern int platform_conversation_begin(void);
extern const char *platform_last_error_code(void);
extern const char *call_client_start_error(void);
extern const char *platform_conversation_bot_id(void);
extern const char *platform_conversation_update_json(void);
extern int platform_conversation_started(const char *provider_connection_ref);
extern int platform_conversation_poll(void);
extern bool platform_conversation_audio_authorized(void);
extern int platform_conversation_turn(const char *speaker, const char *source_id, const char *text);
extern void platform_conversation_finish(bool failed, const char *error_code);
extern bool platform_runtime_rtc(void);
extern const char *platform_runtime_rtc_app_id(void);
extern void platform_runtime_engine_ready(bool ready);
extern void platform_conversation_observe(const char *type);
extern void platform_conversation_stop_result(bool started,int result);
static atomic_bool rtc_connected;
extern const char *volc_current_user_id(volc_engine_t handle,bool remote);
static void publish_call_display(unsigned generation)
{
if (call_display.generation != generation) return;
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,
VOLC_DISPLAY_TEXT, call_display_text(&call_display));
}
static void observe_call_connection(unsigned generation)
{
if (atomic_load(&display_connected)) call_display_connected(&call_display, generation);
publish_call_display(generation);
}
static bool conversation_output_allowed(void)
{
return !stop_requested && platform_conversation_audio_authorized();
}
volc_conv_service_t get_conv_ai_service()
{
@@ -98,11 +253,13 @@ volc_conv_service_t get_conv_ai_service()
// volc_hal_file_t* audioFile = NULL;
// int a_cnt = 0;
static void __audio_capture_cb(volc_hal_capture_t capture, const void* data, int len, volc_hal_frame_info_t* frame_info){
if (diagnostic_uplink_suppressed()) return;
volc_audio_frame_info_t info = {0};
info.commit = false;
info.data_type = frame_info->data_type;
if(!is_stop)
if(!is_stop && !stop_requested && platform_conversation_audio_authorized())
{
if (atomic_load(&local_speech_active)) call_idle_activity();
static uint64_t report_ms, magnitude;
static unsigned samples, peak, frames, failures, max_send_ms;
const int16_t *pcm = data;
@@ -140,6 +297,9 @@ static void __on_volc_event(volc_engine_t handle, volc_event_t *event, void *use
{
case VOLC_EV_CONNECTED:
LOGI("Volc Engine connected");
call_idle_activity();
if (!stop_requested) atomic_store(&display_connected, true);
if(platform_runtime_rtc()){atomic_store(&rtc_connected,true);platform_conversation_observe("MEDIA_CONNECTED");}
break;
case VOLC_EV_DISCONNECTED:
LOGI( "Volc Engine disconnected");
@@ -154,33 +314,34 @@ static void __on_volc_event(volc_engine_t handle, volc_event_t *event, void *use
static void __on_volc_conversation_status(volc_engine_t handle, volc_conv_status_e status, void *user_data)
{
LOGI("AUDIO conversation status=%d",(int)status);
// Turn events are not call lifecycle events. A delayed ANSWER_FINISH must
// neither end the call nor overwrite its terminal status.
if (stop_requested) return;
if (status != VOLC_CONV_STATUS_LISTENING) call_idle_activity();
int index = MMAP_EAF_HAPPY_EAF;
switch (status)
{
case VOLC_CONV_STATUS_LISTENING:
index = MMAP_EAF_EMOTION_BLINK_FAST_284_126_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体聆听中");
break;
case VOLC_CONV_STATUS_THINKING:
index = MMAP_EAF_EMOTION_DIZZY_284_126_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体思考中");
break;
case VOLC_CONV_STATUS_ANSWERING:
index = MMAP_EAF_HAPPY_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体说话中");
break;
case VOLC_CONV_STATUS_INTERRUPTED:
{
volc_hal_context_t *hal = volc_get_global_hal_context();
if (hal) volc_hal_player_discard_pending(hal->player_handle[VOLC_HAL_PLAYER_AUDIO]);
}
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体被打断");
break;
case VOLC_CONV_STATUS_ANSWER_FINISH:
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体说话完成");
atomic_fetch_add(&voice_finish_serial, 1);
break;
default:
break;
@@ -204,7 +365,13 @@ static void __on_volc_audio_data(volc_engine_t handle, const void *data_ptr, siz
LOGE("player pipeline is NULL");
return;
}
if(!is_stop){
if(!is_stop && !stop_requested && platform_conversation_audio_authorized()){
if (info_ptr && info_ptr->data_type == VOLC_AUDIO_DATA_TYPE_PCM &&
call_idle_pcm_active(data_ptr, data_len)) {
call_idle_activity();
atomic_store(&last_voice_received_ms, (uint32_t)volc_osal_get_time_ms());
}
if(data_len)platform_conversation_observe("FIRST_AUDIO_RECEIVED");
volc_hal_player_play_data(player_handle, data_ptr, data_len);
}
}
@@ -241,13 +408,15 @@ static void __on_subtitle_message_received(const cJSON* root) {
cJSON* text_obj = cJSON_GetObjectItem(obji, "text");
cJSON* definite_obj = cJSON_GetObjectItem(obji, "definite");
char* sub = cJSON_GetStringValue(text_obj);
if(!sub)continue;
if (sub[0] && !stop_requested) call_idle_activity();
if(sub_offset<0||(size_t)sub_offset>strlen(sub))sub_offset=0;
// print_string(sub + sub_offset -1);
if(cJSON_IsTrue(definite_obj)){
sub_offset = sub_offset == 0 ? 0 : sub_offset -1;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,sub + sub_offset);
sub_offset = 0;
aios_event_pub(VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE, sub, NULL);
} else if(strlen(sub) - sub_offset > 18){
sub_offset = sub_offset == 0 ? 0 : sub_offset -1;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,sub + sub_offset);
@@ -259,6 +428,62 @@ static void __on_subtitle_message_received(const cJSON* root) {
}
}
/* Called only by the conversation task, never from an SDK callback. */
static struct {
char text[512];
double round, sequence;
bool active, complete, overflow;
} voice_user_subtitle;
static void process_voice_end_transcript(const char *speaker, const char *text, bool complete)
{
if (!complete || !text || !*text || is_stop || stop_requested) return;
if (!strcmp(speaker, "subject")) {
call_voice_end_user(&voice_end, text, (uint32_t)volc_osal_get_time_ms(),
atomic_load(&voice_finish_serial));
} else if (!strcmp(speaker, "agent") && voice_end.pending) {
voice_end.reply_complete = true;
}
}
static void process_voice_end_rtc_user(const cJSON *line)
{
cJSON *round = cJSON_GetObjectItemCaseSensitive(line, "roundId");
cJSON *sequence = cJSON_GetObjectItemCaseSensitive(line, "sequence");
const char *text = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(line, "text"));
if (!cJSON_IsNumber(round) || !cJSON_IsNumber(sequence) || !text ||
!cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line, "definite")) || is_stop || stop_requested) return;
if (voice_user_subtitle.active && round->valuedouble == voice_user_subtitle.round &&
sequence->valuedouble <= voice_user_subtitle.sequence) return;
if (!voice_user_subtitle.active || round->valuedouble != voice_user_subtitle.round || voice_user_subtitle.complete) {
memset(&voice_user_subtitle, 0, sizeof(voice_user_subtitle));
voice_user_subtitle.active = true;
voice_user_subtitle.round = round->valuedouble;
}
voice_user_subtitle.sequence = sequence->valuedouble;
size_t used = strlen(voice_user_subtitle.text), len = strlen(text);
if (used + len + 1 >= sizeof(voice_user_subtitle.text)) voice_user_subtitle.overflow = true;
if (!voice_user_subtitle.overflow) {
memcpy(voice_user_subtitle.text + used, text, len + 1);
}
if (cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line, "paragraph"))) {
voice_user_subtitle.complete = true;
/* Overflow disables matching but still cancels a pending goodbye. */
process_voice_end_transcript("subject", voice_user_subtitle.overflow ? "continue" : voice_user_subtitle.text, true);
}
}
static bool voice_end_ready(void)
{
if (!voice_end.pending) return false;
volc_hal_context_t *hal = volc_get_global_hal_context();
uint32_t now = (uint32_t)volc_osal_get_time_ms();
bool quiet = now - atomic_load(&last_voice_received_ms) >= 1200u && hal &&
volc_hal_player_voice_quiet(hal->player_handle[VOLC_HAL_PLAYER_AUDIO], 600u);
return call_voice_end_due(&voice_end, now, atomic_load(&voice_finish_serial),
quiet, atomic_load(&local_speech_active));
}
/* HTTPS reporting must never run inside the WebSocket receive mutex. */
static void process_platform_reports(void)
{
@@ -266,6 +491,24 @@ static void process_platform_reports(void)
while (platform_report_queue && xQueueReceive(platform_report_queue, &root, 0) == pdTRUE) {
cJSON *type_obj = cJSON_GetObjectItemCaseSensitive(root, "type");
const char *type = cJSON_IsString(type_obj) ? type_obj->valuestring : "";
if (!strcmp(type,"subtitle") && platform_runtime_rtc()) {
cJSON *line=NULL;cJSON_ArrayForEach(line,cJSON_GetObjectItemCaseSensitive(root,"data")){
const char *text=cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(line,"text"));
const char *user=cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(line,"userId"));
cJSON *seq=cJSON_GetObjectItemCaseSensitive(line,"sequence");
if(text&&user&&cJSON_IsNumber(seq)&&cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line,"definite"))){
char key[240];snprintf(key,sizeof(key),"%s:%.0f",user,seq->valuedouble);
const char *local=volc_current_user_id(conv_service.engine,false),*remote=volc_current_user_id(conv_service.engine,true);
const char *speaker=local&&!strcmp(user,local)?"subject":remote&&!strcmp(user,remote)?"agent":"unknown";
/* definite is a final clause; paragraph confirms the whole
* utterance, so "拜拜,不对,我还想聊" cannot end early. */
if (!strcmp(speaker, "subject")) process_voice_end_rtc_user(line);
else process_voice_end_transcript(speaker, text,
cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line,"paragraph")));
(void)platform_conversation_turn(speaker,key,text);
}
}
} else
if ((strcmp(type, "session.updated") == 0 || strcmp(type, "session.created") == 0) && !session_started_reported) {
cJSON *session = cJSON_GetObjectItemCaseSensitive(root, "session");
cJSON *id = cJSON_IsObject(session) ? cJSON_GetObjectItemCaseSensitive(session, "id") : NULL;
@@ -280,12 +523,14 @@ static void process_platform_reports(void)
cJSON *text = cJSON_GetObjectItemCaseSensitive(root, "transcript");
cJSON *source = cJSON_GetObjectItemCaseSensitive(root, "event_id");
if (cJSON_IsString(text) && text->valuestring[0]) {
process_voice_end_transcript("subject", text->valuestring, true);
(void)platform_conversation_turn("subject", cJSON_IsString(source) ? source->valuestring : "subject", text->valuestring);
}
} else if (strcmp(type, "response.audio_transcript.done") == 0) {
cJSON *text = cJSON_GetObjectItemCaseSensitive(root, "transcript");
cJSON *source = cJSON_GetObjectItemCaseSensitive(root, "response_id");
if (cJSON_IsString(text) && text->valuestring[0]) {
process_voice_end_transcript("agent", text->valuestring, true);
(void)platform_conversation_turn("agent", cJSON_IsString(source) ? source->valuestring : "agent", text->valuestring);
}
}
@@ -293,9 +538,19 @@ static void process_platform_reports(void)
}
}
static bool subtitle_has_final_clause(const cJSON *root)
{
cJSON *line = NULL;
cJSON_ArrayForEach(line, cJSON_GetObjectItemCaseSensitive(root, "data")) {
if (cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line, "definite"))) return true;
}
return false;
}
static void __on_volc_message_data(volc_engine_t handle, const void *message, size_t size, volc_message_info_t *info_ptr, void *user_data)
{
static char message_buffer[4096];
if(size>=sizeof(message_buffer)-1){platform_conversation_observe("GAP_DECLARED");return;}
if (size > 0 && size < sizeof(message_buffer) - 1) {
memcpy(message_buffer, message, size);
message_buffer[size] = 0;
@@ -306,9 +561,10 @@ static void __on_volc_message_data(volc_engine_t handle, const void *message, si
if (standard_protocol) {
const char *type = cJSON_GetStringValue(cJSON_GetObjectItem(root, "type"));
if (type && (!strcmp(type, "error") || !strcmp(type, "conversation.item.created"))) {
LOGI("Provider control event: %s", message_buffer);
LOGI("Provider control event type: %s", type);
}
if (type && !strcmp(type, "response.audio.done") && !is_stop) {
atomic_fetch_add(&voice_finish_serial, 1);
/* Flush the final sub-frame through the PCM decoder. */
static const uint8_t silence[2048] = {0};
volc_hal_context_t *hal = volc_get_global_hal_context();
@@ -320,8 +576,10 @@ static void __on_volc_message_data(volc_engine_t handle, const void *message, si
}
if (type && (strstr(type, "speech_") || strstr(type, "response."))) {
conv_service.wait_time = 0;
if (!stop_requested) call_idle_activity();
}
bool report = type && (!strcmp(type, "session.created") || !strcmp(type, "session.updated") ||
(!strcmp(type, "subtitle") && platform_runtime_rtc() && subtitle_has_final_clause(root)) ||
!strcmp(type, "conversation.item.input_audio_transcription.completed") ||
!strcmp(type, "response.audio_transcript.done"));
if (report && platform_report_queue && xQueueSend(platform_report_queue, &root, 0) == pdTRUE) {
@@ -333,6 +591,13 @@ static void __on_volc_message_data(volc_engine_t handle, const void *message, si
} else if (message_buffer[0] == 's' && message_buffer[1] == 'u' && message_buffer[2] == 'b' && message_buffer[3] == 'v') {
__on_subtitle_message_received(root);
conv_service.wait_time = 0;
/* Interim ASR updates must not fill the bounded control queue
* and evict the actual completed goodbye. Display still gets all. */
if(platform_runtime_rtc() && subtitle_has_final_clause(root)){
if(platform_report_queue&&xQueueSend(platform_report_queue,&root,0)==pdTRUE)root=NULL;
else platform_conversation_observe("GAP_DECLARED");
}
cJSON_Delete(root);
} else if (message_buffer[0] == 't' && message_buffer[1] == 'o' && message_buffer[2] == 'o' && message_buffer[3] == 'l') {
// function calling 消息
aios_event_pub(VOLC_FUNCTION_CALL_EXEC, root, NULL);
@@ -360,7 +625,16 @@ void conv_ai_service_init()
.on_volc_message_data = __on_volc_message_data,
};
int ret = volc_create(&conv_service.engine, config_buf, &conv_service.volc_event_handler, &conv_service);
cJSON *runtime=cJSON_Parse(config_buf);
if(platform_runtime_rtc()){
cJSON_AddStringToObject(cJSON_GetObjectItem(runtime,"iot"),"rtc_app_id",platform_runtime_rtc_app_id());
cJSON_SetNumberValue(cJSON_GetObjectItem(cJSON_GetObjectItem(cJSON_GetObjectItem(runtime,"rtc"),"audio"),"codec"),1);
}
else cJSON_DeleteItemFromObjectCaseSensitive(runtime,"rtc");
char *runtime_config=cJSON_PrintUnformatted(runtime);cJSON_Delete(runtime);
int ret = volc_create(&conv_service.engine, runtime_config, &conv_service.volc_event_handler, &conv_service);
if(runtime_config){memset(runtime_config,0,strlen(runtime_config));free(runtime_config);}
platform_runtime_engine_ready(ret==0);
if (ret != 0) {
LOGE("volc_create failed, ret: %d", ret);
return;
@@ -369,14 +643,42 @@ void conv_ai_service_init()
}
}
void conv_ai_service_prepare_task(void)
{
// Called synchronously by the serialized manager before creating the task.
diagnostic_prepare();
stop_requested = false;
is_stop = true;
atomic_store(&user_stop_requested, false);
atomic_store(&display_connected, false);
atomic_store(&local_speech_active, false);
call_voice_end_reset(&voice_end);
memset(&voice_user_subtitle, 0, sizeof(voice_user_subtitle));
atomic_store(&voice_finish_serial, 0);
atomic_store(&last_voice_received_ms, (uint32_t)volc_osal_get_time_ms());
/* The previous SDK has stopped before the manager admits a new call. */
cJSON *stale;
while (platform_report_queue && xQueueReceive(platform_report_queue, &stale, 0) == pdTRUE) cJSON_Delete(stale);
call_idle_activity();
call_display_begin(&call_display);
}
void conv_ai_service_task(void *pvParameters)
{
LOGI("TASK_DIAGNOSTIC name=conv_ai present=1 priority=%u core=%d",
(unsigned)uxTaskPriorityGet(NULL), (int)xTaskGetCoreID(NULL));
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
char* status_str = NULL;
bool provider_started = false;
bool audio_started = false;
bool local_audio_stopped = true;
call_display_stop_t display_stop = CALL_DISPLAY_STOP_FAILED;
unsigned display_generation = call_display.generation;
char *params = NULL;
is_stop = true;
session_started_reported = false;
provider_disconnected = false;
atomic_store(&rtc_connected,false);
if (g_hal_context == NULL) {
LOGE("volc_get_global_hal_context failed");
status_str = "系统未初始化,ai对话连接失败";
@@ -384,8 +686,9 @@ void conv_ai_service_task(void *pvParameters)
}
global_display = g_hal_context->display_handle;
int ret = 0;
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT, "ai对话创建中");
publish_call_display(display_generation);
if (stop_requested) goto CONV_AI_QUIT;
ret = platform_conversation_begin();
if (ret != 0) {
LOGE("platform_conversation_begin failed, ret: %d", ret);
@@ -393,8 +696,14 @@ void conv_ai_service_task(void *pvParameters)
if (strcmp(platform_last_error_code(), "FIRMWARE_UPDATE_REQUIRED") == 0) {
status_str = "平台要求更新固件";
}
if (strcmp(call_client_start_error(), "DEVICE_UNBOUND") == 0) {
status_str = "设备没有绑定人员,请先绑定";
} else if (strcmp(call_client_start_error(), "VOICE_AGENT_NOT_READY") == 0) {
status_str = "老人的语音助手尚未就绪";
}
goto CONV_AI_QUIT;
}
if (stop_requested || !platform_conversation_audio_authorized()) goto CONV_AI_QUIT;
/* Capture may run while the WebSocket is still handshaking. Hold frames
* until volc_start has moved the engine into STARTED state. */
@@ -404,7 +713,7 @@ void conv_ai_service_task(void *pvParameters)
volc_hal_capture_config_t capture_audio_config = {0};
capture_audio_config.media_type = VOLC_MEDIA_TYPE_AUDIO;
capture_audio_config.data_cb = __audio_capture_cb;
// capture_audio_config.audio_wakeup_cb = (volc_hal_audio_wakeup_cb_t)rec_engine_cb;
capture_audio_config.audio_wakeup_cb = call_idle_capture_event;
capture_audio_config.user_data = g_hal_context;
volc_hal_capture_t capture_handle = g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO];
@@ -413,9 +722,13 @@ void conv_ai_service_task(void *pvParameters)
capture_handle = g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO];
}
audio_started = true;
volc_hal_capture_stop(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO]);
volc_hal_capture_update_config(capture_handle, &capture_audio_config);
volc_hal_capture_start(g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO], VOLC_AUDIO_MODE_CAPTURE);
if (!capture_handle ||
volc_hal_capture_stop(capture_handle) != 0 ||
volc_hal_capture_update_config(capture_handle, &capture_audio_config) != 0 ||
volc_hal_capture_start(capture_handle, VOLC_AUDIO_MODE_CAPTURE) != 0) {
status_str = "收音启动失败";
goto CONV_AI_QUIT;
}
volc_hal_player_config_t player_audio_config = {0};
player_audio_config.media_type = VOLC_MEDIA_TYPE_AUDIO;
@@ -423,20 +736,22 @@ void conv_ai_service_task(void *pvParameters)
g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO] = volc_hal_player_create(&player_audio_config);
}
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, "ai对话连接中");
publish_call_display(display_generation);
volc_hal_player_set_authorization_gate(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO], conversation_output_allowed);
volc_hal_player_start(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
// step 2: start ai conversation
const char *session_update = platform_conversation_update_json();
size_t params_size = strlen(session_update) + 64;
char *params = volc_osal_malloc(params_size);
params = volc_osal_malloc(params_size);
if (params == NULL) {
status_str = "会话配置内存不足";
goto CONV_AI_QUIT;
}
snprintf(params, params_size, "{\"audio\":{\"codec\":0},\"session_update\":%s}", session_update);
if(platform_runtime_rtc())snprintf(params,params_size,"%s",session_update);
else snprintf(params, params_size, "{\"audio\":{\"codec\":0},\"session_update\":%s}", session_update);
volc_opt_t opt = {
.mode = VOLC_MODE_WS,
.mode = platform_runtime_rtc()?VOLC_MODE_RTC:VOLC_MODE_WS,
.bot_id = (char *)platform_conversation_bot_id(),
.params = params,
};
@@ -444,46 +759,91 @@ void conv_ai_service_task(void *pvParameters)
aios_event_pub(VOLC_LOCAL_LOGIC_PLAY_WELCOME, NULL, NULL);
#endif
ret = volc_start(conv_service.engine, &opt);
volc_osal_free(params);
if (ret != 0) {
LOGE("volc_start failed, ret: %d", ret);
status_str = "ai对话连接失败,准备退出";
goto CONV_AI_QUIT;
}
provider_started = true;
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, "ai对话已连接");
call_idle_activity();
if (stop_requested || !platform_conversation_audio_authorized()) goto CONV_AI_QUIT;
// A successful start request does not prove that the media transport joined.
observe_call_connection(display_generation);
is_stop = false;
while (!is_stop && !provider_disconnected) {
while (!is_stop && !stop_requested && !provider_disconnected && platform_conversation_audio_authorized()) {
observe_call_connection(display_generation);
if(platform_runtime_rtc()&&atomic_load(&rtc_connected))session_started_reported=true;
process_platform_reports();
if (session_started_reported && platform_conversation_poll()) {
status_str = "平台已结束本次对话";
if (voice_end_ready()) {
LOGI("VOICE_END user request completed");
status_str = "本次对话已结束";
atomic_store(&user_stop_requested, true);
display_stop = CALL_DISPLAY_STOP_NORMAL;
goto CONV_AI_QUIT;
}
if (diagnostic_say_pending && session_started_reported) {
diagnostic_say_pending = false;
if ((platform_runtime_rtc()||session_started_reported) && platform_conversation_poll()) {
status_str = "平台已结束本次对话";
display_stop = atomic_load(&display_connected) ? CALL_DISPLAY_STOP_NORMAL : CALL_DISPLAY_STOP_FAILED;
goto CONV_AI_QUIT;
}
if ((atomic_load(&diagnostic_state) & DIAGNOSTIC_PENDING_MASK) != DIAGNOSTIC_NONE && session_started_reported && (!platform_runtime_rtc() || volc_current_user_id(conv_service.engine,true))) {
unsigned diagnostic = diagnostic_take_pending();
if (diagnostic == DIAGNOSTIC_TTS) {
diagnostic_send_tts();
} else if (diagnostic == DIAGNOSTIC_SAY) {
char request[640];
snprintf(request, sizeof(request), "{\"event_id\":\"diag-%llu\",\"type\":\"conversation.item.create\",\"item\":{\"type\":\"message\",\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"请只说这句话:你好,这是语音播放测试。现在使用自然的语速,连续清楚地说完这句话。\"}],\"interrupt_mode\":1}}", (unsigned long long)volc_osal_get_time_ms());
volc_message_info_t info = {.is_binary = false};
int sent = volc_send_message(conv_service.engine, request, strlen(request), &info);
int sent = platform_runtime_rtc()
? volc_send_text_to_agent(conv_service.engine,"请只说这句话:你好,这是语音播放测试。",VOLC_AGENT_TYPE_LLM,1)
: volc_send_message(conv_service.engine, request, strlen(request), &info);
LOGI("PLAYBACK diagnostic sentence requested ret=%d", sent);
const char *respond = "{\"type\":\"response.create\",\"response\":{\"modalities\":[\"text\",\"audio\"]}}";
volc_send_message(conv_service.engine, respond, strlen(respond), &info);
if (!platform_runtime_rtc()) volc_send_message(conv_service.engine, respond, strlen(respond), &info);
conv_service.wait_time = 0;
call_idle_activity();
}
}
sleep(1);
conv_service.wait_time++;
if(conv_service.wait_time >= CONV_SERVICE_TIMEOUT){
if (call_idle_expired()) {
LOGI("CALL_IDLE timeout seconds=%d", CONV_SERVICE_TIMEOUT);
status_str = "ai对话已超时,准备退出";
display_stop = atomic_load(&display_connected) ? CALL_DISPLAY_STOP_NORMAL : CALL_DISPLAY_STOP_FAILED;
goto CONV_AI_QUIT;
}
}
status_str = "ai对话已断开";
display_stop = atomic_load(&provider_disconnected) ? CALL_DISPLAY_STOP_INTERRUPTED :
atomic_load(&display_connected) ? CALL_DISPLAY_STOP_NORMAL : CALL_DISPLAY_STOP_FAILED;
CONV_AI_QUIT:
// Latch a connection that arrived immediately before hangup, without
// publishing a misleading new TALKING frame while teardown has started.
if (atomic_load(&display_connected)) call_display_connected(&call_display, display_generation);
is_stop = true;
// volc_hal_file_close(audioFile);
volc_stop(conv_service.engine);
stop_requested = true;
diagnostic_close();
call_display_stopping(&call_display, display_generation);
publish_call_display(display_generation);
// Local privacy boundary precedes SDK shutdown and all HTTP requests.
if (g_hal_context && audio_started) {
local_audio_stopped = volc_hal_capture_stop(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO]) == 0;
if (g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]) {
volc_hal_player_discard_pending(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
if (volc_hal_player_stop(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]) != 0) local_audio_stopped = false;
}
}
int stop_result=volc_stop(conv_service.engine);
platform_conversation_stop_result(provider_started,stop_result);
if (atomic_load(&user_stop_requested)) display_stop = CALL_DISPLAY_STOP_NORMAL;
if (provider_started && stop_result != 0) display_stop = CALL_DISPLAY_STOP_INTERRUPTED;
// Show the local terminal state before an end-report HTTP request can block.
call_display_finished(&call_display, display_generation, display_stop,
local_audio_stopped, volc_osal_get_time_ms());
publish_call_display(display_generation);
// The SDK may retain opt.params until stop has released the session.
if (params) { volc_osal_free(params); params = NULL; }
process_platform_reports();
platform_conversation_finish(!provider_started || !session_started_reported,
!provider_started ? "VOLC_START_FAILED" : (!session_started_reported ? "SESSION_UPDATE_NOT_CONFIRMED" : NULL));
@@ -497,25 +857,40 @@ CONV_AI_QUIT:
}
}
if(status_str != NULL){
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, status_str);
}
if (status_str != NULL) LOGI("Call finished: %s", status_str);
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE, VOLC_DISPLAY_TEXT, "");
conv_service.wait_time = 0;
// just show the status: Ai对话已断开
sleep(1);
// No detached timer: the old task owns this brief terminal display. Restore
// idle before publishing OVER, so a new call can never be overwritten by an
// old timeout. Slow end-report requests already count toward the hold time.
if (local_audio_stopped) {
while (!call_display_idle_if_due(&call_display, display_generation, volc_osal_get_time_ms())) {
vTaskDelay(pdMS_TO_TICKS(50));
}
publish_call_display(display_generation);
}
if (g_hal_context && audio_started) {
volc_hal_capture_start(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO],VOLC_AUDIO_MODE_WAKEUP);
}
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_OVER, NULL, NULL);
#ifdef ESP_PLATFORM
// The manager releases this WithCaps task from another task's context.
// Plain vTaskDelete(NULL) does not free WithCaps stacks/TCBs.
for (;;) vTaskSuspend(NULL);
#else
volc_osal_thread_exit(NULL);
#endif
}
void conv_ai_service_task_stop(void)
{
// sleep for AI Agent say goodbye
sleep(1);
atomic_store(&user_stop_requested, true);
stop_requested = true;
is_stop = true;
diagnostic_close();
// Capture callbacks drop frames immediately. The independent audio worker
// closes its feeder through the authorization gate, including blocked HTTP.
// Only the conversation task destroys handles; hook callbacks never race it.
}
#if __cplusplus
@@ -11,6 +11,11 @@
#include "volc_service_common.h"
#include "volc_conv_service.h"
#include "volc_conv.h"
#ifdef ESP_PLATFORM
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#endif
extern void conv_ai_service_diagnostic_reset(void);
#if __cplusplus
extern "C" {
@@ -31,6 +36,14 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
}
volc_hal_display_t global_display = g_hal_context->display_handle;
switch (e->id) {
case VOLC_SERVICE_AI_CONVERSATION_TOGGLE:
// This manager owns admission. Connecting and stopping tasks remain
// active until OVER, so repeated taps cannot create another call.
if (me->conv_thread_id != NULL) {
conv_ai_service_task_stop();
return AIOS_Ret_Handled;
}
/* fall through */
case VOLC_SERVICE_AI_CONVERSATION_START:
if (me->conv_thread_id == NULL) {
volc_osal_thread_param_t param = {0};
@@ -39,7 +52,10 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
param.stack_size = 8*1024;
param.priority = 5;
param.stack_in_ext = 1;
volc_osal_thread_create(&me->conv_thread_id, &param, conv_ai_service_task,NULL);
conv_ai_service_prepare_task();
if (volc_osal_thread_create(&me->conv_thread_id, &param, conv_ai_service_task,NULL) != 0) {
conv_ai_service_diagnostic_reset();
}
}
return AIOS_Ret_Handled;
case VOLC_SERVICE_AI_CONVERSATION_INTERRUPT:
@@ -48,12 +64,20 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
case VOLC_SERVICE_AI_CONVERSATION_QUIT:
if (me->conv_thread_id != NULL) {
conv_ai_service_task_stop();
me->conv_thread_id = NULL;
// The old task owns SDK teardown and reports OVER after cleanup.
// Keep admission closed until then, including rapid hook toggles.
}
return AIOS_Ret_Handled;
// notice the Ai_Conversation over by task
case VOLC_SERVICE_AI_CONVERSATION_OVER:
if (me->conv_thread_id != NULL) {
#ifdef ESP_PLATFORM
// OSAL stores the native handle in an allocated handle slot.
// OVER is the worker's final action; reclaim its WithCaps
// stack and TCB before admitting another conversation.
vTaskDeleteWithCaps(*(TaskHandle_t *)me->conv_thread_id);
#endif
volc_osal_thread_destroy(me->conv_thread_id);
me->conv_thread_id = NULL;
}
return AIOS_Ret_Handled;
@@ -65,6 +89,7 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
static aios_ret_t __state_init(volc_conv_service_manager_t * const me, aios_event_t const * const e)
{
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_START); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_TOGGLE);
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_INTERRUPT); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_QUIT); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_OVER); // 订阅事件
@@ -14,6 +14,7 @@ enum {
VOLC_SERVICE_AI_CONVERSATION_INTERRUPT = 101,
VOLC_SERVICE_AI_CONVERSATION_QUIT = 102, // control the Ai_Conversation task quit
VOLC_SERVICE_AI_CONVERSATION_OVER = 103, // notice the Ai_Conversation over by task
VOLC_SERVICE_AI_CONVERSATION_TOGGLE = 104, // screen tap: start idle or stop active
// function_call event:
VOLC_FUNCTION_CALL_EXEC = 200,
@@ -21,7 +21,6 @@ typedef struct volc_local_logic_service {
} volc_local_logic_service_t;
volc_local_logic_service_t g_volc_local_logic_service;
static int has_quit = 0;
static void __play_welcome(){
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
if(g_hal_context == NULL) return;
@@ -42,50 +41,20 @@ static void __play_welcome(){
return;
}
static int __has_substring(const char *haystack, const char *needle) {
// 处理边界:子串为空串,按惯例返回包含(可根据需求调整)
if (needle[0] == '\0') {
return 1;
}
// strstr 返回非 NULL 表示找到子串
return strstr(haystack, needle) != NULL ? 1 : 0;
}
static void __parser_and_process_subtitle(char *subtitle)
{
volc_conv_service_t conv_service = get_conv_ai_service();
if (__has_substring(subtitle, "退下吧") || __has_substring(subtitle, "拜拜"))
{
if (!has_quit)
{
volc_send_text_to_agent(conv_service.engine, "好的拜拜", VOLC_AGENT_TYPE_TTS, 1);
has_quit = 1;
}
}
if (__has_substring(subtitle, "好的拜拜"))
{
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_QUIT, NULL, NULL);
has_quit = 0;
}
}
static aios_ret_t __state_conversation(volc_local_logic_service_t * const me, aios_event_t const * const e)
{
switch (e->id) {
case VOLC_LOCAL_LOGIC_PLAY_WELCOME:
__play_welcome();
return AIOS_Ret_Handled;
case VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE:
__parser_and_process_subtitle(e->data);
return AIOS_Ret_Handled;
}
}
static aios_ret_t __state_init(volc_local_logic_service_t * const me, aios_event_t const * const e)
{
AIOS_EVENT_SUB(VOLC_LOCAL_LOGIC_PLAY_WELCOME); // 订阅事件
AIOS_EVENT_SUB(VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE); // 订阅事件
// Voice ending is handled by the conversation task with speaker identity,
// complete user transcripts and playback drain; never by substring events.
return AIOS_TRAN(__state_conversation);
}
@@ -28,6 +28,8 @@
#include "av_pipeline_pool.h"
#include "esp_gmf_fifo.h"
#include "esp_codec_dev.h"
#include <stdatomic.h>
#include "freertos/semphr.h"
#include "esp_gmf_new_databus.h"
#include "esp_vad.h"
@@ -104,6 +106,18 @@ typedef struct {
static audio_manager_t audio_manager;
static audio_recorder_t audio_recorder;
static atomic_bool microphone_blocked, recorder_closing, recorder_suspended;
static StaticSemaphore_t recorder_mutex_storage;
static SemaphoreHandle_t recorder_mutex;
static portMUX_TYPE recorder_init_lock=portMUX_INITIALIZER_UNLOCKED;
static SemaphoreHandle_t recorder_control_lock(void) {
portENTER_CRITICAL(&recorder_init_lock);
if(!recorder_mutex)recorder_mutex=xSemaphoreCreateMutexStatic(&recorder_mutex_storage);
portEXIT_CRITICAL(&recorder_init_lock);
return recorder_mutex;
}
static atomic_uint microphone_inflight, microphone_reads;
static atomic_uint recorder_readers;
static audio_feeder_t audio_feeder;
static audio_playback_t audio_playback;
static audio_prompt_t audio_prompt;
@@ -163,8 +177,10 @@ static inline esp_opus_enc_frame_duration_t convert_opus_enc_frame_duration(int
static int audio_render_write_hdlr(uint8_t *pcm_data, uint32_t len, void *ctx)
{
ESP_LOGD(TAG, "Audio render write handler, data size: %ld", len);
esp_codec_dev_write(audio_manager.config.play_dev, pcm_data, len);
return ESP_OK;
int result=esp_codec_dev_write(audio_manager.config.play_dev, pcm_data, len);
extern void platform_conversation_observe(const char *type);
if(result==ESP_OK&&len)platform_conversation_observe("FIRST_PLAYBACK_STARTED");
return result;
}
static int playback_out_data_callback(uint8_t *data, int data_size, void *ctx)
@@ -240,7 +256,18 @@ static int recorder_outport_release_write(void *handle, esp_gmf_data_bus_block_t
static int recorder_inport_acquire_read(void *handle, esp_gmf_data_bus_block_t *blk, int wanted_size, int block_ticks)
{
// Increment before checking the gate, so standby can await every read that
// crossed the gate. While suspended no microphone samples enter the AFE.
for (;;) {
atomic_fetch_add(&microphone_inflight,1);
if (!atomic_load(&microphone_blocked)) break;
atomic_fetch_sub(&microphone_inflight,1);
if(atomic_load(&recorder_closing)){blk->valid_size=0;return ESP_GMF_IO_FAIL;}
vTaskDelay(pdMS_TO_TICKS(10));
}
int read_ret = esp_codec_dev_read(audio_manager.config.rec_dev, blk->buf, wanted_size);
atomic_fetch_add(&microphone_reads,1);
atomic_fetch_sub(&microphone_inflight,1);
if (read_ret != ESP_CODEC_DEV_OK) {
ESP_LOGE(TAG, "Microphone read failed: %d", read_ret);
blk->valid_size = 0;
@@ -531,6 +558,10 @@ static void recorder_cleanup_on_error(void)
esp_gmf_afe_manager_destroy(audio_recorder.afe_manager);
audio_recorder.afe_manager = NULL;
}
if (audio_recorder.pool) {
av_audio_recorder_pool_deinit(audio_recorder.pool);
audio_recorder.pool = NULL;
}
}
static esp_err_t recorder_configure_all_encoders(void)
@@ -662,7 +693,9 @@ static int feeder_outport_release_write(void *handle, esp_gmf_data_bus_block_t *
if (audio_manager.enable_mixer) {
esp_audio_render_stream_write(audio_manager.stream[AUDIO_RENDER_STREAM_FEEDER], blk->buf, blk->valid_size);
} else {
esp_codec_dev_write(audio_manager.config.play_dev, blk->buf, blk->valid_size);
int result=esp_codec_dev_write(audio_manager.config.play_dev, blk->buf, blk->valid_size);
extern void platform_conversation_observe(const char *type);
if(result==ESP_OK&&blk->valid_size)platform_conversation_observe("FIRST_PLAYBACK_STARTED");
}
return blk->valid_size;
}
@@ -1122,7 +1155,7 @@ esp_err_t audio_prompt_play(const char *url)
return ESP_OK;
}
esp_err_t audio_recorder_open(audio_recorder_config_t *config)
static esp_err_t recorder_open_internal(audio_recorder_config_t *config)
{
ESP_LOGI(TAG, "Opening audio recorder...");
@@ -1170,10 +1203,24 @@ esp_err_t audio_recorder_open(audio_recorder_config_t *config)
cleanup_and_exit:
ESP_LOGE(TAG, "Failed to open audio recorder, cleaning up...");
atomic_store(&recorder_closing,true);
recorder_cleanup_on_error();
atomic_store(&recorder_closing,false);
return ESP_FAIL;
}
esp_err_t audio_recorder_open(audio_recorder_config_t *config)
{
SemaphoreHandle_t lock=recorder_control_lock();xSemaphoreTake(lock,portMAX_DELAY);
esp_err_t e=recorder_open_internal(config);
atomic_store(&recorder_suspended,false);
if(e==ESP_OK && atomic_load(&microphone_blocked)) {
e=esp_gmf_afe_manager_suspend(audio_recorder.afe_manager,true)==ESP_GMF_ERR_OK?ESP_OK:ESP_FAIL;
atomic_store(&recorder_suspended,e==ESP_OK);
}
xSemaphoreGive(lock);return e;
}
esp_err_t audio_recorder_pause(void)
{
if (audio_recorder.state == AUDIO_RUN_STATE_CLOSED) {
@@ -1227,7 +1274,7 @@ esp_err_t audio_recorder_get_afe_manager_handle(esp_gmf_afe_manager_handle_t *af
}
esp_err_t audio_recorder_close(void)
static esp_err_t recorder_close_internal(void)
{
if (audio_recorder.state == AUDIO_RUN_STATE_CLOSED) {
ESP_LOGW(TAG, "Audio recorder is already closed");
@@ -1241,8 +1288,10 @@ esp_err_t audio_recorder_close(void)
if (audio_recorder.pipe) {
esp_gmf_fifo_abort(audio_recorder.fifo);
esp_gmf_pipeline_stop(audio_recorder.pipe);
audio_recorder.pipe = NULL;
}
// Abort wakes blocked readers. Keep the FIFO and pipeline alive until each
// reader has released its block; retain the pipeline handle for destroy.
while (atomic_load(&recorder_readers)) vTaskDelay(1);
if (audio_recorder.task) {
esp_gmf_task_deinit(audio_recorder.task);
audio_recorder.task = NULL;
@@ -1277,20 +1326,31 @@ esp_err_t audio_recorder_close(void)
return ESP_OK;
}
esp_err_t audio_recorder_close(void)
{
SemaphoreHandle_t lock=recorder_control_lock();xSemaphoreTake(lock,portMAX_DELAY);
atomic_store(&recorder_closing,true);
esp_err_t e=recorder_close_internal();
atomic_store(&recorder_closing,false);
xSemaphoreGive(lock);return e;
}
esp_err_t audio_recorder_read_data(uint8_t *data, int data_size)
{
if (!data || data_size <= 0) {
ESP_LOGE(TAG, "Invalid parameters for recorder read");
return ESP_FAIL;
}
if (audio_recorder.state == AUDIO_RUN_STATE_CLOSED || !audio_recorder.fifo) {
ESP_LOGE(TAG, "Recorder not available for reading");
atomic_fetch_add(&recorder_readers, 1);
if (atomic_load(&recorder_closing) || audio_recorder.state == AUDIO_RUN_STATE_CLOSED || !audio_recorder.fifo) {
atomic_fetch_sub(&recorder_readers, 1);
return ESP_FAIL;
}
esp_gmf_data_bus_block_t blk = {0};
int ret = esp_gmf_fifo_acquire_read(audio_recorder.fifo, &blk, data_size, portMAX_DELAY);
if (ret < 0) {
// ESP_LOGE(TAG, "Failed to acquire read from recorder FIFO (%d)", ret);
atomic_fetch_sub(&recorder_readers, 1);
return ret;
}
int bytes_to_copy = blk.valid_size;
@@ -1299,15 +1359,37 @@ esp_err_t audio_recorder_read_data(uint8_t *data, int data_size)
bytes_to_copy = data_size;
}
memcpy(data, blk.buf, bytes_to_copy);
esp_gmf_fifo_release_read(audio_recorder.fifo, &blk, portMAX_DELAY);
#if CONFIG_MEDIA_DUMP_ENABLE && CONFIG_MEDIA_DUMP_AUDIO_AFTER_AEC
media_dump_feed(NULL, (uint8_t *)blk.buf, blk.valid_size);
#endif /* CONFIG_MEDIA_DUMP_ENABLE && CONFIG_MEDIA_DUMP_AUDIO_AFTER_AEC */
esp_gmf_fifo_release_read(audio_recorder.fifo, &blk, portMAX_DELAY);
atomic_fetch_sub(&recorder_readers, 1);
return bytes_to_copy;
}
esp_err_t audio_feeder_feed_data(uint8_t *data, int data_size)
{
return audio_feeder_feed_data_timeout(data, data_size, UINT32_MAX);
}
esp_err_t audio_recorder_set_standby(bool standby)
{
SemaphoreHandle_t lock=recorder_control_lock();xSemaphoreTake(lock,portMAX_DELAY);
atomic_store(&microphone_blocked,standby);
esp_err_t e=ESP_OK;
if(audio_recorder.afe_manager && esp_gmf_afe_manager_suspend(audio_recorder.afe_manager,standby)!=ESP_GMF_ERR_OK)e=ESP_FAIL;
atomic_store(&recorder_suspended,standby && e==ESP_OK);
if(standby && e==ESP_OK) {
for(int i=0;i<100 && atomic_load(&microphone_inflight);i++)vTaskDelay(pdMS_TO_TICKS(10));
if(atomic_load(&microphone_inflight))e=ESP_ERR_TIMEOUT;
}
xSemaphoreGive(lock);return e;
}
bool audio_recorder_is_standby(void) { return atomic_load(&microphone_blocked) && atomic_load(&recorder_suspended) && !atomic_load(&microphone_inflight); }
uint32_t audio_recorder_microphone_reads(void) { return atomic_load(&microphone_reads); }
esp_err_t audio_feeder_feed_data_timeout(uint8_t *data, int data_size, uint32_t timeout_ms)
{
TickType_t wait_ticks = timeout_ms == UINT32_MAX ? portMAX_DELAY : pdMS_TO_TICKS(timeout_ms);
if (!data || data_size <= 0) {
ESP_LOGE(TAG, "Invalid parameters for playback feed");
return ESP_FAIL;
@@ -1319,17 +1401,17 @@ esp_err_t audio_feeder_feed_data(uint8_t *data, int data_size)
esp_gmf_data_bus_block_t blk = {0};
if (audio_feeder.config.decoder_cfg.format == AV_PROCESSOR_FORMAT_ID_PCM) {
int ret = esp_gmf_db_acquire_write(audio_feeder.fifo, &blk, data_size, portMAX_DELAY);
int ret = esp_gmf_db_acquire_write(audio_feeder.fifo, &blk, data_size, wait_ticks);
if (ret < 0) {
ESP_LOGE(TAG, "Failed to acquire write to playback FIFO (0x%x)", ret);
return ESP_FAIL;
}
blk.buf = (uint8_t *)data;
blk.valid_size = data_size;
ret = esp_gmf_db_release_write(audio_feeder.fifo, &blk, portMAX_DELAY);
ret = esp_gmf_db_release_write(audio_feeder.fifo, &blk, wait_ticks);
ESP_GMF_RET_ON_NOT_OK(TAG, ret, {return ESP_FAIL;}, "Failed to release write to playback FIFO");
} else {
int ret = esp_gmf_db_acquire_write(audio_feeder.fifo, &blk, data_size, portMAX_DELAY);
int ret = esp_gmf_db_acquire_write(audio_feeder.fifo, &blk, data_size, wait_ticks);
if (ret < 0) {
ESP_LOGE(TAG, "Failed to acquire write to playback FIFO (0x%x)", ret);
return ESP_FAIL;
@@ -1338,7 +1420,7 @@ esp_err_t audio_feeder_feed_data(uint8_t *data, int data_size)
memcpy(blk.buf, data, bytes_to_copy);
blk.valid_size = bytes_to_copy;
ret = esp_gmf_db_release_write(audio_feeder.fifo, &blk, portMAX_DELAY);
ret = esp_gmf_db_release_write(audio_feeder.fifo, &blk, wait_ticks);
ESP_GMF_RET_ON_NOT_OK(TAG, ret, {return ESP_FAIL;}, "Failed to release write to playback FIFO");
}
return ESP_OK;
@@ -251,6 +251,10 @@ esp_err_t audio_recorder_pause(void);
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_resume(void);
// Pause microphone reads and AFE wake detection without destroying audio tasks.
esp_err_t audio_recorder_set_standby(bool standby);
bool audio_recorder_is_standby(void);
uint32_t audio_recorder_microphone_reads(void);
/**
* @brief Get the AFE manager handle of the audio recorder
@@ -333,6 +337,8 @@ esp_err_t audio_feeder_stop(void);
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_feeder_feed_data(uint8_t *data, int data_size);
/* Bounded queue waits for clients with a local audio authorization deadline. */
esp_err_t audio_feeder_feed_data_timeout(uint8_t *data, int data_size, uint32_t timeout_ms);
/**
* @brief Open the audio playback module
@@ -36,7 +36,7 @@ endif()
idf_component_register(
SRCS ${VOLC_SRCS}
INCLUDE_DIRS ${VOLC_INCS}
REQUIRES mbedtls json lwip esp_netif
REQUIRES mbedtls json lwip esp_netif esp_audio_codec
)
if(CONFIG_VOLC_RTC_MODE)
@@ -98,7 +98,7 @@ int volc_device_register(volc_iot_info_t* info, char** output)
char* json_str = cJSON_PrintUnformatted(root);
snprintf(url, sizeof(url), "%s%s?%s&%s", VOLC_IOT_HOST, VOLC_DYNAMIC_REGISTER_PATH, VOLC_API_ACTION_DYNAMIC_REGISTER, VOLC_API_VERSION_QUERY_PARAM);
LOGD("url: %s, body: %s", url, json_str);
LOGD("device registration request prepared");
char* response = volc_http_post(url, json_str, strlen(json_str));
if (response == NULL) {
@@ -140,8 +140,8 @@ int volc_device_register(volc_iot_info_t* info, char** output)
err_out_label:
HAL_SAFE_FREE(payload);
HAL_SAFE_FREE(response);
HAL_SAFE_FREE(root);
HAL_SAFE_FREE(response_json);
cJSON_Delete(root);
cJSON_Delete(response_json);
HAL_SAFE_FREE(signature);
HAL_SAFE_FREE(json_str);
return ret;
@@ -170,17 +170,19 @@ int volc_get_rtc_config(volc_iot_info_t* info, int audio_codec, const char* bot_
if(params){
cJSON* params_config = cJSON_Parse(params);
cJSON* agent_config = cJSON_CreateObject();
cJSON* agent_config = NULL;
volc_json_read_object(params_config,"AgentConfig",&agent_config);
cJSON_AddItemToObject(root, "agent_config", agent_config);
if (agent_config) cJSON_AddItemToObject(root, "agent_config", agent_config);
cJSON* config = cJSON_CreateObject();
cJSON* config = NULL;
volc_json_read_object(params_config,"Config",&config);
cJSON_AddItemToObject(root, "config", config);
// Omit Config when using the elder's already-configured bot defaults.
if (config) cJSON_AddItemToObject(root, "config", config);
cJSON_Delete(params_config);
}
char* json_str = cJSON_PrintUnformatted(root);
snprintf(url, sizeof(url), "%s%s?%s&%s", VOLC_IOT_HOST, VOLC_GET_RTC_CONFIG_PATH, VOLC_API_ACTION_GET_RTC_CONFIG, VOLC_API_VERSION_QUERY_PARAM);
LOGI("url: %s, body: %s", url, json_str);
LOGI("RTC configuration request prepared");
char* response = volc_http_post(url, json_str, strlen(json_str));
if (response == NULL) {
LOGE("Failed to get response from server");
@@ -193,7 +195,12 @@ int volc_get_rtc_config(volc_iot_info_t* info, int audio_codec, const char* bot_
volc_json_read_string(response_json, "Result.Token", &room_info->rtc_opt.p_token);
volc_json_read_string(response_json, "Result.TaskID", &room_info->task_id);
if (room_info->rtc_opt.p_channel_name == NULL || room_info->rtc_opt.p_uid == NULL || room_info->rtc_opt.p_token == NULL || room_info->task_id == NULL) {
LOGE("Failed to get RTC config from server: %s", response);
char *provider_code = NULL;
volc_json_read_string(response_json, "ResponseMetadata.Error.Code", &provider_code);
LOGE("RTC config rejected code=%s fields room=%d user=%d token=%d task=%d", provider_code ? provider_code : "NO_ERROR_CODE",
room_info->rtc_opt.p_channel_name != NULL, room_info->rtc_opt.p_uid != NULL,
room_info->rtc_opt.p_token != NULL, room_info->task_id != NULL);
HAL_SAFE_FREE(provider_code);
ret = -1;
goto err_out_label;
}
@@ -26,6 +26,7 @@ void volc_rtc_destroy(volc_rtc_t rtc);
int volc_rtc_start(volc_rtc_t rtc, const char* bot_id, volc_iot_info_t* iot_info, const char* params);
int volc_rtc_stop(volc_rtc_t rtc);
const char *volc_rtc_user_id(volc_rtc_t rtc,bool remote);
int volc_rtc_send(volc_rtc_t rtc, const void* data, int size, volc_data_info_t* data_info);
@@ -6,6 +6,30 @@
#include <string.h>
#include <unistd.h>
#include <inttypes.h>
#include <stdatomic.h>
#ifdef ESP_PLATFORM
#include "esp_opus_enc.h"
#include "esp_opus_dec.h"
#include "esp_heap_caps.h"
#include "esp_timer.h"
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "freertos/idf_additions.h"
/* Own complete transport packets: SDK callback memory is borrowed. Storage and
* the decoder task stack use PSRAM; only small RTOS control objects use DRAM. */
#define RTC_RX_PACKET_MAX 4096
#define RTC_RX_PACKET_COUNT 48
#define RTC_RX_ROOM_MAX 256
typedef struct {
uint32_t generation;
uint32_t arrival_ms;
size_t size;
uint8_t data[RTC_RX_PACKET_MAX];
} rtc_rx_packet_t;
#endif
#include "volc_osal.h"
#include "util/volc_list.h"
@@ -18,12 +42,17 @@
#define MAGIC_OFFSET 8
const char* interrupt_str = "{\"Command\":\"interrupt\"}";
// Optional platform control hook. Audio remains on the RTC transport. Invoke
// before joining/publishing so the cloud recorder can capture the opening turn.
extern int volc_platform_recording_prepare(const char *app, const char *room, const char *user) __attribute__((weak));
typedef struct {
bool b_pipeline_started;
bool b_user_joined;
bool b_channel_joined;
bool b_first_keyframe_received;
bool b_fini;
atomic_bool b_fini;
atomic_bool closing;
bool b_audio_publish;
bool b_video_publish;
bool b_audio_subscribe;
@@ -40,8 +69,46 @@ typedef struct {
volc_data_cb data_callback;
byte_rtc_engine_t rtc;
byte_rtc_event_handler_t event_handler;
uint8_t pcm_frame[640]; // 20 ms, 16 kHz, mono PCM16
size_t pcm_used;
uint64_t rx_report_ms, rx_previous_ms;
unsigned rx_frames, rx_pcm_bytes, rx_gap_ms, rx_decode_ms, rx_errors;
#ifdef ESP_PLATFORM
void *opus_encoder;
void *opus_decoder;
uint8_t *opus_packet;
int opus_packet_size;
uint8_t decoded_pcm[3840]; // Maximum 120 ms at 16 kHz, mono PCM16.
rtc_rx_packet_t *rx_packets;
QueueHandle_t rx_free, rx_pending;
SemaphoreHandle_t rx_work_lock, rx_worker_stopped;
TaskHandle_t rx_worker;
atomic_bool rx_accepting, rx_shutdown;
atomic_uint rx_generation, rx_producers;
atomic_uint rx_callbacks, rx_callback_us, rx_callback_max_us;
atomic_uint rx_free_max_us, rx_copy_max_us, rx_send_max_us, rx_post_max_us;
atomic_uint rx_overflow, rx_oversized, rx_stale, rx_peak;
char rx_room[RTC_RX_ROOM_MAX];
uint32_t rx_decoder_generation;
unsigned rx_applied_generation;
unsigned rx_decode_total_us;
#endif
} rtc_impl_t;
#ifdef ESP_PLATFORM
static int __rtc_rx_init(rtc_impl_t *rtc);
static int __rtc_rx_start(rtc_impl_t *rtc, const char *room);
static void __rtc_rx_stop(rtc_impl_t *rtc);
static void __rtc_rx_destroy(rtc_impl_t *rtc);
#endif
static void __clear_room_info(rtc_impl_t *rtc) {
HAL_SAFE_FREE(rtc->info.rtc_opt.p_channel_name);
HAL_SAFE_FREE(rtc->info.rtc_opt.p_uid);
HAL_SAFE_FREE(rtc->info.rtc_opt.p_token);
HAL_SAFE_FREE(rtc->info.task_id);
}
static bool __is_first_keyframe_not_received(rtc_impl_t* rtc, int is_key_frame)
{
return (!rtc->b_first_keyframe_received && !is_key_frame);
@@ -80,7 +147,7 @@ static int __rtc_start(rtc_impl_t* rtc, volc_rtc_option_t* option)
{
volc_opt_t* p_opt = NULL;
byte_rtc_room_options_t room_opt = {0};
if (!rtc || !option) {
if (!rtc || !option || !option->p_channel_name || !option->p_uid || rtc->b_pipeline_started) {
LOGE("rtc or option is NULL");
return -1;
}
@@ -89,43 +156,74 @@ static int __rtc_start(rtc_impl_t* rtc, volc_rtc_option_t* option)
room_opt.auto_publish_video = rtc->b_video_publish;
room_opt.auto_subscribe_audio = rtc->b_audio_subscribe;
room_opt.auto_subscribe_video = rtc->b_video_subscribe;
LOGI("Joining channel: %s, uid: %s, token: %s, vpub: %d, vsub: %d, apub: %d, asub: %d", option->p_channel_name, option->p_uid, option->p_token, (int)room_opt.auto_publish_video, (int)room_opt.auto_subscribe_video, (int)room_opt.auto_publish_audio, (int)room_opt.auto_subscribe_audio);
#ifndef ESP_PLATFORM
rtc->rx_report_ms=rtc->rx_previous_ms=0;
rtc->rx_frames=rtc->rx_pcm_bytes=rtc->rx_gap_ms=rtc->rx_decode_ms=rtc->rx_errors=0;
#endif
rtc->p_channel_name = strdup(option->p_channel_name);
rtc->p_user_id = strdup(option->p_uid);
if (!rtc->p_channel_name || !rtc->p_user_id) {
HAL_SAFE_FREE(rtc->p_channel_name);
HAL_SAFE_FREE(rtc->p_user_id);
return -1;
}
#ifdef ESP_PLATFORM
if (rtc->opus_encoder && __rtc_rx_start(rtc, option->p_channel_name) != 0) {
HAL_SAFE_FREE(rtc->p_channel_name);
HAL_SAFE_FREE(rtc->p_user_id);
return -1;
}
#endif
rtc->b_pipeline_started = true;
LOGI("Joining RTC channel: vpub=%d vsub=%d apub=%d asub=%d", (int)room_opt.auto_publish_video, (int)room_opt.auto_subscribe_video, (int)room_opt.auto_publish_audio, (int)room_opt.auto_subscribe_audio);
int ret = byte_rtc_join_room(rtc->rtc, option->p_channel_name, option->p_uid, option->p_token, &room_opt);
if (ret != 0) {
LOGE("Failed to join room: %d", ret);
#ifdef ESP_PLATFORM
__rtc_rx_stop(rtc);
#endif
rtc->b_pipeline_started = rtc->b_channel_joined = rtc->b_user_joined = false;
HAL_SAFE_FREE(rtc->p_channel_name);
HAL_SAFE_FREE(rtc->p_user_id);
return ret;
}
rtc->b_pipeline_started = true;
rtc->p_channel_name = strdup(option->p_channel_name);
rtc->p_user_id = strdup(option->p_uid);
return 0;
}
static void __rtc_stop(rtc_impl_t* rtc)
static int __rtc_stop(rtc_impl_t* rtc)
{
if (!rtc) {
LOGE("rtc instance is NULL");
return;
return -1;
}
#ifdef ESP_PLATFORM
/* Close delivery even when leave_room subsequently fails. */
__rtc_rx_stop(rtc);
#endif
if (!rtc->b_pipeline_started) {
LOGI("pipeline not started");
return;
return 0;
}
int ret = byte_rtc_leave_room(rtc->rtc, rtc->p_channel_name);
if (ret != 0) {
LOGE("Failed to leave room: %d", ret);
return;
return ret;
}
rtc->b_pipeline_started = false;
rtc->pcm_used = 0;
rtc->b_channel_joined = false;
rtc->b_user_joined = false;
HAL_SAFE_FREE(rtc->p_channel_name);
HAL_SAFE_FREE(rtc->p_user_id);
HAL_SAFE_FREE(rtc->p_remote_user_id);
return;
return 0;
}
static void __send_data_2_user(rtc_impl_t* rtc, const void* data, int data_len, volc_data_info_t* info) {
if (rtc->data_callback) {
if (!atomic_load(&rtc->closing) && rtc->data_callback) {
rtc->data_callback(rtc->context, data, data_len, info);
}
}
@@ -137,7 +235,7 @@ static void _register_message_router(rtc_impl_t* rtc, volc_msg_cb callback)
static void _send_message_2_user(rtc_impl_t* rtc, volc_msg_t* msg)
{
if (rtc->message_callback) {
if (!atomic_load(&rtc->closing) && rtc->message_callback) {
rtc->message_callback(rtc->context, msg);
}
}
@@ -164,7 +262,7 @@ static int _on_conversion_status_message_parsed(uint8_t* message, rtc_impl_t* rt
volc_json_read_int(root, "Stage.Code", &c);
volc_json_read_string(root, "ErrorInfo.Reason", &error_reason);
if(error_reason != NULL) {
LOGW("ErrorInfo.Reason %s",error_reason);
LOGW("RTC error detail withheld");
if(strcmp(error_reason, "preParamCheck:AI license duration quota exceeded") == 0) {
msg.code = VOLC_MSG_QUOTA_EXCEEDED;
msg.data.conv_status = 0;
@@ -177,8 +275,52 @@ static int _on_conversion_status_message_parsed(uint8_t* message, rtc_impl_t* rt
return c;
}
/* RTC_SELF_PRIORITY_BEGIN: the SDK must not yield to its priority-5 decoder
* while publishing a packet. Apply a base-priority floor in the SDK's own
* callbacks, without borrowing task handles or lowering a higher base. */
#ifdef ESP_PLATFORM
#define RTC_TASK_PRIORITY_FLOOR 6
static atomic_bool diagnostic_rtc_seen;
static atomic_uint diagnostic_rtc_effective_priority, diagnostic_rtc_base, diagnostic_rtc_observed_ms;
static atomic_int diagnostic_rtc_core;
void volc_rtc_diagnostic_snapshot(void)
{
bool seen = atomic_load(&diagnostic_rtc_seen);
unsigned base = atomic_load(&diagnostic_rtc_base);
LOGI("RTC_TASK_SNAPSHOT seen=%d priority=%u base=%u core=%d floor=%u observed_age_ms=%u floor_met=%d",
seen, atomic_load(&diagnostic_rtc_effective_priority), base, atomic_load(&diagnostic_rtc_core), RTC_TASK_PRIORITY_FLOOR,
seen ? (unsigned)((uint32_t)(esp_timer_get_time()/1000) - atomic_load(&diagnostic_rtc_observed_ms)) : 0,
seen && base >= RTC_TASK_PRIORITY_FLOOR);
}
static void __rtc_diagnostic_priority_apply(void)
{
if (strcmp(pcTaskGetName(NULL), "VolcRTCMain")) return;
TaskStatus_t status;
vTaskGetInfo(NULL, &status, pdFALSE, eInvalid);
UBaseType_t before = uxTaskPriorityGet(NULL);
UBaseType_t desired = status.uxBasePriority < RTC_TASK_PRIORITY_FLOOR ? RTC_TASK_PRIORITY_FLOOR : status.uxBasePriority;
if (status.uxBasePriority != desired) vTaskPrioritySet(NULL, desired);
if (!atomic_load(&diagnostic_rtc_seen) || status.uxBasePriority != desired) {
LOGI("RTC_PRIORITY_FLOOR before=%u base_before=%u after=%u base_after=%u floor=%u core=%d",
(unsigned)before, (unsigned)status.uxBasePriority, (unsigned)uxTaskPriorityGet(NULL), (unsigned)desired, RTC_TASK_PRIORITY_FLOOR,
(int)xTaskGetCoreID(NULL));
}
atomic_store(&diagnostic_rtc_effective_priority, uxTaskPriorityGet(NULL));
atomic_store(&diagnostic_rtc_base, desired);
atomic_store(&diagnostic_rtc_core, (int)xTaskGetCoreID(NULL));
atomic_store(&diagnostic_rtc_observed_ms, (uint32_t)(esp_timer_get_time()/1000));
atomic_store(&diagnostic_rtc_seen, true);
}
#else
static void __rtc_diagnostic_priority_apply(void) {}
#endif
/* RTC_SELF_PRIORITY_END */
static void _on_join_channel_success(byte_rtc_engine_t engine, const char* channel, int elapsed_ms, bool rejoin)
{
__rtc_diagnostic_priority_apply();
volc_msg_t msg = {0};
LOGI("join channel success %s elapsed %d ms\n", channel, elapsed_ms);
rtc_impl_t* rtc = (rtc_impl_t*) byte_rtc_get_user_data(engine);
@@ -196,8 +338,9 @@ static void _on_user_joined(byte_rtc_engine_t engine, const char* channel, const
volc_msg_t msg = {0};
LOGI("remote user joined %s:%s elapsed %d ms\n", channel, user_name, elapsed_ms);
rtc->b_user_joined = true;
HAL_SAFE_FREE(rtc->p_remote_user_id);
rtc->p_remote_user_id = strdup(user_name);
rtc->b_user_joined = rtc->p_remote_user_id != NULL;
msg.code = VOLC_MSG_USER_JOINED;
_send_message_2_user(rtc, &msg);
@@ -205,6 +348,7 @@ static void _on_user_joined(byte_rtc_engine_t engine, const char* channel, const
static void _on_user_offline(byte_rtc_engine_t engine, const char* channel, const char* user_name, int reason)
{
__rtc_diagnostic_priority_apply();
rtc_impl_t* rtc = (rtc_impl_t*) byte_rtc_get_user_data(engine);
volc_msg_t msg = {0};
LOGI("remote user offline %s:%s reason %d\n", channel, user_name, reason);
@@ -224,17 +368,296 @@ static void _on_user_mute_video(byte_rtc_engine_t engine, const char* channel, c
LOGD("remote user mute video %s:%s %d\n", channel, user_name, muted);
};
#ifdef ESP_PLATFORM
/* RTC_RX_HOST_BEGIN: host tests extract these production functions verbatim. */
static void __rtc_rx_max(atomic_uint *value, unsigned candidate)
{
unsigned old = atomic_load(value);
while (old < candidate && !atomic_compare_exchange_weak(value, &old, candidate)) {}
}
static bool __rtc_rx_current(rtc_impl_t *rtc, unsigned generation)
{
return atomic_load(&rtc->rx_accepting) && generation == atomic_load(&rtc->rx_generation);
}
/* No decoder, allocator, logging, or waiting on the SDK's synchronous thread. */
static void __rtc_rx_enqueue(rtc_impl_t *rtc, const char *room, const void *data, size_t size)
{
uint64_t started = esp_timer_get_time();
atomic_fetch_add(&rtc->rx_producers, 1);
unsigned generation = atomic_load(&rtc->rx_generation);
if (!atomic_load(&rtc->rx_accepting) || !room || strcmp(room, rtc->rx_room)) {
atomic_fetch_add(&rtc->rx_stale, 1);
goto done;
}
if (!data || !size || size > RTC_RX_PACKET_MAX) {
atomic_fetch_add(&rtc->rx_oversized, 1);
goto done;
}
unsigned slot;
uint64_t stage = esp_timer_get_time();
BaseType_t acquired = xQueueReceive(rtc->rx_free, &slot, 0);
__rtc_rx_max(&rtc->rx_free_max_us, (unsigned)(esp_timer_get_time() - stage));
if (acquired != pdTRUE) {
atomic_fetch_add(&rtc->rx_overflow, 1);
goto done;
}
stage = esp_timer_get_time();
rtc_rx_packet_t *packet = &rtc->rx_packets[slot];
packet->generation = generation;
packet->arrival_ms = (uint32_t)(started / 1000);
packet->size = size;
memcpy(packet->data, data, size);
__rtc_rx_max(&rtc->rx_copy_max_us, (unsigned)(esp_timer_get_time() - stage));
if (!__rtc_rx_current(rtc, generation)) {
atomic_fetch_add(&rtc->rx_stale, 1);
xQueueSend(rtc->rx_free, &slot, 0);
} else {
stage = esp_timer_get_time();
BaseType_t sent = xQueueSend(rtc->rx_pending, &slot, 0);
__rtc_rx_max(&rtc->rx_send_max_us, (unsigned)(esp_timer_get_time() - stage));
stage = esp_timer_get_time();
if (sent != pdTRUE) {
atomic_fetch_add(&rtc->rx_overflow, 1);
xQueueSend(rtc->rx_free, &slot, 0);
} else {
__rtc_rx_max(&rtc->rx_peak, (unsigned)uxQueueMessagesWaiting(rtc->rx_pending));
}
__rtc_rx_max(&rtc->rx_post_max_us, (unsigned)(esp_timer_get_time() - stage));
}
done: ;
unsigned elapsed = (unsigned)(esp_timer_get_time() - started);
atomic_fetch_add(&rtc->rx_callbacks, 1);
atomic_fetch_add(&rtc->rx_callback_us, elapsed);
__rtc_rx_max(&rtc->rx_callback_max_us, elapsed);
atomic_fetch_sub(&rtc->rx_producers, 1);
}
/* The worker serializes the service's PCM discard with its own delivery. A
* generation check alone has a check/emit race if the worker is preempted. */
static void __rtc_rx_interrupt(rtc_impl_t *rtc)
{
atomic_fetch_add(&rtc->rx_generation, 1);
}
/* Only the worker calls this while holding rx_work_lock. */
static void __rtc_rx_apply_interrupt(rtc_impl_t *rtc)
{
unsigned generation = atomic_load(&rtc->rx_generation);
if (rtc->rx_applied_generation == generation) return;
rtc->rx_applied_generation = generation;
if (atomic_load(&rtc->rx_accepting)) {
volc_msg_t msg = {.code=VOLC_MSG_CONV_STATUS};
msg.data.conv_status = VOLC_CONV_STATUS_INTERRUPTED;
_send_message_2_user(rtc, &msg);
}
}
/* A transport callback may contain more than one encoded frame. The decoder
* reports consumed input separately from decoded PCM; do not discard its tail. */
static int __rtc_decode_opus(rtc_impl_t *rtc, const void *data, size_t length, volc_data_info_t *info, unsigned generation)
{
esp_audio_dec_in_raw_t raw = {.buffer=(uint8_t *)data, .len=length};
int total=0;
while(raw.len && __rtc_rx_current(rtc, generation)) {
esp_audio_dec_out_frame_t frame={.buffer=rtc->decoded_pcm,.len=sizeof(rtc->decoded_pcm)};
esp_audio_dec_info_t decoded_info={0};
raw.consumed=0;
int ret=esp_opus_dec_decode(rtc->opus_decoder,&raw,&frame,&decoded_info);
if(ret!=ESP_AUDIO_ERR_OK || !raw.consumed || raw.consumed>raw.len || frame.decoded_size>sizeof(rtc->decoded_pcm)) {
rtc->rx_errors++;
LOGE("RTC Opus decode failed code=%d remaining=%u consumed=%u",ret,(unsigned)raw.len,(unsigned)raw.consumed);
return total;
}
if(frame.decoded_size && __rtc_rx_current(rtc, generation)) {
info->info.audio.data_type=VOLC_AUDIO_DATA_TYPE_PCM;
__send_data_2_user(rtc,frame.buffer,frame.decoded_size,info);
total+=frame.decoded_size;
}
raw.buffer+=raw.consumed;
raw.len-=raw.consumed;
}
return total;
}
static int __rtc_rx_decoder_reset(rtc_impl_t *rtc, unsigned generation)
{
if (rtc->opus_decoder) esp_opus_dec_close(rtc->opus_decoder);
rtc->opus_decoder = NULL;
esp_opus_dec_cfg_t config = ESP_OPUS_DEC_CONFIG_DEFAULT();
config.sample_rate = 16000;
config.channel = 1;
config.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_120_MS;
if (esp_opus_dec_open(&config, sizeof(config), &rtc->opus_decoder) != ESP_AUDIO_ERR_OK) {
rtc->rx_errors++;
return -1;
}
rtc->rx_decoder_generation = generation;
return 0;
}
static void __rtc_rx_report(rtc_impl_t *rtc)
{
uint64_t now = volc_osal_get_time_ms();
if (!rtc->rx_report_ms) rtc->rx_report_ms = now;
if (now - rtc->rx_report_ms < 1000) return;
LOGI("DOWNLINK span_ms=%u frames=%u pcm_ms=%u max_gap_ms=%u max_decode_ms=%u errors=%u",
(unsigned)(now-rtc->rx_report_ms), rtc->rx_frames, rtc->rx_pcm_bytes/32,
rtc->rx_gap_ms, rtc->rx_decode_ms, rtc->rx_errors);
LOGI("RTC_RX callbacks=%u callback_total_us=%u callback_max_us=%u decode_total_us=%u queued=%u peak=%u overflow=%u oversized=%u stale=%u",
atomic_exchange(&rtc->rx_callbacks,0), atomic_exchange(&rtc->rx_callback_us,0),
atomic_exchange(&rtc->rx_callback_max_us,0), rtc->rx_decode_total_us,
(unsigned)uxQueueMessagesWaiting(rtc->rx_pending), atomic_exchange(&rtc->rx_peak,0),
atomic_exchange(&rtc->rx_overflow,0), atomic_exchange(&rtc->rx_oversized,0), atomic_exchange(&rtc->rx_stale,0));
/* Wall time includes preemption after a queue wakes another task. These are
* per-stage maxima, not CPU costs or additive parts of the same callback. */
LOGI("RTC_RX_STAGE free_max_us=%u copy_max_us=%u send_max_us=%u post_max_us=%u worker_priority=%u worker_core=%d",
atomic_exchange(&rtc->rx_free_max_us,0), atomic_exchange(&rtc->rx_copy_max_us,0),
atomic_exchange(&rtc->rx_send_max_us,0), atomic_exchange(&rtc->rx_post_max_us,0),
(unsigned)uxTaskPriorityGet(NULL), (int)xTaskGetCoreID(NULL));
rtc->rx_report_ms = now;
rtc->rx_frames=rtc->rx_pcm_bytes=rtc->rx_gap_ms=rtc->rx_decode_ms=rtc->rx_errors=rtc->rx_decode_total_us=0;
}
static void __rtc_rx_process(rtc_impl_t *rtc, unsigned slot)
{
rtc_rx_packet_t *packet = &rtc->rx_packets[slot];
xSemaphoreTake(rtc->rx_work_lock, portMAX_DELAY);
__rtc_rx_apply_interrupt(rtc);
if (__rtc_rx_current(rtc, packet->generation)) {
unsigned gap = rtc->rx_previous_ms ? (uint32_t)(packet->arrival_ms - (uint32_t)rtc->rx_previous_ms) : 0;
rtc->rx_previous_ms = packet->arrival_ms;
if (gap > rtc->rx_gap_ms) rtc->rx_gap_ms = gap;
rtc->rx_frames++;
uint64_t started = esp_timer_get_time();
if ((rtc->opus_decoder && rtc->rx_decoder_generation == packet->generation) ||
__rtc_rx_decoder_reset(rtc, packet->generation) == 0) {
volc_data_info_t info = {.type=VOLC_DATA_TYPE_AUDIO};
rtc->rx_pcm_bytes += __rtc_decode_opus(rtc, packet->data, packet->size, &info, packet->generation);
}
unsigned elapsed = (unsigned)(esp_timer_get_time()-started);
rtc->rx_decode_total_us += elapsed;
if (elapsed/1000 > rtc->rx_decode_ms) rtc->rx_decode_ms = elapsed/1000;
} else {
atomic_fetch_add(&rtc->rx_stale, 1);
}
__rtc_rx_report(rtc);
xSemaphoreGive(rtc->rx_work_lock);
xQueueSend(rtc->rx_free, &slot, 0);
}
static void __rtc_rx_task(void *arg)
{
rtc_impl_t *rtc = arg;
while (!atomic_load(&rtc->rx_shutdown)) {
xSemaphoreTake(rtc->rx_work_lock, portMAX_DELAY);
__rtc_rx_apply_interrupt(rtc);
xSemaphoreGive(rtc->rx_work_lock);
unsigned slot;
if (xQueueReceive(rtc->rx_pending, &slot, pdMS_TO_TICKS(20)) == pdTRUE) {
__rtc_rx_process(rtc, slot);
}
}
xSemaphoreGive(rtc->rx_worker_stopped);
/* Control deletes this task with its original WithCaps handle. Self-delete
* would allocate an IDF cleanup task, unsafe when internal RAM is scarce. */
for (;;) vTaskSuspend(NULL);
}
static int __rtc_rx_init(rtc_impl_t *rtc)
{
rtc->rx_packets = heap_caps_calloc(RTC_RX_PACKET_COUNT, sizeof(*rtc->rx_packets), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
rtc->rx_free = xQueueCreate(RTC_RX_PACKET_COUNT, sizeof(unsigned));
rtc->rx_pending = xQueueCreate(RTC_RX_PACKET_COUNT, sizeof(unsigned));
rtc->rx_work_lock = xSemaphoreCreateMutex();
rtc->rx_worker_stopped = xSemaphoreCreateBinary();
if (!rtc->rx_packets || !rtc->rx_free || !rtc->rx_pending || !rtc->rx_work_lock || !rtc->rx_worker_stopped) return -1;
for (unsigned i=0; i<RTC_RX_PACKET_COUNT; i++) xQueueSend(rtc->rx_free, &i, 0);
if (xTaskCreatePinnedToCoreWithCaps(__rtc_rx_task, "rtc_opus_rx", 16*1024, rtc, 5, &rtc->rx_worker,
tskNO_AFFINITY, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT) != pdPASS) {
rtc->rx_worker = NULL;
return -1;
}
return 0;
}
static void __rtc_rx_stop(rtc_impl_t *rtc)
{
atomic_store(&rtc->rx_accepting, false);
__rtc_rx_interrupt(rtc);
while (atomic_load(&rtc->rx_producers)) vTaskDelay(1);
if (!rtc->rx_work_lock) return;
/* Wait for a decoder/user callback already in progress. Future work checks
* accepting/generation under this lock, so nothing is emitted after return. */
xSemaphoreTake(rtc->rx_work_lock, portMAX_DELAY);
unsigned slot;
if (rtc->rx_pending && rtc->rx_free) {
while (xQueueReceive(rtc->rx_pending, &slot, 0) == pdTRUE) xQueueSend(rtc->rx_free, &slot, 0);
}
rtc->rx_applied_generation = atomic_load(&rtc->rx_generation);
xSemaphoreGive(rtc->rx_work_lock);
}
static int __rtc_rx_start(rtc_impl_t *rtc, const char *room)
{
if (!room || strlen(room) >= sizeof(rtc->rx_room) || !rtc->rx_worker) return -1;
__rtc_rx_stop(rtc);
xSemaphoreTake(rtc->rx_work_lock, portMAX_DELAY);
memcpy(rtc->rx_room, room, strlen(room)+1);
rtc->rx_report_ms=rtc->rx_previous_ms=0;
rtc->rx_frames=rtc->rx_pcm_bytes=rtc->rx_gap_ms=rtc->rx_decode_ms=rtc->rx_errors=rtc->rx_decode_total_us=0;
rtc->rx_applied_generation = atomic_load(&rtc->rx_generation);
atomic_store(&rtc->rx_accepting, true);
xSemaphoreGive(rtc->rx_work_lock);
return 0;
}
static void __rtc_rx_destroy(rtc_impl_t *rtc)
{
__rtc_rx_stop(rtc);
atomic_store(&rtc->rx_shutdown, true);
if (rtc->rx_worker) {
xSemaphoreTake(rtc->rx_worker_stopped, portMAX_DELAY);
vTaskDeleteWithCaps(rtc->rx_worker);
rtc->rx_worker = NULL;
}
if (rtc->rx_free) vQueueDelete(rtc->rx_free);
if (rtc->rx_pending) vQueueDelete(rtc->rx_pending);
if (rtc->rx_work_lock) vSemaphoreDelete(rtc->rx_work_lock);
if (rtc->rx_worker_stopped) vSemaphoreDelete(rtc->rx_worker_stopped);
heap_caps_free(rtc->rx_packets);
rtc->rx_packets = NULL;
rtc->rx_free = rtc->rx_pending = NULL;
rtc->rx_work_lock = rtc->rx_worker_stopped = NULL;
}
/* RTC_RX_HOST_END */
#endif
static void _on_audio_data(byte_rtc_engine_t engine, const char* channel, const char* user_name, uint16_t sent_ts, audio_data_type_e data_type,
const void* data_ptr, size_t data_len, const uint8_t* extra_info, size_t extra_info_size)
{
__rtc_diagnostic_priority_apply();
volc_data_info_t info = {0};
rtc_impl_t* rtc = (rtc_impl_t*)byte_rtc_get_user_data(engine);
#ifdef ESP_PLATFORM
if (rtc && data_type == AUDIO_DATA_TYPE_OPUS) {
__rtc_rx_enqueue(rtc, channel, data_ptr, data_len);
return;
}
#endif
if (NULL == rtc || !rtc->b_pipeline_started || !rtc->b_channel_joined || !rtc->b_user_joined) {
LOGE("pipeline not started or channel not joined or user not joined");
return;
}
info.type = VOLC_DATA_TYPE_AUDIO;
uint64_t now=volc_osal_get_time_ms();
if(!rtc->rx_report_ms)rtc->rx_report_ms=now;
unsigned gap=rtc->rx_previous_ms?(unsigned)(now-rtc->rx_previous_ms):0;
if(gap>rtc->rx_gap_ms)rtc->rx_gap_ms=gap;
rtc->rx_previous_ms=now;rtc->rx_frames++;
info.info.audio.data_type = (volc_audio_data_type_e) data_type;
// info.info.audio.sent_ts = sent_ts;
__send_data_2_user(rtc, data_ptr, data_len, &info);
@@ -268,7 +691,7 @@ static void _on_channel_error(byte_rtc_engine_t engine, const char* channel, int
{
volc_msg_t msg_data = {0};
rtc_impl_t* rtc = (rtc_impl_t*) byte_rtc_get_user_data(engine);
LOGE("channel error %s:%d %s\n", channel, code, msg ? msg : "");
LOGE("RTC channel error code=%d", code);
msg_data.code = code;
msg_data.data.msg = (char *)msg;
_send_message_2_user(rtc, &msg_data);
@@ -281,7 +704,7 @@ static void _on_global_error(byte_rtc_engine_t engine, int code, const char* mes
rtc->b_channel_joined = false;
rtc->b_first_keyframe_received = false;
LOGI("global error %d %s\n", code, message ? message : "");
LOGI("RTC global error code=%d", code);
msg_data.code = VOLC_MSG_DISCONNECTED;
_send_message_2_user(rtc, &msg_data);
};
@@ -309,7 +732,8 @@ static void _on_target_bitrate_changed(byte_rtc_engine_t engine, const char* cha
static void _on_token_privilege_will_expire(byte_rtc_engine_t engine, const char* token)
{
LOGI("\ntoken privilege will expire %s", token ? token : "");
(void)token;
LOGI("RTC token privilege will expire");
rtc_impl_t* rtc = (rtc_impl_t*) byte_rtc_get_user_data(engine);
volc_msg_t msg_data = {0};
msg_data.code = VOLC_MSG_TOKEN_EXPIRED;
@@ -319,6 +743,7 @@ static void _on_token_privilege_will_expire(byte_rtc_engine_t engine, const char
static void _on_message_received(byte_rtc_engine_t engine, const char* channel_name, const char* src, const uint8_t* message, int size, bool binary)
{
__rtc_diagnostic_priority_apply();
int ret = 0;
volc_msg_t msg = { 0 };
@@ -331,22 +756,47 @@ static void _on_message_received(byte_rtc_engine_t engine, const char* channel_n
LOGE("pipeline not started or channel not joined or user not joined");
return;
}
#ifdef ESP_PLATFORM
/* rx_room is only changed after in-flight SDK readers have left. */
bool rx_guard = false;
if (rtc->opus_encoder) {
atomic_fetch_add(&rtc->rx_producers, 1);
bool current_room = atomic_load(&rtc->rx_accepting) && channel_name && !strcmp(channel_name, rtc->rx_room);
if (!current_room) {
atomic_fetch_sub(&rtc->rx_producers, 1);
return;
}
rx_guard = true;
}
#endif
if (_is_target_message(message, MAGIC_CONV)) {
ret = _on_conversion_status_message_parsed((uint8_t *)message + MAGIC_OFFSET,rtc);
if(ret != -1){
#ifdef ESP_PLATFORM
if (ret == VOLC_CONV_STATUS_INTERRUPTED && rtc->opus_encoder) {
__rtc_rx_interrupt(rtc);
goto message_done;
}
#endif
msg.code = VOLC_MSG_CONV_STATUS;
msg.data.conv_status = ret;
_send_message_2_user(rtc, &msg);
}
return;
goto message_done;
}
__send_data_2_user(rtc, message, size, &info);
message_done:
#ifdef ESP_PLATFORM
if (rx_guard) atomic_fetch_sub(&rtc->rx_producers, 1);
#endif
return;
};
static void _on_message_send_result(byte_rtc_engine_t engine, const char* channel_name, int64_t msgid, int error, const char* extencontent)
{
LOGD("----------------------->MessageSendResult msg id %" PRId64 ", error %d, extencontent %s \n", msgid, error, extencontent ? extencontent : "");
__rtc_diagnostic_priority_apply();
LOGD("MessageSendResult id=%" PRId64 " error=%d", msgid, error);
};
static void _on_license_will_expire(byte_rtc_engine_t engine, int daysleft)
@@ -361,7 +811,7 @@ static void _on_license_will_expire(byte_rtc_engine_t engine, int daysleft)
static void _on_fini_notify(byte_rtc_engine_t engine)
{
rtc_impl_t* rtc = (rtc_impl_t*) byte_rtc_get_user_data(engine);
rtc->b_fini = true;
atomic_store(&rtc->b_fini, true);
}
/*
@@ -444,6 +894,7 @@ static int __rtc_init(rtc_impl_t* engine, cJSON* p_config)
}
engine->rtc = byte_rtc_create(engine->p_appid, &rtc_event_handler);
if (!engine->rtc) return -1;
byte_rtc_set_user_data(engine->rtc, (void*) engine);
byte_rtc_set_log_level(engine->rtc, log_level);
ret = volc_json_read_object(p_config, "params", &p_params);
@@ -455,16 +906,20 @@ static int __rtc_init(rtc_impl_t* engine, cJSON* p_config)
if (p_param && cJSON_IsString(p_param)) {
const char* param_str = cJSON_GetStringValue(p_param);
if (param_str && strlen(param_str) > 0) {
byte_rtc_set_params(engine->rtc, param_str);
LOGI("volc_rtc_create: set param[%d]: %s", i, param_str);
int param_ret = byte_rtc_set_params(engine->rtc, param_str);
LOGI("volc_rtc_create: set param[%d] result=%d", i, param_ret);
}
}
}
}
cJSON_Delete(p_params);
}
byte_rtc_init(engine->rtc);
byte_rtc_set_audio_codec(engine->rtc, __volc_to_rtc_audio_codec(engine->audio_codec));
ret = byte_rtc_init(engine->rtc);
LOGI("RTC init result=%d", ret);
if (ret != 0) return ret;
ret = byte_rtc_set_audio_codec(engine->rtc, __volc_to_rtc_audio_codec(engine->audio_codec));
LOGI("RTC codec=%d result=%d", engine->audio_codec, ret);
if (ret != 0) return ret;
byte_rtc_set_video_codec(engine->rtc, video_codec - 1); // -1 for default codec
return 0;
}
@@ -490,11 +945,28 @@ volc_rtc_t volc_rtc_create(const char* appid, void* context, cJSON* p_config, vo
goto err_out_label;
}
#ifdef ESP_PLATFORM
if (rtc->audio_codec == VOLC_AUDIO_CODEC_TYPE_OPUS) {
esp_opus_enc_config_t enc = ESP_OPUS_ENC_CONFIG_DEFAULT();
enc.sample_rate=16000; enc.channel=1; enc.bitrate=24000;
enc.complexity=0; enc.enable_vbr=true;
int input_size=0;
if (esp_opus_enc_open(&enc,sizeof(enc),&rtc->opus_encoder)!=ESP_AUDIO_ERR_OK ||
esp_opus_enc_get_frame_size(rtc->opus_encoder,&input_size,&rtc->opus_packet_size)!=ESP_AUDIO_ERR_OK ||
input_size != sizeof(rtc->pcm_frame) || rtc->opus_packet_size <= 0 ||
!(rtc->opus_packet=volc_osal_malloc(rtc->opus_packet_size)) || __rtc_rx_init(rtc) != 0) {
LOGE("RTC Opus codec initialization failed");
volc_rtc_destroy(rtc);
return NULL;
}
LOGI("RTC Opus ready: 16000 Hz mono, 20 ms uplink");
}
#endif
LOGD("rtc create success");
return (volc_rtc_t)rtc;
err_out_label:
HAL_SAFE_FREE(rtc->p_appid);
HAL_SAFE_FREE(rtc);
volc_rtc_destroy(rtc);
return NULL;
}
@@ -505,13 +977,31 @@ void volc_rtc_destroy(volc_rtc_t handle)
LOGE("rtc instance is NULL");
return;
}
atomic_store(&rtc->closing, true);
__rtc_stop(rtc);
byte_rtc_fini(rtc->rtc);
while (!rtc->b_fini) {
usleep(1000 * 10);
if (rtc->rtc) {
int fini_result = byte_rtc_fini(rtc->rtc);
if (fini_result != 0) {
/* The SDK only documents destroy as safe after fini_notify. Keep
* callback context alive rather than free memory it might access. */
LOGE("RTC cleanup deferred: fini failed code=%d", fini_result);
return;
}
while (!atomic_load(&rtc->b_fini)) {
usleep(1000 * 10);
}
}
byte_rtc_destroy(rtc->rtc);
if (rtc->rtc) byte_rtc_destroy(rtc->rtc);
__clear_room_info(rtc);
HAL_SAFE_FREE(rtc->p_appid);
HAL_SAFE_FREE(rtc->p_user_id);
#ifdef ESP_PLATFORM
__rtc_rx_destroy(rtc);
if (rtc->opus_encoder) esp_opus_enc_close(rtc->opus_encoder);
if (rtc->opus_decoder) esp_opus_dec_close(rtc->opus_decoder);
HAL_SAFE_FREE(rtc->opus_packet);
#endif
HAL_SAFE_FREE(rtc->p_channel_name);
HAL_SAFE_FREE(rtc->p_remote_user_id);
HAL_SAFE_FREE(rtc->p_token);
@@ -527,12 +1017,23 @@ int volc_rtc_start(volc_rtc_t rtc, const char* bot_id, volc_iot_info_t* iot_info
LOGE("rtc instance is NULL");
return -1;
}
char* task_id = "test";
if (volc_get_rtc_config(iot_info, __volc_to_rtc_audio_codec(rtc_impl->audio_codec), bot_id, task_id, &rtc_impl->info, params)) {
cJSON *request=cJSON_Parse(params);
const char *task_id=cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(request,"PlatformTaskID"));
if(!task_id||strlen(task_id)!=36){cJSON_Delete(request);return -1;}
// Never reuse a previous room/token if the next provider request fails.
__clear_room_info(rtc_impl);
int config_result=volc_get_rtc_config(iot_info, __volc_to_rtc_audio_codec(rtc_impl->audio_codec), bot_id, task_id, &rtc_impl->info, params);
cJSON_Delete(request);
if (config_result) {
__clear_room_info(rtc_impl);
LOGE("get rtc config failed");
return -1;
}
opt = &rtc_impl->info.rtc_opt;
if (volc_platform_recording_prepare) {
int recording_result=volc_platform_recording_prepare(rtc_impl->p_appid,opt->p_channel_name,opt->p_uid);
if (recording_result) LOGW("Cloud conversation recording not ready (%d)",recording_result);
}
return __rtc_start(rtc_impl, opt);
}
@@ -542,7 +1043,38 @@ int volc_rtc_stop(volc_rtc_t handle) {
LOGE("rtc instance is NULL");
return -1;
}
__rtc_stop(rtc);
return __rtc_stop(rtc);
}
const char *volc_rtc_user_id(volc_rtc_t handle,bool remote) {
rtc_impl_t *rtc=handle;if(!rtc)return NULL;return remote?rtc->p_remote_user_id:rtc->p_user_id;
}
static int __rtc_send_pcm(rtc_impl_t *rtc, const uint8_t *data, size_t size) {
audio_frame_info_t info = {.data_type = AUDIO_DATA_TYPE_PCM};
while (size) {
size_t take = sizeof(rtc->pcm_frame) - rtc->pcm_used;
if (take > size) take = size;
memcpy(rtc->pcm_frame + rtc->pcm_used, data, take);
rtc->pcm_used += take; data += take; size -= take;
if (rtc->pcm_used == sizeof(rtc->pcm_frame)) {
int ret;
#ifdef ESP_PLATFORM
if (rtc->opus_encoder) {
esp_audio_enc_in_frame_t in = {.buffer=rtc->pcm_frame,.len=sizeof(rtc->pcm_frame)};
esp_audio_enc_out_frame_t out = {.buffer=rtc->opus_packet,.len=rtc->opus_packet_size};
ret=esp_opus_enc_process(rtc->opus_encoder,&in,&out);
if(ret==ESP_AUDIO_ERR_OK){
info.data_type=AUDIO_DATA_TYPE_OPUS;
ret=byte_rtc_send_audio_data(rtc->rtc,rtc->p_channel_name,out.buffer,out.encoded_bytes,&info);
}
} else
#endif
ret = byte_rtc_send_audio_data(rtc->rtc,rtc->p_channel_name,rtc->pcm_frame,sizeof(rtc->pcm_frame),&info);
rtc->pcm_used = 0;
if (ret != 0) return ret;
}
}
return 0;
}
@@ -550,14 +1082,19 @@ int volc_rtc_send(volc_rtc_t handle, const void* data, int size, volc_data_info_
rtc_impl_t* rtc = (rtc_impl_t *)handle;
audio_frame_info_t audio_info = {0};
video_frame_info_t video_info = {0};
if (NULL == rtc || NULL == data || NULL == data_info) {
if (NULL == rtc || NULL == data || NULL == data_info || size <= 0) {
LOGE("input args is invalid, rtc(%p), data(%p), data_info(%p)", rtc, data, data_info);
return -1;
}
switch (data_info->type) {
case VOLC_DATA_TYPE_AUDIO: {
// Join is asynchronous. Do not enqueue media into the vendor
// transport until both room and agent are actually present.
if (!rtc->b_pipeline_started || !rtc->b_channel_joined || !rtc->b_user_joined) return 0;
if (data_info->info.audio.data_type == VOLC_AUDIO_DATA_TYPE_PCM && size > 0)
return __rtc_send_pcm(rtc,data,(size_t)size);
audio_info.data_type = (audio_data_type_e) data_info->info.audio.data_type;
byte_rtc_send_audio_data(rtc->rtc, rtc->p_channel_name, data, size, &audio_info);
return byte_rtc_send_audio_data(rtc->rtc, rtc->p_channel_name, data, size, &audio_info);
break;
}
case VOLC_DATA_TYPE_VIDEO: {
@@ -125,16 +125,16 @@ static void __ws_recv_data(ws_impl_t* ws, const char* data, int data_len)
p_json = cJSON_ParseWithLength(data, data_len);
if (!p_json) {
LOGE("Failed to parse json, data_len: %d, data: %s", data_len, data);
LOGE("Failed to parse json, data_len: %d", data_len);
return;
}
// LOGI("json: %s", data);
// Protocol payloads contain private text and must not be logged.
volc_json_read_string(p_json, "type", &p_type);
volc_json_read_string(p_json, "delta", &p_delta);
volc_json_read_string(p_json, "response.status", &p_status);
if (strcmp(p_type, "response.audio.delta") == 0 && p_delta) {
if (strlen(p_delta) == 0) {
LOGE("delta is empty, data: %s", data);
LOGE("delta is empty");
goto err_out_label;
}
if (NULL == ws || !ws->b_pipeline_started) {
@@ -182,11 +182,11 @@ static void __ws_recv_data(ws_impl_t* ws, const char* data, int data_len)
ws->conv_status = VOLC_CONV_STATUS_INTERRUPTED;
msg.data.conv_status = VOLC_CONV_STATUS_INTERRUPTED;
}
LOGI("data: %s", data);
LOGI("conversation status updated");
__send_message_2_user(ws, &msg);
} else {
if (strcmp(p_type, "session.created") == 0) {
LOGI("%s", data);
LOGI("provider session created");
} else if (strcmp(p_type, "response.audio_transcript.delta") == 0 || strcmp(p_type, "response.audio_transcript.done") == 0 || strcmp(p_type, "response.audio.done") == 0) {
volc_json_read_string(p_json, "response_id", &p_response_id);
if (__ws_drop_for_interrupted(ws, p_response_id)) {
@@ -471,7 +471,7 @@ static int __ws_input_audio_buffer_commit(ws_impl_t* ws) {
root = cJSON_CreateObject();
cJSON_AddStringToObject(root, "type", "input_audio_buffer.commit");
char* json = cJSON_PrintUnformatted(root);
LOGD("json: %s", json);
LOGD("WebSocket message prepared");
cJSON_Delete(root);
ret = volc_ws_client_send_text(ws->client, json, strlen(json), 1000);
if (ret >= 0) {
@@ -500,7 +500,7 @@ static int __ws_response_create(ws_impl_t* ws) {
cJSON_AddItemToObject(response_obj, "modalities", modalities_array);
// 生成 JSON 字符串
const char* json = cJSON_PrintUnformatted(root);
LOGD("json: %s", json);
LOGD("WebSocket message prepared");
// 释放内存
cJSON_Delete(root);
ret = volc_ws_client_send_text(ws->client, json, strlen(json), 1000);
@@ -555,7 +555,7 @@ static void __ws_parse_params(const char* params, ws_params_t* out_params) {
}
p_json = cJSON_Parse(params);
if (!p_json) {
LOGW("Failed to parse json params: %s", params);
LOGW("Failed to parse json params");
return;
}
volc_json_read_int(p_json, "audio.codec", (int*)&out_params->audio_codec_type);
@@ -416,7 +416,7 @@ static int ws_read(volc_ws_client_t* client, char* buffer, int len, int timeout_
transport_ws_t* ws = client->ws_transport;
if (ws->frame_state.bytes_remaining <= 0) {
if ((rlen = ws_read_header(client, buffer, len, timeout_ms)) < 0) {
LOGE("Error reading header %d, %s\r\n", rlen, buffer);
LOGE("Error reading header, length=%d", rlen);
ws->frame_state.bytes_remaining = 0;
return rlen;
}
@@ -835,7 +835,7 @@ static int ws_connect(volc_ws_client_t* client, const char* host, int port, int
"Sec-WebSocket-Key: %s\r\n"
"Sec-WebSocket-Version: 13\r\n",
ws->path, host, host, user_agent_ptr, client_key);
LOGD("http request: %s", ws->buffer);
LOGD("WebSocket handshake prepared");
if (len <= 0 || len >= WS_BUFFER_SIZE) {
LOGE("Error in request generation, desired request len: %d, buffer size: %d\r\n", len, WS_BUFFER_SIZE);
@@ -871,7 +871,7 @@ static int ws_connect(volc_ws_client_t* client, const char* host, int port, int
r, len, WS_BUFFER_SIZE);
return -1;
}
LOGD("ws->buffer:\r\n%s", ws->buffer);
LOGD("WebSocket handshake sent");
if (ws_tcp_write(client, ws->buffer, len, timeout_ms) < 0) {
LOGE("Write FAIL");
}
@@ -879,12 +879,12 @@ static int ws_connect(volc_ws_client_t* client, const char* host, int port, int
int header_len = 0;
do {
if ((len = ws_tcp_read(client, ws->buffer + header_len, WS_BUFFER_SIZE - header_len, timeout_ms)) <= 0) {
LOGE("read FAIL for upgrade header:%s\r\n", ws->buffer);
LOGE("Failed to read upgrade header");
return -1;
}
header_len += len;
ws->buffer[header_len] = '\0';
LOGD("Read header chunk %d, current header size: %d, header: %s\r\n", len, header_len, ws->buffer);
LOGD("Read header chunk %d, current header size: %d", len, header_len);
} while (NULL == strstr(ws->buffer, "\r\n\r\n") && header_len < WS_BUFFER_SIZE);
char* server_key = get_http_header(ws->buffer, "Sec-WebSocket-Accept:");
@@ -1117,7 +1117,7 @@ static int __parse_url(volc_ws_client_t* client, char* url) {
return -1;
}
url_end = url + strlen(url);
LOGD("url: %s", url);
LOGD("WebSocket URL configured");
// 1. parse scheme
filed_end = strstr(pos, "://");
@@ -1209,7 +1209,7 @@ static int __parse_url(volc_ws_client_t* client, char* url) {
}
}
LOGD("scheme: %s, host: %s, port: %d, path: %s", client->scheme, client->host, (int)client->port, client->path);
LOGD("WebSocket endpoint: %s://%s:%d", client->scheme, client->host, (int)client->port);
return 0;
}
@@ -13,6 +13,7 @@
#include <mbedtls/sha256.h>
#include <mbedtls/md5.h>
#include <mbedtls/base64.h>
#include <mbedtls/platform_util.h>
#include "volc_osal.h"
#include "util/volc_base64.h"
@@ -53,63 +54,32 @@ static int __aes_cbc(bool encrypt, const unsigned char* key, unsigned char* iv,
int mode = encrypt ? MBEDTLS_AES_ENCRYPT : MBEDTLS_AES_DECRYPT;
mbedtls_aes_context aes_ctx;
mbedtls_aes_init(&aes_ctx);
mbedtls_aes_setkey_dec(&aes_ctx, key, key_bits);
mbedtls_aes_crypt_cbc(&aes_ctx, mode, ilen, iv, (const unsigned char *)input, output);
int result=encrypt?mbedtls_aes_setkey_enc(&aes_ctx,key,key_bits):mbedtls_aes_setkey_dec(&aes_ctx, key, key_bits);
if(!result)result=mbedtls_aes_crypt_cbc(&aes_ctx, mode, ilen, iv, (const unsigned char *)input, output);
mbedtls_aes_free(&aes_ctx);
return 0;
return result;
}
#define AES_KEY_SIZE 16 // AES key size in bytes
#define AES_KEY_SIZE_24 24
char* volc_aes_decode(const char* secret, const char* input, const bool partial_secret) {
size_t decoded_len = 0;
size_t secret_len = 0;
int key_bits = 0;
char* key = NULL;
unsigned char iv[16] = {0};
unsigned char* decoded_payload = NULL;
if (input == NULL || strlen(input) == 0) {
LOGE("Input is NULL or empty");
return NULL;
size_t key_len=partial_secret?16:24,decoded_len=0;
if(!secret||strlen(secret)<key_len||!input||!input[0]||strlen(input)>8192)return NULL;
size_t capacity=strlen(input)+1;
uint8_t *decoded=volc_osal_calloc(capacity,1),*output=volc_osal_calloc(capacity,1);
if(!decoded||!output){HAL_SAFE_FREE(decoded);HAL_SAFE_FREE(output);return NULL;}
unsigned char key[24]={0},iv[16];memcpy(key,secret,key_len);memcpy(iv,secret,16);
bool valid=mbedtls_base64_decode(decoded,capacity,&decoded_len,(const unsigned char*)input,strlen(input))==0&&decoded_len>0&&decoded_len%16==0;
if(valid)valid=__aes_cbc(false,key,iv,decoded,decoded_len,key_len*8,output)==0;
if(valid){size_t padding=output[decoded_len-1];valid=padding>=1&&padding<=16&&padding<=decoded_len;
if(valid){for(size_t i=decoded_len-padding;i<decoded_len;i++)if(output[i]!=padding)valid=false;
if(valid){size_t length=decoded_len-padding;valid=length>=16&&memchr(output,0,length)==NULL;output[length]=0;}}
}
if (partial_secret) {
// If partial secret is used, we only use the first 8 bytes of the key and iv
key = (char *)volc_osal_calloc(AES_KEY_SIZE, 1);
memcpy(key, secret, AES_KEY_SIZE);
key_bits = 128; // AES-128
} else {
secret_len = strlen(secret);
if (secret_len < AES_KEY_SIZE_24) {
LOGE("Secret length is less than 24, secret: %s", secret);
return NULL;
}
key = (char *)volc_osal_calloc(AES_KEY_SIZE_24, 1); // +1 for null terminator
memcpy(key, secret, AES_KEY_SIZE_24);
key_bits = 192; // AES-192
}
decoded_len = volc_base64_decoded_length((const uint8_t*)input, strlen(input));
decoded_payload = (unsigned char *)volc_osal_calloc(decoded_len + 1, 1);
if (!decoded_payload) {
LOGE("Failed to allocate memory for decoded payload");
return NULL;
}
volc_base64_decode((unsigned char *)decoded_payload, decoded_len, &decoded_len, (const unsigned char *)input, strlen(input));
LOGV("decoded payload: %s, len: %zu", decoded_payload, decoded_len);
uint8_t *output = volc_osal_calloc(decoded_len + 16, 1); // +1 for null terminator
memcpy(iv, secret, AES_KEY_SIZE); // Use the same secret for IV
__aes_cbc(false, (const unsigned char *)key, (unsigned char *)iv, (const unsigned char *)decoded_payload, decoded_len, key_bits, output);
_right_trim_pred(output, _url_trim_pred);
LOGV("trimmed decrypted payload: %s, len: %d", output, (int)strlen((const char*)output));
HAL_SAFE_FREE(decoded_payload);
HAL_SAFE_FREE(key);
return (char *)output;
mbedtls_platform_zeroize(key,sizeof(key));mbedtls_platform_zeroize(iv,sizeof(iv));
mbedtls_platform_zeroize(decoded,capacity);HAL_SAFE_FREE(decoded);
if(!valid){mbedtls_platform_zeroize(output,capacity);HAL_SAFE_FREE(output);return NULL;}
return (char*)output;
}
void volc_sha256_hmac(const unsigned char* key, int keylen, const unsigned char* input, int ilen, unsigned char* output, int* plen) {
@@ -30,7 +30,7 @@ char* volc_http_post(const char* uri, const char* post_data, int data_len)
}
webclient_response(session, (void**) &buffer, &res_len);
LOGD("url: %s, request: %s, response: %s", uri, post_data, buffer);
LOGD("HTTP exchange completed, status: %d", resp_status);
err_out_label:
if (session) {
webclient_close(session);
@@ -14,7 +14,8 @@
#if defined(ENABLE_RTC_MODE)
#include "VolcEngineRTCLite.h"
#include "volc_rtc.h"
#elif defined(ENABLE_WS_MODE)
#endif
#if defined(ENABLE_WS_MODE)
#include "volc_ws.h"
#endif
@@ -195,6 +196,7 @@ static int __config_iot_parse(cJSON* iot_config, volc_engine_impl_t* engine) {
* Product Secret is deliberately not accepted by the firmware path.
*/
ret |= volc_json_read_string(iot_config, "device_secret", &engine->info.device_secret);
(void)volc_json_read_string(iot_config,"rtc_app_id",&engine->info.rtc_app_id);
return ret == 0 ? 0 : -1;
}
@@ -263,7 +265,9 @@ int volc_create(volc_engine_t* handle, const char* config_json, volc_event_handl
cJSON* rtc_cfg = cJSON_GetObjectItem(config, "rtc");
if (rtc_cfg) {
#if defined(ENABLE_RTC_MODE)
if(!engine->info.rtc_app_id||!engine->info.rtc_app_id[0]){ret=VOLC_ERR_FAILED;goto err_out_label;}
engine->rtc = volc_rtc_create(engine->info.rtc_app_id, engine, rtc_cfg, __realtime_user_event_router, __realtime_data_router);
if(!engine->rtc){ret=VOLC_ERR_FAILED;goto err_out_label;}
#else
LOGW("RTC mode is not enabled");
#endif
@@ -365,6 +369,14 @@ int volc_start(volc_engine_t handle, volc_opt_t* opt) {
return ret;
}
const char *volc_current_user_id(volc_engine_t handle,bool remote) {
#if defined(ENABLE_RTC_MODE)
volc_engine_impl_t *engine=handle;return engine?volc_rtc_user_id(engine->rtc,remote):NULL;
#else
return NULL;
#endif
}
int volc_stop(volc_engine_t handle) {
int ret = 0;
volc_engine_impl_t* engine = (volc_engine_impl_t*)handle;
@@ -19,6 +19,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#if defined(MBEDTLS_USER_CONFIG_FILE)
#include MBEDTLS_USER_CONFIG_FILE
@@ -162,7 +163,7 @@ int mbedtls_client_connect(MbedTLSSession *session)
ret = mbedtls_net_connect(&session->server_fd, session->host, session->port,
MBEDTLS_NET_PROTO_TCP);
if (ret != 0) {
LOGE("mbedtls_net_connect error, return -0x%x", -ret);
LOGE("mbedtls_net_connect error, return -0x%x errno=%d", -ret, errno);
return ret;
}