feat: expand companion platform, device controls, and memoir workflows
ci / verify (push) Canceled after 0s
ci / verify (push) Canceled after 0s
Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
This commit is contained in:
@@ -16,6 +16,8 @@ set(VOLC_SERVICE_INCS "${CMAKE_CURRENT_LIST_DIR}/../service/inc"
|
||||
set(VOLC_SERVICE_SRCS "${CMAKE_CURRENT_LIST_DIR}/../service/src/volc_manager_service.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/volc_conv_service.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/volc_conv.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/call_display_state.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/conv_service/src/call_voice_end.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/src/volc_function_call_service.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/function_call_service/src/volc_local_function_list.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../service/local_logic_service/src/local_logic_service.c"
|
||||
|
||||
@@ -15,6 +15,7 @@ set(VOLC_ESP_SRCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/volc_hal.
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/basic_board.c"
|
||||
)
|
||||
set(VOLC_ESP_DISPLAY_SRCS "${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/echoear_font.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/echoear_call_status_font.c"
|
||||
"${CMAKE_CURRENT_LIST_DIR}/../platform/src/espressif/lvgl_source/img_app_aiPhone.c"
|
||||
)
|
||||
set(VOLC_APP_SRCS ${VOLC_FRAMEWORK_SRCS}
|
||||
@@ -39,7 +40,7 @@ idf_component_register(
|
||||
SRCS ${VOLC_APP_SRCS}
|
||||
INCLUDE_DIRS ${VOLC_APP_INCS}
|
||||
REQUIRES volc_conv_ai
|
||||
PRIV_REQUIRES lvgl button av_processor esp_board_manager hw_init esp_mmap_assets esp_emote_gfx esp_vocat_base_control
|
||||
PRIV_REQUIRES nvs_flash lvgl button av_processor esp_board_manager hw_init esp_mmap_assets esp_emote_gfx esp_vocat_base_control
|
||||
)
|
||||
|
||||
# Compiler flags
|
||||
|
||||
+2
-1
@@ -60,6 +60,7 @@ volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config);
|
||||
* @return volc_hal_display_t Display instance pointer
|
||||
*/
|
||||
int volc_hal_display_set_brightness(volc_hal_display_t display, int brightness);
|
||||
int volc_hal_display_get_brightness(volc_hal_display_t display);
|
||||
|
||||
/**
|
||||
* @brief Destroy a Display instance
|
||||
@@ -82,4 +83,4 @@ int volc_hal_display_set_content(volc_hal_display_t display, volc_hal_display_ob
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@@ -48,6 +48,10 @@ int volc_hal_set_audio_player_volume(volc_hal_player_t player, int volume);
|
||||
* @return the volume
|
||||
*/
|
||||
int volc_hal_get_audio_player_volume(volc_hal_player_t player);
|
||||
// Board codec exists while idle; cloud volume must not depend on a temporary
|
||||
// conversation player, which is created and destroyed for each call.
|
||||
int volc_hal_set_device_volume(int volume);
|
||||
int volc_hal_get_device_volume(void);
|
||||
|
||||
/**
|
||||
* @brief Destroy a player instance
|
||||
@@ -90,6 +94,10 @@ int volc_hal_player_play_data(volc_hal_player_t player, const void* data, int le
|
||||
void volc_hal_player_discard_pending(volc_hal_player_t player);
|
||||
/* Allow a fully received short response to start without filling the prebuffer. */
|
||||
void volc_hal_player_finish_segment(volc_hal_player_t player);
|
||||
/* Conversation task only, while it owns a live player. Includes feeder tail. */
|
||||
bool volc_hal_player_voice_quiet(volc_hal_player_t player, unsigned quiet_ms);
|
||||
/* Set before start. NULL leaves non-call audio unrestricted. */
|
||||
void volc_hal_player_set_authorization_gate(volc_hal_player_t player, bool (*allowed)(void));
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
/* Generated by tools/generate_call_status_font.py; do not hand edit.
|
||||
* Source SHA256: abfffdf0cd809cf2e3cd9425988b8945786fb5166bf4c5e91d2b6e1fb3839f74
|
||||
* 2x nearest-neighbor glyphs; packed 1bpp, GFX 1/256-pixel advance.
|
||||
* The original source font's leading notice is preserved below.
|
||||
*/
|
||||
|
||||
/*******************************************************************************
|
||||
* Size: 16 px
|
||||
* Bpp: 1
|
||||
* Opts: --bpp 1 --size 16 --no-compress --stride 1 --align 1 --font 方正兰亭纤黑.TTF --range 32-126,19968-40869,12288-12544,65280-65312 --format lvgl -o zh_font.c
|
||||
******************************************************************************/
|
||||
|
||||
#ifdef LVGL_VERSION_MAJOR
|
||||
#error "Build this font in its own GFX-only translation unit"
|
||||
#endif
|
||||
#include "widget/gfx_font_lvgl.h"
|
||||
|
||||
#if LVGL_VERSION_MAJOR != 8 || LV_FONT_FMT_TXT_LARGE != 1
|
||||
#error "Unexpected GFX font compatibility ABI"
|
||||
#endif
|
||||
_Static_assert(sizeof(lv_font_fmt_txt_glyph_dsc_t) == 16, "GFX LARGE glyph ABI required");
|
||||
|
||||
static const uint8_t status_bitmap[] = {
|
||||
/* U+003F ? */
|
||||
15,240,15,240,48,12,48,12,192,15,192,15,192,3,192,3,0,15,0,15,0,12,0,12,
|
||||
0,240,0,240,3,192,3,192,3,0,3,0,3,0,3,0,0,0,0,0,0,0,0,0,
|
||||
3,0,3,0,
|
||||
/* U+4E2D 中 */
|
||||
0,12,0,0,3,0,0,0,192,0,0,48,0,0,12,0,0,3,0,15,255,255,255,255,
|
||||
255,255,192,12,0,240,3,0,60,0,192,15,0,48,3,192,12,0,240,3,0,60,0,192,
|
||||
15,0,48,3,255,255,255,255,255,255,252,0,192,15,0,48,3,0,12,0,0,3,0,0,
|
||||
0,192,0,0,48,0,0,12,0,0,3,0,0,0,192,0,0,48,0,0,12,0,0,3,
|
||||
0,0,
|
||||
/* U+5728 在 */
|
||||
0,12,0,0,0,48,0,0,3,0,0,0,12,0,0,63,255,255,252,255,255,255,240,3,
|
||||
0,0,0,12,0,0,0,192,48,0,3,0,192,0,48,3,0,0,192,12,0,3,0,48,
|
||||
0,12,0,192,0,243,255,255,3,207,255,252,51,0,48,0,204,0,192,0,48,3,0,0,
|
||||
192,12,0,3,0,48,0,12,0,192,0,48,3,0,0,192,12,0,3,0,48,0,12,0,
|
||||
192,0,63,255,255,192,255,255,255,3,0,0,0,12,0,0,0,
|
||||
/* U+5931 失 */
|
||||
0,3,0,0,0,12,0,0,48,48,0,0,192,192,0,3,3,0,0,12,12,0,0,255,
|
||||
255,255,3,255,255,252,60,3,0,0,240,12,0,3,0,48,0,12,0,192,0,0,3,0,
|
||||
0,0,12,0,15,255,255,255,255,255,255,255,0,3,0,0,0,12,0,0,0,204,0,0,
|
||||
3,48,0,0,12,192,0,0,51,0,0,3,3,0,0,12,12,0,3,192,15,0,15,0,
|
||||
60,3,192,0,15,207,0,0,63,192,0,0,3,0,0,0,0,
|
||||
/* U+5F85 待 */
|
||||
0,0,48,0,0,0,192,0,48,3,0,0,192,12,0,15,0,48,0,60,0,192,0,192,
|
||||
255,255,3,3,255,252,48,0,48,0,192,0,192,12,60,3,0,48,240,12,0,3,63,255,
|
||||
252,12,255,255,240,192,0,0,3,0,0,0,60,0,3,0,240,0,12,12,195,255,255,243,
|
||||
15,255,255,12,0,3,0,48,0,12,0,192,192,48,3,3,0,192,12,3,3,0,48,12,
|
||||
12,0,192,48,48,3,0,192,192,12,0,3,0,48,0,12,0,192,15,240,3,0,63,192,
|
||||
/* U+63A5 接 */
|
||||
12,0,48,0,192,3,0,12,0,48,0,192,3,0,12,63,255,240,195,255,255,255,12,3,
|
||||
207,240,192,60,12,3,3,0,192,48,48,12,255,255,240,207,255,255,12,0,0,0,192,0,
|
||||
0,63,3,0,3,240,48,0,204,255,255,252,207,255,255,12,12,3,0,192,192,48,12,12,
|
||||
3,0,192,192,48,12,15,12,0,192,240,192,12,0,252,0,192,15,192,12,15,195,192,192,
|
||||
252,60,240,240,0,63,15,0,3,
|
||||
/* U+65AD 断 */
|
||||
192,192,0,12,12,0,0,192,192,255,252,12,15,255,204,204,192,12,204,204,0,204,204,192,
|
||||
12,204,204,0,192,192,192,12,12,12,0,192,192,192,12,12,12,0,207,252,255,252,255,207,
|
||||
255,195,192,192,204,60,12,12,195,240,192,204,63,12,12,204,204,192,204,204,204,12,240,192,
|
||||
192,207,12,12,12,192,192,192,204,12,12,12,192,3,0,204,0,48,12,255,255,0,207,255,
|
||||
240,12,0,15,0,192,0,240,12,
|
||||
/* U+673A 机 */
|
||||
12,0,0,0,48,0,0,0,192,63,240,3,0,255,192,12,3,3,0,48,12,12,15,252,
|
||||
48,48,63,240,192,192,12,3,3,0,48,12,12,0,192,48,48,3,0,192,192,63,3,3,
|
||||
0,252,12,12,3,204,48,48,15,48,192,192,204,3,3,3,48,12,12,12,192,48,48,51,
|
||||
0,192,192,12,3,3,0,48,12,12,0,192,192,48,195,3,0,195,12,12,3,12,48,48,
|
||||
12,48,195,0,48,195,12,0,195,12,192,3,240,51,0,15,192,
|
||||
/* U+675F 束 */
|
||||
0,3,0,0,0,12,0,15,255,255,255,255,255,255,255,0,3,0,0,0,12,0,0,0,
|
||||
48,0,0,0,192,0,15,255,255,192,63,255,255,3,0,48,12,12,0,192,48,48,3,0,
|
||||
192,192,12,3,3,0,48,12,12,0,192,48,15,255,255,192,63,255,255,0,0,252,0,0,
|
||||
3,240,0,0,51,48,0,0,204,192,0,12,48,192,0,48,195,0,3,3,3,192,12,12,
|
||||
15,15,192,48,3,255,0,192,15,0,3,0,0,0,12,0,0,
|
||||
/* U+6B63 正 */
|
||||
63,255,255,252,255,255,255,240,0,12,0,0,0,48,0,0,0,192,0,0,3,0,0,0,
|
||||
12,0,0,0,48,0,3,0,192,0,12,3,0,0,48,12,0,0,192,48,0,3,0,255,
|
||||
240,12,3,255,192,48,12,0,0,192,48,0,3,0,192,0,12,3,0,0,48,12,0,0,
|
||||
192,48,0,3,0,192,0,12,3,0,0,48,12,0,0,192,48,0,255,255,255,255,255,255,
|
||||
255,240,
|
||||
/* U+7ED3 结 */
|
||||
3,0,12,0,12,0,48,0,48,0,192,0,192,3,0,12,15,255,252,48,63,255,240,192,
|
||||
0,192,3,0,3,0,48,192,12,0,195,0,48,15,252,0,192,63,240,3,0,3,15,255,
|
||||
240,12,63,255,192,192,0,0,3,0,0,0,12,0,0,0,48,0,0,3,60,63,255,12,
|
||||
240,255,252,60,3,0,48,240,12,0,192,0,48,3,0,0,192,12,3,243,0,48,15,204,
|
||||
0,195,192,63,255,15,0,255,252,0,3,0,48,0,12,0,192,
|
||||
/* U+8BDD 话 */
|
||||
48,0,0,0,192,0,0,3,3,255,255,12,15,255,252,12,0,48,0,48,0,192,0,0,
|
||||
3,0,0,0,12,0,0,255,255,252,3,255,255,255,192,3,0,63,0,12,0,12,0,48,
|
||||
0,48,0,192,0,192,3,0,3,0,12,0,12,15,255,240,48,63,255,192,192,192,3,3,
|
||||
3,0,12,12,12,0,48,48,48,0,192,192,192,3,3,3,0,12,12,204,0,48,51,48,
|
||||
0,192,240,255,255,3,195,255,252,0,12,0,48,0,48,0,192,
|
||||
/* U+8D25 败 */
|
||||
0,0,12,0,0,0,12,0,63,252,12,0,63,252,12,0,48,12,48,0,48,12,48,0,
|
||||
51,12,63,255,51,12,63,255,51,12,48,48,51,12,48,48,51,12,240,48,51,12,240,48,
|
||||
51,15,48,48,51,15,48,48,51,12,48,48,51,12,48,48,51,12,48,192,51,12,48,192,
|
||||
51,12,12,192,51,12,12,192,51,12,12,192,51,12,12,192,12,192,3,0,12,192,3,0,
|
||||
12,60,12,192,12,60,12,192,60,0,48,48,60,0,48,48,240,3,192,15,240,3,192,15,
|
||||
0,3,0,0,0,3,0,0,
|
||||
/* U+8FDE 连 */
|
||||
48,0,192,0,192,3,0,0,192,12,0,3,0,48,0,12,63,255,252,48,255,255,240,0,
|
||||
48,0,0,0,192,0,0,3,12,0,0,12,48,0,0,192,192,0,3,3,0,252,12,12,
|
||||
3,240,48,48,0,195,255,255,195,15,255,255,12,0,12,0,48,0,48,0,192,0,192,3,
|
||||
0,3,0,12,255,255,252,51,255,255,240,192,0,192,3,0,3,0,12,0,12,0,48,0,
|
||||
48,3,60,0,0,12,240,0,0,192,255,255,255,3,255,255,240,
|
||||
/* U+901A 通 */
|
||||
60,63,255,240,240,255,255,192,240,48,12,3,192,192,48,3,0,255,0,12,3,252,0,3,
|
||||
255,255,0,15,255,252,252,48,48,51,240,192,192,192,195,255,255,3,15,255,252,12,48,48,
|
||||
48,48,192,192,192,195,3,3,3,12,12,12,12,63,255,240,48,255,255,192,195,3,3,3,
|
||||
12,12,12,12,48,48,48,48,192,192,195,243,0,63,15,204,0,252,192,255,255,255,3,255,
|
||||
255,240,
|
||||
};
|
||||
|
||||
static const lv_font_fmt_txt_glyph_dsc_t status_glyphs[] = {
|
||||
{0}, /* glyph zero is reserved */
|
||||
{0,4960,16,26,2,-2}, /* ? */
|
||||
{52,8192,26,30,2,-4}, /* 中 */
|
||||
{150,8192,30,30,0,-4}, /* 在 */
|
||||
{263,8192,30,30,2,-4}, /* 失 */
|
||||
{376,8192,30,32,2,-4}, /* 待 */
|
||||
{496,8192,28,30,2,-4}, /* 接 */
|
||||
{601,8192,28,30,2,-4}, /* 断 */
|
||||
{706,8192,30,30,2,-4}, /* 机 */
|
||||
{819,8192,30,30,0,-4}, /* 束 */
|
||||
{932,8192,30,26,0,-2}, /* 正 */
|
||||
{1030,8192,30,30,0,-4}, /* 结 */
|
||||
{1143,8192,30,30,0,-4}, /* 话 */
|
||||
{1256,8192,32,32,0,-6}, /* 败 */
|
||||
{1384,8192,30,30,0,-4}, /* 连 */
|
||||
{1497,8192,30,26,0,-2}, /* 通 */
|
||||
};
|
||||
|
||||
#define STATUS_CMAP(cp, id) { .range_start=(cp), .range_length=1, .glyph_id_start=(id), .unicode_list=NULL, .glyph_id_ofs_list=NULL, .list_length=0, .type=LV_FONT_FMT_TXT_CMAP_FORMAT0_TINY }
|
||||
static const lv_font_fmt_txt_cmap_t status_cmaps[] = {
|
||||
STATUS_CMAP(0x003F, 1),
|
||||
STATUS_CMAP(0x4E2D, 2),
|
||||
STATUS_CMAP(0x5728, 3),
|
||||
STATUS_CMAP(0x5931, 4),
|
||||
STATUS_CMAP(0x5F85, 5),
|
||||
STATUS_CMAP(0x63A5, 6),
|
||||
STATUS_CMAP(0x65AD, 7),
|
||||
STATUS_CMAP(0x673A, 8),
|
||||
STATUS_CMAP(0x675F, 9),
|
||||
STATUS_CMAP(0x6B63, 10),
|
||||
STATUS_CMAP(0x7ED3, 11),
|
||||
STATUS_CMAP(0x8BDD, 12),
|
||||
STATUS_CMAP(0x8D25, 13),
|
||||
STATUS_CMAP(0x8FDE, 14),
|
||||
STATUS_CMAP(0x901A, 15),
|
||||
};
|
||||
#undef STATUS_CMAP
|
||||
|
||||
static const lv_font_fmt_txt_dsc_t status_font_dsc = {
|
||||
.glyph_bitmap=status_bitmap, .glyph_dsc=status_glyphs, .cmaps=status_cmaps,
|
||||
.kern_dsc=NULL, .kern_scale=0, .cmap_num=15, .bpp=1,
|
||||
.kern_classes=0, .bitmap_format=0, .cache=NULL
|
||||
};
|
||||
|
||||
static const lv_font_t status_font = {
|
||||
.get_glyph_dsc=lv_font_get_glyph_dsc_fmt_txt, .get_glyph_bitmap=lv_font_get_bitmap_fmt_txt,
|
||||
.line_height=38, .base_line=10, .subpx=LV_FONT_SUBPX_NONE, .static_bitmap=1,
|
||||
.underline_position=-6, .underline_thickness=2, .dsc=&status_font_dsc, .fallback=NULL, .user_data=NULL
|
||||
};
|
||||
|
||||
const void *echoear_call_status_font(void) { return &status_font; }
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
/* Generated by tools/generate_call_status_font.py; do not hand edit.
|
||||
* Source SHA256: abfffdf0cd809cf2e3cd9425988b8945786fb5166bf4c5e91d2b6e1fb3839f74
|
||||
* 2x nearest-neighbor glyphs; packed 1bpp, GFX 1/256-pixel advance.
|
||||
* The original source font's leading notice is preserved below.
|
||||
*/
|
||||
#pragma once
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
/* GFX compatibility font; deliberately exposes no LVGL types. */
|
||||
const void *echoear_call_status_font(void);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
+49
-55
@@ -2,6 +2,10 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdatomic.h>
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "freertos/semphr.h"
|
||||
|
||||
#include "volc_hal.h"
|
||||
#include "volc_hal_capture.h"
|
||||
@@ -39,8 +43,10 @@ typedef struct {
|
||||
volc_hal_capture_data_cb_t data_cb; // Data callback function
|
||||
volc_hal_audio_wakeup_cb_t audio_wakeup_cb; // audio wakeup cb
|
||||
void* user_data; // User data pointer
|
||||
volc_osal_tid_t capture_thread;
|
||||
volatile bool is_started;
|
||||
TaskHandle_t capture_thread;
|
||||
SemaphoreHandle_t capture_stopped;
|
||||
uint8_t *read_buf;
|
||||
atomic_bool is_started;
|
||||
} volc_hal_capture_impl_t;
|
||||
|
||||
extern basic_board_periph_t global_periph;
|
||||
@@ -61,13 +67,12 @@ static void __volc_audio_capture_task(void *arg)
|
||||
volc_hal_capture_impl_t *impl = (volc_hal_capture_impl_t *)arg;
|
||||
LOGI("capture task started...");
|
||||
int read_size = __volc_capture_get_default_read_size(impl);
|
||||
uint8_t *read_buf = volc_osal_malloc(read_size);
|
||||
// mem_assert(read_buf);
|
||||
uint8_t *read_buf = impl->read_buf;
|
||||
|
||||
while(impl->is_started) {
|
||||
int ret = audio_recorder_read_data(read_buf, read_size);
|
||||
|
||||
if (ret > 0 && impl->data_cb && impl->audio_capture_config.state == CAPTURE) {
|
||||
if (impl->is_started && ret > 0 && impl->data_cb && impl->audio_capture_config.state == CAPTURE) {
|
||||
#ifdef CONFIG_VOLC_AUDIO_G711A
|
||||
volc_hal_frame_info_t frame_info = {
|
||||
.data_type = VOLC_AUDIO_DATA_TYPE_G711A,
|
||||
@@ -83,13 +88,10 @@ static void __volc_audio_capture_task(void *arg)
|
||||
#endif
|
||||
}
|
||||
}
|
||||
// audio_recorder_pause();
|
||||
if (impl->capture_thread) {
|
||||
volc_osal_thread_destroy(impl->capture_thread);
|
||||
impl->capture_thread = NULL;
|
||||
}
|
||||
volc_osal_free(read_buf);
|
||||
volc_osal_thread_exit(NULL);
|
||||
// Stop owns deletion with the original WithCaps handle. No old reader may
|
||||
// survive into the next recorder, or erase a newly created task's handle.
|
||||
xSemaphoreGive(impl->capture_stopped);
|
||||
for (;;) vTaskSuspend(NULL);
|
||||
}
|
||||
|
||||
volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t *config)
|
||||
@@ -100,6 +102,9 @@ volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t *config)
|
||||
return NULL;
|
||||
}
|
||||
volc_hal_capture_impl_t *capture = (volc_hal_capture_impl_t *)volc_osal_calloc(1, sizeof(volc_hal_capture_impl_t));
|
||||
if (!capture) return NULL;
|
||||
capture->capture_stopped = xSemaphoreCreateBinary();
|
||||
if (!capture->capture_stopped) { volc_osal_free(capture); return NULL; }
|
||||
// config
|
||||
capture->is_started = false;
|
||||
capture->media_type = config->media_type;
|
||||
@@ -132,53 +137,37 @@ volc_hal_capture_t volc_hal_capture_create(volc_hal_capture_config_t *config)
|
||||
return (volc_hal_capture_t)capture;
|
||||
}
|
||||
|
||||
static void __start_audio_capture(volc_hal_capture_impl_t *capture)
|
||||
static int __start_audio(volc_hal_capture_impl_t *capture, bool wakeup)
|
||||
{
|
||||
// audio_pipeline_unlink(impl->audio_capture_config.audio_pipeline);
|
||||
av_processor_afe_config_t afe_config = DEFAULT_AV_PROCESSOR_AFE_CONFIG();
|
||||
afe_config.ai_mode_wakeup = false;
|
||||
afe_config.ai_mode_wakeup = wakeup;
|
||||
audio_recorder_config_t recorder_config = DEFAULT_AUDIO_RECORDER_CONFIG();
|
||||
memcpy((void *)&recorder_config.encoder_cfg, &capture->audio_capture_config.recorder_cfg, sizeof(av_processor_encoder_config_t));
|
||||
memcpy(&recorder_config.afe_config, &afe_config, sizeof(av_processor_afe_config_t));
|
||||
recorder_config.recorder_event_cb = capture->audio_wakeup_cb;
|
||||
recorder_config.recorder_ctx = NULL;
|
||||
audio_recorder_open(&recorder_config);
|
||||
|
||||
volc_osal_thread_param_t param = {0};
|
||||
snprintf(param.name, sizeof(param.name), "%s", "volc_capture_task");
|
||||
param.stack_size = 4 * 1024;
|
||||
param.priority = 5;
|
||||
param.stack_in_ext = 1;
|
||||
capture->read_buf = volc_osal_malloc(__volc_capture_get_default_read_size(capture));
|
||||
if (!capture->read_buf) return -1;
|
||||
if (audio_recorder_open(&recorder_config) != ESP_OK) {
|
||||
volc_osal_free(capture->read_buf); capture->read_buf = NULL;
|
||||
return -1;
|
||||
}
|
||||
capture->is_started = true;
|
||||
capture->audio_capture_config.state = CAPTURE;
|
||||
esp_err_t ret = volc_osal_thread_create(&capture->capture_thread, ¶m, __volc_audio_capture_task, capture);
|
||||
|
||||
LOGI("capture task start... mode: CAPTURE");
|
||||
return;
|
||||
}
|
||||
|
||||
static void __start_audio_wake(volc_hal_capture_impl_t *capture)
|
||||
{
|
||||
av_processor_afe_config_t afe_config = DEFAULT_AV_PROCESSOR_AFE_CONFIG();
|
||||
afe_config.ai_mode_wakeup = true;
|
||||
audio_recorder_config_t recorder_config = DEFAULT_AUDIO_RECORDER_CONFIG();
|
||||
memcpy((void *)&recorder_config.encoder_cfg, &capture->audio_capture_config.recorder_cfg, sizeof(av_processor_encoder_config_t));
|
||||
memcpy(&recorder_config.afe_config, &afe_config, sizeof(av_processor_afe_config_t));
|
||||
recorder_config.recorder_event_cb = capture->audio_wakeup_cb;
|
||||
recorder_config.recorder_ctx = NULL;
|
||||
audio_recorder_open(&recorder_config);
|
||||
|
||||
volc_osal_thread_param_t param = {0};
|
||||
snprintf(param.name, sizeof(param.name), "%s", "volc_capture_task");
|
||||
param.stack_size = 4 * 1024;
|
||||
param.priority = 5;
|
||||
param.stack_in_ext = 1;
|
||||
capture->is_started = true;
|
||||
capture->audio_capture_config.state = WAKEUP;
|
||||
esp_err_t ret = volc_osal_thread_create(&capture->capture_thread, ¶m, __volc_audio_capture_task, capture);
|
||||
|
||||
LOGI("capture task start... mode: WAKEUP");
|
||||
return;
|
||||
capture->audio_capture_config.state = wakeup ? WAKEUP : CAPTURE;
|
||||
// RTC Opus encoding needs the larger stack only during conversation.
|
||||
if (xTaskCreatePinnedToCoreWithCaps(__volc_audio_capture_task, "volc_capture_task",
|
||||
wakeup ? 4*1024 : 48*1024, capture, 5, &capture->capture_thread,
|
||||
1, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT) != pdPASS) {
|
||||
capture->is_started = false;
|
||||
capture->capture_thread = NULL;
|
||||
audio_recorder_close();
|
||||
capture->audio_capture_config.state = INITED;
|
||||
volc_osal_free(capture->read_buf); capture->read_buf = NULL;
|
||||
return -1;
|
||||
}
|
||||
LOGI("capture task start... mode: %s", wakeup ? "WAKEUP" : "CAPTURE");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int volc_hal_capture_start(volc_hal_capture_t capture,volc_hal_audio_capture_mode_e mode)
|
||||
@@ -191,13 +180,11 @@ int volc_hal_capture_start(volc_hal_capture_t capture,volc_hal_audio_capture_mod
|
||||
switch (mode)
|
||||
{
|
||||
case VOLC_AUDIO_MODE_WAKEUP:
|
||||
__start_audio_wake(impl);
|
||||
break;
|
||||
return __start_audio(impl, true);
|
||||
case VOLC_AUDIO_MODE_CAPTURE:
|
||||
__start_audio_capture(impl);
|
||||
break;
|
||||
return __start_audio(impl, false);
|
||||
default:
|
||||
break;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
@@ -217,6 +204,12 @@ int volc_hal_capture_stop(volc_hal_capture_t capture)
|
||||
// audio_recorder_pause();
|
||||
// audio_recorder_resume();
|
||||
audio_recorder_close();
|
||||
if (impl->capture_thread) {
|
||||
xSemaphoreTake(impl->capture_stopped, portMAX_DELAY);
|
||||
vTaskDeleteWithCaps(impl->capture_thread);
|
||||
impl->capture_thread = NULL;
|
||||
}
|
||||
volc_osal_free(impl->read_buf); impl->read_buf = NULL;
|
||||
impl->audio_capture_config.state = INITED;
|
||||
break;
|
||||
case VOLC_MEDIA_TYPE_VIDEO:
|
||||
@@ -252,6 +245,7 @@ void volc_hal_capture_destroy(volc_hal_capture_t capture)
|
||||
default:
|
||||
break;
|
||||
}
|
||||
vSemaphoreDelete(impl->capture_stopped);
|
||||
volc_osal_free(impl);
|
||||
}
|
||||
|
||||
|
||||
+83
-30
@@ -3,7 +3,7 @@
|
||||
|
||||
#include "volc_hal.h"
|
||||
#include "volc_hal_display.h"
|
||||
#include "volc_lvgl_source.h"
|
||||
#include "echoear_call_status_font.h"
|
||||
#include "hw_init.h"
|
||||
#include "esp_lcd_panel_ops.h"
|
||||
#include "esp_mmap_assets.h"
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <stddef.h>
|
||||
#include "volc_osal.h"
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
#include "basic_board.h"
|
||||
|
||||
#include "mmap_generate_eaf.h"
|
||||
@@ -31,6 +32,8 @@ typedef struct volc_hal_display_impl {
|
||||
gfx_obj_t *s_label;
|
||||
gfx_obj_t *s_subtitle;
|
||||
gfx_obj_t *s_setup_hint;
|
||||
atomic_int requested_status;
|
||||
int rendered_status;
|
||||
esp_lcd_panel_io_handle_t display_io_handle;
|
||||
SemaphoreHandle_t flush_done;
|
||||
int s_current_emote;
|
||||
@@ -40,6 +43,10 @@ typedef struct volc_hal_display_impl {
|
||||
static volc_hal_display_impl_t* global_display_impl = NULL;
|
||||
extern basic_board_periph_t global_periph;
|
||||
|
||||
static const char *const call_status_texts[] = {
|
||||
"", "待机", "正在连接", "通话中", "正在结束", "通话结束", "连接失败", "连接中断"
|
||||
};
|
||||
|
||||
static esp_lv_adapter_rotation_t __get_configured_rotation(void)
|
||||
{
|
||||
#if CONFIG_EXAMPLE_DISPLAY_ROTATION_0
|
||||
@@ -157,6 +164,43 @@ static esp_err_t __create_animation(void)
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
/* Keep status outside the eyes, in the safe area of the round panel. GFX
|
||||
* caches the glyph mask; update it only when the status actually changes. */
|
||||
static esp_err_t __create_status_label(void)
|
||||
{
|
||||
gfx_emote_lock(global_display_impl->s_gfx);
|
||||
gfx_obj_t *label = gfx_label_create(global_display_impl->s_gfx);
|
||||
if (!label) {
|
||||
gfx_emote_unlock(global_display_impl->s_gfx);
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
global_display_impl->s_label = label;
|
||||
gfx_obj_set_size(label, 240, 42);
|
||||
gfx_obj_align(label, GFX_ALIGN_TOP_MID, 0, 58);
|
||||
gfx_label_set_font(label, (gfx_font_t)echoear_call_status_font());
|
||||
gfx_label_set_text_align(label, GFX_TEXT_ALIGN_CENTER);
|
||||
gfx_label_set_long_mode(label, GFX_LABEL_LONG_CLIP);
|
||||
gfx_label_set_color(label, GFX_COLOR_HEX(0xFFFFFF));
|
||||
gfx_label_set_bg_color(label, GFX_COLOR_HEX(0x000000));
|
||||
gfx_label_set_bg_enable(label, true);
|
||||
gfx_obj_set_visible(label, false);
|
||||
esp_err_t ret = gfx_label_set_text(label, "");
|
||||
gfx_emote_unlock(global_display_impl->s_gfx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void __render_status(volc_hal_display_impl_t *display)
|
||||
{
|
||||
int requested = atomic_load(&display->requested_status);
|
||||
if (requested == display->rendered_status) return;
|
||||
gfx_emote_lock(display->s_gfx);
|
||||
if (gfx_label_set_text(display->s_label, call_status_texts[requested]) == ESP_OK) {
|
||||
gfx_obj_set_visible(display->s_label, requested != 0);
|
||||
display->rendered_status = requested;
|
||||
}
|
||||
gfx_emote_unlock(display->s_gfx);
|
||||
}
|
||||
|
||||
static void __emote_task(void *arg)
|
||||
{
|
||||
/* Start animation playback */
|
||||
@@ -166,6 +210,7 @@ static void __emote_task(void *arg)
|
||||
|
||||
/* The display task owns animation playback for the lifetime of the app. */
|
||||
while (1) {
|
||||
__render_status(global_display_impl);
|
||||
vTaskDelay(pdMS_TO_TICKS(100));
|
||||
}
|
||||
}
|
||||
@@ -267,7 +312,10 @@ volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config)
|
||||
* animation instead; application status text explains the setup state.
|
||||
*/
|
||||
global_display_impl->s_current_emote = MMAP_EAF_HAPPY_EAF;
|
||||
atomic_init(&global_display_impl->requested_status, 0);
|
||||
global_display_impl->rendered_status = -1;
|
||||
__create_animation();
|
||||
ESP_ERROR_CHECK(__create_status_label());
|
||||
volc_osal_thread_param_t param = {0};
|
||||
snprintf(param.name, sizeof(param.name), "%s", "volc_capture_task");
|
||||
param.stack_size = 8 * 1024;
|
||||
@@ -282,36 +330,31 @@ volc_hal_display_t volc_hal_display_create(volc_hal_display_config_t* config)
|
||||
return (volc_hal_display_t)global_display_impl;
|
||||
}
|
||||
|
||||
static int display_backlight(volc_hal_display_t display, periph_ledc_handle_t **handle, uint32_t *maximum)
|
||||
{
|
||||
if(!display || esp_board_manager_get_device_handle("lcd_brightness",(void **)handle)!=ESP_OK || !*handle)return -1;
|
||||
dev_ledc_ctrl_config_t *device=NULL;periph_ledc_config_t *config=NULL;
|
||||
if(esp_board_manager_get_device_config("lcd_brightness",(void *)&device)!=ESP_OK || !device)return -1;
|
||||
if(esp_board_manager_get_periph_config(device->ledc_name,(void **)&config)!=ESP_OK || !config)return -1;
|
||||
if(config->duty_resolution<1 || config->duty_resolution>20)return -1;
|
||||
*maximum=(1U<<(uint32_t)config->duty_resolution)-1;
|
||||
return 0;
|
||||
}
|
||||
int volc_hal_display_set_brightness(volc_hal_display_t display, int brightness)
|
||||
{
|
||||
if(display == NULL) return -1;
|
||||
if (brightness > 100) {
|
||||
brightness = 100;
|
||||
}
|
||||
if (brightness < 0) {
|
||||
brightness = 0;
|
||||
}
|
||||
|
||||
periph_ledc_handle_t *ledc_handle = NULL;
|
||||
|
||||
esp_board_manager_get_device_handle("lcd_brightness", (void **)&ledc_handle);
|
||||
|
||||
dev_ledc_ctrl_config_t *dev_ledc_cfg = NULL;
|
||||
esp_err_t config_ret = esp_board_manager_get_device_config("lcd_brightness", (void*)&dev_ledc_cfg);
|
||||
if (config_ret != ESP_OK) {
|
||||
return -1;
|
||||
}
|
||||
// ESP_LOGI(TAG, "dev_ledc_cfg.ledc_name: %s, name: %s, type: %s", dev_ledc_cfg->ledc_name, dev_ledc_cfg->name, dev_ledc_cfg->type);
|
||||
periph_ledc_config_t *ledc_config = NULL;
|
||||
esp_board_manager_get_periph_config(dev_ledc_cfg->ledc_name, (void**)&ledc_config);
|
||||
uint32_t duty = (brightness * ((1 << (uint32_t)ledc_config->duty_resolution) - 1)) / 100;
|
||||
// ESP_LOGI(TAG, "duty_cycle: %" PRIu32 ", speed_mode: %d, channel: %d, duty_resolution: %d", duty, ledc_handle->speed_mode, ledc_handle->channel, ledc_config->duty_resolution);
|
||||
// ESP_BOARD_RETURN_ON_ERROR(ledc_set_duty(ledc_handle->speed_mode, ledc_handle->channel, duty), TAG, "LEDC set duty failed");
|
||||
ledc_set_duty(ledc_handle->speed_mode, ledc_handle->channel, duty);
|
||||
// ESP_BOARD_RETURN_ON_ERROR(ledc_update_duty(ledc_handle->speed_mode, ledc_handle->channel), TAG, "LEDC update duty failed");
|
||||
ledc_update_duty(ledc_handle->speed_mode, ledc_handle->channel);
|
||||
return 0;
|
||||
// return bsp_display_brightness_set(brightness);
|
||||
periph_ledc_handle_t *handle=NULL;uint32_t maximum=0;
|
||||
if(display_backlight(display,&handle,&maximum))return -1;
|
||||
if(brightness<0)brightness=0;if(brightness>100)brightness=100;
|
||||
if(ledc_set_duty(handle->speed_mode,handle->channel,brightness*maximum/100)!=ESP_OK)return -1;
|
||||
return ledc_update_duty(handle->speed_mode,handle->channel)==ESP_OK?0:-1;
|
||||
}
|
||||
int volc_hal_display_get_brightness(volc_hal_display_t display)
|
||||
{
|
||||
periph_ledc_handle_t *handle=NULL;uint32_t maximum=0;
|
||||
if(display_backlight(display,&handle,&maximum))return -1;
|
||||
uint32_t duty=ledc_get_duty(handle->speed_mode,handle->channel);
|
||||
if(duty>maximum)return -1;
|
||||
return (duty*100+maximum/2)/maximum;
|
||||
}
|
||||
|
||||
void volc_hal_display_destroy(volc_hal_display_t display)
|
||||
@@ -339,7 +382,17 @@ int volc_hal_display_set_content(volc_hal_display_t display, volc_hal_display_ob
|
||||
volc_hal_display_impl_t* global_display_impl = (volc_hal_display_impl_t*)(g_hal_context->display_handle);
|
||||
|
||||
if(obj == VOLC_DISPLAY_OBJ_STATUS && type == VOLC_DISPLAY_TEXT){
|
||||
ESP_LOGI("DISPLAY", "Status: %s", (const char *)content);
|
||||
if (!content) return -1;
|
||||
int requested = 0;
|
||||
for (int i = 1; i < sizeof(call_status_texts) / sizeof(call_status_texts[0]); ++i) {
|
||||
if (strcmp(content, call_status_texts[i]) == 0) { requested = i; break; }
|
||||
}
|
||||
if (strcmp(content, "轻触屏幕开始聊天") == 0 ||
|
||||
strcmp(content, "轻触屏幕或说嗨,乐鑫") == 0) requested = 1;
|
||||
/* RTC callbacks never wait on the renderer's full-frame lock. Other
|
||||
* setup text keeps its previous behavior, with no subtitle rendering. */
|
||||
atomic_store(&global_display_impl->requested_status, requested);
|
||||
return 0;
|
||||
}
|
||||
if(obj == VOLC_DISPLAY_OBJ_SUBTITLE && type == VOLC_DISPLAY_TEXT){
|
||||
ESP_LOGI("DISPLAY", "Subtitle: %s", (const char *)content);
|
||||
|
||||
+93
-11
@@ -2,6 +2,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
#include "volc_hal.h"
|
||||
#include "volc_hal_player.h"
|
||||
@@ -38,17 +39,33 @@ typedef struct {
|
||||
audio_player_config_t audio_player_config;
|
||||
video_player_config_t video_player_config;
|
||||
};
|
||||
volatile bool is_started;
|
||||
atomic_bool is_started;
|
||||
atomic_int feeder_stop_phase;
|
||||
StreamBufferHandle_t pcm_stream;
|
||||
StaticStreamBuffer_t stream_state;
|
||||
uint8_t *stream_storage;
|
||||
SemaphoreHandle_t worker_stopped;
|
||||
volatile bool discard_pending;
|
||||
volatile bool segment_complete;
|
||||
bool (*authorization_gate)(void);
|
||||
atomic_bool write_in_progress;
|
||||
atomic_uint last_voice_write_ms;
|
||||
} volc_player_impl_t;
|
||||
|
||||
extern basic_board_periph_t global_periph;
|
||||
|
||||
static int stop_feeder_once(volc_player_impl_t *impl)
|
||||
{
|
||||
int expected = 0;
|
||||
if (atomic_compare_exchange_strong(&impl->feeder_stop_phase, &expected, 1)) {
|
||||
int ret = audio_feeder_stop();
|
||||
atomic_store(&impl->feeder_stop_phase, 2);
|
||||
return ret;
|
||||
}
|
||||
while (atomic_load(&impl->feeder_stop_phase) == 1) vTaskDelay(1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void pcm_playback_task(void *arg)
|
||||
{
|
||||
volc_player_impl_t *impl = arg;
|
||||
@@ -56,7 +73,22 @@ static void pcm_playback_task(void *arg)
|
||||
bool primed = false;
|
||||
uint64_t first_byte_ms = 0;
|
||||
uint64_t last_feed_ms = 0;
|
||||
bool authorization_closed = false;
|
||||
uint64_t report_ms=volc_osal_get_time_ms();
|
||||
unsigned fed_bytes=0,max_write_ms=0,write_errors=0;
|
||||
while (impl->is_started) {
|
||||
uint64_t tick_ms=volc_osal_get_time_ms();
|
||||
if(tick_ms-report_ms>=1000){
|
||||
LOGI("PLAYBACK stats span_ms=%u fed_ms=%u queued_ms=%u max_write_ms=%u errors=%u primed=%d",(unsigned)(tick_ms-report_ms),fed_bytes/32,(unsigned)xStreamBufferBytesAvailable(impl->pcm_stream)/32,max_write_ms,write_errors,primed);
|
||||
report_ms=tick_ms;fed_bytes=max_write_ms=write_errors=0;
|
||||
}
|
||||
// Runs in the audio task even while the control task is blocked on HTTP.
|
||||
if (authorization_closed || (impl->authorization_gate && !impl->authorization_gate())) {
|
||||
if (!authorization_closed) { stop_feeder_once(impl); authorization_closed = true; }
|
||||
while (xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), 0)) {}
|
||||
vTaskDelay(pdMS_TO_TICKS(10));
|
||||
continue;
|
||||
}
|
||||
if (impl->discard_pending) {
|
||||
while (xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), 0)) {}
|
||||
impl->discard_pending = false;
|
||||
@@ -80,8 +112,10 @@ static void pcm_playback_task(void *arg)
|
||||
LOGI("PLAYBACK start source=16000Hz mono PCM16 buffered=%u wait_ms=%u complete=%d",
|
||||
(unsigned)available, (unsigned)(volc_osal_get_time_ms() - first_byte_ms), impl->segment_complete);
|
||||
}
|
||||
size_t count = xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), pdMS_TO_TICKS(30));
|
||||
atomic_store(&impl->write_in_progress, true);
|
||||
size_t count = xStreamBufferReceive(impl->pcm_stream, pcm, sizeof(pcm), 0);
|
||||
if (!count) {
|
||||
atomic_store(&impl->write_in_progress, false);
|
||||
/* An empty ingress queue does not mean the speaker has run dry:
|
||||
* the downstream feeder may still hold 256 ms of PCM. Keep feeding
|
||||
* arriving chunks immediately instead of inserting another prefill
|
||||
@@ -91,13 +125,35 @@ static void pcm_playback_task(void *arg)
|
||||
first_byte_ms = 0;
|
||||
LOGI("PLAYBACK idle after %u ms without PCM", PCM_REPRIME_IDLE_MS);
|
||||
}
|
||||
vTaskDelay(pdMS_TO_TICKS(10));
|
||||
continue;
|
||||
}
|
||||
if (audio_feeder_feed_data(pcm, count) == ESP_OK) {
|
||||
if (impl->authorization_gate && !impl->authorization_gate()) {
|
||||
atomic_store(&impl->write_in_progress, false);
|
||||
continue;
|
||||
}
|
||||
uint64_t magnitude = 0;
|
||||
for (size_t i = 0; i + 1 < count; i += 2) {
|
||||
int16_t sample;
|
||||
memcpy(&sample, pcm + i, sizeof(sample));
|
||||
magnitude += sample < 0 ? -(int)sample : sample;
|
||||
}
|
||||
bool voiced = count >= 2 && magnitude / (count / 2) >= 64;
|
||||
if (voiced) atomic_store(&impl->last_voice_write_ms, (uint32_t)volc_osal_get_time_ms());
|
||||
// A full 1024-byte chunk represents 32 ms at 16 kHz PCM16. Allow
|
||||
// the sink to consume it without dropping audio at a 20 ms timeout.
|
||||
uint64_t write_ms=volc_osal_get_time_ms();
|
||||
if (audio_feeder_feed_data_timeout(pcm, count, 50) == ESP_OK) {
|
||||
fed_bytes+=count;
|
||||
last_feed_ms = volc_osal_get_time_ms();
|
||||
} else if (impl->is_started) {
|
||||
write_errors++;
|
||||
LOGE("PLAYBACK feeder write failed");
|
||||
}
|
||||
unsigned elapsed_write=(unsigned)(volc_osal_get_time_ms()-write_ms);
|
||||
if (voiced) atomic_store(&impl->last_voice_write_ms, (uint32_t)volc_osal_get_time_ms());
|
||||
atomic_store(&impl->write_in_progress, false);
|
||||
if(elapsed_write>max_write_ms)max_write_ms=elapsed_write;
|
||||
}
|
||||
xSemaphoreGive(impl->worker_stopped);
|
||||
vTaskDelete(NULL);
|
||||
@@ -115,10 +171,7 @@ volc_hal_player_t volc_hal_player_create(volc_hal_player_config_t* config)
|
||||
impl->media_type = config->media_type;
|
||||
switch (impl->media_type) {
|
||||
case VOLC_MEDIA_TYPE_AUDIO:
|
||||
if(global_periph.play_dev){
|
||||
esp_codec_dev_set_out_vol(global_periph.play_dev, DEFAULT_AUDIO_VOL);
|
||||
}
|
||||
impl->audio_player_config.volume = DEFAULT_AUDIO_VOL;
|
||||
impl->audio_player_config.volume = volc_hal_get_device_volume();
|
||||
#if (CONFIG_VOLC_AUDIO_G711A)
|
||||
impl->audio_player_config.feeder_cfg.format = AV_PROCESSOR_FORMAT_ID_G711A;
|
||||
impl->audio_player_config.feeder_cfg.params.g711.audio_info.sample_rate = 8000;
|
||||
@@ -168,14 +221,24 @@ volc_hal_player_t volc_hal_player_create(volc_hal_player_config_t* config)
|
||||
return (volc_hal_player_t)impl;
|
||||
}
|
||||
|
||||
int volc_hal_set_device_volume(int volume){
|
||||
if(volume<0 || volume>100 || !global_periph.play_dev)return -1;
|
||||
return esp_codec_dev_set_out_vol(global_periph.play_dev,volume);
|
||||
}
|
||||
int volc_hal_get_device_volume(void){
|
||||
int volume=-1;
|
||||
if(!global_periph.play_dev || esp_codec_dev_get_out_vol(global_periph.play_dev,&volume)!=ESP_OK)return -1;
|
||||
return volume;
|
||||
}
|
||||
int volc_hal_set_audio_player_volume(volc_hal_player_t player, int volume){
|
||||
volc_player_impl_t* impl = (volc_player_impl_t*)player;
|
||||
if (impl == NULL) {
|
||||
return -1;
|
||||
}
|
||||
if(impl->media_type == VOLC_MEDIA_TYPE_AUDIO){
|
||||
impl->audio_player_config.volume = volume;
|
||||
return esp_codec_dev_set_out_vol(global_periph.play_dev, volume);
|
||||
int result=volc_hal_set_device_volume(volume);
|
||||
if (result==ESP_OK) impl->audio_player_config.volume = volume;
|
||||
return result;
|
||||
} else if(impl->media_type == VOLC_MEDIA_TYPE_VIDEO){
|
||||
LOGE("media type %d not supported set volume", impl->media_type);
|
||||
return -1;
|
||||
@@ -187,7 +250,7 @@ int volc_hal_get_audio_player_volume(volc_hal_player_t player){
|
||||
if (impl == NULL) {
|
||||
return -1;
|
||||
}
|
||||
return impl->audio_player_config.volume;
|
||||
return volc_hal_get_device_volume();
|
||||
}
|
||||
|
||||
void volc_hal_player_destroy(volc_hal_player_t player)
|
||||
@@ -231,6 +294,9 @@ int volc_hal_player_start(volc_hal_player_t player)
|
||||
case VOLC_MEDIA_TYPE_AUDIO:
|
||||
if (audio_feeder_run() != ESP_OK) return -1;
|
||||
xStreamBufferReset(impl->pcm_stream);
|
||||
atomic_store(&impl->feeder_stop_phase, 0);
|
||||
atomic_store(&impl->write_in_progress, false);
|
||||
atomic_store(&impl->last_voice_write_ms, (uint32_t)volc_osal_get_time_ms());
|
||||
impl->is_started = true;
|
||||
if (xTaskCreatePinnedToCore(pcm_playback_task, "pcm_playback", 4096, impl, 5, NULL, 1) != pdPASS) {
|
||||
impl->is_started = false;
|
||||
@@ -260,7 +326,7 @@ int volc_hal_player_stop(volc_hal_player_t player)
|
||||
case VOLC_MEDIA_TYPE_AUDIO:
|
||||
impl->is_started = false;
|
||||
/* Stop the feeder first to release any in-flight blocking write. */
|
||||
int ret = audio_feeder_stop();
|
||||
int ret = stop_feeder_once(impl);
|
||||
xSemaphoreTake(impl->worker_stopped, portMAX_DELAY);
|
||||
return ret;
|
||||
break;
|
||||
@@ -313,3 +379,19 @@ void volc_hal_player_finish_segment(volc_hal_player_t player)
|
||||
volc_player_impl_t *impl = player;
|
||||
if (impl && impl->is_started) impl->segment_complete = true;
|
||||
}
|
||||
|
||||
bool volc_hal_player_voice_quiet(volc_hal_player_t player, unsigned quiet_ms)
|
||||
{
|
||||
volc_player_impl_t *impl = player;
|
||||
if (!impl || !impl->is_started || !impl->pcm_stream) return false;
|
||||
uint32_t now = (uint32_t)volc_osal_get_time_ms();
|
||||
return !atomic_load(&impl->write_in_progress) &&
|
||||
xStreamBufferBytesAvailable(impl->pcm_stream) == 0 &&
|
||||
now - atomic_load(&impl->last_voice_write_ms) >= quiet_ms;
|
||||
}
|
||||
|
||||
void volc_hal_player_set_authorization_gate(volc_hal_player_t player, bool (*allowed)(void))
|
||||
{
|
||||
volc_player_impl_t *impl = player;
|
||||
if (impl && !impl->is_started) impl->authorization_gate = allowed;
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
#ifndef CALL_DISPLAY_STATE_H
|
||||
#define CALL_DISPLAY_STATE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef enum {
|
||||
CALL_DISPLAY_IDLE,
|
||||
CALL_DISPLAY_CONNECTING,
|
||||
CALL_DISPLAY_TALKING,
|
||||
CALL_DISPLAY_ENDING,
|
||||
CALL_DISPLAY_ENDED,
|
||||
CALL_DISPLAY_FAILED,
|
||||
CALL_DISPLAY_INTERRUPTED,
|
||||
} call_display_phase_t;
|
||||
|
||||
typedef enum {
|
||||
CALL_DISPLAY_STOP_NORMAL,
|
||||
CALL_DISPLAY_STOP_FAILED,
|
||||
CALL_DISPLAY_STOP_INTERRUPTED,
|
||||
} call_display_stop_t;
|
||||
|
||||
// Owned by the conversation task. SDK callbacks only latch atomic observations;
|
||||
// they never render text or mutate this lifecycle while the task is stopping.
|
||||
typedef struct {
|
||||
call_display_phase_t phase;
|
||||
unsigned generation;
|
||||
bool connected;
|
||||
uint64_t terminal_until_ms;
|
||||
} call_display_state_t;
|
||||
|
||||
unsigned call_display_begin(call_display_state_t *state);
|
||||
void call_display_connected(call_display_state_t *state, unsigned generation);
|
||||
void call_display_stopping(call_display_state_t *state, unsigned generation);
|
||||
void call_display_finished(call_display_state_t *state, unsigned generation,
|
||||
call_display_stop_t reason, bool audio_stopped, uint64_t now_ms);
|
||||
bool call_display_idle_if_due(call_display_state_t *state, unsigned generation, uint64_t now_ms);
|
||||
const char *call_display_text(const call_display_state_t *state);
|
||||
|
||||
#endif
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Owned by the conversation task. SDK callbacks only publish atomic counters. */
|
||||
typedef struct {
|
||||
bool pending;
|
||||
bool reply_complete;
|
||||
uint32_t requested_ms;
|
||||
unsigned finish_baseline;
|
||||
} call_voice_end_t;
|
||||
|
||||
bool call_voice_end_command(const char *text);
|
||||
void call_voice_end_reset(call_voice_end_t *state);
|
||||
void call_voice_end_user(call_voice_end_t *state, const char *text,
|
||||
uint32_t now_ms, unsigned finish_serial);
|
||||
bool call_voice_end_due(const call_voice_end_t *state, uint32_t now_ms,
|
||||
unsigned finish_serial, bool playback_quiet,
|
||||
bool user_speaking);
|
||||
+1
@@ -12,6 +12,7 @@ typedef struct {
|
||||
} volc_conv_service_t;
|
||||
|
||||
void conv_ai_service_task(void *pvParameters);
|
||||
void conv_ai_service_prepare_task(void);
|
||||
void conv_ai_service_task_stop();
|
||||
void conv_ai_service_init();
|
||||
volc_conv_service_t get_conv_ai_service();
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
#include "call_display_state.h"
|
||||
|
||||
unsigned call_display_begin(call_display_state_t *state)
|
||||
{
|
||||
++state->generation;
|
||||
state->phase = CALL_DISPLAY_CONNECTING;
|
||||
state->connected = false;
|
||||
state->terminal_until_ms = 0;
|
||||
return state->generation;
|
||||
}
|
||||
|
||||
void call_display_connected(call_display_state_t *state, unsigned generation)
|
||||
{
|
||||
if (state->generation == generation && state->phase == CALL_DISPLAY_CONNECTING) {
|
||||
state->connected = true;
|
||||
state->phase = CALL_DISPLAY_TALKING;
|
||||
}
|
||||
}
|
||||
|
||||
void call_display_stopping(call_display_state_t *state, unsigned generation)
|
||||
{
|
||||
if (state->generation == generation &&
|
||||
(state->phase == CALL_DISPLAY_CONNECTING || state->phase == CALL_DISPLAY_TALKING)) {
|
||||
state->phase = CALL_DISPLAY_ENDING;
|
||||
}
|
||||
}
|
||||
|
||||
void call_display_finished(call_display_state_t *state, unsigned generation,
|
||||
call_display_stop_t reason, bool audio_stopped, uint64_t now_ms)
|
||||
{
|
||||
if (state->generation != generation || state->phase != CALL_DISPLAY_ENDING || !audio_stopped) return;
|
||||
state->phase = reason == CALL_DISPLAY_STOP_NORMAL ? CALL_DISPLAY_ENDED :
|
||||
state->connected ? CALL_DISPLAY_INTERRUPTED : CALL_DISPLAY_FAILED;
|
||||
state->terminal_until_ms = now_ms + 2000;
|
||||
}
|
||||
|
||||
bool call_display_idle_if_due(call_display_state_t *state, unsigned generation, uint64_t now_ms)
|
||||
{
|
||||
if (state->generation != generation ||
|
||||
(state->phase != CALL_DISPLAY_ENDED && state->phase != CALL_DISPLAY_FAILED &&
|
||||
state->phase != CALL_DISPLAY_INTERRUPTED) || now_ms < state->terminal_until_ms) return false;
|
||||
state->phase = CALL_DISPLAY_IDLE;
|
||||
return true;
|
||||
}
|
||||
|
||||
const char *call_display_text(const call_display_state_t *state)
|
||||
{
|
||||
switch (state->phase) {
|
||||
case CALL_DISPLAY_CONNECTING: return "正在连接";
|
||||
case CALL_DISPLAY_TALKING: return "通话中";
|
||||
case CALL_DISPLAY_ENDING: return "正在结束";
|
||||
case CALL_DISPLAY_ENDED: return "通话结束";
|
||||
case CALL_DISPLAY_FAILED: return "连接失败";
|
||||
case CALL_DISPLAY_INTERRUPTED: return "连接中断";
|
||||
default: return "待机";
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
#include "call_voice_end.h"
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Match a whole short request, never a substring in a story, quote, question
|
||||
* or negated instruction. This intentionally is a bounded command vocabulary. */
|
||||
bool call_voice_end_command(const char *text)
|
||||
{
|
||||
if (!text) return false;
|
||||
char normalized[256];
|
||||
size_t n = 0;
|
||||
static const char *punctuation[] = {",", "。", "!", "、"};
|
||||
for (const char *p = text; *p;) {
|
||||
bool skipped = false;
|
||||
for (size_t i = 0; i < sizeof(punctuation) / sizeof(*punctuation); ++i) {
|
||||
size_t len = strlen(punctuation[i]);
|
||||
if (!strncmp(p, punctuation[i], len)) { p += len; skipped = true; break; }
|
||||
}
|
||||
if (skipped) continue;
|
||||
if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n' ||
|
||||
*p == ',' || *p == '.' || *p == '!') { ++p; continue; }
|
||||
/* Preserve question marks, quotation marks and all other words. */
|
||||
if (n + 1 >= sizeof(normalized)) return false;
|
||||
normalized[n++] = *p++;
|
||||
}
|
||||
normalized[n] = 0;
|
||||
static const char *prefixes[] = {
|
||||
"好了", "好啦", "好的", "嗯", "那就", "那", "我们", "咱们",
|
||||
"今天", "现在", "就先", "就", "先", "请", "麻烦你", "谢谢你", "谢谢"
|
||||
};
|
||||
const char *p = normalized;
|
||||
for (unsigned pass = 0; pass < 8; ++pass) {
|
||||
bool found = false;
|
||||
for (size_t i = 0; i < sizeof(prefixes) / sizeof(*prefixes); ++i) {
|
||||
size_t len = strlen(prefixes[i]);
|
||||
if (!strncmp(p, prefixes[i], len)) { p += len; found = true; break; }
|
||||
}
|
||||
if (!found) break;
|
||||
}
|
||||
static const char *commands[] = {
|
||||
"拜拜", "再见", "下次再聊", "下回再聊", "回头再聊",
|
||||
"聊到这里", "聊到这儿", "聊到这", "不聊了", "我不聊了",
|
||||
"结束聊天", "结束对话", "结束通话", "结束这次聊天", "结束本次通话",
|
||||
"挂断电话", "挂断通话", "挂断", "挂了", "我挂了", "我先挂了",
|
||||
"我先走了", "我要休息了", "我去休息了"
|
||||
};
|
||||
static const char *tails[] = {
|
||||
"吧", "把", "啦", "了", "啊", "呀", "哦", "哈", "谢谢你", "谢谢",
|
||||
"拜拜", "再见", "下次再聊", "下回再聊", "回头再聊"
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(commands) / sizeof(*commands); ++i) {
|
||||
size_t len = strlen(commands[i]);
|
||||
if (strncmp(p, commands[i], len)) continue;
|
||||
const char *tail = p + len;
|
||||
for (unsigned pass = 0; *tail && pass < 8; ++pass) {
|
||||
bool found = false;
|
||||
for (size_t j = 0; j < sizeof(tails) / sizeof(*tails); ++j) {
|
||||
size_t tlen = strlen(tails[j]);
|
||||
if (!strncmp(tail, tails[j], tlen)) { tail += tlen; found = true; break; }
|
||||
}
|
||||
if (!found) break;
|
||||
}
|
||||
if (!*tail) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void call_voice_end_reset(call_voice_end_t *state)
|
||||
{
|
||||
memset(state, 0, sizeof(*state));
|
||||
}
|
||||
|
||||
void call_voice_end_user(call_voice_end_t *state, const char *text,
|
||||
uint32_t now_ms, unsigned finish_serial)
|
||||
{
|
||||
if (!call_voice_end_command(text)) {
|
||||
/* A later complete user utterance cancels the pending goodbye. */
|
||||
call_voice_end_reset(state);
|
||||
} else if (!state->pending) {
|
||||
state->pending = true;
|
||||
state->reply_complete = false;
|
||||
state->requested_ms = now_ms;
|
||||
state->finish_baseline = finish_serial;
|
||||
}
|
||||
/* Repeated subtitles must not extend the stop deadline. */
|
||||
}
|
||||
|
||||
bool call_voice_end_due(const call_voice_end_t *state, uint32_t now_ms,
|
||||
unsigned finish_serial, bool playback_quiet,
|
||||
bool user_speaking)
|
||||
{
|
||||
if (!state->pending || user_speaking) return false;
|
||||
uint32_t elapsed = now_ms - state->requested_ms;
|
||||
/* Missing response events/audio must not leave an ended chat open forever. */
|
||||
if (elapsed >= 12000u) return true;
|
||||
return elapsed >= 1500u && playback_quiet &&
|
||||
(state->reply_complete || finish_serial != state->finish_baseline);
|
||||
}
|
||||
+421
-46
@@ -2,6 +2,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
#include "aios.h"
|
||||
#include "cJSON.h"
|
||||
@@ -10,6 +11,9 @@
|
||||
#include "util/volc_log.h"
|
||||
#include "volc_service_common.h"
|
||||
#include "volc_conv.h"
|
||||
#include "call_display_state.h"
|
||||
#include "call_voice_end.h"
|
||||
#include "esp_gmf_afe.h"
|
||||
#include "volc_hal.h"
|
||||
#include "volc_hal_capture.h"
|
||||
#include "volc_hal_display.h"
|
||||
@@ -20,12 +24,18 @@
|
||||
#include "mmap_generate_eaf.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/queue.h"
|
||||
#include "freertos/task.h"
|
||||
#include "freertos/idf_additions.h"
|
||||
extern int8_t get_rssi(void);
|
||||
|
||||
#if __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Opus RTP uses a 48 kHz clock in BOTH directions (RFC 7587). We submit
|
||||
* externally encoded 20 ms packets, so the SDK must timestamp them with
|
||||
* 960 ticks. The separate PCM encoder/decoder still uses 16 kHz / 320 samples;
|
||||
* using that PCM rate for RTP compresses the transmitted audio timeline. */
|
||||
#define CONV_AI_CONFIG_FORMAT "{\
|
||||
\"ver\": 1,\
|
||||
\"iot\": {\
|
||||
@@ -50,32 +60,150 @@ extern "C" {
|
||||
\"{\\\"debug\\\":{\\\"log_to_console\\\":1}}\",\
|
||||
\"{\\\"rtc\\\":{\\\"access\\\":{\\\"concurrent_requests\\\":1}}}\",\
|
||||
\"{\\\"rtc\\\":{\\\"ice\\\":{\\\"concurrent_agents\\\":1}}}\",\
|
||||
\"{\\\"rtc\\\":{\\\"thread\\\":{\\\"stack_in_ext\\\":1}}}\",\
|
||||
\"{\\\"rtc\\\":{\\\"thread\\\":{\\\"stack_in_ext\\\":1,\\\"stack_size\\\":49152}}}\",\
|
||||
\"{\\\"rtc\\\":{\\\"network\\\":{\\\"enable_audio_jitter2\\\":0}}}\",\
|
||||
\"{\\\"rtc\\\":{\\\"report\\\":{\\\"enable\\\":0}}}\",\
|
||||
\"{\\\"audio\\\":{\\\"codec\\\":{\\\"pcma\\\":{\\\"s_samples_per_frame\\\":480}}}}\"\
|
||||
\"{\\\"audio\\\":{\\\"codec\\\":{\\\"opus\\\":{\\\"s_sample_rate\\\":48000,\\\"s_samples_per_frame\\\":960,\\\"r_sample_rate\\\":48000,\\\"r_samples_per_frame\\\":960}}}}\"\
|
||||
]\
|
||||
}\
|
||||
}"
|
||||
|
||||
#define CONV_SERVICE_TIMEOUT 60
|
||||
|
||||
/* CALL_IDLE_BEGIN: callbacks only update atomic activity, never stop the SDK.
|
||||
* Continuous microphone/silence packets and lease renewals are not activity.
|
||||
* Unsigned monotonic milliseconds also handle the 32-bit clock wrap. */
|
||||
static atomic_uint last_call_activity_ms;
|
||||
static atomic_bool local_speech_active;
|
||||
|
||||
static void call_idle_activity(void)
|
||||
{
|
||||
atomic_store(&last_call_activity_ms, (uint32_t)volc_osal_get_time_ms());
|
||||
}
|
||||
|
||||
static bool call_idle_expired(void)
|
||||
{
|
||||
uint32_t now = (uint32_t)volc_osal_get_time_ms();
|
||||
return (uint32_t)(now - atomic_load(&last_call_activity_ms)) >= CONV_SERVICE_TIMEOUT * 1000u;
|
||||
}
|
||||
|
||||
static void call_idle_capture_event(void *event, void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
const esp_gmf_afe_evt_t *vad = event;
|
||||
if (!vad) return;
|
||||
if (vad->type == ESP_GMF_AFE_EVT_VAD_START || vad->type == ESP_GMF_AFE_EVT_VAD_END) {
|
||||
atomic_store(&local_speech_active, vad->type == ESP_GMF_AFE_EVT_VAD_START);
|
||||
call_idle_activity();
|
||||
}
|
||||
}
|
||||
|
||||
static bool call_idle_pcm_active(const void *data, size_t bytes)
|
||||
{
|
||||
if (!data || bytes < sizeof(int16_t) || (bytes & 1)) return false;
|
||||
const int16_t *pcm = data;
|
||||
uint64_t magnitude = 0;
|
||||
size_t samples = bytes / sizeof(*pcm);
|
||||
for (size_t i = 0; i < samples; ++i) {
|
||||
magnitude += pcm[i] < 0 ? -(int)pcm[i] : pcm[i];
|
||||
}
|
||||
// RTC sends decoded silence continuously, even when the agent is quiet.
|
||||
// Measure synthesized audio before the user's speaker volume is applied.
|
||||
return magnitude / samples >= 64;
|
||||
}
|
||||
/* CALL_IDLE_END */
|
||||
|
||||
static char config_buf[1024] = {0};
|
||||
static volc_conv_service_t conv_service = {0};
|
||||
|
||||
static volatile bool is_stop = false;
|
||||
static volatile bool provider_disconnected = false;
|
||||
static atomic_bool is_stop = true;
|
||||
static atomic_bool stop_requested = false;
|
||||
static atomic_bool provider_disconnected = false;
|
||||
static atomic_bool display_connected = false;
|
||||
static atomic_bool user_stop_requested = false;
|
||||
static call_display_state_t call_display;
|
||||
static volc_hal_display_t global_display = NULL;
|
||||
static bool session_started_reported = false;
|
||||
static QueueHandle_t platform_report_queue;
|
||||
static volatile bool diagnostic_say_pending;
|
||||
static call_voice_end_t voice_end;
|
||||
static atomic_uint voice_finish_serial;
|
||||
static atomic_uint last_voice_received_ms;
|
||||
/* VOICE_DIAGNOSTIC_BEGIN: local USB requests only. TTS keeps microphone uplink
|
||||
* suppressed until this call ends or an explicit diag-say restores it. */
|
||||
enum { DIAGNOSTIC_NONE, DIAGNOSTIC_SAY, DIAGNOSTIC_TTS, DIAGNOSTIC_PENDING_MASK = 3,
|
||||
DIAGNOSTIC_MUTE = 4, DIAGNOSTIC_CLOSING = 8, DIAGNOSTIC_ACTIVE = 16 };
|
||||
static atomic_uint diagnostic_state;
|
||||
extern bool platform_runtime_rtc(void);
|
||||
|
||||
bool conv_ai_service_diagnostic_active(void)
|
||||
{
|
||||
return (atomic_load(&diagnostic_state) & DIAGNOSTIC_ACTIVE) != 0;
|
||||
}
|
||||
|
||||
void conv_ai_service_diagnostic_reset(void)
|
||||
{
|
||||
atomic_fetch_and(&diagnostic_state, DIAGNOSTIC_CLOSING);
|
||||
}
|
||||
|
||||
static void diagnostic_close(void)
|
||||
{
|
||||
atomic_store(&diagnostic_state, DIAGNOSTIC_CLOSING);
|
||||
}
|
||||
|
||||
static void diagnostic_prepare(void)
|
||||
{
|
||||
atomic_fetch_and(&diagnostic_state, ~DIAGNOSTIC_CLOSING);
|
||||
}
|
||||
|
||||
static bool diagnostic_uplink_suppressed(void)
|
||||
{
|
||||
return (atomic_load(&diagnostic_state) & DIAGNOSTIC_MUTE) != 0;
|
||||
}
|
||||
|
||||
static unsigned diagnostic_take_pending(void)
|
||||
{
|
||||
return atomic_fetch_and(&diagnostic_state, ~DIAGNOSTIC_PENDING_MASK) & DIAGNOSTIC_PENDING_MASK;
|
||||
}
|
||||
|
||||
static void diagnostic_request(int mode)
|
||||
{
|
||||
unsigned state = atomic_load(&diagnostic_state);
|
||||
do {
|
||||
if ((state & DIAGNOSTIC_CLOSING) && xTaskGetHandle("conv_ai")) {
|
||||
LOGI("PLAYBACK diagnostic request rejected: call closing");
|
||||
return;
|
||||
}
|
||||
} while (!atomic_compare_exchange_weak(&diagnostic_state, &state,
|
||||
(unsigned)mode | DIAGNOSTIC_ACTIVE |
|
||||
(mode == DIAGNOSTIC_TTS ? DIAGNOSTIC_MUTE : 0)));
|
||||
/* Repeating the fixed passage reuses the existing room. */
|
||||
if (!xTaskGetHandle("conv_ai") && aios_event_pub(VOLC_SERVICE_AI_CONVERSATION, NULL, NULL) != 0) {
|
||||
conv_ai_service_diagnostic_reset();
|
||||
}
|
||||
}
|
||||
|
||||
void conv_ai_service_diagnostic_say(void)
|
||||
{
|
||||
diagnostic_say_pending = true;
|
||||
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION, NULL, NULL);
|
||||
diagnostic_request(DIAGNOSTIC_SAY);
|
||||
}
|
||||
|
||||
void conv_ai_service_diagnostic_tts(void)
|
||||
{
|
||||
diagnostic_request(DIAGNOSTIC_TTS);
|
||||
}
|
||||
|
||||
static void diagnostic_send_tts(void)
|
||||
{
|
||||
static const char *passage = "这是连续语音播放测试。请您听一听,这段声音是否清楚、连贯,中间有没有突然停顿。我们用自然的语速,把这段话平稳地读完。";
|
||||
int sent = platform_runtime_rtc()
|
||||
? volc_send_text_to_agent(conv_service.engine, passage, VOLC_AGENT_TYPE_TTS, 1) : -2;
|
||||
LOGI("PLAYBACK diagnostic TTS requested ret=%d mic_suppressed=1", sent);
|
||||
if (sent != 0) conv_ai_service_diagnostic_reset();
|
||||
conv_service.wait_time = 0;
|
||||
call_idle_activity();
|
||||
}
|
||||
/* VOICE_DIAGNOSTIC_END */
|
||||
|
||||
/* Implemented by the platform client in the top-level firmware component. */
|
||||
extern const char *platform_runtime_instance_id(void);
|
||||
extern const char *platform_runtime_product_key(void);
|
||||
@@ -83,12 +211,39 @@ extern const char *platform_runtime_device_name(void);
|
||||
extern const char *platform_runtime_device_secret(void);
|
||||
extern int platform_conversation_begin(void);
|
||||
extern const char *platform_last_error_code(void);
|
||||
extern const char *call_client_start_error(void);
|
||||
extern const char *platform_conversation_bot_id(void);
|
||||
extern const char *platform_conversation_update_json(void);
|
||||
extern int platform_conversation_started(const char *provider_connection_ref);
|
||||
extern int platform_conversation_poll(void);
|
||||
extern bool platform_conversation_audio_authorized(void);
|
||||
extern int platform_conversation_turn(const char *speaker, const char *source_id, const char *text);
|
||||
extern void platform_conversation_finish(bool failed, const char *error_code);
|
||||
extern bool platform_runtime_rtc(void);
|
||||
extern const char *platform_runtime_rtc_app_id(void);
|
||||
extern void platform_runtime_engine_ready(bool ready);
|
||||
extern void platform_conversation_observe(const char *type);
|
||||
extern void platform_conversation_stop_result(bool started,int result);
|
||||
static atomic_bool rtc_connected;
|
||||
extern const char *volc_current_user_id(volc_engine_t handle,bool remote);
|
||||
|
||||
static void publish_call_display(unsigned generation)
|
||||
{
|
||||
if (call_display.generation != generation) return;
|
||||
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,
|
||||
VOLC_DISPLAY_TEXT, call_display_text(&call_display));
|
||||
}
|
||||
|
||||
static void observe_call_connection(unsigned generation)
|
||||
{
|
||||
if (atomic_load(&display_connected)) call_display_connected(&call_display, generation);
|
||||
publish_call_display(generation);
|
||||
}
|
||||
|
||||
static bool conversation_output_allowed(void)
|
||||
{
|
||||
return !stop_requested && platform_conversation_audio_authorized();
|
||||
}
|
||||
|
||||
volc_conv_service_t get_conv_ai_service()
|
||||
{
|
||||
@@ -98,11 +253,13 @@ volc_conv_service_t get_conv_ai_service()
|
||||
// volc_hal_file_t* audioFile = NULL;
|
||||
// int a_cnt = 0;
|
||||
static void __audio_capture_cb(volc_hal_capture_t capture, const void* data, int len, volc_hal_frame_info_t* frame_info){
|
||||
if (diagnostic_uplink_suppressed()) return;
|
||||
volc_audio_frame_info_t info = {0};
|
||||
info.commit = false;
|
||||
info.data_type = frame_info->data_type;
|
||||
if(!is_stop)
|
||||
if(!is_stop && !stop_requested && platform_conversation_audio_authorized())
|
||||
{
|
||||
if (atomic_load(&local_speech_active)) call_idle_activity();
|
||||
static uint64_t report_ms, magnitude;
|
||||
static unsigned samples, peak, frames, failures, max_send_ms;
|
||||
const int16_t *pcm = data;
|
||||
@@ -140,6 +297,9 @@ static void __on_volc_event(volc_engine_t handle, volc_event_t *event, void *use
|
||||
{
|
||||
case VOLC_EV_CONNECTED:
|
||||
LOGI("Volc Engine connected");
|
||||
call_idle_activity();
|
||||
if (!stop_requested) atomic_store(&display_connected, true);
|
||||
if(platform_runtime_rtc()){atomic_store(&rtc_connected,true);platform_conversation_observe("MEDIA_CONNECTED");}
|
||||
break;
|
||||
case VOLC_EV_DISCONNECTED:
|
||||
LOGI( "Volc Engine disconnected");
|
||||
@@ -154,33 +314,34 @@ static void __on_volc_event(volc_engine_t handle, volc_event_t *event, void *use
|
||||
|
||||
static void __on_volc_conversation_status(volc_engine_t handle, volc_conv_status_e status, void *user_data)
|
||||
{
|
||||
LOGI("AUDIO conversation status=%d",(int)status);
|
||||
// Turn events are not call lifecycle events. A delayed ANSWER_FINISH must
|
||||
// neither end the call nor overwrite its terminal status.
|
||||
if (stop_requested) return;
|
||||
if (status != VOLC_CONV_STATUS_LISTENING) call_idle_activity();
|
||||
int index = MMAP_EAF_HAPPY_EAF;
|
||||
switch (status)
|
||||
{
|
||||
case VOLC_CONV_STATUS_LISTENING:
|
||||
index = MMAP_EAF_EMOTION_BLINK_FAST_284_126_EAF;
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体聆听中");
|
||||
break;
|
||||
case VOLC_CONV_STATUS_THINKING:
|
||||
index = MMAP_EAF_EMOTION_DIZZY_284_126_EAF;
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体思考中");
|
||||
break;
|
||||
case VOLC_CONV_STATUS_ANSWERING:
|
||||
index = MMAP_EAF_HAPPY_EAF;
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体说话中");
|
||||
break;
|
||||
case VOLC_CONV_STATUS_INTERRUPTED:
|
||||
{
|
||||
volc_hal_context_t *hal = volc_get_global_hal_context();
|
||||
if (hal) volc_hal_player_discard_pending(hal->player_handle[VOLC_HAL_PLAYER_AUDIO]);
|
||||
}
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体被打断");
|
||||
break;
|
||||
case VOLC_CONV_STATUS_ANSWER_FINISH:
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体说话完成");
|
||||
atomic_fetch_add(&voice_finish_serial, 1);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
@@ -204,7 +365,13 @@ static void __on_volc_audio_data(volc_engine_t handle, const void *data_ptr, siz
|
||||
LOGE("player pipeline is NULL");
|
||||
return;
|
||||
}
|
||||
if(!is_stop){
|
||||
if(!is_stop && !stop_requested && platform_conversation_audio_authorized()){
|
||||
if (info_ptr && info_ptr->data_type == VOLC_AUDIO_DATA_TYPE_PCM &&
|
||||
call_idle_pcm_active(data_ptr, data_len)) {
|
||||
call_idle_activity();
|
||||
atomic_store(&last_voice_received_ms, (uint32_t)volc_osal_get_time_ms());
|
||||
}
|
||||
if(data_len)platform_conversation_observe("FIRST_AUDIO_RECEIVED");
|
||||
volc_hal_player_play_data(player_handle, data_ptr, data_len);
|
||||
}
|
||||
}
|
||||
@@ -241,13 +408,15 @@ static void __on_subtitle_message_received(const cJSON* root) {
|
||||
cJSON* text_obj = cJSON_GetObjectItem(obji, "text");
|
||||
cJSON* definite_obj = cJSON_GetObjectItem(obji, "definite");
|
||||
char* sub = cJSON_GetStringValue(text_obj);
|
||||
if(!sub)continue;
|
||||
if (sub[0] && !stop_requested) call_idle_activity();
|
||||
if(sub_offset<0||(size_t)sub_offset>strlen(sub))sub_offset=0;
|
||||
|
||||
// print_string(sub + sub_offset -1);
|
||||
if(cJSON_IsTrue(definite_obj)){
|
||||
sub_offset = sub_offset == 0 ? 0 : sub_offset -1;
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,sub + sub_offset);
|
||||
sub_offset = 0;
|
||||
aios_event_pub(VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE, sub, NULL);
|
||||
} else if(strlen(sub) - sub_offset > 18){
|
||||
sub_offset = sub_offset == 0 ? 0 : sub_offset -1;
|
||||
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,sub + sub_offset);
|
||||
@@ -259,6 +428,62 @@ static void __on_subtitle_message_received(const cJSON* root) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Called only by the conversation task, never from an SDK callback. */
|
||||
static struct {
|
||||
char text[512];
|
||||
double round, sequence;
|
||||
bool active, complete, overflow;
|
||||
} voice_user_subtitle;
|
||||
|
||||
static void process_voice_end_transcript(const char *speaker, const char *text, bool complete)
|
||||
{
|
||||
if (!complete || !text || !*text || is_stop || stop_requested) return;
|
||||
if (!strcmp(speaker, "subject")) {
|
||||
call_voice_end_user(&voice_end, text, (uint32_t)volc_osal_get_time_ms(),
|
||||
atomic_load(&voice_finish_serial));
|
||||
} else if (!strcmp(speaker, "agent") && voice_end.pending) {
|
||||
voice_end.reply_complete = true;
|
||||
}
|
||||
}
|
||||
|
||||
static void process_voice_end_rtc_user(const cJSON *line)
|
||||
{
|
||||
cJSON *round = cJSON_GetObjectItemCaseSensitive(line, "roundId");
|
||||
cJSON *sequence = cJSON_GetObjectItemCaseSensitive(line, "sequence");
|
||||
const char *text = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(line, "text"));
|
||||
if (!cJSON_IsNumber(round) || !cJSON_IsNumber(sequence) || !text ||
|
||||
!cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line, "definite")) || is_stop || stop_requested) return;
|
||||
if (voice_user_subtitle.active && round->valuedouble == voice_user_subtitle.round &&
|
||||
sequence->valuedouble <= voice_user_subtitle.sequence) return;
|
||||
if (!voice_user_subtitle.active || round->valuedouble != voice_user_subtitle.round || voice_user_subtitle.complete) {
|
||||
memset(&voice_user_subtitle, 0, sizeof(voice_user_subtitle));
|
||||
voice_user_subtitle.active = true;
|
||||
voice_user_subtitle.round = round->valuedouble;
|
||||
}
|
||||
voice_user_subtitle.sequence = sequence->valuedouble;
|
||||
size_t used = strlen(voice_user_subtitle.text), len = strlen(text);
|
||||
if (used + len + 1 >= sizeof(voice_user_subtitle.text)) voice_user_subtitle.overflow = true;
|
||||
if (!voice_user_subtitle.overflow) {
|
||||
memcpy(voice_user_subtitle.text + used, text, len + 1);
|
||||
}
|
||||
if (cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line, "paragraph"))) {
|
||||
voice_user_subtitle.complete = true;
|
||||
/* Overflow disables matching but still cancels a pending goodbye. */
|
||||
process_voice_end_transcript("subject", voice_user_subtitle.overflow ? "continue" : voice_user_subtitle.text, true);
|
||||
}
|
||||
}
|
||||
|
||||
static bool voice_end_ready(void)
|
||||
{
|
||||
if (!voice_end.pending) return false;
|
||||
volc_hal_context_t *hal = volc_get_global_hal_context();
|
||||
uint32_t now = (uint32_t)volc_osal_get_time_ms();
|
||||
bool quiet = now - atomic_load(&last_voice_received_ms) >= 1200u && hal &&
|
||||
volc_hal_player_voice_quiet(hal->player_handle[VOLC_HAL_PLAYER_AUDIO], 600u);
|
||||
return call_voice_end_due(&voice_end, now, atomic_load(&voice_finish_serial),
|
||||
quiet, atomic_load(&local_speech_active));
|
||||
}
|
||||
|
||||
/* HTTPS reporting must never run inside the WebSocket receive mutex. */
|
||||
static void process_platform_reports(void)
|
||||
{
|
||||
@@ -266,6 +491,24 @@ static void process_platform_reports(void)
|
||||
while (platform_report_queue && xQueueReceive(platform_report_queue, &root, 0) == pdTRUE) {
|
||||
cJSON *type_obj = cJSON_GetObjectItemCaseSensitive(root, "type");
|
||||
const char *type = cJSON_IsString(type_obj) ? type_obj->valuestring : "";
|
||||
if (!strcmp(type,"subtitle") && platform_runtime_rtc()) {
|
||||
cJSON *line=NULL;cJSON_ArrayForEach(line,cJSON_GetObjectItemCaseSensitive(root,"data")){
|
||||
const char *text=cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(line,"text"));
|
||||
const char *user=cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(line,"userId"));
|
||||
cJSON *seq=cJSON_GetObjectItemCaseSensitive(line,"sequence");
|
||||
if(text&&user&&cJSON_IsNumber(seq)&&cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line,"definite"))){
|
||||
char key[240];snprintf(key,sizeof(key),"%s:%.0f",user,seq->valuedouble);
|
||||
const char *local=volc_current_user_id(conv_service.engine,false),*remote=volc_current_user_id(conv_service.engine,true);
|
||||
const char *speaker=local&&!strcmp(user,local)?"subject":remote&&!strcmp(user,remote)?"agent":"unknown";
|
||||
/* definite is a final clause; paragraph confirms the whole
|
||||
* utterance, so "拜拜,不对,我还想聊" cannot end early. */
|
||||
if (!strcmp(speaker, "subject")) process_voice_end_rtc_user(line);
|
||||
else process_voice_end_transcript(speaker, text,
|
||||
cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line,"paragraph")));
|
||||
(void)platform_conversation_turn(speaker,key,text);
|
||||
}
|
||||
}
|
||||
} else
|
||||
if ((strcmp(type, "session.updated") == 0 || strcmp(type, "session.created") == 0) && !session_started_reported) {
|
||||
cJSON *session = cJSON_GetObjectItemCaseSensitive(root, "session");
|
||||
cJSON *id = cJSON_IsObject(session) ? cJSON_GetObjectItemCaseSensitive(session, "id") : NULL;
|
||||
@@ -280,12 +523,14 @@ static void process_platform_reports(void)
|
||||
cJSON *text = cJSON_GetObjectItemCaseSensitive(root, "transcript");
|
||||
cJSON *source = cJSON_GetObjectItemCaseSensitive(root, "event_id");
|
||||
if (cJSON_IsString(text) && text->valuestring[0]) {
|
||||
process_voice_end_transcript("subject", text->valuestring, true);
|
||||
(void)platform_conversation_turn("subject", cJSON_IsString(source) ? source->valuestring : "subject", text->valuestring);
|
||||
}
|
||||
} else if (strcmp(type, "response.audio_transcript.done") == 0) {
|
||||
cJSON *text = cJSON_GetObjectItemCaseSensitive(root, "transcript");
|
||||
cJSON *source = cJSON_GetObjectItemCaseSensitive(root, "response_id");
|
||||
if (cJSON_IsString(text) && text->valuestring[0]) {
|
||||
process_voice_end_transcript("agent", text->valuestring, true);
|
||||
(void)platform_conversation_turn("agent", cJSON_IsString(source) ? source->valuestring : "agent", text->valuestring);
|
||||
}
|
||||
}
|
||||
@@ -293,9 +538,19 @@ static void process_platform_reports(void)
|
||||
}
|
||||
}
|
||||
|
||||
static bool subtitle_has_final_clause(const cJSON *root)
|
||||
{
|
||||
cJSON *line = NULL;
|
||||
cJSON_ArrayForEach(line, cJSON_GetObjectItemCaseSensitive(root, "data")) {
|
||||
if (cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(line, "definite"))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void __on_volc_message_data(volc_engine_t handle, const void *message, size_t size, volc_message_info_t *info_ptr, void *user_data)
|
||||
{
|
||||
static char message_buffer[4096];
|
||||
if(size>=sizeof(message_buffer)-1){platform_conversation_observe("GAP_DECLARED");return;}
|
||||
if (size > 0 && size < sizeof(message_buffer) - 1) {
|
||||
memcpy(message_buffer, message, size);
|
||||
message_buffer[size] = 0;
|
||||
@@ -306,9 +561,10 @@ static void __on_volc_message_data(volc_engine_t handle, const void *message, si
|
||||
if (standard_protocol) {
|
||||
const char *type = cJSON_GetStringValue(cJSON_GetObjectItem(root, "type"));
|
||||
if (type && (!strcmp(type, "error") || !strcmp(type, "conversation.item.created"))) {
|
||||
LOGI("Provider control event: %s", message_buffer);
|
||||
LOGI("Provider control event type: %s", type);
|
||||
}
|
||||
if (type && !strcmp(type, "response.audio.done") && !is_stop) {
|
||||
atomic_fetch_add(&voice_finish_serial, 1);
|
||||
/* Flush the final sub-frame through the PCM decoder. */
|
||||
static const uint8_t silence[2048] = {0};
|
||||
volc_hal_context_t *hal = volc_get_global_hal_context();
|
||||
@@ -320,8 +576,10 @@ static void __on_volc_message_data(volc_engine_t handle, const void *message, si
|
||||
}
|
||||
if (type && (strstr(type, "speech_") || strstr(type, "response."))) {
|
||||
conv_service.wait_time = 0;
|
||||
if (!stop_requested) call_idle_activity();
|
||||
}
|
||||
bool report = type && (!strcmp(type, "session.created") || !strcmp(type, "session.updated") ||
|
||||
(!strcmp(type, "subtitle") && platform_runtime_rtc() && subtitle_has_final_clause(root)) ||
|
||||
!strcmp(type, "conversation.item.input_audio_transcription.completed") ||
|
||||
!strcmp(type, "response.audio_transcript.done"));
|
||||
if (report && platform_report_queue && xQueueSend(platform_report_queue, &root, 0) == pdTRUE) {
|
||||
@@ -333,6 +591,13 @@ static void __on_volc_message_data(volc_engine_t handle, const void *message, si
|
||||
} else if (message_buffer[0] == 's' && message_buffer[1] == 'u' && message_buffer[2] == 'b' && message_buffer[3] == 'v') {
|
||||
__on_subtitle_message_received(root);
|
||||
conv_service.wait_time = 0;
|
||||
/* Interim ASR updates must not fill the bounded control queue
|
||||
* and evict the actual completed goodbye. Display still gets all. */
|
||||
if(platform_runtime_rtc() && subtitle_has_final_clause(root)){
|
||||
if(platform_report_queue&&xQueueSend(platform_report_queue,&root,0)==pdTRUE)root=NULL;
|
||||
else platform_conversation_observe("GAP_DECLARED");
|
||||
}
|
||||
cJSON_Delete(root);
|
||||
} else if (message_buffer[0] == 't' && message_buffer[1] == 'o' && message_buffer[2] == 'o' && message_buffer[3] == 'l') {
|
||||
// function calling 消息
|
||||
aios_event_pub(VOLC_FUNCTION_CALL_EXEC, root, NULL);
|
||||
@@ -360,7 +625,16 @@ void conv_ai_service_init()
|
||||
.on_volc_message_data = __on_volc_message_data,
|
||||
};
|
||||
|
||||
int ret = volc_create(&conv_service.engine, config_buf, &conv_service.volc_event_handler, &conv_service);
|
||||
cJSON *runtime=cJSON_Parse(config_buf);
|
||||
if(platform_runtime_rtc()){
|
||||
cJSON_AddStringToObject(cJSON_GetObjectItem(runtime,"iot"),"rtc_app_id",platform_runtime_rtc_app_id());
|
||||
cJSON_SetNumberValue(cJSON_GetObjectItem(cJSON_GetObjectItem(cJSON_GetObjectItem(runtime,"rtc"),"audio"),"codec"),1);
|
||||
}
|
||||
else cJSON_DeleteItemFromObjectCaseSensitive(runtime,"rtc");
|
||||
char *runtime_config=cJSON_PrintUnformatted(runtime);cJSON_Delete(runtime);
|
||||
int ret = volc_create(&conv_service.engine, runtime_config, &conv_service.volc_event_handler, &conv_service);
|
||||
if(runtime_config){memset(runtime_config,0,strlen(runtime_config));free(runtime_config);}
|
||||
platform_runtime_engine_ready(ret==0);
|
||||
if (ret != 0) {
|
||||
LOGE("volc_create failed, ret: %d", ret);
|
||||
return;
|
||||
@@ -369,14 +643,42 @@ void conv_ai_service_init()
|
||||
}
|
||||
}
|
||||
|
||||
void conv_ai_service_prepare_task(void)
|
||||
{
|
||||
// Called synchronously by the serialized manager before creating the task.
|
||||
diagnostic_prepare();
|
||||
stop_requested = false;
|
||||
is_stop = true;
|
||||
atomic_store(&user_stop_requested, false);
|
||||
atomic_store(&display_connected, false);
|
||||
atomic_store(&local_speech_active, false);
|
||||
call_voice_end_reset(&voice_end);
|
||||
memset(&voice_user_subtitle, 0, sizeof(voice_user_subtitle));
|
||||
atomic_store(&voice_finish_serial, 0);
|
||||
atomic_store(&last_voice_received_ms, (uint32_t)volc_osal_get_time_ms());
|
||||
/* The previous SDK has stopped before the manager admits a new call. */
|
||||
cJSON *stale;
|
||||
while (platform_report_queue && xQueueReceive(platform_report_queue, &stale, 0) == pdTRUE) cJSON_Delete(stale);
|
||||
call_idle_activity();
|
||||
call_display_begin(&call_display);
|
||||
}
|
||||
|
||||
void conv_ai_service_task(void *pvParameters)
|
||||
{
|
||||
LOGI("TASK_DIAGNOSTIC name=conv_ai present=1 priority=%u core=%d",
|
||||
(unsigned)uxTaskPriorityGet(NULL), (int)xTaskGetCoreID(NULL));
|
||||
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
|
||||
char* status_str = NULL;
|
||||
bool provider_started = false;
|
||||
bool audio_started = false;
|
||||
bool local_audio_stopped = true;
|
||||
call_display_stop_t display_stop = CALL_DISPLAY_STOP_FAILED;
|
||||
unsigned display_generation = call_display.generation;
|
||||
char *params = NULL;
|
||||
is_stop = true;
|
||||
session_started_reported = false;
|
||||
provider_disconnected = false;
|
||||
atomic_store(&rtc_connected,false);
|
||||
if (g_hal_context == NULL) {
|
||||
LOGE("volc_get_global_hal_context failed");
|
||||
status_str = "系统未初始化,ai对话连接失败";
|
||||
@@ -384,8 +686,9 @@ void conv_ai_service_task(void *pvParameters)
|
||||
}
|
||||
global_display = g_hal_context->display_handle;
|
||||
int ret = 0;
|
||||
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT, "ai对话创建中");
|
||||
publish_call_display(display_generation);
|
||||
|
||||
if (stop_requested) goto CONV_AI_QUIT;
|
||||
ret = platform_conversation_begin();
|
||||
if (ret != 0) {
|
||||
LOGE("platform_conversation_begin failed, ret: %d", ret);
|
||||
@@ -393,8 +696,14 @@ void conv_ai_service_task(void *pvParameters)
|
||||
if (strcmp(platform_last_error_code(), "FIRMWARE_UPDATE_REQUIRED") == 0) {
|
||||
status_str = "平台要求更新固件";
|
||||
}
|
||||
if (strcmp(call_client_start_error(), "DEVICE_UNBOUND") == 0) {
|
||||
status_str = "设备没有绑定人员,请先绑定";
|
||||
} else if (strcmp(call_client_start_error(), "VOICE_AGENT_NOT_READY") == 0) {
|
||||
status_str = "老人的语音助手尚未就绪";
|
||||
}
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
if (stop_requested || !platform_conversation_audio_authorized()) goto CONV_AI_QUIT;
|
||||
|
||||
/* Capture may run while the WebSocket is still handshaking. Hold frames
|
||||
* until volc_start has moved the engine into STARTED state. */
|
||||
@@ -404,7 +713,7 @@ void conv_ai_service_task(void *pvParameters)
|
||||
volc_hal_capture_config_t capture_audio_config = {0};
|
||||
capture_audio_config.media_type = VOLC_MEDIA_TYPE_AUDIO;
|
||||
capture_audio_config.data_cb = __audio_capture_cb;
|
||||
// capture_audio_config.audio_wakeup_cb = (volc_hal_audio_wakeup_cb_t)rec_engine_cb;
|
||||
capture_audio_config.audio_wakeup_cb = call_idle_capture_event;
|
||||
|
||||
capture_audio_config.user_data = g_hal_context;
|
||||
volc_hal_capture_t capture_handle = g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO];
|
||||
@@ -413,9 +722,13 @@ void conv_ai_service_task(void *pvParameters)
|
||||
capture_handle = g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO];
|
||||
}
|
||||
audio_started = true;
|
||||
volc_hal_capture_stop(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO]);
|
||||
volc_hal_capture_update_config(capture_handle, &capture_audio_config);
|
||||
volc_hal_capture_start(g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO], VOLC_AUDIO_MODE_CAPTURE);
|
||||
if (!capture_handle ||
|
||||
volc_hal_capture_stop(capture_handle) != 0 ||
|
||||
volc_hal_capture_update_config(capture_handle, &capture_audio_config) != 0 ||
|
||||
volc_hal_capture_start(capture_handle, VOLC_AUDIO_MODE_CAPTURE) != 0) {
|
||||
status_str = "收音启动失败";
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
|
||||
volc_hal_player_config_t player_audio_config = {0};
|
||||
player_audio_config.media_type = VOLC_MEDIA_TYPE_AUDIO;
|
||||
@@ -423,20 +736,22 @@ void conv_ai_service_task(void *pvParameters)
|
||||
g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO] = volc_hal_player_create(&player_audio_config);
|
||||
}
|
||||
|
||||
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, "ai对话连接中");
|
||||
publish_call_display(display_generation);
|
||||
volc_hal_player_set_authorization_gate(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO], conversation_output_allowed);
|
||||
volc_hal_player_start(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
|
||||
|
||||
// step 2: start ai conversation
|
||||
const char *session_update = platform_conversation_update_json();
|
||||
size_t params_size = strlen(session_update) + 64;
|
||||
char *params = volc_osal_malloc(params_size);
|
||||
params = volc_osal_malloc(params_size);
|
||||
if (params == NULL) {
|
||||
status_str = "会话配置内存不足";
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
snprintf(params, params_size, "{\"audio\":{\"codec\":0},\"session_update\":%s}", session_update);
|
||||
if(platform_runtime_rtc())snprintf(params,params_size,"%s",session_update);
|
||||
else snprintf(params, params_size, "{\"audio\":{\"codec\":0},\"session_update\":%s}", session_update);
|
||||
volc_opt_t opt = {
|
||||
.mode = VOLC_MODE_WS,
|
||||
.mode = platform_runtime_rtc()?VOLC_MODE_RTC:VOLC_MODE_WS,
|
||||
.bot_id = (char *)platform_conversation_bot_id(),
|
||||
.params = params,
|
||||
};
|
||||
@@ -444,46 +759,91 @@ void conv_ai_service_task(void *pvParameters)
|
||||
aios_event_pub(VOLC_LOCAL_LOGIC_PLAY_WELCOME, NULL, NULL);
|
||||
#endif
|
||||
ret = volc_start(conv_service.engine, &opt);
|
||||
volc_osal_free(params);
|
||||
if (ret != 0) {
|
||||
LOGE("volc_start failed, ret: %d", ret);
|
||||
status_str = "ai对话连接失败,准备退出";
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
provider_started = true;
|
||||
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, "ai对话已连接");
|
||||
call_idle_activity();
|
||||
if (stop_requested || !platform_conversation_audio_authorized()) goto CONV_AI_QUIT;
|
||||
// A successful start request does not prove that the media transport joined.
|
||||
observe_call_connection(display_generation);
|
||||
is_stop = false;
|
||||
|
||||
while (!is_stop && !provider_disconnected) {
|
||||
while (!is_stop && !stop_requested && !provider_disconnected && platform_conversation_audio_authorized()) {
|
||||
observe_call_connection(display_generation);
|
||||
if(platform_runtime_rtc()&&atomic_load(&rtc_connected))session_started_reported=true;
|
||||
process_platform_reports();
|
||||
if (session_started_reported && platform_conversation_poll()) {
|
||||
status_str = "平台已结束本次对话";
|
||||
if (voice_end_ready()) {
|
||||
LOGI("VOICE_END user request completed");
|
||||
status_str = "本次对话已结束";
|
||||
atomic_store(&user_stop_requested, true);
|
||||
display_stop = CALL_DISPLAY_STOP_NORMAL;
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
if (diagnostic_say_pending && session_started_reported) {
|
||||
diagnostic_say_pending = false;
|
||||
if ((platform_runtime_rtc()||session_started_reported) && platform_conversation_poll()) {
|
||||
status_str = "平台已结束本次对话";
|
||||
display_stop = atomic_load(&display_connected) ? CALL_DISPLAY_STOP_NORMAL : CALL_DISPLAY_STOP_FAILED;
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
if ((atomic_load(&diagnostic_state) & DIAGNOSTIC_PENDING_MASK) != DIAGNOSTIC_NONE && session_started_reported && (!platform_runtime_rtc() || volc_current_user_id(conv_service.engine,true))) {
|
||||
unsigned diagnostic = diagnostic_take_pending();
|
||||
if (diagnostic == DIAGNOSTIC_TTS) {
|
||||
diagnostic_send_tts();
|
||||
} else if (diagnostic == DIAGNOSTIC_SAY) {
|
||||
char request[640];
|
||||
snprintf(request, sizeof(request), "{\"event_id\":\"diag-%llu\",\"type\":\"conversation.item.create\",\"item\":{\"type\":\"message\",\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"请只说这句话:你好,这是语音播放测试。现在使用自然的语速,连续清楚地说完这句话。\"}],\"interrupt_mode\":1}}", (unsigned long long)volc_osal_get_time_ms());
|
||||
volc_message_info_t info = {.is_binary = false};
|
||||
int sent = volc_send_message(conv_service.engine, request, strlen(request), &info);
|
||||
int sent = platform_runtime_rtc()
|
||||
? volc_send_text_to_agent(conv_service.engine,"请只说这句话:你好,这是语音播放测试。",VOLC_AGENT_TYPE_LLM,1)
|
||||
: volc_send_message(conv_service.engine, request, strlen(request), &info);
|
||||
LOGI("PLAYBACK diagnostic sentence requested ret=%d", sent);
|
||||
const char *respond = "{\"type\":\"response.create\",\"response\":{\"modalities\":[\"text\",\"audio\"]}}";
|
||||
volc_send_message(conv_service.engine, respond, strlen(respond), &info);
|
||||
if (!platform_runtime_rtc()) volc_send_message(conv_service.engine, respond, strlen(respond), &info);
|
||||
conv_service.wait_time = 0;
|
||||
call_idle_activity();
|
||||
}
|
||||
}
|
||||
sleep(1);
|
||||
conv_service.wait_time++;
|
||||
if(conv_service.wait_time >= CONV_SERVICE_TIMEOUT){
|
||||
if (call_idle_expired()) {
|
||||
LOGI("CALL_IDLE timeout seconds=%d", CONV_SERVICE_TIMEOUT);
|
||||
status_str = "ai对话已超时,准备退出";
|
||||
display_stop = atomic_load(&display_connected) ? CALL_DISPLAY_STOP_NORMAL : CALL_DISPLAY_STOP_FAILED;
|
||||
goto CONV_AI_QUIT;
|
||||
}
|
||||
}
|
||||
status_str = "ai对话已断开";
|
||||
display_stop = atomic_load(&provider_disconnected) ? CALL_DISPLAY_STOP_INTERRUPTED :
|
||||
atomic_load(&display_connected) ? CALL_DISPLAY_STOP_NORMAL : CALL_DISPLAY_STOP_FAILED;
|
||||
|
||||
CONV_AI_QUIT:
|
||||
// Latch a connection that arrived immediately before hangup, without
|
||||
// publishing a misleading new TALKING frame while teardown has started.
|
||||
if (atomic_load(&display_connected)) call_display_connected(&call_display, display_generation);
|
||||
is_stop = true;
|
||||
// volc_hal_file_close(audioFile);
|
||||
volc_stop(conv_service.engine);
|
||||
stop_requested = true;
|
||||
diagnostic_close();
|
||||
call_display_stopping(&call_display, display_generation);
|
||||
publish_call_display(display_generation);
|
||||
// Local privacy boundary precedes SDK shutdown and all HTTP requests.
|
||||
if (g_hal_context && audio_started) {
|
||||
local_audio_stopped = volc_hal_capture_stop(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO]) == 0;
|
||||
if (g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]) {
|
||||
volc_hal_player_discard_pending(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
|
||||
if (volc_hal_player_stop(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]) != 0) local_audio_stopped = false;
|
||||
}
|
||||
}
|
||||
int stop_result=volc_stop(conv_service.engine);
|
||||
platform_conversation_stop_result(provider_started,stop_result);
|
||||
if (atomic_load(&user_stop_requested)) display_stop = CALL_DISPLAY_STOP_NORMAL;
|
||||
if (provider_started && stop_result != 0) display_stop = CALL_DISPLAY_STOP_INTERRUPTED;
|
||||
// Show the local terminal state before an end-report HTTP request can block.
|
||||
call_display_finished(&call_display, display_generation, display_stop,
|
||||
local_audio_stopped, volc_osal_get_time_ms());
|
||||
publish_call_display(display_generation);
|
||||
// The SDK may retain opt.params until stop has released the session.
|
||||
if (params) { volc_osal_free(params); params = NULL; }
|
||||
process_platform_reports();
|
||||
platform_conversation_finish(!provider_started || !session_started_reported,
|
||||
!provider_started ? "VOLC_START_FAILED" : (!session_started_reported ? "SESSION_UPDATE_NOT_CONFIRMED" : NULL));
|
||||
@@ -497,25 +857,40 @@ CONV_AI_QUIT:
|
||||
}
|
||||
}
|
||||
|
||||
if(status_str != NULL){
|
||||
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, status_str);
|
||||
}
|
||||
if (status_str != NULL) LOGI("Call finished: %s", status_str);
|
||||
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE, VOLC_DISPLAY_TEXT, "");
|
||||
conv_service.wait_time = 0;
|
||||
// just show the status: Ai对话已断开
|
||||
sleep(1);
|
||||
// No detached timer: the old task owns this brief terminal display. Restore
|
||||
// idle before publishing OVER, so a new call can never be overwritten by an
|
||||
// old timeout. Slow end-report requests already count toward the hold time.
|
||||
if (local_audio_stopped) {
|
||||
while (!call_display_idle_if_due(&call_display, display_generation, volc_osal_get_time_ms())) {
|
||||
vTaskDelay(pdMS_TO_TICKS(50));
|
||||
}
|
||||
publish_call_display(display_generation);
|
||||
}
|
||||
if (g_hal_context && audio_started) {
|
||||
volc_hal_capture_start(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO],VOLC_AUDIO_MODE_WAKEUP);
|
||||
}
|
||||
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_OVER, NULL, NULL);
|
||||
#ifdef ESP_PLATFORM
|
||||
// The manager releases this WithCaps task from another task's context.
|
||||
// Plain vTaskDelete(NULL) does not free WithCaps stacks/TCBs.
|
||||
for (;;) vTaskSuspend(NULL);
|
||||
#else
|
||||
volc_osal_thread_exit(NULL);
|
||||
#endif
|
||||
}
|
||||
|
||||
void conv_ai_service_task_stop(void)
|
||||
{
|
||||
// sleep for AI Agent say goodbye
|
||||
sleep(1);
|
||||
atomic_store(&user_stop_requested, true);
|
||||
stop_requested = true;
|
||||
is_stop = true;
|
||||
diagnostic_close();
|
||||
// Capture callbacks drop frames immediately. The independent audio worker
|
||||
// closes its feeder through the authorization gate, including blocked HTTP.
|
||||
// Only the conversation task destroys handles; hook callbacks never race it.
|
||||
}
|
||||
|
||||
#if __cplusplus
|
||||
|
||||
+27
-2
@@ -11,6 +11,11 @@
|
||||
#include "volc_service_common.h"
|
||||
#include "volc_conv_service.h"
|
||||
#include "volc_conv.h"
|
||||
#ifdef ESP_PLATFORM
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#endif
|
||||
extern void conv_ai_service_diagnostic_reset(void);
|
||||
|
||||
#if __cplusplus
|
||||
extern "C" {
|
||||
@@ -31,6 +36,14 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
|
||||
}
|
||||
volc_hal_display_t global_display = g_hal_context->display_handle;
|
||||
switch (e->id) {
|
||||
case VOLC_SERVICE_AI_CONVERSATION_TOGGLE:
|
||||
// This manager owns admission. Connecting and stopping tasks remain
|
||||
// active until OVER, so repeated taps cannot create another call.
|
||||
if (me->conv_thread_id != NULL) {
|
||||
conv_ai_service_task_stop();
|
||||
return AIOS_Ret_Handled;
|
||||
}
|
||||
/* fall through */
|
||||
case VOLC_SERVICE_AI_CONVERSATION_START:
|
||||
if (me->conv_thread_id == NULL) {
|
||||
volc_osal_thread_param_t param = {0};
|
||||
@@ -39,7 +52,10 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
|
||||
param.stack_size = 8*1024;
|
||||
param.priority = 5;
|
||||
param.stack_in_ext = 1;
|
||||
volc_osal_thread_create(&me->conv_thread_id, ¶m, conv_ai_service_task,NULL);
|
||||
conv_ai_service_prepare_task();
|
||||
if (volc_osal_thread_create(&me->conv_thread_id, ¶m, conv_ai_service_task,NULL) != 0) {
|
||||
conv_ai_service_diagnostic_reset();
|
||||
}
|
||||
}
|
||||
return AIOS_Ret_Handled;
|
||||
case VOLC_SERVICE_AI_CONVERSATION_INTERRUPT:
|
||||
@@ -48,12 +64,20 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
|
||||
case VOLC_SERVICE_AI_CONVERSATION_QUIT:
|
||||
if (me->conv_thread_id != NULL) {
|
||||
conv_ai_service_task_stop();
|
||||
me->conv_thread_id = NULL;
|
||||
// The old task owns SDK teardown and reports OVER after cleanup.
|
||||
// Keep admission closed until then, including rapid hook toggles.
|
||||
}
|
||||
return AIOS_Ret_Handled;
|
||||
// notice the Ai_Conversation over by task
|
||||
case VOLC_SERVICE_AI_CONVERSATION_OVER:
|
||||
if (me->conv_thread_id != NULL) {
|
||||
#ifdef ESP_PLATFORM
|
||||
// OSAL stores the native handle in an allocated handle slot.
|
||||
// OVER is the worker's final action; reclaim its WithCaps
|
||||
// stack and TCB before admitting another conversation.
|
||||
vTaskDeleteWithCaps(*(TaskHandle_t *)me->conv_thread_id);
|
||||
#endif
|
||||
volc_osal_thread_destroy(me->conv_thread_id);
|
||||
me->conv_thread_id = NULL;
|
||||
}
|
||||
return AIOS_Ret_Handled;
|
||||
@@ -65,6 +89,7 @@ static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, a
|
||||
static aios_ret_t __state_init(volc_conv_service_manager_t * const me, aios_event_t const * const e)
|
||||
{
|
||||
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_START); // 订阅事件
|
||||
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_TOGGLE);
|
||||
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_INTERRUPT); // 订阅事件
|
||||
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_QUIT); // 订阅事件
|
||||
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_OVER); // 订阅事件
|
||||
|
||||
+1
@@ -14,6 +14,7 @@ enum {
|
||||
VOLC_SERVICE_AI_CONVERSATION_INTERRUPT = 101,
|
||||
VOLC_SERVICE_AI_CONVERSATION_QUIT = 102, // control the Ai_Conversation task quit
|
||||
VOLC_SERVICE_AI_CONVERSATION_OVER = 103, // notice the Ai_Conversation over by task
|
||||
VOLC_SERVICE_AI_CONVERSATION_TOGGLE = 104, // screen tap: start idle or stop active
|
||||
|
||||
// function_call event:
|
||||
VOLC_FUNCTION_CALL_EXEC = 200,
|
||||
|
||||
+2
-33
@@ -21,7 +21,6 @@ typedef struct volc_local_logic_service {
|
||||
} volc_local_logic_service_t;
|
||||
|
||||
volc_local_logic_service_t g_volc_local_logic_service;
|
||||
static int has_quit = 0;
|
||||
static void __play_welcome(){
|
||||
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
|
||||
if(g_hal_context == NULL) return;
|
||||
@@ -42,50 +41,20 @@ static void __play_welcome(){
|
||||
return;
|
||||
}
|
||||
|
||||
static int __has_substring(const char *haystack, const char *needle) {
|
||||
// 处理边界:子串为空串,按惯例返回包含(可根据需求调整)
|
||||
if (needle[0] == '\0') {
|
||||
return 1;
|
||||
}
|
||||
// strstr 返回非 NULL 表示找到子串
|
||||
return strstr(haystack, needle) != NULL ? 1 : 0;
|
||||
}
|
||||
|
||||
static void __parser_and_process_subtitle(char *subtitle)
|
||||
{
|
||||
volc_conv_service_t conv_service = get_conv_ai_service();
|
||||
|
||||
if (__has_substring(subtitle, "退下吧") || __has_substring(subtitle, "拜拜"))
|
||||
{
|
||||
if (!has_quit)
|
||||
{
|
||||
volc_send_text_to_agent(conv_service.engine, "好的拜拜", VOLC_AGENT_TYPE_TTS, 1);
|
||||
has_quit = 1;
|
||||
}
|
||||
}
|
||||
if (__has_substring(subtitle, "好的拜拜"))
|
||||
{
|
||||
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_QUIT, NULL, NULL);
|
||||
has_quit = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static aios_ret_t __state_conversation(volc_local_logic_service_t * const me, aios_event_t const * const e)
|
||||
{
|
||||
switch (e->id) {
|
||||
case VOLC_LOCAL_LOGIC_PLAY_WELCOME:
|
||||
__play_welcome();
|
||||
return AIOS_Ret_Handled;
|
||||
case VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE:
|
||||
__parser_and_process_subtitle(e->data);
|
||||
return AIOS_Ret_Handled;
|
||||
}
|
||||
}
|
||||
|
||||
static aios_ret_t __state_init(volc_local_logic_service_t * const me, aios_event_t const * const e)
|
||||
{
|
||||
AIOS_EVENT_SUB(VOLC_LOCAL_LOGIC_PLAY_WELCOME); // 订阅事件
|
||||
AIOS_EVENT_SUB(VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE); // 订阅事件
|
||||
// Voice ending is handled by the conversation task with speaker identity,
|
||||
// complete user transcripts and playback drain; never by substring events.
|
||||
return AIOS_TRAN(__state_conversation);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user