chore: checkpoint 第一版
ci / verify (push) Canceled after 0s

This commit is contained in:
Codex Local
2026-09-06 17:04:11 +08:00
parent a8cdec98d1
commit 0c2151ace9
659 changed files with 185855 additions and 554 deletions
@@ -0,0 +1,12 @@
# Copyright (2025) Beijing Volcano Engine Technology Ltd.
# SPDX-License-Identifier: Apache License 2.0
file(GLOB SERVICE_SRCS_COMMON "${CMAKE_CURRENT_LIST_DIR}/src/*.c")
file(GLOB SERVICE_SRCS_CONV "${CMAKE_CURRENT_LIST_DIR}/conv_service/src/*.c")
file(GLOB SERVICE_SRCS_FUNCTION_CALL "${CMAKE_CURRENT_LIST_DIR}/function_call_service/src/*.c")
set(SERVICE_SRCS ${SERVICE_SRCS_COMMON} ${SERVICE_SRCS_CONV} ${SERVICE_SRCS_FUNCTION_CALL})
set(SERVICE_INC "${CMAKE_CURRENT_LIST_DIR}/inc"
"${CMAKE_CURRENT_LIST_DIR}/conv_service/inc"
"${CMAKE_CURRENT_LIST_DIR}/function_call_service/inc")
@@ -0,0 +1,10 @@
# Service
## brief introduction
This directory presents the business module division of the Hardware Conversational Agent Development Kit for AI conversation services. All business modules are named "service", and different services communicate through event subscription (sub) and publication (pub) at the framework layer. The main modules are as follows:
- conv_service:The core module of the AI Conversational Agent, responsible for the initiation, operation, and termination of AI conversation services.
- function_call_service:Interacts with the conv\_service module, responsible for executing function calls and returning the results to the agent.
- local_logic_service:A local module based on predefined business logic (not the agent's business logic). It performs tasks such as parsing subtitles and taking certain actions based on quick responses to the subtitles (different from function calls).
- src: An externally exposed module responsible for initiating the AI conversation.
@@ -0,0 +1,11 @@
# Service
## 目录简介
本目录是硬件对话智能体开发套件对于ai对话业务进行的业务模块划分,不同的业务模块我们以service命名;不同的service通过framework层event的sub和pub进行通信。主要分为一下模块
- conv_service: 智能体ai对话的主要模块,负责ai对话业务的开始,运行,与结束。
- function_call_service: 与conv_service模块交互,负责function call的执行并将结果返回给智能体
- local_logic_service: 本地基于既定业务逻辑(非智能体的业务逻辑)的模块,例如解析字幕并根据字幕快速响应做出某些action等(区别于function call)
- src: 对外暴露的模块,负责ai对话的拉起
@@ -0,0 +1,19 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_CONV_SERVICE_H__
#define VOLC_CONV_SERVICE_H__
#include "volc_conv_ai.h"
typedef struct {
volc_event_handler_t volc_event_handler;
volc_engine_t engine;
int wait_time;
} volc_conv_service_t;
void conv_ai_service_task(void *pvParameters);
void conv_ai_service_task_stop();
void conv_ai_service_init();
volc_conv_service_t get_conv_ai_service();
#endif /* VOLC_CONV_SERVICE_H__ */
@@ -0,0 +1,12 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_CONV_SERVICE_MANAGER_H__
#define VOLC_CONV_SERVICE_MANAGER_H__
#include <stdio.h>
#include "aios.h"
void volc_conv_service_manager_init(void);
#endif // VOLC_CONV_SERVICE_MANAGER_H__
@@ -0,0 +1,523 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#include <stdio.h>
#include <string.h>
#include "aios.h"
#include "cJSON.h"
#include "volc_hal.h"
#include "volc_osal.h"
#include "util/volc_log.h"
#include "volc_service_common.h"
#include "volc_conv.h"
#include "volc_hal.h"
#include "volc_hal_capture.h"
#include "volc_hal_display.h"
#include "volc_hal_player.h"
// #include "volc_hal_file.h"
#include "volc_local_function_list.h"
#include "mmap_generate_eaf.h"
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
extern int8_t get_rssi(void);
#if __cplusplus
extern "C" {
#endif
#define CONV_AI_CONFIG_FORMAT "{\
\"ver\": 1,\
\"iot\": {\
\"instance_id\": \"%s\",\
\"product_key\": \"%s\",\
\"device_secret\": \"%s\",\
\"device_name\": \"%s\"\
},\
\"rtc\": {\
\"log_level\": 3,\
\"audio\": {\
\"publish\": true,\
\"subscribe\": true,\
\"codec\": 3\
},\
\"video\": {\
\"publish\": false,\
\"subscribe\": false,\
\"codec\": 1\
},\
\"params\":[\
\"{\\\"debug\\\":{\\\"log_to_console\\\":1}}\",\
\"{\\\"rtc\\\":{\\\"access\\\":{\\\"concurrent_requests\\\":1}}}\",\
\"{\\\"rtc\\\":{\\\"ice\\\":{\\\"concurrent_agents\\\":1}}}\",\
\"{\\\"rtc\\\":{\\\"thread\\\":{\\\"stack_in_ext\\\":1}}}\",\
\"{\\\"rtc\\\":{\\\"network\\\":{\\\"enable_audio_jitter2\\\":0}}}\",\
\"{\\\"rtc\\\":{\\\"report\\\":{\\\"enable\\\":0}}}\",\
\"{\\\"audio\\\":{\\\"codec\\\":{\\\"pcma\\\":{\\\"s_samples_per_frame\\\":480}}}}\"\
]\
}\
}"
#define CONV_SERVICE_TIMEOUT 60
static char config_buf[1024] = {0};
static volc_conv_service_t conv_service = {0};
static volatile bool is_stop = false;
static volatile bool provider_disconnected = false;
static volc_hal_display_t global_display = NULL;
static bool session_started_reported = false;
static QueueHandle_t platform_report_queue;
static volatile bool diagnostic_say_pending;
void conv_ai_service_diagnostic_say(void)
{
diagnostic_say_pending = true;
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION, NULL, NULL);
}
/* Implemented by the platform client in the top-level firmware component. */
extern const char *platform_runtime_instance_id(void);
extern const char *platform_runtime_product_key(void);
extern const char *platform_runtime_device_name(void);
extern const char *platform_runtime_device_secret(void);
extern int platform_conversation_begin(void);
extern const char *platform_last_error_code(void);
extern const char *platform_conversation_bot_id(void);
extern const char *platform_conversation_update_json(void);
extern int platform_conversation_started(const char *provider_connection_ref);
extern int platform_conversation_poll(void);
extern int platform_conversation_turn(const char *speaker, const char *source_id, const char *text);
extern void platform_conversation_finish(bool failed, const char *error_code);
volc_conv_service_t get_conv_ai_service()
{
return conv_service;
}
// use dump audio
// volc_hal_file_t* audioFile = NULL;
// int a_cnt = 0;
static void __audio_capture_cb(volc_hal_capture_t capture, const void* data, int len, volc_hal_frame_info_t* frame_info){
volc_audio_frame_info_t info = {0};
info.commit = false;
info.data_type = frame_info->data_type;
if(!is_stop)
{
static uint64_t report_ms, magnitude;
static unsigned samples, peak, frames, failures, max_send_ms;
const int16_t *pcm = data;
if (frame_info->data_type == VOLC_AUDIO_DATA_TYPE_PCM) {
for (int i = 0; i < len / sizeof(int16_t); ++i) {
unsigned value = pcm[i] < 0 ? -(int)pcm[i] : pcm[i];
magnitude += value;
if (value > peak) peak = value;
samples++;
}
}
uint64_t before = volc_osal_get_time_ms();
int sent = volc_send_audio_data(conv_service.engine, data, len, &info);
unsigned elapsed = volc_osal_get_time_ms() - before;
if (elapsed > max_send_ms) max_send_ms = elapsed;
frames++;
if (sent < 0) failures++;
if (before - report_ms >= 2000) {
int rssi = get_rssi();
LOGI("UPLINK mean=%u peak=%u frames=%u failed=%u max_send_ms=%u rssi=%d",
samples ? (unsigned)(magnitude / samples) : 0, peak, frames, failures, max_send_ms, rssi);
report_ms = before;
magnitude = samples = peak = frames = failures = max_send_ms = 0;
}
// if(audioFile == NULL){
// audioFile = volc_hal_file_open("/sdcard/cap.pcm","wb");
// volc_hal_file_write(audioFile,data,len);
// }
}
}
static void __on_volc_event(volc_engine_t handle, volc_event_t *event, void *user_data)
{
switch (event->code)
{
case VOLC_EV_CONNECTED:
LOGI("Volc Engine connected");
break;
case VOLC_EV_DISCONNECTED:
LOGI( "Volc Engine disconnected");
provider_disconnected = true;
is_stop = true;
break;
default:
LOGI("Volc Engine event: %d", event->code);
break;
}
}
static void __on_volc_conversation_status(volc_engine_t handle, volc_conv_status_e status, void *user_data)
{
int index = MMAP_EAF_HAPPY_EAF;
switch (status)
{
case VOLC_CONV_STATUS_LISTENING:
index = MMAP_EAF_EMOTION_BLINK_FAST_284_126_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体聆听中");
break;
case VOLC_CONV_STATUS_THINKING:
index = MMAP_EAF_EMOTION_DIZZY_284_126_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体思考中");
break;
case VOLC_CONV_STATUS_ANSWERING:
index = MMAP_EAF_HAPPY_EAF;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_MAIN,VOLC_DISPLAY_IMAGE,&index);
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体说话中");
break;
case VOLC_CONV_STATUS_INTERRUPTED:
{
volc_hal_context_t *hal = volc_get_global_hal_context();
if (hal) volc_hal_player_discard_pending(hal->player_handle[VOLC_HAL_PLAYER_AUDIO]);
}
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体被打断");
break;
case VOLC_CONV_STATUS_ANSWER_FINISH:
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"智能体说话完成");
break;
default:
break;
}
return;
}
static void __on_volc_audio_data(volc_engine_t handle, const void *data_ptr, size_t data_len, volc_audio_frame_info_t *info_ptr, void *user_data)
{
int error = 0;
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
if (g_hal_context == NULL) {
LOGE("volc_get_global_hal_context failed");
return;
}
volc_hal_player_t player_handle = g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO];
if (player_handle == NULL)
{
LOGE("player pipeline is NULL");
return;
}
if(!is_stop){
volc_hal_player_play_data(player_handle, data_ptr, data_len);
}
}
static void __on_volc_video_data(volc_engine_t handle, const void *data_ptr, size_t data_len, volc_video_frame_info_t *info_ptr, void *user_data)
{
}
static int sub_offset = 0;
static void __on_subtitle_message_received(const cJSON* root) {
/*
{
"data" :
[
{
"definite" : false,
"language" : "zh",
"mode" : 1,
"paragraph" : false,
"sequence" : 0,
"text" : "\\u4f60\\u597d",
"userId" : "voiceChat_xxxxx"
}
],
"type" : "subtitle"
}
*/
cJSON * type_obj = cJSON_GetObjectItem(root, "type");
if (type_obj != NULL && strcmp("subtitle", cJSON_GetStringValue(type_obj)) == 0) {
cJSON* data_obj_arr = cJSON_GetObjectItem(root, "data");
cJSON* obji = NULL;
cJSON_ArrayForEach(obji, data_obj_arr) {
cJSON* user_id_obj = cJSON_GetObjectItem(obji, "userId");
cJSON* text_obj = cJSON_GetObjectItem(obji, "text");
cJSON* definite_obj = cJSON_GetObjectItem(obji, "definite");
char* sub = cJSON_GetStringValue(text_obj);
// print_string(sub + sub_offset -1);
if(cJSON_IsTrue(definite_obj)){
sub_offset = sub_offset == 0 ? 0 : sub_offset -1;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,sub + sub_offset);
sub_offset = 0;
aios_event_pub(VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE, sub, NULL);
} else if(strlen(sub) - sub_offset > 18){
sub_offset = sub_offset == 0 ? 0 : sub_offset -1;
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_SUBTITLE,VOLC_DISPLAY_TEXT,sub + sub_offset);
if(strlen(sub) - sub_offset > 36){
sub_offset = strlen(sub);
}
}
}
}
}
/* HTTPS reporting must never run inside the WebSocket receive mutex. */
static void process_platform_reports(void)
{
cJSON *root;
while (platform_report_queue && xQueueReceive(platform_report_queue, &root, 0) == pdTRUE) {
cJSON *type_obj = cJSON_GetObjectItemCaseSensitive(root, "type");
const char *type = cJSON_IsString(type_obj) ? type_obj->valuestring : "";
if ((strcmp(type, "session.updated") == 0 || strcmp(type, "session.created") == 0) && !session_started_reported) {
cJSON *session = cJSON_GetObjectItemCaseSensitive(root, "session");
cJSON *id = cJSON_IsObject(session) ? cJSON_GetObjectItemCaseSensitive(session, "id") : NULL;
cJSON *job_id = cJSON_GetObjectItemCaseSensitive(root, "job_id");
const char *provider_ref = cJSON_IsString(id) && id->valuestring[0] ? id->valuestring :
(cJSON_IsString(job_id) ? job_id->valuestring : NULL);
if (provider_ref && provider_ref[0] && platform_conversation_started(provider_ref) == 0) {
session_started_reported = true;
LOGI("Platform confirmed dynamic session configuration");
}
} else if (strcmp(type, "conversation.item.input_audio_transcription.completed") == 0) {
cJSON *text = cJSON_GetObjectItemCaseSensitive(root, "transcript");
cJSON *source = cJSON_GetObjectItemCaseSensitive(root, "event_id");
if (cJSON_IsString(text) && text->valuestring[0]) {
(void)platform_conversation_turn("subject", cJSON_IsString(source) ? source->valuestring : "subject", text->valuestring);
}
} else if (strcmp(type, "response.audio_transcript.done") == 0) {
cJSON *text = cJSON_GetObjectItemCaseSensitive(root, "transcript");
cJSON *source = cJSON_GetObjectItemCaseSensitive(root, "response_id");
if (cJSON_IsString(text) && text->valuestring[0]) {
(void)platform_conversation_turn("agent", cJSON_IsString(source) ? source->valuestring : "agent", text->valuestring);
}
}
cJSON_Delete(root);
}
}
static void __on_volc_message_data(volc_engine_t handle, const void *message, size_t size, volc_message_info_t *info_ptr, void *user_data)
{
static char message_buffer[4096];
if (size > 0 && size < sizeof(message_buffer) - 1) {
memcpy(message_buffer, message, size);
message_buffer[size] = 0;
const bool standard_protocol = message_buffer[0] == '{';
if (!standard_protocol && size < 8) return;
cJSON *root = cJSON_Parse(standard_protocol ? message_buffer : message_buffer + 8);
if (root != NULL) {
if (standard_protocol) {
const char *type = cJSON_GetStringValue(cJSON_GetObjectItem(root, "type"));
if (type && (!strcmp(type, "error") || !strcmp(type, "conversation.item.created"))) {
LOGI("Provider control event: %s", message_buffer);
}
if (type && !strcmp(type, "response.audio.done") && !is_stop) {
/* Flush the final sub-frame through the PCM decoder. */
static const uint8_t silence[2048] = {0};
volc_hal_context_t *hal = volc_get_global_hal_context();
if (hal) {
volc_hal_player_t player = hal->player_handle[VOLC_HAL_PLAYER_AUDIO];
volc_hal_player_play_data(player, silence, sizeof(silence));
volc_hal_player_finish_segment(player);
}
}
if (type && (strstr(type, "speech_") || strstr(type, "response."))) {
conv_service.wait_time = 0;
}
bool report = type && (!strcmp(type, "session.created") || !strcmp(type, "session.updated") ||
!strcmp(type, "conversation.item.input_audio_transcription.completed") ||
!strcmp(type, "response.audio_transcript.done"));
if (report && platform_report_queue && xQueueSend(platform_report_queue, &root, 0) == pdTRUE) {
root = NULL; /* Conversation task owns queued messages. */
} else if (report) {
LOGE("Platform report queue unavailable or full");
}
cJSON_Delete(root);
} else if (message_buffer[0] == 's' && message_buffer[1] == 'u' && message_buffer[2] == 'b' && message_buffer[3] == 'v') {
__on_subtitle_message_received(root);
conv_service.wait_time = 0;
} else if (message_buffer[0] == 't' && message_buffer[1] == 'o' && message_buffer[2] == 'o' && message_buffer[3] == 'l') {
// function calling 消息
aios_event_pub(VOLC_FUNCTION_CALL_EXEC, root, NULL);
}else if (message_buffer[0] == 'i' && message_buffer[1] == 'n' && message_buffer[2] == 'f' && message_buffer[3] == 'o') {
aios_event_pub(VOLC_FUNCTION_CALL_TRIGGER, root, NULL);
}
}
}
}
void conv_ai_service_init()
{
if (!platform_report_queue) platform_report_queue = xQueueCreate(8, sizeof(cJSON *));
snprintf(config_buf, sizeof(config_buf), CONV_AI_CONFIG_FORMAT,
platform_runtime_instance_id(),
platform_runtime_product_key(),
platform_runtime_device_secret(),
platform_runtime_device_name());
conv_service.volc_event_handler = (volc_event_handler_t){
.on_volc_event = __on_volc_event,
.on_volc_conversation_status = __on_volc_conversation_status,
.on_volc_audio_data = __on_volc_audio_data,
.on_volc_video_data = __on_volc_video_data,
.on_volc_message_data = __on_volc_message_data,
};
int ret = volc_create(&conv_service.engine, config_buf, &conv_service.volc_event_handler, &conv_service);
if (ret != 0) {
LOGE("volc_create failed, ret: %d", ret);
return;
} else {
LOGE("volc_create success");
}
}
void conv_ai_service_task(void *pvParameters)
{
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
char* status_str = NULL;
bool provider_started = false;
bool audio_started = false;
session_started_reported = false;
provider_disconnected = false;
if (g_hal_context == NULL) {
LOGE("volc_get_global_hal_context failed");
status_str = "系统未初始化,ai对话连接失败";
goto CONV_AI_QUIT;
}
global_display = g_hal_context->display_handle;
int ret = 0;
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT, "ai对话创建中");
ret = platform_conversation_begin();
if (ret != 0) {
LOGE("platform_conversation_begin failed, ret: %d", ret);
status_str = "平台会话创建失败";
if (strcmp(platform_last_error_code(), "FIRMWARE_UPDATE_REQUIRED") == 0) {
status_str = "平台要求更新固件";
}
goto CONV_AI_QUIT;
}
/* Capture may run while the WebSocket is still handshaking. Hold frames
* until volc_start has moved the engine into STARTED state. */
is_stop = true;
// step 1: start audio capture & play
volc_hal_capture_config_t capture_audio_config = {0};
capture_audio_config.media_type = VOLC_MEDIA_TYPE_AUDIO;
capture_audio_config.data_cb = __audio_capture_cb;
// capture_audio_config.audio_wakeup_cb = (volc_hal_audio_wakeup_cb_t)rec_engine_cb;
capture_audio_config.user_data = g_hal_context;
volc_hal_capture_t capture_handle = g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO];
if (g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO] == NULL) {
g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO] = volc_hal_capture_create(&capture_audio_config);
capture_handle = g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO];
}
audio_started = true;
volc_hal_capture_stop(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO]);
volc_hal_capture_update_config(capture_handle, &capture_audio_config);
volc_hal_capture_start(g_hal_context->capture_handle[VOLC_HAL_PLAYER_AUDIO], VOLC_AUDIO_MODE_CAPTURE);
volc_hal_player_config_t player_audio_config = {0};
player_audio_config.media_type = VOLC_MEDIA_TYPE_AUDIO;
if (g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO] == NULL) {
g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO] = volc_hal_player_create(&player_audio_config);
}
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, "ai对话连接中");
volc_hal_player_start(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
// step 2: start ai conversation
const char *session_update = platform_conversation_update_json();
size_t params_size = strlen(session_update) + 64;
char *params = volc_osal_malloc(params_size);
if (params == NULL) {
status_str = "会话配置内存不足";
goto CONV_AI_QUIT;
}
snprintf(params, params_size, "{\"audio\":{\"codec\":0},\"session_update\":%s}", session_update);
volc_opt_t opt = {
.mode = VOLC_MODE_WS,
.bot_id = (char *)platform_conversation_bot_id(),
.params = params,
};
#if defined ENABLE_PLAY_WELCOME
aios_event_pub(VOLC_LOCAL_LOGIC_PLAY_WELCOME, NULL, NULL);
#endif
ret = volc_start(conv_service.engine, &opt);
volc_osal_free(params);
if (ret != 0) {
LOGE("volc_start failed, ret: %d", ret);
status_str = "ai对话连接失败,准备退出";
goto CONV_AI_QUIT;
}
provider_started = true;
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, "ai对话已连接");
is_stop = false;
while (!is_stop && !provider_disconnected) {
process_platform_reports();
if (session_started_reported && platform_conversation_poll()) {
status_str = "平台已结束本次对话";
goto CONV_AI_QUIT;
}
if (diagnostic_say_pending && session_started_reported) {
diagnostic_say_pending = false;
char request[640];
snprintf(request, sizeof(request), "{\"event_id\":\"diag-%llu\",\"type\":\"conversation.item.create\",\"item\":{\"type\":\"message\",\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"请只说这句话:你好,这是语音播放测试。现在使用自然的语速,连续清楚地说完这句话。\"}],\"interrupt_mode\":1}}", (unsigned long long)volc_osal_get_time_ms());
volc_message_info_t info = {.is_binary = false};
int sent = volc_send_message(conv_service.engine, request, strlen(request), &info);
LOGI("PLAYBACK diagnostic sentence requested ret=%d", sent);
const char *respond = "{\"type\":\"response.create\",\"response\":{\"modalities\":[\"text\",\"audio\"]}}";
volc_send_message(conv_service.engine, respond, strlen(respond), &info);
conv_service.wait_time = 0;
}
sleep(1);
conv_service.wait_time++;
if(conv_service.wait_time >= CONV_SERVICE_TIMEOUT){
status_str = "ai对话已超时,准备退出";
goto CONV_AI_QUIT;
}
}
status_str = "ai对话已断开";
CONV_AI_QUIT:
is_stop = true;
// volc_hal_file_close(audioFile);
volc_stop(conv_service.engine);
process_platform_reports();
platform_conversation_finish(!provider_started || !session_started_reported,
!provider_started ? "VOLC_START_FAILED" : (!session_started_reported ? "SESSION_UPDATE_NOT_CONFIRMED" : NULL));
if (g_hal_context && audio_started) {
volc_hal_capture_stop(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO]);
if (g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]) {
volc_hal_player_stop(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
volc_hal_player_destroy(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO] = NULL;
}
}
if(status_str != NULL){
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_STATUS, VOLC_DISPLAY_TEXT, status_str);
}
volc_hal_display_set_content(global_display, VOLC_DISPLAY_OBJ_SUBTITLE, VOLC_DISPLAY_TEXT, "");
conv_service.wait_time = 0;
// just show the status: Ai对话已断开
sleep(1);
if (g_hal_context && audio_started) {
volc_hal_capture_start(g_hal_context->capture_handle[VOLC_HAL_CAPTURE_AUDIO],VOLC_AUDIO_MODE_WAKEUP);
}
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_OVER, NULL, NULL);
volc_osal_thread_exit(NULL);
}
void conv_ai_service_task_stop(void)
{
// sleep for AI Agent say goodbye
sleep(1);
is_stop = true;
}
#if __cplusplus
}
#endif
@@ -0,0 +1,84 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#include <stdio.h>
#include "aios.h"
#include "volc_hal.h"
#include "volc_osal.h"
#include "volc_hal_display.h"
#include "volc_hal_file.h"
#include "volc_service_common.h"
#include "volc_conv_service.h"
#include "volc_conv.h"
#if __cplusplus
extern "C" {
#endif
typedef struct volc_conv_service_manager {
aios_session_t super;
volc_osal_tid_t conv_thread_id;
} volc_conv_service_manager_t;
volc_conv_service_manager_t volc_conv_service_manager;
static aios_ret_t __state_conversation(volc_conv_service_manager_t * const me, aios_event_t const * const e)
{
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
if (g_hal_context == NULL) {
return AIOS_Ret_NotHandled;
}
volc_hal_display_t global_display = g_hal_context->display_handle;
switch (e->id) {
case VOLC_SERVICE_AI_CONVERSATION_START:
if (me->conv_thread_id == NULL) {
volc_osal_thread_param_t param = {0};
volc_hal_display_set_content(global_display,VOLC_DISPLAY_OBJ_STATUS,VOLC_DISPLAY_TEXT,"ai对话启动中");
snprintf(param.name, sizeof(param.name), "%s", "conv_ai");
param.stack_size = 8*1024;
param.priority = 5;
param.stack_in_ext = 1;
volc_osal_thread_create(&me->conv_thread_id, &param, conv_ai_service_task,NULL);
}
return AIOS_Ret_Handled;
case VOLC_SERVICE_AI_CONVERSATION_INTERRUPT:
return AIOS_Ret_Handled;
// control the Ai_Conversation task quit
case VOLC_SERVICE_AI_CONVERSATION_QUIT:
if (me->conv_thread_id != NULL) {
conv_ai_service_task_stop();
me->conv_thread_id = NULL;
}
return AIOS_Ret_Handled;
// notice the Ai_Conversation over by task
case VOLC_SERVICE_AI_CONVERSATION_OVER:
if (me->conv_thread_id != NULL) {
me->conv_thread_id = NULL;
}
return AIOS_Ret_Handled;
default:
return AIOS_Ret_NotHandled;
}
}
static aios_ret_t __state_init(volc_conv_service_manager_t * const me, aios_event_t const * const e)
{
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_START); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_INTERRUPT); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_QUIT); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION_OVER); // 订阅事件
me->conv_thread_id = NULL;
conv_ai_service_init();
return AIOS_TRAN(__state_conversation);
}
void volc_conv_service_manager_init(void)
{
aios_session_init(&volc_conv_service_manager.super, VOLC_SERVICE_PRIORITY_AI_CONVERSATION, NULL);
aios_session_start(&volc_conv_service_manager.super, AIOS_STATE_CAST(__state_init));
}
#if __cplusplus
}
#endif
@@ -0,0 +1,9 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_FUNCTION_CALL_SERVICE_H__
#define VOLC_FUNCTION_CALL_SERVICE_H__
void function_call_service_init();
#endif /* VOLC_FUNCTION_CALL_SERVICE_H__ */
@@ -0,0 +1,9 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_LOCAL_FUNCTION_LIST_H__
#define VOLC_LOCAL_FUNCTION_LIST_H__
void adjust_audio_val(const char* action);
#endif
@@ -0,0 +1,134 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#include <string.h>
#include "cJSON.h"
#include "volc_service_common.h"
#include "volc_conv.h"
#include "volc_function_call_service.h"
#include "volc_local_function_list.h"
#include "aios.h"
#include <stdio.h>
#if __cplusplus
extern "C" {
#endif
typedef struct volc_function_call_service {
aios_session_t super;
} volc_function_call_service_t;
volc_function_call_service_t g_volc_function_call_service;
static void __on_function_calling_message_received(const cJSON *root)
{
/*
{
"subscriber_user_id" : "",
"tool_calls" :
[
{
"function" :
{
"arguments" : "{\\"location\\": \\"\\u5317\\u4eac\\u5e02\\"}",
"name" : "get_current_weather"
},
"id" : "call_py400kek0e3pczrqdxgnb3lo",
"type" : "function"
}
]
}
*/
cJSON *tool_obj_arr = cJSON_GetObjectItem(root, "tool_calls");
cJSON *obji = NULL;
cJSON_ArrayForEach(obji, tool_obj_arr)
{
cJSON *id_obj = cJSON_GetObjectItem(obji, "id");
cJSON *function_obj = cJSON_GetObjectItem(obji, "function");
if (id_obj && function_obj)
{
cJSON *arguments_obj = cJSON_GetObjectItem(function_obj, "arguments");
cJSON *name_obj = cJSON_GetObjectItem(function_obj, "name");
char *arguments_json_str = cJSON_GetStringValue(arguments_obj);
const char *func_name = cJSON_GetStringValue(name_obj);
const char *func_id = cJSON_GetStringValue(id_obj);
if (strcmp(func_name, "adjust_audio_val") == 0 && arguments_json_str)
{
cJSON *arguments_json = cJSON_Parse(arguments_json_str);
cJSON *action_obj = cJSON_GetObjectItem(arguments_json, "action");
const char *action = (action_obj->valuestring);
adjust_audio_val(action);
volc_conv_service_t conv_service = get_conv_ai_service();
cJSON *fc_obj = cJSON_CreateObject();
cJSON_AddStringToObject(fc_obj, "ToolCallID", func_id);
cJSON_AddStringToObject(fc_obj, "Content", "音量已经调整了");
char *json_string = cJSON_Print(fc_obj);
static char fc_message_buffer[256] = {'f', 'u', 'n', 'c'};
int json_str_len = strlen(json_string);
fc_message_buffer[4] = (json_str_len >> 24) & 0xff;
fc_message_buffer[5] = (json_str_len >> 16) & 0xff;
fc_message_buffer[6] = (json_str_len >> 8) & 0xff;
fc_message_buffer[7] = (json_str_len >> 0) & 0xff;
memcpy(fc_message_buffer + 8, json_string, json_str_len);
cJSON_Delete(fc_obj);
volc_message_info_t info = {0};
info.is_binary = 1;
volc_send_message(conv_service.engine, fc_message_buffer, json_str_len + 8, &info);
}
}
}
cJSON_Delete(root);
}
void __on_function_calling_trigger(const cJSON *root)
{
/*
{
"event_type": "function_calling", // 事件类型,固定为此值
"function": "get_current_weather", // 被触发的函数名,与您在 StartVoiceChat 中定义的名称一致
"tool_call_id": "call_12345", // 本次函数调用的唯一ID,后续回传结果时需要用到
"response_id": "resp_abcde" // 本次响应的唯一ID,可用于问题排查
}
*/
cJSON *event_type_obj = cJSON_GetObjectItemCaseSensitive(root, "event_type");
if(event_type_obj && strcmp(event_type_obj->valuestring, "function_calling") == 0 ) {
cJSON *function_obj = cJSON_GetObjectItemCaseSensitive(root, "function");
if(function_obj && strcmp(function_obj->valuestring, "adjust_audio_val") == 0){
volc_conv_service_t conv_service = get_conv_ai_service();
volc_send_text_to_agent(conv_service.engine, "别着急,我来给你调整音量", VOLC_AGENT_TYPE_TTS,2);
}
}
cJSON_Delete(root);
}
static aios_ret_t __state_conversation(volc_function_call_service_t * const me, aios_event_t const * const e)
{
switch (e->id) {
case VOLC_FUNCTION_CALL_EXEC:
__on_function_calling_message_received((const cJSON *)e->data);
return AIOS_Ret_Handled;
case VOLC_FUNCTION_CALL_TRIGGER:
__on_function_calling_trigger((const cJSON *)e->data);
return AIOS_Ret_Handled;
}
}
static aios_ret_t __state_init(volc_function_call_service_t * const me, aios_event_t const * const e)
{
AIOS_EVENT_SUB(VOLC_FUNCTION_CALL_EXEC); // 订阅事件
AIOS_EVENT_SUB(VOLC_FUNCTION_CALL_TRIGGER); // 订阅事件
return AIOS_TRAN(__state_conversation);
}
void function_call_service_init(void){
aios_session_init(&g_volc_function_call_service.super, VOLC_SERVICE_PRIORITY_FUNCTION_CALL, NULL);
aios_session_start(&g_volc_function_call_service.super, AIOS_STATE_CAST(__state_init));
}
#if __cplusplus
}
#endif
@@ -0,0 +1,41 @@
#include "volc_local_function_list.h"
#include "volc_hal.h"
#include "volc_hal_player.h"
#include <string.h>
void __volc_up_audio_val()
{
volc_hal_context_t *g_hal_context = volc_get_global_hal_context();
if (g_hal_context == NULL)
{
return;
}
int volc_audio_vol = volc_hal_get_audio_player_volume(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
volc_audio_vol = (volc_audio_vol + 10) <= 100 ? (volc_audio_vol + 10) : 100;
volc_hal_set_audio_player_volume(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO], volc_audio_vol);
}
void __volc_down_audio_val()
{
volc_hal_context_t *g_hal_context = volc_get_global_hal_context();
if (g_hal_context == NULL)
{
return;
}
int volc_audio_vol = volc_hal_get_audio_player_volume(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO]);
volc_audio_vol = (volc_audio_vol - 10) <= 0 ? 0 : (volc_audio_vol - 10);
volc_hal_set_audio_player_volume(g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO], volc_audio_vol);
}
void adjust_audio_val(const char *action)
{
if (strcmp(action, "up") == 0)
{
__volc_up_audio_val();
}
else if (strcmp(action, "down") == 0)
{
__volc_down_audio_val();
}
}
@@ -0,0 +1,11 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_SERVICE_MANAGER_H__
#define VOLC_SERVICE_MANAGER_H__
#include <stdbool.h>
#include "aios.h"
void volc_service_manager_init(void);
#endif /* VOLC_SERVICE_MANAGER_H__ */
@@ -0,0 +1,38 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_CONV_COMMON_H__
#define VOLC_CONV_COMMON_H__
#include "aios.h"
enum {
VOLC_SERVICE_EVENT_TEST = Event_User,
VOLC_SERVICE_NETWORK_CONFIG,
VOLC_SERVICE_AI_CONVERSATION,
// Ai_Conversation event:
VOLC_SERVICE_AI_CONVERSATION_START = 100,
VOLC_SERVICE_AI_CONVERSATION_INTERRUPT = 101,
VOLC_SERVICE_AI_CONVERSATION_QUIT = 102, // control the Ai_Conversation task quit
VOLC_SERVICE_AI_CONVERSATION_OVER = 103, // notice the Ai_Conversation over by task
// function_call event:
VOLC_FUNCTION_CALL_EXEC = 200,
VOLC_FUNCTION_CALL_TRIGGER = 201,
// local logic event:
VOLC_LOCAL_LOGIC_PLAY_WELCOME = 300,
VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE = 301,
VOLC_SERVICE_EVENT_MAX,
};
enum {
VOLC_SERVICE_PRIORITY_AI_CONVERSATION = 0,
VOLC_SERVICE_PRIORITY_FUNCTION_CALL,
VOLC_SERVICE_PRIORITY_LOCAL_LOGIC,
VOLC_SERVICE_PRIORITY_APP_MANAGER,
};
#endif /* VOLC_CONV_COMMON_H__ */
@@ -0,0 +1,9 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#ifndef VOLC_LOCAL_LOGIC_SERVICE_H__
#define VOLC_LOCAL_LOGIC_SERVICE_H__
void local_logic_service_init();
#endif /* VOLC_LOCAL_LOGIC_SERVICE_H__ */
@@ -0,0 +1,99 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#include <string.h>
#include "volc_conv_ai.h"
#include "volc_service_common.h"
#include "aios.h"
#include "volc_conv.h"
#include "volc_hal.h"
#include "volc_hal_file.h"
#include "volc_hal_player.h"
#if __cplusplus
extern "C" {
#endif
static const char *welcome_audio_file_name = "/sdcard/wel.pcm";
typedef struct volc_local_logic_service {
aios_session_t super;
} volc_local_logic_service_t;
volc_local_logic_service_t g_volc_local_logic_service;
static int has_quit = 0;
static void __play_welcome(){
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
if(g_hal_context == NULL) return;
volc_hal_player_t player = g_hal_context->player_handle[VOLC_HAL_PLAYER_AUDIO];
volc_hal_file_t* audioFile = volc_hal_file_open(welcome_audio_file_name, "rb");
if(audioFile == NULL) return;
char audio_data[160];
if(player){
int offset = 0;
uint64_t size = volc_hal_file_get_size(audioFile);
while(offset < size - 160){
volc_hal_file_read(audioFile,audio_data,160);
volc_hal_player_play_data(player, audio_data, 160);
offset += 160;
}
}
volc_hal_file_close(audioFile);
return;
}
static int __has_substring(const char *haystack, const char *needle) {
// 处理边界:子串为空串,按惯例返回包含(可根据需求调整)
if (needle[0] == '\0') {
return 1;
}
// strstr 返回非 NULL 表示找到子串
return strstr(haystack, needle) != NULL ? 1 : 0;
}
static void __parser_and_process_subtitle(char *subtitle)
{
volc_conv_service_t conv_service = get_conv_ai_service();
if (__has_substring(subtitle, "退下吧") || __has_substring(subtitle, "拜拜"))
{
if (!has_quit)
{
volc_send_text_to_agent(conv_service.engine, "好的拜拜", VOLC_AGENT_TYPE_TTS, 1);
has_quit = 1;
}
}
if (__has_substring(subtitle, "好的拜拜"))
{
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_QUIT, NULL, NULL);
has_quit = 0;
}
}
static aios_ret_t __state_conversation(volc_local_logic_service_t * const me, aios_event_t const * const e)
{
switch (e->id) {
case VOLC_LOCAL_LOGIC_PLAY_WELCOME:
__play_welcome();
return AIOS_Ret_Handled;
case VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE:
__parser_and_process_subtitle(e->data);
return AIOS_Ret_Handled;
}
}
static aios_ret_t __state_init(volc_local_logic_service_t * const me, aios_event_t const * const e)
{
AIOS_EVENT_SUB(VOLC_LOCAL_LOGIC_PLAY_WELCOME); // 订阅事件
AIOS_EVENT_SUB(VOLC_LOCAL_LOGIC_PROCESS_SUBTITLE); // 订阅事件
return AIOS_TRAN(__state_conversation);
}
void local_logic_service_init(void){
aios_session_init(&g_volc_local_logic_service.super, VOLC_SERVICE_PRIORITY_LOCAL_LOGIC, NULL);
aios_session_start(&g_volc_local_logic_service.super, AIOS_STATE_CAST(__state_init));
}
#if __cplusplus
}
#endif
@@ -0,0 +1,43 @@
// Copyright (2025) Beijing Volcano Engine Technology Ltd.
// SPDX-License-Identifier: Apache-2.0
#include "volc_hal.h"
#include "volc_manager_service.h"
#include "volc_conv_service.h"
#include "volc_service_common.h"
typedef struct volc_service_manager {
aios_session_t super;
} volc_service_manager_t;
volc_service_manager_t volc_service_manager;
static aios_ret_t __state_wait_service_event(volc_service_manager_t* const me, aios_event_t const * const e)
{
volc_hal_context_t* g_hal_context = volc_get_global_hal_context();
if(g_hal_context == NULL){
return AIOS_Ret_NotHandled;
}
switch ((int)e->id) {
case VOLC_SERVICE_NETWORK_CONFIG:
return AIOS_Ret_Handled;
case VOLC_SERVICE_AI_CONVERSATION:
aios_event_pub(VOLC_SERVICE_AI_CONVERSATION_START, NULL, NULL);
return AIOS_Ret_Handled;
default:
return AIOS_Ret_NotHandled;
}
}
static aios_ret_t __state_init(volc_service_manager_t* const me, aios_event_t const * const e)
{
AIOS_EVENT_SUB(VOLC_SERVICE_NETWORK_CONFIG); // 订阅事件
AIOS_EVENT_SUB(VOLC_SERVICE_AI_CONVERSATION); // 订阅事件
return AIOS_TRAN(__state_wait_service_event);
}
void volc_service_manager_init(void)
{
aios_session_init(&volc_service_manager.super, VOLC_SERVICE_PRIORITY_APP_MANAGER, NULL);
aios_session_start(&volc_service_manager.super, AIOS_STATE_CAST(__state_init));
}