chore: checkpoint 第一版
ci / verify (push) Canceled after 0s

This commit is contained in:
Codex Local
2026-09-06 17:04:11 +08:00
parent a8cdec98d1
commit 0c2151ace9
659 changed files with 185855 additions and 554 deletions
@@ -0,0 +1 @@
**/gen_bmgr_codes/
@@ -0,0 +1,7 @@
# Changelog
## v0.1.0
### Features
- Initial version of `av_processor`
@@ -0,0 +1,7 @@
idf_component_register(
INCLUDE_DIRS "include" "./"
PRIV_INCLUDE_DIRS "private_include"
SRC_DIRS "./"
REQUIRES esp_capture esp_video_codec
)
@@ -0,0 +1,82 @@
menu "Audio/Video Processor Configuration"
# Media Dump Configuration
menu "Media Dump"
config MEDIA_DUMP_ENABLE
bool "Enable Media Dump"
default n
help
Enable media dump feature. When disabled, related options are hidden.
# Audio AEC Save Point
choice MEDIA_DUMP_AUDIO_POINT
prompt "Audio Save Mode (AEC point)"
depends on MEDIA_DUMP_ENABLE
default MEDIA_DUMP_AUDIO_NONE
help
Select which audio to save for media dump with respect to AEC processing.
Only one can be selected, or select None to disable both.
config MEDIA_DUMP_AUDIO_BEFORE_AEC
bool "Save Before AEC"
help
Save raw microphone audio before AEC processing for media dump.
config MEDIA_DUMP_AUDIO_AFTER_AEC
bool "Save After AEC"
help
Save audio after AEC processing for media dump.
config MEDIA_DUMP_AUDIO_NONE
bool "None"
help
Do not save audio for media dump.
endchoice
config MEDIA_DUMP_DURATION_SEC
int "Dump Duration (seconds)"
depends on MEDIA_DUMP_ENABLE
default 20
range 1 3600
help
Maximum duration to capture media dump in seconds. Default 20s.
choice MEDIA_DUMP_SINK
prompt "Media Dump Output Method"
depends on MEDIA_DUMP_ENABLE
default MEDIA_DUMP_SINK_SDCARD
help
Select where to save media dump data.
config MEDIA_DUMP_SINK_UDP
bool "UDP"
config MEDIA_DUMP_SINK_SDCARD
bool "SD Card File"
endchoice
if MEDIA_DUMP_SINK_UDP
config MEDIA_DUMP_UDP_IP
string "UDP Destination IP"
default "192.168.1.100"
help
Destination IPv4 address for UDP media dump.
config MEDIA_DUMP_UDP_PORT
int "UDP Destination Port"
default 5000
range 1 65535
help
Destination UDP port for media dump.
endif
if MEDIA_DUMP_SINK_SDCARD
config MEDIA_DUMP_SDCARD_FILENAME
string "SD Card File Name"
default "/sdcard/media_dump.bin"
help
File path on SD card to save media dump data. Ensure SD is mounted.
endif
endmenu
endmenu
@@ -0,0 +1,22 @@
Espressif Modified MIT License
Copyright (c) 2025 Espressif Systems (Shanghai) CO., LTD
Permission is hereby granted for use EXCLUSIVELY with Espressif Systems products.
This includes the right to use, copy, modify, merge, publish, distribute, and sublicense
the Software, subject to the following conditions:
1. This Software MUST BE USED IN CONJUNCTION WITH ESPRESSIF SYSTEMS PRODUCTS.
2. The above copyright notice and this permission notice shall be included in all copies
or substantial portions of the Software.
3. Redistribution of the Software in source or binary form FOR USE WITH NON-ESPRESSIF PRODUCTS
is strictly prohibited.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
@@ -0,0 +1,332 @@
# AV Processor Component (Audio/Video Processing)
This component provides a unified encapsulation for audio recording/playback/data feeding and video capture/decoding/rendering on ESP platforms, implemented based on middleware such as GMF (General Media Framework), esp-audio-render, esp-video-codec, and esp-capture.
## Feature Overview 🎛️
- **Audio 🎧**:
- Recording (AEC/VAD/AFE optional)
- Playback (supports URL and local files, decoding depends on configuration)
- Data feeding (feeder, suitable for scenarios like RTC/streaming where receiving and playing happen simultaneously)
- Optional mixer with volume fade and focus control between multiple streams
- **Video 🎥**:
- Multi-channel video capture
- Video decoding and rendering callbacks
---
## Quick Start
### 1. Audio Usage
1. Initialize audio manager:
```c
// Method 1: Use default configuration macro (recommended)
audio_manager_config_t cfg = DEFAULT_AUDIO_MANAGER_CONFIG();
cfg.play_dev = your_play_dev;
cfg.rec_dev = your_rec_dev;
strcpy(cfg.mic_layout, "RMNM"); // Microphone layout (optional, for AFE configuration)
cfg.board_sample_rate = 16000;
cfg.board_bits = 32;
cfg.board_channels = 2;
cfg.play_volume = 80;
cfg.rec_volume = 60;
cfg.rec_ref_volume = 60;
cfg.enable_mixer = true; // Enable when mixing feeder and playback is needed
audio_manager_init(&cfg);
```
2. Recording (polling read):
```c
// Recording event callback (optional)
void recorder_event_cb(void *event, void *ctx) {
// Handle recording events (e.g., AFE events)
}
// Configure recorder (using default configuration macro)
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
// Optional: Configure AFE (runtime configuration, takes priority over Kconfig)
recorder_cfg.afe_config.vad_enable = true;
recorder_cfg.afe_config.vad_mode = 4;
recorder_cfg.afe_config.agc_enable = true;
recorder_cfg.recorder_event_cb = recorder_event_cb;
// Open recorder
audio_recorder_open(&recorder_cfg);
uint8_t buf[2048];
audio_recorder_read_data(buf, sizeof(buf));
audio_recorder_close();
```
3. Feeder playback (suitable for external stream → local playback):
```c
// Configure feeder (using default configuration macro)
audio_feeder_config_t feeder_cfg = DEFAULT_AUDIO_FEEDER_CONFIG();
// Optional: When using OPUS decoder, task_stack is recommended to be >= 4096 * 10
// feeder_cfg.feeder_task_config.task_stack = 4096 * 10;
audio_feeder_open(&feeder_cfg);
audio_feeder_run();
// Feed external data blocks multiple times as needed
audio_feeder_feed_data(pkt, pkt_len);
audio_feeder_stop();
audio_feeder_close();
```
4. Normal playback (URL or local):
```c
// Configure player (using default configuration macro)
audio_playback_config_t playback_cfg = DEFAULT_AUDIO_PLAYBACK_CONFIG();
audio_playback_open(&playback_cfg);
audio_playback_play("http://<ip>:<port>/audio.mp3");
audio_playback_pause();
audio_playback_resume();
audio_playback_stop();
audio_playback_close();
```
5. Focus/fade control (when mixer is enabled):
```c
// Note: Mixer must be opened after both playback and feeder are opened
audio_processor_mixer_open();
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_FEEDER); // Focus on feeder audio
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_PLAYBACK); // Focus on playback audio
audio_processor_mixer_close();
```
### 2. Video Usage
1. Render (decode) callback:
```c
void decoded_cb(void *ctx, const uint8_t *data, size_t size) {
// Handle decoded frame data
}
video_render_config_t rcfg = {
.decode_cfg = your_vdec_cfg,
.resolution = {.width = 640, .height = 480},
.decode_cb = decoded_cb,
};
video_render_handle_t r = video_render_open(&rcfg);
video_render_start(r);
video_frame_t f = {.data = enc_frame, .size = enc_size};
video_render_frame_feed(r, &f);
video_render_stop(r);
video_render_close(r);
```
1. Capture (multi-channel):
```c
void capture_cb(void *ctx, int index, video_frame_t *frame) {
// Handle captured video frame
}
video_capture_config_t ccfg = {0};
ccfg.camera_config = &your_cam_cfg;
ccfg.sink_num = 2;
ccfg.sink_cfg[0] = your_sink0_cfg;
ccfg.sink_cfg[1] = your_sink1_cfg;
ccfg.capture_frame_cb = capture_cb;
video_capture_handle_t c = video_capture_open(&ccfg);
video_capture_start(c);
video_capture_stop(c);
video_capture_close(c);
```
---
## Media Dump 💾
- Enable condition ✅: Takes effect after enabling `MEDIA_DUMP_ENABLE` in `menuconfig`.
- Save methods 📤: Supports SD card and UDP outputs (choose one).
- SD card 💾: Enable `CONFIG_MEDIA_DUMP_SINK_SDCARD`, output to file `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`, duration controlled by `CONFIG_MEDIA_DUMP_DURATION_SEC`.
- UDP 📡: Enable `CONFIG_MEDIA_DUMP_SINK_UDP`, send raw media data via `CONFIG_MEDIA_DUMP_UDP_IP` and `CONFIG_MEDIA_DUMP_UDP_PORT` (can use `script/udp_reciver.py` script).
- Save point 🎚️ (before/after AEC processing): Select in "Audio Save Mode (AEC point)" in `menuconfig`
- `MEDIA_DUMP_AUDIO_BEFORE_AEC` (Save Before AEC): Save raw microphone audio before AEC (for analyzing noise/feedback)
- `MEDIA_DUMP_AUDIO_AFTER_AEC` (Save After AEC): Save audio after AEC processing (for evaluating AEC effectiveness)
- Typical usage 🔍: Export raw audio/video data when reproducing issues, check saturation, feedback, and format correctness offline using tools like Audacity.
## Configuration ⚙️
### Runtime Configuration (Recommended)
AFE configuration can be set at runtime via the `afe_config` field in the `audio_recorder_config_t` structure, which takes priority over Kconfig configuration:
```c
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
// Configure AFE
recorder_cfg.afe_config.ai_mode_wakeup = false; // AI mode: true=wakeup mode, false=direct mode
recorder_cfg.afe_config.vad_enable = true; // Enable VAD
recorder_cfg.afe_config.vad_mode = 4; // VAD mode (1-4), larger values are more sensitive
recorder_cfg.afe_config.vad_min_speech_ms = 64; // Minimum speech duration (ms)
recorder_cfg.afe_config.vad_min_noise_ms = 1000; // Minimum noise duration (ms)
recorder_cfg.afe_config.agc_enable = true; // Enable AGC
recorder_cfg.afe_config.enable_vcmd_detect = false; // Enable VCMD
recorder_cfg.afe_config.vcmd_timeout_ms = 5000; // VCMD timeout (ms)
recorder_cfg.afe_config.mn_language = "cn"; // Model language: "cn" or "en"
recorder_cfg.afe_config.wakeup_time_ms = 10000; // Wakeup time (ms)
recorder_cfg.afe_config.wakeup_end_time_ms = 2000; // Wakeup end time (ms)
// Pass configuration when opening recorder
audio_recorder_open(&recorder_cfg);
```
### Kconfig Configuration (Alternative)
If fields in `audio_recorder_config_t.afe_config` use default values or are not set, Kconfig configuration will be used. Configure via `idf.py menuconfig`:
#### 💾 Media Dump Configuration
Configure under **"Component config" -> "Audio/Video Processor Configuration" -> "Media Dump"** in `menuconfig`:
- **`MEDIA_DUMP_ENABLE`**: Enable media data dump functionality, disabled by default
- When enabled, raw audio/video data can be saved for debugging and analysis
- **`MEDIA_DUMP_AUDIO_POINT`**: Audio save point selection (choice type)
- **`MEDIA_DUMP_AUDIO_BEFORE_AEC`**: Save raw audio before AEC processing
- Useful for analyzing noise and feedback issues
- **`MEDIA_DUMP_AUDIO_AFTER_AEC`**: Save audio after AEC processing
- Useful for evaluating AEC processing effectiveness
- **`MEDIA_DUMP_AUDIO_NONE`**: Do not save audio (default)
- **`MEDIA_DUMP_DURATION_SEC`**: Save duration in seconds, default 20
- Type: integer (int)
- Range: 1-3600 seconds
- **`MEDIA_DUMP_SINK`**: Save method selection (choice type)
- **`MEDIA_DUMP_SINK_SDCARD`**: Save to SD card file (default)
- File path configuration: `CONFIG_MEDIA_DUMP_SDCARD_FILENAME` (default: `/sdcard/media_dump.bin`)
- Ensure SD card is properly mounted
- **`MEDIA_DUMP_SINK_UDP`**: Send via UDP
- Target IP configuration: `CONFIG_MEDIA_DUMP_UDP_IP` (default: `192.168.1.100`)
- Target port configuration: `CONFIG_MEDIA_DUMP_UDP_PORT` (default: 5000)
- Can use `script/udp_reciver.py` script to receive data
### Task Configuration
The audio processing component uses multiple FreeRTOS tasks internally, which can be customized via task configuration fields in each module's configuration structure:
#### Task Configuration for Each Module
- **Recorder** (`audio_recorder_config_t`):
- `afe_feed_task_config`: AFE feed task configuration (default stack size 3KB)
- `afe_fetch_task_config`: AFE fetch task configuration (default stack size 3KB)
- `recorder_task_config`: Recorder task configuration (default stack size 5KB, recommend >= 40KB when using OPUS encoder)
- **Player** (`audio_playback_config_t`):
- `playback_task_config`: Playback task configuration (default stack size 4KB)
- **Feeder** (`audio_feeder_config_t`):
- `feeder_task_config`: Feeder task configuration (default stack size 5KB, recommend >= 40KB when using OPUS decoder)
**Important Notes**:
- When using OPUS encoder or decoder, `task_stack` of `recorder_task_config` and `feeder_task_config` should be set to at least `4096 * 10` bytes (40KB).
- When `task_stack` in task configuration is set to 0, default values will be used.
- When `task_stack_in_ext` is set to `true`, task stack will be allocated in external memory, helping to save internal RAM.
- Task configuration should be set in each module's configuration structure (e.g., `audio_recorder_config_t`, `audio_playback_config_t`, `audio_feeder_config_t`), not in `audio_manager_config_t`.
---
## API Reference 📚
Header files:
- `include/audio_processor.h`
- `include/video_processor.h`
- `include/av_processor_type.h`
Common functions (selection):
- Audio management:
- `audio_manager_init`/`audio_manager_deinit`: Initialize/deinitialize audio manager
- Recording:
- `audio_recorder_open`: Open recorder (supports encoder configuration and event callback)
- `audio_recorder_read_data`: Read recording data
- `audio_recorder_pause`/`audio_recorder_resume`: Pause/resume recording
- `audio_recorder_get_afe_manager_handle`: Get AFE manager handle
- `audio_recorder_close`: Close recorder
- Feeder:
- `audio_feeder_open`: Open feeder (supports decoder configuration)
- `audio_feeder_run`: Start feeder
- `audio_feeder_feed_data`: Feed audio data
- `audio_feeder_stop`: Stop feeder
- `audio_feeder_close`: Close feeder
- Playback:
- `audio_playback_open`/`audio_playback_close`: Open/close player
- `audio_playback_play`: Play audio (supports URL or local file, e.g., `http://...` or `file:///sdcard/...`)
- `audio_playback_stop`: Stop playback
- `audio_playback_pause`/`audio_playback_resume`: Pause/resume playback
- `audio_playback_get_state`: Get playback state
- Mixer:
- `audio_processor_mixer_open`: Open mixer (must be called after both playback and feeder are opened)
- `audio_processor_mixer_close`: Close mixer
- `audio_processor_ramp_control`: Control audio focus and volume fade
- Video rendering:
- `video_render_open`/`video_render_close`: Open/close video renderer
- `video_render_start`/`video_render_stop`: Start/stop rendering
- `video_render_frame_feed`: Feed video frame
- Video capture:
- `video_capture_open`/`video_capture_close`: Open/close video capturer
- `video_capture_start`/`video_capture_stop`: Start/stop capture
---
## FAQ ❓
### Audio Issues
- **"Self-questioning" phenomenon occurs (device playback sound is repeatedly captured by microphone)**:
- Enable `MEDIA_DUMP_ENABLE` in `menuconfig`, reproduce the issue and export saved audio data for further analysis
- Open the exported audio file using [Audacity](https://www.audacityteam.org/download/), observe waveform/spectrum to determine if saturation or feedback exists
- If saturation clipping occurs, reduce microphone gain (`esp_codec_dev_set_in_gain`); or appropriately reduce speaker volume (`esp_codec_dev_set_out_vol`) to avoid excessive feedback
- **Stack overflow when using OPUS encoder/decoder**:
- Ensure `task_stack` of `recorder_task_config` and `feeder_task_config` is set to at least `4096 * 10` bytes (40KB)
- Consider allocating task stack in external memory (set `task_stack_in_ext = true`)
- **Mixer not working properly**:
- Ensure that both player (`audio_playback_open`) and feeder (`audio_feeder_open`) are opened before calling `audio_processor_mixer_open()`
- Ensure `enable_mixer = true` is set when calling `audio_manager_init`
### Development and Debugging
- **Modifying component source code (espressif__av_processor)**:
- If you need to modify code within `espressif__av_processor`, you can execute the `script/move_av_processor.py` script
- The script will copy `managed_components/espressif__av_processor` to your project's `PROJECT_DIR/components/espressif__av_processor` directory
- The script will automatically delete the original directory under `managed_components` and update `idf_component.yml`
- Before using, set the environment variable: `export PROJECT_DIR=/path/to/project`
- **Media data dump (Media Dump)**:
- UDP method: Can use `script/udp_reciver.py` script to receive UDP data
- SD card method: Ensure SD card is properly mounted and file path is accessible
---
## Version Information
Current version: v0.5.0
## License
This component follows the Apache-2.0 license. See the `LICENSE` file included in the repository for details.
## Related Resources
- Project repository: https://github.com/espressif/esp-gmf/tree/main/components/av_processor
- Issue reporting: https://github.com/espressif/esp-gmf/issues
- Documentation: https://github.com/espressif/esp-gmf/blob/main/components/av_processor/README.md
@@ -0,0 +1,346 @@
# AV Processor 组件(音视频处理)
本组件提供在 ESP 平台上进行音频录制/播放/喂数以及视频捕获/解码/渲染的统一封装,基于 GMF(General Media Framework)、esp-audio-render、esp-video-codec、esp-capture 等中间组件实现。
## 功能概览 🎛️
- **音频 🎧**:
- 录音(AEC/VAD/AFE 可选)
- 播放(支持 URL、本地文件,解码由配置决定)
- 数据喂入(feeder,适合边接收边播放的场景,如 RTC/流媒体)
- 可选混音器,多路流间音量渐变(fade)与焦点控制
- **视频 🎥**:
- 多路视频采集
- 视频解码与渲染回调
---
## 快速开始
### 1. 音频使用
1. 初始化音频管理器:
```c
// 方式一:使用默认配置宏(推荐)
audio_manager_config_t cfg = DEFAULT_AUDIO_MANAGER_CONFIG();
cfg.play_dev = your_play_dev;
cfg.rec_dev = your_rec_dev;
strcpy(cfg.mic_layout, "RMNM"); // 麦克风布局(可选,用于 AFE 配置)
cfg.board_sample_rate = 16000;
cfg.board_bits = 32;
cfg.board_channels = 2;
cfg.play_volume = 80;
cfg.rec_volume = 60;
cfg.rec_ref_volume = 60;
cfg.enable_mixer = true; // 需要 feeder 与 playback 混音时开启
audio_manager_init(&cfg);
```
2. 录音(可轮询读取):
```c
// 录音事件回调(可选)
void recorder_event_cb(void *event, void *ctx) {
// 处理录音事件(如 AFE 事件)
}
// 配置录音器(使用默认配置宏)
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
// 可选:配置 AFE(运行时配置,优先级高于 Kconfig)
recorder_cfg.afe_config.vad_enable = true;
recorder_cfg.afe_config.vad_mode = 4;
recorder_cfg.afe_config.agc_enable = true;
recorder_cfg.recorder_event_cb = recorder_event_cb;
// 打开录音器
audio_recorder_open(&recorder_cfg);
uint8_t buf[2048];
audio_recorder_read_data(buf, sizeof(buf));
audio_recorder_close();
```
3. 喂数播放(feeder,适合外部码流→本地播放):
```c
// 配置 feeder(使用默认配置宏)
audio_feeder_config_t feeder_cfg = DEFAULT_AUDIO_FEEDER_CONFIG();
// 可选:使用 OPUS 解码器时,task_stack 建议 >= 4096 * 10
// feeder_cfg.feeder_task_config.task_stack = 4096 * 10;
audio_feeder_open(&feeder_cfg);
audio_feeder_run();
// 按需多次喂入外部数据块
audio_feeder_feed_data(pkt, pkt_len);
audio_feeder_stop();
audio_feeder_close();
```
4. 普通播放(URL 或本地):
```c
// 配置播放器(使用默认配置宏)
audio_playback_config_t playback_cfg = DEFAULT_AUDIO_PLAYBACK_CONFIG();
audio_playback_open(&playback_cfg);
audio_playback_play("http://<ip>:<port>/audio.mp3");
audio_playback_pause();
audio_playback_resume();
audio_playback_stop();
audio_playback_close();
```
5. 提示播放
```c
// 配置播放器(使用默认配置宏)
audio_prompt_config_t prompt_cfg = DEFAULT_AUDIO_PROMPT_CONFIG();
audio_prompt_open(&prompt_cfg);
audio_prompt_play("spiffs://audio.mp3");
audio_prompt_close();
```
> 提示播放 和 配置播放器是相似的,只是 `提示播放` 是阻塞式的播放,适合使用短音频的播放。目的是在使用 普通播放 的播放的时候又需要播放
> 提示音,可以有很好的相应
6. 焦点/渐变控制(启用混音器时):
```c
// 注意:使用混音器前需要先打开播放器和 feeder
audio_processor_mixer_open();
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_FEEDER); // 侧重喂数音频
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_PLAYBACK); // 侧重音频播放
audio_processor_mixer_close();
```
### 2. 视频使用
1. 渲染(解码)回调:
```c
void decoded_cb(void *ctx, const uint8_t *data, size_t size) {
// 处理解码后的帧数据
}
video_render_config_t rcfg = {
.decode_cfg = your_vdec_cfg,
.resolution = {.width = 640, .height = 480},
.decode_cb = decoded_cb,
};
video_render_handle_t r = video_render_open(&rcfg);
video_render_start(r);
video_frame_t f = {.data = enc_frame, .size = enc_size};
video_render_frame_feed(r, &f);
video_render_stop(r);
video_render_close(r);
```
1. 采集(多路):
```c
void capture_cb(void *ctx, int index, video_frame_t *frame) {
// 处理采集到的视频帧
}
video_capture_config_t ccfg = {0};
ccfg.camera_config = &your_cam_cfg;
ccfg.sink_num = 2;
ccfg.sink_cfg[0] = your_sink0_cfg;
ccfg.sink_cfg[1] = your_sink1_cfg;
ccfg.capture_frame_cb = capture_cb;
video_capture_handle_t c = video_capture_open(&ccfg);
video_capture_start(c);
video_capture_stop(c);
video_capture_close(c);
```
---
## 媒体数据保存(Media Dump)💾
- 使能条件 ✅:在 `menuconfig` 勾选 `MEDIA_DUMP_ENABLE` 后生效。
- 保存方式 📤:支持 SD 卡与 UDP 两种输出(可二选一)。
- SD 卡 💾:开启 `CONFIG_MEDIA_DUMP_SINK_SDCARD`,输出到文件 `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`,时长由 `CONFIG_MEDIA_DUMP_DURATION_SEC` 控制。
- UDP 📡:开启 `CONFIG_MEDIA_DUMP_SINK_UDP`,通过 `CONFIG_MEDIA_DUMP_UDP_IP` 与 `CONFIG_MEDIA_DUMP_UDP_PORT` 发送原始媒体数据(可使用 `script/udp_reciver.py` 脚本)。
- 保存点位 🎚️(AEC 处理前/后):在 `menuconfig` 的“Audio Save Mode (AEC point)”中选择
- `MEDIA_DUMP_AUDIO_BEFORE_AEC`(Save Before AEC):保存 AEC 前的麦克风原始音频(便于分析底噪/回授)
- `MEDIA_DUMP_AUDIO_AFTER_AEC`(Save After AEC):保存 AEC 处理后的音频(便于评估 AEC 效果)
- 典型用途 🔍:问题复现时导出原始音/视频数据,离线用 Audacity 等工具检查饱和、回授、格式是否正确。
## 配置说明 ⚙️
### 运行时配置(推荐)
AFE 配置可以通过 `audio_recorder_config_t` 结构体中的 `afe_config` 字段在运行时配置,优先级高于 Kconfig 配置:
```c
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
// 配置 AFE
recorder_cfg.afe_config.ai_mode_wakeup = false; // AI 模式:true=唤醒模式,false=直连模式
recorder_cfg.afe_config.vad_enable = true; // 使能 VAD
recorder_cfg.afe_config.vad_mode = 4; // VAD 模式(1-4),值越大越敏感
recorder_cfg.afe_config.vad_min_speech_ms = 64; // 最小语音时长(ms)
recorder_cfg.afe_config.vad_min_noise_ms = 1000; // 最小噪声时长(ms)
recorder_cfg.afe_config.agc_enable = true; // 使能 AGC
recorder_cfg.afe_config.enable_vcmd_detect = false; // 使能 VCMD
recorder_cfg.afe_config.vcmd_timeout_ms = 5000; // VCMD 超时(ms)
recorder_cfg.afe_config.mn_language = "cn"; // 模型语言:"cn" 或 "en"
recorder_cfg.afe_config.wakeup_time_ms = 10000; // 唤醒时间(ms)
recorder_cfg.afe_config.wakeup_end_time_ms = 2000; // 唤醒结束时间(ms)
// 打开录音器时传入配置
audio_recorder_open(&recorder_cfg);
```
### Kconfig 配置(备选)
如果 `audio_recorder_config_t.afe_config` 中的字段使用默认值或未设置,则使用 Kconfig 配置。通过 `idf.py menuconfig` 进入配置菜单进行配置。
#### 💾 Media Dump 配置
在 `menuconfig` 的 **"Component config" -> "Audio/Video Processor Configuration" -> "Media Dump"** 下配置:
- **`MEDIA_DUMP_ENABLE`**:使能媒体数据保存功能,默认关闭
- 启用后可以保存原始音频/视频数据用于调试分析
- **`MEDIA_DUMP_AUDIO_POINT`**:音频保存点位选择(choice 类型)
- **`MEDIA_DUMP_AUDIO_BEFORE_AEC`**:保存 AEC 处理前的原始音频
- 便于分析底噪、回授等问题
- **`MEDIA_DUMP_AUDIO_AFTER_AEC`**:保存 AEC 处理后的音频
- 便于评估 AEC 处理效果
- **`MEDIA_DUMP_AUDIO_NONE`**:不保存音频(默认)
- **`MEDIA_DUMP_DURATION_SEC`**:保存时长(秒),默认 20
- 类型:整数(int)
- 范围:1-3600 秒
- **`MEDIA_DUMP_SINK`**:保存方式选择(choice 类型)
- **`MEDIA_DUMP_SINK_SDCARD`**:保存到 SD 卡文件(默认)
- 文件路径配置项:`CONFIG_MEDIA_DUMP_SDCARD_FILENAME`(默认:`/sdcard/media_dump.bin`)
- 需要确保 SD 卡已正确挂载
- **`MEDIA_DUMP_SINK_UDP`**:通过 UDP 发送
- 目标 IP 配置项:`CONFIG_MEDIA_DUMP_UDP_IP`(默认:`192.168.1.100`)
- 目标端口配置项:`CONFIG_MEDIA_DUMP_UDP_PORT`(默认:5000)
- 可使用 `script/udp_reciver.py` 脚本接收数据
### 任务配置说明
音频处理组件内部使用多个 FreeRTOS 任务,可以通过各功能模块的配置结构体中的任务配置字段进行自定义:
#### 各功能模块任务配置
- **录音器**(`audio_recorder_config_t`):
- `afe_feed_task_config`:AFE feed 任务配置(默认栈大小 3KB)
- `afe_fetch_task_config`:AFE fetch 任务配置(默认栈大小 3KB)
- `recorder_task_config`:录音任务配置(默认栈大小 5KB,使用 OPUS 编码器时建议 >= 40KB)
- **播放器**(`audio_playback_config_t`):
- `playback_task_config`:播放任务配置(默认栈大小 4KB)
- **Feeder**(`audio_feeder_config_t`):
- `feeder_task_config`:Feeder 任务配置(默认栈大小 5KB,使用 OPUS 解码器时建议 >= 40KB)
**重要提示**:
- 当使用 OPUS 编码器或解码器时,`recorder_task_config` 和 `feeder_task_config` 的 `task_stack` 需要设置为至少 `4096 * 10` 字节(40KB)。
- 任务配置中的 `task_stack` 设置为 0 时,将使用默认值。
- `task_stack_in_ext` 设置为 `true` 时,任务栈将分配在外部内存中,有助于节省内部 RAM。
- 任务配置应在各自模块的配置结构体中设置(如 `audio_recorder_config_t`、`audio_playback_config_t`、`audio_feeder_config_t`),而不是在 `audio_manager_config_t` 中。
---
## API 参考 📚
头文件:
- `include/audio_processor.h`
- `include/video_processor.h`
- `include/av_processor_type.h`
常用函数(节选):
- 音频管理:
- `audio_manager_init`/`audio_manager_deinit`:初始化/反初始化音频管理器
- 录音:
- `audio_recorder_open`:打开录音器(支持编码器配置和事件回调)
- `audio_recorder_read_data`:读取录音数据
- `audio_recorder_pause`/`audio_recorder_resume`:暂停/恢复录音
- `audio_recorder_get_afe_manager_handle`:获取 AFE manager 句柄
- `audio_recorder_close`:关闭录音器
- 喂数:
- `audio_feeder_open`:打开 feeder(支持解码器配置)
- `audio_feeder_run`:启动 feeder
- `audio_feeder_feed_data`:喂入音频数据
- `audio_feeder_stop`:停止 feeder
- `audio_feeder_close`:关闭 feeder
- 播放:
- `audio_playback_open`/`audio_playback_close`:打开/关闭播放器
- `audio_playback_play`:播放音频(支持 URL 或本地文件,如 `http://...` 或 `file:///sdcard/...`)
- `audio_playback_stop`:停止播放
- `audio_playback_pause`/`audio_playback_resume`:暂停/恢复播放
- `audio_playback_get_state`:获取播放状态
- 混音器:
- `audio_processor_mixer_open`:打开混音器(必须在 playback 和 feeder 都打开后调用)
- `audio_processor_mixer_close`:关闭混音器
- `audio_processor_ramp_control`:控制音频焦点和音量渐变
- 视频渲染:
- `video_render_open`/`video_render_close`:打开/关闭视频渲染器
- `video_render_start`/`video_render_stop`:启动/停止渲染
- `video_render_frame_feed`:喂入视频帧
- 视频采集:
- `video_capture_open`/`video_capture_close`:打开/关闭视频采集器
- `video_capture_start`/`video_capture_stop`:启动/停止采集
---
## 常见问题 ❓
### 音频问题
- **出现"自问自答"现象(设备播放的声音被麦克风重复收录)**:
- 在 `menuconfig` 使能 `MEDIA_DUMP_ENABLE`,复现问题并导出保存的音频数据以便进一步分析
- 使用 [Audacity](https://www.audacityteam.org/download/) 打开导出的音频文件,观察波形/频谱判断是否存在饱和或回授
- 如出现饱和剪切,调低麦克风增益(`esp_codec_dev_set_in_gain`);或适当减小扬声器音量(`esp_codec_dev_set_out_vol`),避免过度回授
- **OPUS 编解码器使用时出现栈溢出**:
- 确保 `recorder_task_config` 和 `feeder_task_config` 的 `task_stack` 设置为至少 `4096 * 10` 字节(40KB)
- 考虑将任务栈分配在外部内存(设置 `task_stack_in_ext = true`)
- **混音器无法正常工作**:
- 确保在调用 `audio_processor_mixer_open()` 之前,已经打开了播放器(`audio_playback_open`)和 feeder(`audio_feeder_open`)
- 确保在 `audio_manager_init` 时设置了 `enable_mixer = true`
### 开发调试
- **修改组件源码(espressif__av_processor)**:
- 如需修改 `espressif__av_processor` 内的代码,可执行 `script/move_av_processor.py` 脚本
- 脚本会将 `managed_components/espressif__av_processor` 复制到你的工程 `PROJECT_DIR/components/espressif__av_processor` 下
- 脚本会自动删除 `managed_components` 下的原目录,并更新 `idf_component.yml`
- 使用前需要设置环境变量:`export PROJECT_DIR=/path/to/project`
- **媒体数据保存(Media Dump)**:
- UDP 方式:可使用 `script/udp_reciver.py` 脚本接收 UDP 数据
- SD 卡方式:确保 SD 卡已正确挂载,文件路径可访问
---
## 版本信息
当前版本:v0.5.0
## 许可证
本组件遵循 Apache-2.0 许可证,详见仓库随附的 `LICENSE`。
## 相关资源
- 项目仓库:https://github.com/espressif/esp-gmf/tree/main/components/av_processor
- 问题反馈:https://github.com/espressif/esp-gmf/issues
- 文档:https://github.com/espressif/esp-gmf/blob/main/components/av_processor/README.md
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,182 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "esp_err.h"
#include "esp_log.h"
#include "esp_audio_dec_default.h"
#include "esp_audio_enc_default.h"
#include "esp_audio_simple_dec_default.h"
#include "esp_gmf_audio_dec.h"
#include "esp_gmf_audio_enc.h"
#include "esp_gmf_ch_cvt.h"
#include "esp_gmf_bit_cvt.h"
#include "esp_gmf_rate_cvt.h"
#include "esp_gmf_pool.h"
#include "esp_gmf_element.h"
#include "esp_gmf_fade.h"
#include "esp_gmf_err.h"
#include "av_processor_type.h"
static char *TAG = "AV_PIPELINE_POOL";
esp_err_t av_audio_playback_pool_init(esp_gmf_pool_handle_t *pool)
{
*pool = NULL;
esp_gmf_err_t ret = ESP_GMF_ERR_OK;
esp_gmf_element_handle_t hd = NULL;
esp_err_t err = esp_gmf_pool_init(pool);
ESP_GMF_RET_ON_NOT_OK(TAG, err, {return err;}, "Failed to initialize GMF pool");
esp_audio_simple_dec_cfg_t es_dec_cfg = DEFAULT_ESP_GMF_AUDIO_DEC_CONFIG();
esp_gmf_audio_dec_init(&es_dec_cfg, &hd);
esp_gmf_pool_register_element(*pool, hd, NULL);
return ret;
}
esp_err_t av_audio_playback_pool_deinit(esp_gmf_pool_handle_t pool)
{
return esp_gmf_pool_deinit(pool);
}
esp_err_t av_audio_feeder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_decoder_config_t *decoder_cfg)
{
*pool = NULL;
esp_err_t ret = ESP_OK;
esp_audio_simple_dec_cfg_t dec_cfg = DEFAULT_ESP_GMF_AUDIO_DEC_CONFIG();
esp_gmf_element_handle_t el = NULL;
esp_gmf_pool_init(pool);
// Default to PCM if no config provided
av_processor_format_id_t format = AV_PROCESSOR_FORMAT_ID_PCM;
if (decoder_cfg) {
format = decoder_cfg->format;
}
switch (format) {
case AV_PROCESSOR_FORMAT_ID_PCM:
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_PCM;
break;
case AV_PROCESSOR_FORMAT_ID_G711A: {
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_G711A;
esp_g711_dec_cfg_t g711_dec_cfg = {0};
g711_dec_cfg.channel = decoder_cfg ? decoder_cfg->params.g711.audio_info.channels : 1;
dec_cfg.dec_cfg = &g711_dec_cfg;
dec_cfg.cfg_size = sizeof(esp_g711_dec_cfg_t);
break;
}
case AV_PROCESSOR_FORMAT_ID_G711U: {
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_G711U;
esp_g711_dec_cfg_t g711_dec_cfg = {0};
g711_dec_cfg.channel = decoder_cfg ? decoder_cfg->params.g711.audio_info.channels : 1;
dec_cfg.dec_cfg = &g711_dec_cfg;
dec_cfg.cfg_size = sizeof(esp_g711_dec_cfg_t);
break;
}
case AV_PROCESSOR_FORMAT_ID_OPUS: {
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_RAW_OPUS;
esp_opus_dec_cfg_t opus_dec_cfg = ESP_OPUS_DEC_CONFIG_DEFAULT();
if (decoder_cfg) {
// Convert frame_duration from int to esp_opus_enc_frame_duration_t
int frame_duration = decoder_cfg->params.opus.audio_info.frame_duration;
switch (frame_duration) {
case 2: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_2_5_MS; break;
case 5: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_5_MS; break;
case 10: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_10_MS; break;
case 20: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_20_MS; break;
case 40: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_40_MS; break;
case 60: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_60_MS; break;
case 80: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_80_MS; break;
case 100: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_100_MS; break;
case 120: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_120_MS; break;
default: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_60_MS; break;
}
opus_dec_cfg.channel = decoder_cfg->params.opus.audio_info.channels;
opus_dec_cfg.sample_rate = decoder_cfg->params.opus.audio_info.sample_rate;
}
dec_cfg.dec_cfg = &opus_dec_cfg;
dec_cfg.cfg_size = sizeof(esp_opus_dec_cfg_t);
break;
}
case AV_PROCESSOR_FORMAT_ID_AAC:
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_AAC;
esp_aac_dec_cfg_t aac_dec_cfg = {0};
aac_dec_cfg.no_adts_header = decoder_cfg->params.aac.no_adts_header;
aac_dec_cfg.aac_plus_enable = decoder_cfg->params.aac.aac_plus_enable;
dec_cfg.dec_cfg = &aac_dec_cfg;
dec_cfg.cfg_size = sizeof(esp_aac_dec_cfg_t);
break;
default:
ESP_LOGE(TAG, "Unsupported format: %d", format);
return ESP_ERR_NOT_SUPPORTED;
}
esp_gmf_audio_dec_init(&dec_cfg, &el);
esp_gmf_pool_register_element(*pool, el, NULL);
esp_ae_ch_cvt_cfg_t es_ch_cvt_cfg = DEFAULT_ESP_GMF_CH_CVT_CONFIG();
ret = esp_gmf_ch_cvt_init(&es_ch_cvt_cfg, &el);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio ch cvt");
ret = esp_gmf_pool_register_element(*pool, el, NULL);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
esp_ae_bit_cvt_cfg_t es_bit_cvt_cfg = DEFAULT_ESP_GMF_BIT_CVT_CONFIG();
ret = esp_gmf_bit_cvt_init(&es_bit_cvt_cfg, &el);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio bit cvt");
ret = esp_gmf_pool_register_element(*pool, el, NULL);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
esp_ae_rate_cvt_cfg_t es_rate_cvt_cfg = DEFAULT_ESP_GMF_RATE_CVT_CONFIG();
esp_gmf_rate_cvt_init(&es_rate_cvt_cfg, &el);
esp_gmf_pool_register_element(*pool, el, NULL);
return ret;
}
esp_err_t av_audio_feeder_pool_deinit(esp_gmf_pool_handle_t pool)
{
return esp_gmf_pool_deinit(pool);
}
esp_err_t av_audio_recorder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_encoder_config_t *encoder_cfg)
{
*pool = NULL;
esp_gmf_err_t ret = ESP_GMF_ERR_OK;
esp_audio_enc_config_t enc_cfg = DEFAULT_ESP_GMF_AUDIO_ENC_CONFIG();
esp_gmf_element_handle_t el = NULL;
esp_err_t err = esp_gmf_pool_init(pool);
ESP_GMF_RET_ON_NOT_OK(TAG, err, {return err;}, "Failed to initialize GMF pool");
esp_gmf_audio_enc_init(&enc_cfg, &el);
esp_gmf_pool_register_element(*pool, el, NULL);
esp_ae_ch_cvt_cfg_t es_ch_cvt_cfg = DEFAULT_ESP_GMF_CH_CVT_CONFIG();
ret = esp_gmf_ch_cvt_init(&es_ch_cvt_cfg, &el);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio ch cvt");
ret = esp_gmf_pool_register_element(*pool, el, NULL);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
esp_ae_bit_cvt_cfg_t es_bit_cvt_cfg = DEFAULT_ESP_GMF_BIT_CVT_CONFIG();
ret = esp_gmf_bit_cvt_init(&es_bit_cvt_cfg, &el);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio bit cvt");
ret = esp_gmf_pool_register_element(*pool, el, NULL);
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
esp_ae_rate_cvt_cfg_t es_rate_cvt_cfg = DEFAULT_ESP_GMF_RATE_CVT_CONFIG();
esp_gmf_rate_cvt_init(&es_rate_cvt_cfg, &el);
esp_gmf_pool_register_element(*pool, el, NULL);
return err;
}
esp_err_t av_audio_recorder_pool_deinit(esp_gmf_pool_handle_t pool)
{
return esp_gmf_pool_deinit(pool);
}
@@ -0,0 +1,31 @@
dependencies:
espressif/gmf_ai_audio:
version: '~0.7'
public: true
rules:
- if: "target in [esp32p4, esp32s3, esp32]"
espressif/esp_audio_simple_player:
version: '^0.9.4'
public: true
espressif/gmf_loader:
version: "^0.7"
public: true
espressif/esp_video_codec:
public: true
version: "^0.5"
espressif/esp_capture:
public: true
version: "^0.7"
espressif/esp_audio_render:
public: true
version: "^0.7.0"
description: Audio/Video Processor module for ESP-ADF
documentation: https://github.com/espressif/esp-adf/blob/master/components/av_processor/README.md
issues: https://github.com/espressif/esp-adf/issues
repository: git://github.com/espressif/esp-adf.git
tags:
- gmf_multimedia
- av_processor
url: https://github.com/espressif/esp-adf/tree/master/components/av_processor
version: 0.1.0
@@ -0,0 +1,488 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
#include "esp_err.h"
#include "esp_gmf_afe_manager.h"
#include "esp_audio_simple_player_advance.h"
#include "av_processor_type.h"
/* Brief: Default audio manager configuration
* Note: When use OPUS decoder or OPUS encoder, the recorder_task_config and feeder_task_config
* task stack size need be set to more than 4096 * 10 bytes.
*/
#define DEFAULT_AUDIO_MANAGER_CONFIG() { \
.play_dev = NULL, \
.rec_dev = NULL, \
.mic_layout = {0}, \
.board_sample_rate = 16000, \
.board_bits = 32, \
.board_channels = 2, \
}
#define DEFAULT_AUDIO_FEEDER_CONFIG() { \
.feeder_task_config = { \
.task_stack = 0, \
.task_prio = 0, \
.task_core = 0, \
.task_stack_in_ext = true, \
}, \
.feeder_mixer_gain = { \
.initial_gain = 0.0, \
.target_gain = 0.5, \
.transition_time = 1500 \
}, \
}
#define DEFAULT_AUDIO_PLAYBACK_CONFIG() { \
.playback_task_config = { \
.task_stack = 4096, \
.task_prio = 5, \
.task_core = 0, \
.task_stack_in_ext = true, \
}, \
.playback_mixer_gain = { \
.initial_gain = 0.0, \
.target_gain = 1.0, \
.transition_time = 1500 \
} \
}
#define DEFAULT_AUDIO_PROMPT_CONFIG() { \
.prompt_task_config = { \
.task_stack = 4096, \
.task_prio = 5, \
.task_core = 0, \
.task_stack_in_ext = true \
} \
}
#define DEFAULT_AUDIO_RECORDER_CONFIG() { \
.afe_feed_task_config = { \
.task_stack = 40 * 1024, \
.task_prio = 5, \
.task_core = 0, \
.task_stack_in_ext = true, \
}, \
.afe_fetch_task_config = { \
.task_stack = 6 * 1024, \
.task_prio = 5, \
.task_core = 1, \
.task_stack_in_ext = true \
}, \
.recorder_task_config = { \
.task_stack = 4096, \
.task_prio = 5, \
.task_core = 0, \
.task_stack_in_ext = true \
}, \
.recorder_event_cb = NULL, \
.recorder_ctx = NULL, \
}
/**
* @brief Default AFE configuration macro
*/
#define DEFAULT_AV_PROCESSOR_AFE_CONFIG() { \
.model = "model", \
.ai_mode_wakeup = false, \
.vad_enable = true, \
.vad_mode = 4, \
.vad_min_speech_ms = 64, \
.vad_min_noise_ms = 1000, \
.agc_enable = true, \
.afe_type = AFE_TYPE_SR, \
.enable_vcmd_detect = false, \
.vcmd_timeout_ms = 5000, \
.mn_language = "cn", \
.wakeup_end_time_ms = 2000, \
.wakeup_time_ms = 10000 \
}
/**
* @brief Type definition for the audio recorder event callback function
*
* @param[in] event Pointer to the event data, which can contain information about the recorder's state
* @param[in] ctx User-defined context pointer, passed when registering the callback
*/
typedef void (*recorder_event_callback_t)(void *event, void *ctx);
/**
* @brief AFE (Audio Front-End) configuration structure
*
* This structure contains all AFE-related configuration parameters that were previously
* configured via Kconfig. These parameters control the behavior of the audio front-end
* processing including wake word detection, VAD, AGC, and other audio processing features.
*/
typedef struct {
const char *model; /*!< Model path/name for SR model initialization */
bool ai_mode_wakeup; /*!< AI mode: true for WAKEUP mode, false for DIRECT mode */
bool vad_enable; /*!< Enable Voice Activity Detection (VAD) */
int vad_mode; /*!< VAD mode value (1-4), larger values are more sensitive to human voice */
int vad_min_speech_ms; /*!< Minimum speech duration in milliseconds */
int vad_min_noise_ms; /*!< Minimum noise duration in milliseconds */
bool agc_enable; /*!< Enable Automatic Gain Control (AGC) */
afe_type_t afe_type; /*!< AFE type: AFE_TYPE_SR (0) or AFE_TYPE_VC (1) */
bool enable_vcmd_detect; /*!< Enable Voice Command Detection */
int vcmd_timeout_ms; /*!< Voice command detection timeout in milliseconds */
const char *mn_language; /*!< Model language (e.g., "cn", "en") */
int wakeup_end_time_ms; /*!< Wakeup end time in milliseconds */
int wakeup_time_ms; /*!< Wakeup time in milliseconds */
} av_processor_afe_config_t;
/* Brief: Audio mixer gain configuration */
typedef struct {
float initial_gain; /*!< Initial gain value (0.0 - 1.0) */
float target_gain; /*!< Target gain value (0.0 - 1.0) */
int transition_time; /*!< Transition time in milliseconds */
} audio_mixer_gain_config_t;
/* Brief: Common task configuration */
typedef struct {
int task_stack; /*!< Task stack size (0 means use default) */
int task_prio; /*!< Task priority */
int task_core; /*!< Task core ID */
bool task_stack_in_ext; /*!< Whether to allocate task stack in external memory */
} audio_task_config_t;
typedef struct {
audio_task_config_t feeder_task_config; /*!< Feeder task configuration */
av_processor_decoder_config_t decoder_cfg; /*!< Decoder configuration */
audio_mixer_gain_config_t feeder_mixer_gain; /*!< Feeder mixer gain configuration */
} audio_feeder_config_t;
typedef struct {
audio_task_config_t playback_task_config; /*!< Playback task configuration */
audio_mixer_gain_config_t playback_mixer_gain; /*!< Playback mixer gain configuration */
} audio_playback_config_t;
typedef struct {
audio_task_config_t prompt_task_config; /*!< Prompt task configuration */
} audio_prompt_config_t;
typedef struct {
audio_task_config_t afe_feed_task_config; /*!< AFE feed task configuration */
av_processor_afe_config_t afe_config; /*!< AFE configuration */
audio_task_config_t afe_fetch_task_config; /*!< AFE fetch task configuration */
audio_task_config_t recorder_task_config; /*!< Recorder task configuration */
av_processor_encoder_config_t encoder_cfg; /*!< Encoder configuration */
recorder_event_callback_t recorder_event_cb; /*!< Recorder event callback */
void *recorder_ctx; /*!< Recorder context */
} audio_recorder_config_t;
/* Brief: Audio manager configuration */
typedef struct {
void *play_dev;
void *rec_dev;
char mic_layout[8];
int board_sample_rate;
int board_bits;
int board_channels;
} audio_manager_config_t;
/**
* @brief Audio focus control mode - determines which audio stream gets priority
*/
typedef enum {
AUDIO_MIXER_FOCUS_FEEDER = 0, /*!< Focus on FEEDER stream - boost feeder, reduce playback */
AUDIO_MIXER_FOCUS_PLAYBACK = 1, /*!< Focus on PLAYBACK stream - boost playback, reduce feeder */
AUDIO_MIXER_FOCUS_BALANCED = 2, /*!< Balanced mode - both streams at equal levels */
} audio_mixer_focus_mode_t;
enum audio_player_state_e {
AUDIO_RUN_STATE_IDLE,
AUDIO_RUN_STATE_PLAYING,
AUDIO_RUN_STATE_CLOSED,
AUDIO_RUN_STATE_PAUSED,
};
/**
* @brief Initializes the audio manager module.
*
* @param[in] config Pointer to the audio manager configuration structure
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_manager_init(audio_manager_config_t *config);
/**
* @brief Deinitializes the audio manager component
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_manager_deinit();
/**
* @brief Opens the audio recorder and registers an event callback
*
* @param[in] config Pointer to the audio recorder configuration structure.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_open(audio_recorder_config_t *config);
/**
* @brief Pauses the audio recorder system
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_pause(void);
/**
* @brief Resumes the audio recorder system
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_resume(void);
/**
* @brief Get the AFE manager handle of the audio recorder
*
* @param[out] afe_manager_handle Pointer to the AFE manager handle
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_get_afe_manager_handle(esp_gmf_afe_manager_handle_t *afe_manager_handle);
/**
* @brief Closes the audio recorder
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_close(void);
/**
* @brief Reads audio data from the recorder.
*
* @param[out] data Pointer to the buffer where the audio data will be stored
* @param[in] data_size Size of the buffer to store the audio data.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_recorder_read_data(uint8_t *data, int data_size);
/**
* @brief Opens the audio feeder system
*
* @param[in] config Pointer to the audio feeder configuration structure.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_feeder_open(audio_feeder_config_t *config);
/**
* @brief Closes the audio feeder module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_feeder_close(void);
/**
* @brief Starts the audio feeder operation.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_feeder_run(void);
/**
* @brief Stops the ongoing audio feeder
*
* @return
* - ESP_OK On successfully stopping the audio playback
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_feeder_stop(void);
/**
* @brief Feeds audio data into the feeder system
*
* @param[in] data Pointer to the audio data to be fed into the feeder system
* @param[in] data_size Size of the audio data to be fed into the feeder system
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_feeder_feed_data(uint8_t *data, int data_size);
/**
* @brief Open the audio playback module
*
* @param[in] config Pointer to the audio playback configuration structure.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_open(audio_playback_config_t *config);
/**
* @brief Close the audio playback module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_close(void);
/**
* @brief Play an audio playback from the specified URL. Eg: "http://192.168.1.100:8000/audio.mp3", "file:///sdcard/audio.mp3"
*
* @param[in] url URL pointing to the audio playback file to be played
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_play(const char *url);
/**
* @brief Wait for the audio playback to end. This function will block until the audio playback is ended.
*
* @param[in] url URL pointing to the audio playback file to be played
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_play_wait_end(const char *url);
/**
* @brief Pauses the audio playback module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_pause(void);
/**
* @brief Resumes the audio playback module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_resume(void);
/**
* @brief Get the state of the audio playback module
*
* @return
* - enum audio_player_state_e Playback state
*/
enum audio_player_state_e audio_playback_get_state(void);
/**
* @brief Stop the currently playing audio playback module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_playback_stop(void);
/**
* @brief Open the audio prompt module
* @note This function not provide pause and resume functionality, because the prompt is played to end automatically.
* This function's priority is higher than the audio playback and feeder, so when the prompt is playing, the audio playback
* and feeder decoded data will be ignored.
*
* @param[in] config Pointer to the audio prompt configuration structure.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_prompt_open(audio_prompt_config_t *config);
/**
* @brief Close the audio prompt module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_prompt_close(void);
/**
* @brief Play an audio prompt from the specified URL.
*
* @param[in] url URL pointing to the audio prompt file to be played
* @note Audio prompt is not supported mixer, so when play prompt, the audio playback and feeder decoded data will be ignored.
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_prompt_play(const char *url);
/**
* @brief Open the audio mixer module
*
* @note This function must be called after audio_playback_open and audio_feeder_open and the audio_mixer must be enabled
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_processor_mixer_open(void);
/**
* @brief Close the audio mixer module
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t audio_processor_mixer_close(void);
/**
* @brief Control audio focus between FEEDER and PLAYBACK streams
*
* @note This function controls the fade processors for both feeder and playback streams
* to create smooth volume transitions and focus on the desired audio source.
*
* @param[in] mode The focus control mode:
* - AUDIO_FOCUS_FEEDER: Focus on FEEDER stream (boost feeder, reduce playback)
* - AUDIO_FOCUS_PLAYBACK: Focus on PLAYBACK stream (boost playback, reduce feeder)
* - AUDIO_FOCUS_BALANCED: Balanced mode (both streams at equal levels)
*
* @return
* - ESP_OK On success
* - ESP_ERR_INVALID_ARG Invalid mode parameter
* - ESP_ERR_INVALID_STATE Audio render not initialized or streams not available
* - ESP_FAIL Failed to control fade processors
*/
esp_err_t audio_processor_ramp_control(audio_mixer_focus_mode_t mode);
#ifdef __cplusplus
}
#endif /* __cplusplus */
@@ -0,0 +1,145 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
#include <stdint.h>
#include <stdbool.h>
#include "esp_afe_config.h"
/**
* @brief AV Processor FourCC definition
*/
#define AV_PROCESSOR_FOURCC(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
/**
* @brief AV Processor format ID definition
*
* @note Aligned with GMF FourCC definition for audio video codecs and formats
* Detailed info refer to `https://github.com/espressif/esp-gmf/blob/main/gmf_core/helpers/include/esp_fourcc.h`
*/
typedef enum {
AV_PROCESSOR_FORMAT_ID_PCM = AV_PROCESSOR_FOURCC('P', 'C', 'M', ' '), /*!< PCM (Uncompressed) encoder */
AV_PROCESSOR_FORMAT_ID_G711A = AV_PROCESSOR_FOURCC('A', 'L', 'A', 'U'), /*!< G.711 A-law encoder */
AV_PROCESSOR_FORMAT_ID_G711U = AV_PROCESSOR_FOURCC('U', 'L', 'A', 'W'), /*!< G.711 μ-law encoder */
AV_PROCESSOR_FORMAT_ID_OPUS = AV_PROCESSOR_FOURCC('O', 'P', 'U', 'S'), /*!< OPUS encoder */
AV_PROCESSOR_FORMAT_ID_AAC = AV_PROCESSOR_FOURCC('A', 'A', 'C', ' '), /*!< AAC encoder */
} av_processor_format_id_t;
/**
* @brief Audio encoder format type
*/
typedef enum {
AV_PROCESSOR_ENCODER_PCM = 0, /*!< PCM (Uncompressed) encoder */
AV_PROCESSOR_ENCODER_G711A, /*!< G.711 A-law encoder */
AV_PROCESSOR_ENCODER_G711U, /*!< G.711 μ-law encoder */
AV_PROCESSOR_ENCODER_OPUS, /*!< OPUS encoder */
AV_PROCESSOR_ENCODER_MAX, /*!< Maximum encoder type */
} av_processor_encoder_format_t;
/**
* @brief Audio decoder format type
*/
typedef enum {
AV_PROCESSOR_DECODER_PCM = 0, /*!< PCM (Uncompressed) decoder */
AV_PROCESSOR_DECODER_G711A, /*!< G.711 A-law decoder */
AV_PROCESSOR_DECODER_G711U, /*!< G.711 μ-law decoder */
AV_PROCESSOR_DECODER_OPUS, /*!< OPUS decoder */
AV_PROCESSOR_DECODER_AAC, /*!< AAC decoder */
AV_PROCESSOR_DECODER_MAX, /*!< Maximum decoder type */
} av_processor_decoder_format_t;
typedef struct {
uint32_t sample_rate; /*!< Sample rate in Hz (e.g., 8000, 16000, 24000, 32000, 44100, 48000) */
uint8_t sample_bits; /*!< Sample bits in bits (e.g., 8, 16, 24, 32) */
uint8_t channels; /*!< Number of audio channels (1-4) */
int frame_duration; /*!< Frame duration in milliseconds */
} av_processor_audio_info_t;
/**
* @brief PCM encoder/decoder configuration
*/
typedef struct {
av_processor_audio_info_t audio_info;
} av_processor_pcm_config_t;
/**
* @brief OPUS encoder configuration
*/
typedef struct {
av_processor_audio_info_t audio_info;
bool enable_vbr; /*!< Enable Variable Bit Rate (VBR) */
int bitrate; /*!< Bitrate in bps */
} av_processor_opus_encoder_config_t;
/**
* @brief OPUS decoder configuration
*/
typedef struct {
av_processor_audio_info_t audio_info;
} av_processor_opus_decoder_config_t;
/**
* @brief AAC decoder configuration
*/
typedef struct {
av_processor_audio_info_t audio_info;
bool no_adts_header; /*!< Set if AAC frame data not contain ADTS header */
bool aac_plus_enable; /*!< Set if need AAC-Plus support */
} av_processor_aac_decoder_config_t;
/**
* @brief G.711 encoder/decoder configuration
*/
typedef struct {
av_processor_audio_info_t audio_info;
} av_processor_g711_config_t;
typedef struct {
av_processor_audio_info_t audio_info;
int bitrate; /*!< Bitrate in bps */
bool adts_used; /*!< Whether write ADTS header: true - add ADTS header, false - raw aac data only */
} av_processor_aac_encoder_config_t;
/**
* @brief Audio encoder configuration structure
*
* This structure contains the encoder format type and corresponding format-specific parameters.
* Only the parameters for the selected format (indicated by format) should be used.
*/
typedef struct {
av_processor_format_id_t format; /*!< Encoder format ID */
union {
av_processor_pcm_config_t pcm; /*!< PCM encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_PCM) */
av_processor_opus_encoder_config_t opus; /*!< OPUS encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_OPUS) */
av_processor_g711_config_t g711; /*!< G.711 encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_G711A or AV_PROCESSOR_FORMAT_ID_G711U) */
av_processor_aac_encoder_config_t aac; /*!< AAC encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_AAC) */
} params; /*!< Format-specific encoder parameters */
} av_processor_encoder_config_t;
/**
* @brief Audio decoder configuration structure
*
* This structure contains the decoder format type and corresponding format-specific parameters.
* Only the parameters for the selected format (indicated by format) should be used.
*/
typedef struct {
av_processor_format_id_t format; /*!< Decoder format ID */
union {
av_processor_pcm_config_t pcm; /*!< PCM decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_PCM) */
av_processor_opus_decoder_config_t opus; /*!< OPUS decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_OPUS) */
av_processor_g711_config_t g711; /*!< G.711 decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_G711A or AV_PROCESSOR_FORMAT_ID_G711U) */
av_processor_aac_decoder_config_t aac; /*!< AAC decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_AAC) */
} params; /*!< Format-specific decoder parameters */
} av_processor_decoder_config_t;
#ifdef __cplusplus
}
#endif /* __cplusplus */
@@ -0,0 +1,174 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
*
* See LICENSE file for details.
*/
#pragma once
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
#include "esp_capture_sink.h"
#include "esp_err.h"
#include "esp_video_dec.h"
/**
* @brief Default maximum number of video paths
*/
#define DEFAULT_VID_MAX_PATH 2
/**
* @brief Video capture handle type
*/
typedef void *video_capture_handle_t;
/**
* @brief Video render handle type
*/
typedef void *video_render_handle_t;
/**
* @brief Video frame structure
*/
typedef struct {
uint8_t *data; /*!< Frame data buffer */
size_t size; /*!< Frame data size */
} video_frame_t;
/**
* @brief Video render decode callback function type
*
* @param[in] ctx User context pointer
* @param[in] data Decoded frame data
* @param[in] size Size of decoded frame data
*/
typedef void (*video_render_decode_callback_t)(void *ctx, const uint8_t *data, size_t size);
/**
* @brief Video capture frame callback function type
*
* @param[in] ctx User context pointer
* @param[in] index Frame index
* @param[in] vid_frame Captured video frame
*/
typedef void (*video_capture_frame_callback_t)(void *ctx, int index, video_frame_t *vid_frame);
/**
* @brief Video render configuration structure
*/
typedef struct {
esp_video_dec_cfg_t decode_cfg; /*!< Video decoder configuration */
esp_video_codec_resolution_t resolution; /*!< Video resolution */
video_render_decode_callback_t decode_cb; /*!< Decode callback function */
void *decode_cb_ctx; /*!< Decode callback context */
} video_render_config_t;
/**
* @brief Video capture configuration structure
*/
typedef struct {
void *camera_config; /*!< Camera configuration */
int sink_num; /*!< Number of sinks */
esp_capture_sink_cfg_t sink_cfg[DEFAULT_VID_MAX_PATH]; /*!< Sink configurations */
video_capture_frame_callback_t capture_frame_cb; /*!< Capture frame callback function */
void *capture_frame_cb_ctx; /*!< Capture frame callback context */
} video_capture_config_t;
/**
* @brief Opens a video capture session
*
* @param[in] config Video capture configuration
*
* @return
* - video_capture_handle On success
* - NULL On failure
*/
video_capture_handle_t video_capture_open(video_capture_config_t *config);
/**
* @brief Closes a video capture session
*
* @param[in] handle Video capture handle
*/
void video_capture_close(video_capture_handle_t handle);
/**
* @brief Starts video capture
*
* @param[in] handle Video capture handle
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t video_capture_start(video_capture_handle_t handle);
/**
* @brief Stops video capture
*
* @param[in] handle Video capture handle
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t video_capture_stop(video_capture_handle_t handle);
/**
* @brief Opens a video render session
*
* @param[in] config Video render configuration
*
* @return
* - video_render_handle On success
* - NULL On failure
*/
video_render_handle_t video_render_open(video_render_config_t *config);
/**
* @brief Closes a video render session
*
* @param[in] handle Video render handle
*/
void video_render_close(video_render_handle_t handle);
/**
* @brief Starts video rendering
*
* @param[in] handle Video render handle
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t video_render_start(video_render_handle_t handle);
/**
* @brief Stops video rendering
*
* @param[in] handle Video render handle
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t video_render_stop(video_render_handle_t handle);
/**
* @brief Feeds a video frame to the render pipeline
*
* @param[in] handle Video render handle
* @param[in] frame Video frame to feed
*
* @return
* - ESP_OK On success
* - Other Appropriate esp_err_t error code on failure
*/
esp_err_t video_render_frame_feed(video_render_handle_t handle, video_frame_t *frame);
#ifdef __cplusplus
}
#endif /* __cplusplus */
@@ -0,0 +1,224 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
*
* See LICENSE file for details.
*/
#include <stdio.h>
#include <string.h>
#include <stdbool.h>
#include <stdint.h>
#include <stddef.h>
#include <errno.h>
#include <sys/types.h>
#include <unistd.h>
#include "esp_log.h"
#include "esp_timer.h"
#include "sdkconfig.h"
#include "media_dump.h"
#if CONFIG_MEDIA_DUMP_SINK_UDP
#include "lwip/inet.h"
#include "lwip/sockets.h"
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
#if CONFIG_MEDIA_DUMP_ENABLE
static const char *TAG = "MEDIA_DUMP";
typedef struct {
bool opened;
int64_t start_time_us;
int64_t max_duration_us;
bool expired;
bool notified;
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
FILE *fp;
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
#if CONFIG_MEDIA_DUMP_SINK_UDP
int sock;
struct sockaddr_in dest;
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
} media_dump_ctx_t;
static media_dump_ctx_t s_ctx = {0};
static inline bool media_dump_time_exceeded(void)
{
media_dump_ctx_t *ctx = &s_ctx;
if (!ctx->opened) {
return false;
}
int64_t now = esp_timer_get_time();
if ((now - ctx->start_time_us) >= ctx->max_duration_us) {
ctx->expired = true;
if (!ctx->notified) {
ESP_LOGI(TAG, "Dump duration reached (%ds)", (int)CONFIG_MEDIA_DUMP_DURATION_SEC);
ctx->notified = true;
}
return true;
}
return false;
}
static void media_dump_release(media_dump_ctx_t *ctx)
{
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
if (ctx->fp) {
fflush(ctx->fp);
fclose(ctx->fp);
ctx->fp = NULL;
}
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
#if CONFIG_MEDIA_DUMP_SINK_UDP
if (ctx->sock >= 0) {
close(ctx->sock);
ctx->sock = -1;
}
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
ctx->opened = false;
}
int media_dump_open(const char *filename)
{
media_dump_ctx_t *ctx = &s_ctx;
/* allow re-open anytime: close existing and start a new session */
if (ctx->opened) {
media_dump_release(ctx);
}
/* reset state and start fresh */
memset(ctx, 0, sizeof(*ctx));
ctx->max_duration_us = (int64_t)CONFIG_MEDIA_DUMP_DURATION_SEC * 1000000LL;
ctx->start_time_us = esp_timer_get_time();
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
const char *path = NULL;
if (filename) {
path = filename;
} else {
path = CONFIG_MEDIA_DUMP_SDCARD_FILENAME;
}
ctx->fp = fopen(path, "wb");
if (!ctx->fp) {
ESP_LOGE(TAG, "Dump open file failed: %s, error: %s", path, strerror(errno));
return -1;
}
ESP_LOGI(TAG, "Dump file: %s, %ds", path, CONFIG_MEDIA_DUMP_DURATION_SEC);
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
#if CONFIG_MEDIA_DUMP_SINK_UDP
ctx->sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (ctx->sock < 0) {
ESP_LOGE(TAG, "Dump create UDP socket failed");
return -1;
}
memset(&ctx->dest, 0, sizeof(ctx->dest));
ctx->dest.sin_family = AF_INET;
ctx->dest.sin_port = htons(CONFIG_MEDIA_DUMP_UDP_PORT);
ctx->dest.sin_addr.s_addr = inet_addr(CONFIG_MEDIA_DUMP_UDP_IP);
ESP_LOGI(TAG, "Dump UDP %s:%d, %ds", CONFIG_MEDIA_DUMP_UDP_IP,
CONFIG_MEDIA_DUMP_UDP_PORT, CONFIG_MEDIA_DUMP_DURATION_SEC);
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
ctx->opened = true;
return 0;
}
int media_dump_write(const uint8_t *data, size_t size)
{
media_dump_ctx_t *ctx = &s_ctx;
if (!ctx->opened || ctx->expired || !data || size == 0) {
return -1;
}
if (media_dump_time_exceeded()) {
ESP_LOGW(TAG, "Dump duration reached, ignore write");
return 0;
}
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
if (ctx->fp) {
size_t w = fwrite(data, 1, size, ctx->fp);
if (w != size) {
ESP_LOGW(TAG, "Dump short file write: %u/%u", (unsigned)w, (unsigned)size);
}
return (int)w;
}
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
#if CONFIG_MEDIA_DUMP_SINK_UDP
if (ctx->sock >= 0) {
int sent = sendto(ctx->sock, data, size, 0, (struct sockaddr *)&ctx->dest, sizeof(ctx->dest));
if (sent < 0) {
ESP_LOGW(TAG, "Dump UDP send failed");
return -1;
}
return sent;
}
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
return -1;
}
int media_dump_close(void)
{
media_dump_ctx_t *ctx = &s_ctx;
if (!ctx->opened) {
return 0;
}
/* normal close: always release immediately */
media_dump_release(ctx);
return 0;
}
int media_dump_feed(const char *filename, const uint8_t *data, size_t size)
{
media_dump_ctx_t *ctx = &s_ctx;
/* feed mode: maintain a single open session across calls */
if (ctx->expired) {
return 0;
}
if (!ctx->opened) {
if (media_dump_open(filename) != 0) {
return -1;
}
}
ESP_LOGI(TAG, "Dump duration not reached, write now");
int ret = media_dump_write(data, size);
/* If duration reached during this write, close now */
if (ctx->expired && ctx->opened) {
(void)media_dump_close();
ESP_LOGI(TAG, "Dump duration reached, close now");
}
return ret;
}
#else /* !CONFIG_MEDIA_DUMP_ENABLE */
int media_dump_open(const char *filename)
{
return 0;
}
int media_dump_write(const uint8_t *data, size_t size)
{
(void)data;
(void)size;
return 0;
}
int media_dump_close(void)
{
return 0;
}
int media_dump_feed(const char *filename, const uint8_t *data, size_t size)
{
(void)filename;
(void)data;
(void)size;
return 0;
}
#endif /* CONFIG_MEDIA_DUMP_ENABLE */
@@ -0,0 +1,101 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
*
* See LICENSE file for details.
*/
#pragma once
#include "esp_err.h"
#include "esp_gmf_pool.h"
#include "av_processor_type.h"
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
/**
* @brief Initialize audio playback pool
*
* Creates and initializes a memory pool for audio playback operations.
* This pool manages memory for audio data that is being played back from
* various sources (files, streams, etc.).
*
* @param pool Pointer to store the created pool handle
* @return
* - ESP_OK Pool initialized successfully
* - ESP_ERR_NO_MEM Insufficient memory to create pool
* - ESP_ERR_INVALID_ARG Invalid parameter provided
*/
esp_err_t av_audio_playback_pool_init(esp_gmf_pool_handle_t *pool);
/**
* @brief Deinitialize audio playback pool
*
* Destroys the audio playback pool and frees all associated memory.
*
* @param pool Handle of the pool to deinitialize
* @return
* - ESP_OK Pool deinitialized successfully
* - ESP_ERR_INVALID_ARG Invalid pool handle
*/
esp_err_t av_audio_playback_pool_deinit(esp_gmf_pool_handle_t pool);
/**
* @brief Initialize audio recorder pool
*
* @param pool Pointer to store the created pool handle
* @param encoder_cfg Pointer to encoder configuration. If NULL, default PCM configuration will be used.
* @return
* - ESP_OK Pool initialized successfully
* - ESP_ERR_NO_MEM Insufficient memory to create pool
* - ESP_ERR_INVALID_ARG Invalid parameter provided
*/
esp_err_t av_audio_recorder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_encoder_config_t *encoder_cfg);
/**
* @brief Deinitialize audio recorder pool
*
* Destroys the audio recorder pool and frees all associated memory.
* Make sure encoder is not running or else may cause memory violation error
*
* @param pool Handle of the pool to deinitialize
* @return
* - ESP_OK Pool deinitialized successfully
* - ESP_ERR_INVALID_ARG Invalid pool handle
*/
esp_err_t av_audio_recorder_pool_deinit(esp_gmf_pool_handle_t pool);
/**
* @brief Initialize audio feeder pool
*
* Creates and initializes a memory pool for audio feeder operations.
* This pool manages memory for audio data that is being fed into
* various processing pipelines or output streams.
*
* @param pool Pointer to store the created pool handle
* @param decoder_cfg Pointer to decoder configuration. If NULL, default PCM configuration will be used.
* @return
* - ESP_OK Pool initialized successfully
* - ESP_ERR_NO_MEM Insufficient memory to create pool
* - ESP_ERR_INVALID_ARG Invalid parameter provided
*/
esp_err_t av_audio_feeder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_decoder_config_t *decoder_cfg);
/**
* @brief Deinitialize audio feeder pool
*
* Destroys the audio feeder pool and frees all associated memory.
* Make sure decoder is not running or else may cause memory violation error
*
* @param pool Handle of the pool to deinitialize
* @return
* - ESP_OK Pool deinitialized successfully
* - ESP_ERR_INVALID_ARG Invalid pool handle
*/
esp_err_t av_audio_feeder_pool_deinit(esp_gmf_pool_handle_t pool);
#ifdef __cplusplus
}
#endif /* __cplusplus */
@@ -0,0 +1,84 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
*
* See LICENSE file for details.
*/
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
#include <stddef.h>
#include <stdint.h>
/**
* @brief Open a media dump session
*
* When enabled by Kconfig, subsequent writes are dumped to the selected sink.
* If a session is already open, it will be closed and a new one started.
* The session duration is limited by `CONFIG_MEDIA_DUMP_DURATION_SEC`; after
* the timeout is reached, additional writes are not accepted.
*
* Sinks:
* - SDCARD: write to `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`
* - UDP: send to `CONFIG_MEDIA_DUMP_UDP_ADDRESS` (format: IP:Port, e.g., "192.168.1.100:5000")
*
* @param[in] filename Optional filename for the dump file
* - If NULL, use `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`
* - If not NULL, use the filename as the dump file name
* @return
* - 0: Session opened successfully
* - -1: Failed to open (e.g., file/socket creation failure)
*/
int media_dump_open(const char *filename);
/**
* @brief Write media data to the dump sink
*
* Effective only when a session is open and not expired.
*
* @param[in] data Pointer to the data buffer to write
* @param[in] size Number of bytes in the buffer
* @return
* - >0: Bytes actually written/sent (depends on sink)
* - 0: Maximum duration reached, this write is ignored
* - -1: Session not open, already expired, or parameters invalid
*/
int media_dump_write(const uint8_t *data, size_t size);
/**
* @brief Close the current media dump session and release resources
*
* Calling multiple times is safe; if no session is open, it returns immediately.
*
* @return
* - 0: Always returns 0
*/
int media_dump_close(void);
/**
* @brief Feed data with auto open/close behavior
*
* If no session is open, media_dump_open() is called automatically. After the
* write, if the session duration has been reached, the session is closed
* automatically. If the session is already expired, this function returns 0.
* If want to call this function, can use `extern int media_dump_feed(const char *filename, const uint8_t *data, size_t size);` in the file.
*
* @param[in] filename Optional filename for the dump file
* - If NULL, use `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`
* - If not NULL, use the filename as the dump file name
* @param[in] data Pointer to the data buffer to write
* @param[in] size Number of bytes in the buffer
* @return
* - Same return semantics as media_dump_write()
* - -1: Auto open failed
*/
int media_dump_feed(const char *filename, const uint8_t *data, size_t size);
#ifdef __cplusplus
}
#endif
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
import os
import shutil
import sys
import logging
from pathlib import Path
def main() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(levelname)s: %(message)s",
)
# Compute source directory (parent of this script's directory, i.e. espressif__av_processor)
script_path = Path(__file__).resolve()
av_processor_dir = script_path.parent.parent
if av_processor_dir.name != "espressif__av_processor":
logging.error("Failed to resolve source directory. Expected 'espressif__av_processor', got: %s", av_processor_dir)
return 1
# Ensure source espressif__av_processor is under managed_components
if av_processor_dir.parent.name != "managed_components":
logging.error("This script must be run when espressif__av_processor is under 'managed_components'. Current path: %s", av_processor_dir)
return 1
# Read PROJECT_DIR environment variable
project_dir_env = os.environ.get("PROJECT_DIR")
if not project_dir_env:
logging.error("PROJECT_DIR is not set. Example: export PROJECT_DIR=/path/to/project")
return 1
project_dir = Path(project_dir_env).resolve()
if not project_dir.exists() or not project_dir.is_dir():
logging.error("PROJECT_DIR does not exist or is not a directory: %s", project_dir)
return 1
# Target components directory
components_dir = project_dir / "components"
try:
components_dir.mkdir(parents=True, exist_ok=True)
except Exception as e:
logging.error("Failed to create target directory: %s -> %s", components_dir, e)
return 1
# Destination espressif__av_processor path
dest_dir = components_dir / "espressif__av_processor"
# If destination exists, remove it before copying
if dest_dir.exists():
logging.info("Destination exists, removing: %s", dest_dir)
try:
shutil.rmtree(dest_dir)
except Exception as e:
logging.error("Failed to remove existing destination: %s -> %s", dest_dir, e)
return 1
# Perform copy
logging.info("Copying %s -> %s", av_processor_dir, dest_dir)
try:
shutil.copytree(av_processor_dir, dest_dir)
except Exception as e:
logging.error("Copy failed: %s", e)
return 1
# Switch to a safe working directory to avoid deleting the running/current dir
try:
os.chdir(project_dir)
except Exception as e:
logging.error("Failed to change working directory: %s -> %s", project_dir, e)
return 1
# Remove source directory (parent av_processor)
logging.info("Removing source directory: %s", av_processor_dir)
try:
shutil.rmtree(av_processor_dir)
except Exception as e:
logging.error("Failed to remove source directory: %s -> %s", av_processor_dir, e)
logging.info("Copy completed, but source removal failed. Please clean up manually.")
return 1
# Update PROJECT_DIR/main/idf_component.yml to remove 'espressif/av_processor'
idf_component_path = project_dir / "main" / "idf_component.yml"
if idf_component_path.exists():
try:
content = idf_component_path.read_text(encoding="utf-8")
lines = content.splitlines()
new_lines = [line for line in lines if "espressif/av_processor" not in line]
if len(new_lines) != len(lines):
idf_component_path.write_text("\n".join(new_lines) + ("\n" if content.endswith("\n") else ""), encoding="utf-8")
logging.info("Removed 'espressif/av_processor' from %s", idf_component_path)
else:
logging.info("No 'espressif/av_processor' entry found in %s", idf_component_path)
except Exception as e:
logging.error("Failed to update %s: %s", idf_component_path, e)
return 1
else:
logging.info("idf_component.yml not found at %s, skipping update", idf_component_path)
logging.info("Copied espressif__av_processor to PROJECT_DIR/components, removed original directory, and updated idf_component.yml.")
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,93 @@
# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
# SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
#
# See LICENSE file for details.
#!/usr/bin/env python3
import argparse
import socket
import sys
from pathlib import Path
import os
from datetime import datetime
import logging
def get_primary_ip():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
s.connect(("8.8.8.8", 80))
return s.getsockname()[0]
except Exception:
return "127.0.0.1"
finally:
s.close()
def gen_filename():
ts = datetime.now().strftime("%Y%m%d%H%M%S")
return f"media_dump_{ts}.bin"
BUF_SIZE = 10240
def main():
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
parser = argparse.ArgumentParser(description="Receive UDP media dump and save to file")
parser.add_argument("--port", type=int, default=5000, help="UDP port (default: 5000)")
args = parser.parse_args()
primary_ip = get_primary_ip()
logging.info(f"Host primary IP: {primary_ip}, port: {args.port} (set MEDIA_DUMP_UDP_IP:PORT to this)")
out_path = Path(gen_filename())
out_path.parent.mkdir(parents=True, exist_ok=True)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
sock.bind(("0.0.0.0", args.port))
except OSError as e:
logging.error(f"Bind failed on 0.0.0.0:{args.port} - {e}")
sys.exit(1)
logging.info(f"Listening on 0.0.0.0:{args.port}, output file: {out_path.name} (created on first packet), buf={BUF_SIZE}")
total = 0
f = None
try:
while True:
try:
data, addr = sock.recvfrom(BUF_SIZE)
except OSError:
break
if not data:
logging.info("\nClose received, finishing and exiting")
break
if f is None:
f = open(out_path, "wb")
logging.info(f"Receiving... writing to {out_path.resolve()}")
f.write(data)
total += len(data)
sys.stdout.write(f"\r[INFO] recv={len(data)} bytes, total={total} bytes")
sys.stdout.flush()
except KeyboardInterrupt:
pass
finally:
sock.close()
# ensure the progress line ends with newline
try:
sys.stdout.write("\n")
sys.stdout.flush()
except Exception:
pass
try:
if f is not None:
f.flush()
os.fsync(f.fileno())
f.close()
logging.info(f"Saved file: {out_path.resolve()}")
else:
logging.info("No packets received, no file saved")
except Exception as e:
logging.error(f"Finalize file failed: {e}")
logging.info(f"Done. Total bytes received: {total}")
sys.exit(0)
if __name__ == "__main__":
main()
@@ -0,0 +1,470 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "freertos/queue.h"
#include "esp_log.h"
#include "driver/gpio.h"
#include "esp_gmf_oal_thread.h"
#include "esp_gmf_oal_mem.h"
#include "esp_video_dec.h"
#include "esp_video_dec_reg.h"
#include "esp_video_enc_default.h"
#include "esp_video_codec_utils.h"
#include "esp_video_dec_default.h"
#include "esp_video_codec_version.h"
#include "esp_video_codec_types.h"
#include "esp_capture.h"
#include "esp_capture_defaults.h"
#include "video_processor.h"
static const char *TAG = "VIDEO_PROCESSOR";
#define VIDEO_RENDER_QUEUE_DEFAULT_SIZE (5)
#define _EVENT_GROUP_STOP_BIT (1 << 0)
#define VIDEO_DECODER_WORKER_TASK_STACK_SIZE (4096 * 10)
#define VIDEO_DECODER_WORKER_TASK_PRIORITY (12)
#define VIDEO_DECODER_WORKER_TASK_CORE (0)
#define VIDEO_DECODER_WORKER_TASK_NAME "video_decoder_worker"
#define VIDEO_CAPTURE_WORKER_TASK_STACK_SIZE (4096 * 10)
#define VIDEO_CAPTURE_WORKER_TASK_PRIORITY (12)
#define VIDEO_CAPTURE_WORKER_TASK_CORE (0)
#define VIDEO_CAPTURE_WORKER_TASK_NAME "video_capture_worker"
typedef enum {
VID_PROCESSOR_CMD_STOP,
VID_PROCESSOR_CMD_DATA,
VID_PROCESSOR_CMD_MAX,
} video_processor_cmd_t;
typedef struct {
video_processor_cmd_t cmd;
video_frame_t frame;
} video_processor_event_t;
/**
* @brief Video render internal structure
*/
typedef struct {
video_render_config_t config; /*!< Render configuration */
esp_gmf_oal_thread_t vdec_thread; /*!< Decoder thread handle */
QueueHandle_t frame_queue; /*!< Frame queue handle */
EventGroupHandle_t event_group; /*!< Event group handle */
video_frame_t out_frame; /*!< Output frame buffer */
void *ctx; /*!< User context */
esp_video_dec_handle_t dec_handle; /*!< Decoder handle */
bool is_running; /*!< Running state flag */
} video_render_t;
/**
* @brief Video capture internal structure
*/
typedef struct {
video_capture_config_t config; /*!< Capture configuration */
esp_capture_handle_t capture; /*!< Capture handle */
esp_capture_audio_src_if_t *aud_src; /*!< Audio source interface */
esp_capture_video_src_if_t *vid_src; /*!< Video source interface */
esp_capture_sink_handle_t sink[DEFAULT_VID_MAX_PATH]; /*!< Sink handles */
esp_gmf_oal_thread_t capture_thread; /*!< Capture thread handle */
EventGroupHandle_t event_group; /*!< Event group handle */
QueueHandle_t frame_queue; /*!< Frame queue handle */
bool is_running; /*!< Running state flag */
} video_capture_t;
static esp_capture_video_src_if_t *create_video_source(void *cam_config)
{
#if CONFIG_IDF_TARGET_ESP32S3
return esp_capture_new_video_dvp_src((esp_capture_video_dvp_src_cfg_t *)cam_config);
#else
ESP_LOGE(TAG, "Video source not supported for this target");
return NULL;
#endif /* CONFIG_IDF_TARGET_ESP32S3 */
}
static void decoder_worker_thread(void *arg)
{
video_render_t *render = (video_render_t *)arg;
if (!render) {
ESP_LOGE(TAG, "Invalid render handle");
esp_gmf_oal_thread_delete(NULL);
}
if (!render->out_frame.data) {
ESP_LOGE(TAG, "Failed to allocate memory for output frame");
esp_gmf_oal_thread_delete(NULL);
}
render->is_running = true;
video_processor_event_t evt_frame = {0};
while (render->is_running) {
if (xQueueReceive(render->frame_queue, &evt_frame, portMAX_DELAY) == pdPASS) {
ESP_LOGD(TAG, "vdec_task: %d", evt_frame.cmd);
switch (evt_frame.cmd) {
case VID_PROCESSOR_CMD_DATA:
ESP_LOGD(TAG, "Processing video frame, size: %zu", evt_frame.frame.size);
esp_video_dec_in_frame_t in_frame = {
.pts = 0,
.data = evt_frame.frame.data,
.size = evt_frame.frame.size,
};
esp_video_dec_out_frame_t out_frame = {
.data = render->out_frame.data,
.size = render->out_frame.size,
};
int ret = esp_video_dec_process(render->dec_handle, &in_frame, &out_frame);
if (ret != ESP_VC_ERR_OK) {
ESP_LOGE(TAG, "Failed to decode frame. ret: %d", ret);
break;
}
if (render->config.decode_cb) {
render->config.decode_cb(render->config.decode_cb_ctx, out_frame.data, out_frame.size);
}
break;
case VID_PROCESSOR_CMD_STOP:
ESP_LOGI(TAG, "Received stop command, exiting decoder thread");
render->is_running = false;
break;
default:
ESP_LOGE(TAG, "Unknown command: %d", evt_frame.cmd);
break;
}
if (evt_frame.frame.data) {
esp_gmf_oal_free(evt_frame.frame.data);
evt_frame.frame.data = NULL;
}
}
}
xEventGroupClearBits(render->event_group, _EVENT_GROUP_STOP_BIT);
ESP_LOGI(TAG, "Decoder thread exiting");
vTaskDelete(NULL);
}
static void capture_worker_thread(void *arg)
{
video_capture_t *capture = (video_capture_t *)arg;
if (!capture) {
ESP_LOGE(TAG, "Invalid capture handle");
esp_gmf_oal_thread_delete(NULL);
}
for (int i = 0; i < capture->config.sink_num; i++) {
esp_capture_sink_enable(capture->sink[i], ESP_CAPTURE_RUN_MODE_ALWAYS);
}
capture->is_running = true;
ESP_LOGI(TAG, "Capture thread started");
uint32_t gcd_fps = capture->config.sink_cfg[0].video_info.fps;
for (int i = 1; i < capture->config.sink_num; i++) {
uint32_t current_fps = capture->config.sink_cfg[i].video_info.fps;
uint32_t a = gcd_fps, b = current_fps;
while (b != 0) {
uint32_t temp = b;
b = a % b;
a = temp;
}
gcd_fps = a;
}
uint32_t base_delay_ms = 1000 / gcd_fps;
ESP_LOGI(TAG, "Multi-path capture: %d paths, gcd_fps: %" PRIu32 ", base_delay: %" PRIu32 "ms",
capture->config.sink_num, gcd_fps, base_delay_ms);
while (capture->is_running) {
for (int i = 0; i < capture->config.sink_num; i++) {
esp_capture_stream_frame_t frame = {0};
frame.stream_type = ESP_CAPTURE_STREAM_TYPE_VIDEO;
// Acquire frame from sink
esp_err_t ret = esp_capture_sink_acquire_frame(capture->sink[i], &frame, true);
if (ret != ESP_CAPTURE_ERR_OK) {
ESP_LOGD(TAG, "Acquire frame from sink %d timeout", i);
vTaskDelay(pdMS_TO_TICKS(base_delay_ms));
continue;
}
if (frame.stream_type == ESP_CAPTURE_STREAM_TYPE_VIDEO && frame.data && frame.size > 0) {
if (capture->config.capture_frame_cb) {
video_frame_t vid_frame = {
.data = frame.data,
.size = frame.size,
};
capture->config.capture_frame_cb(capture->config.capture_frame_cb_ctx, i, &vid_frame);
}
}
esp_capture_sink_release_frame(capture->sink[i], &frame);
}
}
ESP_LOGI(TAG, "Capture thread exiting");
xEventGroupSetBits(capture->event_group, _EVENT_GROUP_STOP_BIT);
esp_gmf_oal_thread_delete(NULL);
}
video_render_handle_t video_render_open(video_render_config_t *config)
{
if (!config) {
ESP_LOGE(TAG, "Invalid config");
return NULL;
}
esp_video_dec_register_default();
video_render_t *render = esp_gmf_oal_calloc(1, sizeof(video_render_t));
if (!render) {
ESP_LOGE(TAG, "Failed to allocate memory for render");
return NULL;
}
render->config.decode_cb_ctx = config->decode_cb_ctx;
render->config.decode_cb = config->decode_cb;
render->config.decode_cfg = config->decode_cfg;
render->config.resolution.width = config->resolution.width;
render->config.resolution.height = config->resolution.height;
render->frame_queue = xQueueCreate(VIDEO_RENDER_QUEUE_DEFAULT_SIZE, sizeof(video_processor_event_t));
if (!render->frame_queue) {
goto __exit;
}
render->event_group = xEventGroupCreate();
if (!render->event_group) {
goto __exit;
}
ESP_LOGI(TAG, "Video render opened successfully");
return render;
__exit:
if (render->frame_queue) {
vQueueDelete(render->frame_queue);
render->frame_queue = NULL;
}
if (render->event_group) {
vEventGroupDelete(render->event_group);
render->event_group = NULL;
}
if (render) {
esp_gmf_oal_free(render);
render = NULL;
}
return NULL;
}
esp_err_t video_render_start(video_render_handle_t handle)
{
video_render_t *render = (video_render_t *)handle;
if (!render) {
return ESP_ERR_INVALID_ARG;
}
uint32_t actual_size = 0;
render->out_frame.size = esp_video_codec_get_image_size(render->config.decode_cfg.out_fmt, &render->config.resolution);
render->out_frame.data = esp_video_codec_align_alloc(128, render->out_frame.size, &actual_size);
if (!render->out_frame.data) {
ESP_LOGE(TAG, "Failed to allocate memory for output frame");
return ESP_ERR_NO_MEM;
}
ESP_LOGI(TAG, "Output frame buffer allocated, size: %zu", render->out_frame.size);
int ret = esp_video_dec_open(&render->config.decode_cfg, &render->dec_handle);
if (!render->dec_handle || ret != ESP_VC_ERR_OK) {
ESP_LOGE(TAG, "Failed to open video decoder, error: %d", ret);
esp_gmf_oal_free(render->out_frame.data);
render->out_frame.data = NULL;
return ESP_ERR_NO_MEM;
}
esp_gmf_oal_thread_create(&render->vdec_thread, VIDEO_DECODER_WORKER_TASK_NAME,
decoder_worker_thread, (void *)render,
VIDEO_DECODER_WORKER_TASK_STACK_SIZE, VIDEO_DECODER_WORKER_TASK_PRIORITY,
true, VIDEO_DECODER_WORKER_TASK_CORE);
ESP_LOGI(TAG, "Video render started successfully");
return ESP_OK;
}
esp_err_t video_render_frame_feed(video_render_handle_t handle, video_frame_t *vid_frame)
{
video_render_t *render = (video_render_t *)handle;
if (!render) {
ESP_LOGE(TAG, "Invalid render handle");
return ESP_ERR_INVALID_ARG;
}
if (!vid_frame || !vid_frame->data || vid_frame->size == 0) {
ESP_LOGE(TAG, "Invalid frame data");
return ESP_ERR_INVALID_ARG;
}
video_processor_event_t evt_frame = {0};
evt_frame.cmd = VID_PROCESSOR_CMD_DATA;
evt_frame.frame.data = esp_gmf_oal_malloc(vid_frame->size);
if (!evt_frame.frame.data) {
ESP_LOGE(TAG, "Failed to allocate memory for frame data");
return ESP_ERR_NO_MEM;
}
memcpy(evt_frame.frame.data, vid_frame->data, vid_frame->size);
evt_frame.frame.size = vid_frame->size;
if (xQueueSend(render->frame_queue, &evt_frame, portMAX_DELAY) != pdPASS) {
ESP_LOGE(TAG, "Failed to send frame to queue");
esp_gmf_oal_free(evt_frame.frame.data);
return ESP_FAIL;
}
return ESP_OK;
}
esp_err_t video_render_stop(video_render_handle_t handle)
{
video_render_t *render = (video_render_t *)handle;
if (!render) {
ESP_LOGE(TAG, "Invalid render handle");
return ESP_ERR_INVALID_ARG;
}
if (!render->is_running) {
ESP_LOGW(TAG, "Video render is not running");
return ESP_OK;
}
video_processor_event_t evt_frame = {0};
evt_frame.cmd = VID_PROCESSOR_CMD_STOP;
if (xQueueSend(render->frame_queue, &evt_frame, portMAX_DELAY) != pdPASS) {
ESP_LOGE(TAG, "Failed to send stop command to queue");
return ESP_FAIL;
}
xEventGroupWaitBits(render->event_group, _EVENT_GROUP_STOP_BIT, pdTRUE, pdFALSE, portMAX_DELAY);
ESP_LOGI(TAG, "Video render stopped successfully");
return ESP_OK;
}
void video_render_close(video_render_handle_t handle)
{
video_render_t *render = (video_render_t *)handle;
if (!render) {
ESP_LOGE(TAG, "Invalid render handle");
return;
}
if (render->is_running) {
video_render_stop(handle);
}
if (render->dec_handle) {
esp_video_dec_close(render->dec_handle);
render->dec_handle = NULL;
}
if (render->out_frame.data) {
esp_gmf_oal_free(render->out_frame.data);
render->out_frame.data = NULL;
}
if (render->frame_queue) {
vQueueDelete(render->frame_queue);
render->frame_queue = NULL;
}
esp_gmf_oal_free(render);
ESP_LOGI(TAG, "Video render closed successfully");
}
video_capture_handle_t video_capture_open(video_capture_config_t *config)
{
if (!config) {
ESP_LOGE(TAG, "Invalid config");
return NULL;
}
// capture structure
video_capture_t *capture = esp_gmf_oal_calloc(1, sizeof(video_capture_t));
if (!capture) {
ESP_LOGE(TAG, "Failed to allocate memory for capture");
return NULL;
}
capture->config = *config;
capture->frame_queue = xQueueCreate(2, sizeof(video_processor_event_t));
if (!capture->frame_queue) {
ESP_LOGE(TAG, "Failed to create frame queue");
goto __exit;
}
capture->event_group = xEventGroupCreate();
if (!capture->event_group) {
goto __exit;
}
esp_video_enc_register_default();
return capture;
__exit:
if (capture->frame_queue) {
vQueueDelete(capture->frame_queue);
capture->frame_queue = NULL;
}
if (capture->event_group) {
vEventGroupDelete(capture->event_group);
capture->event_group = NULL;
}
if (capture) {
esp_gmf_oal_free(capture);
capture = NULL;
}
return NULL;
}
esp_err_t video_capture_start(video_capture_handle_t handle)
{
video_capture_t *capture = (video_capture_t *)handle;
if (!capture) {
ESP_LOGE(TAG, "Invalid capture handle");
return ESP_ERR_INVALID_ARG;
}
capture->vid_src = create_video_source(capture->config.camera_config);
if (!capture->vid_src) {
ESP_LOGE(TAG, "Failed to create video source or not support");
return ESP_ERR_NO_MEM;
}
for (int i = 0; i < capture->config.sink_num; i++) {
// TODO: support other format
esp_capture_video_info_t fixed_caps = {
.format_id = ESP_CAPTURE_FMT_ID_RGB565_BE,
.width = capture->config.sink_cfg[i].video_info.width,
.height = capture->config.sink_cfg[i].video_info.height,
.fps = capture->config.sink_cfg[i].video_info.fps,
};
capture->vid_src->set_fixed_caps(capture->vid_src, &fixed_caps);
}
esp_capture_cfg_t capture_cfg = {
.video_src = capture->vid_src,
};
esp_capture_open(&capture_cfg, &capture->capture);
if (capture->capture == NULL) {
ESP_LOGE(TAG, "Fail to create capture");
return ESP_ERR_NO_MEM;
}
for (int i = 0; i < capture->config.sink_num; i++) {
esp_capture_sink_cfg_t sink_cfg = capture->config.sink_cfg[i];
esp_capture_sink_setup(capture->capture, i, &sink_cfg, &capture->sink[i]);
esp_capture_sink_enable(capture->sink[i], ESP_CAPTURE_RUN_MODE_ALWAYS);
}
esp_capture_start(capture->capture);
esp_gmf_oal_thread_create(&capture->capture_thread, VIDEO_CAPTURE_WORKER_TASK_NAME, capture_worker_thread,
(void *)capture, VIDEO_CAPTURE_WORKER_TASK_STACK_SIZE, VIDEO_CAPTURE_WORKER_TASK_PRIORITY,
true, VIDEO_CAPTURE_WORKER_TASK_CORE);
return ESP_OK;
}
void video_capture_close(video_capture_handle_t handle)
{
video_capture_t *capture = (video_capture_t *)handle;
if (!capture) {
ESP_LOGE(TAG, "Invalid capture handle");
return;
}
if (capture->capture) {
esp_capture_stop(capture->capture);
esp_capture_close(capture->capture);
}
capture->is_running = false;
xEventGroupWaitBits(capture->event_group, _EVENT_GROUP_STOP_BIT, pdTRUE, pdFALSE, portMAX_DELAY);
if (capture->vid_src) {
capture->vid_src = NULL;
}
if (capture->frame_queue) {
vQueueDelete(capture->frame_queue);
}
esp_gmf_oal_free(capture);
}