@@ -0,0 +1 @@
|
||||
**/gen_bmgr_codes/
|
||||
@@ -0,0 +1,7 @@
|
||||
# Changelog
|
||||
|
||||
## v0.1.0
|
||||
|
||||
### Features
|
||||
|
||||
- Initial version of `av_processor`
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
idf_component_register(
|
||||
INCLUDE_DIRS "include" "./"
|
||||
PRIV_INCLUDE_DIRS "private_include"
|
||||
SRC_DIRS "./"
|
||||
REQUIRES esp_capture esp_video_codec
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
menu "Audio/Video Processor Configuration"
|
||||
|
||||
# Media Dump Configuration
|
||||
menu "Media Dump"
|
||||
config MEDIA_DUMP_ENABLE
|
||||
bool "Enable Media Dump"
|
||||
default n
|
||||
help
|
||||
Enable media dump feature. When disabled, related options are hidden.
|
||||
|
||||
# Audio AEC Save Point
|
||||
choice MEDIA_DUMP_AUDIO_POINT
|
||||
prompt "Audio Save Mode (AEC point)"
|
||||
depends on MEDIA_DUMP_ENABLE
|
||||
default MEDIA_DUMP_AUDIO_NONE
|
||||
help
|
||||
Select which audio to save for media dump with respect to AEC processing.
|
||||
Only one can be selected, or select None to disable both.
|
||||
|
||||
config MEDIA_DUMP_AUDIO_BEFORE_AEC
|
||||
bool "Save Before AEC"
|
||||
help
|
||||
Save raw microphone audio before AEC processing for media dump.
|
||||
|
||||
config MEDIA_DUMP_AUDIO_AFTER_AEC
|
||||
bool "Save After AEC"
|
||||
help
|
||||
Save audio after AEC processing for media dump.
|
||||
|
||||
config MEDIA_DUMP_AUDIO_NONE
|
||||
bool "None"
|
||||
help
|
||||
Do not save audio for media dump.
|
||||
endchoice
|
||||
|
||||
config MEDIA_DUMP_DURATION_SEC
|
||||
int "Dump Duration (seconds)"
|
||||
depends on MEDIA_DUMP_ENABLE
|
||||
default 20
|
||||
range 1 3600
|
||||
help
|
||||
Maximum duration to capture media dump in seconds. Default 20s.
|
||||
|
||||
choice MEDIA_DUMP_SINK
|
||||
prompt "Media Dump Output Method"
|
||||
depends on MEDIA_DUMP_ENABLE
|
||||
default MEDIA_DUMP_SINK_SDCARD
|
||||
help
|
||||
Select where to save media dump data.
|
||||
|
||||
config MEDIA_DUMP_SINK_UDP
|
||||
bool "UDP"
|
||||
|
||||
config MEDIA_DUMP_SINK_SDCARD
|
||||
bool "SD Card File"
|
||||
endchoice
|
||||
|
||||
if MEDIA_DUMP_SINK_UDP
|
||||
config MEDIA_DUMP_UDP_IP
|
||||
string "UDP Destination IP"
|
||||
default "192.168.1.100"
|
||||
help
|
||||
Destination IPv4 address for UDP media dump.
|
||||
|
||||
config MEDIA_DUMP_UDP_PORT
|
||||
int "UDP Destination Port"
|
||||
default 5000
|
||||
range 1 65535
|
||||
help
|
||||
Destination UDP port for media dump.
|
||||
endif
|
||||
|
||||
if MEDIA_DUMP_SINK_SDCARD
|
||||
config MEDIA_DUMP_SDCARD_FILENAME
|
||||
string "SD Card File Name"
|
||||
default "/sdcard/media_dump.bin"
|
||||
help
|
||||
File path on SD card to save media dump data. Ensure SD is mounted.
|
||||
endif
|
||||
endmenu
|
||||
|
||||
endmenu
|
||||
@@ -0,0 +1,22 @@
|
||||
Espressif Modified MIT License
|
||||
|
||||
Copyright (c) 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
|
||||
Permission is hereby granted for use EXCLUSIVELY with Espressif Systems products.
|
||||
This includes the right to use, copy, modify, merge, publish, distribute, and sublicense
|
||||
the Software, subject to the following conditions:
|
||||
|
||||
1. This Software MUST BE USED IN CONJUNCTION WITH ESPRESSIF SYSTEMS PRODUCTS.
|
||||
2. The above copyright notice and this permission notice shall be included in all copies
|
||||
or substantial portions of the Software.
|
||||
3. Redistribution of the Software in source or binary form FOR USE WITH NON-ESPRESSIF PRODUCTS
|
||||
is strictly prohibited.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
||||
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
|
||||
FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
|
||||
@@ -0,0 +1,332 @@
|
||||
# AV Processor Component (Audio/Video Processing)
|
||||
|
||||
This component provides a unified encapsulation for audio recording/playback/data feeding and video capture/decoding/rendering on ESP platforms, implemented based on middleware such as GMF (General Media Framework), esp-audio-render, esp-video-codec, and esp-capture.
|
||||
|
||||
## Feature Overview 🎛️
|
||||
|
||||
- **Audio 🎧**:
|
||||
- Recording (AEC/VAD/AFE optional)
|
||||
- Playback (supports URL and local files, decoding depends on configuration)
|
||||
- Data feeding (feeder, suitable for scenarios like RTC/streaming where receiving and playing happen simultaneously)
|
||||
- Optional mixer with volume fade and focus control between multiple streams
|
||||
- **Video 🎥**:
|
||||
- Multi-channel video capture
|
||||
- Video decoding and rendering callbacks
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Audio Usage
|
||||
|
||||
1. Initialize audio manager:
|
||||
|
||||
```c
|
||||
// Method 1: Use default configuration macro (recommended)
|
||||
audio_manager_config_t cfg = DEFAULT_AUDIO_MANAGER_CONFIG();
|
||||
cfg.play_dev = your_play_dev;
|
||||
cfg.rec_dev = your_rec_dev;
|
||||
strcpy(cfg.mic_layout, "RMNM"); // Microphone layout (optional, for AFE configuration)
|
||||
cfg.board_sample_rate = 16000;
|
||||
cfg.board_bits = 32;
|
||||
cfg.board_channels = 2;
|
||||
cfg.play_volume = 80;
|
||||
cfg.rec_volume = 60;
|
||||
cfg.rec_ref_volume = 60;
|
||||
cfg.enable_mixer = true; // Enable when mixing feeder and playback is needed
|
||||
|
||||
audio_manager_init(&cfg);
|
||||
```
|
||||
|
||||
2. Recording (polling read):
|
||||
|
||||
```c
|
||||
// Recording event callback (optional)
|
||||
void recorder_event_cb(void *event, void *ctx) {
|
||||
// Handle recording events (e.g., AFE events)
|
||||
}
|
||||
|
||||
// Configure recorder (using default configuration macro)
|
||||
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
|
||||
// Optional: Configure AFE (runtime configuration, takes priority over Kconfig)
|
||||
recorder_cfg.afe_config.vad_enable = true;
|
||||
recorder_cfg.afe_config.vad_mode = 4;
|
||||
recorder_cfg.afe_config.agc_enable = true;
|
||||
recorder_cfg.recorder_event_cb = recorder_event_cb;
|
||||
|
||||
// Open recorder
|
||||
audio_recorder_open(&recorder_cfg);
|
||||
uint8_t buf[2048];
|
||||
audio_recorder_read_data(buf, sizeof(buf));
|
||||
|
||||
audio_recorder_close();
|
||||
```
|
||||
|
||||
3. Feeder playback (suitable for external stream → local playback):
|
||||
|
||||
```c
|
||||
// Configure feeder (using default configuration macro)
|
||||
audio_feeder_config_t feeder_cfg = DEFAULT_AUDIO_FEEDER_CONFIG();
|
||||
// Optional: When using OPUS decoder, task_stack is recommended to be >= 4096 * 10
|
||||
// feeder_cfg.feeder_task_config.task_stack = 4096 * 10;
|
||||
|
||||
audio_feeder_open(&feeder_cfg);
|
||||
audio_feeder_run();
|
||||
// Feed external data blocks multiple times as needed
|
||||
audio_feeder_feed_data(pkt, pkt_len);
|
||||
|
||||
audio_feeder_stop();
|
||||
audio_feeder_close();
|
||||
```
|
||||
|
||||
4. Normal playback (URL or local):
|
||||
|
||||
```c
|
||||
// Configure player (using default configuration macro)
|
||||
audio_playback_config_t playback_cfg = DEFAULT_AUDIO_PLAYBACK_CONFIG();
|
||||
|
||||
audio_playback_open(&playback_cfg);
|
||||
audio_playback_play("http://<ip>:<port>/audio.mp3");
|
||||
|
||||
audio_playback_pause();
|
||||
audio_playback_resume();
|
||||
audio_playback_stop();
|
||||
audio_playback_close();
|
||||
```
|
||||
|
||||
5. Focus/fade control (when mixer is enabled):
|
||||
|
||||
```c
|
||||
// Note: Mixer must be opened after both playback and feeder are opened
|
||||
audio_processor_mixer_open();
|
||||
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_FEEDER); // Focus on feeder audio
|
||||
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_PLAYBACK); // Focus on playback audio
|
||||
audio_processor_mixer_close();
|
||||
```
|
||||
|
||||
### 2. Video Usage
|
||||
|
||||
1. Render (decode) callback:
|
||||
|
||||
```c
|
||||
void decoded_cb(void *ctx, const uint8_t *data, size_t size) {
|
||||
// Handle decoded frame data
|
||||
}
|
||||
|
||||
video_render_config_t rcfg = {
|
||||
.decode_cfg = your_vdec_cfg,
|
||||
.resolution = {.width = 640, .height = 480},
|
||||
.decode_cb = decoded_cb,
|
||||
};
|
||||
video_render_handle_t r = video_render_open(&rcfg);
|
||||
video_render_start(r);
|
||||
|
||||
video_frame_t f = {.data = enc_frame, .size = enc_size};
|
||||
video_render_frame_feed(r, &f);
|
||||
|
||||
video_render_stop(r);
|
||||
video_render_close(r);
|
||||
```
|
||||
|
||||
1. Capture (multi-channel):
|
||||
|
||||
```c
|
||||
void capture_cb(void *ctx, int index, video_frame_t *frame) {
|
||||
// Handle captured video frame
|
||||
}
|
||||
|
||||
video_capture_config_t ccfg = {0};
|
||||
ccfg.camera_config = &your_cam_cfg;
|
||||
ccfg.sink_num = 2;
|
||||
ccfg.sink_cfg[0] = your_sink0_cfg;
|
||||
ccfg.sink_cfg[1] = your_sink1_cfg;
|
||||
ccfg.capture_frame_cb = capture_cb;
|
||||
|
||||
video_capture_handle_t c = video_capture_open(&ccfg);
|
||||
video_capture_start(c);
|
||||
video_capture_stop(c);
|
||||
video_capture_close(c);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Media Dump 💾
|
||||
|
||||
- Enable condition ✅: Takes effect after enabling `MEDIA_DUMP_ENABLE` in `menuconfig`.
|
||||
- Save methods 📤: Supports SD card and UDP outputs (choose one).
|
||||
- SD card 💾: Enable `CONFIG_MEDIA_DUMP_SINK_SDCARD`, output to file `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`, duration controlled by `CONFIG_MEDIA_DUMP_DURATION_SEC`.
|
||||
- UDP 📡: Enable `CONFIG_MEDIA_DUMP_SINK_UDP`, send raw media data via `CONFIG_MEDIA_DUMP_UDP_IP` and `CONFIG_MEDIA_DUMP_UDP_PORT` (can use `script/udp_reciver.py` script).
|
||||
- Save point 🎚️ (before/after AEC processing): Select in "Audio Save Mode (AEC point)" in `menuconfig`
|
||||
- `MEDIA_DUMP_AUDIO_BEFORE_AEC` (Save Before AEC): Save raw microphone audio before AEC (for analyzing noise/feedback)
|
||||
- `MEDIA_DUMP_AUDIO_AFTER_AEC` (Save After AEC): Save audio after AEC processing (for evaluating AEC effectiveness)
|
||||
- Typical usage 🔍: Export raw audio/video data when reproducing issues, check saturation, feedback, and format correctness offline using tools like Audacity.
|
||||
|
||||
## Configuration ⚙️
|
||||
|
||||
### Runtime Configuration (Recommended)
|
||||
|
||||
AFE configuration can be set at runtime via the `afe_config` field in the `audio_recorder_config_t` structure, which takes priority over Kconfig configuration:
|
||||
|
||||
```c
|
||||
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
|
||||
// Configure AFE
|
||||
recorder_cfg.afe_config.ai_mode_wakeup = false; // AI mode: true=wakeup mode, false=direct mode
|
||||
recorder_cfg.afe_config.vad_enable = true; // Enable VAD
|
||||
recorder_cfg.afe_config.vad_mode = 4; // VAD mode (1-4), larger values are more sensitive
|
||||
recorder_cfg.afe_config.vad_min_speech_ms = 64; // Minimum speech duration (ms)
|
||||
recorder_cfg.afe_config.vad_min_noise_ms = 1000; // Minimum noise duration (ms)
|
||||
recorder_cfg.afe_config.agc_enable = true; // Enable AGC
|
||||
recorder_cfg.afe_config.enable_vcmd_detect = false; // Enable VCMD
|
||||
recorder_cfg.afe_config.vcmd_timeout_ms = 5000; // VCMD timeout (ms)
|
||||
recorder_cfg.afe_config.mn_language = "cn"; // Model language: "cn" or "en"
|
||||
recorder_cfg.afe_config.wakeup_time_ms = 10000; // Wakeup time (ms)
|
||||
recorder_cfg.afe_config.wakeup_end_time_ms = 2000; // Wakeup end time (ms)
|
||||
|
||||
// Pass configuration when opening recorder
|
||||
audio_recorder_open(&recorder_cfg);
|
||||
```
|
||||
|
||||
### Kconfig Configuration (Alternative)
|
||||
|
||||
If fields in `audio_recorder_config_t.afe_config` use default values or are not set, Kconfig configuration will be used. Configure via `idf.py menuconfig`:
|
||||
|
||||
#### 💾 Media Dump Configuration
|
||||
|
||||
Configure under **"Component config" -> "Audio/Video Processor Configuration" -> "Media Dump"** in `menuconfig`:
|
||||
|
||||
- **`MEDIA_DUMP_ENABLE`**: Enable media data dump functionality, disabled by default
|
||||
- When enabled, raw audio/video data can be saved for debugging and analysis
|
||||
|
||||
- **`MEDIA_DUMP_AUDIO_POINT`**: Audio save point selection (choice type)
|
||||
- **`MEDIA_DUMP_AUDIO_BEFORE_AEC`**: Save raw audio before AEC processing
|
||||
- Useful for analyzing noise and feedback issues
|
||||
- **`MEDIA_DUMP_AUDIO_AFTER_AEC`**: Save audio after AEC processing
|
||||
- Useful for evaluating AEC processing effectiveness
|
||||
- **`MEDIA_DUMP_AUDIO_NONE`**: Do not save audio (default)
|
||||
|
||||
- **`MEDIA_DUMP_DURATION_SEC`**: Save duration in seconds, default 20
|
||||
- Type: integer (int)
|
||||
- Range: 1-3600 seconds
|
||||
|
||||
- **`MEDIA_DUMP_SINK`**: Save method selection (choice type)
|
||||
- **`MEDIA_DUMP_SINK_SDCARD`**: Save to SD card file (default)
|
||||
- File path configuration: `CONFIG_MEDIA_DUMP_SDCARD_FILENAME` (default: `/sdcard/media_dump.bin`)
|
||||
- Ensure SD card is properly mounted
|
||||
- **`MEDIA_DUMP_SINK_UDP`**: Send via UDP
|
||||
- Target IP configuration: `CONFIG_MEDIA_DUMP_UDP_IP` (default: `192.168.1.100`)
|
||||
- Target port configuration: `CONFIG_MEDIA_DUMP_UDP_PORT` (default: 5000)
|
||||
- Can use `script/udp_reciver.py` script to receive data
|
||||
|
||||
### Task Configuration
|
||||
|
||||
The audio processing component uses multiple FreeRTOS tasks internally, which can be customized via task configuration fields in each module's configuration structure:
|
||||
|
||||
#### Task Configuration for Each Module
|
||||
|
||||
- **Recorder** (`audio_recorder_config_t`):
|
||||
- `afe_feed_task_config`: AFE feed task configuration (default stack size 3KB)
|
||||
- `afe_fetch_task_config`: AFE fetch task configuration (default stack size 3KB)
|
||||
- `recorder_task_config`: Recorder task configuration (default stack size 5KB, recommend >= 40KB when using OPUS encoder)
|
||||
|
||||
- **Player** (`audio_playback_config_t`):
|
||||
- `playback_task_config`: Playback task configuration (default stack size 4KB)
|
||||
|
||||
- **Feeder** (`audio_feeder_config_t`):
|
||||
- `feeder_task_config`: Feeder task configuration (default stack size 5KB, recommend >= 40KB when using OPUS decoder)
|
||||
|
||||
**Important Notes**:
|
||||
- When using OPUS encoder or decoder, `task_stack` of `recorder_task_config` and `feeder_task_config` should be set to at least `4096 * 10` bytes (40KB).
|
||||
- When `task_stack` in task configuration is set to 0, default values will be used.
|
||||
- When `task_stack_in_ext` is set to `true`, task stack will be allocated in external memory, helping to save internal RAM.
|
||||
- Task configuration should be set in each module's configuration structure (e.g., `audio_recorder_config_t`, `audio_playback_config_t`, `audio_feeder_config_t`), not in `audio_manager_config_t`.
|
||||
|
||||
---
|
||||
|
||||
## API Reference 📚
|
||||
|
||||
Header files:
|
||||
|
||||
- `include/audio_processor.h`
|
||||
- `include/video_processor.h`
|
||||
- `include/av_processor_type.h`
|
||||
|
||||
Common functions (selection):
|
||||
|
||||
- Audio management:
|
||||
- `audio_manager_init`/`audio_manager_deinit`: Initialize/deinitialize audio manager
|
||||
- Recording:
|
||||
- `audio_recorder_open`: Open recorder (supports encoder configuration and event callback)
|
||||
- `audio_recorder_read_data`: Read recording data
|
||||
- `audio_recorder_pause`/`audio_recorder_resume`: Pause/resume recording
|
||||
- `audio_recorder_get_afe_manager_handle`: Get AFE manager handle
|
||||
- `audio_recorder_close`: Close recorder
|
||||
- Feeder:
|
||||
- `audio_feeder_open`: Open feeder (supports decoder configuration)
|
||||
- `audio_feeder_run`: Start feeder
|
||||
- `audio_feeder_feed_data`: Feed audio data
|
||||
- `audio_feeder_stop`: Stop feeder
|
||||
- `audio_feeder_close`: Close feeder
|
||||
- Playback:
|
||||
- `audio_playback_open`/`audio_playback_close`: Open/close player
|
||||
- `audio_playback_play`: Play audio (supports URL or local file, e.g., `http://...` or `file:///sdcard/...`)
|
||||
- `audio_playback_stop`: Stop playback
|
||||
- `audio_playback_pause`/`audio_playback_resume`: Pause/resume playback
|
||||
- `audio_playback_get_state`: Get playback state
|
||||
- Mixer:
|
||||
- `audio_processor_mixer_open`: Open mixer (must be called after both playback and feeder are opened)
|
||||
- `audio_processor_mixer_close`: Close mixer
|
||||
- `audio_processor_ramp_control`: Control audio focus and volume fade
|
||||
- Video rendering:
|
||||
- `video_render_open`/`video_render_close`: Open/close video renderer
|
||||
- `video_render_start`/`video_render_stop`: Start/stop rendering
|
||||
- `video_render_frame_feed`: Feed video frame
|
||||
- Video capture:
|
||||
- `video_capture_open`/`video_capture_close`: Open/close video capturer
|
||||
- `video_capture_start`/`video_capture_stop`: Start/stop capture
|
||||
|
||||
---
|
||||
|
||||
## FAQ ❓
|
||||
|
||||
### Audio Issues
|
||||
|
||||
- **"Self-questioning" phenomenon occurs (device playback sound is repeatedly captured by microphone)**:
|
||||
- Enable `MEDIA_DUMP_ENABLE` in `menuconfig`, reproduce the issue and export saved audio data for further analysis
|
||||
- Open the exported audio file using [Audacity](https://www.audacityteam.org/download/), observe waveform/spectrum to determine if saturation or feedback exists
|
||||
- If saturation clipping occurs, reduce microphone gain (`esp_codec_dev_set_in_gain`); or appropriately reduce speaker volume (`esp_codec_dev_set_out_vol`) to avoid excessive feedback
|
||||
|
||||
- **Stack overflow when using OPUS encoder/decoder**:
|
||||
- Ensure `task_stack` of `recorder_task_config` and `feeder_task_config` is set to at least `4096 * 10` bytes (40KB)
|
||||
- Consider allocating task stack in external memory (set `task_stack_in_ext = true`)
|
||||
|
||||
- **Mixer not working properly**:
|
||||
- Ensure that both player (`audio_playback_open`) and feeder (`audio_feeder_open`) are opened before calling `audio_processor_mixer_open()`
|
||||
- Ensure `enable_mixer = true` is set when calling `audio_manager_init`
|
||||
|
||||
### Development and Debugging
|
||||
|
||||
- **Modifying component source code (espressif__av_processor)**:
|
||||
- If you need to modify code within `espressif__av_processor`, you can execute the `script/move_av_processor.py` script
|
||||
- The script will copy `managed_components/espressif__av_processor` to your project's `PROJECT_DIR/components/espressif__av_processor` directory
|
||||
- The script will automatically delete the original directory under `managed_components` and update `idf_component.yml`
|
||||
- Before using, set the environment variable: `export PROJECT_DIR=/path/to/project`
|
||||
|
||||
- **Media data dump (Media Dump)**:
|
||||
- UDP method: Can use `script/udp_reciver.py` script to receive UDP data
|
||||
- SD card method: Ensure SD card is properly mounted and file path is accessible
|
||||
|
||||
---
|
||||
|
||||
## Version Information
|
||||
|
||||
Current version: v0.5.0
|
||||
|
||||
## License
|
||||
|
||||
This component follows the Apache-2.0 license. See the `LICENSE` file included in the repository for details.
|
||||
|
||||
## Related Resources
|
||||
|
||||
- Project repository: https://github.com/espressif/esp-gmf/tree/main/components/av_processor
|
||||
- Issue reporting: https://github.com/espressif/esp-gmf/issues
|
||||
- Documentation: https://github.com/espressif/esp-gmf/blob/main/components/av_processor/README.md
|
||||
@@ -0,0 +1,346 @@
|
||||
# AV Processor 组件(音视频处理)
|
||||
|
||||
本组件提供在 ESP 平台上进行音频录制/播放/喂数以及视频捕获/解码/渲染的统一封装,基于 GMF(General Media Framework)、esp-audio-render、esp-video-codec、esp-capture 等中间组件实现。
|
||||
|
||||
## 功能概览 🎛️
|
||||
|
||||
- **音频 🎧**:
|
||||
- 录音(AEC/VAD/AFE 可选)
|
||||
- 播放(支持 URL、本地文件,解码由配置决定)
|
||||
- 数据喂入(feeder,适合边接收边播放的场景,如 RTC/流媒体)
|
||||
- 可选混音器,多路流间音量渐变(fade)与焦点控制
|
||||
- **视频 🎥**:
|
||||
- 多路视频采集
|
||||
- 视频解码与渲染回调
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 音频使用
|
||||
|
||||
1. 初始化音频管理器:
|
||||
|
||||
```c
|
||||
// 方式一:使用默认配置宏(推荐)
|
||||
audio_manager_config_t cfg = DEFAULT_AUDIO_MANAGER_CONFIG();
|
||||
cfg.play_dev = your_play_dev;
|
||||
cfg.rec_dev = your_rec_dev;
|
||||
strcpy(cfg.mic_layout, "RMNM"); // 麦克风布局(可选,用于 AFE 配置)
|
||||
cfg.board_sample_rate = 16000;
|
||||
cfg.board_bits = 32;
|
||||
cfg.board_channels = 2;
|
||||
cfg.play_volume = 80;
|
||||
cfg.rec_volume = 60;
|
||||
cfg.rec_ref_volume = 60;
|
||||
cfg.enable_mixer = true; // 需要 feeder 与 playback 混音时开启
|
||||
|
||||
audio_manager_init(&cfg);
|
||||
```
|
||||
|
||||
2. 录音(可轮询读取):
|
||||
|
||||
```c
|
||||
// 录音事件回调(可选)
|
||||
void recorder_event_cb(void *event, void *ctx) {
|
||||
// 处理录音事件(如 AFE 事件)
|
||||
}
|
||||
|
||||
// 配置录音器(使用默认配置宏)
|
||||
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
|
||||
// 可选:配置 AFE(运行时配置,优先级高于 Kconfig)
|
||||
recorder_cfg.afe_config.vad_enable = true;
|
||||
recorder_cfg.afe_config.vad_mode = 4;
|
||||
recorder_cfg.afe_config.agc_enable = true;
|
||||
recorder_cfg.recorder_event_cb = recorder_event_cb;
|
||||
|
||||
// 打开录音器
|
||||
audio_recorder_open(&recorder_cfg);
|
||||
uint8_t buf[2048];
|
||||
audio_recorder_read_data(buf, sizeof(buf));
|
||||
|
||||
audio_recorder_close();
|
||||
```
|
||||
|
||||
3. 喂数播放(feeder,适合外部码流→本地播放):
|
||||
|
||||
```c
|
||||
// 配置 feeder(使用默认配置宏)
|
||||
audio_feeder_config_t feeder_cfg = DEFAULT_AUDIO_FEEDER_CONFIG();
|
||||
// 可选:使用 OPUS 解码器时,task_stack 建议 >= 4096 * 10
|
||||
// feeder_cfg.feeder_task_config.task_stack = 4096 * 10;
|
||||
|
||||
audio_feeder_open(&feeder_cfg);
|
||||
audio_feeder_run();
|
||||
// 按需多次喂入外部数据块
|
||||
audio_feeder_feed_data(pkt, pkt_len);
|
||||
|
||||
audio_feeder_stop();
|
||||
audio_feeder_close();
|
||||
```
|
||||
|
||||
4. 普通播放(URL 或本地):
|
||||
|
||||
```c
|
||||
// 配置播放器(使用默认配置宏)
|
||||
audio_playback_config_t playback_cfg = DEFAULT_AUDIO_PLAYBACK_CONFIG();
|
||||
|
||||
audio_playback_open(&playback_cfg);
|
||||
audio_playback_play("http://<ip>:<port>/audio.mp3");
|
||||
|
||||
audio_playback_pause();
|
||||
audio_playback_resume();
|
||||
audio_playback_stop();
|
||||
audio_playback_close();
|
||||
```
|
||||
|
||||
5. 提示播放
|
||||
|
||||
```c
|
||||
// 配置播放器(使用默认配置宏)
|
||||
audio_prompt_config_t prompt_cfg = DEFAULT_AUDIO_PROMPT_CONFIG();
|
||||
|
||||
audio_prompt_open(&prompt_cfg);
|
||||
audio_prompt_play("spiffs://audio.mp3");
|
||||
audio_prompt_close();
|
||||
```
|
||||
> 提示播放 和 配置播放器是相似的,只是 `提示播放` 是阻塞式的播放,适合使用短音频的播放。目的是在使用 普通播放 的播放的时候又需要播放
|
||||
> 提示音,可以有很好的相应
|
||||
|
||||
6. 焦点/渐变控制(启用混音器时):
|
||||
|
||||
```c
|
||||
// 注意:使用混音器前需要先打开播放器和 feeder
|
||||
audio_processor_mixer_open();
|
||||
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_FEEDER); // 侧重喂数音频
|
||||
audio_processor_ramp_control(AUDIO_MIXER_FOCUS_PLAYBACK); // 侧重音频播放
|
||||
audio_processor_mixer_close();
|
||||
```
|
||||
|
||||
### 2. 视频使用
|
||||
|
||||
1. 渲染(解码)回调:
|
||||
|
||||
```c
|
||||
void decoded_cb(void *ctx, const uint8_t *data, size_t size) {
|
||||
// 处理解码后的帧数据
|
||||
}
|
||||
|
||||
video_render_config_t rcfg = {
|
||||
.decode_cfg = your_vdec_cfg,
|
||||
.resolution = {.width = 640, .height = 480},
|
||||
.decode_cb = decoded_cb,
|
||||
};
|
||||
video_render_handle_t r = video_render_open(&rcfg);
|
||||
video_render_start(r);
|
||||
|
||||
video_frame_t f = {.data = enc_frame, .size = enc_size};
|
||||
video_render_frame_feed(r, &f);
|
||||
|
||||
video_render_stop(r);
|
||||
video_render_close(r);
|
||||
```
|
||||
|
||||
1. 采集(多路):
|
||||
|
||||
```c
|
||||
void capture_cb(void *ctx, int index, video_frame_t *frame) {
|
||||
// 处理采集到的视频帧
|
||||
}
|
||||
|
||||
video_capture_config_t ccfg = {0};
|
||||
ccfg.camera_config = &your_cam_cfg;
|
||||
ccfg.sink_num = 2;
|
||||
ccfg.sink_cfg[0] = your_sink0_cfg;
|
||||
ccfg.sink_cfg[1] = your_sink1_cfg;
|
||||
ccfg.capture_frame_cb = capture_cb;
|
||||
|
||||
video_capture_handle_t c = video_capture_open(&ccfg);
|
||||
video_capture_start(c);
|
||||
video_capture_stop(c);
|
||||
video_capture_close(c);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 媒体数据保存(Media Dump)💾
|
||||
|
||||
- 使能条件 ✅:在 `menuconfig` 勾选 `MEDIA_DUMP_ENABLE` 后生效。
|
||||
- 保存方式 📤:支持 SD 卡与 UDP 两种输出(可二选一)。
|
||||
- SD 卡 💾:开启 `CONFIG_MEDIA_DUMP_SINK_SDCARD`,输出到文件 `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`,时长由 `CONFIG_MEDIA_DUMP_DURATION_SEC` 控制。
|
||||
- UDP 📡:开启 `CONFIG_MEDIA_DUMP_SINK_UDP`,通过 `CONFIG_MEDIA_DUMP_UDP_IP` 与 `CONFIG_MEDIA_DUMP_UDP_PORT` 发送原始媒体数据(可使用 `script/udp_reciver.py` 脚本)。
|
||||
- 保存点位 🎚️(AEC 处理前/后):在 `menuconfig` 的“Audio Save Mode (AEC point)”中选择
|
||||
- `MEDIA_DUMP_AUDIO_BEFORE_AEC`(Save Before AEC):保存 AEC 前的麦克风原始音频(便于分析底噪/回授)
|
||||
- `MEDIA_DUMP_AUDIO_AFTER_AEC`(Save After AEC):保存 AEC 处理后的音频(便于评估 AEC 效果)
|
||||
- 典型用途 🔍:问题复现时导出原始音/视频数据,离线用 Audacity 等工具检查饱和、回授、格式是否正确。
|
||||
|
||||
## 配置说明 ⚙️
|
||||
|
||||
### 运行时配置(推荐)
|
||||
|
||||
AFE 配置可以通过 `audio_recorder_config_t` 结构体中的 `afe_config` 字段在运行时配置,优先级高于 Kconfig 配置:
|
||||
|
||||
```c
|
||||
audio_recorder_config_t recorder_cfg = DEFAULT_AUDIO_RECORDER_CONFIG();
|
||||
// 配置 AFE
|
||||
recorder_cfg.afe_config.ai_mode_wakeup = false; // AI 模式:true=唤醒模式,false=直连模式
|
||||
recorder_cfg.afe_config.vad_enable = true; // 使能 VAD
|
||||
recorder_cfg.afe_config.vad_mode = 4; // VAD 模式(1-4),值越大越敏感
|
||||
recorder_cfg.afe_config.vad_min_speech_ms = 64; // 最小语音时长(ms)
|
||||
recorder_cfg.afe_config.vad_min_noise_ms = 1000; // 最小噪声时长(ms)
|
||||
recorder_cfg.afe_config.agc_enable = true; // 使能 AGC
|
||||
recorder_cfg.afe_config.enable_vcmd_detect = false; // 使能 VCMD
|
||||
recorder_cfg.afe_config.vcmd_timeout_ms = 5000; // VCMD 超时(ms)
|
||||
recorder_cfg.afe_config.mn_language = "cn"; // 模型语言:"cn" 或 "en"
|
||||
recorder_cfg.afe_config.wakeup_time_ms = 10000; // 唤醒时间(ms)
|
||||
recorder_cfg.afe_config.wakeup_end_time_ms = 2000; // 唤醒结束时间(ms)
|
||||
|
||||
// 打开录音器时传入配置
|
||||
audio_recorder_open(&recorder_cfg);
|
||||
```
|
||||
|
||||
### Kconfig 配置(备选)
|
||||
|
||||
如果 `audio_recorder_config_t.afe_config` 中的字段使用默认值或未设置,则使用 Kconfig 配置。通过 `idf.py menuconfig` 进入配置菜单进行配置。
|
||||
|
||||
|
||||
#### 💾 Media Dump 配置
|
||||
|
||||
在 `menuconfig` 的 **"Component config" -> "Audio/Video Processor Configuration" -> "Media Dump"** 下配置:
|
||||
|
||||
- **`MEDIA_DUMP_ENABLE`**:使能媒体数据保存功能,默认关闭
|
||||
- 启用后可以保存原始音频/视频数据用于调试分析
|
||||
|
||||
- **`MEDIA_DUMP_AUDIO_POINT`**:音频保存点位选择(choice 类型)
|
||||
- **`MEDIA_DUMP_AUDIO_BEFORE_AEC`**:保存 AEC 处理前的原始音频
|
||||
- 便于分析底噪、回授等问题
|
||||
- **`MEDIA_DUMP_AUDIO_AFTER_AEC`**:保存 AEC 处理后的音频
|
||||
- 便于评估 AEC 处理效果
|
||||
- **`MEDIA_DUMP_AUDIO_NONE`**:不保存音频(默认)
|
||||
|
||||
- **`MEDIA_DUMP_DURATION_SEC`**:保存时长(秒),默认 20
|
||||
- 类型:整数(int)
|
||||
- 范围:1-3600 秒
|
||||
|
||||
- **`MEDIA_DUMP_SINK`**:保存方式选择(choice 类型)
|
||||
- **`MEDIA_DUMP_SINK_SDCARD`**:保存到 SD 卡文件(默认)
|
||||
- 文件路径配置项:`CONFIG_MEDIA_DUMP_SDCARD_FILENAME`(默认:`/sdcard/media_dump.bin`)
|
||||
- 需要确保 SD 卡已正确挂载
|
||||
- **`MEDIA_DUMP_SINK_UDP`**:通过 UDP 发送
|
||||
- 目标 IP 配置项:`CONFIG_MEDIA_DUMP_UDP_IP`(默认:`192.168.1.100`)
|
||||
- 目标端口配置项:`CONFIG_MEDIA_DUMP_UDP_PORT`(默认:5000)
|
||||
- 可使用 `script/udp_reciver.py` 脚本接收数据
|
||||
|
||||
### 任务配置说明
|
||||
|
||||
音频处理组件内部使用多个 FreeRTOS 任务,可以通过各功能模块的配置结构体中的任务配置字段进行自定义:
|
||||
|
||||
#### 各功能模块任务配置
|
||||
|
||||
- **录音器**(`audio_recorder_config_t`):
|
||||
- `afe_feed_task_config`:AFE feed 任务配置(默认栈大小 3KB)
|
||||
- `afe_fetch_task_config`:AFE fetch 任务配置(默认栈大小 3KB)
|
||||
- `recorder_task_config`:录音任务配置(默认栈大小 5KB,使用 OPUS 编码器时建议 >= 40KB)
|
||||
|
||||
- **播放器**(`audio_playback_config_t`):
|
||||
- `playback_task_config`:播放任务配置(默认栈大小 4KB)
|
||||
|
||||
- **Feeder**(`audio_feeder_config_t`):
|
||||
- `feeder_task_config`:Feeder 任务配置(默认栈大小 5KB,使用 OPUS 解码器时建议 >= 40KB)
|
||||
|
||||
**重要提示**:
|
||||
- 当使用 OPUS 编码器或解码器时,`recorder_task_config` 和 `feeder_task_config` 的 `task_stack` 需要设置为至少 `4096 * 10` 字节(40KB)。
|
||||
- 任务配置中的 `task_stack` 设置为 0 时,将使用默认值。
|
||||
- `task_stack_in_ext` 设置为 `true` 时,任务栈将分配在外部内存中,有助于节省内部 RAM。
|
||||
- 任务配置应在各自模块的配置结构体中设置(如 `audio_recorder_config_t`、`audio_playback_config_t`、`audio_feeder_config_t`),而不是在 `audio_manager_config_t` 中。
|
||||
|
||||
---
|
||||
|
||||
## API 参考 📚
|
||||
|
||||
头文件:
|
||||
|
||||
- `include/audio_processor.h`
|
||||
- `include/video_processor.h`
|
||||
- `include/av_processor_type.h`
|
||||
|
||||
常用函数(节选):
|
||||
|
||||
- 音频管理:
|
||||
- `audio_manager_init`/`audio_manager_deinit`:初始化/反初始化音频管理器
|
||||
- 录音:
|
||||
- `audio_recorder_open`:打开录音器(支持编码器配置和事件回调)
|
||||
- `audio_recorder_read_data`:读取录音数据
|
||||
- `audio_recorder_pause`/`audio_recorder_resume`:暂停/恢复录音
|
||||
- `audio_recorder_get_afe_manager_handle`:获取 AFE manager 句柄
|
||||
- `audio_recorder_close`:关闭录音器
|
||||
- 喂数:
|
||||
- `audio_feeder_open`:打开 feeder(支持解码器配置)
|
||||
- `audio_feeder_run`:启动 feeder
|
||||
- `audio_feeder_feed_data`:喂入音频数据
|
||||
- `audio_feeder_stop`:停止 feeder
|
||||
- `audio_feeder_close`:关闭 feeder
|
||||
- 播放:
|
||||
- `audio_playback_open`/`audio_playback_close`:打开/关闭播放器
|
||||
- `audio_playback_play`:播放音频(支持 URL 或本地文件,如 `http://...` 或 `file:///sdcard/...`)
|
||||
- `audio_playback_stop`:停止播放
|
||||
- `audio_playback_pause`/`audio_playback_resume`:暂停/恢复播放
|
||||
- `audio_playback_get_state`:获取播放状态
|
||||
- 混音器:
|
||||
- `audio_processor_mixer_open`:打开混音器(必须在 playback 和 feeder 都打开后调用)
|
||||
- `audio_processor_mixer_close`:关闭混音器
|
||||
- `audio_processor_ramp_control`:控制音频焦点和音量渐变
|
||||
- 视频渲染:
|
||||
- `video_render_open`/`video_render_close`:打开/关闭视频渲染器
|
||||
- `video_render_start`/`video_render_stop`:启动/停止渲染
|
||||
- `video_render_frame_feed`:喂入视频帧
|
||||
- 视频采集:
|
||||
- `video_capture_open`/`video_capture_close`:打开/关闭视频采集器
|
||||
- `video_capture_start`/`video_capture_stop`:启动/停止采集
|
||||
|
||||
---
|
||||
|
||||
## 常见问题 ❓
|
||||
|
||||
### 音频问题
|
||||
|
||||
- **出现"自问自答"现象(设备播放的声音被麦克风重复收录)**:
|
||||
- 在 `menuconfig` 使能 `MEDIA_DUMP_ENABLE`,复现问题并导出保存的音频数据以便进一步分析
|
||||
- 使用 [Audacity](https://www.audacityteam.org/download/) 打开导出的音频文件,观察波形/频谱判断是否存在饱和或回授
|
||||
- 如出现饱和剪切,调低麦克风增益(`esp_codec_dev_set_in_gain`);或适当减小扬声器音量(`esp_codec_dev_set_out_vol`),避免过度回授
|
||||
|
||||
- **OPUS 编解码器使用时出现栈溢出**:
|
||||
- 确保 `recorder_task_config` 和 `feeder_task_config` 的 `task_stack` 设置为至少 `4096 * 10` 字节(40KB)
|
||||
- 考虑将任务栈分配在外部内存(设置 `task_stack_in_ext = true`)
|
||||
|
||||
- **混音器无法正常工作**:
|
||||
- 确保在调用 `audio_processor_mixer_open()` 之前,已经打开了播放器(`audio_playback_open`)和 feeder(`audio_feeder_open`)
|
||||
- 确保在 `audio_manager_init` 时设置了 `enable_mixer = true`
|
||||
|
||||
### 开发调试
|
||||
|
||||
- **修改组件源码(espressif__av_processor)**:
|
||||
- 如需修改 `espressif__av_processor` 内的代码,可执行 `script/move_av_processor.py` 脚本
|
||||
- 脚本会将 `managed_components/espressif__av_processor` 复制到你的工程 `PROJECT_DIR/components/espressif__av_processor` 下
|
||||
- 脚本会自动删除 `managed_components` 下的原目录,并更新 `idf_component.yml`
|
||||
- 使用前需要设置环境变量:`export PROJECT_DIR=/path/to/project`
|
||||
|
||||
- **媒体数据保存(Media Dump)**:
|
||||
- UDP 方式:可使用 `script/udp_reciver.py` 脚本接收 UDP 数据
|
||||
- SD 卡方式:确保 SD 卡已正确挂载,文件路径可访问
|
||||
|
||||
---
|
||||
|
||||
## 版本信息
|
||||
|
||||
当前版本:v0.5.0
|
||||
|
||||
## 许可证
|
||||
|
||||
本组件遵循 Apache-2.0 许可证,详见仓库随附的 `LICENSE`。
|
||||
|
||||
## 相关资源
|
||||
|
||||
- 项目仓库:https://github.com/espressif/esp-gmf/tree/main/components/av_processor
|
||||
- 问题反馈:https://github.com/espressif/esp-gmf/issues
|
||||
- 文档:https://github.com/espressif/esp-gmf/blob/main/components/av_processor/README.md
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "esp_err.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_audio_dec_default.h"
|
||||
#include "esp_audio_enc_default.h"
|
||||
#include "esp_audio_simple_dec_default.h"
|
||||
#include "esp_gmf_audio_dec.h"
|
||||
#include "esp_gmf_audio_enc.h"
|
||||
#include "esp_gmf_ch_cvt.h"
|
||||
#include "esp_gmf_bit_cvt.h"
|
||||
#include "esp_gmf_rate_cvt.h"
|
||||
#include "esp_gmf_pool.h"
|
||||
#include "esp_gmf_element.h"
|
||||
#include "esp_gmf_fade.h"
|
||||
#include "esp_gmf_err.h"
|
||||
#include "av_processor_type.h"
|
||||
|
||||
static char *TAG = "AV_PIPELINE_POOL";
|
||||
|
||||
esp_err_t av_audio_playback_pool_init(esp_gmf_pool_handle_t *pool)
|
||||
{
|
||||
*pool = NULL;
|
||||
esp_gmf_err_t ret = ESP_GMF_ERR_OK;
|
||||
esp_gmf_element_handle_t hd = NULL;
|
||||
esp_err_t err = esp_gmf_pool_init(pool);
|
||||
ESP_GMF_RET_ON_NOT_OK(TAG, err, {return err;}, "Failed to initialize GMF pool");
|
||||
esp_audio_simple_dec_cfg_t es_dec_cfg = DEFAULT_ESP_GMF_AUDIO_DEC_CONFIG();
|
||||
esp_gmf_audio_dec_init(&es_dec_cfg, &hd);
|
||||
esp_gmf_pool_register_element(*pool, hd, NULL);
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t av_audio_playback_pool_deinit(esp_gmf_pool_handle_t pool)
|
||||
{
|
||||
return esp_gmf_pool_deinit(pool);
|
||||
}
|
||||
|
||||
esp_err_t av_audio_feeder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_decoder_config_t *decoder_cfg)
|
||||
{
|
||||
*pool = NULL;
|
||||
esp_err_t ret = ESP_OK;
|
||||
esp_audio_simple_dec_cfg_t dec_cfg = DEFAULT_ESP_GMF_AUDIO_DEC_CONFIG();
|
||||
esp_gmf_element_handle_t el = NULL;
|
||||
esp_gmf_pool_init(pool);
|
||||
|
||||
// Default to PCM if no config provided
|
||||
av_processor_format_id_t format = AV_PROCESSOR_FORMAT_ID_PCM;
|
||||
if (decoder_cfg) {
|
||||
format = decoder_cfg->format;
|
||||
}
|
||||
|
||||
switch (format) {
|
||||
case AV_PROCESSOR_FORMAT_ID_PCM:
|
||||
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_PCM;
|
||||
break;
|
||||
|
||||
case AV_PROCESSOR_FORMAT_ID_G711A: {
|
||||
|
||||
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_G711A;
|
||||
esp_g711_dec_cfg_t g711_dec_cfg = {0};
|
||||
g711_dec_cfg.channel = decoder_cfg ? decoder_cfg->params.g711.audio_info.channels : 1;
|
||||
dec_cfg.dec_cfg = &g711_dec_cfg;
|
||||
dec_cfg.cfg_size = sizeof(esp_g711_dec_cfg_t);
|
||||
break;
|
||||
}
|
||||
|
||||
case AV_PROCESSOR_FORMAT_ID_G711U: {
|
||||
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_G711U;
|
||||
esp_g711_dec_cfg_t g711_dec_cfg = {0};
|
||||
g711_dec_cfg.channel = decoder_cfg ? decoder_cfg->params.g711.audio_info.channels : 1;
|
||||
dec_cfg.dec_cfg = &g711_dec_cfg;
|
||||
dec_cfg.cfg_size = sizeof(esp_g711_dec_cfg_t);
|
||||
break;
|
||||
}
|
||||
|
||||
case AV_PROCESSOR_FORMAT_ID_OPUS: {
|
||||
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_RAW_OPUS;
|
||||
esp_opus_dec_cfg_t opus_dec_cfg = ESP_OPUS_DEC_CONFIG_DEFAULT();
|
||||
if (decoder_cfg) {
|
||||
// Convert frame_duration from int to esp_opus_enc_frame_duration_t
|
||||
int frame_duration = decoder_cfg->params.opus.audio_info.frame_duration;
|
||||
switch (frame_duration) {
|
||||
case 2: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_2_5_MS; break;
|
||||
case 5: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_5_MS; break;
|
||||
case 10: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_10_MS; break;
|
||||
case 20: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_20_MS; break;
|
||||
case 40: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_40_MS; break;
|
||||
case 60: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_60_MS; break;
|
||||
case 80: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_80_MS; break;
|
||||
case 100: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_100_MS; break;
|
||||
case 120: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_120_MS; break;
|
||||
default: opus_dec_cfg.frame_duration = ESP_OPUS_DEC_FRAME_DURATION_60_MS; break;
|
||||
}
|
||||
opus_dec_cfg.channel = decoder_cfg->params.opus.audio_info.channels;
|
||||
opus_dec_cfg.sample_rate = decoder_cfg->params.opus.audio_info.sample_rate;
|
||||
}
|
||||
dec_cfg.dec_cfg = &opus_dec_cfg;
|
||||
dec_cfg.cfg_size = sizeof(esp_opus_dec_cfg_t);
|
||||
break;
|
||||
}
|
||||
|
||||
case AV_PROCESSOR_FORMAT_ID_AAC:
|
||||
dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_AAC;
|
||||
esp_aac_dec_cfg_t aac_dec_cfg = {0};
|
||||
aac_dec_cfg.no_adts_header = decoder_cfg->params.aac.no_adts_header;
|
||||
aac_dec_cfg.aac_plus_enable = decoder_cfg->params.aac.aac_plus_enable;
|
||||
dec_cfg.dec_cfg = &aac_dec_cfg;
|
||||
dec_cfg.cfg_size = sizeof(esp_aac_dec_cfg_t);
|
||||
break;
|
||||
|
||||
default:
|
||||
ESP_LOGE(TAG, "Unsupported format: %d", format);
|
||||
|
||||
return ESP_ERR_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
esp_gmf_audio_dec_init(&dec_cfg, &el);
|
||||
esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
|
||||
esp_ae_ch_cvt_cfg_t es_ch_cvt_cfg = DEFAULT_ESP_GMF_CH_CVT_CONFIG();
|
||||
ret = esp_gmf_ch_cvt_init(&es_ch_cvt_cfg, &el);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio ch cvt");
|
||||
ret = esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
|
||||
|
||||
esp_ae_bit_cvt_cfg_t es_bit_cvt_cfg = DEFAULT_ESP_GMF_BIT_CVT_CONFIG();
|
||||
ret = esp_gmf_bit_cvt_init(&es_bit_cvt_cfg, &el);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio bit cvt");
|
||||
ret = esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
|
||||
|
||||
esp_ae_rate_cvt_cfg_t es_rate_cvt_cfg = DEFAULT_ESP_GMF_RATE_CVT_CONFIG();
|
||||
esp_gmf_rate_cvt_init(&es_rate_cvt_cfg, &el);
|
||||
esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t av_audio_feeder_pool_deinit(esp_gmf_pool_handle_t pool)
|
||||
{
|
||||
return esp_gmf_pool_deinit(pool);
|
||||
}
|
||||
|
||||
esp_err_t av_audio_recorder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_encoder_config_t *encoder_cfg)
|
||||
{
|
||||
*pool = NULL;
|
||||
esp_gmf_err_t ret = ESP_GMF_ERR_OK;
|
||||
esp_audio_enc_config_t enc_cfg = DEFAULT_ESP_GMF_AUDIO_ENC_CONFIG();
|
||||
esp_gmf_element_handle_t el = NULL;
|
||||
esp_err_t err = esp_gmf_pool_init(pool);
|
||||
ESP_GMF_RET_ON_NOT_OK(TAG, err, {return err;}, "Failed to initialize GMF pool");
|
||||
|
||||
esp_gmf_audio_enc_init(&enc_cfg, &el);
|
||||
esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
|
||||
esp_ae_ch_cvt_cfg_t es_ch_cvt_cfg = DEFAULT_ESP_GMF_CH_CVT_CONFIG();
|
||||
ret = esp_gmf_ch_cvt_init(&es_ch_cvt_cfg, &el);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio ch cvt");
|
||||
ret = esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
|
||||
|
||||
esp_ae_bit_cvt_cfg_t es_bit_cvt_cfg = DEFAULT_ESP_GMF_BIT_CVT_CONFIG();
|
||||
ret = esp_gmf_bit_cvt_init(&es_bit_cvt_cfg, &el);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio bit cvt");
|
||||
ret = esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register element in pool");
|
||||
|
||||
esp_ae_rate_cvt_cfg_t es_rate_cvt_cfg = DEFAULT_ESP_GMF_RATE_CVT_CONFIG();
|
||||
esp_gmf_rate_cvt_init(&es_rate_cvt_cfg, &el);
|
||||
esp_gmf_pool_register_element(*pool, el, NULL);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
esp_err_t av_audio_recorder_pool_deinit(esp_gmf_pool_handle_t pool)
|
||||
{
|
||||
return esp_gmf_pool_deinit(pool);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
dependencies:
|
||||
espressif/gmf_ai_audio:
|
||||
version: '~0.7'
|
||||
public: true
|
||||
rules:
|
||||
- if: "target in [esp32p4, esp32s3, esp32]"
|
||||
espressif/esp_audio_simple_player:
|
||||
version: '^0.9.4'
|
||||
public: true
|
||||
espressif/gmf_loader:
|
||||
version: "^0.7"
|
||||
public: true
|
||||
espressif/esp_video_codec:
|
||||
public: true
|
||||
version: "^0.5"
|
||||
espressif/esp_capture:
|
||||
public: true
|
||||
version: "^0.7"
|
||||
espressif/esp_audio_render:
|
||||
public: true
|
||||
version: "^0.7.0"
|
||||
|
||||
description: Audio/Video Processor module for ESP-ADF
|
||||
documentation: https://github.com/espressif/esp-adf/blob/master/components/av_processor/README.md
|
||||
issues: https://github.com/espressif/esp-adf/issues
|
||||
repository: git://github.com/espressif/esp-adf.git
|
||||
tags:
|
||||
- gmf_multimedia
|
||||
- av_processor
|
||||
url: https://github.com/espressif/esp-adf/tree/master/components/av_processor
|
||||
version: 0.1.0
|
||||
@@ -0,0 +1,488 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#include "esp_err.h"
|
||||
#include "esp_gmf_afe_manager.h"
|
||||
#include "esp_audio_simple_player_advance.h"
|
||||
#include "av_processor_type.h"
|
||||
|
||||
/* Brief: Default audio manager configuration
|
||||
* Note: When use OPUS decoder or OPUS encoder, the recorder_task_config and feeder_task_config
|
||||
* task stack size need be set to more than 4096 * 10 bytes.
|
||||
*/
|
||||
#define DEFAULT_AUDIO_MANAGER_CONFIG() { \
|
||||
.play_dev = NULL, \
|
||||
.rec_dev = NULL, \
|
||||
.mic_layout = {0}, \
|
||||
.board_sample_rate = 16000, \
|
||||
.board_bits = 32, \
|
||||
.board_channels = 2, \
|
||||
}
|
||||
|
||||
#define DEFAULT_AUDIO_FEEDER_CONFIG() { \
|
||||
.feeder_task_config = { \
|
||||
.task_stack = 0, \
|
||||
.task_prio = 0, \
|
||||
.task_core = 0, \
|
||||
.task_stack_in_ext = true, \
|
||||
}, \
|
||||
.feeder_mixer_gain = { \
|
||||
.initial_gain = 0.0, \
|
||||
.target_gain = 0.5, \
|
||||
.transition_time = 1500 \
|
||||
}, \
|
||||
}
|
||||
|
||||
#define DEFAULT_AUDIO_PLAYBACK_CONFIG() { \
|
||||
.playback_task_config = { \
|
||||
.task_stack = 4096, \
|
||||
.task_prio = 5, \
|
||||
.task_core = 0, \
|
||||
.task_stack_in_ext = true, \
|
||||
}, \
|
||||
.playback_mixer_gain = { \
|
||||
.initial_gain = 0.0, \
|
||||
.target_gain = 1.0, \
|
||||
.transition_time = 1500 \
|
||||
} \
|
||||
}
|
||||
|
||||
#define DEFAULT_AUDIO_PROMPT_CONFIG() { \
|
||||
.prompt_task_config = { \
|
||||
.task_stack = 4096, \
|
||||
.task_prio = 5, \
|
||||
.task_core = 0, \
|
||||
.task_stack_in_ext = true \
|
||||
} \
|
||||
}
|
||||
|
||||
#define DEFAULT_AUDIO_RECORDER_CONFIG() { \
|
||||
.afe_feed_task_config = { \
|
||||
.task_stack = 40 * 1024, \
|
||||
.task_prio = 5, \
|
||||
.task_core = 0, \
|
||||
.task_stack_in_ext = true, \
|
||||
}, \
|
||||
.afe_fetch_task_config = { \
|
||||
.task_stack = 6 * 1024, \
|
||||
.task_prio = 5, \
|
||||
.task_core = 1, \
|
||||
.task_stack_in_ext = true \
|
||||
}, \
|
||||
.recorder_task_config = { \
|
||||
.task_stack = 4096, \
|
||||
.task_prio = 5, \
|
||||
.task_core = 0, \
|
||||
.task_stack_in_ext = true \
|
||||
}, \
|
||||
.recorder_event_cb = NULL, \
|
||||
.recorder_ctx = NULL, \
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Default AFE configuration macro
|
||||
*/
|
||||
#define DEFAULT_AV_PROCESSOR_AFE_CONFIG() { \
|
||||
.model = "model", \
|
||||
.ai_mode_wakeup = false, \
|
||||
.vad_enable = true, \
|
||||
.vad_mode = 4, \
|
||||
.vad_min_speech_ms = 64, \
|
||||
.vad_min_noise_ms = 1000, \
|
||||
.agc_enable = true, \
|
||||
.afe_type = AFE_TYPE_SR, \
|
||||
.enable_vcmd_detect = false, \
|
||||
.vcmd_timeout_ms = 5000, \
|
||||
.mn_language = "cn", \
|
||||
.wakeup_end_time_ms = 2000, \
|
||||
.wakeup_time_ms = 10000 \
|
||||
}
|
||||
/**
|
||||
* @brief Type definition for the audio recorder event callback function
|
||||
*
|
||||
* @param[in] event Pointer to the event data, which can contain information about the recorder's state
|
||||
* @param[in] ctx User-defined context pointer, passed when registering the callback
|
||||
*/
|
||||
typedef void (*recorder_event_callback_t)(void *event, void *ctx);
|
||||
|
||||
/**
|
||||
* @brief AFE (Audio Front-End) configuration structure
|
||||
*
|
||||
* This structure contains all AFE-related configuration parameters that were previously
|
||||
* configured via Kconfig. These parameters control the behavior of the audio front-end
|
||||
* processing including wake word detection, VAD, AGC, and other audio processing features.
|
||||
*/
|
||||
typedef struct {
|
||||
const char *model; /*!< Model path/name for SR model initialization */
|
||||
bool ai_mode_wakeup; /*!< AI mode: true for WAKEUP mode, false for DIRECT mode */
|
||||
bool vad_enable; /*!< Enable Voice Activity Detection (VAD) */
|
||||
int vad_mode; /*!< VAD mode value (1-4), larger values are more sensitive to human voice */
|
||||
int vad_min_speech_ms; /*!< Minimum speech duration in milliseconds */
|
||||
int vad_min_noise_ms; /*!< Minimum noise duration in milliseconds */
|
||||
bool agc_enable; /*!< Enable Automatic Gain Control (AGC) */
|
||||
afe_type_t afe_type; /*!< AFE type: AFE_TYPE_SR (0) or AFE_TYPE_VC (1) */
|
||||
bool enable_vcmd_detect; /*!< Enable Voice Command Detection */
|
||||
int vcmd_timeout_ms; /*!< Voice command detection timeout in milliseconds */
|
||||
const char *mn_language; /*!< Model language (e.g., "cn", "en") */
|
||||
int wakeup_end_time_ms; /*!< Wakeup end time in milliseconds */
|
||||
int wakeup_time_ms; /*!< Wakeup time in milliseconds */
|
||||
} av_processor_afe_config_t;
|
||||
|
||||
/* Brief: Audio mixer gain configuration */
|
||||
typedef struct {
|
||||
float initial_gain; /*!< Initial gain value (0.0 - 1.0) */
|
||||
float target_gain; /*!< Target gain value (0.0 - 1.0) */
|
||||
int transition_time; /*!< Transition time in milliseconds */
|
||||
} audio_mixer_gain_config_t;
|
||||
|
||||
/* Brief: Common task configuration */
|
||||
typedef struct {
|
||||
int task_stack; /*!< Task stack size (0 means use default) */
|
||||
int task_prio; /*!< Task priority */
|
||||
int task_core; /*!< Task core ID */
|
||||
bool task_stack_in_ext; /*!< Whether to allocate task stack in external memory */
|
||||
} audio_task_config_t;
|
||||
|
||||
typedef struct {
|
||||
audio_task_config_t feeder_task_config; /*!< Feeder task configuration */
|
||||
av_processor_decoder_config_t decoder_cfg; /*!< Decoder configuration */
|
||||
audio_mixer_gain_config_t feeder_mixer_gain; /*!< Feeder mixer gain configuration */
|
||||
} audio_feeder_config_t;
|
||||
|
||||
typedef struct {
|
||||
audio_task_config_t playback_task_config; /*!< Playback task configuration */
|
||||
audio_mixer_gain_config_t playback_mixer_gain; /*!< Playback mixer gain configuration */
|
||||
} audio_playback_config_t;
|
||||
|
||||
typedef struct {
|
||||
audio_task_config_t prompt_task_config; /*!< Prompt task configuration */
|
||||
} audio_prompt_config_t;
|
||||
|
||||
typedef struct {
|
||||
audio_task_config_t afe_feed_task_config; /*!< AFE feed task configuration */
|
||||
av_processor_afe_config_t afe_config; /*!< AFE configuration */
|
||||
audio_task_config_t afe_fetch_task_config; /*!< AFE fetch task configuration */
|
||||
audio_task_config_t recorder_task_config; /*!< Recorder task configuration */
|
||||
av_processor_encoder_config_t encoder_cfg; /*!< Encoder configuration */
|
||||
recorder_event_callback_t recorder_event_cb; /*!< Recorder event callback */
|
||||
void *recorder_ctx; /*!< Recorder context */
|
||||
} audio_recorder_config_t;
|
||||
|
||||
/* Brief: Audio manager configuration */
|
||||
typedef struct {
|
||||
void *play_dev;
|
||||
void *rec_dev;
|
||||
char mic_layout[8];
|
||||
int board_sample_rate;
|
||||
int board_bits;
|
||||
int board_channels;
|
||||
} audio_manager_config_t;
|
||||
|
||||
/**
|
||||
* @brief Audio focus control mode - determines which audio stream gets priority
|
||||
*/
|
||||
typedef enum {
|
||||
AUDIO_MIXER_FOCUS_FEEDER = 0, /*!< Focus on FEEDER stream - boost feeder, reduce playback */
|
||||
AUDIO_MIXER_FOCUS_PLAYBACK = 1, /*!< Focus on PLAYBACK stream - boost playback, reduce feeder */
|
||||
AUDIO_MIXER_FOCUS_BALANCED = 2, /*!< Balanced mode - both streams at equal levels */
|
||||
} audio_mixer_focus_mode_t;
|
||||
|
||||
enum audio_player_state_e {
|
||||
AUDIO_RUN_STATE_IDLE,
|
||||
AUDIO_RUN_STATE_PLAYING,
|
||||
AUDIO_RUN_STATE_CLOSED,
|
||||
AUDIO_RUN_STATE_PAUSED,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Initializes the audio manager module.
|
||||
*
|
||||
* @param[in] config Pointer to the audio manager configuration structure
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_manager_init(audio_manager_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Deinitializes the audio manager component
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_manager_deinit();
|
||||
|
||||
/**
|
||||
* @brief Opens the audio recorder and registers an event callback
|
||||
*
|
||||
* @param[in] config Pointer to the audio recorder configuration structure.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_recorder_open(audio_recorder_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Pauses the audio recorder system
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_recorder_pause(void);
|
||||
|
||||
/**
|
||||
* @brief Resumes the audio recorder system
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_recorder_resume(void);
|
||||
|
||||
/**
|
||||
* @brief Get the AFE manager handle of the audio recorder
|
||||
*
|
||||
* @param[out] afe_manager_handle Pointer to the AFE manager handle
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_recorder_get_afe_manager_handle(esp_gmf_afe_manager_handle_t *afe_manager_handle);
|
||||
|
||||
/**
|
||||
* @brief Closes the audio recorder
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_recorder_close(void);
|
||||
|
||||
/**
|
||||
* @brief Reads audio data from the recorder.
|
||||
*
|
||||
* @param[out] data Pointer to the buffer where the audio data will be stored
|
||||
* @param[in] data_size Size of the buffer to store the audio data.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_recorder_read_data(uint8_t *data, int data_size);
|
||||
|
||||
/**
|
||||
* @brief Opens the audio feeder system
|
||||
*
|
||||
* @param[in] config Pointer to the audio feeder configuration structure.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_feeder_open(audio_feeder_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Closes the audio feeder module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_feeder_close(void);
|
||||
|
||||
/**
|
||||
* @brief Starts the audio feeder operation.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_feeder_run(void);
|
||||
|
||||
/**
|
||||
* @brief Stops the ongoing audio feeder
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On successfully stopping the audio playback
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_feeder_stop(void);
|
||||
|
||||
/**
|
||||
* @brief Feeds audio data into the feeder system
|
||||
*
|
||||
* @param[in] data Pointer to the audio data to be fed into the feeder system
|
||||
* @param[in] data_size Size of the audio data to be fed into the feeder system
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_feeder_feed_data(uint8_t *data, int data_size);
|
||||
|
||||
/**
|
||||
* @brief Open the audio playback module
|
||||
*
|
||||
* @param[in] config Pointer to the audio playback configuration structure.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_open(audio_playback_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Close the audio playback module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_close(void);
|
||||
|
||||
/**
|
||||
* @brief Play an audio playback from the specified URL. Eg: "http://192.168.1.100:8000/audio.mp3", "file:///sdcard/audio.mp3"
|
||||
*
|
||||
* @param[in] url URL pointing to the audio playback file to be played
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_play(const char *url);
|
||||
|
||||
/**
|
||||
* @brief Wait for the audio playback to end. This function will block until the audio playback is ended.
|
||||
*
|
||||
* @param[in] url URL pointing to the audio playback file to be played
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_play_wait_end(const char *url);
|
||||
|
||||
/**
|
||||
* @brief Pauses the audio playback module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_pause(void);
|
||||
/**
|
||||
* @brief Resumes the audio playback module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_resume(void);
|
||||
|
||||
/**
|
||||
* @brief Get the state of the audio playback module
|
||||
*
|
||||
* @return
|
||||
* - enum audio_player_state_e Playback state
|
||||
*/
|
||||
enum audio_player_state_e audio_playback_get_state(void);
|
||||
|
||||
/**
|
||||
* @brief Stop the currently playing audio playback module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_playback_stop(void);
|
||||
|
||||
/**
|
||||
* @brief Open the audio prompt module
|
||||
* @note This function not provide pause and resume functionality, because the prompt is played to end automatically.
|
||||
* This function's priority is higher than the audio playback and feeder, so when the prompt is playing, the audio playback
|
||||
* and feeder decoded data will be ignored.
|
||||
*
|
||||
* @param[in] config Pointer to the audio prompt configuration structure.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_prompt_open(audio_prompt_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Close the audio prompt module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_prompt_close(void);
|
||||
|
||||
/**
|
||||
* @brief Play an audio prompt from the specified URL.
|
||||
*
|
||||
* @param[in] url URL pointing to the audio prompt file to be played
|
||||
* @note Audio prompt is not supported mixer, so when play prompt, the audio playback and feeder decoded data will be ignored.
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_prompt_play(const char *url);
|
||||
|
||||
/**
|
||||
* @brief Open the audio mixer module
|
||||
*
|
||||
* @note This function must be called after audio_playback_open and audio_feeder_open and the audio_mixer must be enabled
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_processor_mixer_open(void);
|
||||
|
||||
/**
|
||||
* @brief Close the audio mixer module
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t audio_processor_mixer_close(void);
|
||||
|
||||
/**
|
||||
* @brief Control audio focus between FEEDER and PLAYBACK streams
|
||||
*
|
||||
* @note This function controls the fade processors for both feeder and playback streams
|
||||
* to create smooth volume transitions and focus on the desired audio source.
|
||||
*
|
||||
* @param[in] mode The focus control mode:
|
||||
* - AUDIO_FOCUS_FEEDER: Focus on FEEDER stream (boost feeder, reduce playback)
|
||||
* - AUDIO_FOCUS_PLAYBACK: Focus on PLAYBACK stream (boost playback, reduce feeder)
|
||||
* - AUDIO_FOCUS_BALANCED: Balanced mode (both streams at equal levels)
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - ESP_ERR_INVALID_ARG Invalid mode parameter
|
||||
* - ESP_ERR_INVALID_STATE Audio render not initialized or streams not available
|
||||
* - ESP_FAIL Failed to control fade processors
|
||||
*/
|
||||
esp_err_t audio_processor_ramp_control(audio_mixer_focus_mode_t mode);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include "esp_afe_config.h"
|
||||
|
||||
/**
|
||||
* @brief AV Processor FourCC definition
|
||||
*/
|
||||
#define AV_PROCESSOR_FOURCC(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
|
||||
|
||||
/**
|
||||
* @brief AV Processor format ID definition
|
||||
*
|
||||
* @note Aligned with GMF FourCC definition for audio video codecs and formats
|
||||
* Detailed info refer to `https://github.com/espressif/esp-gmf/blob/main/gmf_core/helpers/include/esp_fourcc.h`
|
||||
*/
|
||||
typedef enum {
|
||||
AV_PROCESSOR_FORMAT_ID_PCM = AV_PROCESSOR_FOURCC('P', 'C', 'M', ' '), /*!< PCM (Uncompressed) encoder */
|
||||
AV_PROCESSOR_FORMAT_ID_G711A = AV_PROCESSOR_FOURCC('A', 'L', 'A', 'U'), /*!< G.711 A-law encoder */
|
||||
AV_PROCESSOR_FORMAT_ID_G711U = AV_PROCESSOR_FOURCC('U', 'L', 'A', 'W'), /*!< G.711 μ-law encoder */
|
||||
AV_PROCESSOR_FORMAT_ID_OPUS = AV_PROCESSOR_FOURCC('O', 'P', 'U', 'S'), /*!< OPUS encoder */
|
||||
AV_PROCESSOR_FORMAT_ID_AAC = AV_PROCESSOR_FOURCC('A', 'A', 'C', ' '), /*!< AAC encoder */
|
||||
} av_processor_format_id_t;
|
||||
|
||||
/**
|
||||
* @brief Audio encoder format type
|
||||
*/
|
||||
typedef enum {
|
||||
AV_PROCESSOR_ENCODER_PCM = 0, /*!< PCM (Uncompressed) encoder */
|
||||
AV_PROCESSOR_ENCODER_G711A, /*!< G.711 A-law encoder */
|
||||
AV_PROCESSOR_ENCODER_G711U, /*!< G.711 μ-law encoder */
|
||||
AV_PROCESSOR_ENCODER_OPUS, /*!< OPUS encoder */
|
||||
AV_PROCESSOR_ENCODER_MAX, /*!< Maximum encoder type */
|
||||
} av_processor_encoder_format_t;
|
||||
|
||||
/**
|
||||
* @brief Audio decoder format type
|
||||
*/
|
||||
typedef enum {
|
||||
AV_PROCESSOR_DECODER_PCM = 0, /*!< PCM (Uncompressed) decoder */
|
||||
AV_PROCESSOR_DECODER_G711A, /*!< G.711 A-law decoder */
|
||||
AV_PROCESSOR_DECODER_G711U, /*!< G.711 μ-law decoder */
|
||||
AV_PROCESSOR_DECODER_OPUS, /*!< OPUS decoder */
|
||||
AV_PROCESSOR_DECODER_AAC, /*!< AAC decoder */
|
||||
AV_PROCESSOR_DECODER_MAX, /*!< Maximum decoder type */
|
||||
} av_processor_decoder_format_t;
|
||||
|
||||
typedef struct {
|
||||
uint32_t sample_rate; /*!< Sample rate in Hz (e.g., 8000, 16000, 24000, 32000, 44100, 48000) */
|
||||
uint8_t sample_bits; /*!< Sample bits in bits (e.g., 8, 16, 24, 32) */
|
||||
uint8_t channels; /*!< Number of audio channels (1-4) */
|
||||
int frame_duration; /*!< Frame duration in milliseconds */
|
||||
} av_processor_audio_info_t;
|
||||
|
||||
/**
|
||||
* @brief PCM encoder/decoder configuration
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_audio_info_t audio_info;
|
||||
} av_processor_pcm_config_t;
|
||||
|
||||
/**
|
||||
* @brief OPUS encoder configuration
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_audio_info_t audio_info;
|
||||
bool enable_vbr; /*!< Enable Variable Bit Rate (VBR) */
|
||||
int bitrate; /*!< Bitrate in bps */
|
||||
} av_processor_opus_encoder_config_t;
|
||||
|
||||
/**
|
||||
* @brief OPUS decoder configuration
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_audio_info_t audio_info;
|
||||
} av_processor_opus_decoder_config_t;
|
||||
|
||||
/**
|
||||
* @brief AAC decoder configuration
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_audio_info_t audio_info;
|
||||
bool no_adts_header; /*!< Set if AAC frame data not contain ADTS header */
|
||||
bool aac_plus_enable; /*!< Set if need AAC-Plus support */
|
||||
} av_processor_aac_decoder_config_t;
|
||||
|
||||
/**
|
||||
* @brief G.711 encoder/decoder configuration
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_audio_info_t audio_info;
|
||||
} av_processor_g711_config_t;
|
||||
|
||||
typedef struct {
|
||||
av_processor_audio_info_t audio_info;
|
||||
int bitrate; /*!< Bitrate in bps */
|
||||
bool adts_used; /*!< Whether write ADTS header: true - add ADTS header, false - raw aac data only */
|
||||
} av_processor_aac_encoder_config_t;
|
||||
|
||||
/**
|
||||
* @brief Audio encoder configuration structure
|
||||
*
|
||||
* This structure contains the encoder format type and corresponding format-specific parameters.
|
||||
* Only the parameters for the selected format (indicated by format) should be used.
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_format_id_t format; /*!< Encoder format ID */
|
||||
union {
|
||||
av_processor_pcm_config_t pcm; /*!< PCM encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_PCM) */
|
||||
av_processor_opus_encoder_config_t opus; /*!< OPUS encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_OPUS) */
|
||||
av_processor_g711_config_t g711; /*!< G.711 encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_G711A or AV_PROCESSOR_FORMAT_ID_G711U) */
|
||||
av_processor_aac_encoder_config_t aac; /*!< AAC encoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_AAC) */
|
||||
} params; /*!< Format-specific encoder parameters */
|
||||
} av_processor_encoder_config_t;
|
||||
|
||||
/**
|
||||
* @brief Audio decoder configuration structure
|
||||
*
|
||||
* This structure contains the decoder format type and corresponding format-specific parameters.
|
||||
* Only the parameters for the selected format (indicated by format) should be used.
|
||||
*/
|
||||
typedef struct {
|
||||
av_processor_format_id_t format; /*!< Decoder format ID */
|
||||
union {
|
||||
av_processor_pcm_config_t pcm; /*!< PCM decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_PCM) */
|
||||
av_processor_opus_decoder_config_t opus; /*!< OPUS decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_OPUS) */
|
||||
av_processor_g711_config_t g711; /*!< G.711 decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_G711A or AV_PROCESSOR_FORMAT_ID_G711U) */
|
||||
av_processor_aac_decoder_config_t aac; /*!< AAC decoder configuration (used when format == AV_PROCESSOR_FORMAT_ID_AAC) */
|
||||
} params; /*!< Format-specific decoder parameters */
|
||||
} av_processor_decoder_config_t;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
|
||||
*
|
||||
* See LICENSE file for details.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#include "esp_capture_sink.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_video_dec.h"
|
||||
|
||||
/**
|
||||
* @brief Default maximum number of video paths
|
||||
*/
|
||||
#define DEFAULT_VID_MAX_PATH 2
|
||||
|
||||
/**
|
||||
* @brief Video capture handle type
|
||||
*/
|
||||
typedef void *video_capture_handle_t;
|
||||
|
||||
/**
|
||||
* @brief Video render handle type
|
||||
*/
|
||||
typedef void *video_render_handle_t;
|
||||
|
||||
/**
|
||||
* @brief Video frame structure
|
||||
*/
|
||||
typedef struct {
|
||||
uint8_t *data; /*!< Frame data buffer */
|
||||
size_t size; /*!< Frame data size */
|
||||
} video_frame_t;
|
||||
|
||||
/**
|
||||
* @brief Video render decode callback function type
|
||||
*
|
||||
* @param[in] ctx User context pointer
|
||||
* @param[in] data Decoded frame data
|
||||
* @param[in] size Size of decoded frame data
|
||||
*/
|
||||
typedef void (*video_render_decode_callback_t)(void *ctx, const uint8_t *data, size_t size);
|
||||
|
||||
/**
|
||||
* @brief Video capture frame callback function type
|
||||
*
|
||||
* @param[in] ctx User context pointer
|
||||
* @param[in] index Frame index
|
||||
* @param[in] vid_frame Captured video frame
|
||||
*/
|
||||
typedef void (*video_capture_frame_callback_t)(void *ctx, int index, video_frame_t *vid_frame);
|
||||
|
||||
/**
|
||||
* @brief Video render configuration structure
|
||||
*/
|
||||
typedef struct {
|
||||
esp_video_dec_cfg_t decode_cfg; /*!< Video decoder configuration */
|
||||
esp_video_codec_resolution_t resolution; /*!< Video resolution */
|
||||
video_render_decode_callback_t decode_cb; /*!< Decode callback function */
|
||||
void *decode_cb_ctx; /*!< Decode callback context */
|
||||
} video_render_config_t;
|
||||
|
||||
/**
|
||||
* @brief Video capture configuration structure
|
||||
*/
|
||||
typedef struct {
|
||||
void *camera_config; /*!< Camera configuration */
|
||||
int sink_num; /*!< Number of sinks */
|
||||
esp_capture_sink_cfg_t sink_cfg[DEFAULT_VID_MAX_PATH]; /*!< Sink configurations */
|
||||
video_capture_frame_callback_t capture_frame_cb; /*!< Capture frame callback function */
|
||||
void *capture_frame_cb_ctx; /*!< Capture frame callback context */
|
||||
} video_capture_config_t;
|
||||
|
||||
/**
|
||||
* @brief Opens a video capture session
|
||||
*
|
||||
* @param[in] config Video capture configuration
|
||||
*
|
||||
* @return
|
||||
* - video_capture_handle On success
|
||||
* - NULL On failure
|
||||
*/
|
||||
video_capture_handle_t video_capture_open(video_capture_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Closes a video capture session
|
||||
*
|
||||
* @param[in] handle Video capture handle
|
||||
*/
|
||||
void video_capture_close(video_capture_handle_t handle);
|
||||
|
||||
/**
|
||||
* @brief Starts video capture
|
||||
*
|
||||
* @param[in] handle Video capture handle
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t video_capture_start(video_capture_handle_t handle);
|
||||
|
||||
/**
|
||||
* @brief Stops video capture
|
||||
*
|
||||
* @param[in] handle Video capture handle
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t video_capture_stop(video_capture_handle_t handle);
|
||||
|
||||
/**
|
||||
* @brief Opens a video render session
|
||||
*
|
||||
* @param[in] config Video render configuration
|
||||
*
|
||||
* @return
|
||||
* - video_render_handle On success
|
||||
* - NULL On failure
|
||||
*/
|
||||
video_render_handle_t video_render_open(video_render_config_t *config);
|
||||
|
||||
/**
|
||||
* @brief Closes a video render session
|
||||
*
|
||||
* @param[in] handle Video render handle
|
||||
*/
|
||||
void video_render_close(video_render_handle_t handle);
|
||||
|
||||
/**
|
||||
* @brief Starts video rendering
|
||||
*
|
||||
* @param[in] handle Video render handle
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t video_render_start(video_render_handle_t handle);
|
||||
|
||||
/**
|
||||
* @brief Stops video rendering
|
||||
*
|
||||
* @param[in] handle Video render handle
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t video_render_stop(video_render_handle_t handle);
|
||||
|
||||
/**
|
||||
* @brief Feeds a video frame to the render pipeline
|
||||
*
|
||||
* @param[in] handle Video render handle
|
||||
* @param[in] frame Video frame to feed
|
||||
*
|
||||
* @return
|
||||
* - ESP_OK On success
|
||||
* - Other Appropriate esp_err_t error code on failure
|
||||
*/
|
||||
esp_err_t video_render_frame_feed(video_render_handle_t handle, video_frame_t *frame);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
@@ -0,0 +1,224 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
|
||||
*
|
||||
* See LICENSE file for details.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <errno.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "esp_log.h"
|
||||
#include "esp_timer.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
#include "media_dump.h"
|
||||
|
||||
#if CONFIG_MEDIA_DUMP_SINK_UDP
|
||||
#include "lwip/inet.h"
|
||||
#include "lwip/sockets.h"
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
|
||||
|
||||
#if CONFIG_MEDIA_DUMP_ENABLE
|
||||
|
||||
static const char *TAG = "MEDIA_DUMP";
|
||||
|
||||
typedef struct {
|
||||
bool opened;
|
||||
int64_t start_time_us;
|
||||
int64_t max_duration_us;
|
||||
bool expired;
|
||||
bool notified;
|
||||
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
|
||||
FILE *fp;
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
|
||||
#if CONFIG_MEDIA_DUMP_SINK_UDP
|
||||
int sock;
|
||||
struct sockaddr_in dest;
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
|
||||
} media_dump_ctx_t;
|
||||
|
||||
static media_dump_ctx_t s_ctx = {0};
|
||||
|
||||
static inline bool media_dump_time_exceeded(void)
|
||||
{
|
||||
media_dump_ctx_t *ctx = &s_ctx;
|
||||
if (!ctx->opened) {
|
||||
return false;
|
||||
}
|
||||
int64_t now = esp_timer_get_time();
|
||||
if ((now - ctx->start_time_us) >= ctx->max_duration_us) {
|
||||
ctx->expired = true;
|
||||
if (!ctx->notified) {
|
||||
ESP_LOGI(TAG, "Dump duration reached (%ds)", (int)CONFIG_MEDIA_DUMP_DURATION_SEC);
|
||||
ctx->notified = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void media_dump_release(media_dump_ctx_t *ctx)
|
||||
{
|
||||
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
|
||||
if (ctx->fp) {
|
||||
fflush(ctx->fp);
|
||||
fclose(ctx->fp);
|
||||
ctx->fp = NULL;
|
||||
}
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
|
||||
#if CONFIG_MEDIA_DUMP_SINK_UDP
|
||||
if (ctx->sock >= 0) {
|
||||
close(ctx->sock);
|
||||
ctx->sock = -1;
|
||||
}
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
|
||||
ctx->opened = false;
|
||||
}
|
||||
|
||||
int media_dump_open(const char *filename)
|
||||
{
|
||||
media_dump_ctx_t *ctx = &s_ctx;
|
||||
/* allow re-open anytime: close existing and start a new session */
|
||||
if (ctx->opened) {
|
||||
media_dump_release(ctx);
|
||||
}
|
||||
/* reset state and start fresh */
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
ctx->max_duration_us = (int64_t)CONFIG_MEDIA_DUMP_DURATION_SEC * 1000000LL;
|
||||
ctx->start_time_us = esp_timer_get_time();
|
||||
|
||||
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
|
||||
const char *path = NULL;
|
||||
if (filename) {
|
||||
path = filename;
|
||||
} else {
|
||||
path = CONFIG_MEDIA_DUMP_SDCARD_FILENAME;
|
||||
}
|
||||
ctx->fp = fopen(path, "wb");
|
||||
if (!ctx->fp) {
|
||||
ESP_LOGE(TAG, "Dump open file failed: %s, error: %s", path, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
ESP_LOGI(TAG, "Dump file: %s, %ds", path, CONFIG_MEDIA_DUMP_DURATION_SEC);
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
|
||||
|
||||
#if CONFIG_MEDIA_DUMP_SINK_UDP
|
||||
ctx->sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
|
||||
if (ctx->sock < 0) {
|
||||
ESP_LOGE(TAG, "Dump create UDP socket failed");
|
||||
return -1;
|
||||
}
|
||||
memset(&ctx->dest, 0, sizeof(ctx->dest));
|
||||
ctx->dest.sin_family = AF_INET;
|
||||
ctx->dest.sin_port = htons(CONFIG_MEDIA_DUMP_UDP_PORT);
|
||||
ctx->dest.sin_addr.s_addr = inet_addr(CONFIG_MEDIA_DUMP_UDP_IP);
|
||||
ESP_LOGI(TAG, "Dump UDP %s:%d, %ds", CONFIG_MEDIA_DUMP_UDP_IP,
|
||||
CONFIG_MEDIA_DUMP_UDP_PORT, CONFIG_MEDIA_DUMP_DURATION_SEC);
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
|
||||
|
||||
ctx->opened = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int media_dump_write(const uint8_t *data, size_t size)
|
||||
{
|
||||
media_dump_ctx_t *ctx = &s_ctx;
|
||||
if (!ctx->opened || ctx->expired || !data || size == 0) {
|
||||
return -1;
|
||||
}
|
||||
if (media_dump_time_exceeded()) {
|
||||
ESP_LOGW(TAG, "Dump duration reached, ignore write");
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if CONFIG_MEDIA_DUMP_SINK_SDCARD
|
||||
if (ctx->fp) {
|
||||
size_t w = fwrite(data, 1, size, ctx->fp);
|
||||
if (w != size) {
|
||||
ESP_LOGW(TAG, "Dump short file write: %u/%u", (unsigned)w, (unsigned)size);
|
||||
}
|
||||
return (int)w;
|
||||
}
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_SDCARD */
|
||||
|
||||
#if CONFIG_MEDIA_DUMP_SINK_UDP
|
||||
if (ctx->sock >= 0) {
|
||||
int sent = sendto(ctx->sock, data, size, 0, (struct sockaddr *)&ctx->dest, sizeof(ctx->dest));
|
||||
if (sent < 0) {
|
||||
ESP_LOGW(TAG, "Dump UDP send failed");
|
||||
return -1;
|
||||
}
|
||||
return sent;
|
||||
}
|
||||
#endif /* CONFIG_MEDIA_DUMP_SINK_UDP */
|
||||
return -1;
|
||||
}
|
||||
|
||||
int media_dump_close(void)
|
||||
{
|
||||
media_dump_ctx_t *ctx = &s_ctx;
|
||||
if (!ctx->opened) {
|
||||
return 0;
|
||||
}
|
||||
/* normal close: always release immediately */
|
||||
media_dump_release(ctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int media_dump_feed(const char *filename, const uint8_t *data, size_t size)
|
||||
{
|
||||
media_dump_ctx_t *ctx = &s_ctx;
|
||||
/* feed mode: maintain a single open session across calls */
|
||||
if (ctx->expired) {
|
||||
return 0;
|
||||
}
|
||||
if (!ctx->opened) {
|
||||
if (media_dump_open(filename) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
ESP_LOGI(TAG, "Dump duration not reached, write now");
|
||||
int ret = media_dump_write(data, size);
|
||||
/* If duration reached during this write, close now */
|
||||
if (ctx->expired && ctx->opened) {
|
||||
(void)media_dump_close();
|
||||
ESP_LOGI(TAG, "Dump duration reached, close now");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
#else /* !CONFIG_MEDIA_DUMP_ENABLE */
|
||||
|
||||
int media_dump_open(const char *filename)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int media_dump_write(const uint8_t *data, size_t size)
|
||||
{
|
||||
(void)data;
|
||||
(void)size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int media_dump_close(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int media_dump_feed(const char *filename, const uint8_t *data, size_t size)
|
||||
{
|
||||
(void)filename;
|
||||
(void)data;
|
||||
(void)size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* CONFIG_MEDIA_DUMP_ENABLE */
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
|
||||
*
|
||||
* See LICENSE file for details.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "esp_err.h"
|
||||
#include "esp_gmf_pool.h"
|
||||
#include "av_processor_type.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif /* __cplusplus */
|
||||
|
||||
/**
|
||||
* @brief Initialize audio playback pool
|
||||
*
|
||||
* Creates and initializes a memory pool for audio playback operations.
|
||||
* This pool manages memory for audio data that is being played back from
|
||||
* various sources (files, streams, etc.).
|
||||
*
|
||||
* @param pool Pointer to store the created pool handle
|
||||
* @return
|
||||
* - ESP_OK Pool initialized successfully
|
||||
* - ESP_ERR_NO_MEM Insufficient memory to create pool
|
||||
* - ESP_ERR_INVALID_ARG Invalid parameter provided
|
||||
*/
|
||||
esp_err_t av_audio_playback_pool_init(esp_gmf_pool_handle_t *pool);
|
||||
|
||||
/**
|
||||
* @brief Deinitialize audio playback pool
|
||||
*
|
||||
* Destroys the audio playback pool and frees all associated memory.
|
||||
*
|
||||
* @param pool Handle of the pool to deinitialize
|
||||
* @return
|
||||
* - ESP_OK Pool deinitialized successfully
|
||||
* - ESP_ERR_INVALID_ARG Invalid pool handle
|
||||
*/
|
||||
esp_err_t av_audio_playback_pool_deinit(esp_gmf_pool_handle_t pool);
|
||||
|
||||
/**
|
||||
* @brief Initialize audio recorder pool
|
||||
*
|
||||
* @param pool Pointer to store the created pool handle
|
||||
* @param encoder_cfg Pointer to encoder configuration. If NULL, default PCM configuration will be used.
|
||||
* @return
|
||||
* - ESP_OK Pool initialized successfully
|
||||
* - ESP_ERR_NO_MEM Insufficient memory to create pool
|
||||
* - ESP_ERR_INVALID_ARG Invalid parameter provided
|
||||
*/
|
||||
esp_err_t av_audio_recorder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_encoder_config_t *encoder_cfg);
|
||||
|
||||
/**
|
||||
* @brief Deinitialize audio recorder pool
|
||||
*
|
||||
* Destroys the audio recorder pool and frees all associated memory.
|
||||
* Make sure encoder is not running or else may cause memory violation error
|
||||
*
|
||||
* @param pool Handle of the pool to deinitialize
|
||||
* @return
|
||||
* - ESP_OK Pool deinitialized successfully
|
||||
* - ESP_ERR_INVALID_ARG Invalid pool handle
|
||||
*/
|
||||
esp_err_t av_audio_recorder_pool_deinit(esp_gmf_pool_handle_t pool);
|
||||
|
||||
/**
|
||||
* @brief Initialize audio feeder pool
|
||||
*
|
||||
* Creates and initializes a memory pool for audio feeder operations.
|
||||
* This pool manages memory for audio data that is being fed into
|
||||
* various processing pipelines or output streams.
|
||||
*
|
||||
* @param pool Pointer to store the created pool handle
|
||||
* @param decoder_cfg Pointer to decoder configuration. If NULL, default PCM configuration will be used.
|
||||
* @return
|
||||
* - ESP_OK Pool initialized successfully
|
||||
* - ESP_ERR_NO_MEM Insufficient memory to create pool
|
||||
* - ESP_ERR_INVALID_ARG Invalid parameter provided
|
||||
*/
|
||||
esp_err_t av_audio_feeder_pool_init(esp_gmf_pool_handle_t *pool, const av_processor_decoder_config_t *decoder_cfg);
|
||||
|
||||
/**
|
||||
* @brief Deinitialize audio feeder pool
|
||||
*
|
||||
* Destroys the audio feeder pool and frees all associated memory.
|
||||
* Make sure decoder is not running or else may cause memory violation error
|
||||
*
|
||||
* @param pool Handle of the pool to deinitialize
|
||||
* @return
|
||||
* - ESP_OK Pool deinitialized successfully
|
||||
* - ESP_ERR_INVALID_ARG Invalid pool handle
|
||||
*/
|
||||
esp_err_t av_audio_feeder_pool_deinit(esp_gmf_pool_handle_t pool);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
* SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
|
||||
*
|
||||
* See LICENSE file for details.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/**
|
||||
* @brief Open a media dump session
|
||||
*
|
||||
* When enabled by Kconfig, subsequent writes are dumped to the selected sink.
|
||||
* If a session is already open, it will be closed and a new one started.
|
||||
* The session duration is limited by `CONFIG_MEDIA_DUMP_DURATION_SEC`; after
|
||||
* the timeout is reached, additional writes are not accepted.
|
||||
*
|
||||
* Sinks:
|
||||
* - SDCARD: write to `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`
|
||||
* - UDP: send to `CONFIG_MEDIA_DUMP_UDP_ADDRESS` (format: IP:Port, e.g., "192.168.1.100:5000")
|
||||
*
|
||||
* @param[in] filename Optional filename for the dump file
|
||||
* - If NULL, use `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`
|
||||
* - If not NULL, use the filename as the dump file name
|
||||
* @return
|
||||
* - 0: Session opened successfully
|
||||
* - -1: Failed to open (e.g., file/socket creation failure)
|
||||
*/
|
||||
int media_dump_open(const char *filename);
|
||||
|
||||
/**
|
||||
* @brief Write media data to the dump sink
|
||||
*
|
||||
* Effective only when a session is open and not expired.
|
||||
*
|
||||
* @param[in] data Pointer to the data buffer to write
|
||||
* @param[in] size Number of bytes in the buffer
|
||||
* @return
|
||||
* - >0: Bytes actually written/sent (depends on sink)
|
||||
* - 0: Maximum duration reached, this write is ignored
|
||||
* - -1: Session not open, already expired, or parameters invalid
|
||||
*/
|
||||
int media_dump_write(const uint8_t *data, size_t size);
|
||||
|
||||
/**
|
||||
* @brief Close the current media dump session and release resources
|
||||
*
|
||||
* Calling multiple times is safe; if no session is open, it returns immediately.
|
||||
*
|
||||
* @return
|
||||
* - 0: Always returns 0
|
||||
*/
|
||||
int media_dump_close(void);
|
||||
|
||||
/**
|
||||
* @brief Feed data with auto open/close behavior
|
||||
*
|
||||
* If no session is open, media_dump_open() is called automatically. After the
|
||||
* write, if the session duration has been reached, the session is closed
|
||||
* automatically. If the session is already expired, this function returns 0.
|
||||
|
||||
* If want to call this function, can use `extern int media_dump_feed(const char *filename, const uint8_t *data, size_t size);` in the file.
|
||||
*
|
||||
* @param[in] filename Optional filename for the dump file
|
||||
* - If NULL, use `CONFIG_MEDIA_DUMP_SDCARD_FILENAME`
|
||||
* - If not NULL, use the filename as the dump file name
|
||||
* @param[in] data Pointer to the data buffer to write
|
||||
* @param[in] size Number of bytes in the buffer
|
||||
* @return
|
||||
* - Same return semantics as media_dump_write()
|
||||
* - -1: Auto open failed
|
||||
*/
|
||||
int media_dump_feed(const char *filename, const uint8_t *data, size_t size);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(levelname)s: %(message)s",
|
||||
)
|
||||
# Compute source directory (parent of this script's directory, i.e. espressif__av_processor)
|
||||
script_path = Path(__file__).resolve()
|
||||
av_processor_dir = script_path.parent.parent
|
||||
|
||||
if av_processor_dir.name != "espressif__av_processor":
|
||||
logging.error("Failed to resolve source directory. Expected 'espressif__av_processor', got: %s", av_processor_dir)
|
||||
return 1
|
||||
|
||||
# Ensure source espressif__av_processor is under managed_components
|
||||
if av_processor_dir.parent.name != "managed_components":
|
||||
logging.error("This script must be run when espressif__av_processor is under 'managed_components'. Current path: %s", av_processor_dir)
|
||||
return 1
|
||||
|
||||
# Read PROJECT_DIR environment variable
|
||||
project_dir_env = os.environ.get("PROJECT_DIR")
|
||||
if not project_dir_env:
|
||||
logging.error("PROJECT_DIR is not set. Example: export PROJECT_DIR=/path/to/project")
|
||||
return 1
|
||||
|
||||
project_dir = Path(project_dir_env).resolve()
|
||||
if not project_dir.exists() or not project_dir.is_dir():
|
||||
logging.error("PROJECT_DIR does not exist or is not a directory: %s", project_dir)
|
||||
return 1
|
||||
|
||||
# Target components directory
|
||||
components_dir = project_dir / "components"
|
||||
try:
|
||||
components_dir.mkdir(parents=True, exist_ok=True)
|
||||
except Exception as e:
|
||||
logging.error("Failed to create target directory: %s -> %s", components_dir, e)
|
||||
return 1
|
||||
|
||||
# Destination espressif__av_processor path
|
||||
dest_dir = components_dir / "espressif__av_processor"
|
||||
|
||||
# If destination exists, remove it before copying
|
||||
if dest_dir.exists():
|
||||
logging.info("Destination exists, removing: %s", dest_dir)
|
||||
try:
|
||||
shutil.rmtree(dest_dir)
|
||||
except Exception as e:
|
||||
logging.error("Failed to remove existing destination: %s -> %s", dest_dir, e)
|
||||
return 1
|
||||
|
||||
# Perform copy
|
||||
logging.info("Copying %s -> %s", av_processor_dir, dest_dir)
|
||||
try:
|
||||
shutil.copytree(av_processor_dir, dest_dir)
|
||||
except Exception as e:
|
||||
logging.error("Copy failed: %s", e)
|
||||
return 1
|
||||
|
||||
# Switch to a safe working directory to avoid deleting the running/current dir
|
||||
try:
|
||||
os.chdir(project_dir)
|
||||
except Exception as e:
|
||||
logging.error("Failed to change working directory: %s -> %s", project_dir, e)
|
||||
return 1
|
||||
|
||||
# Remove source directory (parent av_processor)
|
||||
logging.info("Removing source directory: %s", av_processor_dir)
|
||||
try:
|
||||
shutil.rmtree(av_processor_dir)
|
||||
except Exception as e:
|
||||
logging.error("Failed to remove source directory: %s -> %s", av_processor_dir, e)
|
||||
logging.info("Copy completed, but source removal failed. Please clean up manually.")
|
||||
return 1
|
||||
|
||||
# Update PROJECT_DIR/main/idf_component.yml to remove 'espressif/av_processor'
|
||||
idf_component_path = project_dir / "main" / "idf_component.yml"
|
||||
if idf_component_path.exists():
|
||||
try:
|
||||
content = idf_component_path.read_text(encoding="utf-8")
|
||||
lines = content.splitlines()
|
||||
new_lines = [line for line in lines if "espressif/av_processor" not in line]
|
||||
if len(new_lines) != len(lines):
|
||||
idf_component_path.write_text("\n".join(new_lines) + ("\n" if content.endswith("\n") else ""), encoding="utf-8")
|
||||
logging.info("Removed 'espressif/av_processor' from %s", idf_component_path)
|
||||
else:
|
||||
logging.info("No 'espressif/av_processor' entry found in %s", idf_component_path)
|
||||
except Exception as e:
|
||||
logging.error("Failed to update %s: %s", idf_component_path, e)
|
||||
return 1
|
||||
else:
|
||||
logging.info("idf_component.yml not found at %s, skipping update", idf_component_path)
|
||||
|
||||
logging.info("Copied espressif__av_processor to PROJECT_DIR/components, removed original directory, and updated idf_component.yml.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
# SPDX-License-Identifier: LicenseRef-Espressif-Modified-MIT
|
||||
#
|
||||
# See LICENSE file for details.
|
||||
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import socket
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import os
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
def get_primary_ip():
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
try:
|
||||
s.connect(("8.8.8.8", 80))
|
||||
return s.getsockname()[0]
|
||||
except Exception:
|
||||
return "127.0.0.1"
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
def gen_filename():
|
||||
ts = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
return f"media_dump_{ts}.bin"
|
||||
|
||||
BUF_SIZE = 10240
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
parser = argparse.ArgumentParser(description="Receive UDP media dump and save to file")
|
||||
parser.add_argument("--port", type=int, default=5000, help="UDP port (default: 5000)")
|
||||
args = parser.parse_args()
|
||||
|
||||
primary_ip = get_primary_ip()
|
||||
logging.info(f"Host primary IP: {primary_ip}, port: {args.port} (set MEDIA_DUMP_UDP_IP:PORT to this)")
|
||||
|
||||
out_path = Path(gen_filename())
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
try:
|
||||
sock.bind(("0.0.0.0", args.port))
|
||||
except OSError as e:
|
||||
logging.error(f"Bind failed on 0.0.0.0:{args.port} - {e}")
|
||||
sys.exit(1)
|
||||
|
||||
logging.info(f"Listening on 0.0.0.0:{args.port}, output file: {out_path.name} (created on first packet), buf={BUF_SIZE}")
|
||||
total = 0
|
||||
f = None
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
data, addr = sock.recvfrom(BUF_SIZE)
|
||||
except OSError:
|
||||
break
|
||||
if not data:
|
||||
logging.info("\nClose received, finishing and exiting")
|
||||
break
|
||||
if f is None:
|
||||
f = open(out_path, "wb")
|
||||
logging.info(f"Receiving... writing to {out_path.resolve()}")
|
||||
f.write(data)
|
||||
total += len(data)
|
||||
sys.stdout.write(f"\r[INFO] recv={len(data)} bytes, total={total} bytes")
|
||||
sys.stdout.flush()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
sock.close()
|
||||
# ensure the progress line ends with newline
|
||||
try:
|
||||
sys.stdout.write("\n")
|
||||
sys.stdout.flush()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if f is not None:
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
f.close()
|
||||
logging.info(f"Saved file: {out_path.resolve()}")
|
||||
else:
|
||||
logging.info("No packets received, no file saved")
|
||||
except Exception as e:
|
||||
logging.error(f"Finalize file failed: {e}")
|
||||
logging.info(f"Done. Total bytes received: {total}")
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,470 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO., LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "freertos/queue.h"
|
||||
|
||||
#include "esp_log.h"
|
||||
#include "driver/gpio.h"
|
||||
#include "esp_gmf_oal_thread.h"
|
||||
#include "esp_gmf_oal_mem.h"
|
||||
#include "esp_video_dec.h"
|
||||
#include "esp_video_dec_reg.h"
|
||||
#include "esp_video_enc_default.h"
|
||||
#include "esp_video_codec_utils.h"
|
||||
#include "esp_video_dec_default.h"
|
||||
#include "esp_video_codec_version.h"
|
||||
#include "esp_video_codec_types.h"
|
||||
#include "esp_capture.h"
|
||||
#include "esp_capture_defaults.h"
|
||||
|
||||
#include "video_processor.h"
|
||||
|
||||
static const char *TAG = "VIDEO_PROCESSOR";
|
||||
|
||||
#define VIDEO_RENDER_QUEUE_DEFAULT_SIZE (5)
|
||||
#define _EVENT_GROUP_STOP_BIT (1 << 0)
|
||||
|
||||
#define VIDEO_DECODER_WORKER_TASK_STACK_SIZE (4096 * 10)
|
||||
#define VIDEO_DECODER_WORKER_TASK_PRIORITY (12)
|
||||
#define VIDEO_DECODER_WORKER_TASK_CORE (0)
|
||||
#define VIDEO_DECODER_WORKER_TASK_NAME "video_decoder_worker"
|
||||
|
||||
#define VIDEO_CAPTURE_WORKER_TASK_STACK_SIZE (4096 * 10)
|
||||
#define VIDEO_CAPTURE_WORKER_TASK_PRIORITY (12)
|
||||
#define VIDEO_CAPTURE_WORKER_TASK_CORE (0)
|
||||
#define VIDEO_CAPTURE_WORKER_TASK_NAME "video_capture_worker"
|
||||
|
||||
typedef enum {
|
||||
VID_PROCESSOR_CMD_STOP,
|
||||
VID_PROCESSOR_CMD_DATA,
|
||||
VID_PROCESSOR_CMD_MAX,
|
||||
} video_processor_cmd_t;
|
||||
|
||||
typedef struct {
|
||||
video_processor_cmd_t cmd;
|
||||
video_frame_t frame;
|
||||
} video_processor_event_t;
|
||||
|
||||
/**
|
||||
* @brief Video render internal structure
|
||||
*/
|
||||
typedef struct {
|
||||
video_render_config_t config; /*!< Render configuration */
|
||||
esp_gmf_oal_thread_t vdec_thread; /*!< Decoder thread handle */
|
||||
QueueHandle_t frame_queue; /*!< Frame queue handle */
|
||||
EventGroupHandle_t event_group; /*!< Event group handle */
|
||||
video_frame_t out_frame; /*!< Output frame buffer */
|
||||
void *ctx; /*!< User context */
|
||||
esp_video_dec_handle_t dec_handle; /*!< Decoder handle */
|
||||
bool is_running; /*!< Running state flag */
|
||||
} video_render_t;
|
||||
|
||||
/**
|
||||
* @brief Video capture internal structure
|
||||
*/
|
||||
typedef struct {
|
||||
video_capture_config_t config; /*!< Capture configuration */
|
||||
esp_capture_handle_t capture; /*!< Capture handle */
|
||||
esp_capture_audio_src_if_t *aud_src; /*!< Audio source interface */
|
||||
esp_capture_video_src_if_t *vid_src; /*!< Video source interface */
|
||||
esp_capture_sink_handle_t sink[DEFAULT_VID_MAX_PATH]; /*!< Sink handles */
|
||||
esp_gmf_oal_thread_t capture_thread; /*!< Capture thread handle */
|
||||
EventGroupHandle_t event_group; /*!< Event group handle */
|
||||
QueueHandle_t frame_queue; /*!< Frame queue handle */
|
||||
bool is_running; /*!< Running state flag */
|
||||
} video_capture_t;
|
||||
|
||||
static esp_capture_video_src_if_t *create_video_source(void *cam_config)
|
||||
{
|
||||
#if CONFIG_IDF_TARGET_ESP32S3
|
||||
return esp_capture_new_video_dvp_src((esp_capture_video_dvp_src_cfg_t *)cam_config);
|
||||
#else
|
||||
ESP_LOGE(TAG, "Video source not supported for this target");
|
||||
return NULL;
|
||||
#endif /* CONFIG_IDF_TARGET_ESP32S3 */
|
||||
}
|
||||
|
||||
static void decoder_worker_thread(void *arg)
|
||||
{
|
||||
video_render_t *render = (video_render_t *)arg;
|
||||
if (!render) {
|
||||
ESP_LOGE(TAG, "Invalid render handle");
|
||||
esp_gmf_oal_thread_delete(NULL);
|
||||
}
|
||||
if (!render->out_frame.data) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory for output frame");
|
||||
esp_gmf_oal_thread_delete(NULL);
|
||||
}
|
||||
render->is_running = true;
|
||||
video_processor_event_t evt_frame = {0};
|
||||
while (render->is_running) {
|
||||
if (xQueueReceive(render->frame_queue, &evt_frame, portMAX_DELAY) == pdPASS) {
|
||||
ESP_LOGD(TAG, "vdec_task: %d", evt_frame.cmd);
|
||||
switch (evt_frame.cmd) {
|
||||
case VID_PROCESSOR_CMD_DATA:
|
||||
ESP_LOGD(TAG, "Processing video frame, size: %zu", evt_frame.frame.size);
|
||||
esp_video_dec_in_frame_t in_frame = {
|
||||
.pts = 0,
|
||||
.data = evt_frame.frame.data,
|
||||
.size = evt_frame.frame.size,
|
||||
};
|
||||
esp_video_dec_out_frame_t out_frame = {
|
||||
.data = render->out_frame.data,
|
||||
.size = render->out_frame.size,
|
||||
};
|
||||
int ret = esp_video_dec_process(render->dec_handle, &in_frame, &out_frame);
|
||||
if (ret != ESP_VC_ERR_OK) {
|
||||
ESP_LOGE(TAG, "Failed to decode frame. ret: %d", ret);
|
||||
break;
|
||||
}
|
||||
if (render->config.decode_cb) {
|
||||
render->config.decode_cb(render->config.decode_cb_ctx, out_frame.data, out_frame.size);
|
||||
}
|
||||
break;
|
||||
case VID_PROCESSOR_CMD_STOP:
|
||||
ESP_LOGI(TAG, "Received stop command, exiting decoder thread");
|
||||
render->is_running = false;
|
||||
break;
|
||||
|
||||
default:
|
||||
ESP_LOGE(TAG, "Unknown command: %d", evt_frame.cmd);
|
||||
break;
|
||||
}
|
||||
if (evt_frame.frame.data) {
|
||||
esp_gmf_oal_free(evt_frame.frame.data);
|
||||
evt_frame.frame.data = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
xEventGroupClearBits(render->event_group, _EVENT_GROUP_STOP_BIT);
|
||||
ESP_LOGI(TAG, "Decoder thread exiting");
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
static void capture_worker_thread(void *arg)
|
||||
{
|
||||
video_capture_t *capture = (video_capture_t *)arg;
|
||||
if (!capture) {
|
||||
ESP_LOGE(TAG, "Invalid capture handle");
|
||||
esp_gmf_oal_thread_delete(NULL);
|
||||
}
|
||||
for (int i = 0; i < capture->config.sink_num; i++) {
|
||||
esp_capture_sink_enable(capture->sink[i], ESP_CAPTURE_RUN_MODE_ALWAYS);
|
||||
}
|
||||
capture->is_running = true;
|
||||
ESP_LOGI(TAG, "Capture thread started");
|
||||
|
||||
uint32_t gcd_fps = capture->config.sink_cfg[0].video_info.fps;
|
||||
for (int i = 1; i < capture->config.sink_num; i++) {
|
||||
uint32_t current_fps = capture->config.sink_cfg[i].video_info.fps;
|
||||
uint32_t a = gcd_fps, b = current_fps;
|
||||
while (b != 0) {
|
||||
uint32_t temp = b;
|
||||
b = a % b;
|
||||
a = temp;
|
||||
}
|
||||
gcd_fps = a;
|
||||
}
|
||||
uint32_t base_delay_ms = 1000 / gcd_fps;
|
||||
ESP_LOGI(TAG, "Multi-path capture: %d paths, gcd_fps: %" PRIu32 ", base_delay: %" PRIu32 "ms",
|
||||
capture->config.sink_num, gcd_fps, base_delay_ms);
|
||||
|
||||
while (capture->is_running) {
|
||||
for (int i = 0; i < capture->config.sink_num; i++) {
|
||||
esp_capture_stream_frame_t frame = {0};
|
||||
frame.stream_type = ESP_CAPTURE_STREAM_TYPE_VIDEO;
|
||||
// Acquire frame from sink
|
||||
esp_err_t ret = esp_capture_sink_acquire_frame(capture->sink[i], &frame, true);
|
||||
if (ret != ESP_CAPTURE_ERR_OK) {
|
||||
ESP_LOGD(TAG, "Acquire frame from sink %d timeout", i);
|
||||
vTaskDelay(pdMS_TO_TICKS(base_delay_ms));
|
||||
continue;
|
||||
}
|
||||
if (frame.stream_type == ESP_CAPTURE_STREAM_TYPE_VIDEO && frame.data && frame.size > 0) {
|
||||
if (capture->config.capture_frame_cb) {
|
||||
video_frame_t vid_frame = {
|
||||
.data = frame.data,
|
||||
.size = frame.size,
|
||||
};
|
||||
capture->config.capture_frame_cb(capture->config.capture_frame_cb_ctx, i, &vid_frame);
|
||||
}
|
||||
}
|
||||
esp_capture_sink_release_frame(capture->sink[i], &frame);
|
||||
}
|
||||
}
|
||||
ESP_LOGI(TAG, "Capture thread exiting");
|
||||
xEventGroupSetBits(capture->event_group, _EVENT_GROUP_STOP_BIT);
|
||||
esp_gmf_oal_thread_delete(NULL);
|
||||
}
|
||||
|
||||
video_render_handle_t video_render_open(video_render_config_t *config)
|
||||
{
|
||||
if (!config) {
|
||||
ESP_LOGE(TAG, "Invalid config");
|
||||
return NULL;
|
||||
}
|
||||
esp_video_dec_register_default();
|
||||
|
||||
video_render_t *render = esp_gmf_oal_calloc(1, sizeof(video_render_t));
|
||||
if (!render) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory for render");
|
||||
return NULL;
|
||||
}
|
||||
render->config.decode_cb_ctx = config->decode_cb_ctx;
|
||||
render->config.decode_cb = config->decode_cb;
|
||||
render->config.decode_cfg = config->decode_cfg;
|
||||
render->config.resolution.width = config->resolution.width;
|
||||
render->config.resolution.height = config->resolution.height;
|
||||
render->frame_queue = xQueueCreate(VIDEO_RENDER_QUEUE_DEFAULT_SIZE, sizeof(video_processor_event_t));
|
||||
if (!render->frame_queue) {
|
||||
goto __exit;
|
||||
}
|
||||
render->event_group = xEventGroupCreate();
|
||||
if (!render->event_group) {
|
||||
goto __exit;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Video render opened successfully");
|
||||
return render;
|
||||
|
||||
__exit:
|
||||
if (render->frame_queue) {
|
||||
vQueueDelete(render->frame_queue);
|
||||
render->frame_queue = NULL;
|
||||
}
|
||||
if (render->event_group) {
|
||||
vEventGroupDelete(render->event_group);
|
||||
render->event_group = NULL;
|
||||
}
|
||||
if (render) {
|
||||
esp_gmf_oal_free(render);
|
||||
render = NULL;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
esp_err_t video_render_start(video_render_handle_t handle)
|
||||
{
|
||||
video_render_t *render = (video_render_t *)handle;
|
||||
if (!render) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
uint32_t actual_size = 0;
|
||||
render->out_frame.size = esp_video_codec_get_image_size(render->config.decode_cfg.out_fmt, &render->config.resolution);
|
||||
render->out_frame.data = esp_video_codec_align_alloc(128, render->out_frame.size, &actual_size);
|
||||
if (!render->out_frame.data) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory for output frame");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
ESP_LOGI(TAG, "Output frame buffer allocated, size: %zu", render->out_frame.size);
|
||||
|
||||
int ret = esp_video_dec_open(&render->config.decode_cfg, &render->dec_handle);
|
||||
if (!render->dec_handle || ret != ESP_VC_ERR_OK) {
|
||||
ESP_LOGE(TAG, "Failed to open video decoder, error: %d", ret);
|
||||
esp_gmf_oal_free(render->out_frame.data);
|
||||
render->out_frame.data = NULL;
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
esp_gmf_oal_thread_create(&render->vdec_thread, VIDEO_DECODER_WORKER_TASK_NAME,
|
||||
decoder_worker_thread, (void *)render,
|
||||
VIDEO_DECODER_WORKER_TASK_STACK_SIZE, VIDEO_DECODER_WORKER_TASK_PRIORITY,
|
||||
true, VIDEO_DECODER_WORKER_TASK_CORE);
|
||||
|
||||
ESP_LOGI(TAG, "Video render started successfully");
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t video_render_frame_feed(video_render_handle_t handle, video_frame_t *vid_frame)
|
||||
{
|
||||
video_render_t *render = (video_render_t *)handle;
|
||||
if (!render) {
|
||||
ESP_LOGE(TAG, "Invalid render handle");
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
if (!vid_frame || !vid_frame->data || vid_frame->size == 0) {
|
||||
ESP_LOGE(TAG, "Invalid frame data");
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
video_processor_event_t evt_frame = {0};
|
||||
evt_frame.cmd = VID_PROCESSOR_CMD_DATA;
|
||||
evt_frame.frame.data = esp_gmf_oal_malloc(vid_frame->size);
|
||||
if (!evt_frame.frame.data) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory for frame data");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
memcpy(evt_frame.frame.data, vid_frame->data, vid_frame->size);
|
||||
evt_frame.frame.size = vid_frame->size;
|
||||
|
||||
if (xQueueSend(render->frame_queue, &evt_frame, portMAX_DELAY) != pdPASS) {
|
||||
ESP_LOGE(TAG, "Failed to send frame to queue");
|
||||
esp_gmf_oal_free(evt_frame.frame.data);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t video_render_stop(video_render_handle_t handle)
|
||||
{
|
||||
video_render_t *render = (video_render_t *)handle;
|
||||
if (!render) {
|
||||
ESP_LOGE(TAG, "Invalid render handle");
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
if (!render->is_running) {
|
||||
ESP_LOGW(TAG, "Video render is not running");
|
||||
return ESP_OK;
|
||||
}
|
||||
video_processor_event_t evt_frame = {0};
|
||||
evt_frame.cmd = VID_PROCESSOR_CMD_STOP;
|
||||
|
||||
if (xQueueSend(render->frame_queue, &evt_frame, portMAX_DELAY) != pdPASS) {
|
||||
ESP_LOGE(TAG, "Failed to send stop command to queue");
|
||||
return ESP_FAIL;
|
||||
}
|
||||
xEventGroupWaitBits(render->event_group, _EVENT_GROUP_STOP_BIT, pdTRUE, pdFALSE, portMAX_DELAY);
|
||||
|
||||
ESP_LOGI(TAG, "Video render stopped successfully");
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
void video_render_close(video_render_handle_t handle)
|
||||
{
|
||||
video_render_t *render = (video_render_t *)handle;
|
||||
if (!render) {
|
||||
ESP_LOGE(TAG, "Invalid render handle");
|
||||
return;
|
||||
}
|
||||
if (render->is_running) {
|
||||
video_render_stop(handle);
|
||||
}
|
||||
if (render->dec_handle) {
|
||||
esp_video_dec_close(render->dec_handle);
|
||||
render->dec_handle = NULL;
|
||||
}
|
||||
if (render->out_frame.data) {
|
||||
esp_gmf_oal_free(render->out_frame.data);
|
||||
render->out_frame.data = NULL;
|
||||
}
|
||||
if (render->frame_queue) {
|
||||
vQueueDelete(render->frame_queue);
|
||||
render->frame_queue = NULL;
|
||||
}
|
||||
esp_gmf_oal_free(render);
|
||||
ESP_LOGI(TAG, "Video render closed successfully");
|
||||
}
|
||||
|
||||
video_capture_handle_t video_capture_open(video_capture_config_t *config)
|
||||
{
|
||||
if (!config) {
|
||||
ESP_LOGE(TAG, "Invalid config");
|
||||
return NULL;
|
||||
}
|
||||
// capture structure
|
||||
video_capture_t *capture = esp_gmf_oal_calloc(1, sizeof(video_capture_t));
|
||||
if (!capture) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory for capture");
|
||||
return NULL;
|
||||
}
|
||||
capture->config = *config;
|
||||
capture->frame_queue = xQueueCreate(2, sizeof(video_processor_event_t));
|
||||
if (!capture->frame_queue) {
|
||||
ESP_LOGE(TAG, "Failed to create frame queue");
|
||||
goto __exit;
|
||||
}
|
||||
capture->event_group = xEventGroupCreate();
|
||||
if (!capture->event_group) {
|
||||
goto __exit;
|
||||
}
|
||||
esp_video_enc_register_default();
|
||||
return capture;
|
||||
|
||||
__exit:
|
||||
if (capture->frame_queue) {
|
||||
vQueueDelete(capture->frame_queue);
|
||||
capture->frame_queue = NULL;
|
||||
}
|
||||
if (capture->event_group) {
|
||||
vEventGroupDelete(capture->event_group);
|
||||
capture->event_group = NULL;
|
||||
}
|
||||
if (capture) {
|
||||
esp_gmf_oal_free(capture);
|
||||
capture = NULL;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
esp_err_t video_capture_start(video_capture_handle_t handle)
|
||||
{
|
||||
video_capture_t *capture = (video_capture_t *)handle;
|
||||
if (!capture) {
|
||||
ESP_LOGE(TAG, "Invalid capture handle");
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
capture->vid_src = create_video_source(capture->config.camera_config);
|
||||
if (!capture->vid_src) {
|
||||
ESP_LOGE(TAG, "Failed to create video source or not support");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
for (int i = 0; i < capture->config.sink_num; i++) {
|
||||
// TODO: support other format
|
||||
esp_capture_video_info_t fixed_caps = {
|
||||
.format_id = ESP_CAPTURE_FMT_ID_RGB565_BE,
|
||||
.width = capture->config.sink_cfg[i].video_info.width,
|
||||
.height = capture->config.sink_cfg[i].video_info.height,
|
||||
.fps = capture->config.sink_cfg[i].video_info.fps,
|
||||
};
|
||||
capture->vid_src->set_fixed_caps(capture->vid_src, &fixed_caps);
|
||||
}
|
||||
esp_capture_cfg_t capture_cfg = {
|
||||
.video_src = capture->vid_src,
|
||||
};
|
||||
esp_capture_open(&capture_cfg, &capture->capture);
|
||||
if (capture->capture == NULL) {
|
||||
ESP_LOGE(TAG, "Fail to create capture");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
for (int i = 0; i < capture->config.sink_num; i++) {
|
||||
esp_capture_sink_cfg_t sink_cfg = capture->config.sink_cfg[i];
|
||||
esp_capture_sink_setup(capture->capture, i, &sink_cfg, &capture->sink[i]);
|
||||
esp_capture_sink_enable(capture->sink[i], ESP_CAPTURE_RUN_MODE_ALWAYS);
|
||||
}
|
||||
esp_capture_start(capture->capture);
|
||||
|
||||
esp_gmf_oal_thread_create(&capture->capture_thread, VIDEO_CAPTURE_WORKER_TASK_NAME, capture_worker_thread,
|
||||
(void *)capture, VIDEO_CAPTURE_WORKER_TASK_STACK_SIZE, VIDEO_CAPTURE_WORKER_TASK_PRIORITY,
|
||||
true, VIDEO_CAPTURE_WORKER_TASK_CORE);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
void video_capture_close(video_capture_handle_t handle)
|
||||
{
|
||||
video_capture_t *capture = (video_capture_t *)handle;
|
||||
if (!capture) {
|
||||
ESP_LOGE(TAG, "Invalid capture handle");
|
||||
return;
|
||||
}
|
||||
if (capture->capture) {
|
||||
esp_capture_stop(capture->capture);
|
||||
esp_capture_close(capture->capture);
|
||||
}
|
||||
capture->is_running = false;
|
||||
xEventGroupWaitBits(capture->event_group, _EVENT_GROUP_STOP_BIT, pdTRUE, pdFALSE, portMAX_DELAY);
|
||||
if (capture->vid_src) {
|
||||
capture->vid_src = NULL;
|
||||
}
|
||||
if (capture->frame_queue) {
|
||||
vQueueDelete(capture->frame_queue);
|
||||
}
|
||||
esp_gmf_oal_free(capture);
|
||||
}
|
||||
Reference in New Issue
Block a user