diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index 0b5472cd282a..631d6fa23eea 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -17,6 +17,10 @@ CONFIG_FORMAT_CONVERT_HIFI3=n CONFIG_SAMPLE_KEYPHRASE=y CONFIG_COMP_PHASE_VOCODER=y CONFIG_COMP_STFT_PROCESS=y +CONFIG_SOF_STAGING=y +CONFIG_CPP=y +CONFIG_STD_CPP17=y +CONFIG_COMP_TENSORFLOW=m # SOF / audio modules / mocks # This mock is part of official sof-bin releases because the CI that diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index b6ac41938398..b514c5e1e756 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -15,6 +15,12 @@ CONFIG_FORMAT_CONVERT_HIFI3=n CONFIG_COMP_GOOGLE_RTC_AUDIO_PROCESSING=m CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_MOCK=y CONFIG_COMP_STFT_PROCESS=y +CONFIG_SOF_STAGING=y +CONFIG_COMP_KPB=y +CONFIG_COMP_TENSORFLOW=y +CONFIG_COMP_MFCC=y +CONFIG_COMP_VOLUME=y +CONFIG_COMP_GAIN=y # SOF / infrastructure CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n @@ -29,7 +35,7 @@ CONFIG_COLD_STORE_EXECUTE_DRAM=y CONFIG_INTEL_MODULES=y CONFIG_LIBRARY_AUTH_SUPPORT=y CONFIG_LIBRARY_MANAGER=y -CONFIG_LIBRARY_BASE_ADDRESS=0xa0688000 +CONFIG_LIBRARY_BASE_ADDRESS=0xa0700000 CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y diff --git a/app/llext_relocatable.conf b/app/llext_relocatable.conf index 76b5339e1bb0..ce8dafe3a6aa 100644 --- a/app/llext_relocatable.conf +++ b/app/llext_relocatable.conf @@ -1 +1,2 @@ CONFIG_LLEXT_TYPE_ELF_RELOCATABLE=y +CONFIG_LLEXT_EXPORT_BUILTINS_BY_SLID=y diff --git a/scripts/llext_link_helper.py b/scripts/llext_link_helper.py index f10777c9918f..333c863d1d21 100755 --- a/scripts/llext_link_helper.py +++ b/scripts/llext_link_helper.py @@ -77,6 +77,8 @@ def main(): command = [args.command] + is_relocatable = '-r' in args.params + executable = [] writable = [] readonly = [] @@ -111,7 +113,8 @@ def main(): text_found = True text_addr = max_alignment(text_addr, 0x1000, s_alignment) text_size = s_size - command.append(f'-Wl,-Ttext=0x{text_addr:x}') + if not is_relocatable: + command.append(f'-Wl,-Ttext=0x{text_addr:x}') else: executable.append(section) @@ -164,7 +167,8 @@ def main(): dram_addr = align_up(dram_addr, s_alignment) - command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') + if not is_relocatable: + command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') dram_addr += section.header['sh_size'] @@ -177,7 +181,8 @@ def main(): dram_addr = align_up(dram_addr, s_alignment) - command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') + if not is_relocatable: + command.append(f'-Wl,--section-start={s_name}=0x{dram_addr:x}') dram_addr += section.header['sh_size'] @@ -189,7 +194,8 @@ def main(): start_addr = align_up(start_addr, s_alignment) - command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') + if not is_relocatable: + command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') start_addr += section.header['sh_size'] @@ -201,10 +207,11 @@ def main(): start_addr = align_up(start_addr, s_alignment) - if s_name == '.data': - command.append(f'-Wl,-Tdata=0x{start_addr:x}') - else: - command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') + if not is_relocatable: + if s_name == '.data': + command.append(f'-Wl,-Tdata=0x{start_addr:x}') + else: + command.append(f'-Wl,--section-start={s_name}=0x{start_addr:x}') start_addr += section.header['sh_size'] diff --git a/scripts/llext_offset_calc.py b/scripts/llext_offset_calc.py index 0f302a8cbe12..2a07984b725f 100755 --- a/scripts/llext_offset_calc.py +++ b/scripts/llext_offset_calc.py @@ -47,6 +47,9 @@ def get_elf_size(elf_name): if section.header['sh_addr'] + section.header['sh_size'] > end: end = section.header['sh_addr'] + section.header['sh_size'] + if start == 0xffffffff: + return 0 + size = end - start return size diff --git a/src/audio/tensorflow/README.md b/src/audio/tensorflow/README.md index a8e3f59ba3ce..7246f5d63c77 100644 --- a/src/audio/tensorflow/README.md +++ b/src/audio/tensorflow/README.md @@ -1,22 +1,154 @@ -# TensorFlow Lite Micro (TFLM) Architecture +# TensorFlow Lite Micro (TFLM) & Wake-on-Voice (WoV) Architecture -This directory acts as the bridge for running ML models. +This directory provides the TensorFlow Lite for Microcontrollers (TFLM) classification module (`TFLMCLY`) for Sound Open Firmware (SOF), including integration with MFCC feature extraction, mtrace logging, IPC host notifications, and Key Phrase Buffer (KPB) Wake-on-Voice (WoV) trigger infrastructure. + +--- ## Overview -Integrates TensorFlow Lite for Microcontrollers into the SOF audio pipeline. Evaluates pre-trained neural network topologies inline with the audio stream for tasks like wake-word, noise cancellation, or sound classification. +The TFLM module evaluates pre-trained micro speech neural network models inline within the SOF audio processing graph. It receives pre-processed audio feature tensors (e.g. 40-bin mel spectrograms from the MFCC component), runs model inference, logs keyword detections to `mtrace`, issues IPC4 notifications to the host audio driver, and signals the KPB module to drain buffered pre-keyword audio for Wake-on-Voice. + +--- + +## Architecture & Data Flow -## Architecture Diagram +### Dual-Path Wake-on-Voice (WoV) Architecture + +To allow continuous keyword evaluation without streaming audio to the host until a keyword is detected, the pipeline separates real-time keyword detection from host PCM draining via KPB: ```mermaid -graph LR - Feat[Audio Features] --> TFLM[TFLM Inference Engine] - SubGraph[FlatBuffer Model] -.-> TFLM - TFLM --> Out[Inference Labels/Scores] +graph TD + DAI[HDA Mic DAI] --> Gain[Gain Component] + Gain --> KPB[KPB Buffer Module] + + subgraph "Real-Time Detection Path (KPB Pin 1)" + KPB -- Live Audio Stream --> SRC[SRC: 48kHz -> 16kHz] + SRC --> MFCC[MFCC Feature Extractor] + MFCC -- 40-bin Mel Tensors --> TFLM[TFLM Classifier: tflmcly] + TFLM --> VSink[Virtual Sink: virtual.tflm_sink] + end + + subgraph "Host Draining Path (KPB Pin 2)" + KPB -- History Draining Stream --> Host[Host Copier: PCM Capture] + end + + TFLM -- "1. Log to mtrace (comp_info)" --> MTrace[mtrace / SOF Trace Log] + TFLM -- "2. IPC4 Host Notification" --> IPC[Host Audio Driver] + TFLM -- "3. KPB_EVENT_BEGIN_DRAINING (notifier_event)" --> KPB +``` + +--- + +## Pipeline Execution & Event Flow + +1. **Continuous Real-Time Listening**: + - Live microphone audio is captured by the HDA DAI and passed into `KPB` (`kpb.2.1`). + - KPB Output Pin 1 continuously streams audio to `SRC` (resampling 48kHz $\to$ 16kHz), `MFCC` (generating 40-bin `int8_t` mel spectrogram features), and `TFLM` (`tflmcly.1.1`). + - KPB stores the raw PCM audio continuously in its circular history buffer (e.g. 2100ms – 3000ms history). + +2. **Inference & Keyword Detection**: + - `tflm_process()` feeds 1960-byte feature tensors ($40 \text{ features} \times 49 \text{ windows}$) into the Micro Speech TFLM interpreter (`[1, 49, 40]` `int8_t` input tensor). + - Upon `TF_ProcessClassify()`, predictions are evaluated across categories: + - `0`: `silence` + - `1`: `unknown` + - `2`: `yes` (Keyword) + - `3`: `no` (Keyword) + +3. **Wake-on-Voice Trigger & Notification**: + When a keyword (`yes` or `no`) is detected with $\ge 0.70$ confidence: + - **mtrace Logging**: Logs detection with keyword label and confidence score via `comp_info()`: + `"TFLM keyword detected: yes (confidence 0.852)"` + - **Host IPC Notification**: Sends an IPC4 module notification (`SOF_IPC4_MODULE_NOTIFICATION`) to inform the host driver. + - **KPB Draining Signal**: Fires a system notification event (`NOTIFIER_ID_KPB_CLIENT_EVT`) with `KPB_EVENT_BEGIN_DRAINING`. + - **Host PCM Draining**: KPB opens Output Pin 2 to `host-copier`, draining pre-keyword history buffer audio followed by live mic audio to the host capture stream. + +--- + +## Topology v2 Integration & Usage + +### 1. Component Widget (`include/components/tflm.conf`) + +Defines `Class.Widget."tflmcly"`: +- **UUID**: `42:c6:1d:c5:e1:a2:df:48:a4:90:e2:74:8c:b6:36:3e` (`c51dc642-a2e1-48df-a490e2748cb6363e`) +- **Type**: `effect` + +### 2. Detection Pipeline Template (`include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf`) + +Instantiates the real-time detection graph: +```conf +Object.Widget { + virtual."1" { name "virtual.tflm_sink" } + src."1" { ... } + mfcc."1" { ... } + tflmcly."1" { ... } +} ``` -## Configuration and Scripts +### 3. Top-Level Topology Configuration (`sof-hda-tflm.conf`) + +Instantiates the complete HDA Mic WoV topology with dual-path KPB routing: +```conf +Object.Base.route [ + # DAI -> Gain -> KPB + { source "dai-copier.HDA.Analog.capture"; sink "gain.2.1" } + { source "gain.2.1"; sink "kpb.2.1" } + + # KPB Pin 1 -> Real-time Detection Path + { source "kpb.2.1"; sink "src.1.1" } + + # KPB Pin 2 -> Host WoV Draining Path + { source "kpb.2.1"; sink "host-copier.0.capture" } +] +``` + +--- + +## Building and Testing Topologies + +### Pre-processing and Compiling with `alsatplg` + +Build `sof-hda-tflm.tplg` using the Topology v2 pre-processor: + +```bash +ALSA_CONFIG_DIR=tools/topology/topology2 \ + tools/bin/alsatplg \ + -I tools/topology/topology2/ \ + -p -c tools/topology/topology2/sof-hda-tflm.conf \ + -o build/sof-hda-tflm.tplg +``` + +### Inspecting Decoded Topology Graphs + +Decode and verify the compiled `.tplg` binary: + +```bash +tools/bin/alsatplg -d build/sof-hda-tflm.tplg -o decoded.txt +grep -A 20 "SectionGraph" decoded.txt +``` + +Expected graph output: +```text +SectionGraph { + set0 { + gain.2.1 <- dai-copier.HDA.Analog.capture + kpb.2.1 <- gain.2.1 + src.1.1 <- kpb.2.1 (Real-Time Detection Path) + host-copier.0.capture <- kpb.2.1 (Host WoV Draining Path) + } + set1 { + mfcc.1.1 <- src.1.1 + tflmcly.1.1 <- mfcc.1.1 + virtual.tflm_sink <- tflmcly.1.1 (Detection Sink Termination) + } +} +``` + +--- + +## Source Files -- **Kconfig**: Enforces requirements for C++17 support and core framework staging logic (`COMP_TENSORFLOW`). -- **CMakeLists.txt**: An intricate build specification linking the Tensilica neural network library block computations (`nn_hifi_lib`) and the TensorFlow Lite micro core engine (`tflm_lib`). Also hooks the `tflm-classify.c` SOF adapter via compiler flags explicitly enforcing memory, precision, and XTENSA optimizations. -- **tflmcly.toml**: Topology definition for the specific TFLM Classifier implementation binding the engine against the UUID `UUIDREG_STR_TFLMCLY`. +- **[tflm-classify.c](file:///home/lrg/work/sof-ptl/sof/src/audio/tensorflow/tflm-classify.c)**: SOF module adapter implementation for TFLM. +- **[speech.cc](file:///home/lrg/work/sof-ptl/sof/src/audio/tensorflow/speech.cc)** / **[speech.h](file:///home/lrg/work/sof-ptl/sof/src/audio/tensorflow/speech.h)**: TFLM C++ API bridge & micro speech tensor wrapper. +- **[tflm.conf](file:///home/lrg/work/sof-ptl/sof/tools/topology/topology2/include/components/tflm.conf)**: Topology v2 widget class definition. +- **[host-gateway-src-mfcc-tflm-capture.conf](file:///home/lrg/work/sof-ptl/sof/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf)**: Detection pipeline template. +- **[sof-hda-tflm.conf](file:///home/lrg/work/sof-ptl/sof/tools/topology/topology2/sof-hda-tflm.conf)**: Top-level WoV topology configuration. diff --git a/src/audio/tensorflow/llext/CMakeLists.txt b/src/audio/tensorflow/llext/CMakeLists.txt index 22bfc22da748..f6a65d66821a 100644 --- a/src/audio/tensorflow/llext/CMakeLists.txt +++ b/src/audio/tensorflow/llext/CMakeLists.txt @@ -10,6 +10,8 @@ set(FLATBUFFERS_PATH ${sof_top_dir}/../flatbuffers) set(GEMMLOWP_PATH ${sof_top_dir}/../gemmlowp) set(RUY_PATH ${sof_top_dir}/../ruy) +if(CMAKE_C_COMPILER_ID STREQUAL "Xtensa") +set(TFLM_EXTRA_LIBS nn_hifi_lib) add_library(nn_hifi_lib STATIC ${NN_HIFI_PATH}/algo/common/src/xa_nnlib_common_api.c ${NN_HIFI_PATH}/algo/kernels/reorg/hifi4/xa_nn_pad_8.c @@ -43,6 +45,7 @@ add_library(nn_hifi_lib STATIC ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_pointwise_sym8sxsym16s.c ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_pointwise_8x16.c ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_circ_buf.c + ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_std_8x8.c ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_conv2d_depthwise.c ${NN_HIFI_PATH}/algo/kernels/cnn/hifi4/xa_nn_matXvec_16x16_16_circ.c @@ -206,18 +209,79 @@ target_compile_options(nn_hifi_lib PRIVATE -fsigned-char -fno-exceptions -mlongcalls - -INLINE:requested - -mcoproc -fno-zero-initialized-in-bss -mtext-section-literals -Wsign-compare - -m32 -DMODEL_INT16 -DNNLIB_V2 -Dhifi4 -DTFLITE_SINGLE_ROUNDING=1 ) +target_compile_options(nn_hifi_lib PRIVATE + -INLINE:requested + -mcoproc + -m32 +) +else() +set(TFLM_EXTRA_LIBS "") +endif() + + + +if(CMAKE_C_COMPILER_ID STREQUAL "Xtensa") + set(TFLM_KERNEL_SOURCES + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/sub.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_hifi.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pooling_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/softmax_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/unidirectional_sequence_lstm.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/transpose_conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected_common_xtensa.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/lstm_eval_hifi.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/add_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/depthwise_conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reshape.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/softmax.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/depthwise_conv_hifi.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_int16_reference.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_int8_reference.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected_int8.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pad_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_int8_int16.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/softmax_int8_int16.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/logistic.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reduce.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/lstm_eval.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reshape_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pad.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reduce_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/dequantize.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/add.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pooling_int8.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/depthwise_conv_vision.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/strided_slice.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_common_xtensa.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/svdf.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/leaky_relu.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/quantize.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pooling.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_vision.cc + ) +else() + set(TFLM_KERNEL_SOURCES + ${TFLM_PATH}/tensorflow/lite/micro/kernels/sub.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/depthwise_conv.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/fully_connected.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/reshape.cc + ${TFLM_PATH}/tensorflow/lite/micro/kernels/softmax.cc + ) +endif() + # TODO: complete sources have been added here from userspace build but # not all are needed so this is a list of "needed" sources to build # a memory and performance optimized TFLM for SOF. @@ -325,46 +389,9 @@ add_library(tflm_lib STATIC #${TFLM_PATH}/tensorflow/lite/micro/kernels/var_handle.cc #${TFLM_PATH}/tensorflow/lite/micro/kernels/while.cc #${TFLM_PATH}/tensorflow/lite/micro/kernels/zeros_like.cc - # xtensa kernels - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/sub.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_hifi.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pooling_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/softmax_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/unidirectional_sequence_lstm.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/transpose_conv.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected_common_xtensa.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/lstm_eval_hifi.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/add_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/depthwise_conv.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reshape.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/softmax.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/depthwise_conv_hifi.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_int16_reference.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_int8_reference.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected_int8.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pad_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_int8_int16.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/softmax_int8_int16.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/logistic.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reduce.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/lstm_eval.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reshape_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pad.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/reduce_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/dequantize.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/add.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pooling_int8.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/depthwise_conv_vision.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/strided_slice.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_common_xtensa.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/svdf.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/fully_connected.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/leaky_relu.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/quantize.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/pooling.cc - ${TFLM_PATH}/tensorflow/lite/micro/kernels/xtensa/conv_vision.cc + + ${TFLM_KERNEL_SOURCES} + ${TFLM_PATH}/tensorflow/lite/micro/mock_micro_graph.cc ${TFLM_PATH}/tensorflow/lite/micro/flatbuffer_utils.cc ${TFLM_PATH}/tensorflow/lite/micro/micro_interpreter_graph.cc @@ -420,7 +447,6 @@ target_include_directories(tflm_lib PRIVATE #endif target_compile_options(tflm_lib PRIVATE -std=c++17 - -stdlib=libc++ -fno-rtti -fno-exceptions -fno-threadsafe-statics @@ -443,14 +469,30 @@ target_compile_options(tflm_lib PRIVATE -DKERNELS_OPTIMIZED_FOR_SPEED -DTF_LITE_MCU_DEBUG_LOG -DTF_LITE_USE_CTIME - --xtensa-core=ace10_LX7HiFi4_2022_10 - -mcoproc -DHIFI4 -mlongcalls -DNNLIB_V2 -Wno-shadow ) +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(tflm_lib PRIVATE -stdlib=libc++) +endif() + +if(CMAKE_C_COMPILER_ID STREQUAL "Xtensa") + target_compile_options(tflm_lib PRIVATE + --xtensa-core=ace10_LX7HiFi4_2022_10 + -mcoproc + ) +endif() + + +if(CMAKE_C_COMPILER_ID STREQUAL "Xtensa") + set(TFLM_STD_LIBS c++ hal) +else() + set(TFLM_STD_LIBS stdc++) +endif() + sof_llext_build("tflm" SOURCES ../tflm-classify.c ../llext-wrap.c @@ -459,7 +501,8 @@ sof_llext_build("tflm" # TFLM is a C++ userspace application so hence we need to link # userspace C++, C, maths and C runtime for llext module usage. - LIBS tflm_lib nn_hifi_lib c++ m c gcc hal + LIBS tflm_lib ${TFLM_EXTRA_LIBS} ${TFLM_STD_LIBS} m c gcc LIBS_PATH . ) + diff --git a/src/audio/tensorflow/tflm-classify.c b/src/audio/tensorflow/tflm-classify.c index 9c93329bce1c..15fbe6e34d2a 100644 --- a/src/audio/tensorflow/tflm-classify.c +++ b/src/audio/tensorflow/tflm-classify.c @@ -32,6 +32,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include "speech.h" SOF_DEFINE_REG_UUID(tflmcly); @@ -44,8 +51,87 @@ static const char * const prediction[] = TFLM_CATEGORY_DATA; struct tflm_comp_data { struct comp_data_blob_handler *model_handler; struct tf_classify tfc; + struct ipc_msg *msg; + struct kpb_event_data event_data; + struct kpb_client client_data; }; +static void tflm_notify_kpb(struct processing_module *mod) +{ + struct tflm_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + + comp_info(dev, "TFLM keyword trigger -> notifying KPB to begin draining"); + + cd->client_data.r_ptr = NULL; + cd->client_data.sink = NULL; + cd->client_data.id = 0; + cd->event_data.event_id = KPB_EVENT_BEGIN_DRAINING; + cd->event_data.client_data = &cd->client_data; + + notifier_event(dev, NOTIFIER_ID_KPB_CLIENT_EVT, + NOTIFIER_TARGET_CORE_ALL_MASK, &cd->event_data, + sizeof(cd->event_data)); +} + +static int tflm_ipc_notification_init(struct processing_module *mod) +{ + struct tflm_comp_data *cd = module_get_private_data(mod); + struct ipc_msg msg_proto; + struct comp_dev *dev = mod->dev; + struct comp_ipc_config *ipc_config = &dev->ipc_config; + union ipc4_notification_header *primary = + (union ipc4_notification_header *)&msg_proto.header; + struct sof_ipc4_notify_module_data *msg_module_data; + struct sof_ipc4_control_msg_payload *msg_payload; + + memset_s(&msg_proto, sizeof(msg_proto), 0, sizeof(msg_proto)); + primary->r.notif_type = SOF_IPC4_MODULE_NOTIFICATION; + primary->r.type = SOF_IPC4_GLB_NOTIFICATION; + primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; + primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; + cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + + sizeof(struct sof_ipc4_control_msg_payload) + + sizeof(struct sof_ipc4_ctrl_value_chan)); + if (!cd->msg) { + comp_err(dev, "Failed to initialize TFLM notification"); + return -ENOMEM; + } + + msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data; + msg_module_data->instance_id = IPC4_INST_ID(ipc_config->id); + msg_module_data->module_id = IPC4_MOD_ID(ipc_config->id); + msg_module_data->event_id = SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL | + SOF_IPC4_SWITCH_CONTROL_PARAM_ID; + msg_module_data->event_data_size = sizeof(struct sof_ipc4_control_msg_payload) + + sizeof(struct sof_ipc4_ctrl_value_chan); + + msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data; + msg_payload->id = 0; + msg_payload->num_elems = 1; + msg_payload->chanv[0].channel = 0; + + comp_dbg(dev, "TFLM notification init: instance_id = 0x%08x, module_id = 0x%08x", + msg_module_data->instance_id, msg_module_data->module_id); + return 0; +} + +static void tflm_send_keyword_notification(struct processing_module *mod, uint32_t category_idx) +{ + struct tflm_comp_data *cd = module_get_private_data(mod); + struct sof_ipc4_notify_module_data *msg_module_data; + struct sof_ipc4_control_msg_payload *msg_payload; + + if (!cd->msg) + return; + + msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data; + msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data; + msg_payload->chanv[0].value = category_idx; + ipc_msg_send(cd->msg, NULL, false); +} + __cold static int tflm_init(struct processing_module *mod) { struct module_data *md = &mod->priv; @@ -97,11 +183,19 @@ __cold static int tflm_init(struct processing_module *mod) goto fail; } + ret = tflm_ipc_notification_init(mod); + if (ret < 0) { + comp_err(dev, "failed to init notification"); + goto fail; + } + return ret; fail: /* Passing NULL pointer to free functions is Ok */ mod_data_blob_handler_free(mod, cd->model_handler); + if (cd->msg) + ipc_msg_free(cd->msg); mod_free(mod, cd); return ret; } @@ -112,6 +206,8 @@ __cold static int tflm_free(struct processing_module *mod) assert_can_be_cold(); + if (cd->msg) + ipc_msg_free(cd->msg); mod_data_blob_handler_free(mod, cd->model_handler); mod_free(mod, cd); return 0; @@ -180,24 +276,23 @@ static int tflm_process(struct processing_module *mod, { struct tflm_comp_data *cd = module_get_private_data(mod); struct comp_dev *dev = mod->dev; - size_t frame_bytes = source_get_frame_bytes(sources[0]); - int features = source_get_data_frames_available(sources[0]); + int features_available = source_get_data_frames_available(sources[0]); const void *data_ptr, *buf_start; size_t buf_size; - int ret; + int ret = 0; comp_dbg(dev, "entry"); - /* Window size is TFLM_FEATURE_ELEM_COUNT and we increment - * by TFLM_FEATURE_SIZE until buffer empty. + /* Window size is TFLM_FEATURE_ELEM_COUNT (1960 bytes int8_t) and we increment + * by TFLM_FEATURE_SIZE (40 bytes int8_t) until buffer empty. */ - while (features >= TFLM_FEATURE_ELEM_COUNT) { - ret = source_get_data(sources[0], TFLM_FEATURE_ELEM_COUNT * frame_bytes, + while (features_available >= TFLM_FEATURE_ELEM_COUNT) { + ret = source_get_data(sources[0], TFLM_FEATURE_ELEM_COUNT, &data_ptr, &buf_start, &buf_size); if (ret) return ret; - cd->tfc.audio_features = data_ptr; + cd->tfc.audio_features = (int8_t *)data_ptr; cd->tfc.audio_data_size = TFLM_FEATURE_ELEM_COUNT; ret = TF_ProcessClassify(&cd->tfc); if (!ret) { @@ -207,15 +302,30 @@ static int tflm_process(struct processing_module *mod, return ret; } - /* debug - dump the output */ + /* Evaluate predictions */ + int max_idx = 0; + float max_score = cd->tfc.predictions[0]; + for (int i = 0; i < cd->tfc.categories; i++) { comp_dbg(dev, "tf: predictions %1.3f %s", cd->tfc.predictions[i], prediction[i]); + if (cd->tfc.predictions[i] > max_score) { + max_score = cd->tfc.predictions[i]; + max_idx = i; + } + } + + /* Check if a keyword ("yes" or "no", category indices 2 and 3) is detected */ + if (max_idx >= 2 && max_score >= 0.70f) { + comp_info(dev, "TFLM keyword detected: %s (confidence %1.3f)", + prediction[max_idx], (double)max_score); + tflm_send_keyword_notification(mod, max_idx); + tflm_notify_kpb(mod); } - /* advance by one stride */ - source_release_data(sources[0], TFLM_FEATURE_SIZE * frame_bytes); - features = source_get_data_frames_available(sources[0]); + /* advance by one 20ms stride (40 int8_t features) */ + source_release_data(sources[0], TFLM_FEATURE_SIZE); + features_available = source_get_data_frames_available(sources[0]); } return ret; diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 29c226eb61a7..87ddc082d409 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -116,6 +116,8 @@ struct lib_manager_module { unsigned int n_dependent; /* For auxiliary modules: number of dependents */ bool mapped; struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS]; + uintptr_t vma_base; + size_t vma_size; }; struct lib_manager_mod_ctx { diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 4c1e4f02d5b5..a098b7b61bcd 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -50,6 +50,101 @@ extern struct tr_ctx lib_manager_tr; #define PAGE_SZ CONFIG_MM_DRV_PAGE_SIZE +#include +#include + +#define LLEXT_LIB_PAGES (CONFIG_LIBRARY_REGION_SIZE / PAGE_SZ) +SYS_BITARRAY_DEFINE_STATIC(lib_vma_bitarray, LLEXT_LIB_PAGES); + +static uintptr_t llext_manager_alloc_vma(size_t size) +{ + size_t num_pages = ALIGN_UP(size, PAGE_SZ) / PAGE_SZ; + size_t offset; + int ret; + + ret = sys_bitarray_alloc(&lib_vma_bitarray, num_pages, &offset); + if (ret < 0) { + tr_err(&lib_manager_tr, "llext_manager_alloc_vma: failed to allocate %zu pages", num_pages); + return 0; + } + + return CONFIG_LIBRARY_BASE_ADDRESS + offset * PAGE_SZ; +} + +void llext_manager_free_vma(uintptr_t vma, size_t size) +{ + if (!vma || !size) + return; + + size_t num_pages = ALIGN_UP(size, PAGE_SZ) / PAGE_SZ; + size_t offset = (vma - CONFIG_LIBRARY_BASE_ADDRESS) / PAGE_SZ; + + sys_bitarray_free(&lib_vma_bitarray, num_pages, offset); +} + +static enum llext_mem llext_manager_get_sec_mem_idx(const char *name, const elf_shdr_t *shdr) +{ + if (strcmp(name, ".exported_sym") == 0) + return LLEXT_MEM_EXPORT; + + switch (shdr->sh_type) { + case SHT_NOBITS: + return LLEXT_MEM_BSS; + case SHT_PROGBITS: + if (shdr->sh_flags & SHF_EXECINSTR) + return LLEXT_MEM_TEXT; + else if (shdr->sh_flags & SHF_WRITE) + return LLEXT_MEM_DATA; + else + return LLEXT_MEM_RODATA; + case SHT_PREINIT_ARRAY: + return LLEXT_MEM_PREINIT; + case SHT_INIT_ARRAY: + return LLEXT_MEM_INIT; + case SHT_FINI_ARRAY: + return LLEXT_MEM_FINI; + default: + return LLEXT_MEM_COUNT; + } +} + +static size_t llext_manager_layout_sections(uint8_t *elf_buf, uintptr_t vma_base) +{ + elf_ehdr_t *hdr = (elf_ehdr_t *)elf_buf; + elf_shdr_t *shdrs = (elf_shdr_t *)(elf_buf + hdr->e_shoff); + elf_shdr_t *shstr_shdr = shdrs + hdr->e_shstrndx; + const char *shstrtab = (const char *)(elf_buf + shstr_shdr->sh_offset); + + uintptr_t current_vma = vma_base; + enum llext_mem last_region = LLEXT_MEM_COUNT; + + for (int i = 0; i < hdr->e_shnum; i++) { + elf_shdr_t *shdr = shdrs + i; + + if (!(shdr->sh_flags & SHF_ALLOC) || shdr->sh_size == 0) + continue; + + const char *name = shstrtab + shdr->sh_name; + enum llext_mem s_region = llext_manager_get_sec_mem_idx(name, shdr); + + if (s_region == LLEXT_MEM_COUNT) + continue; + + if (last_region != LLEXT_MEM_COUNT && last_region != s_region) { + current_vma = ALIGN_UP(current_vma, PAGE_SZ); + } + last_region = s_region; + + current_vma = ALIGN_UP(current_vma, shdr->sh_addralign); + if (vma_base) { + shdr->sh_addr = current_vma; + } + current_vma += shdr->sh_size; + } + + return current_vma - vma_base; +} + static int llext_manager_update_flags(void __sparse_cache *vma, size_t size, uint32_t flags) { size_t pre_pad_size = (uintptr_t)vma & (PAGE_SZ - 1); @@ -383,6 +478,26 @@ static int llext_manager_link(const char *name, } if (!*llext || mctx->mapped) { + if (!*llext) { + uint8_t *elf_buf = (uint8_t *)mctx->ebl->buf; + size_t total_size = llext_manager_layout_sections(elf_buf, 0); + if (total_size == 0) { + tr_err(&lib_manager_tr, "llext_manager_link: layout sections failed"); + return -EINVAL; + } + + uintptr_t vma_base = llext_manager_alloc_vma(total_size); + if (!vma_base) { + tr_err(&lib_manager_tr, "llext_manager_link: VMA allocation failed"); + return -ENOMEM; + } + + mctx->vma_base = vma_base; + mctx->vma_size = total_size; + + llext_manager_layout_sections(elf_buf, vma_base); + } + /* * Either the very first time loading this module, or the module * is already mapped, we just call llext_load() to refcount it @@ -395,8 +510,15 @@ static int llext_manager_link(const char *name, }; ret = llext_load(ldr, name, llext, &ldr_parm); - if (ret) + if (ret) { + tr_err(&lib_manager_tr, "llext_load failed: ret=%d", ret); + if (mctx->vma_base) { + llext_manager_free_vma(mctx->vma_base, mctx->vma_size); + mctx->vma_base = 0; + mctx->vma_size = 0; + } return ret; + } } /* All code sections */ diff --git a/src/library_manager/llext_manager_dram.c b/src/library_manager/llext_manager_dram.c index 2f6cff2b3501..1cc8fad942f8 100644 --- a/src/library_manager/llext_manager_dram.c +++ b/src/library_manager/llext_manager_dram.c @@ -14,6 +14,8 @@ LOG_MODULE_DECLARE(lib_manager, CONFIG_SOF_LOG_LEVEL); +void llext_manager_free_vma(uintptr_t vma, size_t size); + struct lib_manager_dram_storage { struct ext_library ext_lib; struct lib_manager_mod_ctx *ctx; @@ -332,6 +334,9 @@ int llext_manager_restore_from_dram(void) if (mod[k].llext) llext_unload(&mod[k].llext); + if (mod[k].vma_base) + llext_manager_free_vma(mod[k].vma_base, mod[k].vma_size); + if (mod[k].ebl) rfree(mod[k].ebl); } diff --git a/tools/rimage/config/ptl.toml.h b/tools/rimage/config/ptl.toml.h index e7a53d1b1820..dbd843349cb9 100644 --- a/tools/rimage/config/ptl.toml.h +++ b/tools/rimage/config/ptl.toml.h @@ -50,9 +50,7 @@ index = __COUNTER__ #include