From c294fc11d5da43d1b7c21099001ef8d3823d43ea Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 31 Jul 2026 12:02:22 +0930 Subject: [PATCH 1/9] ESP/Speaker: Create log queue --- esp/speaker-netsender/main/Kconfig.projbuild | 26 +++++- esp/speaker-netsender/main/include/log.hpp | 58 +------------ .../main/include/log_udp.hpp | 83 +++++++++++++++++++ esp/speaker-netsender/main/log.cpp | 65 +++++++++++++++ esp/speaker-netsender/sdkconfig | 9 ++ 5 files changed, 180 insertions(+), 61 deletions(-) create mode 100644 esp/speaker-netsender/main/include/log_udp.hpp create mode 100644 esp/speaker-netsender/main/log.cpp diff --git a/esp/speaker-netsender/main/Kconfig.projbuild b/esp/speaker-netsender/main/Kconfig.projbuild index d0f6d10..6789e7f 100644 --- a/esp/speaker-netsender/main/Kconfig.projbuild +++ b/esp/speaker-netsender/main/Kconfig.projbuild @@ -68,7 +68,7 @@ menu "Amplifier" int "AMP I2C Address" range 0 127 # The default amp address is 0x2D (45) - default 45 + default 45 help I2C address of the attached Amplifier. @@ -99,7 +99,7 @@ menu "Amplifier" default 18 help SCL Pin GPIO number. - + endmenu menu "I2S Settings" @@ -117,7 +117,7 @@ menu "Amplifier" default 32 help WS (LRCLK) GPIO number - + config AMP_I2S_DOUT int "I2S Data Out GPIO Pin number" range 1 255 @@ -153,4 +153,22 @@ menu "Amplifier" endmenu -endmenu \ No newline at end of file +endmenu + +menu "Logging" + + config LOG_LEN + int "Log line length" + range 64 512 + default 128 + help + The maximum length of string for a single entry. + + config LOG_QUEUE_LEN + int "Logging queue length" + range 64 512 + default 64 + help + The maximum number of logs to be stored in the queue. Any additional logs will be dropped until the queue frees space. + +endmenu diff --git a/esp/speaker-netsender/main/include/log.hpp b/esp/speaker-netsender/main/include/log.hpp index 2cbed7c..2737ff4 100644 --- a/esp/speaker-netsender/main/include/log.hpp +++ b/esp/speaker-netsender/main/include/log.hpp @@ -1,6 +1,6 @@ /* Name: - log.hpp - Functions for managing logging. + log.hpp - logging layer to split logs between different outputs. Authors: David Sutton @@ -25,59 +25,3 @@ */ #pragma once - -#include -#include - -#include "esp_log_write.h" - -#include "cc.h" -#include "lwip/inet.h" -#include "lwip/sockets.h" - -static auto log_socket = -1; -static struct sockaddr_in dest_addr; - -// Broadcast logging on port 4040. -const constexpr auto LOG_UDP_IP = "255.255.255.255"; -const constexpr auto LOG_UDP_PORT = 4040; - -/** - * @brief vprintf replacement to log via UDP and to serial - * - * udp_logging_vprintf is used to replace the default vprintf - * implementation for ESP logs. - * - * @param[in] fmt format string. - * @param[in] args arguments for format string. - * - * @return bytes written to serial log - */ -inline int udp_logging_vprintf(const char *fmt, va_list args) -{ - char log_buf[512]; - int len = vsnprintf(log_buf, sizeof(log_buf), fmt, args); - - if (len > 0 && log_socket >= 0) { - sendto(log_socket, log_buf, len, 0, (struct sockaddr *)&dest_addr, sizeof(dest_addr)); - } - - // Log to serial. - return vprintf(fmt, args); -} - -/** - * @brief initialise UDP logging - */ -inline void init_udp_logging() -{ - // Create the UDP socket - log_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); - - dest_addr.sin_addr.s_addr = inet_addr(LOG_UDP_IP); - dest_addr.sin_family = AF_INET; - dest_addr.sin_port = htons(LOG_UDP_PORT); - - // Set the log hook - esp_log_set_vprintf(udp_logging_vprintf); -} diff --git a/esp/speaker-netsender/main/include/log_udp.hpp b/esp/speaker-netsender/main/include/log_udp.hpp new file mode 100644 index 0000000..2cbed7c --- /dev/null +++ b/esp/speaker-netsender/main/include/log_udp.hpp @@ -0,0 +1,83 @@ +/* + Name: + log.hpp - Functions for managing logging. + + Authors: + David Sutton + + License: + Copyright (C) 2026 The Australian Ocean Lab (AusOcean). + + This file is part of NetSender. NetSender is free software: you can + redistribute it and/or modify it under the terms of the GNU + General Public License as published by the Free Software + Foundation, either version 3 of the License, or (at your option) + any later version. + + NetSender is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with NetSender in gpl.txt. If not, see + . +*/ + +#pragma once + +#include +#include + +#include "esp_log_write.h" + +#include "cc.h" +#include "lwip/inet.h" +#include "lwip/sockets.h" + +static auto log_socket = -1; +static struct sockaddr_in dest_addr; + +// Broadcast logging on port 4040. +const constexpr auto LOG_UDP_IP = "255.255.255.255"; +const constexpr auto LOG_UDP_PORT = 4040; + +/** + * @brief vprintf replacement to log via UDP and to serial + * + * udp_logging_vprintf is used to replace the default vprintf + * implementation for ESP logs. + * + * @param[in] fmt format string. + * @param[in] args arguments for format string. + * + * @return bytes written to serial log + */ +inline int udp_logging_vprintf(const char *fmt, va_list args) +{ + char log_buf[512]; + int len = vsnprintf(log_buf, sizeof(log_buf), fmt, args); + + if (len > 0 && log_socket >= 0) { + sendto(log_socket, log_buf, len, 0, (struct sockaddr *)&dest_addr, sizeof(dest_addr)); + } + + // Log to serial. + return vprintf(fmt, args); +} + +/** + * @brief initialise UDP logging + */ +inline void init_udp_logging() +{ + // Create the UDP socket + log_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); + + dest_addr.sin_addr.s_addr = inet_addr(LOG_UDP_IP); + dest_addr.sin_family = AF_INET; + dest_addr.sin_port = htons(LOG_UDP_PORT); + + // Set the log hook + esp_log_set_vprintf(udp_logging_vprintf); +} diff --git a/esp/speaker-netsender/main/log.cpp b/esp/speaker-netsender/main/log.cpp new file mode 100644 index 0000000..edbe303 --- /dev/null +++ b/esp/speaker-netsender/main/log.cpp @@ -0,0 +1,65 @@ +/* + Name: + log.cpp - logging layer to split logs between different outputs. + + Authors: + David Sutton + + License: + Copyright (C) 2026 The Australian Ocean Lab (AusOcean). + + This file is part of NetSender. NetSender is free software: you can + redistribute it and/or modify it under the terms of the GNU + General Public License as published by the Free Software + Foundation, either version 3 of the License, or (at your option) + any later version. + + NetSender is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with NetSender in gpl.txt. If not, see + . +*/ + +#pragma once + +#include "include/log.hpp" + +#include +#include +#include +#include +#include + +#include "esp_err.h" +#include "esp_log.h" +#include "esp_log_write.h" +#include "freertos/idf_additions.h" +#include "freertos/projdefs.h" + +// Logging Tag. +static constexpr auto TAG = "log"; + +// Queue item size (max CONFIG_LOG_LEN characters per line) +static constexpr auto QUEUE_ITEM_SIZE = CONFIG_LOG_LEN * sizeof(char); + +// Storage buffer for logs. +static uint8_t queue_storage[CONFIG_LOG_QUEUE_LEN * (QUEUE_ITEM_SIZE)]; +static StaticQueue_t queue_buf; +QueueHandle_t log_queue = NULL; + +esp_err_t init_log_queue(void) +{ + // Create static queue. + log_queue = xQueueCreateStatic(CONFIG_LOG_QUEUE_LEN, CONFIG_LOG_LEN * sizeof(char), queue_storage, &queue_buf); + + if (log_queue == NULL) { + ESP_LOGE(TAG, "Failed to create static log queue"); + return ESP_FAIL; + } + + return ESP_OK; +} diff --git a/esp/speaker-netsender/sdkconfig b/esp/speaker-netsender/sdkconfig index 630b1ef..d5c0146 100644 --- a/esp/speaker-netsender/sdkconfig +++ b/esp/speaker-netsender/sdkconfig @@ -755,6 +755,15 @@ CONFIG_AMP_I2S_SAMPLE_RATE=44100 # end of I2S Settings # end of Amplifier +# +# Logging +# +# default: +CONFIG_LOG_LEN=128 +# default: +CONFIG_LOG_QUEUE_LEN=64 +# end of Logging + # # Netsender # From 1d1111ba1e299f8da51e00fe2ba25cb6b2ccd1ed Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 31 Jul 2026 12:29:06 +0930 Subject: [PATCH 2/9] Speaker: Ignore cache dir --- esp/speaker-netsender/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/esp/speaker-netsender/.gitignore b/esp/speaker-netsender/.gitignore index c500d89..dd30b71 100644 --- a/esp/speaker-netsender/.gitignore +++ b/esp/speaker-netsender/.gitignore @@ -78,3 +78,4 @@ Desktop.ini *.vscode *.zed/ +*.cache From 122aaae879e80e6238da98c5258e058cc4f5fcaf Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 31 Jul 2026 12:35:37 +0930 Subject: [PATCH 3/9] ESP/Speaker: Initialise Logging --- esp/speaker-netsender/main/CMakeLists.txt | 2 +- esp/speaker-netsender/main/include/log.hpp | 14 +++ .../main/include/log_udp.hpp | 30 ++----- esp/speaker-netsender/main/log.cpp | 90 ++++++++++++++++++- esp/speaker-netsender/main/main.cpp | 7 ++ 5 files changed, 116 insertions(+), 27 deletions(-) diff --git a/esp/speaker-netsender/main/CMakeLists.txt b/esp/speaker-netsender/main/CMakeLists.txt index c82750f..14987e3 100755 --- a/esp/speaker-netsender/main/CMakeLists.txt +++ b/esp/speaker-netsender/main/CMakeLists.txt @@ -1,4 +1,4 @@ -idf_component_register(SRCS "main.cpp" "audio.cpp" "ethernet.cpp" "sd.cpp" "self_check.cpp" "time.cpp" +idf_component_register(SRCS "main.cpp" "audio.cpp" "ethernet.cpp" "sd.cpp" "self_check.cpp" "time.cpp" "log.cpp" INCLUDE_DIRS "." "include" REQUIRES esp_driver_sdspi esp_eth esp_netif fatfs sdmmc driver tas5805 netsender ota_tcp pipi) diff --git a/esp/speaker-netsender/main/include/log.hpp b/esp/speaker-netsender/main/include/log.hpp index 2737ff4..9e4b927 100644 --- a/esp/speaker-netsender/main/include/log.hpp +++ b/esp/speaker-netsender/main/include/log.hpp @@ -25,3 +25,17 @@ */ #pragma once + +#include +#include +#include + +/** + * @brief initialise logging. + * + * @returns a read function which returns the number of bytes read or NULL on + * error. + * Initialises multi-output logging. This setup logs to file and via UDP. + * + */ +std::function>()> init_logging(); diff --git a/esp/speaker-netsender/main/include/log_udp.hpp b/esp/speaker-netsender/main/include/log_udp.hpp index 2cbed7c..1ecd177 100644 --- a/esp/speaker-netsender/main/include/log_udp.hpp +++ b/esp/speaker-netsender/main/include/log_udp.hpp @@ -29,8 +29,6 @@ #include #include -#include "esp_log_write.h" - #include "cc.h" #include "lwip/inet.h" #include "lwip/sockets.h" @@ -42,28 +40,16 @@ static struct sockaddr_in dest_addr; const constexpr auto LOG_UDP_IP = "255.255.255.255"; const constexpr auto LOG_UDP_PORT = 4040; -/** - * @brief vprintf replacement to log via UDP and to serial - * - * udp_logging_vprintf is used to replace the default vprintf - * implementation for ESP logs. - * - * @param[in] fmt format string. - * @param[in] args arguments for format string. - * - * @return bytes written to serial log - */ -inline int udp_logging_vprintf(const char *fmt, va_list args) +// Broadcasts the passed message over the configured UDP logging port. +inline int udp_log_send(const char *msg) { - char log_buf[512]; - int len = vsnprintf(log_buf, sizeof(log_buf), fmt, args); + auto len = strlen(msg); - if (len > 0 && log_socket >= 0) { - sendto(log_socket, log_buf, len, 0, (struct sockaddr *)&dest_addr, sizeof(dest_addr)); + if (log_socket >= 0) { + return sendto(log_socket, msg, len, 0, (struct sockaddr *)&dest_addr, sizeof(dest_addr)); } - // Log to serial. - return vprintf(fmt, args); + return -1; } /** @@ -74,10 +60,8 @@ inline void init_udp_logging() // Create the UDP socket log_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); + // Configure destination. dest_addr.sin_addr.s_addr = inet_addr(LOG_UDP_IP); dest_addr.sin_family = AF_INET; dest_addr.sin_port = htons(LOG_UDP_PORT); - - // Set the log hook - esp_log_set_vprintf(udp_logging_vprintf); } diff --git a/esp/speaker-netsender/main/log.cpp b/esp/speaker-netsender/main/log.cpp index edbe303..d6b8f9f 100644 --- a/esp/speaker-netsender/main/log.cpp +++ b/esp/speaker-netsender/main/log.cpp @@ -24,8 +24,6 @@ . */ -#pragma once - #include "include/log.hpp" #include @@ -40,6 +38,8 @@ #include "freertos/idf_additions.h" #include "freertos/projdefs.h" +#include "include/log_udp.hpp" + // Logging Tag. static constexpr auto TAG = "log"; @@ -51,7 +51,49 @@ static uint8_t queue_storage[CONFIG_LOG_QUEUE_LEN * (QUEUE_ITEM_SIZE)]; static StaticQueue_t queue_buf; QueueHandle_t log_queue = NULL; -esp_err_t init_log_queue(void) +// Original vprintf handler so we can still print directly to stdout/serial +static vprintf_like_t default_vprintf = nullptr; + +// custom intercept for logs sent through ESP logging pipeline. +static int log_vprintf(const char *fmt, va_list args) +{ + char log_buf[CONFIG_LOG_LEN]; + + // Format the incoming log string + int len = vsnprintf(log_buf, sizeof(log_buf), fmt, args); + + // Write to standard stdout/UART immediately if original handler is preserved + if (default_vprintf) { + default_vprintf(fmt, args); + } + + // Push formatted string to the log queue + if (log_queue != nullptr && len > 0) { + // Use 0 ticks to prevent blocking calling tasks if the queue gets full + xQueueSend(log_queue, log_buf, 0); + } + + return len; +} + +// Task to handle logs sent through the queue. This is handled asynchronously +// to logging to prevent blocking network and file writes. +static void log_processor_task(void *pvParameters) +{ + char msg[CONFIG_LOG_LEN]; + + while (true) { + // Block until a new log message arrives in the queue. + if (xQueueReceive(log_queue, msg, portMAX_DELAY) == pdTRUE) { + udp_log_send(msg); + // TODO: Log to file. + } + } +} + +// Initialises logging queue which is used to queue logs to prevent blocking +// calls to file or network. +esp_err_t init_log_queue() { // Create static queue. log_queue = xQueueCreateStatic(CONFIG_LOG_QUEUE_LEN, CONFIG_LOG_LEN * sizeof(char), queue_storage, &queue_buf); @@ -63,3 +105,45 @@ esp_err_t init_log_queue(void) return ESP_OK; } + +std::function>()> init_logging() +{ + // Initialise queue. + auto err = init_log_queue(); + if (err != ESP_OK) { + ESP_LOGE(TAG, "unable to init log queue: %s", esp_err_to_name(err)); + return NULL; + } + + // Initialise UDP logging. + init_udp_logging(); + + // TODO: Initialise file logging. + + // Start logging task. + if (xTaskCreatePinnedToCore(log_processor_task, "logging_task", 4096, NULL, 5, NULL, 1) != pdPASS) { + ESP_LOGE(TAG, "log task creation failed"); + return NULL; + } + + default_vprintf = esp_log_set_vprintf(log_vprintf); + + p = std::make_unique("/sdcard/logs"); + + return []() -> std::optional> { + auto &stream = p->get_logs(); + + std::vector buffer(1024); // Allocate space for logs + + // Read logs from stream into buffer + stream.read(reinterpret_cast(buffer.data()), buffer.size()); + + auto bytes_read = stream.gcount(); + if (bytes_read == 0) { + return std::nullopt; + } + + buffer.resize(bytes_read); // Shrink to actual payload size + return buffer; + }; +} diff --git a/esp/speaker-netsender/main/main.cpp b/esp/speaker-netsender/main/main.cpp index 6d92bc1..102e25f 100644 --- a/esp/speaker-netsender/main/main.cpp +++ b/esp/speaker-netsender/main/main.cpp @@ -44,6 +44,7 @@ #include "include/sd.hpp" #include "include/self_check.hpp" #include "include/time.hpp" +#include "log.hpp" #include "netsender.hpp" #include "ota_tcp.hpp" #include "tas5805.hpp" @@ -142,6 +143,12 @@ void app_main(void) initialise_sntp(); ESP_LOGI(TAG, "SNTP Initialised"); + ESP_LOGI(TAG, "Initialising Logging"); + auto read_bin_func = init_logging(); + char log_pin_name[4] = "T0"; + ns.register_binary_input(log_pin_name, read_bin_func); + ESP_LOGI(TAG, "Logging Initialised"); + ESP_LOGI(TAG, "Initialising OTA listener"); init_ota_tcp(); ESP_LOGI(TAG, "OTA Initialised"); From a1a2a32d6fb9f611cf6d4654c5f69392d524f383 Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 7 Aug 2026 09:03:52 +0930 Subject: [PATCH 4/9] Speaker: Handle intercept logging This change adds a new logging layer which handles the intercept logging from the ESP logging abstraction. --- .../components/pipi/include/pipi.hpp | 41 ++---------- .../components/pipi/pipi.cpp | 66 ++++++++----------- esp/speaker-netsender/main/log.cpp | 8 ++- 3 files changed, 36 insertions(+), 79 deletions(-) diff --git a/esp/speaker-netsender/components/pipi/include/pipi.hpp b/esp/speaker-netsender/components/pipi/include/pipi.hpp index 190fac9..3afb722 100644 --- a/esp/speaker-netsender/components/pipi/include/pipi.hpp +++ b/esp/speaker-netsender/components/pipi/include/pipi.hpp @@ -130,36 +130,12 @@ class FileLogger { FileLogger(); /** - * @brief Log an info level log. + * @brief Writes an ESP pre-formatted log. * - * @param[in] msg format string for log message. - * @param[in] args arguments for format string. + * This handles cutting the formatted log to get the correct + * logging level and removing the timestamp. */ - esp_err_t info(const char *msg, ...); - - /** - * @brief Log a warn level log. - * - * @param[in] msg format string for log message. - * @param[in] args arguments for format string. - */ - esp_err_t warn(const char *msg, ...); - - /** - * @brief Log an error level log. - * - * @param[in] msg format string for log message. - * @param[in] args arguments for format string. - */ - esp_err_t error(const char *msg, ...); - - /** - * @brief Log a fatal level log. - * - * @param[in] msg format string for log message. - * @param[in] args arguments for format string. - */ - esp_err_t fatal(const char *msg, ...); + esp_err_t log(char *msg); /** * @brief maximum path length. @@ -177,15 +153,6 @@ class FileLogger { FRIEND_TEST(TestFileLogger, Write_Logs_Direct); #endif - /** - * @brief Log a formatted message for a given log level. - * - * @param[in] level enum log level. - * @param[in] msg format string for log message. - * @param[in] args arguments for format string. - */ - esp_err_t log(const Level level, const char *msg, ...); - /** * @brief Create a new log file with a timestamped name. * diff --git a/esp/speaker-netsender/components/pipi/pipi.cpp b/esp/speaker-netsender/components/pipi/pipi.cpp index 1dc99f3..bd4d7e3 100644 --- a/esp/speaker-netsender/components/pipi/pipi.cpp +++ b/esp/speaker-netsender/components/pipi/pipi.cpp @@ -163,50 +163,38 @@ esp_err_t Pipi::FileLogger::new_file() return ESP_OK; } -esp_err_t Pipi::FileLogger::info(const char *fmt, ...) +esp_err_t Pipi::FileLogger::log(char *msg) { - va_list args; - va_start(args, fmt); - return this->log(Level::INFO, fmt, args); -} -esp_err_t Pipi::FileLogger::warn(const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - return this->log(Level::WARN, fmt, args); -} -esp_err_t Pipi::FileLogger::error(const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - return this->log(Level::ERROR, fmt, args); -} -esp_err_t Pipi::FileLogger::fatal(const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - return this->log(Level::FATAL, fmt, args); -} - -esp_err_t Pipi::FileLogger::log(const Pipi::Level level, const char *fmt, ...) -{ - va_list args; - va_start(args, fmt); - if (!this->ready) { - va_end(args); - return ESP_FAIL; + // Example log: + // I (01:02:03.040) : + // ^ + // | + // 17th character + constexpr auto msg_start = 17; + + // Determine log level based on first character. + auto level = INFO; + switch (msg[0]) { + case 'I': + level = INFO; + break; + case 'W': + level = WARN; + break; + case 'E': + level = ERROR; + break; + case 'F': + level = FATAL; + break; } - char msg[Entry::MAX_LOG_LENGTH]; - auto written = vsnprintf(msg, Entry::MAX_LOG_LENGTH, fmt, args); - va_end(args); - if (written < 0) { - ESP_LOGE(TAG, "unable to format log message"); - return ESP_FAIL; - } + // Cut the level and time from the passed message. + auto cut_log = &msg[msg_start]; + cut_log[strlen(cut_log) - 1] = '\0'; std::chrono::system_clock::now(); - auto e = Entry(time(nullptr), level, msg); + auto e = Entry(time(nullptr), level, cut_log); return e.write(curr_file); } diff --git a/esp/speaker-netsender/main/log.cpp b/esp/speaker-netsender/main/log.cpp index d6b8f9f..cbf26a8 100644 --- a/esp/speaker-netsender/main/log.cpp +++ b/esp/speaker-netsender/main/log.cpp @@ -39,6 +39,8 @@ #include "freertos/projdefs.h" #include "include/log_udp.hpp" +#include "netsender.hpp" +#include "pipi.hpp" // Logging Tag. static constexpr auto TAG = "log"; @@ -51,6 +53,8 @@ static uint8_t queue_storage[CONFIG_LOG_QUEUE_LEN * (QUEUE_ITEM_SIZE)]; static StaticQueue_t queue_buf; QueueHandle_t log_queue = NULL; +std::unique_ptr p; + // Original vprintf handler so we can still print directly to stdout/serial static vprintf_like_t default_vprintf = nullptr; @@ -86,7 +90,7 @@ static void log_processor_task(void *pvParameters) // Block until a new log message arrives in the queue. if (xQueueReceive(log_queue, msg, portMAX_DELAY) == pdTRUE) { udp_log_send(msg); - // TODO: Log to file. + p->log(msg); } } } @@ -118,8 +122,6 @@ std::function>()> init_logging() // Initialise UDP logging. init_udp_logging(); - // TODO: Initialise file logging. - // Start logging task. if (xTaskCreatePinnedToCore(log_processor_task, "logging_task", 4096, NULL, 5, NULL, 1) != pdPASS) { ESP_LOGE(TAG, "log task creation failed"); From ee1f84545f2e4c7c3d26db9c7049eb4607150d78 Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 14 Aug 2026 16:24:46 +0930 Subject: [PATCH 5/9] Speaker: Add registry and handling for binary pins This change adds the required functionality to allow binary pins to be registered, and read from to send them in poll requests. --- .../netsender/include/netsender.hpp | 9 ++ .../components/netsender/netsender.cpp | 94 +++++++++++++++++-- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/esp/speaker-netsender/components/netsender/include/netsender.hpp b/esp/speaker-netsender/components/netsender/include/netsender.hpp index b6ccf89..75d4f6d 100644 --- a/esp/speaker-netsender/components/netsender/include/netsender.hpp +++ b/esp/speaker-netsender/components/netsender/include/netsender.hpp @@ -29,11 +29,13 @@ #pragma once +#include #include #include #include #include #include +#include #include "esp_err.h" @@ -116,7 +118,9 @@ struct netsender_configuration_t { struct netsender_pin_t { char name[NETSENDER_PIN_SIZE]; std::function()> read; + std::function>()> read_binary; std::optional value; + std::vector data_storage; uint8_t *data; }; @@ -143,6 +147,11 @@ class Netsender { */ esp_err_t register_input(char *pin_name, std::function()> read_func); + /** + * @brief append a binary read function to an associated pin. + */ + esp_err_t register_binary_input(char *pin_name, std::function>()> read_func); + /** * @brief register a callback to handle variable parsing. * diff --git a/esp/speaker-netsender/components/netsender/netsender.cpp b/esp/speaker-netsender/components/netsender/netsender.cpp index d643557..d5cf2c8 100644 --- a/esp/speaker-netsender/components/netsender/netsender.cpp +++ b/esp/speaker-netsender/components/netsender/netsender.cpp @@ -29,12 +29,14 @@ #include "freertos/FreeRTOS.h" // IWYU pragma: keep +#include #include #include #include #include #include #include +#include #include "esp_err.h" #include "esp_http_client.h" @@ -202,6 +204,26 @@ esp_err_t Netsender::register_input(char pin_name[NETSENDER_PIN_SIZE], return ESP_OK; } +esp_err_t Netsender::register_binary_input(char pin_name[NETSENDER_PIN_SIZE], + std::function>()> read_binary_func) +{ + if (input_cnt >= CONFIG_NETSENDER_MAX_PINS) { + ESP_LOGE(TAG, "cannot register more than %d inputs", CONFIG_NETSENDER_MAX_PINS); + return ESP_FAIL; + } + + ESP_LOGI(TAG, "registering new input: %s", pin_name); + + auto pin = netsender_pin_t{}; + memcpy(pin.name, pin_name, NETSENDER_PIN_SIZE); + pin.read_binary = read_binary_func; + + inputs[input_cnt] = pin; + input_cnt++; + + return ESP_OK; +} + esp_err_t Netsender::register_variable_parser(std::function parser_func) { ESP_LOGD(TAG, "registering new variable callback"); @@ -461,13 +483,30 @@ esp_err_t Netsender::req_poll() for (auto i = 0; i < input_cnt; i++) { auto &pin = inputs[i]; - pin.value = pin.read(); - if (pin.value.has_value()) { - ESP_LOGI(TAG, "read pin %s: %d", pin.name, pin.value.value()); - append_pin_to_url(url, pin); - } else { - ESP_LOGE(TAG, "failed to read pin %s", pin.name); + if (pin.read != NULL) { + // Read non-binary pins. + pin.value = pin.read(); + pin.value = pin.read(); + if (!pin.value.has_value()) { + ESP_LOGE(TAG, "failed to read pin %s", pin.name); + continue; + } + ESP_LOGI(TAG, "read non-binary pin %s: %d", pin.name, pin.value.value()); + pin.data = NULL; + } else if (pin.read_binary != NULL) { + // Read binary pins. + auto out = pin.read_binary(); + if (!out.has_value()) { + ESP_LOGE(TAG, "failed to read pin %s", pin.name); + continue; + } + pin.data_storage = std::move(out.value()); + pin.data = pin.data_storage.data(); + pin.value = pin.data_storage.size(); + ESP_LOGI(TAG, "read binary pin %s: %d", pin.name, pin.value.value()); } + + append_pin_to_url(url, pin); } // Initialise the request. @@ -478,8 +517,51 @@ esp_err_t Netsender::req_poll() .event_handler = http_event_handler, .user_data = this->resp_buf, }; + + // Create body data to append to request. + static constexpr auto MAX_BODY_LEN = 1024 * 4; + static char body[1024 * 4]; + auto pos = 0; + for (auto i = 0; i < input_cnt; i++) { + auto &pin = inputs[i]; + if (pin.data == NULL) { + continue; + } + + if (pos + pin.value.value() >= MAX_BODY_LEN) { + ESP_LOGI(TAG, "binary data would exceed max body length: %d bytes over", + pos + pin.value.value() - MAX_BODY_LEN); + continue; + } + memcpy(body + pos, pin.data, pin.value.value()); + pos += pin.value.value(); + } + + // Init http client. auto http_handle = esp_http_client_init(&http_config); + // Append body if any. + if (pos != 0) { + auto err = esp_http_client_set_method(http_handle, HTTP_METHOD_POST); + if (err != ESP_OK) { + ESP_LOGE(TAG, "unable to set client method: %s", esp_err_to_name(err)); + return err; + } + + err = esp_http_client_set_header(http_handle, "Content-Type", "application/json"); + if (err != ESP_OK) { + ESP_LOGE(TAG, "unable to set content-type: %s", esp_err_to_name(err)); + return err; + } + + err = esp_http_client_set_post_field(http_handle, (const char *)body, pos); + if (err != ESP_OK) { + ESP_LOGE(TAG, "unable to set post field: %s", esp_err_to_name(err)); + esp_http_client_cleanup(http_handle); + return err; + } + } + // Send the request. auto err = esp_http_client_perform(http_handle); if (err != ESP_OK) { From 3488a9d9c4bf3e2ca9194aaffc0c8337d9b469bc Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 14 Aug 2026 16:27:59 +0930 Subject: [PATCH 6/9] Speaker: Get logs and update format This change adds a get_logs function which is used to get the last log file from the SD card, allowing the caller to write these to the remote. This change also updates the log format which is being written to file to improve compatibility with the existing logging service. --- .../components/pipi/include/pipi.hpp | 8 ++++++++ esp/speaker-netsender/components/pipi/pipi.cpp | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/esp/speaker-netsender/components/pipi/include/pipi.hpp b/esp/speaker-netsender/components/pipi/include/pipi.hpp index 3afb722..4bbe795 100644 --- a/esp/speaker-netsender/components/pipi/include/pipi.hpp +++ b/esp/speaker-netsender/components/pipi/include/pipi.hpp @@ -23,6 +23,7 @@ along with NetSender in gpl.txt. If not, see . */ +#pragma once #include #include @@ -142,6 +143,13 @@ class FileLogger { */ static constexpr auto MAX_PATH_LENGTH = 256; + /** + * @brief returns a pointer to the last unread log file + * + * NOTE: The caller should not close this file. + */ + std::ifstream &get_logs(); + private: #ifdef RUNNING_UNIT_TESTS friend class TestFileLogger; diff --git a/esp/speaker-netsender/components/pipi/pipi.cpp b/esp/speaker-netsender/components/pipi/pipi.cpp index bd4d7e3..a2d8304 100644 --- a/esp/speaker-netsender/components/pipi/pipi.cpp +++ b/esp/speaker-netsender/components/pipi/pipi.cpp @@ -54,14 +54,19 @@ esp_err_t Pipi::Entry::write(std::ostream &stream) } else if (len < 0) { return ESP_FAIL; } + + // Keep in sync with level enums. + auto LevelStrings = {"info", "warn", "error", "fatal"}; + char marshalled[Pipi::Entry::MAX_LOG_LENGTH + 100]; auto written = snprintf(marshalled, Pipi::Entry::MAX_LOG_LENGTH + 100, "{" + "\"caller\":\"speaker-netsender\"," "\"timestamp\":%" PRId64 "," - "\"level\":%d," + "\"level\":\"%s\"," "\"message\":\"%s\"" - "}", - this->timestamp, this->level, this->data); + "}\n", + this->timestamp, LevelStrings.begin()[this->level], this->data); if (written < 0 || written >= sizeof(marshalled)) { return ESP_FAIL; } @@ -199,6 +204,12 @@ esp_err_t Pipi::FileLogger::log(char *msg) return e.write(curr_file); } +std::ifstream &Pipi::FileLogger::get_logs() +{ + this->new_file(); + return this->prev_file; +} + void Pipi::FileLogger::close() { if (this->curr_file.is_open()) { From 2efe680b1e9b097015429f06f7d7b364877e8551 Mon Sep 17 00:00:00 2001 From: David Sutton Date: Fri, 21 Aug 2026 10:42:56 +0930 Subject: [PATCH 7/9] ESP/Speaker: Use persistent http_handle This change adds a persistent http_handle to the NetSender class to reduce request overhead. --- .../netsender/include/netsender.hpp | 17 ++ .../components/netsender/netsender.cpp | 161 +++++++++++++----- 2 files changed, 132 insertions(+), 46 deletions(-) diff --git a/esp/speaker-netsender/components/netsender/include/netsender.hpp b/esp/speaker-netsender/components/netsender/include/netsender.hpp index 75d4f6d..0e2b336 100644 --- a/esp/speaker-netsender/components/netsender/include/netsender.hpp +++ b/esp/speaker-netsender/components/netsender/include/netsender.hpp @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include "esp_err.h" +#include "esp_http_client.h" #include "sdkconfig.h" @@ -186,6 +188,21 @@ class Netsender { */ bool configured; + /** + * Persistent handle for http requests. + */ + esp_http_client_handle_t http_handle; + + /** + * Mutex for persistent http_handle. + */ + std::timed_mutex http_handle_mu; + + /** + * @brief init function for http_handle. + */ + esp_err_t init_http_handle(); + /** * Latest varsum value. */ diff --git a/esp/speaker-netsender/components/netsender/netsender.cpp b/esp/speaker-netsender/components/netsender/netsender.cpp index d5cf2c8..bb78f63 100644 --- a/esp/speaker-netsender/components/netsender/netsender.cpp +++ b/esp/speaker-netsender/components/netsender/netsender.cpp @@ -29,8 +29,12 @@ #include "freertos/FreeRTOS.h" // IWYU pragma: keep +#include +#include #include #include +#include +#include #include #include #include @@ -108,6 +112,9 @@ Netsender::Netsender() } else { this->configured = true; } + + // Initialise HTTP handler. + ESP_ERROR_CHECK(init_http_handle()); } esp_err_t Netsender::read_nvs_config() @@ -374,25 +381,38 @@ esp_err_t Netsender::req_config() CONFIG_NETSENDER_REMOTE_HOST, netsender_endpoint::CONFIG, NETSENDER_VERSION, this->mac, uptime(), netsender_mode::ONLINE); - // Initialise the request. - esp_http_client_config_t http_config = { - .url = url, - .method = HTTP_METHOD_GET, - .disable_auto_redirect = true, - .event_handler = http_event_handler, - .user_data = this->resp_buf, - }; - auto http_handle = esp_http_client_init(&http_config); + // Get lock for http_client. + using namespace std::chrono_literals; + std::unique_lock lock(this->http_handle_mu, 500ms); + + if (!lock.owns_lock()) { + // The http handle is busy. + ESP_LOGE(TAG, "Unable to acquire lock for config request"); + return ESP_FAIL; + } + + auto err = esp_http_client_set_url(this->http_handle, url); + if (err != ESP_OK) { + lock.unlock(); + return err; + } + err = esp_http_client_set_method(this->http_handle, HTTP_METHOD_GET); + if (err != ESP_OK) { + lock.unlock(); + return err; + } // Send the request. - auto err = esp_http_client_perform(http_handle); + err = esp_http_client_perform(this->http_handle); if (err != ESP_OK) { + lock.unlock(); return err; } // Check the status code. // TODO: Handle other status codes. - if (auto status_code = esp_http_client_get_status_code(http_handle) != 200) { + if (auto status_code = esp_http_client_get_status_code(this->http_handle) != 200) { + lock.unlock(); ESP_LOGE(TAG, "got non 200 status code: %d", status_code); return ESP_FAIL; } @@ -402,6 +422,7 @@ esp_err_t Netsender::req_config() // Parse the incoming config. std::string param; std::string json_resp(resp_buf); + lock.unlock(); if (netsender_extract_json(json_resp, "mp", param) && std::stoi(param) != this->config.monPeriod) { this->config.monPeriod = std::stoi(param); ESP_LOGI(TAG, "monPeriod changed: %d", this->config.monPeriod); @@ -447,8 +468,6 @@ esp_err_t Netsender::req_config() print_config(); } - ESP_ERROR_CHECK(esp_http_client_cleanup(http_handle)); - return ESP_OK; } @@ -486,7 +505,6 @@ esp_err_t Netsender::req_poll() if (pin.read != NULL) { // Read non-binary pins. pin.value = pin.read(); - pin.value = pin.read(); if (!pin.value.has_value()) { ESP_LOGE(TAG, "failed to read pin %s", pin.name); continue; @@ -509,14 +527,26 @@ esp_err_t Netsender::req_poll() append_pin_to_url(url, pin); } - // Initialise the request. - esp_http_client_config_t http_config = { - .url = url, - .method = HTTP_METHOD_GET, - .disable_auto_redirect = true, - .event_handler = http_event_handler, - .user_data = this->resp_buf, - }; + // Get lock for http_client. + using namespace std::chrono_literals; + std::unique_lock lock(this->http_handle_mu, 500ms); + + if (!lock.owns_lock()) { + // The http handle is busy. + ESP_LOGE(TAG, "Unable to acquire lock for config request"); + return ESP_FAIL; + } + + auto err = esp_http_client_set_url(this->http_handle, url); + if (err != ESP_OK) { + lock.unlock(); + return err; + } + err = esp_http_client_set_method(this->http_handle, HTTP_METHOD_GET); + if (err != ESP_OK) { + lock.unlock(); + return err; + } // Create body data to append to request. static constexpr auto MAX_BODY_LEN = 1024 * 4; @@ -537,40 +567,50 @@ esp_err_t Netsender::req_poll() pos += pin.value.value(); } - // Init http client. - auto http_handle = esp_http_client_init(&http_config); - // Append body if any. if (pos != 0) { - auto err = esp_http_client_set_method(http_handle, HTTP_METHOD_POST); + auto err = esp_http_client_set_method(this->http_handle, HTTP_METHOD_POST); if (err != ESP_OK) { + lock.unlock(); ESP_LOGE(TAG, "unable to set client method: %s", esp_err_to_name(err)); return err; } - err = esp_http_client_set_header(http_handle, "Content-Type", "application/json"); + err = esp_http_client_set_header(this->http_handle, "Content-Type", "application/json"); if (err != ESP_OK) { + lock.unlock(); ESP_LOGE(TAG, "unable to set content-type: %s", esp_err_to_name(err)); return err; } - err = esp_http_client_set_post_field(http_handle, (const char *)body, pos); + err = esp_http_client_set_post_field(this->http_handle, (const char *)body, pos); if (err != ESP_OK) { + lock.unlock(); ESP_LOGE(TAG, "unable to set post field: %s", esp_err_to_name(err)); - esp_http_client_cleanup(http_handle); + return err; + } + } else { + err = esp_http_client_set_method(this->http_handle, HTTP_METHOD_GET); + if (err != ESP_OK) { + lock.unlock(); return err; } } // Send the request. - auto err = esp_http_client_perform(http_handle); + err = esp_http_client_perform(this->http_handle); if (err != ESP_OK) { + lock.unlock(); return err; } + // Clear post data. + err = esp_http_client_set_post_field(this->http_handle, NULL, 0); + // Check the status code. // TODO: Handle other status codes. if (auto status_code = esp_http_client_get_status_code(http_handle) != 200) { + lock.unlock(); ESP_LOGE(TAG, "got non 200 status code: %d", status_code); return ESP_FAIL; } @@ -579,6 +619,7 @@ esp_err_t Netsender::req_poll() ESP_LOGI(TAG, "poll response: %s", this->resp_buf); std::string rc; const auto resp = std::string(resp_buf); + lock.unlock(); const auto has_rc = netsender_extract_json(resp, "rc", rc); if (has_rc) { ESP_LOGD(TAG, "got response code: %s", rc.c_str()); @@ -597,9 +638,24 @@ esp_err_t Netsender::req_poll() } } - // Cleanup http_handle. - ESP_ERROR_CHECK(esp_http_client_cleanup(http_handle)); + return ESP_OK; +} +esp_err_t Netsender::init_http_handle() +{ + // Configure client. + esp_http_client_config_t config = {}; + config.url = CONFIG_NETSENDER_REMOTE_HOST; + config.disable_auto_redirect = true; + config.event_handler = http_event_handler; + config.user_data = this->resp_buf; + + // Init http client. + this->http_handle = esp_http_client_init(&config); + + if (this->http_handle == NULL) { + return ESP_FAIL; + } return ESP_OK; } @@ -616,25 +672,38 @@ esp_err_t Netsender::req_vars() "&dk=%s", // Device Key. CONFIG_NETSENDER_REMOTE_HOST, netsender_endpoint::VARS, this->mac, this->config.dkey); - // Initialise the request. - esp_http_client_config_t http_config = { - .url = url, - .method = HTTP_METHOD_GET, - .disable_auto_redirect = true, - .event_handler = http_event_handler, - .user_data = this->resp_buf, - }; - auto http_handle = esp_http_client_init(&http_config); + using namespace std::chrono_literals; + std::unique_lock lock(this->http_handle_mu, 500ms); + + if (!lock.owns_lock()) { + // The http handle is busy. + ESP_LOGE(TAG, "Unable to acquire lock for vars request"); + return ESP_FAIL; + } + + auto err = esp_http_client_set_url(this->http_handle, url); + if (err != ESP_OK) { + lock.unlock(); + return err; + } + err = esp_http_client_set_method(this->http_handle, HTTP_METHOD_GET); + if (err != ESP_OK) { + lock.unlock(); + return err; + } // Send the request. - auto err = esp_http_client_perform(http_handle); + err = esp_http_client_perform(this->http_handle); if (err != ESP_OK) { + lock.unlock(); return err; } // Check the status code. // TODO: Handle other status codes. - if (auto status_code = esp_http_client_get_status_code(http_handle) != 200) { + auto status_code = esp_http_client_get_status_code(this->http_handle); + if (status_code != 200) { + lock.unlock(); ESP_LOGE(TAG, "got non 200 status code: %d", status_code); return ESP_FAIL; } @@ -643,6 +712,9 @@ esp_err_t Netsender::req_vars() std::string vs; const auto resp = std::string(resp_buf); + // Release the lock on the HTTP client handle. + lock.unlock(); + err = this->parse_variable_callback(resp); if (err != ESP_OK) { ESP_LOGE(TAG, "unable to parse variables in callback"); @@ -656,9 +728,6 @@ esp_err_t Netsender::req_vars() this->varsum = std::stoi(vs); } - // Cleanup http_handle. - ESP_ERROR_CHECK(esp_http_client_cleanup(http_handle)); - return ESP_OK; } From 706c59c67ff37ec448fc314274191595083b1808 Mon Sep 17 00:00:00 2001 From: David Sutton Date: Mon, 7 Sep 2026 12:27:44 +0930 Subject: [PATCH 8/9] ESP/Speaker: Improve Logging Performance This change changes Pipi to use file descriptors instead of fstreams to improve performance. --- esp/speaker-netsender/README.md | 24 +++++- .../host_tests/test_pipi/dependencies.lock | 2 +- .../pipi/host_tests/test_pipi/sdkconfig | 29 +++++-- .../components/pipi/include/pipi.hpp | 19 ++--- .../components/pipi/pipi.cpp | 85 ++++++++++--------- esp/speaker-netsender/main/log.cpp | 26 ++++-- 6 files changed, 119 insertions(+), 66 deletions(-) diff --git a/esp/speaker-netsender/README.md b/esp/speaker-netsender/README.md index 1d1c0c0..7a862a0 100644 --- a/esp/speaker-netsender/README.md +++ b/esp/speaker-netsender/README.md @@ -30,7 +30,29 @@ Alternatively this can also be run manually via the command line: clang-tidy --fix ``` -**Keeping includes that clang-tidy wants to remove:** +## Running the Pipi host tests + +The `pipi` logging component has host tests that build and run natively on your Linux machine, so no ESP32 hardware is required. They live in `components/pipi/host_tests/test_pipi` and are a gtest-based ESP-IDF project that targets the `linux` host. + +In addition to the ESP-IDF setup described above, your ESP-IDF installation must ship the clang Linux toolchain (`tools/cmake/toolchain-clang-linux.cmake`), as the tests are built with the clang toolchain. The `clang`, `clang++` and `lld` binaries also need to be installed on your system. + +To build and run the tests: + +```sh +cd components/pipi/host_tests/test_pipi +idf.py build +./build/test_pipi.elf +``` + +The test target is already pinned to `linux` in the committed `sdkconfig`, so no `set-target` step is required. Run the tests from the `test_pipi` directory, as the suite creates and `chdir`s into a `test/` subdirectory to isolate its log files. Individual tests can be selected with gtest flags, e.g.: + +```sh +./build/test_pipi.elf --gtest_filter=TestEntry.* +``` + +Note that some ESP-IDF point releases do not include `toolchain-clang-linux.cmake` (e.g. v6.0.2), which will cause the build to fail with a "toolchain file not found" error. If that happens, use an ESP-IDF version that provides it. + +## Keeping includes that clang-tidy wants to remove: clang-tidy sometimes suggests removing ESP-IDF driver headers that are actually needed as direct includes, because it can see them transitively through other headers. diff --git a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/dependencies.lock b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/dependencies.lock index e0ec445..a47ef57 100644 --- a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/dependencies.lock +++ b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/dependencies.lock @@ -2,7 +2,7 @@ dependencies: idf: source: type: idf - version: 6.1.0 + version: 6.0.2 direct_dependencies: - idf manifest_hash: 23e4fa146fe3e7046fcad9313a66ff459ece216944aa64c3f6f45db59f9f4a4a diff --git a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/sdkconfig b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/sdkconfig index 571d3a2..d06865d 100644 --- a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/sdkconfig +++ b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/sdkconfig @@ -1,6 +1,6 @@ # # Automatically generated file. DO NOT EDIT. -# Espressif IoT Development Framework (ESP-IDF) 6.1.0 Project Configuration +# Espressif IoT Development Framework (ESP-IDF) 6.0.2 Project Configuration # # default: CONFIG_SOC_EFUSE_SUPPORTED=y @@ -84,8 +84,6 @@ CONFIG_ESP_ROM_USB_SERIAL_DEVICE_NUM=-1 CONFIG_ESP_ROM_HAS_ETS_PRINTF_BUG=y # default: CONFIG_ESP_ROM_HAS_OUTPUT_PUTC_FUNC=y -# default: -CONFIG_ESP_ROM_BOOTLOADER_OFFSET_FLASH=0x0 # # Boot ROM Behavior @@ -215,8 +213,6 @@ CONFIG_ESP_SLEEP_WAIT_FLASH_READY_EXTRA_DELAY=0 # CONFIG_ESP_SLEEP_DEBUG is not set # default: CONFIG_ESP_SLEEP_GPIO_ENABLE_INTERNAL_RESISTORS=y -# default: -# CONFIG_ESP_SLEEP_SET_FLASH_DPD is not set # end of Sleep Config # @@ -304,6 +300,20 @@ CONFIG_ESP_MAIN_TASK_AFFINITY=0x0 # default: CONFIG_ESP_MINIMAL_SHARED_STACK_SIZE=2048 # default: +CONFIG_ESP_INT_WDT=y +# default: +CONFIG_ESP_INT_WDT_TIMEOUT_MS=300 +# default: +CONFIG_ESP_TASK_WDT_EN=y +# default: +CONFIG_ESP_TASK_WDT_INIT=y +# default: +# CONFIG_ESP_TASK_WDT_PANIC is not set +# default: +CONFIG_ESP_TASK_WDT_TIMEOUT_S=5 +# default: +CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=y +# default: # CONFIG_ESP_PANIC_HANDLER_IRAM is not set # default: # CONFIG_ESP_DEBUG_STUBS_ENABLE is not set @@ -328,6 +338,8 @@ CONFIG_ESP_IPC_TASK_STACK_SIZE=1024 # Kernel # # default: +# CONFIG_FREERTOS_SMP is not set +# default: CONFIG_FREERTOS_UNICORE=y # default: CONFIG_FREERTOS_HZ=100 @@ -680,6 +692,13 @@ CONFIG_CONSOLE_UART_BAUDRATE=115200 CONFIG_SYSTEM_EVENT_QUEUE_SIZE=32 CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=2304 CONFIG_MAIN_TASK_STACK_SIZE=3584 +CONFIG_INT_WDT=y +CONFIG_INT_WDT_TIMEOUT_MS=300 +CONFIG_TASK_WDT=y +CONFIG_ESP_TASK_WDT=y +# CONFIG_TASK_WDT_PANIC is not set +CONFIG_TASK_WDT_TIMEOUT_S=5 +CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU0=y # CONFIG_ESP32_DEBUG_STUBS_ENABLE is not set CONFIG_IPC_TASK_STACK_SIZE=1024 CONFIG_TIMER_TASK_PRIORITY=1 diff --git a/esp/speaker-netsender/components/pipi/include/pipi.hpp b/esp/speaker-netsender/components/pipi/include/pipi.hpp index 4bbe795..f3d68ea 100644 --- a/esp/speaker-netsender/components/pipi/include/pipi.hpp +++ b/esp/speaker-netsender/components/pipi/include/pipi.hpp @@ -26,11 +26,9 @@ #pragma once #include -#include #ifdef RUNNING_UNIT_TESTS #include #endif -#include #include "esp_err.h" @@ -68,15 +66,14 @@ class Entry { Entry(const int64_t ts, const Level level, const char *msg); /** - * @brief write JSON formatted Entry to stream. + * @brief write JSON formatted Entry to file. * - * Writing the Entry to the given stream will format the entry as a single - * JSON entity to write to the stream. + * Writing the Entry to the given file will format the entry as a single + * JSON entity to write to the file. * - * @param[in] stream an output stream such as a file to write the formatted - * log to + * @param[in] file a file descriptor to write the formatted log to. */ - esp_err_t write(std::ostream &stream); + esp_err_t write(const int fd); private: #ifdef RUNNING_UNIT_TESTS @@ -148,7 +145,7 @@ class FileLogger { * * NOTE: The caller should not close this file. */ - std::ifstream &get_logs(); + int get_logs(); private: #ifdef RUNNING_UNIT_TESTS @@ -171,7 +168,7 @@ class FileLogger { /** * @brief current logging file. */ - std::ofstream curr_file; + int curr_file; /** * @brief path to current log file. @@ -181,7 +178,7 @@ class FileLogger { /** * @brief previous logging file. */ - std::ifstream prev_file; + int prev_file; /** * @brief path for log files. diff --git a/esp/speaker-netsender/components/pipi/pipi.cpp b/esp/speaker-netsender/components/pipi/pipi.cpp index a2d8304..a9b5048 100644 --- a/esp/speaker-netsender/components/pipi/pipi.cpp +++ b/esp/speaker-netsender/components/pipi/pipi.cpp @@ -31,22 +31,22 @@ #include #include +#include #include #include #include -#include -#include +#include #include +#include #include "esp_err.h" -#include "esp_log.h" // Logging Tag. static constexpr auto TAG = "Pipi"; Pipi::Entry::Entry(const int64_t ts, const Level level, const char *msg) : timestamp(ts), level(level), data(msg) {} -esp_err_t Pipi::Entry::write(std::ostream &stream) +esp_err_t Pipi::Entry::write(int fd) { int len = strlen(data); if (len > this->MAX_LOG_LENGTH) { @@ -59,20 +59,20 @@ esp_err_t Pipi::Entry::write(std::ostream &stream) auto LevelStrings = {"info", "warn", "error", "fatal"}; char marshalled[Pipi::Entry::MAX_LOG_LENGTH + 100]; - auto written = snprintf(marshalled, Pipi::Entry::MAX_LOG_LENGTH + 100, - "{" - "\"caller\":\"speaker-netsender\"," - "\"timestamp\":%" PRId64 "," - "\"level\":\"%s\"," - "\"message\":\"%s\"" - "}\n", - this->timestamp, LevelStrings.begin()[this->level], this->data); - if (written < 0 || written >= sizeof(marshalled)) { + auto marshalled_len = snprintf(marshalled, Pipi::Entry::MAX_LOG_LENGTH + 100, + "{" + "\"caller\":\"speaker-netsender\"," + "\"timestamp\":%" PRId64 "," + "\"level\":\"%s\"," + "\"message\":\"%s\"" + "}\n", + this->timestamp, LevelStrings.begin()[this->level], this->data); + if (marshalled_len < 0 || marshalled_len >= sizeof(marshalled)) { return ESP_FAIL; } - stream.write(marshalled, written); - if (stream.fail()) { + auto written = ::write(fd, marshalled, marshalled_len); + if (written != marshalled_len) { return ESP_FAIL; } return ESP_OK; @@ -82,7 +82,7 @@ esp_err_t Pipi::FileLogger::make_path(const char *path) { auto path_len = strnlen(path, MAX_PATH_LENGTH + 1); if (path_len > MAX_PATH_LENGTH - 1) { - ESP_LOGE(TAG, "path too long"); + printf("%s: path too long", TAG); return ESP_FAIL; } @@ -103,7 +103,7 @@ esp_err_t Pipi::FileLogger::make_path(const char *path) auto status = mkdir(parent, 0777); if (status == -1 && errno != EEXIST) { - ESP_LOGE(TAG, "unable to create dir (%s): %d (%s)", parent, errno, strerror(errno)); + printf("%s: unable to create dir (%s): %d (%s)", TAG, parent, errno, strerror(errno)); return ESP_FAIL; } } @@ -111,7 +111,7 @@ esp_err_t Pipi::FileLogger::make_path(const char *path) return ESP_OK; } -Pipi::FileLogger::FileLogger(const char *path) : ready(false) +Pipi::FileLogger::FileLogger(const char *path) : curr_file(-1), prev_file(-1), ready(false) { // Validate the path is a directory. const char *resolved = (path != nullptr && path[0] != '\0') ? path : DEFAULT_PATH; @@ -125,7 +125,7 @@ Pipi::FileLogger::FileLogger(const char *path) : ready(false) // Directory doesn't exist so create it. auto err = make_path(resolved); if (err != ESP_OK) { - ESP_LOGE(TAG, "unable to create path"); + printf("%s: unable to create path", TAG); return; } strncpy(this->path, resolved, this->MAX_PATH_LENGTH); @@ -134,7 +134,7 @@ Pipi::FileLogger::FileLogger(const char *path) : ready(false) // Start a new logging file. auto err = this->new_file(); if (err != ESP_OK) { - ESP_LOGE(TAG, "unable to start new logfile: %s", esp_err_to_name(err)); + printf("%s: unable to start new logfile: %s", TAG, esp_err_to_name(err)); return; } @@ -145,23 +145,24 @@ Pipi::FileLogger::FileLogger() : Pipi::FileLogger(DEFAULT_PATH) {} esp_err_t Pipi::FileLogger::new_file() { - if (this->prev_file.is_open()) { - this->prev_file.close(); - } - if (this->curr_file.is_open()) { - this->curr_file.close(); - this->prev_file.open(curr_file_path); - if (!this->prev_file.is_open()) { - ESP_LOGE(TAG, "unable to open previous file as ifstream"); + if (this->prev_file >= 0) { + if (::close(this->prev_file) != 0) { + printf("%s: unable to close previous file: %s (%d)", TAG, strerror(errno), errno); return ESP_FAIL; } + this->prev_file = -1; + } + if (this->curr_file >= 0) { + this->prev_file = this->curr_file; + this->curr_file = -1; } - snprintf(this->curr_file_path, MAX_PATH_LENGTH, "%s/%" PRId64 ".log", this->path, time(nullptr)); - this->curr_file.open(this->curr_file_path, std::ios::out); - - if (!this->curr_file.is_open()) { - ESP_LOGE(TAG, "unable to open log file: %s", this->curr_file_path); + static int log_counter = 0; + snprintf(this->curr_file_path, MAX_PATH_LENGTH, "%s/test_%d.log", this->path, log_counter++); + this->curr_file = ::open(this->curr_file_path, O_CREAT | O_RDWR | O_TRUNC); + if (this->curr_file < 0) { + printf("%s: unable to open new file (%s): %s (%d)", TAG, this->curr_file_path, strerror(errno), errno); + this->curr_file = -1; return ESP_FAIL; } @@ -204,19 +205,25 @@ esp_err_t Pipi::FileLogger::log(char *msg) return e.write(curr_file); } -std::ifstream &Pipi::FileLogger::get_logs() +int Pipi::FileLogger::get_logs() { - this->new_file(); + ESP_ERROR_CHECK(this->new_file()); return this->prev_file; } void Pipi::FileLogger::close() { - if (this->curr_file.is_open()) { - this->curr_file.close(); + if (this->prev_file >= 0) { + if (::close(this->prev_file) != 0) { + printf("%s: unable to close previous file: %s (%d)", TAG, strerror(errno), errno); + } + this->prev_file = -1; } - if (this->prev_file.is_open()) { - this->prev_file.close(); + if (this->curr_file >= 0) { + if (::close(this->curr_file) != 0) { + printf("%s: unable to close current file: %s (%d)", TAG, strerror(errno), errno); + } + this->curr_file = -1; } this->ready = false; } diff --git a/esp/speaker-netsender/main/log.cpp b/esp/speaker-netsender/main/log.cpp index cbf26a8..9cdded9 100644 --- a/esp/speaker-netsender/main/log.cpp +++ b/esp/speaker-netsender/main/log.cpp @@ -27,9 +27,11 @@ #include "include/log.hpp" #include +#include #include #include #include +#include #include #include "esp_err.h" @@ -128,24 +130,30 @@ std::function>()> init_logging() return NULL; } - default_vprintf = esp_log_set_vprintf(log_vprintf); - p = std::make_unique("/sdcard/logs"); + default_vprintf = esp_log_set_vprintf(log_vprintf); + return []() -> std::optional> { - auto &stream = p->get_logs(); + auto logs_fd = p->get_logs(); - std::vector buffer(1024); // Allocate space for logs + if (::lseek(logs_fd, 0, SEEK_SET) == -1) { + ESP_LOGE(TAG, "unable to seek logs: %s (%d)", strerror(errno), errno); + ::close(logs_fd); + return std::nullopt; + } - // Read logs from stream into buffer - stream.read(reinterpret_cast(buffer.data()), buffer.size()); + std::vector buffer(1024); - auto bytes_read = stream.gcount(); - if (bytes_read == 0) { + // Read logs from stream into buffer + auto bytes_read = ::read(logs_fd, buffer.data(), buffer.size()); + if (bytes_read == -1) { + ESP_LOGE(TAG, "unable to read logs into buffer: %s (%d)", strerror(errno), errno); return std::nullopt; } - buffer.resize(bytes_read); // Shrink to actual payload size + buffer.resize(static_cast(bytes_read)); + return buffer; }; } From c436473120ae69c6e4e1636b87fd63590c426b63 Mon Sep 17 00:00:00 2001 From: David Sutton Date: Mon, 7 Sep 2026 12:53:44 +0930 Subject: [PATCH 9/9] ESP/Speaker: Fix Pipi tests This updates the pipi tests to use the new interface. --- esp/speaker-netsender/README.md | 2 +- .../pipi/host_tests/test_pipi/CMakeLists.txt | 5 + .../host_tests/test_pipi/main/test_pipi.cpp | 120 ++++++++---------- .../components/pipi/pipi.cpp | 4 +- 4 files changed, 63 insertions(+), 68 deletions(-) diff --git a/esp/speaker-netsender/README.md b/esp/speaker-netsender/README.md index 7a862a0..c086b1c 100644 --- a/esp/speaker-netsender/README.md +++ b/esp/speaker-netsender/README.md @@ -44,7 +44,7 @@ idf.py build ./build/test_pipi.elf ``` -The test target is already pinned to `linux` in the committed `sdkconfig`, so no `set-target` step is required. Run the tests from the `test_pipi` directory, as the suite creates and `chdir`s into a `test/` subdirectory to isolate its log files. Individual tests can be selected with gtest flags, e.g.: +The test target is already pinned to `linux` in the committed `sdkconfig`, so no `set-target` step is required. The project's `CMakeLists.txt` enables the local `gtest` component (defining `RUNNING_UNIT_TESTS`) automatically. Run the tests from the `test_pipi` directory, as the suite creates and `chdir`s into a `test/` subdirectory to isolate its log files. Individual tests can be selected with gtest flags, e.g.: ```sh ./build/test_pipi.elf --gtest_filter=TestEntry.* diff --git a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/CMakeLists.txt b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/CMakeLists.txt index 26dbde8..365826f 100644 --- a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/CMakeLists.txt +++ b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/CMakeLists.txt @@ -1,10 +1,15 @@ cmake_minimum_required(VERSION 3.16) +# This is a dedicated host test application, so always enable the gtest +# component, regardless of any cached ENABLE_TESTS value. +set(ENABLE_TESTS ON CACHE BOOL "Enable GoogleTest unit tests" FORCE) + # Append extra component directories so IDF can find our local components. list(APPEND EXTRA_COMPONENT_DIRS "../../../" # Project component directory. ) +# RUNNING_UNIT_TESTS is used by pipi to expose internals to the test fixtures. if(ENABLE_TESTS) add_compile_definitions(RUNNING_UNIT_TESTS) endif() diff --git a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/main/test_pipi.cpp b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/main/test_pipi.cpp index eff9f1c..5cf61a1 100644 --- a/esp/speaker-netsender/components/pipi/host_tests/test_pipi/main/test_pipi.cpp +++ b/esp/speaker-netsender/components/pipi/host_tests/test_pipi/main/test_pipi.cpp @@ -1,11 +1,10 @@ -#include +#include +#include #include +#include +#include #include -#include -#include -#include -#include -#include +#include #include #include "esp_err.h" @@ -15,7 +14,16 @@ namespace Pipi { // Forward declarations -void assert_log_level(std::ifstream &f, int level); +void assert_log_level(const int fd, const char *level); + +// Write a log to the given logger in the ESP log format: +// " (hh:mm:ss.mmm) tag: \n" +esp_err_t log_esp_msg(FileLogger &logger, const char level, const char *message) +{ + char msg[512]; + snprintf(msg, sizeof(msg), "%c (00:00:00.000) tag: %s\n", level, message); + return logger.log(msg); +} class TestEntry : public testing::Test { protected: @@ -55,17 +63,24 @@ TEST_F(TestEntry, Write) auto entry = Entry(this->TEST_TIME, Level::INFO, msg); - std::stringstream out; - entry.write(out); + int fd = ::open("entry.log", O_CREAT | O_RDWR | O_TRUNC, 0666); + ASSERT_GE(fd, 0); + + ASSERT_EQ(entry.write(fd), ESP_OK); char got[Pipi::Entry::MAX_LOG_LENGTH + 100]; - out.get(got, Pipi::Entry::MAX_LOG_LENGTH + 100, '\0'); + ASSERT_NE(::lseek(fd, 0, SEEK_SET), -1); + auto len = ::read(fd, got, sizeof(got) - 1); + ASSERT_GT(len, 0); + got[len] = '\0'; char want[Pipi::Entry::MAX_LOG_LENGTH + 100]; - snprintf(want, Pipi::Entry::MAX_LOG_LENGTH + 100, "{\"timestamp\":%d,\"level\":%d,\"message\":\"%s\"}", - this->TEST_TIME, Level::INFO, msg); + snprintf(want, sizeof(want), + "{\"caller\":\"speaker-netsender\",\"timestamp\":%" PRId64 ",\"level\":\"info\",\"message\":\"%s\"}\n", + static_cast(this->TEST_TIME), msg); EXPECT_STREQ(want, got); + ::close(fd); } class TestFileLogger : public testing::Test { @@ -110,6 +125,7 @@ TEST_F(TestFileLogger, Make_Path) { constexpr auto LOG_DIR = "long/path/to/logs"; auto p = FileLogger(LOG_DIR); + ASSERT_TRUE(p.ready); // FileLogger is ready. struct stat buffer; stat(LOG_DIR, &buffer); @@ -121,71 +137,45 @@ TEST_F(TestFileLogger, Write_Logs_Direct) constexpr auto LOG_DIR = "write_logs_direct"; auto p = FileLogger(LOG_DIR); - constexpr auto TEST_INT = 1234; - auto err = p.log(Level::INFO, "This is an INFO log with an int (%d)", TEST_INT); - ASSERT_EQ(err, ESP_OK); - err = p.log(Level::INFO, "This is an INFO log with no formatting"); - ASSERT_EQ(err, ESP_OK); - err = p.info("This is an INFO log with an int (%d)", TEST_INT); - ASSERT_EQ(err, ESP_OK); - err = p.info("This is an INFO log with no formatting"); - ASSERT_EQ(err, ESP_OK); - err = p.log(Level::WARN, "This is a WARN log with an int (%d)", TEST_INT); - ASSERT_EQ(err, ESP_OK); - err = p.log(Level::WARN, "This is a WARN log with no formatting"); - ASSERT_EQ(err, ESP_OK); - err = p.warn("This is a WARN log with an int (%d)", TEST_INT); - ASSERT_EQ(err, ESP_OK); - err = p.warn("This is a WARN log with no formatting"); + auto err = log_esp_msg(p, 'I', "This is an INFO log"); ASSERT_EQ(err, ESP_OK); - err = p.log(Level::ERROR, "This is an ERROR log with an int (%d)", TEST_INT); + err = log_esp_msg(p, 'I', "This is a second INFO log"); ASSERT_EQ(err, ESP_OK); - err = p.log(Level::ERROR, "This is an ERROR log with no formatting"); + err = log_esp_msg(p, 'W', "This is a WARN log"); ASSERT_EQ(err, ESP_OK); - err = p.error("This is an ERROR log with an int (%d)", TEST_INT); + err = log_esp_msg(p, 'E', "This is an ERROR log"); ASSERT_EQ(err, ESP_OK); - err = p.error("This is an ERROR log with no formatting"); + err = log_esp_msg(p, 'F', "This is a FATAL log"); ASSERT_EQ(err, ESP_OK); - err = p.log(Level::FATAL, "This is a FATAL log with an int (%d)", TEST_INT); - ASSERT_EQ(err, ESP_OK); - err = p.log(Level::FATAL, "This is a FATAL log with no formatting"); - ASSERT_EQ(err, ESP_OK); - err = p.fatal("This is a FATAL log with an int (%d)", TEST_INT); - ASSERT_EQ(err, ESP_OK); - err = p.fatal("This is a FATAL log with no formatting"); - ASSERT_EQ(err, ESP_OK); - - // This is required because logfiles are timestamped, and will overwrite itself if called too quickly. - std::this_thread::sleep_for(std::chrono::seconds(1)); - // // Create a new logfile so we can read from the old one. - p.new_file(); - ASSERT_TRUE(p.prev_file.is_open()); + // Create a new logfile so we can read from the old one. + auto logs_fd = p.get_logs(); + ASSERT_GE(logs_fd, 0); + ASSERT_NE(::lseek(logs_fd, 0, SEEK_SET), -1); // Assert the log levels. - for (auto i = 0; i < 4; i++) { - assert_log_level(p.prev_file, Level::INFO); - } - for (auto i = 0; i < 4; i++) { - assert_log_level(p.prev_file, Level::WARN); - } - for (auto i = 0; i < 4; i++) { - assert_log_level(p.prev_file, Level::ERROR); - } - for (auto i = 0; i < 4; i++) { - assert_log_level(p.prev_file, Level::FATAL); - } + assert_log_level(logs_fd, "info"); + assert_log_level(logs_fd, "info"); + assert_log_level(logs_fd, "warn"); + assert_log_level(logs_fd, "error"); + assert_log_level(logs_fd, "fatal"); } -void assert_log_level(std::ifstream &f, int level) +// Read the next line from the log file and assert it contains the given level string. +void assert_log_level(const int fd, const char *level) { constexpr auto SEARCH_LEN = 1024; - f.ignore(SEARCH_LEN, ','); // Skip over timestamp - char got[SEARCH_LEN]; - f.getline(got, SEARCH_LEN, ','); + char line[SEARCH_LEN]; + size_t i = 0; + char c; + while (::read(fd, &c, 1) == 1 && c != '\n' && i < sizeof(line) - 1) { + line[i++] = c; + } + line[i] = '\0'; + ASSERT_GT(i, 0); - char level_str[10]; - snprintf(level_str, 10, "\"level\":%d", level); - ASSERT_STREQ(got, level_str); + char want[32]; + snprintf(want, sizeof(want), "\"level\":\"%s\"", level); + ASSERT_NE(strstr(line, want), nullptr) << "line: " << line; } } // namespace Pipi diff --git a/esp/speaker-netsender/components/pipi/pipi.cpp b/esp/speaker-netsender/components/pipi/pipi.cpp index a9b5048..023a175 100644 --- a/esp/speaker-netsender/components/pipi/pipi.cpp +++ b/esp/speaker-netsender/components/pipi/pipi.cpp @@ -35,9 +35,9 @@ #include #include #include -#include +#include #include -#include +#include #include "esp_err.h"