Feature/tls keylog

AGENTS.md

@@ -93,14 +93,23 @@ ctest --test-dir build --output-on-failure --no-tests=error --timeout 180   # ne
    omission is invisible until someone flashes a board. They diverge on purpose — `pal_posix.c`
    and `iot_pal_defaults.c` host-only (each IDF app defines its own `get_default_pal()`),
    `pal_freertos.c` IDF-only — so never blind-sync them.
+<<<<<<< HEAD
 5. **Module code goes through the PAL**: `pal->malloc`/`pal->free` for memory, the `log_tag_*`
    macros in `common/log.h` for output (via each module's own family: `IOT_LOG*`,
    `TAI_LOG*`, `TUYA_BLE_HAL_LOG*`). A direct `malloc` or
    `printf` is a porting bug even where it links on the host -- and the `test` CI job greps for
    raw `log_emit(`/`printf(` call sites outside `common/log.{h,c}` and the test trees, so one
    fails the pipeline. The one deliberate gap: `pal_t` has
+=======
+5. **Module code goes through the PAL**: `pal->malloc`/`pal->free` for memory, `log_emit` for
+   output (via each module's prefixed `log_info`/`log_warn`/`log_error`). A direct `malloc` or
+   `printf` is a porting bug even where it links on the host. Two deliberate gaps: `pal_t` has
+>>>>>>> 2629217 (feat(common): opt-in TLS key log, for decrypting captures in Wireshark)
    only a monotonic `time_ms`, so ATOP signing reads libc `time(NULL)` — a port needs a real-time
-   clock the C library can see, or every signed request carries a `t` the cloud rejects.
+   clock the C library can see, or every signed request carries a `t` the cloud rejects; and
+   `pal_t` has no file interface, so the TLS key-log file sink in `common/tls.c` uses libc
+   `fopen`/`fputs`, compiled in only where `TLS_KEYLOG_FILE_SINK` is 1 (POSIX and ESP-IDF by
+   default) so a bare newlib port without `_open`/`_close` stubs still links `tls.o`.
 6. **CHANGELOG entries are terse, and carry a PR number.** One line per change —
    `- <module> — <what changed>(#<PR>).` — with indented sub-bullets only for specifics a reader
    acts on (a new symbol, a changed default, a migration step). `## [0.3.0]` is the reference for

CHANGELOG.md

@@ -35,6 +35,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - A scan provider enables WiFi-list/status capability discovery, not a complete PSK3.0 activation exchange.
 - docs-site — new bilingual guide "DP (Data Point): Definition, Creation, and Usage" under User Guides → Cloud Configuration (#41).
   - Covers the official three-level DP definition, the four elements (DPID/DPCode/type/constraints), the six data types, rw/ro/wr transfer modes, `dps` and cloud command-set message formats, cloud rate limits, platform-side DP creation, and DP usage in agentic-kit (schema intake, type mapping, core `iot_dp_*` APIs, a typical code flow), with official and on-site references.
+- common — opt-in TLS key log, for decrypting a capture of the SDK's cloud
+  traffic in Wireshark(#34).
+  - `tls_keylog_open_file()` / `tls_keylog_close_file()` write NSS
+    `SSLKEYLOGFILE` lines to a file (compiled in on POSIX and ESP-IDF, or with
+    `TLS_KEYLOG_FILE_SINK=1`); `tls_set_keylog_handler()` routes them to your
+    own sink.
+  - Process-wide and off by default: enable it once before the first TLS
+    connection and every channel (MQTT, ATOP, IoT-DNS, RTC/TAI) exports.
+  - A runtime switch for debugging only; enabling it logs a `LOG_WARN`.
+    Needs mbedTLS 3.x.
+  - `docs-site/docs/guides/tls-keylog.md` covers the Wireshark setup.
 
 ### Changed
 

CMakeLists.txt

@@ -392,6 +392,11 @@ if(AGENTIC_KIT_BUILD_TESTS AND AGENTIC_KIT_ENABLE_PROJECT_TESTS)
         "${coremqtt_source_dir}/source/core_mqtt.c"
         "${coremqtt_source_dir}/source/core_mqtt_serializer.c"
         "${coremqtt_source_dir}/source/core_mqtt_state.c"
+        # Same remap story as coreMQTT/coreHTTP above: tls.c lives in the
+        # remap-free agentic_kit_common, but its keylog warning is asserted
+        # by tests -- compiling it here puts its lines behind the sink while
+        # agentic_kit_common's copy stays unreferenced on the link line.
+        "${COMMON_DIR}/tls.c"
         "${IOT_CLIENT_TESTS_DIR}/test_log.c"
     )
     target_include_directories(tuya_iot_client_testlog PUBLIC

common/tls.c

@@ -11,12 +11,37 @@
 
 #include <string.h>
 #include <stdio.h>
+#include <errno.h>
 
 #include "mbedtls/ssl.h"
 #include "mbedtls/x509_crt.h"
 #include "mbedtls/error.h"
 #include "mbedtls/net_sockets.h"
 #include "mbedtls/version.h"
+#include "mbedtls/platform_util.h"   /* mbedtls_platform_zeroize for the key log */
+
+/* The key-log file sink is the only libc file I/O in library code. It is
+ * compiled in where a writable filesystem is the norm (hosts, ESP-IDF's VFS)
+ * and out elsewhere, so a bare newlib port without _open/_close/_lseek stubs
+ * still links tls.o. Override with -DTLS_KEYLOG_FILE_SINK=0/1. */
+#ifndef TLS_KEYLOG_FILE_SINK
+#  if defined(__unix__) || defined(__APPLE__) || defined(ESP_PLATFORM)
+#    define TLS_KEYLOG_FILE_SINK 1
+#  else
+#    define TLS_KEYLOG_FILE_SINK 0
+#  endif
+#endif
+
+/* On POSIX the file is created with open(2) -- mode from creation, no symlink
+ * following, no inheritance across exec -- and fsync'ed per line. */
+#if TLS_KEYLOG_FILE_SINK && (defined(__unix__) || defined(__APPLE__))
+#  define TLS_KEYLOG_POSIX_FILE 1
+#  include <fcntl.h>
+#  include <sys/stat.h>
+#  include <unistd.h>
+#else
+#  define TLS_KEYLOG_POSIX_FILE 0
+#endif
 
 /* =========================================================================
  * tls_t -- per-connection state
@@ -28,6 +53,8 @@ struct tls_conn {
     void                *tcp_handle;
     const pal_t         *pal;
     void                *yield_mutex;  /* serialises ssl_read/ssl_write       */
+    tls_keylog_fn        keylog_fn;    /* key-log sink, snapshotted by tls_connect */
+    void                *keylog_ctx;
 };
 
 /* =========================================================================
@@ -63,6 +90,229 @@ static int tls_map_verify(tls_verify_t v)
     }
 }
 
+/* =========================================================================
+ * TLS key log (NSS SSLKEYLOGFILE format) -- opt-in, off by default.
+ *
+ * Process-wide like the DRBG above: one sink covers every connection. The app
+ * installs it once at startup, single-threaded, before the first tls_connect()
+ * (tls.h); tls_connect() then snapshots the pair into the tls_t on the
+ * connecting thread, and mbedTLS's export callback reads only its own
+ * connection's copy. Thread creation orders that install before any later
+ * handshake, so the globals need neither volatile nor a lock -- and neither
+ * would make a *swap* during a handshake safe, which is why tls.h forbids one.
+ * ========================================================================= */
+
+static tls_keylog_fn g_keylog_fn  = NULL;
+static void         *g_keylog_ctx = NULL;
+
+#if TLS_KEYLOG_FILE_SINK
+/* Set once a write has failed, so the failure is logged once rather than on
+ * every handshake. Reset when the file sink is (re)installed. */
+static bool g_keylog_file_failed = false;
+
+static void tls_keylog_file_write(void *ctx, const char *line)
+{
+    FILE *f = (FILE *)ctx;
+    /* Unbuffered stream (see open_file): fputs() is one write(2). A failed write
+     * -- ENOSPC, a VFS gone read-only -- would otherwise drop the line silently
+     * and leave a capture that will not decrypt, with nothing in the log to say
+     * why. */
+    if (fputs(line, f) == EOF) {
+        if (!g_keylog_file_failed) {
+            g_keylog_file_failed = true;
+            log_tag_error("tls", "key log write failed (%s); later lines are lost",
+                     strerror(errno));
+        }
+        return;
+    }
+#if TLS_KEYLOG_POSIX_FILE
+    /* write(2) alone leaves the line in the page cache; tls.h promises it
+     * survives a reset mid-session, so push it through. */
+    (void)fsync(fileno(f));
+#endif
+}
+#endif /* TLS_KEYLOG_FILE_SINK */
+
+/* The one place the (fn, ctx) pair changes. Detaches the outgoing sink before
+ * releasing anything it owns, publishes the new pair, and emits the single
+ * "key logging ENABLED" line the docs tell operators to grep production logs
+ * for. `where` names the sink for that line. */
+static void keylog_install(tls_keylog_fn fn, void *ctx, const char *where)
+{
+    tls_keylog_fn old_fn  = g_keylog_fn;
+    void         *old_ctx = g_keylog_ctx;
+    g_keylog_fn  = NULL;
+    g_keylog_ctx = NULL;
+#if TLS_KEYLOG_FILE_SINK
+    if (old_fn == tls_keylog_file_write) {
+        fclose((FILE *)old_ctx);
+        g_keylog_file_failed = false;
+    }
+#else
+    (void)old_fn;
+    (void)old_ctx;
+#endif
+    g_keylog_ctx = ctx;
+    g_keylog_fn  = fn;
+    if (fn)
+        log_tag_warn("tls", "key logging ENABLED -- session secrets are "
+                           "being exported to %s; do not use in production", where);
+}
+
+void tls_set_keylog_handler(tls_keylog_fn fn, void *ctx)
+{
+    keylog_install(fn, ctx, "a custom sink");
+}
+
+int tls_keylog_open_file(const char *path)
+{
+    if (!path || path[0] == '\0') return TLS_ERR_ARGS;
+#if TLS_KEYLOG_FILE_SINK
+    if (g_keylog_fn == tls_keylog_file_write) {
+        log_tag_error("tls", "key log file already open");
+        return TLS_ERR_ARGS;
+    }
+    FILE *f = NULL;
+#if TLS_KEYLOG_POSIX_FILE
+    /* The file IS the secret: 0600 from the moment it exists (fopen("a") +
+     * fchmod would leave it at 0666 & ~umask in between), never through a
+     * symlink someone planted at the documented path, never inherited by a
+     * child the app execs. */
+    int fd = open(path, O_WRONLY | O_CREAT | O_APPEND | O_NOFOLLOW | O_CLOEXEC,
+                  S_IRUSR | S_IWUSR);
+    if (fd >= 0) {
+        f = fdopen(fd, "a");
+        if (!f) close(fd);
+    }
+#else
+    f = fopen(path, "a");
+#endif
+    if (!f) {
+        log_tag_error("tls", "cannot open key log file '%s': %s",
+                 path, strerror(errno));
+        return TLS_ERR_ARGS;
+    }
+    /* No stdio buffering: a buffer holding secrets cannot be wiped, and an
+     * unflushed line is a capture that will not decrypt. A stream that cannot be
+     * made unbuffered is refused rather than silently buffered. */
+    if (setvbuf(f, NULL, _IONBF, 0) != 0) {
+        log_tag_error("tls", "cannot make key log file '%s' unbuffered", path);
+        fclose(f);
+        return TLS_ERR_ARGS;
+    }
+    keylog_install(tls_keylog_file_write, f, path);
+    return TLS_OK;
+#else
+    log_tag_error("tls", "key log file sink not compiled in "
+                        "(TLS_KEYLOG_FILE_SINK=0); use tls_set_keylog_handler()");
+    return TLS_ERR_ARGS;
+#endif
+}
+
+void tls_keylog_close_file(void)
+{
+#if TLS_KEYLOG_FILE_SINK
+    if (g_keylog_fn == tls_keylog_file_write)
+        keylog_install(NULL, NULL, NULL);
+#endif
+}
+
+/* The export side needs mbedTLS 3.x (mbedtls_ssl_set_export_keys_cb and the
+ * mbedtls_ssl_key_export_type enum replaced the 2.x conf_export_keys_cb API);
+ * against 2.x the sink can be installed but nothing is exported, and
+ * tls_connect() says so. */
+#if MBEDTLS_VERSION_MAJOR >= 3
+
+/* NSS label for one mbedTLS export type, or NULL for a type Wireshark has no
+ * label for (nothing is written then, rather than a line it would ignore). */
+static const char *tls_keylog_label(mbedtls_ssl_key_export_type type)
+{
+    switch (type) {
+    case MBEDTLS_SSL_KEY_EXPORT_TLS12_MASTER_SECRET:
+        return "CLIENT_RANDOM";
+#if defined(MBEDTLS_SSL_PROTO_TLS1_3)
+    case MBEDTLS_SSL_KEY_EXPORT_TLS1_3_CLIENT_EARLY_SECRET:
+        return "CLIENT_EARLY_TRAFFIC_SECRET";
+    case MBEDTLS_SSL_KEY_EXPORT_TLS1_3_EARLY_EXPORTER_SECRET:
+        return "EARLY_EXPORTER_SECRET";
+    case MBEDTLS_SSL_KEY_EXPORT_TLS1_3_CLIENT_HANDSHAKE_TRAFFIC_SECRET:
+        return "CLIENT_HANDSHAKE_TRAFFIC_SECRET";
+    case MBEDTLS_SSL_KEY_EXPORT_TLS1_3_SERVER_HANDSHAKE_TRAFFIC_SECRET:
+        return "SERVER_HANDSHAKE_TRAFFIC_SECRET";
+    case MBEDTLS_SSL_KEY_EXPORT_TLS1_3_CLIENT_APPLICATION_TRAFFIC_SECRET:
+        return "CLIENT_TRAFFIC_SECRET_0";
+    case MBEDTLS_SSL_KEY_EXPORT_TLS1_3_SERVER_APPLICATION_TRAFFIC_SECRET:
+        return "SERVER_TRAFFIC_SECRET_0";
+#endif
+    default:
+        return NULL;
+    }
+}
+
+/* Longest secret any export type carries: the TLS 1.2 master secret is 48
+ * bytes; a TLS 1.3 traffic secret is one hash, 64 bytes with SHA-512 enabled --
+ * the same bound mbedTLS uses (MBEDTLS_TLS1_3_MD_MAX_SIZE). Anything longer is
+ * not a line Wireshark expects and is dropped rather than truncated into a
+ * silently undecryptable log. */
+#define TLS_KEYLOG_MAX_SECRET 64
+#define TLS_KEYLOG_MAX_LABEL  40
+_Static_assert(sizeof("CLIENT_HANDSHAKE_TRAFFIC_SECRET") - 1 <= TLS_KEYLOG_MAX_LABEL,
+               "longest NSS key-log label must fit TLS_KEYLOG_MAX_LABEL");
+/* label + ' ' + 32-byte client random in hex + ' ' + secret in hex + "\n\0" */
+#define TLS_KEYLOG_LINE_SIZE  (TLS_KEYLOG_MAX_LABEL + 1 + 64 + 1 + \
+                               TLS_KEYLOG_MAX_SECRET * 2 + 2)
+
+/* Same loop as iot_ota_verify.c's bytes_to_hex_lower; duplicated on purpose,
+ * common/ has no hex helper and a module's static is not reachable from here. */
+static void tls_hex(char *out, const unsigned char *in, size_t len)
+{
+    static const char hexdig[] = "0123456789abcdef";
+    for (size_t i = 0; i < len; i++) {
+        out[i * 2]     = hexdig[in[i] >> 4];
+        out[i * 2 + 1] = hexdig[in[i] & 0x0F];
+    }
+}
+
+/* mbedTLS key-export callback: format one NSS key-log line and hand it to this
+ * connection's sink. Runs on the handshaking thread; registered by tls_connect()
+ * only when a sink was installed, so p_expkey is always a tls_t with one. */
+static void tls_keylog_export(void *p_expkey,
+                              mbedtls_ssl_key_export_type type,
+                              const unsigned char *secret,
+                              size_t secret_len,
+                              const unsigned char client_random[32],
+                              const unsigned char server_random[32],
+                              mbedtls_tls_prf_types tls_prf_type)
+{
+    tls_t *t = (tls_t *)p_expkey;
+    (void)server_random;
+    (void)tls_prf_type;
+
+    const char *label = tls_keylog_label(type);
+    if (!label || secret_len == 0) return;
+
+    char line[TLS_KEYLOG_LINE_SIZE];
+    size_t label_len = strlen(label);
+    /* One bound, checked against the real buffer so it cannot drift from it. */
+    if (label_len + 1 + 64 + 1 + secret_len * 2 + 2 > sizeof(line)) return;
+    size_t n = 0;
+
+    memcpy(line + n, label, label_len);      n += label_len;
+    line[n++] = ' ';
+    tls_hex(line + n, client_random, 32);    n += 64;
+    line[n++] = ' ';
+    tls_hex(line + n, secret, secret_len);   n += secret_len * 2;
+    line[n++] = '\n';
+    line[n]   = '\0';
+
+    t->keylog_fn(t->keylog_ctx, line);
+
+    /* The line is a copy of the secret; do not leave it on the stack. */
+    mbedtls_platform_zeroize(line, sizeof(line));
+}
+
+#endif /* MBEDTLS_VERSION_MAJOR >= 3 */
+
 /* =========================================================================
  * BIO callbacks -- adapt PAL tcp_send/recv to mbedTLS f_send/f_recv.
  *
@@ -207,6 +457,21 @@ tls_t *tls_connect(const tls_config_t *cfg)
 
     if (mbedtls_ssl_setup(&t->ssl, &t->conf) != 0) goto fail;
 
+    /* Snapshot the key-log sink on the connecting thread and register the export
+     * callback only when one is installed: a connection with key logging off
+     * carries no callback and pays nothing at the bottom of the handshake. Every
+     * export call site is inside the handshake and tls.c never enables
+     * renegotiation, so this is the "read at handshake time" tls.h documents. */
+    t->keylog_fn  = g_keylog_fn;
+    t->keylog_ctx = g_keylog_ctx;
+    if (t->keylog_fn) {
+#if MBEDTLS_VERSION_MAJOR >= 3
+        mbedtls_ssl_set_export_keys_cb(&t->ssl, tls_keylog_export, t);
+#else
+        log_emit(LOG_WARN, "[tls] key logging needs mbedTLS 3.x; nothing is exported");
+#endif
+    }
+
     if (mbedtls_ssl_set_hostname(&t->ssl, cfg->sni ? cfg->sni : cfg->host) != 0)
         goto fail;