diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a44c4e6..0632b77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,13 +89,13 @@ test:memory-leak: # Leak-check the pure (no-subprocess) binaries, grouped by module: # rtc-tcp-client : tai_unit_tests, tai_log_level_zero_tests, # tai_log_module_zero_tests, tai_integration_tests - # iot-client : iot_cipher_test + # iot-client : iot_cipher_test, iot_ai_ctrl_test # tuya-ble : tuya_ble_test # rtc-client is a prebuilt closed lib (headers + libstm.a) with no offline # test, so there is nothing to leak-check for it here. The mock-driven iot # tests are covered by the `test` job; under valgrind their Python # handshakes would time out. - for t in tai_unit_tests tai_log_level_zero_tests tai_log_module_zero_tests tai_integration_tests iot_cipher_test tuya_ble_test; do + for t in tai_unit_tests tai_log_level_zero_tests tai_log_module_zero_tests tai_integration_tests iot_cipher_test iot_ai_ctrl_test tuya_ble_test; do echo "==================== valgrind: ${t} ====================" "$VG" "./${CMAKE_BUILD_DIR}/${t}" 2>&1 | tee "valgrind-${t}.log" rc=${PIPESTATUS[0]} diff --git a/CHANGELOG.md b/CHANGELOG.md index bc0844c..43924bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- iot-client — MQTT protocol-9000 AI control channel for server-initiated `asrInterrupt` notices, delivered via `ai_ctrl_callback_t` independently of the RTC TCP Connection (#42). + - The notice payload carries `eventId` and the server time `time`; the time is the interruption cutoff. +- rtc-tcp-client — `on_flow_control` admission hook for TCP receive backpressure; pauses all inbound Frames when the application's audio queue is full, including codec-frame callbacks mid-Packet (#42). + - New knobs: `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS` (admission-check period while paused, default 50 ms) and `AGENTIC_KIT_TAI_WORKER_YIELD_MS` (worker drain-pass yield under sustained traffic, default 10 ms). +- rtc-tcp-client — `tai_event_msg_t` gains borrowed `user_data` / `user_data_len` (attr 111), so an interruption notice can be read without the SDK parsing JSON (#42). + - `TAI_EVT_CHAT_BREAK` carries `{"breakAttributes":{"time":""}}`; compare that time with the `timestamp_ms` latched from audio START to discard an interrupted stream. A notice whose time is missing or unusable fails closed onto the in-flight stream. + +### Changed + +- **BREAKING** pal — `pal_t` gains a mandatory `sleep_ms` member; custom PALs must supply it and all consumers must rebuild (#42). +- rtc-tcp-client — audio Packets paused mid-body now retain a pending-delivery cursor instead of silently dropping codec frames; reopening admission resumes from the first unadmitted byte (#42). + - A pinned Packet whose recorded wire length cannot be consumed exactly once fails fast with `TAI_PROTO_ERR_FRAME_DECODE` instead of sliding the receive buffer out of range. + - A header-only START/ONE_SHOT now reaches `on_audio` with `len == 0`, so its server-side `timestamp_ms` can be latched for interruption filtering. +- rtc-tcp-client — `tai_connect()` returns once the session acknowledgement is handled; media the server coalesced into the handshake stays buffered and is first delivered by the receive worker after connect returns (#42). + - docs-site — English edition of the full documentation site, published at `/en/` with Simplified Chinese retained at `/`. - All 29 docs are mirrored under `docs-site/i18n/en/`, with a locale selector in both the landing-page navbar and the custom docs topbar, localized navbar/footer/sidebar catalogs, and English SVG schematics under `current/images/`. - Heading anchors use explicit IDs shared across locales, and `npm run check:i18n` gates path, image, and anchor parity as part of `npm run build`. diff --git a/CMakeLists.txt b/CMakeLists.txt index bc89af3..fc56e36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,6 +193,7 @@ set(IOT_CLIENT_SOURCES "${IOT_CLIENT_DIR}/src/atop_base.c" "${IOT_CLIENT_DIR}/src/cipher_wrapper.c" "${IOT_CLIENT_DIR}/src/http_client_interface.c" + "${IOT_CLIENT_DIR}/src/iot_ai_ctrl.c" "${IOT_CLIENT_DIR}/src/iot_atop.c" "${IOT_CLIENT_DIR}/src/iot_client.c" "${IOT_CLIENT_DIR}/src/iot_client_message.c" @@ -437,11 +438,43 @@ if(AGENTIC_KIT_BUILD_TESTS AND AGENTIC_KIT_ENABLE_PROJECT_TESTS) agentic_kit_add_iot_test(iot_reset_test "${IOT_CLIENT_TESTS_DIR}/iot_reset_test.c") agentic_kit_add_iot_test(iot_mqtt_test "${IOT_CLIENT_TESTS_DIR}/mqtt_test.c") agentic_kit_add_iot_test(iot_message_test "${IOT_CLIENT_TESTS_DIR}/iot_client_message_test.c") + agentic_kit_add_iot_test(iot_ai_ctrl_test "${IOT_CLIENT_TESTS_DIR}/iot_ai_ctrl_test.c") agentic_kit_add_iot_test(iot_on_boarding_test "${IOT_CLIENT_TESTS_DIR}/on_boarding_test.c") agentic_kit_add_iot_test(iot_dp_test "${IOT_CLIENT_TESTS_DIR}/iot_dp_test.c") agentic_kit_add_iot_test(iot_ota_test "${IOT_CLIENT_TESTS_DIR}/iot_ota_test.c") agentic_kit_add_iot_test(iot_ota_verify_test "${IOT_CLIENT_TESTS_DIR}/iot_ota_verify_test.c") + add_executable(iot_tai_control_test + ${AI_TCP_SOURCES} + "${AI_TCP_TESTS_DIR}/tai_pal_loopback.c" + "${IOT_CLIENT_TESTS_DIR}/iot_tai_control_test.c" + "${IOT_CLIENT_TESTS_DIR}/mqtt_interrupt_demo_test.c" + "${IOT_CLIENT_TESTS_DIR}/test_log.c" + ) + target_include_directories(iot_tai_control_test PRIVATE + "${AI_TCP_DIR}/include" + "${AI_TCP_DIR}/src" + "${AI_TCP_TESTS_DIR}" + "${IOT_CLIENT_DIR}/include" + "${IOT_CLIENT_DIR}/src" + "${IOT_CLIENT_TESTS_DIR}" + "${IOT_CLIENT_TESTS_DIR}/log_config" + "${cjson_source_dir}" + "${PAL_DIR}" + "${COMMON_DIR}" + ) + target_link_libraries(iot_tai_control_test PRIVATE + tuya_iot_client_testlog + agentic_kit_pal + Threads::Threads + ) + target_compile_definitions(iot_tai_control_test PRIVATE + PYTHON3_EXEC="${Python3_EXECUTABLE}" + TEST_CONFIG_DIR="${IOT_CLIENT_TESTS_DIR}/config" + MESSAGE_MOCK_PATH="${IOT_CLIENT_TESTS_DIR}/mock/message_mock.py" + ) + add_test(NAME iot_tai_control_test COMMAND iot_tai_control_test) + # The ble sources compile into the executable (not the tuya_ble # library) so they pick up the test sink's log remap from # tuya_iot_client_testlog's include path -- that is what lets diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index 754f406..0e0d35e 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -18,6 +18,12 @@ and decisions. - **IoT Client → RTC TCP Client**: the IoT Client obtains an AI session token (`iot_client_get_session_token`) that the RTC client uses to open a Session. + Applications may also keep MQTT active as an independent control path: + authenticated protocol-9000 notices reach `ai_ctrl_callback_t` even while + receive backpressure stalls the RTC Connection. The application composes the + callbacks; neither client owns or calls the other. Decision records: + `modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md` + and `modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md`. - **Tuya BLE → IoT Client**: BLE provisioning runs first and hands the device its WiFi credentials (incl. the pairing token); once on WiFi, the IoT Client uses them to activate against the cloud. diff --git a/docs-site/docs/guides/vad-and-interrupt.md b/docs-site/docs/guides/vad-and-interrupt.md index 06e63b7..517fcbb 100644 --- a/docs-site/docs/guides/vad-and-interrupt.md +++ b/docs-site/docs/guides/vad-and-interrupt.md @@ -142,6 +142,12 @@ tai_config_t cfg = { ### RTC TCP Client {#rtc-tcp-client} +:::note MQTT 独立控制路径 +设备可同时保持 IoT MQTT 与 RTC TCP Connection。注册 `iot_ai_ctrl_set_callback()` 后，protocol-9000 `asrInterrupt` 不受 RTC TCP 接收背压影响。应用应让一个线程独占 `iot_client_process()` / publish / reconnect，并把 MQTT callback 与 `TAI_EVT_CHAT_BREAK` 汇入同一线程安全的播放策略。 + +打断的判定依据是**服务端时间**，不是 eventId：MQTT `asrInterrupt` 读 `data.time`，TCP `TAI_EVT_CHAT_BREAK` 读 attr 111 中的 `breakAttributes.time`（不是事件负载）。把最大的时间记为截止时间，清空播放队列，再恢复 RTC 接收，让 worker 继续校验和排空旧媒体；不要直接丢弃 TCP 字节，否则会破坏 Frame 边界。`on_audio` 中锁定 START 的 `msg->timestamp_ms`，凡不晚于截止时间的流一律丢弃。服务端打断不要求调用 `tai_chat_break()`，也不要结束或重开云端 VAD 上行流。 +::: + **接收服务端打断（`TAI_EVT_CHAT_BREAK`，type=4）：** `TAI_EVT_CHAT_BREAK` 有双重身份：用户在 AI 回复中插话时它是打断信号；在云端 VAD 模式下它同时也是**回合结束信号**（云端检测到用户停止说话后下发，当前云端不再下发 `TAI_EVT_SERVER_VAD`）。两种情况下的设备处理相同： @@ -152,18 +158,34 @@ void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *ud) if (msg->event_type == TAI_EVT_CHAT_BREAK) { // 1. 停止 TTS 播放 audio_player_stop(); - // 2. 清空播放缓冲区（丢弃本轮在途 TTS，直到下一个 TAI_STREAM_START） + // 2. 从 attr 111 取服务端打断时间，推进截止时间（取最大值） + uint64_t cutoff = parse_break_time(msg->user_data, msg->user_data_len); + if (cutoff) audio_cutoff_ms = cutoff > audio_cutoff_ms ? cutoff : audio_cutoff_ms; + // 3. 清空播放缓冲区；只有晚于截止时间的流才会重新入队 audio_buffer_flush(); - // 3. 忽略本轮后续回调 - set_ignore_current_response(true); // 不要停止麦克风采音，不要调用 tai_send_audio_end()， // 也不要调用 tai_send_audio_start() 重开上行流—— // 云端 VAD 模式下上行 Event 一直保持打开。 - // 若该打断的 eventId 没有对应的本地下行缓存，记录日志并忽略即可。 + // 若 attr 111 缺失或时间无效，按失败关闭处理：把当前流的 START + // 当作截止时间，丢弃在途流，而不是当作无事发生继续播放。 } } ``` +`on_audio` 侧锁定 START 的服务端时间，并据此过滤： + +```c +void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *ud) +{ + if (msg->stream_flag == TAI_STREAM_START || + msg->stream_flag == TAI_STREAM_ONE_SHOT) + stream_start_ms = msg->timestamp_ms; // 服务端时间，非本地时间 + // MIDDLE/END 不带新时间，沿用本轮流锁定的起始时间 + if (msg->len && stream_start_ms > audio_cutoff_ms) + audio_buffer_push(msg->data, msg->len); +} +``` + **发送客户端打断：** ```c diff --git a/docs-site/docs/reference/iot-client.md b/docs-site/docs/reference/iot-client.md index 2150838..285e1ea 100644 --- a/docs-site/docs/reference/iot-client.md +++ b/docs-site/docs/reference/iot-client.md @@ -441,6 +441,20 @@ int iot_client_publish(iot_client_t *client, const uint8_t *data, size_t data_le --- +### `iot_ai_ctrl_set_callback` {#iot_ai_ctrl_set_callback} + +```c +int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, + void *user_data); +``` + +注册经过 P2.3 解密和认证的 MQTT protocol-9000 AI 控制通知。回调在调用 `iot_client_process()` 的线程上触发，`type` 和 `json_data` 只在回调期间有效。回调应仅更新有界状态或通知应用线程，不应在其中断开/销毁 IoT client。 + +若必须接收订阅建立期间立即到达的通知，初始化时设置 `mqtt_disable_auto_connect=true`，先注册回调，再调用 `iot_client_connect()`。传 NULL callback 可注销。RTC TCP 接收背压不会阻塞该 MQTT 路径；应用应把 MQTT 通知和 TAI ChatBreak 汇入同一播放策略。`asrInterrupt` 的负载携带 `eventId` 与服务端时间 `time`（字符串毫秒），后者是打断判定的截止时间；把它与 `on_audio` 中锁定的 `msg->timestamp_ms` 比较来丢弃过期媒体，而不是按 eventId 判断。服务端通知不是 `tai_chat_break()` 的应答，也不应自动结束云端 VAD 上行流。 + +--- + ### `iot_get_qrcode_info` {#iot_get_qrcode_info} ```c diff --git a/docs-site/docs/reference/rtc-tcp-client.md b/docs-site/docs/reference/rtc-tcp-client.md index 2746456..737666b 100644 --- a/docs-site/docs/reference/rtc-tcp-client.md +++ b/docs-site/docs/reference/rtc-tcp-client.md @@ -178,7 +178,7 @@ sidebar_position: 1 | 字段 | 类型 | 说明 | |------|------|------| | `ping_interval_ms` | `uint32_t` | Ping 间隔（0 = 默认 60000ms） | -| `ping_timeout_ms` | `uint32_t` | Ping 超时（0 = 默认 90000ms） | +| `ping_timeout_ms` | `uint32_t` | 接收存活超时，任意入站数据均刷新（0 = 默认 90000ms）；主动背压暂停期间不计超时，恢复时获得完整的新预算 | | `connect_timeout_ms` | `uint32_t` | 连接超时（0 = 默认 5000ms）。分别约束 `tai_connect` 的两个串行等待阶段：先是连接建立（TCP 建连 + TLS 握手，共用一份预算），再是服务端 SessionNew 应答。任一阶段超时即判定连接失败，因此 `tai_connect` 最坏耗时约为该值的 2 倍。 | ### 3.7 测试配置 {#37-测试配置} @@ -205,6 +205,7 @@ sidebar_position: 1 | `on_image` | function pointer | 图像数据回调（云端生成的图片） | | `on_event` | function pointer | 事件回调（MCP、打断、VAD 等） | | `on_disconnect` | function pointer | 断连回调 | +| `on_flow_control` | `int (*)(tai_ctx_t *, void *)` | 可选接收背压钩子；非零允许接收，0 暂停读取与解析，NULL 不启用背压 | | `user_data` | `void *` | 透传到所有回调 | **回调签名：** @@ -217,8 +218,20 @@ void (*on_text) (tai_ctx_t *ctx, const tai_text_msg_t *msg, void *use void (*on_image) (tai_ctx_t *ctx, const tai_image_msg_t *msg, void *user_data); void (*on_event) (tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data); void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *user_data); +int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); ``` +#### 接收背压（`on_flow_control`） + +- worker 在每次读取前及完整 Frame 之间调用钩子。符合规范的服务端不会在握手阶段触发它（应答先于媒体下发）；仅当服务端把媒体排在应答之前才可能在握手期被调用。钩子必须非阻塞；应用负责同步共享的队列状态。 +- 返回 0 会暂停读取和解析，所有入站流量都会停滞，包括 ChatBreak、ASR 文本、Pong 和 EOF 检测。恢复后先处理已缓冲的完整 Frame，再读取；不完整输入回到有上限的阻塞接收。 +- 主动暂停挂起接收存活超时；恢复时获得新的完整 `ping_timeout_ms` 预算。Ping 和停止请求仍执行，Ping 发送失败仍断连。 +- 背压也在 Audio Packet 内的每个编解码帧回调前检查。中途暂停会以零拷贝方式保留该 Packet 的剩余字节，恢复时先交付剩余帧、再处理后续 Packet。应用应在 `on_audio` 中按服务端时间（`msg->timestamp_ms`）丢弃过期音频，而不是改写 SDK 接收状态。 + +默认每 `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS`（50 ms）重查背压，并以 `AGENTIC_KIT_TAI_WORKER_YIELD_MS`（10 ms）在持续流量下让出 CPU。 + +`pal_t` 新增必填的 `sleep_ms` 回调。它必须非忙等地休眠至少请求的毫秒数，0 为 no-op，且不能依赖 socket 可读性。所有自定义 PAL 和使用方必须用新头文件重新编译。 + ### 接收消息结构体 {#接收消息结构体} `tai_audio_msg_t`（音频回调）： @@ -233,7 +246,7 @@ void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *use | `stream_flag` | `uint8_t` | `TAI_STREAM_*`（取自媒体头） | | `data_id` | `uint16_t` | 数据 ID：`AUDIO_DOWN`(2) / `AUDIO_AUX`(7) | | `event_id` | `const char *` | turn id（借用）；无则为 `""` | -| `timestamp_ms` | `uint64_t` | 流起始时间戳（媒体头） | +| `timestamp_ms` | `uint64_t` | 服务端媒体头时间戳，**非**本地时间；过滤流时以 START 的值锁定，与打断时间比较 | `tai_text_msg_t`（文本回调）： @@ -272,6 +285,12 @@ void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *use | `data` | `const uint8_t *` | 事件负载（通常为 JSON） | | `len` | `size_t` | 负载字节数 | | `event_id` | `const char *` | attr 61（借用）；无则为 `""` | +| `user_data` | `const uint8_t *` | attr 111（借用），**非** NUL 结尾；无则为 NULL | +| `user_data_len` | `size_t` | `user_data` 字节数；与事件负载分离，SDK 不解析其 JSON | + +:::note ChatBreak 打断时间 +`TAI_EVT_CHAT_BREAK` 的服务端时间不在事件负载里，而在 attr 111 中：`{"breakAttributes":{"time":""}}`，值为服务端 epoch 毫秒，与 `on_audio` 的 `timestamp_ms` 同钟同单位。读取 `msg->user_data`、解析 `breakAttributes.time`，再与 `on_audio` 中锁定的 `timestamp_ms` 比较，即可判定该流是否已过期。MQTT 侧的 `asrInterrupt` 走另一条路径：时间在其自身负载的 `time` 字段，且是同一个服务端时间值。 +::: `tai_disconnect_msg_t`（断连回调）： diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md index 0a5915d..f303707 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md @@ -144,6 +144,12 @@ When the user speaks again while the AI is responding, the current response need ### RTC TCP Client {#rtc-tcp-client} +:::note Independent MQTT control path +A device can keep the IoT MQTT and RTC TCP Connection active together. After `iot_ai_ctrl_set_callback()` is registered, a protocol-9000 `asrInterrupt` is independent of RTC TCP receive backpressure. One application thread should own `iot_client_process()`, publish, and reconnect, while the MQTT callback and `TAI_EVT_CHAT_BREAK` feed the same thread-safe playback policy. + +An interruption is decided by **server time**, not by event ID: MQTT `asrInterrupt` carries `data.time`, and TCP `TAI_EVT_CHAT_BREAK` carries `breakAttributes.time` inside attr 111 (not in the event payload). Keep the greatest value as the cutoff, flush the playback queue, then release RTC receive pressure so the worker can keep authenticating and draining old media -- never discard raw TCP bytes, which would corrupt Frame boundaries. In `on_audio`, latch START's `msg->timestamp_ms` and drop every stream at or before the cutoff. A server notice does not require `tai_chat_break()`, and it must not end or reopen a Server-VAD uplink. +::: + **Receive a server-initiated chat break (`TAI_EVT_CHAT_BREAK`, type=4):** `TAI_EVT_CHAT_BREAK` has two roles: it signals an interruption when the user speaks during the AI response; in server-VAD mode, it is also the **turn-end signal** (sent when the cloud detects that the user has stopped speaking; the current cloud no longer sends `TAI_EVT_SERVER_VAD`). Device-side handling is the same in both cases: @@ -154,20 +160,34 @@ void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *ud) if (msg->event_type == TAI_EVT_CHAT_BREAK) { // 1. Stop TTS playback audio_player_stop(); - // 2. Clear the playback buffer (discard this turn's in-flight TTS - // until the next TAI_STREAM_START) + // 2. Read the server interruption time from attr 111 and advance the cutoff + uint64_t cutoff = parse_break_time(msg->user_data, msg->user_data_len); + if (cutoff) audio_cutoff_ms = cutoff > audio_cutoff_ms ? cutoff : audio_cutoff_ms; + // 3. Clear the playback buffer; only streams newer than the cutoff requeue audio_buffer_flush(); - // 3. Ignore subsequent callbacks for this turn - set_ignore_current_response(true); // Do not stop microphone capture or call tai_send_audio_end(). // Do not call tai_send_audio_start() to reopen the uplink stream either: // in server-VAD mode, the uplink Event remains open throughout. - // If this chat break's eventId has no corresponding local downlink buffer, - // just log and ignore it. + // If attr 111 is missing or its time is invalid, fail closed: treat the + // in-flight stream's START as the cutoff instead of letting it play on. } } ``` +`on_audio` latches START's server time and filters on it: + +```c +void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *ud) +{ + if (msg->stream_flag == TAI_STREAM_START || + msg->stream_flag == TAI_STREAM_ONE_SHOT) + stream_start_ms = msg->timestamp_ms; // server time, not local time + // MIDDLE/END carry no new time; they reuse this turn's latched start + if (msg->len && stream_start_ms > audio_cutoff_ms) + audio_buffer_push(msg->data, msg->len); +} +``` + **Send a client-initiated chat break:** ```c diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md index a617232..5345bf4 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md @@ -440,6 +440,20 @@ Publishes an encrypted message to `smart/device/out/{deviceid}`. --- +### `iot_ai_ctrl_set_callback` {#iot_ai_ctrl_set_callback} + +```c +int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, + void *user_data); +``` + +Registers for P2.3-decrypted and authenticated MQTT protocol-9000 AI control notices. The callback runs on the thread calling `iot_client_process()`, and `type` plus `json_data` remain valid only during the callback. Keep it bounded to updating state or notifying an application thread; do not disconnect or destroy the IoT client there. + +If notices sent during subscription setup must be observable, initialize with `mqtt_disable_auto_connect=true`, register the callback, and then call `iot_client_connect()`. Pass a NULL callback to deregister. RTC TCP receive backpressure does not stall this MQTT path. The application should merge MQTT notices and TAI ChatBreak into one playback policy. An `asrInterrupt` payload carries an `eventId` and a server time `time` (a string of milliseconds); that time is the cutoff for interruption filtering -- compare it with the `msg->timestamp_ms` latched in `on_audio` to discard obsolete media, rather than deciding by event ID. A server notice is not an acknowledgment request for `tai_chat_break()` and must not automatically end a Server-VAD uplink. + +--- + ### `iot_get_qrcode_info` {#iot_get_qrcode_info} ```c diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md index 6850265..6989467 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md @@ -178,7 +178,7 @@ Populate this structure before calling `tai_ctx_init`. All pointer fields must r | Field | Type | Description | |------|------|------| | `ping_interval_ms` | `uint32_t` | Ping interval (0 = default 60000ms) | -| `ping_timeout_ms` | `uint32_t` | Ping timeout (0 = default 90000ms) | +| `ping_timeout_ms` | `uint32_t` | Receive-liveness timeout refreshed by any inbound data (0 = default 90000ms); intentional backpressure suspends it and resume grants a fresh full budget | | `connect_timeout_ms` | `uint32_t` | Connection timeout (0 = default 5000ms). It separately constrains the two sequential wait stages of `tai_connect`: first Connection establishment (TCP connection + TLS handshake, sharing one budget), then the server's SessionNew response. A timeout in either stage means the Connection failed, so the worst-case duration of `tai_connect` is approximately twice this value. | ### 3.7 Test Configuration {#37-测试配置} @@ -205,6 +205,7 @@ All callbacks are invoked on the background receive thread. | `on_image` | function pointer | Image data callback (images generated by the cloud) | | `on_event` | function pointer | Event callback (MCP commands, Chat break, VAD, and so on) | | `on_disconnect` | function pointer | Disconnect callback | +| `on_flow_control` | `int (*)(tai_ctx_t *, void *)` | Optional receive-backpressure hook; nonzero admits receive, 0 pauses reads and parsing, NULL disables backpressure | | `user_data` | `void *` | Passed through to all callbacks | **Callback signatures:** @@ -217,8 +218,20 @@ void (*on_text) (tai_ctx_t *ctx, const tai_text_msg_t *msg, void *use void (*on_image) (tai_ctx_t *ctx, const tai_image_msg_t *msg, void *user_data); void (*on_event) (tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data); void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *user_data); +int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); ``` +#### Receive Backpressure (`on_flow_control`) + +- The worker calls the hook before each read and between complete Frames. A conforming server never consults it during the connect handshake (the ack precedes media); only a server coalescing media ahead of the ack could. The hook must not block; the application synchronizes shared queue state. +- Returning 0 pauses both reads and parsing, stalling all inbound traffic including ChatBreak, ASR text, Pong, and EOF detection. Resume processes complete buffered Frames before reading again; partial input returns to bounded blocking receive. +- An intentional pause suspends receive-liveness timeout accounting, and resume grants a fresh `ping_timeout_ms` budget. Ping and stop requests remain active; Ping send failure still disconnects. +- Admission is also checked before each codec-frame callback within an Audio Packet. A mid-Packet pause retains that Packet's remaining bytes without copying; resume delivers the remaining frames before any later Packet. Applications discard obsolete audio inside `on_audio` by comparing the server timestamp (`msg->timestamp_ms`), not by mutating SDK receive state. + +Backpressure is rechecked every `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS` (50 ms by default), and sustained receive traffic yields CPU for `AGENTIC_KIT_TAI_WORKER_YIELD_MS` (10 ms by default). + +`pal_t` adds a mandatory `sleep_ms` callback. It must sleep without busy-waiting for at least the requested milliseconds, treat 0 as a no-op, and not depend on socket readability. Recompile all custom PALs and consumers against the updated header. + ### Received Message Structures {#接收消息结构体} `tai_audio_msg_t` (audio callback): @@ -233,7 +246,7 @@ void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *use | `stream_flag` | `uint8_t` | `TAI_STREAM_*` (from the media header) | | `data_id` | `uint16_t` | Data ID: `AUDIO_DOWN`(2) / `AUDIO_AUX`(7) | | `event_id` | `const char *` | Turn ID (borrowed); `""` if absent | -| `timestamp_ms` | `uint64_t` | Stream start timestamp (media header) | +| `timestamp_ms` | `uint64_t` | Server media-header timestamp, **not** local time; latch START's value to filter a stream against an interruption time | `tai_text_msg_t` (text callback): @@ -272,6 +285,12 @@ Received images arrive as a stream of chunks: START (or ONE_SHOT) carries the fi | `data` | `const uint8_t *` | Event payload (usually JSON) | | `len` | `size_t` | Payload length in bytes | | `event_id` | `const char *` | attr 61 (borrowed); `""` if absent | +| `user_data` | `const uint8_t *` | attr 111 (borrowed), **not** NUL-terminated; NULL if absent | +| `user_data_len` | `size_t` | `user_data` length in bytes; separate from the event payload, and the SDK does not parse its JSON | + +:::note ChatBreak interruption time +The server time for `TAI_EVT_CHAT_BREAK` is not in the event payload but in attr 111: `{"breakAttributes":{"time":""}}`, a server epoch in milliseconds — the same clock and unit as `on_audio`'s `timestamp_ms`. Read `msg->user_data`, parse `breakAttributes.time`, and compare it with the `timestamp_ms` latched in `on_audio` to decide whether a stream is obsolete. The MQTT `asrInterrupt` arrives by a different path: its time is the `time` field of its own payload, and it is the same server time value. +::: `tai_disconnect_msg_t` (disconnect callback): diff --git a/examples/esp-idf/components/agentic_kit/CMakeLists.txt b/examples/esp-idf/components/agentic_kit/CMakeLists.txt index 7273f1e..4c04d86 100644 --- a/examples/esp-idf/components/agentic_kit/CMakeLists.txt +++ b/examples/esp-idf/components/agentic_kit/CMakeLists.txt @@ -15,6 +15,7 @@ set(AK_SRCS ${AK_DIR}/modules/iot-client/src/atop_base.c ${AK_DIR}/modules/iot-client/src/cipher_wrapper.c ${AK_DIR}/modules/iot-client/src/http_client_interface.c + ${AK_DIR}/modules/iot-client/src/iot_ai_ctrl.c ${AK_DIR}/modules/iot-client/src/iot_atop.c ${AK_DIR}/modules/iot-client/src/iot_client.c ${AK_DIR}/modules/iot-client/src/iot_client_message.c diff --git a/examples/posix/CMakeLists.txt b/examples/posix/CMakeLists.txt index f32c40f..22f057c 100644 --- a/examples/posix/CMakeLists.txt +++ b/examples/posix/CMakeLists.txt @@ -154,6 +154,9 @@ target_link_libraries(mcp_demo PRIVATE tuya_ai_tcp tuya_iot_client agentic_kit_p add_executable(agent_trigger_demo "${CMAKE_CURRENT_SOURCE_DIR}/ai/rtc-tcp-client/agent_trigger_demo.c") target_link_libraries(agent_trigger_demo PRIVATE tuya_ai_tcp tuya_iot_client agentic_kit_pal) +add_executable(mqtt_interrupt_demo "${CMAKE_CURRENT_SOURCE_DIR}/ai/rtc-tcp-client/mqtt_interrupt_demo.c") +target_link_libraries(mqtt_interrupt_demo PRIVATE tuya_ai_tcp tuya_iot_client agentic_kit_pal) + find_library(OPUS_LIBRARY opus) find_path(OPUS_INCLUDE_DIR opus/opus.h) if(OPUS_LIBRARY AND OPUS_INCLUDE_DIR) diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c new file mode 100644 index 0000000..81e507e --- /dev/null +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -0,0 +1,353 @@ +/* + * mqtt_interrupt_demo.c -- Independent MQTT control plus pressured TAI media. + * + * This demo uses a bounded synthetic playback queue. MQTT is pumped by one + * application-owned thread while TAI's worker independently receives media. + * Interruptions discard audio whose stream started at or before the server's + * cutoff time, reopening admission so retained stale media can be drained. + * + * Test fixture: iot_tai_control_test.c and mqtt_interrupt_demo_test.c include + * this file whole with `main` #defined away, so regressions exercise these + * actual callbacks. Every file-scope symbol must therefore stay static -- the + * same test binary links two copies of this file, and a non-static symbol + * collides at link time. + */ + +#include +#include +#include +#include +#include + +#include "cJSON.h" +#include "demo_json.h" +#include "iot_client.h" +#include "tuya_ai.h" + +extern const pal_t *tai_pal_posix(void); + +#define DEFAULT_DEVID "6cd370251e8be96de8vwoe" +#define DEFAULT_SECRET_KEY "[SPT;N:b@)wPzK/)" +#define DEFAULT_LOCAL_KEY "#d[<4y*N.vE]RAAG" +#define PLAYBACK_CAPACITY 4 +#define MAX_EVENT_ID 64 + +#define WAIT_STEP_MS 50U +#define WAIT_LIMIT_MS 60000U + +typedef struct { + pthread_mutex_t mutex; + iot_client_t *iot; + tai_ctx_t *tai; + int mqtt_running; + int mqtt_failed; + int done; + int playback_running; + size_t queued_audio; + uint64_t audio_cutoff_ms; + uint64_t stream_start_ms; + uint64_t queued_start_ms[PLAYBACK_CAPACITY]; + char current_event_id[MAX_EVENT_ID]; + int current_event_has_audio; +} demo_state_t; + +#define MAX_AUDIO_TIME_MS ((UINT64_C(1) << 42) - 1) + +static uint64_t interrupt_time(const char *data, size_t len, int chat_break) +{ + if (!data || !len) return 0; + /* cJSON strings have no length: reject encoded NUL rather than silently + * treating a timestamp with a NUL suffix as a shorter decimal string. */ + for (size_t i = 0; i < len; i++) { + if (data[i] == '\0') return 0; + if (data[i] == '\\' && i + 1 < len) { + if (len - i >= 6 && memcmp(data + i, "\\u0000", 6) == 0) return 0; + i++; + } + } + const char *end = NULL; + cJSON *root = cJSON_ParseWithLengthOpts(data, len, &end, 0); + if (!root) return 0; + while (end < data + len && + (*end == ' ' || *end == '\t' || *end == '\r' || *end == '\n')) end++; + cJSON *object = chat_break + ? cJSON_GetObjectItemCaseSensitive(root, "breakAttributes") : root; + cJSON *time = cJSON_GetObjectItemCaseSensitive(object, "time"); + uint64_t result = 0; + if (end == data + len && cJSON_IsObject(object)) { + if (cJSON_IsString(time)) { + const char *p = time->valuestring; + for (; *p; p++) { + if (*p < '0' || *p > '9' || + result > (MAX_AUDIO_TIME_MS - (unsigned)(*p - '0')) / 10) { + result = 0; + break; + } + result = result * 10 + (unsigned)(*p - '0'); + } + } else if (cJSON_IsNumber(time) && time->valuedouble > 0 && + time->valuedouble <= (double)MAX_AUDIO_TIME_MS) { + uint64_t value = (uint64_t)time->valuedouble; + if ((double)value == time->valuedouble) result = value; + } + } + cJSON_Delete(root); + return result; +} + +static void interrupt_playback(demo_state_t *state, const char *data, + size_t len, int chat_break) +{ + uint64_t cutoff = interrupt_time(data, len, chat_break); + const char *source = chat_break ? "TAI ChatBreak" : "MQTT asrInterrupt"; + pthread_mutex_lock(&state->mutex); + if (cutoff) { + if (cutoff > state->audio_cutoff_ms) state->audio_cutoff_ms = cutoff; + } else if (state->stream_start_ms > state->audio_cutoff_ms) { + /* Fail closed. An interruption we cannot place on the server timeline + * must not leave the reply playing over the user, so drop the stream + * currently in flight: its own START becomes the cutoff (a START equal + * to the cutoff is discarded), and later streams are still admitted. */ + state->audio_cutoff_ms = state->stream_start_ms; + } + size_t kept = 0; + for (size_t i = 0; i < state->queued_audio; i++) { + if (state->queued_start_ms[i] > state->audio_cutoff_ms) + state->queued_start_ms[kept++] = state->queued_start_ms[i]; + } + state->queued_audio = kept; + pthread_mutex_unlock(&state->mutex); + if (cutoff) + printf("\n[%s] interruption time=%" PRIu64 "\n", source, cutoff); + else + printf("\n[%s] no usable interruption time; dropped the in-flight stream\n", + source); +} + +static void on_ai_control(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + if (strcmp(type, "asrInterrupt") != 0) return; + interrupt_playback((demo_state_t *)user_data, json_data, data_len, 0); +} + +static int on_flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + pthread_mutex_lock(&state->mutex); + int admit = state->queued_audio < PLAYBACK_CAPACITY; + pthread_mutex_unlock(&state->mutex); + return admit; +} + +static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + pthread_mutex_lock(&state->mutex); + if (msg->stream_flag == TAI_STREAM_START || + msg->stream_flag == TAI_STREAM_ONE_SHOT) + state->stream_start_ms = msg->timestamp_ms; + state->current_event_has_audio = 1; + if (msg->len && state->stream_start_ms > state->audio_cutoff_ms && + state->queued_audio < PLAYBACK_CAPACITY) + state->queued_start_ms[state->queued_audio++] = state->stream_start_ms; + pthread_mutex_unlock(&state->mutex); +} + +static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + if (msg->event_type == TAI_EVT_START && msg->event_id && msg->event_id[0]) { + pthread_mutex_lock(&state->mutex); + snprintf(state->current_event_id, sizeof(state->current_event_id), + "%s", msg->event_id); + state->current_event_has_audio = 0; + pthread_mutex_unlock(&state->mutex); + } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { + interrupt_playback(state, (const char *)msg->user_data, + msg->user_data_len, 1); + } else if (msg->event_type == TAI_EVT_END) { + pthread_mutex_lock(&state->mutex); + if (msg->event_id && state->current_event_id[0] && + strcmp(state->current_event_id, msg->event_id) == 0) { + state->current_event_id[0] = '\0'; + if (!state->current_event_has_audio || !state->audio_cutoff_ms || + state->stream_start_ms > state->audio_cutoff_ms) + state->done = 1; + } + pthread_mutex_unlock(&state->mutex); + } +} + +static void on_text(tai_ctx_t *ctx, const tai_text_msg_t *msg, void *user_data) +{ + (void)ctx; + (void)user_data; + fwrite(msg->text, 1, msg->len, stdout); + fflush(stdout); +} + +static void on_disconnect(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, + void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + fprintf(stderr, "\n[TAI disconnected: reason=%u detail=%u]\n", + (unsigned)msg->reason, (unsigned)msg->detail); + pthread_mutex_lock(&state->mutex); + state->done = 1; + pthread_mutex_unlock(&state->mutex); +} + +static void *mqtt_owner(void *arg) +{ + demo_state_t *state = (demo_state_t *)arg; + while (1) { + pthread_mutex_lock(&state->mutex); + int running = state->mqtt_running; + pthread_mutex_unlock(&state->mutex); + if (!running) break; + if (iot_client_process(state->iot, WAIT_STEP_MS) != OPRT_OK) { + pthread_mutex_lock(&state->mutex); + state->mqtt_failed = 1; + state->mqtt_running = 0; + pthread_mutex_unlock(&state->mutex); + break; + } + } + return NULL; +} + +static void *playback_consumer(void *arg) +{ + demo_state_t *state = (demo_state_t *)arg; + while (1) { + state->iot->pal->sleep_ms(100); + pthread_mutex_lock(&state->mutex); + if (!state->playback_running) { + pthread_mutex_unlock(&state->mutex); + break; + } + if (state->queued_audio > 0) { + state->queued_audio--; + memmove(state->queued_start_ms, state->queued_start_ms + 1, + state->queued_audio * sizeof(state->queued_start_ms[0])); + } + pthread_mutex_unlock(&state->mutex); + } + return NULL; +} + +int main(int argc, char **argv) +{ + const char *devid = argc > 1 ? argv[1] : DEFAULT_DEVID; + const char *secret_key = argc > 2 ? argv[2] : DEFAULT_SECRET_KEY; + const char *local_key = argc > 3 ? argv[3] : DEFAULT_LOCAL_KEY; + const pal_t *pal = tai_pal_posix(); + demo_state_t state; + memset(&state, 0, sizeof(state)); + pthread_mutex_init(&state.mutex, NULL); + + if (iot_init_default() != OPRT_OK) return 1; + iot_client_config_t iot_cfg = { + .region = AY, + .env = PROD, + .mqtt_disable_auto_connect = true, + }; + if (demo_copy_field(iot_cfg.devid, sizeof(iot_cfg.devid), devid, "devid") || + demo_copy_field(iot_cfg.secret_key, sizeof(iot_cfg.secret_key), + secret_key, "secret_key") || + demo_copy_field(iot_cfg.local_key, sizeof(iot_cfg.local_key), + local_key, "local_key")) { + return 1; + } + state.iot = iot_client_init(&iot_cfg); + if (!state.iot) return 1; + + char *token = (char *)calloc(1, 4096); + tai_conn_params_t params; + if (!token || iot_client_get_session_token(state.iot, NULL, token, 4096) || + parse_token(token, ¶ms)) { + fprintf(stderr, "failed to obtain/parse TAI session token\n"); + free(token); + iot_client_deinit(state.iot); + return 1; + } + if (params.biz_code == 0) params.biz_code = 65537; + if (params.biz_tag == 0) params.biz_tag = 119; + + iot_ai_ctrl_set_callback(state.iot, on_ai_control, &state); + if (iot_client_connect(state.iot) != OPRT_OK) { + fprintf(stderr, "MQTT connect failed\n"); + free(token); + iot_client_deinit(state.iot); + return 1; + } + + tai_config_t tai_cfg = { + .host = params.host, + .port = params.port, + .tls_sni = params.tls_sni, + .device_id = params.derived_client_id, + .local_key = local_key, + .client_type = TAI_CLIENT_DEVICE, + .protocol_version = TAI_VER_21, + .sign_level = TAI_SIGN_HMAC_SHA256, + .biz_code = (uint32_t)params.biz_code, + .biz_tag = (uint64_t)params.biz_tag, + .agent_token = params.agent_token, + .pal = pal, + .on_audio = on_audio, + .on_text = on_text, + .on_event = on_event, + .on_disconnect = on_disconnect, + .on_flow_control = on_flow_control, + .user_data = &state, + }; + void *tai_mem = pal->malloc(tai_ctx_size()); + state.tai = tai_mem ? tai_ctx_init(tai_mem, &tai_cfg) : NULL; + if (!state.tai || tai_connect(state.tai) != TAI_OK) { + fprintf(stderr, "TAI connect failed\n"); + if (state.tai) tai_ctx_deinit(state.tai); + pal->free(tai_mem); + free(token); + iot_client_deinit(state.iot); + return 1; + } + + pthread_t mqtt_thread; + pthread_t playback_thread; + state.mqtt_running = 1; + state.playback_running = 1; + pthread_create(&mqtt_thread, NULL, mqtt_owner, &state); + pthread_create(&playback_thread, NULL, playback_consumer, &state); + + const char *question = "Please give a detailed explanation of TCP backpressure."; + int send_rc = tai_send_text(state.tai, question, strlen(question)); + uint32_t waited = 0; + while (send_rc == TAI_OK && waited < WAIT_LIMIT_MS) { + pal->sleep_ms(WAIT_STEP_MS); + pthread_mutex_lock(&state.mutex); + int done = state.done || state.mqtt_failed; + pthread_mutex_unlock(&state.mutex); + if (done) break; + waited += WAIT_STEP_MS; + } + + pthread_mutex_lock(&state.mutex); + state.mqtt_running = 0; + state.playback_running = 0; + pthread_mutex_unlock(&state.mutex); + pthread_join(mqtt_thread, NULL); + pthread_join(playback_thread, NULL); + tai_disconnect(state.tai); + tai_ctx_deinit(state.tai); + pal->free(tai_mem); + free(token); + iot_client_deinit(state.iot); + pthread_mutex_destroy(&state.mutex); + return send_rc == TAI_OK && !state.mqtt_failed ? 0 : 1; +} diff --git a/modules/iot-client/CONTEXT.md b/modules/iot-client/CONTEXT.md index a18a2b4..f6ffe79 100644 --- a/modules/iot-client/CONTEXT.md +++ b/modules/iot-client/CONTEXT.md @@ -43,6 +43,13 @@ _Avoid_: publish (that is the transport verb), sync, send. A cloud→device message that sets DP values. _Avoid_: command, control, write. +**AI control notice**: +An authenticated MQTT protocol-9000 business message delivered through +`ai_ctrl_callback_t`, independently of the RTC TCP Connection. The IoT Client +validates and exposes the notice; the application owns playback interruption, +Event correlation, and any TAI action. +_Avoid_: DP set, TAI Event, injected callback. + **Activation** (on-boarding): First-time provisioning that authenticates the device and returns its credentials (devid / secret_key / local_key) together with its schema and schema id. diff --git a/modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md b/modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md new file mode 100644 index 0000000..d7091ce --- /dev/null +++ b/modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md @@ -0,0 +1,22 @@ +# MQTT AI control remains application-composed + +The IoT Client consumes authenticated MQTT protocol-9000 AI control notices and +passes their event type and data to an application callback. It does not depend +on the RTC TCP Client, inject TAI Events, stop playback, or send an automatic +acknowledgement. The application keeps one owner for MQTT process/publish and +combines this callback with RTC receive callbacks in its own synchronized state. + +We chose a separate MQTT control path because intentional RTC receive +backpressure stalls every Frame on that Connection, including a later ChatBreak. +Keeping the modules independent preserves their existing threading models and +lets an interrupt reach the application without waiting behind media bytes. + +## Consequences + +- The application must keep MQTT connected and call `iot_client_process()` while + a TAI Session is active. +- Callback data is borrowed and the callback must return promptly. +- Event correlation, duplicate suppression, queue flushing, and rejection of + stale media are application responsibilities. +- A server notice does not imply calling `tai_chat_break()` or ending a + server-VAD uplink. diff --git a/modules/iot-client/include/iot_client.h b/modules/iot-client/include/iot_client.h index ef2daa1..7a818c3 100644 --- a/modules/iot-client/include/iot_client.h +++ b/modules/iot-client/include/iot_client.h @@ -111,6 +111,27 @@ IOT_API int iot_init(const pal_t *pal); typedef void (*iot_message_callback_t)(const char *topic, size_t topic_len, const uint8_t *data, size_t data_len); +/** + * @brief Callback for authenticated AI control messages delivered over MQTT. + * + * Fires on the thread calling iot_client_process(). The type and JSON data are + * borrowed and remain valid only for the callback duration. Keep the callback + * non-blocking and copy anything retained after it returns. + * + * A protocol-9000 notice this layer cannot scope -- a non-object data or + * data.data, a non-string type, or a re-serialization allocation failure -- + * is left unconsumed and continues to the raw message callback. + * + * @param type Control event type, such as "asrInterrupt". + * @param json_data Serialized event-specific data; not necessarily an object. + * Re-serialized from the parsed notice with cJSON (unformatted), + * so its byte layout can differ from the original wire bytes. + * @param data_len Length of @p json_data, excluding any terminating NUL. + * @param user_data Opaque pointer supplied at registration. + */ +typedef void (*ai_ctrl_callback_t)(const char *type, const char *json_data, + size_t data_len, void *user_data); + /** * @brief Reset type classification (mirrors TuyaOpen TUYA_RESET_TYPE_REMOTE_*). * @@ -273,6 +294,8 @@ struct iot_dp_context; void *reset_user_data; // Opaque pointer passed back to reset_callback iot_ota_confirm_callback_t ota_confirm_callback; // APP-confirmed OTA (protocol 15) callback void *ota_confirm_user_data; // Opaque pointer passed back to ota_confirm_callback + ai_ctrl_callback_t ai_ctrl_callback; // AI control (protocol 9000) callback + void *ai_ctrl_user_data; // Opaque pointer passed back to ai_ctrl_callback struct iot_dp_context *dp; // DP layer state; points into dp_storage, NULL when inactive void *dp_storage[IOT_DP_CONTEXT_STORAGE / sizeof(void *)]; // inline storage for *dp (no heap) @@ -439,6 +462,23 @@ IOT_API int iot_client_process(iot_client_t *client, uint32_t timeout_ms); */ IOT_API int iot_client_publish(iot_client_t *client, const uint8_t *data, size_t data_len); +/** + * @brief Register the MQTT protocol-9000 AI control callback. + * + * Register before the application starts pumping MQTT messages. If commands + * must be observable during initial MQTT subscription, initialize with + * mqtt_disable_auto_connect=true, register here, then call iot_client_connect(). + * Passing NULL as @p callback deregisters the current callback. + * + * @param client IoT client instance. + * @param callback Callback, or NULL to deregister. + * @param user_data Opaque pointer passed to @p callback. + * @return OPRT_OK, or OPRT_INVALID_PARAMETER if @p client is NULL. + */ +IOT_API int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, + void *user_data); + /** * @brief Get AI agent session token from Tuya cloud. * diff --git a/modules/iot-client/src/iot_ai_ctrl.c b/modules/iot-client/src/iot_ai_ctrl.c new file mode 100644 index 0000000..cfcb442 --- /dev/null +++ b/modules/iot-client/src/iot_ai_ctrl.c @@ -0,0 +1,70 @@ +#include "iot_ai_ctrl.h" + +#include + +#include "cJSON.h" + +#define IOT_PROTO_AI_CONTROL 9000 + +bool iot_ai_ctrl_dispatch(iot_client_t *client, + const uint8_t *bytes, size_t len) +{ + if (!client || !client->ai_ctrl_callback || !bytes || len == 0) { + return false; + } + + cJSON *root = cJSON_ParseWithLength((const char *)bytes, len); + if (!root) return false; + + cJSON *protocol = cJSON_GetObjectItem(root, "protocol"); + if (!cJSON_IsNumber(protocol) || + protocol->valuedouble != (double)IOT_PROTO_AI_CONTROL) { + cJSON_Delete(root); + return false; + } + + cJSON *data = cJSON_GetObjectItem(root, "data"); + cJSON *inner = cJSON_IsObject(data) + ? cJSON_GetObjectItem(data, "data") : NULL; + cJSON *type = cJSON_IsObject(inner) + ? cJSON_GetObjectItem(inner, "type") : NULL; + cJSON *payload = cJSON_IsObject(inner) + ? cJSON_GetObjectItem(inner, "data") : NULL; + if (!cJSON_IsString(type)) { + cJSON_Delete(root); + return false; + } + + char *payload_json = NULL; + const char *payload_bytes = ""; + size_t payload_len = 0; + if (payload) { + payload_json = cJSON_PrintUnformatted(payload); + if (!payload_json) { + /* Do not turn a scoped notice into an unscoped empty payload under + * allocation pressure. Leave the authenticated message unconsumed + * so the application can apply its own fallback policy. */ + cJSON_Delete(root); + return false; + } + payload_bytes = payload_json; + payload_len = strlen(payload_json); + } + + client->ai_ctrl_callback(type->valuestring, payload_bytes, payload_len, + client->ai_ctrl_user_data); + + if (payload_json) client->pal->free(payload_json); + cJSON_Delete(root); + return true; +} + +IOT_API int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, void *user_data) +{ + if (!client) return OPRT_INVALID_PARAMETER; + + client->ai_ctrl_callback = callback; + client->ai_ctrl_user_data = user_data; + return OPRT_OK; +} diff --git a/modules/iot-client/src/iot_ai_ctrl.h b/modules/iot-client/src/iot_ai_ctrl.h new file mode 100644 index 0000000..b037e01 --- /dev/null +++ b/modules/iot-client/src/iot_ai_ctrl.h @@ -0,0 +1,13 @@ +#ifndef IOT_AI_CTRL_H +#define IOT_AI_CTRL_H + +#include +#include +#include + +#include "iot_client.h" + +bool iot_ai_ctrl_dispatch(iot_client_t *client, + const uint8_t *bytes, size_t len); + +#endif /* IOT_AI_CTRL_H */ diff --git a/modules/iot-client/src/iot_client_message.c b/modules/iot-client/src/iot_client_message.c index 453040a..7ca4144 100644 --- a/modules/iot-client/src/iot_client_message.c +++ b/modules/iot-client/src/iot_client_message.c @@ -3,6 +3,7 @@ #include "cipher_wrapper.h" #include "iot_internal.h" #include "iot_dp_internal.h" +#include "iot_ai_ctrl.h" #include "cJSON.h" #include @@ -33,6 +34,8 @@ static void mqtt_message_handler(const char *topic, size_t topic_len, /* consumed: reset notices never reach the DP layer or raw callback */ } else if (iot_client_message_handle_ota_confirm(client, decrypted, decrypted_len)) { /* consumed: APP-confirmed OTA notices never reach the DP/raw path */ + } else if (iot_ai_ctrl_dispatch(client, decrypted, decrypted_len)) { + /* consumed: AI control notices never reach the DP/raw path */ } else if (!iot_dp_dispatch_downlink(client, topic, topic_len, decrypted, decrypted_len) && client->message_callback) { client->message_callback(topic, topic_len, decrypted, decrypted_len); diff --git a/modules/iot-client/src/iot_internal.h b/modules/iot-client/src/iot_internal.h index 92538fc..89714bc 100644 --- a/modules/iot-client/src/iot_internal.h +++ b/modules/iot-client/src/iot_internal.h @@ -39,7 +39,8 @@ #ifndef IOT_SDK_BV #define IOT_SDK_BV "2.0" #endif -#define SDK_VERSION "agentic-kit_0.5.0" + +#define SDK_VERSION "agentic-kit_0.5.0-dev" /* ========================================================================= * iot-client: service endpoints & protocol constants (fixed by the Tuya diff --git a/modules/iot-client/test/iot_ai_ctrl_test.c b/modules/iot-client/test/iot_ai_ctrl_test.c new file mode 100644 index 0000000..13dda9e --- /dev/null +++ b/modules/iot-client/test/iot_ai_ctrl_test.c @@ -0,0 +1,351 @@ +/* + * AI control channel (MQTT protocol 9000) dispatch tests. + * + * Network-free: exercises iot_ai_ctrl_dispatch() and + * iot_ai_ctrl_set_callback() directly. + */ + +#include +#include +#include + +#include "iot_ai_ctrl.h" +#include "iot_client.h" +#include "iot_internal.h" +#include "cJSON.h" + +static int tests_run; +static int tests_passed; + +#define RUN_TEST(fn) \ + do { \ + tests_run++; \ + printf("\n--- [%d] %s ---\n", tests_run, #fn); \ + if ((fn)() == 0) { \ + tests_passed++; \ + printf(" PASS\n"); \ + } else { \ + printf(" FAIL\n"); \ + } \ + } while (0) + +static int callback_count; +static char callback_type[64]; +static char callback_data[512]; +static void *callback_user_data; + +static void reset_callback(void) +{ + callback_count = 0; + callback_type[0] = '\0'; + callback_data[0] = '\0'; + callback_user_data = NULL; +} + +static void control_callback(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + callback_count++; + callback_user_data = user_data; + + if (type) { + snprintf(callback_type, sizeof(callback_type), "%s", type); + } + if (json_data && data_len > 0) { + size_t n = data_len < sizeof(callback_data) - 1 + ? data_len : sizeof(callback_data) - 1; + memcpy(callback_data, json_data, n); + callback_data[n] = '\0'; + } +} + +static iot_client_t *make_client(const pal_t *pal) +{ + iot_client_t *client = (iot_client_t *)pal->malloc(sizeof(*client)); + if (!client) return NULL; + memset(client, 0, sizeof(*client)); + client->pal = pal; + return client; +} + +static void destroy_client(iot_client_t *client) +{ + if (client) client->pal->free(client); +} + +static int test_asr_interrupt_dispatch(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + static char user_data; + const char *envelope = + "{\"protocol\":9000,\"t\":1234567890,\"data\":{" + "\"bizType\":\"EVENT\",\"bizId\":\"uuid-1234\",\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{" + "\"eventId\":\"evt-5678\",\"time\":\"2025-01-15T10:30:00Z\"}}}}"; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, &user_data); + + if (!iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope))) { + printf(" protocol-9000 envelope was not consumed\n"); + } else if (callback_count != 1) { + printf(" callback count=%d\n", callback_count); + } else if (strcmp(callback_type, "asrInterrupt") != 0) { + printf(" callback type=%s\n", callback_type); + } else if (!strstr(callback_data, "\"eventId\":\"evt-5678\"")) { + printf(" callback data=%s\n", callback_data); + } else if (callback_user_data != &user_data) { + printf(" callback user data mismatch\n"); + } else { + result = 0; + } + + destroy_client(client); + return result; +} + +static int test_non_9000_passthrough(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + static const char protocol_five[] = + "{\"protocol\":5,\"data\":{\"dps\":{\"1\":true}}}"; + static const char no_protocol[] = "{\"type\":\"test\"}"; + static const char garbage[] = "not json"; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + + if (iot_ai_ctrl_dispatch(client, (const uint8_t *)protocol_five, + sizeof(protocol_five) - 1) || + iot_ai_ctrl_dispatch(client, (const uint8_t *)no_protocol, + sizeof(no_protocol) - 1) || + iot_ai_ctrl_dispatch(client, (const uint8_t *)garbage, + sizeof(garbage) - 1)) { + printf(" non-control input was consumed\n"); + } else if (callback_count != 0) { + printf(" callback fired on passthrough\n"); + } else { + result = 0; + } + + destroy_client(client); + return result; +} + +static int test_no_callback_passthrough(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + int consumed; + + if (!client) return -1; + consumed = iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)); + destroy_client(client); + return consumed ? -1 : 0; +} + +static int test_missing_type_not_consumed(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"data\":{\"key\":\"value\"}}}}"; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + if (!iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)) && callback_count == 0) { + result = 0; + } + destroy_client(client); + return result; +} + +static int test_exact_length_without_nul(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + static const char json[] = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":null}}}"; + size_t len = sizeof(json) - 1; + uint8_t *bytes; + int result = -1; + + if (!client) return -1; + bytes = (uint8_t *)pal->malloc(len); + if (!bytes) { + destroy_client(client); + return -1; + } + memcpy(bytes, json, len); + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + + if (iot_ai_ctrl_dispatch(client, bytes, len) && callback_count == 1 && + strcmp(callback_type, "asrInterrupt") == 0 && + strcmp(callback_data, "null") == 0) { + result = 0; + } + + pal->free(bytes); + destroy_client(client); + return result; +} + +static int test_null_safety(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + int failed; + + if (!client) return -1; + iot_ai_ctrl_set_callback(client, control_callback, NULL); + failed = iot_ai_ctrl_dispatch(NULL, (const uint8_t *)"x", 1) || + iot_ai_ctrl_dispatch(client, NULL, 0); + destroy_client(client); + return failed ? -1 : 0; +} + +static int test_deregister(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + int consumed; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + iot_ai_ctrl_set_callback(client, NULL, NULL); + consumed = iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)); + destroy_client(client); + return consumed || callback_count != 0 ? -1 : 0; +} + +static int test_set_callback_null_client(void) +{ + return iot_ai_ctrl_set_callback(NULL, control_callback, NULL) == + OPRT_INVALID_PARAMETER ? 0 : -1; +} + +static int test_protocol_must_be_exact_integer(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *fractional = + "{\"protocol\":9000.5,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + const char *string_value = + "{\"protocol\":\"9000\",\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + int failed; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + failed = iot_ai_ctrl_dispatch(client, (const uint8_t *)fractional, + strlen(fractional)) || + iot_ai_ctrl_dispatch(client, (const uint8_t *)string_value, + strlen(string_value)) || + callback_count != 0; + destroy_client(client); + return failed ? -1 : 0; +} + +static int cjson_allocation_count; +static int cjson_fail_after; + +static void *counting_malloc(size_t size) +{ + cjson_allocation_count++; + if (cjson_fail_after >= 0 && cjson_allocation_count > cjson_fail_after) + return NULL; + return malloc(size); +} + +static int test_payload_print_allocation_failure_is_unconsumed(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{\"eventId\":\"evt-1\"}}}}"; + cJSON_Hooks counting_hooks = { counting_malloc, pal->free }; + cJSON_Hooks restore_hooks = { pal->malloc, pal->free }; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + + cJSON_InitHooks(&counting_hooks); + cjson_allocation_count = 0; + cjson_fail_after = -1; + cJSON *root = cJSON_ParseWithLength(envelope, strlen(envelope)); + if (root) cJSON_Delete(root); + if (cjson_allocation_count <= 0) goto out; + + /* Sweep one failing allocation index at a time. The successful run proves + * every earlier run failed before the callback; no scoped payload was + * rewritten to an empty string. */ + result = -1; + for (cjson_fail_after = 0; cjson_fail_after < 32; cjson_fail_after++) { + cjson_allocation_count = 0; + int consumed = iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)); + if (consumed) { + result = callback_count == 1 ? 0 : -1; + break; + } + if (callback_count != 0) break; + } + +out: + cJSON_InitHooks(&restore_hooks); + destroy_client(client); + return result; +} + +int main(void) +{ + const pal_t *pal = get_default_pal(); + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + printf("========== IoT AI Control Channel Test Suite ==========\n"); + + if (iot_init(pal) != OPRT_OK) return 1; + + RUN_TEST(test_asr_interrupt_dispatch); + RUN_TEST(test_non_9000_passthrough); + RUN_TEST(test_no_callback_passthrough); + RUN_TEST(test_missing_type_not_consumed); + RUN_TEST(test_exact_length_without_nul); + RUN_TEST(test_null_safety); + RUN_TEST(test_deregister); + RUN_TEST(test_set_callback_null_client); + RUN_TEST(test_protocol_must_be_exact_integer); + RUN_TEST(test_payload_print_allocation_failure_is_unconsumed); + + printf("\n========== Results: %d/%d passed ==========\n", + tests_passed, tests_run); + return tests_passed == tests_run ? 0 : 1; +} diff --git a/modules/iot-client/test/iot_client_message_test.c b/modules/iot-client/test/iot_client_message_test.c index 89f816d..72ac705 100644 --- a/modules/iot-client/test/iot_client_message_test.c +++ b/modules/iot-client/test/iot_client_message_test.c @@ -109,6 +109,29 @@ static volatile int ota_confirm_cb_called = 0; static int ota_confirm_cb_channel = -1; static void *ota_confirm_cb_user_data = NULL; +static volatile int ai_ctrl_cb_called = 0; +static char ai_ctrl_cb_type[64] = {0}; +static char ai_ctrl_cb_data[256] = {0}; + +static void test_ai_ctrl_callback(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + (void)user_data; + ai_ctrl_cb_called++; + snprintf(ai_ctrl_cb_type, sizeof(ai_ctrl_cb_type), "%s", type ? type : ""); + size_t n = data_len < sizeof(ai_ctrl_cb_data) - 1 + ? data_len : sizeof(ai_ctrl_cb_data) - 1; + if (json_data && n > 0) memcpy(ai_ctrl_cb_data, json_data, n); + ai_ctrl_cb_data[n] = '\0'; +} + +static void reset_ai_ctrl_callback_state(void) +{ + ai_ctrl_cb_called = 0; + ai_ctrl_cb_type[0] = '\0'; + ai_ctrl_cb_data[0] = '\0'; +} + static void test_ota_confirm_callback(int channel, void *user_data) { ota_confirm_cb_called++; @@ -1067,6 +1090,61 @@ static int test_ota_confirm_non_matching_payloads_passthrough(void) return rc; } +/* ---------- AI control dispatch on the decrypted MQTT receive path ---------- */ + +static int test_encrypted_ai_control_message(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = (iot_client_t *)pal->malloc(sizeof(*client)); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{\"eventId\":\"evt-mqtt\"}}}}"; + int result = -1; + + if (!client) return -1; + memset(client, 0, sizeof(*client)); + client->pal = pal; + strncpy(client->devid, TEST_DEVID, sizeof(client->devid) - 1); + strncpy(client->secret_key, TEST_SECRET_KEY, sizeof(client->secret_key) - 1); + strncpy(client->local_key, TEST_LOCAL_KEY, sizeof(client->local_key) - 1); + snprintf(client->mqtt_url, sizeof(client->mqtt_url), "%s", TEST_MQTT_URL); + client->cacert = g_cacert; + client->message_callback = test_message_callback; + iot_ai_ctrl_set_callback(client, test_ai_ctrl_callback, NULL); + reset_callback_state(); + reset_ai_ctrl_callback_state(); + + if (iot_client_message_connect(client) != OPRT_OK) goto out; + (void)wait_for_callback(client, 20); + reset_callback_state(); + + if (iot_client_message_publish(client, (const uint8_t *)envelope, + strlen(envelope)) != OPRT_OK) { + goto out; + } + for (int i = 0; i < 30 && ai_ctrl_cb_called == 0; i++) { + if (iot_client_message_process(client, 50) != OPRT_OK) goto out; + } + + if (ai_ctrl_cb_called != 1) { + printf(" AI control callback count=%d\n", ai_ctrl_cb_called); + } else if (strcmp(ai_ctrl_cb_type, "asrInterrupt") != 0) { + printf(" AI control type=%s\n", ai_ctrl_cb_type); + } else if (!strstr(ai_ctrl_cb_data, "\"eventId\":\"evt-mqtt\"")) { + printf(" AI control data=%s\n", ai_ctrl_cb_data); + } else if (cb_called != 0) { + printf(" raw callback received consumed control message\n"); + } else { + result = 0; + } + +out: + iot_client_message_disconnect(client); + client->cacert = NULL; + pal->free(client); + return result; +} + /* ---------- main ---------- */ int main(void) @@ -1103,6 +1181,7 @@ int main(void) /* Success tests */ RUN_TEST(test_raw_message); RUN_TEST(test_encrypted_message); + RUN_TEST(test_encrypted_ai_control_message); /* Failure tests */ RUN_TEST(test_invalid_format_message); diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c new file mode 100644 index 0000000..5ed15ef --- /dev/null +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -0,0 +1,514 @@ +/* + * Cross-module regression: MQTT control remains deliverable while TAI receive + * backpressure stops the media Connection, then stale media drains without + * refilling playback and a new Event remains playable. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iot_client.h" +#include "iot_client_message.h" +#include "tai_internal.h" +#include "tai_pal_loopback.h" + +/* Include the demo whole (its main renamed away) so the regression drives its + * actual static callbacks. mqtt_interrupt_demo_test.c includes the same demo + * in this one binary -- every file-scope symbol in it must stay static, or + * the two copies collide at link time. */ +#define main combined_demo_main +#include "../../../examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c" +#undef main + +#define TEST_DEVID "test_device_msg_001" +#define TEST_SECRET_KEY "abcdef1234567890" +#define TEST_LOCAL_KEY "0123456789abcdef" +#define TEST_MQTT_URL "mqtts://127.0.0.1:11885" +#define TEST_MQTT_PORT 11885 +#define WRONG_KEY_PORT 11887 + +#define EARLY_AUDIO_MS UINT64_C(1700000000000) +#define OLD_AUDIO_MS UINT64_C(1700000002000) +#define NEW_AUDIO_MS UINT64_C(1700000004000) + +extern const pal_t *tai_pal_posix(void); +int mqtt_interrupt_demo_tests(void); + +static int failures; +static const pal_t *posix_pal; +static const pal_t *loopback_pal; +static pal_t routing_pal; + +static int is_loopback_handle(void *tcp) +{ + return tcp == (void *)0x1; +} + +static void *routing_tcp_connect(const char *host, uint16_t port, + uint32_t timeout_ms) +{ + if (host && strcmp(host, "loopback.test") == 0) { + return loopback_pal->tcp_connect(host, port, timeout_ms); + } + return posix_pal->tcp_connect(host, port, timeout_ms); +} + +static int routing_tcp_send(void *tcp, const uint8_t *buf, size_t len, + uint32_t timeout_ms) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + return owner->tcp_send(tcp, buf, len, timeout_ms); +} + +static int routing_tcp_recv(void *tcp, uint8_t *buf, size_t len, + uint32_t timeout_ms) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + return owner->tcp_recv(tcp, buf, len, timeout_ms); +} + +static void routing_tcp_close(void *tcp) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + owner->tcp_close(tcp); +} + +static int routing_tcp_poll(void *tcp, int events, uint32_t timeout_ms) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + return owner->tcp_poll(tcp, events, timeout_ms); +} + +static const pal_t *make_routing_pal(void) +{ + posix_pal = tai_pal_posix(); + loopback_pal = tai_pal_loopback(); + routing_pal = *posix_pal; + routing_pal.tcp_connect = routing_tcp_connect; + routing_pal.tcp_send = routing_tcp_send; + routing_pal.tcp_recv = routing_tcp_recv; + routing_pal.tcp_close = routing_tcp_close; + routing_pal.tcp_poll = routing_tcp_poll; + return &routing_pal; +} + + +#define CHECK(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #expr); \ + failures++; \ + } \ + } while (0) + +static void sleep_ms(uint32_t ms) +{ + struct timespec ts = { ms / 1000U, (long)(ms % 1000U) * 1000000L }; + nanosleep(&ts, NULL); +} + +typedef struct { + /* This mutex serializes counters and wrappers; callbacks also take playback.mutex. */ + pthread_mutex_t mutex; + demo_state_t playback; + int audio_callbacks; + int accepted_old_audio; + int accepted_new_audio; + int accepted_early_audio; + int chat_breaks; + int mqtt_interrupts; + int interrupted_before_tcp_drain; + pthread_t mqtt_callback_thread; +} app_state_t; + +static app_state_t state; +static pid_t mqtt_mock_pid = -1; +static pid_t wrong_key_mock_pid = -1; + +static char *load_file(const pal_t *pal, const char *path) +{ + FILE *file = fopen(path, "rb"); + if (!file) return NULL; + fseek(file, 0, SEEK_END); + long len = ftell(file); + fseek(file, 0, SEEK_SET); + char *data = (char *)pal->malloc((size_t)len + 1); + if (data && fread(data, 1, (size_t)len, file) == (size_t)len) { + data[len] = '\0'; + } else { + pal->free(data); + data = NULL; + } + fclose(file); + return data; +} + +static int wait_for_port(uint16_t port) +{ + for (int attempt = 0; attempt < 100; attempt++) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + struct sockaddr_in address = {0}; + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + int result = connect(fd, (struct sockaddr *)&address, sizeof(address)); + close(fd); + if (result == 0) return 0; + sleep_ms(20); + } + return -1; +} + +static int start_mqtt_mock(void) +{ + mqtt_mock_pid = fork(); + if (mqtt_mock_pid == 0) { + execlp(PYTHON3_EXEC, PYTHON3_EXEC, MESSAGE_MOCK_PATH, NULL); + _exit(1); + } + if (mqtt_mock_pid < 0) return -1; + + wrong_key_mock_pid = fork(); + if (wrong_key_mock_pid == 0) { + setenv("MESSAGE_MOCK_TYPE", "wrong_key_encrypted", 1); + setenv("MESSAGE_MOCK_PORT", "11887", 1); + execlp(PYTHON3_EXEC, PYTHON3_EXEC, MESSAGE_MOCK_PATH, NULL); + _exit(1); + } + if (wrong_key_mock_pid < 0) return -1; + return wait_for_port(TEST_MQTT_PORT) || wait_for_port(WRONG_KEY_PORT); +} + +static void stop_mqtt_mock(void) +{ + if (wrong_key_mock_pid > 0) { + kill(wrong_key_mock_pid, SIGTERM); + waitpid(wrong_key_mock_pid, NULL, 0); + wrong_key_mock_pid = -1; + } + if (mqtt_mock_pid > 0) { + kill(mqtt_mock_pid, SIGTERM); + waitpid(mqtt_mock_pid, NULL, 0); + mqtt_mock_pid = -1; + } +} + +#define WAIT_FOR(expr, timeout_ms) \ + ({ \ + int found = 0; \ + for (uint32_t waited = 0; waited < (timeout_ms); waited += 5) { \ + pthread_mutex_lock(&state.mutex); \ + found = (expr); \ + pthread_mutex_unlock(&state.mutex); \ + if (found) break; \ + sleep_ms(5); \ + } \ + found; \ + }) + +#define STATE_CHECK(expr) \ + do { \ + pthread_mutex_lock(&state.mutex); \ + CHECK(expr); \ + pthread_mutex_unlock(&state.mutex); \ + } while (0) + +static int flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)ctx; + app_state_t *s = (app_state_t *)user_data; + pthread_mutex_lock(&s->mutex); + /* One queued frame is enough to pressure the real worker in this test. */ + int admit = s->playback.queued_audio == 0; + pthread_mutex_unlock(&s->mutex); + return admit; +} + +static void combined_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, + void *user_data) +{ + app_state_t *s = (app_state_t *)user_data; + pthread_mutex_lock(&s->mutex); + s->audio_callbacks++; + size_t before = s->playback.queued_audio; + on_audio(ctx, msg, &s->playback); + if (s->playback.queued_audio > before) { + if (msg->event_id && strcmp(msg->event_id, "old-event") == 0) { + s->accepted_old_audio++; + } else if (msg->event_id && strcmp(msg->event_id, "new-event") == 0) { + s->accepted_new_audio++; + } else if (msg->event_id && strcmp(msg->event_id, "early-event") == 0) { + s->accepted_early_audio++; + } + } + pthread_mutex_unlock(&s->mutex); +} + +static void combined_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, + void *user_data) +{ + app_state_t *s = (app_state_t *)user_data; + pthread_mutex_lock(&s->mutex); + if (msg->event_type == TAI_EVT_CHAT_BREAK) s->chat_breaks++; + on_event(ctx, msg, &s->playback); + pthread_mutex_unlock(&s->mutex); +} + +static void combined_control(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + app_state_t *s = (app_state_t *)user_data; + if (strcmp(type, "asrInterrupt") != 0) return; + pthread_mutex_lock(&s->mutex); + s->mqtt_callback_thread = pthread_self(); + s->mqtt_interrupts++; + if (s->mqtt_interrupts == 2 && s->chat_breaks == 0) + s->interrupted_before_tcp_drain = 1; + on_ai_control(type, json_data, data_len, &s->playback); + pthread_mutex_unlock(&s->mutex); +} + +static int server_push(tai_ctx_t *ctx, uint8_t packet_type, + const tai_attr_t *attrs, int attr_count, + const uint8_t *payload, size_t payload_len, + uint16_t sequence) +{ + uint8_t packet[512]; + uint8_t frame[600]; + int packet_len = tai_packet_encode(TAI_VER_21, packet_type, + attrs, attr_count, + payload, payload_len, + packet, sizeof(packet)); + if (packet_len <= 0) return packet_len; + int frame_len = tai_frame_encode(TAI_FRAG_NONE, sequence, + packet, (size_t)packet_len, + ctx->sign_key, 32, ctx->pal, + frame, sizeof(frame)); + if (frame_len > 0) tai_loopback_push_recv(frame, (size_t)frame_len); + return frame_len; +} + +static int server_push_event(tai_ctx_t *ctx, uint16_t event_type, + const char *event_id, uint16_t sequence) +{ + tai_attr_t attrs[3] = { + tai_attr_strv(TAI_ATTR_SESSION_ID, ctx->session_id), + tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + tai_attr_strv(TAI_ATTR_USER_DATA, + "{\"breakAttributes\":{\"time\":\"1700000003000\"}}"), + }; + uint8_t payload[4]; + int payload_len = tai_pack_event(TAI_VER_21, event_type, + NULL, 0, payload, sizeof(payload)); + if (payload_len <= 0) return payload_len; + return server_push(ctx, TAI_PKT_EVENT, attrs, + event_type == TAI_EVT_CHAT_BREAK ? 3 : 2, + payload, (size_t)payload_len, sequence); +} + +static int server_push_audio(tai_ctx_t *ctx, const char *event_id, + uint64_t timestamp_ms, uint8_t fill, uint16_t sequence) +{ + uint8_t payload[48]; + int header_len = tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, + timestamp_ms, + payload, sizeof(payload)); + if (header_len <= 0) return header_len; + memset(payload + header_len, fill, 40); + + tai_attr_t attrs[2] = { + tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, + "111 1 16 16000 0 16000 20 40"), + }; + return server_push(ctx, TAI_PKT_AUDIO, attrs, 2, + payload, (size_t)header_len + 40, sequence); +} + +int main(void) +{ + const pal_t *pal = make_routing_pal(); + pthread_mutex_init(&state.mutex, NULL); + pthread_mutex_init(&state.playback.mutex, NULL); + CHECK(iot_init(pal) == OPRT_OK); + failures += mqtt_interrupt_demo_tests(); + CHECK(start_mqtt_mock() == 0); + + char *cacert = load_file(pal, TEST_CONFIG_DIR "/root_cert.pem"); + CHECK(cacert != NULL); + iot_client_t iot; + memset(&iot, 0, sizeof(iot)); + iot.pal = pal; + snprintf(iot.devid, sizeof(iot.devid), "%s", TEST_DEVID); + snprintf(iot.secret_key, sizeof(iot.secret_key), "%s", TEST_SECRET_KEY); + snprintf(iot.local_key, sizeof(iot.local_key), "%s", TEST_LOCAL_KEY); + snprintf(iot.mqtt_url, sizeof(iot.mqtt_url), "%s", TEST_MQTT_URL); + iot.cacert = cacert; + CHECK(iot_ai_ctrl_set_callback(&iot, combined_control, &state) == OPRT_OK); + CHECK(iot_client_message_connect(&iot) == OPRT_OK); + /* Consume the mock's initial raw message before the combined scenario. */ + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + + /* An authenticated-looking payload encrypted with another local key must + * never reach the control callback. */ + iot_client_t wrong_key_iot = iot; + wrong_key_iot.mqtt = NULL; + snprintf(wrong_key_iot.mqtt_url, sizeof(wrong_key_iot.mqtt_url), + "mqtts://127.0.0.1:%u", WRONG_KEY_PORT); + CHECK(iot_client_message_connect(&wrong_key_iot) == OPRT_OK); + CHECK(iot_client_message_process(&wrong_key_iot, 50) == OPRT_OK); + STATE_CHECK(state.mqtt_interrupts == 0); + iot_client_message_disconnect(&wrong_key_iot); + + tai_loopback_reset(); + tai_loopback_set_local_key("test-local-key-16"); + + tai_config_t config = { + .host = "loopback.test", + .port = 443, + .device_id = "test-device", + .local_key = "test-local-key-16", + .protocol_version = TAI_VER_21, + .client_type = TAI_CLIENT_DEVICE, + .sign_level = TAI_SIGN_HMAC_SHA256, + .disable_tls = 1, + .pal = pal, + .on_audio = combined_audio, + .on_event = combined_event, + .on_flow_control = flow_control, + .user_data = &state, + }; + static uint8_t context_memory[sizeof(struct tai_ctx)]; + tai_ctx_t *ctx = tai_ctx_init(context_memory, &config); + CHECK(ctx != NULL); + if (!ctx) return 1; + CHECK(tai_connect(ctx) == TAI_OK); + + uint8_t sent[2048]; + (void)tai_loopback_pop_sent(sent, sizeof(sent)); + uint16_t sequence = 10; + static const char early_control[] = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\"," + "\"data\":{\"time\":\"1700000001000\"}}}}"; + CHECK(iot_client_message_publish(&iot, (const uint8_t *)early_control, + sizeof(early_control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 1, 5); attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + STATE_CHECK(state.mqtt_interrupts == 1); + STATE_CHECK(state.playback.queued_audio == 0); + /* MQTT-before-audio ordering: early-event is cancelled before its START; + * its later audio must drain without entering playback. */ + CHECK(server_push_event(ctx, TAI_EVT_START, "early-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "early-event", EARLY_AUDIO_MS, 0x01, sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_END, "early-event", sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_START, "old-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "old-event", OLD_AUDIO_MS, 0x02, sequence++) > 0); + CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); + STATE_CHECK(state.accepted_early_audio == 0); + STATE_CHECK(state.playback.queued_audio == 1); + + pthread_mutex_lock(&state.mutex); + state.playback.queued_audio = 0; + pthread_mutex_unlock(&state.mutex); + CHECK(server_push_audio(ctx, "old-event", OLD_AUDIO_MS, 0x11, sequence++) > 0); + CHECK(WAIT_FOR(state.accepted_old_audio == 2, 1000)); + + CHECK(server_push_audio(ctx, "old-event", OLD_AUDIO_MS, 0x22, sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, + "old-event", sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_END, "old-event", sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_START, "new-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "new-event", NEW_AUDIO_MS, 0x33, sequence++) > 0); + /* Keep admission closed while the delayed duplicate waits on TCP. */ + CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, + "old-event", sequence++) > 0); + sleep_ms(100); + STATE_CHECK(state.audio_callbacks == 3); + STATE_CHECK(state.chat_breaks == 0); + + static const char control[] = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\"," + "\"data\":{\"time\":\"1700000003000\"}}}}"; + pthread_t mqtt_owner = pthread_self(); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 2, 5); attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + STATE_CHECK(state.mqtt_interrupts == 2); + pthread_mutex_lock(&state.mutex); + int same_thread = pthread_equal(mqtt_owner, state.mqtt_callback_thread); + pthread_mutex_unlock(&state.mutex); + CHECK(same_thread); + STATE_CHECK(state.interrupted_before_tcp_drain == 1); + + CHECK(WAIT_FOR(state.accepted_new_audio == 1, 2000)); + STATE_CHECK(state.audio_callbacks == 5); + STATE_CHECK(state.accepted_old_audio == 2); + STATE_CHECK(state.accepted_new_audio == 1); + STATE_CHECK(state.chat_breaks == 1); + STATE_CHECK(state.playback.queued_audio == 1); + + /* Release pressure by consuming new audio; the queued duplicate is ignored. */ + pthread_mutex_lock(&state.mutex); + state.playback.queued_audio = 0; + pthread_mutex_unlock(&state.mutex); + CHECK(WAIT_FOR(state.chat_breaks == 2, 2000)); + STATE_CHECK(state.playback.queued_audio == 0); + STATE_CHECK(state.accepted_new_audio == 1); + + /* Delayed duplicate on MQTT must also not flush the new playback. */ + pthread_mutex_lock(&state.mutex); + state.playback.queued_audio = 1; + pthread_mutex_unlock(&state.mutex); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 3, 5); attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + STATE_CHECK(state.mqtt_interrupts == 3); + STATE_CHECK(state.playback.queued_audio == 1); + STATE_CHECK(state.accepted_new_audio == 1); + + /* MQTT loss removes only the independent control path. TAI stays paused on + * the full playback queue; reconnect restores authenticated delivery. */ + iot_client_message_disconnect(&iot); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_UNINITIALIZED); + STATE_CHECK(state.playback.queued_audio == 1); + CHECK(iot_client_message_connect(&iot) == OPRT_OK); + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 4, 5); attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + STATE_CHECK(state.mqtt_interrupts == 4); + STATE_CHECK(state.playback.queued_audio == 1); + + /* Shutdown while receive admission is still closed must not hang. */ + tai_disconnect(ctx); + tai_ctx_deinit(ctx); + iot_client_message_disconnect(&iot); + iot.cacert = NULL; + pal->free(cacert); + stop_mqtt_mock(); + pthread_mutex_destroy(&state.playback.mutex); + pthread_mutex_destroy(&state.mutex); + printf("iot_tai_control_test: %s\n", failures ? "FAIL" : "PASS"); + return failures ? 1 : 0; +} diff --git a/modules/iot-client/test/mqtt_interrupt_demo_test.c b/modules/iot-client/test/mqtt_interrupt_demo_test.c new file mode 100644 index 0000000..ab1a14a --- /dev/null +++ b/modules/iot-client/test/mqtt_interrupt_demo_test.c @@ -0,0 +1,324 @@ +/* Include the demo so regressions exercise its actual static callbacks rather + * than a test-only copy of the interruption/filtering logic. Never run its + * main. iot_tai_control_test.c includes the same demo in this one binary -- + * every file-scope symbol in it must stay static, or the two copies collide + * at link time. */ +#define main mqtt_interrupt_demo_main +#include "../../../examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c" +#undef main + +#include "tai_internal.h" + +#define DEMO_CHECK(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #expr); \ + failures++; \ + } \ + } while (0) + +#define EPOCH_MS UINT64_C(1700000000000) + +static int demo_test_event(tai_ctx_t *ctx, uint16_t type, const char *event_id, + const char *user_data, const char *body) +{ + uint8_t payload[256]; + tai_attr_t attrs[2] = { + tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + tai_attr_strv(TAI_ATTR_USER_DATA, user_data ? user_data : ""), + }; + int len = tai_pack_event(TAI_VER_21, type, (const uint8_t *)body, + body ? strlen(body) : 0, payload, sizeof(payload)); + if (len <= 0) return TAI_ERR_PROTO; + return tai_proto_dispatch(ctx, TAI_PKT_EVENT, attrs, user_data ? 2 : 1, + payload, (size_t)len); +} + +static int demo_test_audio(tai_ctx_t *ctx, uint8_t flag, uint64_t timestamp, + size_t frames) +{ + /* Keep the paused body's storage pinned in rx_buf, just like the worker. */ + int len = tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + flag, timestamp, ctx->rx_buf, sizeof(ctx->rx_buf)); + if (len <= 0) return TAI_ERR_PROTO; + size_t body_len = frames * 40; + if (body_len > sizeof(ctx->rx_buf) - (size_t)len) return TAI_ERR_MEM; + for (size_t i = 0; i < frames; i++) + memset(ctx->rx_buf + len + i * 40, (int)i + 1, 40); + tai_attr_t attr = tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, + "111 1 16 16000 0 16000 20 40"); + return tai_proto_dispatch(ctx, TAI_PKT_AUDIO, &attr, 1, + ctx->rx_buf, (size_t)len + body_len); +} + +static void demo_test_interrupt(demo_state_t *state, uint64_t timestamp) +{ + char json[64]; + int len = snprintf(json, sizeof(json), "{\"time\":\"%" PRIu64 "\"}", timestamp); + on_ai_control("asrInterrupt", json, (size_t)len, state); +} + +static void *demo_racing_audio(void *arg) +{ + demo_state_t *state = arg; + tai_audio_msg_t audio = { + .stream_flag = TAI_STREAM_START, + .data = (const uint8_t *)"opus", + .len = 4, + }; + for (uint64_t i = 1; i <= 1000; i++) { + audio.timestamp_ms = EPOCH_MS + i; + if (on_flow_control(NULL, state)) on_audio(NULL, &audio, state); + } + return NULL; +} + +static void *demo_racing_interrupt(void *arg) +{ + demo_state_t *state = arg; + static const char cutoff[] = "{\"time\":\"1700000001000\"}"; + on_ai_control("asrInterrupt", cutoff, sizeof(cutoff) - 1, state); + return NULL; +} + +int mqtt_interrupt_demo_tests(void) +{ + int failures = 0; + const pal_t *pal = tai_pal_posix(); + demo_state_t state = {0}; + if (pthread_mutex_init(&state.mutex, NULL) != 0) { + fprintf(stderr, "FAIL mqtt_interrupt_demo_tests: mutex init\n"); + return 1; + } + tai_config_t config = { + .pal = pal, + .protocol_version = TAI_VER_21, + .on_audio = on_audio, + .on_event = on_event, + .on_flow_control = on_flow_control, + .user_data = &state, + }; + void *memory = pal->malloc(tai_ctx_size()); + state.tai = memory ? tai_ctx_init(memory, &config) : NULL; + DEMO_CHECK(state.tai != NULL); + if (!state.tai) goto cleanup; + tai_ctx_t *ctx = state.tai; + + /* The two Event slices are independent. A plausible payload is never a + * fallback for a missing or malformed Attribute 111: an unusable time + * fails closed onto the in-flight stream's own START (EPOCH_MS), never + * onto the payload's timestamp (+900). */ + const char *plausible_body = + "{\"breakAttributes\":{\"time\":\"1700000000900\"}}"; + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS, 1) == TAI_OK); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", NULL, + plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", "{}", + plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000300\\u0000junk\"}}", + plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000100\"}}", plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 100 && state.queued_audio == 0); + + /* No Event ID is required for audio, and stale START cannot revive playback. + * Later per-Packet timestamps do not change the stream's classification. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS, 2) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_MIDDLE, EPOCH_MS + 900, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_END, EPOCH_MS + 1000, 1) == TAI_OK); + DEMO_CHECK(state.stream_start_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_ONE_SHOT, EPOCH_MS + 200, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 1); + + static const char *invalid[] = { + "{}", "null", "[]", "{\"time\":null}", "{\"time\":true}", + "{\"time\":{}}", "{\"time\":\"\"}", "{\"time\":\"0\"}", + "{\"time\":0}", "{\"time\":-1}", "{\"time\":1.5}", + "{\"time\":\"-1\"}", "{\"time\":\"+1700000000300\"}", + "{\"time\":\"1700000000300x\"}", "{\"time\":\" 1700000000300\"}", + "{\"time\":\"1700000000300.0\"}", "{\"time\":\"1e12\"}", + "{\"time\":\"4398046511104\"}", "{\"time\":4398046511104}", + "{\"time\":\"18446744073709551616000\"}", + "{\"time\":\"1700000000300\\u0000junk\"}", + "{\"time\":\"1700000000300\"", "{\"time\":\"1700000000300\"}garbage", + }; + for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) { + on_ai_control("asrInterrupt", invalid[i], strlen(invalid[i]), &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 0); + } + on_ai_control("asrInterrupt", NULL, 0, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 0); + static const char bounded[] = "{\"time\":\"1700000000300\"}ignored"; + on_ai_control("asrInterrupt", bounded, sizeof(bounded) - 10, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 0); + on_ai_control("not-an-interrupt", bounded, sizeof(bounded) - 8, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200); + + /* Duplicates and older notices cannot flush fresh audio. A newer cutoff + * still below the stream start is applied but must preserve it. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 250, 1) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 1); + demo_test_interrupt(&state, EPOCH_MS + 100); + demo_test_interrupt(&state, EPOCH_MS + 50); + static const char numeric[] = "{\"time\":1700000000150} \n"; + on_ai_control("asrInterrupt", numeric, sizeof(numeric) - 1, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 1); + DEMO_CHECK(state.queued_start_ms[0] == EPOCH_MS + 250); + static const char newer[] = "{\"time\":\"1700000000225\"}"; + on_ai_control("asrInterrupt", newer, sizeof(newer) - 1, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 225 && state.queued_audio == 1); + DEMO_CHECK(state.queued_start_ms[0] == EPOCH_MS + 250); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000175\"}}", plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 225 && state.queued_audio == 1); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000100\"}}", plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 225 && state.queued_audio == 1); + + /* A bounded, non-NUL-terminated view is accepted, ignoring adjacent bytes. */ + on_ai_control("asrInterrupt", bounded, sizeof(bounded) - 8, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 300 && state.queued_audio == 0); + + /* Deterministically exercise both callback/admission race orderings. */ + tai_audio_msg_t audio = { + .stream_flag = TAI_STREAM_START, + .timestamp_ms = EPOCH_MS + 400, + .data = (const uint8_t *)"opus", + .len = 4, + }; + DEMO_CHECK(on_flow_control(ctx, &state) == 1); + demo_test_interrupt(&state, EPOCH_MS + 400); + on_audio(ctx, &audio, &state); + DEMO_CHECK(state.queued_audio == 0); + audio.timestamp_ms = EPOCH_MS + 500; + DEMO_CHECK(on_flow_control(ctx, &state) == 1); + on_audio(ctx, &audio, &state); + DEMO_CHECK(state.queued_audio == 1); + demo_test_interrupt(&state, EPOCH_MS + 500); + DEMO_CHECK(state.queued_audio == 0); + + /* Mixed queues retain each item's original stream start, not merely the + * latest stream's timestamp. Preserve order while removing stale items. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 600, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 900, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 700, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_ONE_SHOT, EPOCH_MS + 1000, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + demo_test_interrupt(&state, EPOCH_MS + 700); + DEMO_CHECK(state.queued_audio == 2); + DEMO_CHECK(state.queued_start_ms[0] == EPOCH_MS + 900); + DEMO_CHECK(state.queued_start_ms[1] == EPOCH_MS + 1000); + demo_test_interrupt(&state, EPOCH_MS + 1000); + DEMO_CHECK(state.queued_audio == 0); + + /* Ordinary pause/resume loses no codec frames; repeated START callbacks, + * including resumed ones, all retain the same server timestamp. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1100, + PLAYBACK_CAPACITY + 2) == TAI_OK); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + DEMO_CHECK(ctx->rx_pending_len == 80); + DEMO_CHECK(ctx->rx_pending_body == ctx->rx_buf + 8 + PLAYBACK_CAPACITY * 40); + DEMO_CHECK(ctx->rx_pending_ts_ms == EPOCH_MS + 1100); + for (size_t i = 0; i < state.queued_audio; i++) + DEMO_CHECK(state.queued_start_ms[i] == EPOCH_MS + 1100); + const uint8_t *cursor = ctx->rx_pending_body; + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 1); + DEMO_CHECK(ctx->rx_pending_body == cursor && ctx->rx_pending_len == 80); + /* Consume one frame, resume exactly one, then consume the full queue. */ + state.queued_audio--; + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 1); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + DEMO_CHECK(ctx->rx_pending_body == cursor + 40 && ctx->rx_pending_len == 40); + DEMO_CHECK(state.queued_start_ms[PLAYBACK_CAPACITY - 1] == EPOCH_MS + 1100); + state.queued_audio = 0; + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 0); + DEMO_CHECK(ctx->rx_pending_body == NULL && ctx->rx_pending_len == 0); + DEMO_CHECK(state.queued_audio == 1 && state.queued_start_ms[0] == EPOCH_MS + 1100); + state.queued_audio = 0; + + /* Interrupt a paused START without touching any worker-owned pending state. + * The worker then drains the retained frames through the real callback. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1200, + PLAYBACK_CAPACITY + 2) == TAI_OK); + cursor = ctx->rx_pending_body; + demo_test_interrupt(&state, EPOCH_MS + 1200); + DEMO_CHECK(state.queued_audio == 0 && on_flow_control(ctx, &state) == 1); + DEMO_CHECK(ctx->rx_pending_body == cursor && ctx->rx_pending_len == 80); + DEMO_CHECK(ctx->rx_pending_ts_ms == EPOCH_MS + 1200); + DEMO_CHECK(ctx->rx_pending_flag == TAI_STREAM_START); + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 0); + DEMO_CHECK(ctx->rx_pending_body == NULL && ctx->rx_pending_len == 0); + DEMO_CHECK(state.queued_audio == 0 && state.stream_start_ms == EPOCH_MS + 1200); + + /* Empty START/ONE_SHOT must latch through the real dispatcher but must not + * enter playback. MIDDLE/END use that latch, not their own timestamps. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1300, 0) == TAI_OK); + DEMO_CHECK(state.stream_start_ms == EPOCH_MS + 1300 && state.queued_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_MIDDLE, EPOCH_MS, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 1 && state.queued_start_ms[0] == EPOCH_MS + 1300); + demo_test_interrupt(&state, EPOCH_MS + 1300); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_ONE_SHOT, EPOCH_MS + 1250, 0) == TAI_OK); + DEMO_CHECK(state.stream_start_ms == EPOCH_MS + 1250 && state.queued_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_END, EPOCH_MS + 2000, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 0); + + /* END for interrupted or previous Events cannot terminate the demo. */ + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_START, "old", NULL, NULL) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1200, 1) == TAI_OK); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "old", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_START, "new", NULL, NULL) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1400, 1) == TAI_OK); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "old", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 0 && strcmp(state.current_event_id, "new") == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "new", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 1); + + /* A later text-only Event must not inherit the interrupted audio latch. */ + demo_test_interrupt(&state, EPOCH_MS + 1400); + state.done = 0; + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_START, "text-only", NULL, NULL) == TAI_OK); + tai_text_msg_t text = { .text = "", .len = 0 }; + on_text(ctx, &text, &state); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "text-only", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 1); + + /* The inclusive 42-bit upper bound is valid, without narrowing to 32 bits. */ + demo_test_interrupt(&state, MAX_AUDIO_TIME_MS); + DEMO_CHECK(state.audio_cutoff_ms == MAX_AUDIO_TIME_MS && state.queued_audio == 0); + + /* No outer test mutex serializes these production callbacks: either order + * must leave all <= cutoff audio discarded using only the playback mutex. */ + demo_state_t racing = {0}; + int mutex_rc = pthread_mutex_init(&racing.mutex, NULL); + DEMO_CHECK(mutex_rc == 0); + if (mutex_rc == 0) { + for (int round = 0; round < 16; round++) { + racing.audio_cutoff_ms = 0; + racing.stream_start_ms = 0; + racing.queued_audio = 0; + pthread_t audio_thread, control_thread; + int audio_rc = pthread_create(&audio_thread, NULL, demo_racing_audio, &racing); + int control_rc = pthread_create(&control_thread, NULL, demo_racing_interrupt, &racing); + DEMO_CHECK(audio_rc == 0 && control_rc == 0); + if (audio_rc == 0) pthread_join(audio_thread, NULL); + if (control_rc == 0) pthread_join(control_thread, NULL); + DEMO_CHECK(racing.audio_cutoff_ms == EPOCH_MS + 1000); + DEMO_CHECK(racing.queued_audio == 0); + } + DEMO_CHECK(pthread_mutex_destroy(&racing.mutex) == 0); + } + +cleanup: + if (state.tai) tai_ctx_deinit(state.tai); + pal->free(memory); + DEMO_CHECK(pthread_mutex_destroy(&state.mutex) == 0); + printf("mqtt_interrupt_demo_tests: %d failures\n", failures); + return failures; +} + +#undef DEMO_CHECK diff --git a/modules/rtc-tcp-client/CONTEXT.md b/modules/rtc-tcp-client/CONTEXT.md index 9ef2d42..25c0efd 100644 --- a/modules/rtc-tcp-client/CONTEXT.md +++ b/modules/rtc-tcp-client/CONTEXT.md @@ -77,9 +77,21 @@ The liveness exchange the background thread runs: it sends a Ping every `ping_in (default 60 s) and treats the Connection as dead if no inbound traffic — a Pong *or any* received data — arrives within `ping_timeout_ms` (default 90 s), then fires `on_disconnect`. Counting any receive, not just Pong, keeps a long downstream stream from tripping a spurious -timeout. +timeout. Intentional receive backpressure suspends only receive liveness; resuming grants a +fresh timeout budget without counting as received traffic. Pings and requested stop remain +active; a Ping send failure still disconnects, and the peer may enforce its own timeout. _Avoid_: heartbeat, poll. +**Receive backpressure**: +Application-controlled admission that pauses all inbound traffic, including buffered Frames, +ChatBreak, ASR text, Pong and EOF detection, rather than only media delivery. Admission is +checked between complete Frames and before each codec-frame callback within an Audio Packet: a +mid-Packet pause keeps the Packet's remaining bytes where they are (zero-copy, worker-owned) +and delivers them before any later Packet once admission reopens. Interruption filtering stays +in the application and is time-based — the server media timestamp against the interruption +time. +_Avoid_: audio mute, selective pause, codec-frame flow control. + **Chat break**: A client-sent Event (`TAI_EVT_CHAT_BREAK`) that interrupts the server's in-progress response. Sent standalone, not part of an Event's normal lifecycle. The server also sends @@ -89,6 +101,17 @@ uplink open. The current cloud no longer sends `TAI_EVT_SERVER_VAD`; a device mu an inbound ChatBreak as the turn boundary. _Avoid_: cancel, stop, abort. +**Server-initiated interrupt**: +An application concern delivered independently from the media Connection: either an inbound +ChatBreak Event on this context or an authenticated MQTT protocol-9000 AI control notice from +the IoT Client. Both carry a server time naming the interruption — the MQTT notice in its own +payload, the ChatBreak in the UserData attribute (attr 111, `breakAttributes.time`) — and the +application filters downlink audio by comparing that time against the media timestamp of the +stream's START. Applications route both paths into one playback policy; this module does not +receive or inject the MQTT notice. The independent path remains available while receive +backpressure intentionally stops this Connection. +_Avoid_: MQTT Event, TAI injection, acknowledgement. + **Server VAD**: A server-sent Event (`TAI_EVT_SERVER_VAD`) signalling that voice-activity detection found the end of the user's speech in audio mode. **Legacy**: the current cloud signals the @@ -194,8 +217,11 @@ The worker loops: check the liveness deadline, send a Ping when due, then block `tai_recv_data` until bytes arrive or the next Ping falls due, then drain. The drain is time-bounded (`AGENTIC_KIT_TAI_DRAIN_BUDGET_MS`, default 150 ms) so a sustained downstream flood cannot starve the Ping / liveness / shutdown checks — leftover bytes wait for the next pass; and any -successful receive refreshes the liveness clock. Bytes accumulate in a sliding receive buffer; -EOF or a transport error makes the worker fire `on_disconnect`. +successful receive refreshes the liveness clock. Receive backpressure pauses both reads and +parsing; on resume, a paused Packet's remaining codec frames drain first, then buffered complete +Frames precede the next read or EOF detection, while partial input returns to bounded blocking +reception. Bytes accumulate in a sliding receive buffer; detected EOF or a transport error makes +the worker fire `on_disconnect`. `tai_process_rx` peels complete Frames off the front of that buffer: diff --git a/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md b/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md new file mode 100644 index 0000000..2bb7cc4 --- /dev/null +++ b/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md @@ -0,0 +1,35 @@ +# Receive backpressure pauses the Connection and retains a partial Packet in place + +The RTC TCP Client exposes an optional `on_flow_control` admission hook. When it +returns zero, the worker stops parsing and stops reading the Connection, allowing +the TCP receive window to close. This pauses all inbound traffic rather than +selecting only audio. The worker still runs Ping, shutdown, and bounded admission +checks through `pal_t.sleep_ms`. + +Admission is checked between complete Frames and before each codec-frame callback +within an Audio Packet. Checking only at Frame boundaries would force the +application either to accept a whole Packet or to lose its remainder: the queue +that fills is the application's, and one Audio Packet can emit several codec frames. + +A mid-Packet pause keeps the Packet's bytes where they already are rather than +copying them into a second queue. The worker records the remaining body pointer, its +length, and the wire length of the pinned Frame; only the worker advances that +cursor, and no other Packet dispatches until it is exhausted — so `rx_audio_*` and +`rx_event_id` still describe the paused Packet and need not be duplicated. Teardown +discards a remainder. + +## Consequences + +- ChatBreak, text, Pong, and EOF detection are delayed while admission is closed; + an independent MQTT control path can carry urgent application notices. +- Intentional pauses suspend receive-liveness accounting. Resume starts a fresh + `ping_timeout_ms` budget without pretending traffic arrived. +- Complete buffered Frames are processed before another read after resume, and a + paused Packet's remaining frames are delivered before any later Packet; + incomplete input returns to bounded blocking receive. +- A header-only START/ONE_SHOT reaches `on_audio` with `len == 0` without an + admission check, so its server-side `timestamp_ms` can be latched for filtering. +- Interruption filtering is application-owned and time-based: `on_audio` compares + the server timestamp against the interruption time from the MQTT notice or from + ChatBreak's `user_data` (attr 111). The SDK exposes no receive state to mutate. +- `pal_t.sleep_ms` is mandatory, so custom PALs and consumers must rebuild. diff --git a/modules/rtc-tcp-client/include/tai_config_defaults.h b/modules/rtc-tcp-client/include/tai_config_defaults.h index 57f1e09..6fbf281 100644 --- a/modules/rtc-tcp-client/include/tai_config_defaults.h +++ b/modules/rtc-tcp-client/include/tai_config_defaults.h @@ -130,4 +130,14 @@ #define AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS 2000U #endif +/* Delay between admission checks while on_flow_control pauses all receives. */ +#ifndef AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS +#define AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS 50U +#endif + +/* CPU yield between receive-worker drain passes under sustained traffic. */ +#ifndef AGENTIC_KIT_TAI_WORKER_YIELD_MS +#define AGENTIC_KIT_TAI_WORKER_YIELD_MS 10U +#endif + #endif /* AGENTIC_KIT_TAI_CONFIG_DEFAULTS_H */ diff --git a/modules/rtc-tcp-client/include/tuya_ai.h b/modules/rtc-tcp-client/include/tuya_ai.h index 6d0451c..199e27a 100644 --- a/modules/rtc-tcp-client/include/tuya_ai.h +++ b/modules/rtc-tcp-client/include/tuya_ai.h @@ -165,14 +165,15 @@ typedef struct tai_ctx tai_ctx_t; /* --- Audio --------------------------------------------------------------- */ typedef struct tai_audio_msg { const uint8_t *data; /* Opus frame / PCM bytes; callback-lifetime */ - size_t len; + size_t len; /* may be 0 for a header-only START/ONE_SHOT */ uint8_t codec; /* TAI_AUDIO_OPUS / TAI_AUDIO_PCM / 0=unknown */ uint32_t sample_rate; /* Hz, 0 if unknown */ uint16_t frame_duration; /* ms per Opus frame */ uint8_t stream_flag; /* TAI_STREAM_* (from the media header) */ uint16_t data_id; /* Data ID: AUDIO_DOWN(2) / AUDIO_AUX(7) */ const char *event_id; /* turn id, borrowed; "" if none */ - uint64_t timestamp_ms; /* stream-start ts (media header) */ + uint64_t timestamp_ms; /* server media-header timestamp, not local time; + * latch START's value for stream filtering */ uint8_t _reserved[8]; } tai_audio_msg_t; @@ -212,6 +213,8 @@ typedef struct tai_event_msg { const uint8_t *data; /* event payload (often JSON); callback-life */ size_t len; const char *event_id; /* attr 61, borrowed; "" if absent */ + const uint8_t *user_data; /* attr 111, borrowed, NOT NUL-terminated; NULL if absent */ + size_t user_data_len; /* separate from event payload; SDK does not parse JSON */ uint8_t _reserved[8]; } tai_event_msg_t; @@ -328,6 +331,25 @@ typedef struct tai_config { void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *user_data); void *user_data; + /* Optional TCP receive backpressure. Called on the worker thread before + * each receive and between complete buffered Frames. A conforming server + * never consults it during the synchronous connect handshake (the ack + * precedes media); only a server coalescing media ahead of the ack could. + * Return nonzero to admit another Frame or + * read; return 0 to pause parsing and skip the read, allowing the TCP receive + * window to close. This pauses all inbound traffic, including ChatBreak, + * text, Pong, and EOF detection. Receive-liveness timeout accounting is + * suspended during the pause; reopening admission grants a fresh + * ping_timeout_ms budget. Pings and requested shutdown remain active, and a + * Ping send failure still disconnects. The hook must return promptly. + * + * Admission is also checked before each codec-frame callback within an Audio + * Packet. A pause retains its remaining bytes without copying; resuming + * delivers them before any later Packet. Applications filter obsolete audio + * in on_audio using their own synchronized interruption state, not by mutating + * SDK receive state. NULL means receive continuously. */ + int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); + } tai_config_t; /* ========================================================================= diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index 67517fc..4d9e216 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -341,6 +341,7 @@ tai_ctx_t *tai_ctx_init(void *mem, const tai_config_t *cfg) ctx->on_image = cfg->on_image; ctx->on_event = cfg->on_event; ctx->on_disconnect = cfg->on_disconnect; + ctx->on_flow_control = cfg->on_flow_control; ctx->user_data = cfg->user_data; ctx->ping_interval_ms = cfg->ping_interval_ms ? cfg->ping_interval_ms : 60000U; @@ -403,7 +404,7 @@ void tai_ctx_deinit(tai_ctx_t *ctx) /* Forward declarations (defined below; used by the confirmed-connect wait) */ static void *worker_thread(void *arg); static int tai_recv_data(tai_ctx_t *ctx, uint32_t timeout_ms); -static int tai_process_rx(tai_ctx_t *ctx); +static int tai_process_rx(tai_ctx_t *ctx, int *paused); int tai_connect(tai_ctx_t *ctx) { @@ -536,7 +537,7 @@ int tai_connect(tai_ctx_t *ctx) tai_disconnect(ctx); return TAI_ERR_NET; } if (n > 0) { - int fatal = tai_process_rx(ctx); + int fatal = tai_process_rx(ctx, NULL); if (fatal != TAI_OK) { TAI_LOGE(ctx->pal, TAG, "connect: protocol error before ack (%d)", fatal); tai_disconnect(ctx); return TAI_ERR_PROTO; @@ -639,6 +640,9 @@ void tai_disconnect(tai_ctx_t *ctx) ctx->rx_len = 0; ctx->frag_len = 0; ctx->frag_state = 0; + ctx->rx_pending_len = 0; /* teardown drops a paused Packet remainder */ + ctx->rx_pending_wire_len = 0; + ctx->rx_pending_body = NULL; ctx->connecting = 0; /* clear in case a connect aborted mid-handshake */ ctx->disconnect_emitted = 0; /* re-arm the single-point on_disconnect */ ctx->rx_event_id[0] = '\0'; /* clear latched turn id so a reconnect starts clean */ @@ -714,15 +718,55 @@ static int tai_recv_data(tai_ctx_t *ctx, uint32_t timeout_ms) * Returns a fail-fast cause for the worker: TAI_OK (no fatal; processed zero or * more frames), a TAI_PROTO_ERR_* detail on a protocol error, or * TAI_RX_PEER_CLOSE|code on a server CONNECTION_CLOSE. Never touches the - * connection lifecycle itself — the worker owns that. + * connection lifecycle itself — the worker owns that. Sets @p paused when + * admission closes between complete Frames; NULL disables flow control during + * the synchronous connect handshake. * ========================================================================= */ -static int tai_process_rx(tai_ctx_t *ctx) +static int tai_process_rx(tai_ctx_t *ctx, int *paused) { + if (paused) *paused = 0; + /* Process all complete frames sitting in rx_buf. Any structural error is * fail-fast: on a reliable, ordered TLS stream a desync cannot be recovered * by dropping bytes/frames, so we RETURN the cause and let the worker tear * the connection down (the app reconnects). */ while (ctx->rx_len >= 5) { + /* One receive may contain several complete Frames. Stop before the next + * one when the consumer fills while dispatching the previous Frame. + * The leading paused gate (not the helper) is what keeps this + * checkpoint dark during the connect handshake (paused == NULL). */ + if (paused && tai_rx_admission_paused(ctx)) { + *paused = 1; + break; + } + + /* A codec-frame callback may have filled the app queue mid-Packet. + * Re-dispatching the Frame here would replay accepted bytes; drain only + * its explicit pending remainder instead. Once it drains, slide the + * pinned Frame out from rx_buf so the next loop starts at the Frame + * behind it. */ + if (ctx->rx_pending_len) { + if (tai_proto_drain_pending_audio(ctx)) { + if (paused) *paused = 1; + break; + } + /* The pinned Frame must be consumed exactly once. A zero or + * oversized length would underflow rx_len (a huge memmove) or + * leave the Frame in place to be dispatched again, so fail fast + * instead of silently desyncing the stream. */ + size_t wire_len = ctx->rx_pending_wire_len; + if (wire_len < 5 || wire_len > ctx->rx_len) { + TAI_LOGE(ctx->pal, TAG, + "pending slide out of range: wire=%zu rx=%zu", + wire_len, ctx->rx_len); + return TAI_PROTO_ERR_FRAME_DECODE; + } + memmove(ctx->rx_buf, ctx->rx_buf + wire_len, ctx->rx_len - wire_len); + ctx->rx_len -= wire_len; + ctx->rx_pending_wire_len = 0; + continue; + } + /* Detect version from first byte */ int ver = tai_frame_detect_version(ctx->rx_buf[0]); if (ver < 0) { @@ -813,9 +857,24 @@ static int tai_process_rx(tai_ctx_t *ctx) return fatal; /* PROTOCOL detail or TAI_RX_PEER_CLOSE|code */ } + /* A mid-Packet pause pins this Frame's front; the paused body pointer + * therefore remains valid until the remainder drains. Once it drains, + * the pending-drain path above already slid the Frame, so this code + * runs only for the no-pending fast path. */ + if (ctx->rx_pending_len) { + ctx->rx_pending_wire_len = needed; + if (paused) *paused = 1; + break; + } + /* Consume frame from rx_buf */ memmove(ctx->rx_buf, ctx->rx_buf + needed, ctx->rx_len - needed); ctx->rx_len -= needed; + + /* During tai_connect(), stop immediately after the acknowledgement. + * Coalesced media stays buffered for the worker, preserving the public + * contract that application receive callbacks run on that thread. */ + if (ctx->connecting && ctx->session_ack >= 0) break; } return TAI_OK; @@ -1226,6 +1285,13 @@ int tai_send_mcp_response(tai_ctx_t *ctx, const char *json_rpc_response) * Drives recv + dispatch in a loop, sends periodic pings, detects pong * timeouts. Auto-started by tai_connect(), stopped by tai_disconnect(). * + * Receive backpressure: admission (tai_rx_admission_paused(), i.e. the + * on_flow_control hook) is queried at four checkpoints — pass top, before + * each drain read, between buffered Frames (tai_process_rx), and before each + * codec-frame callback (tai_protocol.c). See tuya_ai.h on_flow_control and + * ADR 0001 for the contract; the paused branches below each consume one + * facet of that state. + * * Locking: the mbedTLS read/write mutex now lives inside the shared TLS module * (tls_write / tls_read, granularity = a single ssl_* call), so the worker no * longer needs to wrap recv+dispatch in ctx_lock. rx_buf / frag_buf are touched @@ -1238,6 +1304,10 @@ static void *worker_thread(void *arg) tai_ctx_t *ctx = (tai_ctx_t *)arg; TAI_LOGD(ctx->pal, TAG, "worker: started"); + int paused = 0; + int resume_buffered = ctx->rx_len > 0; + uint64_t last_resume_ms = 0; + /* The worker owns the disconnect decision: lower layers RETURN a fatal * cause, transport faults are detected here, and we fire on_disconnect once * on exit. f_reason==0xFF means a clean stop (tai_disconnect / request). */ @@ -1246,14 +1316,20 @@ static void *worker_thread(void *arg) uint16_t f_code = 0; while (ctx->running) { + int was_paused = paused; + paused = tai_rx_admission_paused(ctx); + if (!ctx->running) break; + uint64_t now = ctx->pal->time_ms(); + if (was_paused && !paused) last_resume_ms = now; - /* Liveness timeout: any inbound traffic counts as alive, not just - * PONGs. A busy downstream stream proves the link is up even while a - * ping is overdue, so we never kill an actively-receiving connection. */ + /* Any inbound traffic proves liveness. An intentional receive pause + * suspends this deadline; reopening admission grants a fresh budget + * without pretending any bytes were received. */ uint64_t last_alive = (ctx->last_rx_ms > ctx->last_pong_ms) ? ctx->last_rx_ms : ctx->last_pong_ms; - if (now - last_alive > ctx->ping_timeout_ms) { + if (last_resume_ms > last_alive) last_alive = last_resume_ms; + if (!paused && now - last_alive > ctx->ping_timeout_ms) { TAI_LOGW(ctx->pal, TAG, "worker: liveness timeout (%llu ms idle)", (unsigned long long)(now - last_alive)); f_reason = TAI_DISCONNECT_TRANSPORT; @@ -1293,10 +1369,29 @@ static void *worker_thread(void *arg) * ~AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS instead of waiting out a whole ping interval. */ if (wait_ms > AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS) wait_ms = AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS; + /* Pausing skips the read so the receive window can close. Keep Ping and + * shutdown housekeeping active while admission is closed. */ + if (paused) { + ctx->pal->sleep_ms(AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS); + continue; + } + uint64_t drain_start = ctx->pal->time_ms(); - int n = tai_recv_data(ctx, wait_ms); + /* A pending codec-frame remainder is not extra inbound bytes. Enter the + * drain directly so reopening admission resumes it without requiring a + * new receive; a pending rx_buf remainder also keeps the Frame pinned, + * making rx_len a valid resume marker. Partial input returns to a + * bounded blocking receive rather than spinning. */ + int n = ctx->rx_pending_len ? 1 + : resume_buffered ? (int)ctx->rx_len + : tai_recv_data(ctx, wait_ms); + resume_buffered = 0; + /* Set when this pass drained real input; gates the end-of-pass yield + * below so an idle pass (blocking wait timed out, no data) does not + * gain fixed latency on top of it. */ + int did_work = (n > 0); while (n > 0 && ctx->running) { - int fatal = tai_process_rx(ctx); + int fatal = tai_process_rx(ctx, &paused); if (fatal != TAI_OK) { if (fatal & TAI_RX_PEER_CLOSE) { f_reason = TAI_DISCONNECT_CONNECTION_CLOSE; @@ -1307,11 +1402,21 @@ static void *worker_thread(void *arg) } break; } + if (paused) { + /* Do not read past the buffered Frame that closed admission. */ + resume_buffered = 1; + break; + } + if (!ctx->running) break; /* Bound the greedy drain so periodic ping / liveness / shutdown * checks run even under a sustained flood; leftover bytes wait for * the next pass. */ if (ctx->pal->time_ms() - drain_start > AGENTIC_KIT_TAI_DRAIN_BUDGET_MS) break; + ctx->pal->sleep_ms(AGENTIC_KIT_TAI_WORKER_YIELD_MS); + /* Admission is checked immediately before the next drain read. */ + paused = tai_rx_admission_paused(ctx); + if (paused || !ctx->running) break; n = tai_recv_data(ctx, 0); /* drain remainder non-blocking */ } if (f_reason != 0xFF) break; /* fail-fast / CONNECTION_CLOSE during drain */ @@ -1330,7 +1435,11 @@ static void *worker_thread(void *arg) } /* n > 0: budget yield (running still set) -> loop, keep housekeeping. * n == TAI_ERR_AGAIN: wait timed out (ping due) or drain finished. - * running cleared with no fatal: a clean tai_request_disconnect. */ + * running cleared with no fatal: a clean tai_request_disconnect. + * The yield applies only between drain passes that did work; an idle + * pass must not add fixed latency on top of its blocking wait. */ + if (did_work) + ctx->pal->sleep_ms(AGENTIC_KIT_TAI_WORKER_YIELD_MS); } /* Single-point disconnect: fire on_disconnect exactly once if the worker diff --git a/modules/rtc-tcp-client/src/tai_internal.h b/modules/rtc-tcp-client/src/tai_internal.h index ad223af..062a90d 100644 --- a/modules/rtc-tcp-client/src/tai_internal.h +++ b/modules/rtc-tcp-client/src/tai_internal.h @@ -72,8 +72,9 @@ * Buffer sizes — defaults & docs: include/tai_config_defaults.h * (AGENTIC_KIT_TAI_MAX_FRAGMENT_PAYLOAD, AGENTIC_KIT_TAI_FRAG_BUF_SIZE, AGENTIC_KIT_TAI_TX_HDR_BUF_SIZE, * AGENTIC_KIT_TAI_FRAME_COALESCE_LIMIT, AGENTIC_KIT_TAI_TX_CTRL_BUF_SIZE, AGENTIC_KIT_TAI_MAX_ATTRS, - * AGENTIC_KIT_TAI_DRAIN_BUDGET_MS, AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS; reduce for - * memory-constrained targets, e.g. ESP32 without PSRAM.) + * AGENTIC_KIT_TAI_DRAIN_BUDGET_MS, AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS, + * AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS, AGENTIC_KIT_TAI_WORKER_YIELD_MS; + * reduce for memory-constrained targets, e.g. ESP32 without PSRAM.) * ========================================================================= */ /* RX sliding-window buffer. Sized to EXACTLY one maximum wire frame — @@ -241,6 +242,19 @@ struct tai_ctx { void (*on_disconnect)(tai_ctx_t *, const tai_disconnect_msg_t *, void *); void *user_data; + /* Optional TCP receive backpressure (see tuya_ai.h). */ + int (*on_flow_control)(tai_ctx_t *, void *); + + /* Worker-owned audio cursor: nonzero remaining length means pending. + * No other Packet dispatches until it drains, so rx_audio_* / rx_event_id + * stay valid. Pin the current wire Frame even for reassembled Packets. */ + const uint8_t *rx_pending_body; + size_t rx_pending_len; + size_t rx_pending_wire_len; + uint8_t rx_pending_flag; + uint16_t rx_pending_data_id; + uint64_t rx_pending_ts_ms; + /* RX linear buffer (sliding-window: bytes always at buf[0]) */ uint8_t rx_buf[TAI_RX_BUF_SIZE]; size_t rx_len; @@ -500,6 +514,23 @@ int tai_proto_dispatch (tai_ctx_t *ctx, uint8_t pkt_type, const tai_attr_t *attrs, int attr_count, const uint8_t *payload, size_t payload_len); +/* Worker-only: returns 1 while delivery remains paused, 0 once finished. + * The Packet's bytes stay pinned in rx_buf / frag_buf until it drains. */ +int tai_proto_drain_pending_audio(tai_ctx_t *ctx); + +/* Admission query — the single judgment behind receive backpressure, shared + * by the four checkpoints (worker pass top, before each drain read, between + * buffered Frames, before each codec-frame callback). Nonzero = this + * checkpoint must pause; 0 = admit. A NULL hook never pauses and is never + * invoked. Raw query only: the paused-pointer gate that disables flow + * control during the synchronous connect handshake stays at the + * tai_process_rx call site. Runs on the receive path — the hook must return + * promptly and is called at most once per check. */ +static inline int tai_rx_admission_paused(tai_ctx_t *ctx) +{ + return ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data); +} /* Internal sequence helper */ static inline uint16_t tai_next_seq(tai_ctx_t *ctx) { diff --git a/modules/rtc-tcp-client/src/tai_protocol.c b/modules/rtc-tcp-client/src/tai_protocol.c index e0c0f64..960929c 100644 --- a/modules/rtc-tcp-client/src/tai_protocol.c +++ b/modules/rtc-tcp-client/src/tai_protocol.c @@ -522,7 +522,8 @@ static void emit_text(tai_ctx_t *ctx, const char *text, size_t len, } static void emit_event(tai_ctx_t *ctx, uint16_t event_type, - const uint8_t *data, size_t len) + const uint8_t *data, size_t len, + const tai_attr_t *user_data) { if (!ctx->on_event) return; tai_event_msg_t m = {0}; @@ -530,6 +531,10 @@ static void emit_event(tai_ctx_t *ctx, uint16_t event_type, m.data = data; m.len = len; m.event_id = ctx->rx_event_id; + if (user_data) { + m.user_data = user_data->value; + m.user_data_len = user_data->len; + } ctx->on_event(ctx, &m, ctx->user_data); } @@ -604,28 +609,27 @@ static void parse_audio_params_once(tai_ctx_t *ctx, } } -/* Split an audio body into CBR Opus frames of rx_audio_frame_size, emitting a - * final short remainder. Whole frames are emitted zero-copy from `body` (valid - * until the frame is consumed from rx_buf). fs==0 (PCM / unknown) delivers the - * body whole. */ -static void media_audio_body(tai_ctx_t *ctx, const uint8_t *body, size_t body_len, - uint8_t stream_flag, uint16_t data_id, uint64_t ts_ms) +/* Split CBR Opus into codec frames, including a final short remainder. + * A zero frame size (PCM / unknown) delivers the remaining body whole. + * Only the worker advances this zero-copy cursor; the caller pins storage + * and does not dispatch another Packet until the cursor is exhausted. */ +int tai_proto_drain_pending_audio(tai_ctx_t *ctx) { - if (!ctx->on_audio) return; - uint16_t fs = ctx->rx_audio_frame_size; - if (fs == 0) { - if (body_len > 0) - emit_audio(ctx, body, body_len, stream_flag, data_id, ts_ms); - return; - } - - while (body_len >= fs) { - emit_audio(ctx, body, fs, stream_flag, data_id, ts_ms); - body += fs; body_len -= fs; + while (ctx->rx_pending_len) { + if (tai_rx_admission_paused(ctx)) { + return 1; + } + size_t len = fs == 0 || ctx->rx_pending_len < fs + ? ctx->rx_pending_len : fs; + emit_audio(ctx, ctx->rx_pending_body, len, + ctx->rx_pending_flag, ctx->rx_pending_data_id, + ctx->rx_pending_ts_ms); + ctx->rx_pending_body += len; + ctx->rx_pending_len -= len; } - if (body_len > 0) - emit_audio(ctx, body, body_len, stream_flag, data_id, ts_ms); + ctx->rx_pending_body = NULL; + return 0; } /* AUDIO packet payload: [data_id:2][48-bit stream_flag|ts_ms][opus frames…]. */ @@ -654,7 +658,19 @@ static int media_audio(tai_ctx_t *ctx, parse_audio_params_once(ctx, attrs, attr_count); latch_event_id(ctx, attrs, attr_count); - media_audio_body(ctx, payload + 8, payload_len - 8, stream_flag, data_id, ts_ms); + /* Header-only STARTs still establish the server-time playback boundary. */ + if (payload_len == 8 && + (stream_flag == TAI_STREAM_START || stream_flag == TAI_STREAM_ONE_SHOT)) { + emit_audio(ctx, payload + 8, 0, stream_flag, data_id, ts_ms); + } + if (ctx->on_audio) { + ctx->rx_pending_body = payload + 8; + ctx->rx_pending_len = payload_len - 8; + ctx->rx_pending_flag = stream_flag; + ctx->rx_pending_data_id = data_id; + ctx->rx_pending_ts_ms = ts_ms; + tai_proto_drain_pending_audio(ctx); + } return TAI_OK; } @@ -766,7 +782,8 @@ int tai_proto_dispatch(tai_ctx_t *ctx, } latch_event_id(ctx, attrs, attr_count); - emit_event(ctx, evt_type, evt_data, evt_data_len); + emit_event(ctx, evt_type, evt_data, evt_data_len, + tai_attr_find(attrs, attr_count, TAI_ATTR_USER_DATA)); if (evt_type == TAI_EVT_END) ctx->rx_event_id[0] = '\0'; /* turn over: clear after firing END */ break; diff --git a/modules/rtc-tcp-client/test/tai_pal_loopback.c b/modules/rtc-tcp-client/test/tai_pal_loopback.c index 521c1d5..59cb3cb 100644 --- a/modules/rtc-tcp-client/test/tai_pal_loopback.c +++ b/modules/rtc-tcp-client/test/tai_pal_loopback.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "log.h" #include "../src/tai_internal.h" /* frame/packet codec + key derivation for the handshake mock */ @@ -348,11 +349,36 @@ static void lb_hs_feed(const uint8_t *buf, size_t len) ack_na ? ack_attrs : NULL, ack_na, (const uint8_t *)"", 0, app, sizeof(app)); if (alen > 0) { - uint8_t frame[128]; - int flen = tai_frame_encode(TAI_FRAG_NONE, 1, app, (size_t)alen, - sign_key, sig_len, tai_pal_loopback(), - frame, sizeof(frame)); - if (flen > 0) lb_fifo_push(&g_rx, frame, (size_t)flen); + uint8_t batch[384]; + int batch_len = tai_frame_encode(TAI_FRAG_NONE, 1, app, (size_t)alen, + sign_key, sig_len, tai_pal_loopback(), + batch, sizeof(batch)); + if (batch_len > 0 && g_hs.mode == TAI_LB_HS_ACK_WITH_TEXT) { + uint8_t text_payload[32]; + int text_header_len = tai_pack_text_hdr(TAI_VER_21, + TAI_DATA_ID_TEXT_DOWN, + TAI_STREAM_ONE_SHOT, 1, + text_payload, + sizeof(text_payload)); + static const char text[] = "coalesced"; + if (text_header_len > 0 && + (size_t)text_header_len + sizeof(text) - 1 <= sizeof(text_payload)) { + memcpy(text_payload + text_header_len, text, sizeof(text) - 1); + uint8_t text_app[64]; + int text_app_len = tai_packet_encode( + TAI_VER_21, TAI_PKT_TEXT, NULL, 0, + text_payload, (size_t)text_header_len + sizeof(text) - 1, + text_app, sizeof(text_app)); + if (text_app_len > 0) { + int text_frame_len = tai_frame_encode( + TAI_FRAG_NONE, 2, text_app, (size_t)text_app_len, + sign_key, sig_len, tai_pal_loopback(), + batch + batch_len, sizeof(batch) - (size_t)batch_len); + if (text_frame_len > 0) batch_len += text_frame_len; + } + } + } + if (batch_len > 0) lb_fifo_push(&g_rx, batch, (size_t)batch_len); } g_hs.done = 1; } @@ -375,13 +401,14 @@ static int lb_tcp_send(void *tcp, const uint8_t *buf, size_t len, return (int)len; /* PAL contract: >0 = bytes written */ } -/* Sleep helper for timeout emulation. */ +/* Real CPU sleep for timeout emulation; never advances the virtual clock. */ static void lb_sleep_ms(uint32_t ms) { + if (ms == 0) return; struct timespec ts; ts.tv_sec = ms / 1000; ts.tv_nsec = (long)(ms % 1000) * 1000000L; - nanosleep(&ts, NULL); + while (nanosleep(&ts, &ts) < 0 && errno == EINTR) { } } static uint64_t lb_time_ms(void); /* defined below; used for a real-time recv timeout */ @@ -513,6 +540,7 @@ static const pal_t g_loopback_pal = { .mutex_destroy = lb_mutex_destroy, .thread_create = lb_thread_create, .thread_join = lb_thread_join, + .sleep_ms = lb_sleep_ms, }; const pal_t *tai_pal_loopback(void) diff --git a/modules/rtc-tcp-client/test/tai_pal_loopback.h b/modules/rtc-tcp-client/test/tai_pal_loopback.h index 6cc87b2..f20297c 100644 --- a/modules/rtc-tcp-client/test/tai_pal_loopback.h +++ b/modules/rtc-tcp-client/test/tai_pal_loopback.h @@ -76,6 +76,8 @@ void tai_loopback_close_connection(void); #define TAI_LB_HS_AUTH_OK 4 /* confirm via an AuthenticateResponse (pkt 3) with * connection-status-code 200, the way the production * server does, instead of a SessionNew ack. */ +#define TAI_LB_HS_ACK_WITH_TEXT 5 /* push SessionNew ack and a signed Text Frame in + * one receive batch; Text must reach the worker. */ void tai_loopback_set_local_key(const char *local_key); void tai_loopback_set_handshake_mode(int mode); diff --git a/modules/rtc-tcp-client/test/test_core.c b/modules/rtc-tcp-client/test/test_core.c index b80af0f..475710e 100644 --- a/modules/rtc-tcp-client/test/test_core.c +++ b/modules/rtc-tcp-client/test/test_core.c @@ -49,6 +49,7 @@ static const pal_t g_stub_pal = { .mutex_destroy = stub_mutex_destroy, .thread_create = NULL, .thread_join = NULL, + .sleep_ms = NULL, }; /* ------------------------------------------------------------------------- diff --git a/modules/rtc-tcp-client/test/test_integration.c b/modules/rtc-tcp-client/test/test_integration.c index 4380963..3cf0844 100644 --- a/modules/rtc-tcp-client/test/test_integration.c +++ b/modules/rtc-tcp-client/test/test_integration.c @@ -18,11 +18,16 @@ #include #include #include +#include +#include +#include #include "../src/tai_internal.h" #include "tai_pal_loopback.h" #include "test_log.h" +extern const pal_t *tai_pal_posix(void); + /* ========================================================================= * Test harness * ========================================================================= */ @@ -76,7 +81,7 @@ typedef struct { size_t audio_bytes; uint32_t audio_sample_rate; uint16_t audio_frame_duration; - size_t audio_frame_lens[256]; /* per-call msg->len (carry-alignment check) */ + size_t audio_frame_lens[160]; /* per-call msg->len (carry-alignment check) */ uint8_t audio_concat[8192]; /* concatenation of all delivered frames */ size_t audio_concat_len; @@ -91,17 +96,25 @@ typedef struct { int event_count; uint8_t event_payload[4096]; /* last event's data (reassembly check) */ size_t event_payload_len; + uint8_t event_user_data[256]; + size_t event_user_data_len; + int event_user_data_present; + int event_user_data_borrowed; int disconnect_calls; uint16_t disconnect_code; uint8_t disconnect_reason; uint8_t disconnect_detail; + int text_calls_at_disconnect; + uint64_t disconnect_ms; char last_text_event_id[64]; char last_event_event_id[64]; } test_state_t; static test_state_t g_st; +static pthread_t g_connect_thread; +static pthread_t g_text_callback_thread; static void st_reset(void) { @@ -124,10 +137,15 @@ static void st_reset(void) g_st.event_calls = 0; g_st.event_count = 0; g_st.event_payload_len = 0; + g_st.event_user_data_len = 0; + g_st.event_user_data_present = 0; + g_st.event_user_data_borrowed = 0; g_st.disconnect_calls = 0; g_st.disconnect_code = 0; g_st.disconnect_reason = 0xFF; g_st.disconnect_detail = 0xFF; + g_st.text_calls_at_disconnect = 0; + g_st.disconnect_ms = 0; g_st.last_text_event_id[0] = '\0'; g_st.last_event_event_id[0] = '\0'; pthread_mutex_unlock(&g_st.mtx); @@ -137,6 +155,7 @@ static void on_text(tai_ctx_t *ctx, const tai_text_msg_t *msg, void *ud) { (void)ctx; (void)ud; pthread_mutex_lock(&g_st.mtx); + g_text_callback_thread = pthread_self(); g_st.text_calls++; g_st.last_text_flag = msg->stream_flag; if (msg->event_id) { @@ -190,9 +209,19 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *ud) g_st.event_calls++; if (g_st.event_count < (int)(sizeof(g_st.event_types)/sizeof(g_st.event_types[0]))) g_st.event_types[g_st.event_count++] = msg->event_type; - if (msg->data && msg->len <= sizeof(g_st.event_payload)) { + g_st.event_payload_len = msg->len; + if (msg->data && msg->len <= sizeof(g_st.event_payload)) memcpy(g_st.event_payload, msg->data, msg->len); - g_st.event_payload_len = msg->len; + g_st.event_user_data_len = msg->user_data_len; + g_st.event_user_data_present = msg->user_data != NULL; + if (msg->user_data && msg->user_data_len <= sizeof(g_st.event_user_data)) { + memcpy(g_st.event_user_data, msg->user_data, msg->user_data_len); + uintptr_t p = (uintptr_t)msg->user_data; + g_st.event_user_data_borrowed = + (p >= (uintptr_t)ctx->rx_buf && + p + msg->user_data_len <= (uintptr_t)ctx->rx_buf + sizeof(ctx->rx_buf)) || + (p >= (uintptr_t)ctx->frag_buf && + p + msg->user_data_len <= (uintptr_t)ctx->frag_buf + sizeof(ctx->frag_buf)); } if (msg->event_id) { size_t n = strlen(msg->event_id); @@ -211,6 +240,8 @@ static void on_disconnect(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void g_st.disconnect_code = msg->close_code; g_st.disconnect_reason = msg->reason; g_st.disconnect_detail = msg->detail; + g_st.text_calls_at_disconnect = g_st.text_calls; + g_st.disconnect_ms = ctx->pal->time_ms(); pthread_mutex_unlock(&g_st.mtx); } @@ -510,7 +541,7 @@ static int build_event_app(tai_ctx_t *ctx, uint16_t evt_type, /* ========================================================================= * Common setup/teardown * ========================================================================= */ -static tai_ctx_t *setup_ctx(void *mem) +static tai_ctx_t *setup_ctx_config(void *mem, const tai_config_t *options) { tai_loopback_reset(); tai_loopback_seed_random(42); @@ -535,6 +566,12 @@ static tai_ctx_t *setup_ctx(void *mem) cfg.on_image = on_image; cfg.on_event = on_event; cfg.on_disconnect = on_disconnect; + if (options) { + if (options->pal) cfg.pal = options->pal; + cfg.on_flow_control = options->on_flow_control; + cfg.ping_interval_ms = options->ping_interval_ms; + cfg.ping_timeout_ms = options->ping_timeout_ms; + } /* The loopback completes the handshake (signs the SessionNew ack with the * key derived from this local_key), so confirmed-connect succeeds. */ @@ -543,6 +580,11 @@ static tai_ctx_t *setup_ctx(void *mem) return tai_ctx_init(mem, &cfg); } +static tai_ctx_t *setup_ctx(void *mem) +{ + return setup_ctx_config(mem, NULL); +} + /* ========================================================================= * Test 1: text_query happy path * ========================================================================= */ @@ -983,6 +1025,52 @@ static void test_request_disconnect_from_callback(void) * turn's packets that carry no attr 61 (inherited, not reset to ""), and is * cleared after EVT_END. Distinguishes §4 from §1's reset-if-absent behaviour. * ========================================================================= */ +static void test_event_user_data(void) +{ + SECTION("event_user_data"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + tai_ctx_t *ctx = setup_ctx(ctx_mem); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + const uint8_t user_data[] = "{\"serverTime\":123456789}"; + const uint8_t payload[] = { 'p', 0, 'x' }; + tai_attr_t attr = tai_attr_bytesv(TAI_ATTR_USER_DATA, user_data, sizeof(user_data) - 1); + uint16_t seq = 100; + for (int round = 0; round < 4; round++) { + uint8_t evt[64], app[256]; + size_t len = round == 3 ? 0 : sizeof(payload); + int elen = tai_pack_event(TAI_VER_21, TAI_EVT_CHAT_BREAK, + payload, len, evt, sizeof(evt)); + CHECK(elen > 0); + if (round == 1) { + int alen = tai_packet_encode(TAI_VER_21, TAI_PKT_EVENT, &attr, 1, + evt, (size_t)elen, app, sizeof(app)); + CHECK(alen > 10); + if (alen <= 10) break; + size_t cuts[] = { 10, (size_t)alen }; + server_send_app_fragmented(ctx, app, (size_t)alen, cuts, 2, &seq); + } else { + CHECK(server_send(ctx, TAI_PKT_EVENT, round == 2 ? NULL : &attr, + round == 2 ? 0 : 1, evt, (size_t)elen, seq++) > 0); + } + CHECK(WAIT_FOR(g_st.event_calls == round + 1, 1000)); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.event_payload_len, len); + CHECK(memcmp(g_st.event_payload, payload, len) == 0); + CHECK_EQ_INT(g_st.event_user_data_present, round != 2); + CHECK_EQ_INT(g_st.event_user_data_len, round == 2 ? 0 : sizeof(user_data) - 1); + if (round != 2) { + CHECK(g_st.event_user_data_borrowed); + CHECK(memcmp(g_st.event_user_data, user_data, sizeof(user_data) - 1) == 0); + } + pthread_mutex_unlock(&g_st.mtx); + } + tai_disconnect(ctx); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); +} + static void test_event_id_latch(void) { SECTION("event_id_latch"); @@ -1925,6 +2013,23 @@ static void test_confirmed_connect(void) tai_ctx_deinit(ctx); } + /* Coalesced ack + application data: tai_connect consumes only the ack and + * leaves Text buffered for the receive worker. The callback must not run on + * the connecting thread even when both Frames arrived in one recv. */ + { + tai_ctx_t *ctx = setup_ctx(ctx_mem); + CHECK(ctx != NULL); + g_connect_thread = pthread_self(); + memset(&g_text_callback_thread, 0, sizeof(g_text_callback_thread)); + tai_loopback_set_handshake_mode(TAI_LB_HS_ACK_WITH_TEXT); + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(g_st.text_calls == 1, 1000)); + CHECK(strcmp(g_st.text_buf, "coalesced") == 0); + CHECK(!pthread_equal(g_connect_thread, g_text_callback_thread)); + tai_disconnect(ctx); + tai_ctx_deinit(ctx); + } + /* AuthenticateResponse ack: the production server confirms the connect with * an AuthenticateResponse (pkt 3) carrying connection-status-code 200, NOT a * SessionNew ack. tai_connect must complete. (Regression: pkt 3 used to fall @@ -2022,6 +2127,563 @@ static void test_disconnect_latency(void) tai_ctx_deinit(ctx); } +static const pal_t *observed_base_pal; +static pal_t observed_pal; +static size_t observed_recv_calls; +static size_t observed_recv_bytes; +static size_t observed_sleep_calls; +static size_t observed_empty_polls; +static size_t observed_after_bytes; +static size_t observed_blocking_after_bytes; +static size_t observed_nonblocking_after_bytes; +static uint32_t observed_max_timeout_after_bytes; + +static int observed_recv(void *tcp, uint8_t *buf, size_t len, + uint32_t timeout_ms) +{ + pthread_mutex_lock(&g_st.mtx); + observed_recv_calls++; + if (observed_after_bytes && observed_recv_bytes >= observed_after_bytes) { + if (timeout_ms) observed_blocking_after_bytes++; + else observed_nonblocking_after_bytes++; + if (timeout_ms > observed_max_timeout_after_bytes) + observed_max_timeout_after_bytes = timeout_ms; + } + pthread_mutex_unlock(&g_st.mtx); + + int n = observed_base_pal->tcp_recv(tcp, buf, len, timeout_ms); + + pthread_mutex_lock(&g_st.mtx); + if (n > 0) observed_recv_bytes += (size_t)n; + pthread_mutex_unlock(&g_st.mtx); + return n; +} + +static int observed_poll(void *tcp, int events, uint32_t timeout_ms) +{ + pthread_mutex_lock(&g_st.mtx); + if (events == 0) observed_empty_polls++; + pthread_mutex_unlock(&g_st.mtx); + return observed_base_pal->tcp_poll(tcp, events, timeout_ms); +} + +static void observed_sleep(uint32_t ms) +{ + pthread_mutex_lock(&g_st.mtx); + observed_sleep_calls++; + pthread_mutex_unlock(&g_st.mtx); + observed_base_pal->sleep_ms(ms); +} + +static const pal_t *observe_pal(void) +{ + observed_base_pal = tai_pal_loopback(); + observed_pal = *observed_base_pal; + observed_pal.tcp_recv = observed_recv; + observed_pal.tcp_poll = observed_poll; + observed_pal.sleep_ms = observed_sleep; + observed_recv_calls = 0; + observed_recv_bytes = 0; + observed_sleep_calls = 0; + observed_empty_polls = 0; + observed_after_bytes = 0; + observed_blocking_after_bytes = 0; + observed_nonblocking_after_bytes = 0; + observed_max_timeout_after_bytes = 0; + return &observed_pal; +} + +/* Pause after each delivered codec frame, including frames produced by one + * Packet. Resume without new network bytes to prove no pending callback or + * buffered Frame is stranded. */ +static int flow_allowed_calls; +static int flow_paused_queries; +static uint64_t flow_resume_ms; + +static int test_flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)user_data; + pthread_mutex_lock(&g_st.mtx); + int ready = g_st.audio_calls < flow_allowed_calls; + if (!ready) flow_paused_queries++; + else if (flow_resume_ms == 0) flow_resume_ms = ctx->pal->time_ms(); + pthread_mutex_unlock(&g_st.mtx); + return ready; +} + +static size_t audio_frame_len_at(int index) +{ + return index < 160 ? g_st.audio_frame_lens[index] + : g_st.audio_frame_lens[159]; +} + +static void test_receive_backpressure(void) +{ + SECTION("receive_backpressure"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + tai_config_t options = { + .pal = observe_pal(), + .on_flow_control = test_flow_control, + }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(flow_paused_queries > 0, 1000)); + + pthread_mutex_lock(&g_st.mtx); + size_t paused_recv_calls = observed_recv_calls; + pthread_mutex_unlock(&g_st.mtx); + + static uint8_t audio[8][800]; + /* Distinct constant per Packet: a wrong slide lands in another Packet's + * bytes and fails the packet-id check below regardless of position. */ + for (size_t packet = 0; packet < 8; packet++) { + memset(audio[packet], (int)(0x10 + packet), sizeof(audio[packet])); + CHECK(server_send_audio(ctx, + packet == 0 ? TAI_STREAM_START : TAI_STREAM_MIDDLE, + packet == 0 ? "111 1 16 16000 0 16000 20 40" : NULL, + audio[packet], sizeof(audio[packet]), + (uint16_t)(100 + packet)) > 0); + } + sleep_ms(100); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(observed_recv_calls, paused_recv_calls); + CHECK_EQ_INT(g_st.audio_calls, 0); + pthread_mutex_unlock(&g_st.mtx); + + for (int allowed = 1; allowed <= 8 * 20; allowed++) { + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = allowed; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(g_st.audio_calls >= allowed, 1000)); + sleep_ms(30); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.audio_calls, allowed); + pthread_mutex_unlock(&g_st.mtx); + } + + tai_disconnect(ctx); + CHECK(observed_sleep_calls > 0); + CHECK_EQ_INT(observed_empty_polls, 0); + CHECK_EQ_INT(g_st.audio_calls, 160); + CHECK_EQ_INT(g_st.audio_bytes, 8 * sizeof(audio[0])); + for (int frame = 0; frame < 160; frame++) + CHECK_EQ_INT(audio_frame_len_at(frame), 40); + /* Byte-exact across ALL Packets, not just the paused first one: each + * 800-byte Packet carries its own index in every byte, so any mis-slide + * shows up as a wrong packet id. */ + for (size_t i = 0; i < g_st.audio_concat_len; i++) { + size_t packet = i / sizeof(audio[0]); + if (packet >= 8) break; + CHECK_EQ_INT(g_st.audio_concat[i], audio[packet][0]); + } + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); +} + +static size_t pending_remaining, pending_wire_len; +static uintptr_t pending_cursor; +static int pending_fragmented, pending_metadata_ok, pending_audio_ok; + +static int pending_flow_control(tai_ctx_t *ctx, void *user_data) +{ + int ready = test_flow_control(ctx, user_data); + pthread_mutex_lock(&g_st.mtx); + if (!ready && ctx->rx_pending_wire_len) { + pending_remaining = ctx->rx_pending_len; + pending_wire_len = ctx->rx_pending_wire_len; + pending_cursor = (uintptr_t)ctx->rx_pending_body; + pending_metadata_ok = ctx->rx_audio_codec == TAI_AUDIO_OPUS && + ctx->rx_audio_sample_rate == 16000 && ctx->rx_audio_frame_duration == 20 && + ctx->rx_audio_frame_size == 40 && strcmp(ctx->rx_event_id, "pending-event") == 0; + } + pthread_mutex_unlock(&g_st.mtx); + return ready; +} + +static void pending_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *ud) +{ + const uint8_t *storage = pending_fragmented ? ctx->frag_buf : ctx->rx_buf; + uintptr_t p = (uintptr_t)msg->data; + size_t size = pending_fragmented ? sizeof(ctx->frag_buf) : sizeof(ctx->rx_buf); + pthread_mutex_lock(&g_st.mtx); + pending_audio_ok &= p >= (uintptr_t)storage && p + msg->len <= (uintptr_t)storage + size && + msg->codec == TAI_AUDIO_OPUS && msg->sample_rate == 16000 && + msg->frame_duration == 20 && msg->stream_flag == TAI_STREAM_START && + msg->data_id == TAI_DATA_ID_AUDIO_DOWN && msg->timestamp_ms == 123456789 && + strcmp(msg->event_id, "pending-event") == 0; + pthread_mutex_unlock(&g_st.mtx); + on_audio(ctx, msg, ud); +} + +/* A pinned Frame whose recorded wire length cannot be trusted must fail fast: + * an oversized value would underflow rx_len into a huge memmove, and a zero one + * would leave the Frame in place to be dispatched a second time. */ +static void test_pending_bad_wire_len(void) +{ + SECTION("pending_audio_bad_wire_len"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + pending_remaining = pending_wire_len = pending_cursor = 0; + pending_fragmented = 0; + pending_metadata_ok = 0; + pending_audio_ok = 1; + tai_config_t options = { .pal = observe_pal(), + .on_flow_control = pending_flow_control }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + ctx->on_audio = pending_audio; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + + uint8_t payload[8 + 130], app[256]; + CHECK_EQ_INT(tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, 123456789, payload, sizeof(payload)), 8); + for (size_t i = 8; i < sizeof(payload); i++) payload[i] = (uint8_t)i; + tai_attr_t attrs[] = { + tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, "111 1 16 16000 0 16000 20 40"), + tai_attr_strv(TAI_ATTR_EVENT_ID, "pending-event"), + }; + int alen = tai_packet_encode(TAI_VER_21, TAI_PKT_AUDIO, attrs, 2, + payload, sizeof(payload), app, sizeof(app)); + CHECK(alen > 100); + if (alen <= 100) { tai_disconnect(ctx); tai_ctx_deinit(ctx); return; } + + /* Admit one codec frame, then close admission: the Packet pauses mid-body. */ + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = 1; + pthread_mutex_unlock(&g_st.mtx); + uint16_t seq = 200; + CHECK(server_send(ctx, TAI_PKT_AUDIO, attrs, 2, payload, sizeof(payload), + seq++) > 0); + CHECK(WAIT_FOR(pending_wire_len > 0, 1000)); + + /* Corrupt the recorded wire length, then let delivery run to completion. */ + pthread_mutex_lock(&g_st.mtx); + ctx->rx_pending_wire_len = ctx->rx_len + 1; + flow_allowed_calls = 100; + pthread_mutex_unlock(&g_st.mtx); + + CHECK(WAIT_FOR(g_st.disconnect_calls >= 1, 1000)); + CHECK_EQ_INT(g_st.disconnect_reason, TAI_DISCONNECT_PROTOCOL); + CHECK_EQ_INT(g_st.disconnect_detail, TAI_PROTO_ERR_FRAME_DECODE); + tai_disconnect(ctx); + tai_ctx_deinit(ctx); +} + +static void test_pending_audio(void) +{ + for (int mode = 0; mode < 3; mode++) { + SECTION(mode == 0 ? "pending_audio_multiple_pauses" : + mode == 1 ? "pending_audio_fragmented" : "pending_audio_shutdown"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + pending_remaining = pending_wire_len = pending_cursor = 0; + pending_fragmented = mode != 0; + pending_metadata_ok = 0; + pending_audio_ok = 1; + tai_config_t options = { .pal = observe_pal(), .on_flow_control = pending_flow_control }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + ctx->on_audio = pending_audio; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(flow_paused_queries > 0, 1000)); + uint8_t payload[8 + 130], app[256]; + CHECK_EQ_INT(tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, 123456789, payload, sizeof(payload)), 8); + for (size_t i = 8; i < sizeof(payload); i++) payload[i] = (uint8_t)i; + tai_attr_t attrs[] = { + tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, "111 1 16 16000 0 16000 20 40"), + tai_attr_strv(TAI_ATTR_EVENT_ID, "pending-event"), + }; + int alen = tai_packet_encode(TAI_VER_21, TAI_PKT_AUDIO, attrs, 2, + payload, sizeof(payload), app, sizeof(app)); + CHECK(alen > 100); + if (alen <= 100) { tai_disconnect(ctx); tai_ctx_deinit(ctx); return; } + uint16_t seq = 100; + size_t expected_wire_len; + if (pending_fragmented) { + size_t cuts[] = { 50, 100, (size_t)alen }; + server_send_app_fragmented(ctx, app, (size_t)alen, cuts, 3, &seq); + expected_wire_len = (size_t)alen - 100 + 37; + } else { + expected_wire_len = server_send(ctx, TAI_PKT_AUDIO, attrs, 2, + payload, sizeof(payload), seq++); + } + CHECK(server_send_event_id(ctx, TAI_EVT_END, "later-event", seq++) > 0); + uintptr_t first_cursor = 0; + for (int allowed = 1; allowed <= (mode == 2 ? 1 : 3); allowed++) { + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = allowed; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(pending_remaining == 130 - 40 * (size_t)allowed, 1000)); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.audio_calls, allowed); + CHECK_EQ_INT(g_st.event_calls, 0); + CHECK_EQ_INT(pending_wire_len, expected_wire_len); + CHECK(pending_metadata_ok); + if (allowed == 1) first_cursor = pending_cursor; + CHECK_EQ_INT(pending_cursor - first_cursor, 40 * (allowed - 1)); + size_t reads = observed_recv_calls; + pthread_mutex_unlock(&g_st.mtx); + sleep_ms(50); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(observed_recv_calls, reads); + pthread_mutex_unlock(&g_st.mtx); + } + if (mode != 2) { + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = 100; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(g_st.event_calls == 1, 1000)); + } + tai_disconnect(ctx); + CHECK(pending_audio_ok); + CHECK_EQ_INT(g_st.audio_calls, mode == 2 ? 1 : 4); + CHECK_EQ_INT(g_st.audio_concat_len, mode == 2 ? 40 : 130); + CHECK(memcmp(g_st.audio_concat, payload + 8, g_st.audio_concat_len) == 0); + if (mode != 2) { + CHECK_EQ_INT(g_st.audio_frame_lens[3], 10); + CHECK(strcmp(g_st.last_event_event_id, "later-event") == 0); + } + CHECK_EQ_INT(ctx->rx_pending_len, 0); + CHECK_EQ_INT(ctx->rx_pending_wire_len, 0); + CHECK(ctx->rx_pending_body == NULL); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); + } +} + +static int eof_released, eof_hook_phase, eof_buffered_b; +static size_t eof_paused_queries, eof_frame_b_len; +static uint8_t eof_frame_b[256]; + +static int buffered_eof_flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)user_data; + pthread_mutex_lock(&g_st.mtx); + int ready = eof_released; + if (!ready) eof_paused_queries++; + else if (g_st.text_calls == 1 && eof_hook_phase == 0) { + /* This callback runs on the receive worker: inspecting its buffer here + * is safe, unlike sampling rx_len from the test thread. */ + eof_buffered_b = ctx->rx_len == eof_frame_b_len && + memcmp(ctx->rx_buf, eof_frame_b, eof_frame_b_len) == 0; + eof_hook_phase = 1; + tai_loopback_close_connection(); + ready = 0; + } else if (eof_hook_phase == 1) { + eof_hook_phase = 2; /* admission reopens on the very next query */ + } + pthread_mutex_unlock(&g_st.mtx); + return ready; +} + +static void test_buffered_frame_before_eof(void) +{ + SECTION("buffered_frame_before_eof"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + eof_released = eof_hook_phase = eof_buffered_b = 0; + eof_paused_queries = 0; + tai_config_t options = { .pal = observe_pal(), + .on_flow_control = buffered_eof_flow_control }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(eof_paused_queries > 0, 1000)); + + uint8_t app[128], frames[512]; + int alen = build_text_app(ctx, TAI_STREAM_ONE_SHOT, 1, "A", 1, + app, sizeof(app)); + CHECK(alen > 0); + int a = alen > 0 ? tai_frame_encode(TAI_FRAG_NONE, 10, app, (size_t)alen, + ctx->sign_key, 32, ctx->pal, frames, sizeof(frames)) : -1; + alen = build_text_app(ctx, TAI_STREAM_ONE_SHOT, 2, "B", 1, + app, sizeof(app)); + CHECK(alen > 0); + int b = alen > 0 ? tai_frame_encode(TAI_FRAG_NONE, 11, app, (size_t)alen, + ctx->sign_key, 32, ctx->pal, eof_frame_b, sizeof(eof_frame_b)) : -1; + CHECK(a > 0 && b > 0); + if (a <= 0 || b <= 0) { tai_disconnect(ctx); tai_ctx_deinit(ctx); return; } + memcpy(frames + a, eof_frame_b, (size_t)b); + pthread_mutex_lock(&g_st.mtx); + eof_frame_b_len = (size_t)b; + size_t before_bytes = observed_recv_bytes; + tai_loopback_push_recv(frames, (size_t)(a + b)); + eof_released = 1; + pthread_mutex_unlock(&g_st.mtx); + + CHECK(WAIT_FOR(g_st.disconnect_calls > 0, 2000)); + tai_disconnect(ctx); + CHECK_EQ_INT(eof_hook_phase, 2); + CHECK(eof_buffered_b); + CHECK_EQ_INT(observed_recv_bytes - before_bytes, a + b); + CHECK_EQ_INT(g_st.text_calls_at_disconnect, 2); + CHECK(strcmp(g_st.text_buf, "AB") == 0); + CHECK_EQ_INT(g_st.disconnect_calls, 1); + CHECK_EQ_INT(g_st.disconnect_reason, TAI_DISCONNECT_TRANSPORT); + CHECK_EQ_INT(g_st.disconnect_detail, TAI_TRANSPORT_EOF); + tai_ctx_deinit(ctx); +} + +static void test_partial_frame_blocking_recv(void) +{ + const size_t cuts[] = { 2, 8 }; /* incomplete 5-byte header / incomplete body */ + for (size_t i = 0; i < sizeof(cuts) / sizeof(cuts[0]); i++) { + SECTION(i == 0 ? "partial_header_blocking_recv" : "partial_body_blocking_recv"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + tai_config_t options = { .pal = observe_pal(), + .ping_interval_ms = 1000000 }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + tai_loopback_set_recv_cap_ms(AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS + 5000U); + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + uint8_t app[128], frame[256]; + int alen = build_text_app(ctx, TAI_STREAM_ONE_SHOT, 1, + "partial", 7, app, sizeof(app)); + CHECK(alen > 0); + int flen = alen > 0 ? tai_frame_encode(TAI_FRAG_NONE, 10, app, (size_t)alen, + ctx->sign_key, 32, ctx->pal, frame, sizeof(frame)) : -1; + CHECK(flen > (int)cuts[i]); + if (flen <= (int)cuts[i]) { tai_disconnect(ctx); tai_ctx_deinit(ctx); continue; } + pthread_mutex_lock(&g_st.mtx); + observed_after_bytes = observed_recv_bytes + cuts[i]; + pthread_mutex_unlock(&g_st.mtx); + tai_loopback_push_recv(frame, cuts[i]); + CHECK(WAIT_FOR(observed_recv_bytes >= observed_after_bytes, 1000)); + CHECK(WAIT_FOR(observed_blocking_after_bytes > 0, 1000)); + pthread_mutex_lock(&g_st.mtx); + CHECK(observed_nonblocking_after_bytes <= 4); + CHECK(observed_max_timeout_after_bytes <= AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS); + CHECK_EQ_INT(g_st.text_calls, 0); + pthread_mutex_unlock(&g_st.mtx); + tai_loopback_push_recv(frame + cuts[i], (size_t)flen - cuts[i]); + CHECK(WAIT_FOR(g_st.text_calls == 1, 1000)); + tai_disconnect(ctx); + CHECK(strcmp(g_st.text_buf, "partial") == 0); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); + } +} + +static void test_long_receive_pause_liveness(void) +{ + SECTION("long_receive_pause_liveness"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + const uint32_t timeout_ms = 400; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + tai_config_t options = { + .pal = observe_pal(), + .on_flow_control = test_flow_control, + .ping_interval_ms = 100, + .ping_timeout_ms = timeout_ms, + }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(flow_paused_queries > 0, 1000)); + + sleep_ms(timeout_ms + 250); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + flow_allowed_calls = 100; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(flow_resume_ms != 0, 1000)); + sleep_ms(100); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + pthread_mutex_unlock(&g_st.mtx); + + uint8_t audio[40]; + memset(audio, 0x6b, sizeof(audio)); + uint64_t sent_ms = ctx->pal->time_ms(); + CHECK(server_send_audio(ctx, TAI_STREAM_START, + "111 1 16 16000 0 16000 20 40", + audio, sizeof(audio), 10) > 0); + CHECK(WAIT_FOR(g_st.audio_calls == 1, 1000)); + CHECK(WAIT_FOR(g_st.disconnect_calls > 0, timeout_ms + 1500)); + tai_disconnect(ctx); + CHECK_EQ_INT(g_st.disconnect_reason, TAI_DISCONNECT_TRANSPORT); + CHECK_EQ_INT(g_st.disconnect_detail, TAI_TRANSPORT_PING_TIMEOUT); + CHECK(g_st.disconnect_ms >= sent_ms + timeout_ms); + tai_ctx_deinit(ctx); +} + +static void check_pal_sleep(const pal_t *pal, void *tcp) +{ + uint8_t byte = 0; + CHECK_EQ_INT(pal->tcp_poll(tcp, 1, 1000), 1); + uint64_t start = pal->time_ms(); + pal->sleep_ms(120); + uint64_t elapsed = pal->time_ms() - start; + CHECK(elapsed >= 120); + CHECK(elapsed < 2120); + CHECK_EQ_INT(pal->tcp_poll(tcp, 1, 0), 1); + CHECK_EQ_INT(pal->tcp_recv(tcp, &byte, 1, 0), 1); + CHECK_EQ_INT(byte, 0x5a); + start = pal->time_ms(); + pal->sleep_ms(0); + CHECK(pal->time_ms() - start < 1000); +} + +static void test_pal_sleep(void) +{ + SECTION("pal_sleep_required"); + pal_t incomplete = *tai_pal_loopback(); + CHECK(pal_is_valid(&incomplete)); + incomplete.sleep_ms = NULL; + CHECK(!pal_is_valid(&incomplete)); + + SECTION("pal_sleep_loopback"); + tai_loopback_reset(); + const pal_t *pal = tai_pal_loopback(); + void *tcp = pal->tcp_connect("loopback.test", 443, 1000); + const uint8_t byte = 0x5a; + tai_loopback_push_recv(&byte, 1); + check_pal_sleep(pal, tcp); + pal->tcp_close(tcp); + + SECTION("pal_sleep_posix"); + int listener = socket(AF_INET, SOCK_STREAM, 0); + CHECK(listener >= 0); + if (listener < 0) return; + struct sockaddr_in addr = {0}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + CHECK_EQ_INT(bind(listener, (struct sockaddr *)&addr, sizeof(addr)), 0); + socklen_t addr_len = sizeof(addr); + CHECK_EQ_INT(getsockname(listener, (struct sockaddr *)&addr, &addr_len), 0); + CHECK_EQ_INT(listen(listener, 1), 0); + pal = tai_pal_posix(); + tcp = pal->tcp_connect("127.0.0.1", ntohs(addr.sin_port), 1000); + CHECK(tcp != NULL); + if (tcp) { + int peer = accept(listener, NULL, NULL); + CHECK(peer >= 0); + if (peer >= 0) { + CHECK_EQ_INT(send(peer, &byte, 1, 0), 1); + check_pal_sleep(pal, tcp); + close(peer); + } + pal->tcp_close(tcp); + } + close(listener); +} + /* ========================================================================= * main * ========================================================================= */ @@ -2034,6 +2696,14 @@ int main(void) test_log_env_default(); pthread_mutex_init(&g_st.mtx, NULL); + test_event_user_data(); + test_pending_audio(); + test_pending_bad_wire_len(); + test_receive_backpressure(); + test_buffered_frame_before_eof(); + test_partial_frame_blocking_recv(); + test_long_receive_pause_liveness(); + test_pal_sleep(); test_text_query(); test_audio_roundtrip(); test_image_query(); diff --git a/pal/pal.h b/pal/pal.h index 562027c..39e7055 100644 --- a/pal/pal.h +++ b/pal/pal.h @@ -50,6 +50,8 @@ extern "C" { * designated-initializer tables plus pal_is_valid() below: * pal/pal_posix.c, pal/pal_freertos.c, * modules/rtc-tcp-client/test/tai_pal_loopback.c, and .../test/test_core.c + * (test_core intentionally leaves networking/thread/sleep callbacks NULL because + * it never passes that table through pal_is_valid()). * The host build compiles three of them -- pal_freertos.c is in no CMake * target here, only the ESP-IDF component -- and C99 zero-fills whatever a * designated initializer omits, with no -Wmissing-field-initializers @@ -126,6 +128,13 @@ typedef struct pal { int (*thread_create)(void **handle, void *(*func)(void *), void *arg); int (*thread_join)(void *handle); + /* --- Sleep ----------------------------------------------------------- + * Suspend the calling thread without busy-waiting for at least ms + * milliseconds; 0 is a no-op. Scheduler delays and tick rounding may + * extend the sleep. No socket or socket-readiness dependency. + */ + void (*sleep_ms)(uint32_t ms); + } pal_t; /* ------------------------------------------------------------------------- @@ -146,7 +155,8 @@ static inline bool pal_is_valid(const pal_t *p) return p && p->tcp_connect && p->tcp_send && p->tcp_recv && p->tcp_close && p->tcp_poll && p->time_ms && p->malloc && p->free && p->mutex_create && p->mutex_lock && p->mutex_unlock - && p->mutex_destroy && p->thread_create && p->thread_join; + && p->mutex_destroy && p->thread_create && p->thread_join + && p->sleep_ms; } #ifdef __cplusplus diff --git a/pal/pal_freertos.c b/pal/pal_freertos.c index 5057de7..2b2ce21 100644 --- a/pal/pal_freertos.c +++ b/pal/pal_freertos.c @@ -14,6 +14,7 @@ * configSUPPORT_DYNAMIC_ALLOCATION 1 (pvPortMalloc, dynamic semaphores/tasks) * INCLUDE_vTaskSuspend 1 (used by thread-join shim) * INCLUDE_vTaskDelete 1 (joiner deletes the worker task) + * INCLUDE_vTaskDelay 1 (sleep_ms blocks without busy-waiting) * * Also recommended: a FreeRTOS heap implementation that supports vPortFree * (heap_3.c / heap_4.c / heap_5.c -- NOT heap_1.c). @@ -280,6 +281,19 @@ static uint64_t pal_time_ms(void) return (uint64_t)xTaskGetTickCount() * (uint64_t)portTICK_PERIOD_MS; } +static void pal_sleep_ms(uint32_t ms) +{ + if (ms == 0) return; + + uint64_t ticks = ((uint64_t)ms * configTICK_RATE_HZ + 999U) / 1000U; + const uint64_t max_chunk = (uint64_t)portMAX_DELAY - 1U; + while (ticks > 0) { + uint64_t chunk = ticks > max_chunk ? max_chunk : ticks; + vTaskDelay((TickType_t)(chunk + 1U)); + ticks -= chunk; + } +} + /* ------------------------------------------------------------------------- * Memory * ------------------------------------------------------------------------- */ @@ -392,6 +406,7 @@ static const pal_t g_freertos_pal = { .mutex_destroy = pal_mutex_destroy, .thread_create = pal_thread_create, .thread_join = pal_thread_join, + .sleep_ms = pal_sleep_ms, }; const pal_t *tai_pal_freertos(void) diff --git a/pal/pal_posix.c b/pal/pal_posix.c index 0409cb0..fcc5bfa 100644 --- a/pal/pal_posix.c +++ b/pal/pal_posix.c @@ -242,6 +242,13 @@ static uint64_t pal_time_ms(void) return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)(ts.tv_nsec / 1000000); } +static void pal_sleep_ms(uint32_t ms) +{ + if (ms == 0) return; + struct timespec delay = { ms / 1000, (long)(ms % 1000) * 1000000L }; + while (nanosleep(&delay, &delay) < 0 && errno == EINTR) { } +} + /* ------------------------------------------------------------------------- * Memory * ------------------------------------------------------------------------- */ @@ -308,6 +315,7 @@ static const pal_t g_posix_pal = { .mutex_destroy = pal_mutex_destroy, .thread_create = pal_thread_create, .thread_join = pal_thread_join, + .sleep_ms = pal_sleep_ms, }; const pal_t *tai_pal_posix(void)