From 387957bf3dc3a3a70a14cc9619077065ca4c5e45 Mon Sep 17 00:00:00 2001 From: wangsn Date: Thu, 13 Aug 2026 16:56:08 +0800 Subject: [PATCH 01/10] feat(iot-client): add MQTT control channel for server-initiated AI interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a standalone MQTT control channel (protocol 9000) so server-initiated interrupts (asrInterrupt) arrive immediately via MQTT, independent of the TCP data channel's congestion state. Mirrors TuyaOpen's architecture: data over TCP, control over MQTT. - ai_ctrl_callback_t typedef and iot_ai_ctrl_set_callback() API in iot_client.h - iot_ai_ctrl.c: parse protocol-9000 envelopes, extract data.data.type and data.data.data, deliver to registered callback - iot_client_message.c: dispatch order is now ai_ctrl (9000) → DP (5) → raw - 9 network-free unit tests covering dispatch, passthrough, dedup, edge cases --- CMakeLists.txt | 2 + CONTEXT-MAP.md | 4 + .../components/agentic_kit/CMakeLists.txt | 1 + modules/iot-client/CONTEXT.md | 7 + modules/iot-client/include/iot_client.h | 34 ++ modules/iot-client/src/iot_ai_ctrl.c | 58 ++++ modules/iot-client/src/iot_ai_ctrl.h | 13 + modules/iot-client/src/iot_client_message.c | 3 + modules/iot-client/test/iot_ai_ctrl_test.c | 295 ++++++++++++++++++ .../iot-client/test/iot_client_message_test.c | 79 +++++ modules/rtc-tcp-client/CONTEXT.md | 8 + 11 files changed, 504 insertions(+) create mode 100644 modules/iot-client/src/iot_ai_ctrl.c create mode 100644 modules/iot-client/src/iot_ai_ctrl.h create mode 100644 modules/iot-client/test/iot_ai_ctrl_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index bc89af3..21b3926 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,6 +193,7 @@ set(IOT_CLIENT_SOURCES "${IOT_CLIENT_DIR}/src/atop_base.c" "${IOT_CLIENT_DIR}/src/cipher_wrapper.c" "${IOT_CLIENT_DIR}/src/http_client_interface.c" + "${IOT_CLIENT_DIR}/src/iot_ai_ctrl.c" "${IOT_CLIENT_DIR}/src/iot_atop.c" "${IOT_CLIENT_DIR}/src/iot_client.c" "${IOT_CLIENT_DIR}/src/iot_client_message.c" @@ -437,6 +438,7 @@ if(AGENTIC_KIT_BUILD_TESTS AND AGENTIC_KIT_ENABLE_PROJECT_TESTS) agentic_kit_add_iot_test(iot_reset_test "${IOT_CLIENT_TESTS_DIR}/iot_reset_test.c") agentic_kit_add_iot_test(iot_mqtt_test "${IOT_CLIENT_TESTS_DIR}/mqtt_test.c") agentic_kit_add_iot_test(iot_message_test "${IOT_CLIENT_TESTS_DIR}/iot_client_message_test.c") + agentic_kit_add_iot_test(iot_ai_ctrl_test "${IOT_CLIENT_TESTS_DIR}/iot_ai_ctrl_test.c") agentic_kit_add_iot_test(iot_on_boarding_test "${IOT_CLIENT_TESTS_DIR}/on_boarding_test.c") agentic_kit_add_iot_test(iot_dp_test "${IOT_CLIENT_TESTS_DIR}/iot_dp_test.c") agentic_kit_add_iot_test(iot_ota_test "${IOT_CLIENT_TESTS_DIR}/iot_ota_test.c") diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index 754f406..1ca0c91 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -18,6 +18,10 @@ and decisions. - **IoT Client → RTC TCP Client**: the IoT Client obtains an AI session token (`iot_client_get_session_token`) that the RTC client uses to open a Session. + Applications may also keep MQTT active as an independent control path: + authenticated protocol-9000 notices reach `ai_ctrl_callback_t` even while + receive backpressure stalls the RTC Connection. The application composes the + callbacks; neither client owns or calls the other. - **Tuya BLE → IoT Client**: BLE provisioning runs first and hands the device its WiFi credentials (incl. the pairing token); once on WiFi, the IoT Client uses them to activate against the cloud. diff --git a/examples/esp-idf/components/agentic_kit/CMakeLists.txt b/examples/esp-idf/components/agentic_kit/CMakeLists.txt index 7273f1e..4c04d86 100644 --- a/examples/esp-idf/components/agentic_kit/CMakeLists.txt +++ b/examples/esp-idf/components/agentic_kit/CMakeLists.txt @@ -15,6 +15,7 @@ set(AK_SRCS ${AK_DIR}/modules/iot-client/src/atop_base.c ${AK_DIR}/modules/iot-client/src/cipher_wrapper.c ${AK_DIR}/modules/iot-client/src/http_client_interface.c + ${AK_DIR}/modules/iot-client/src/iot_ai_ctrl.c ${AK_DIR}/modules/iot-client/src/iot_atop.c ${AK_DIR}/modules/iot-client/src/iot_client.c ${AK_DIR}/modules/iot-client/src/iot_client_message.c diff --git a/modules/iot-client/CONTEXT.md b/modules/iot-client/CONTEXT.md index a18a2b4..f6ffe79 100644 --- a/modules/iot-client/CONTEXT.md +++ b/modules/iot-client/CONTEXT.md @@ -43,6 +43,13 @@ _Avoid_: publish (that is the transport verb), sync, send. A cloud→device message that sets DP values. _Avoid_: command, control, write. +**AI control notice**: +An authenticated MQTT protocol-9000 business message delivered through +`ai_ctrl_callback_t`, independently of the RTC TCP Connection. The IoT Client +validates and exposes the notice; the application owns playback interruption, +Event correlation, and any TAI action. +_Avoid_: DP set, TAI Event, injected callback. + **Activation** (on-boarding): First-time provisioning that authenticates the device and returns its credentials (devid / secret_key / local_key) together with its schema and schema id. diff --git a/modules/iot-client/include/iot_client.h b/modules/iot-client/include/iot_client.h index ef2daa1..079e32c 100644 --- a/modules/iot-client/include/iot_client.h +++ b/modules/iot-client/include/iot_client.h @@ -111,6 +111,21 @@ IOT_API int iot_init(const pal_t *pal); typedef void (*iot_message_callback_t)(const char *topic, size_t topic_len, const uint8_t *data, size_t data_len); +/** + * @brief Callback for authenticated AI control messages delivered over MQTT. + * + * Fires on the thread calling iot_client_process(). The type and JSON data are + * borrowed and remain valid only for the callback duration. Keep the callback + * non-blocking and copy anything retained after it returns. + * + * @param type Control event type, such as "asrInterrupt". + * @param json_data Serialized event-specific data; not necessarily an object. + * @param data_len Length of @p json_data, excluding any terminating NUL. + * @param user_data Opaque pointer supplied at registration. + */ +typedef void (*ai_ctrl_callback_t)(const char *type, const char *json_data, + size_t data_len, void *user_data); + /** * @brief Reset type classification (mirrors TuyaOpen TUYA_RESET_TYPE_REMOTE_*). * @@ -273,6 +288,8 @@ struct iot_dp_context; void *reset_user_data; // Opaque pointer passed back to reset_callback iot_ota_confirm_callback_t ota_confirm_callback; // APP-confirmed OTA (protocol 15) callback void *ota_confirm_user_data; // Opaque pointer passed back to ota_confirm_callback + ai_ctrl_callback_t ai_ctrl_callback; // AI control (protocol 9000) callback + void *ai_ctrl_user_data; // Opaque pointer passed back to ai_ctrl_callback struct iot_dp_context *dp; // DP layer state; points into dp_storage, NULL when inactive void *dp_storage[IOT_DP_CONTEXT_STORAGE / sizeof(void *)]; // inline storage for *dp (no heap) @@ -439,6 +456,23 @@ IOT_API int iot_client_process(iot_client_t *client, uint32_t timeout_ms); */ IOT_API int iot_client_publish(iot_client_t *client, const uint8_t *data, size_t data_len); +/** + * @brief Register the MQTT protocol-9000 AI control callback. + * + * Register before the application starts pumping MQTT messages. If commands + * must be observable during initial MQTT subscription, initialize with + * mqtt_disable_auto_connect=true, register here, then call iot_client_connect(). + * Passing NULL as @p callback deregisters the current callback. + * + * @param client IoT client instance. + * @param callback Callback, or NULL to deregister. + * @param user_data Opaque pointer passed to @p callback. + * @return OPRT_OK, or OPRT_INVALID_PARAMETER if @p client is NULL. + */ +IOT_API int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, + void *user_data); + /** * @brief Get AI agent session token from Tuya cloud. * diff --git a/modules/iot-client/src/iot_ai_ctrl.c b/modules/iot-client/src/iot_ai_ctrl.c new file mode 100644 index 0000000..1b7280d --- /dev/null +++ b/modules/iot-client/src/iot_ai_ctrl.c @@ -0,0 +1,58 @@ +#include "iot_ai_ctrl.h" + +#include + +#include "cJSON.h" + +#define IOT_PROTO_AI_CONTROL 9000 + +bool iot_ai_ctrl_dispatch(iot_client_t *client, + const uint8_t *bytes, size_t len) +{ + if (!client || !client->ai_ctrl_callback || !bytes || len == 0) { + return false; + } + + cJSON *root = cJSON_ParseWithLength((const char *)bytes, len); + if (!root) return false; + + cJSON *protocol = cJSON_GetObjectItem(root, "protocol"); + if (!cJSON_IsNumber(protocol) || + protocol->valuedouble != (double)IOT_PROTO_AI_CONTROL) { + cJSON_Delete(root); + return false; + } + + cJSON *data = cJSON_GetObjectItem(root, "data"); + cJSON *inner = cJSON_IsObject(data) + ? cJSON_GetObjectItem(data, "data") : NULL; + cJSON *type = cJSON_IsObject(inner) + ? cJSON_GetObjectItem(inner, "type") : NULL; + cJSON *payload = cJSON_IsObject(inner) + ? cJSON_GetObjectItem(inner, "data") : NULL; + if (!cJSON_IsString(type)) { + cJSON_Delete(root); + return false; + } + + char *payload_json = payload ? cJSON_PrintUnformatted(payload) : NULL; + const char *payload_bytes = payload_json ? payload_json : ""; + size_t payload_len = payload_json ? strlen(payload_json) : 0; + + client->ai_ctrl_callback(type->valuestring, payload_bytes, payload_len, + client->ai_ctrl_user_data); + + if (payload_json) client->pal->free(payload_json); + cJSON_Delete(root); + return true; +} + +int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, void *user_data) +{ + if (!client) return OPRT_INVALID_PARAMETER; + + client->ai_ctrl_callback = callback; + client->ai_ctrl_user_data = user_data; + return OPRT_OK; +} diff --git a/modules/iot-client/src/iot_ai_ctrl.h b/modules/iot-client/src/iot_ai_ctrl.h new file mode 100644 index 0000000..b037e01 --- /dev/null +++ b/modules/iot-client/src/iot_ai_ctrl.h @@ -0,0 +1,13 @@ +#ifndef IOT_AI_CTRL_H +#define IOT_AI_CTRL_H + +#include +#include +#include + +#include "iot_client.h" + +bool iot_ai_ctrl_dispatch(iot_client_t *client, + const uint8_t *bytes, size_t len); + +#endif /* IOT_AI_CTRL_H */ diff --git a/modules/iot-client/src/iot_client_message.c b/modules/iot-client/src/iot_client_message.c index 453040a..7ca4144 100644 --- a/modules/iot-client/src/iot_client_message.c +++ b/modules/iot-client/src/iot_client_message.c @@ -3,6 +3,7 @@ #include "cipher_wrapper.h" #include "iot_internal.h" #include "iot_dp_internal.h" +#include "iot_ai_ctrl.h" #include "cJSON.h" #include @@ -33,6 +34,8 @@ static void mqtt_message_handler(const char *topic, size_t topic_len, /* consumed: reset notices never reach the DP layer or raw callback */ } else if (iot_client_message_handle_ota_confirm(client, decrypted, decrypted_len)) { /* consumed: APP-confirmed OTA notices never reach the DP/raw path */ + } else if (iot_ai_ctrl_dispatch(client, decrypted, decrypted_len)) { + /* consumed: AI control notices never reach the DP/raw path */ } else if (!iot_dp_dispatch_downlink(client, topic, topic_len, decrypted, decrypted_len) && client->message_callback) { client->message_callback(topic, topic_len, decrypted, decrypted_len); diff --git a/modules/iot-client/test/iot_ai_ctrl_test.c b/modules/iot-client/test/iot_ai_ctrl_test.c new file mode 100644 index 0000000..121d913 --- /dev/null +++ b/modules/iot-client/test/iot_ai_ctrl_test.c @@ -0,0 +1,295 @@ +/* + * AI control channel (MQTT protocol 9000) dispatch tests. + * + * Network-free: exercises iot_ai_ctrl_dispatch() and + * iot_ai_ctrl_set_callback() directly. + */ + +#include +#include +#include + +#include "iot_ai_ctrl.h" +#include "iot_client.h" +#include "iot_internal.h" + +static int tests_run; +static int tests_passed; + +#define RUN_TEST(fn) \ + do { \ + tests_run++; \ + printf("\n--- [%d] %s ---\n", tests_run, #fn); \ + if ((fn)() == 0) { \ + tests_passed++; \ + printf(" PASS\n"); \ + } else { \ + printf(" FAIL\n"); \ + } \ + } while (0) + +static int callback_count; +static char callback_type[64]; +static char callback_data[512]; +static void *callback_user_data; + +static void reset_callback(void) +{ + callback_count = 0; + callback_type[0] = '\0'; + callback_data[0] = '\0'; + callback_user_data = NULL; +} + +static void control_callback(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + callback_count++; + callback_user_data = user_data; + + if (type) { + snprintf(callback_type, sizeof(callback_type), "%s", type); + } + if (json_data && data_len > 0) { + size_t n = data_len < sizeof(callback_data) - 1 + ? data_len : sizeof(callback_data) - 1; + memcpy(callback_data, json_data, n); + callback_data[n] = '\0'; + } +} + +static iot_client_t *make_client(const pal_t *pal) +{ + iot_client_t *client = (iot_client_t *)pal->malloc(sizeof(*client)); + if (!client) return NULL; + memset(client, 0, sizeof(*client)); + client->pal = pal; + return client; +} + +static void destroy_client(iot_client_t *client) +{ + if (client) client->pal->free(client); +} + +static int test_asr_interrupt_dispatch(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + static char user_data; + const char *envelope = + "{\"protocol\":9000,\"t\":1234567890,\"data\":{" + "\"bizType\":\"EVENT\",\"bizId\":\"uuid-1234\",\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{" + "\"eventId\":\"evt-5678\",\"time\":\"2025-01-15T10:30:00Z\"}}}}"; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, &user_data); + + if (!iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope))) { + printf(" protocol-9000 envelope was not consumed\n"); + } else if (callback_count != 1) { + printf(" callback count=%d\n", callback_count); + } else if (strcmp(callback_type, "asrInterrupt") != 0) { + printf(" callback type=%s\n", callback_type); + } else if (!strstr(callback_data, "\"eventId\":\"evt-5678\"")) { + printf(" callback data=%s\n", callback_data); + } else if (callback_user_data != &user_data) { + printf(" callback user data mismatch\n"); + } else { + result = 0; + } + + destroy_client(client); + return result; +} + +static int test_non_9000_passthrough(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + static const char protocol_five[] = + "{\"protocol\":5,\"data\":{\"dps\":{\"1\":true}}}"; + static const char no_protocol[] = "{\"type\":\"test\"}"; + static const char garbage[] = "not json"; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + + if (iot_ai_ctrl_dispatch(client, (const uint8_t *)protocol_five, + sizeof(protocol_five) - 1) || + iot_ai_ctrl_dispatch(client, (const uint8_t *)no_protocol, + sizeof(no_protocol) - 1) || + iot_ai_ctrl_dispatch(client, (const uint8_t *)garbage, + sizeof(garbage) - 1)) { + printf(" non-control input was consumed\n"); + } else if (callback_count != 0) { + printf(" callback fired on passthrough\n"); + } else { + result = 0; + } + + destroy_client(client); + return result; +} + +static int test_no_callback_passthrough(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + int consumed; + + if (!client) return -1; + consumed = iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)); + destroy_client(client); + return consumed ? -1 : 0; +} + +static int test_missing_type_not_consumed(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"data\":{\"key\":\"value\"}}}}"; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + if (!iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)) && callback_count == 0) { + result = 0; + } + destroy_client(client); + return result; +} + +static int test_exact_length_without_nul(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + static const char json[] = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":null}}}"; + size_t len = sizeof(json) - 1; + uint8_t *bytes; + int result = -1; + + if (!client) return -1; + bytes = (uint8_t *)pal->malloc(len); + if (!bytes) { + destroy_client(client); + return -1; + } + memcpy(bytes, json, len); + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + + if (iot_ai_ctrl_dispatch(client, bytes, len) && callback_count == 1 && + strcmp(callback_type, "asrInterrupt") == 0 && + strcmp(callback_data, "null") == 0) { + result = 0; + } + + pal->free(bytes); + destroy_client(client); + return result; +} + +static int test_null_safety(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + int failed; + + if (!client) return -1; + iot_ai_ctrl_set_callback(client, control_callback, NULL); + failed = iot_ai_ctrl_dispatch(NULL, (const uint8_t *)"x", 1) || + iot_ai_ctrl_dispatch(client, NULL, 0); + destroy_client(client); + return failed ? -1 : 0; +} + +static int test_deregister(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + int consumed; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + iot_ai_ctrl_set_callback(client, NULL, NULL); + consumed = iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)); + destroy_client(client); + return consumed || callback_count != 0 ? -1 : 0; +} + +static int test_set_callback_null_client(void) +{ + return iot_ai_ctrl_set_callback(NULL, control_callback, NULL) == + OPRT_INVALID_PARAMETER ? 0 : -1; +} + +static int test_protocol_must_be_exact_integer(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *fractional = + "{\"protocol\":9000.5,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + const char *string_value = + "{\"protocol\":\"9000\",\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{}}}}"; + int failed; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + failed = iot_ai_ctrl_dispatch(client, (const uint8_t *)fractional, + strlen(fractional)) || + iot_ai_ctrl_dispatch(client, (const uint8_t *)string_value, + strlen(string_value)) || + callback_count != 0; + destroy_client(client); + return failed ? -1 : 0; +} + +int main(void) +{ + const pal_t *pal = get_default_pal(); + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + printf("========== IoT AI Control Channel Test Suite ==========\n"); + + if (iot_init(pal) != OPRT_OK) return 1; + + RUN_TEST(test_asr_interrupt_dispatch); + RUN_TEST(test_non_9000_passthrough); + RUN_TEST(test_no_callback_passthrough); + RUN_TEST(test_missing_type_not_consumed); + RUN_TEST(test_exact_length_without_nul); + RUN_TEST(test_null_safety); + RUN_TEST(test_deregister); + RUN_TEST(test_set_callback_null_client); + RUN_TEST(test_protocol_must_be_exact_integer); + + printf("\n========== Results: %d/%d passed ==========\n", + tests_passed, tests_run); + return tests_passed == tests_run ? 0 : 1; +} diff --git a/modules/iot-client/test/iot_client_message_test.c b/modules/iot-client/test/iot_client_message_test.c index 89f816d..72ac705 100644 --- a/modules/iot-client/test/iot_client_message_test.c +++ b/modules/iot-client/test/iot_client_message_test.c @@ -109,6 +109,29 @@ static volatile int ota_confirm_cb_called = 0; static int ota_confirm_cb_channel = -1; static void *ota_confirm_cb_user_data = NULL; +static volatile int ai_ctrl_cb_called = 0; +static char ai_ctrl_cb_type[64] = {0}; +static char ai_ctrl_cb_data[256] = {0}; + +static void test_ai_ctrl_callback(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + (void)user_data; + ai_ctrl_cb_called++; + snprintf(ai_ctrl_cb_type, sizeof(ai_ctrl_cb_type), "%s", type ? type : ""); + size_t n = data_len < sizeof(ai_ctrl_cb_data) - 1 + ? data_len : sizeof(ai_ctrl_cb_data) - 1; + if (json_data && n > 0) memcpy(ai_ctrl_cb_data, json_data, n); + ai_ctrl_cb_data[n] = '\0'; +} + +static void reset_ai_ctrl_callback_state(void) +{ + ai_ctrl_cb_called = 0; + ai_ctrl_cb_type[0] = '\0'; + ai_ctrl_cb_data[0] = '\0'; +} + static void test_ota_confirm_callback(int channel, void *user_data) { ota_confirm_cb_called++; @@ -1067,6 +1090,61 @@ static int test_ota_confirm_non_matching_payloads_passthrough(void) return rc; } +/* ---------- AI control dispatch on the decrypted MQTT receive path ---------- */ + +static int test_encrypted_ai_control_message(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = (iot_client_t *)pal->malloc(sizeof(*client)); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{\"eventId\":\"evt-mqtt\"}}}}"; + int result = -1; + + if (!client) return -1; + memset(client, 0, sizeof(*client)); + client->pal = pal; + strncpy(client->devid, TEST_DEVID, sizeof(client->devid) - 1); + strncpy(client->secret_key, TEST_SECRET_KEY, sizeof(client->secret_key) - 1); + strncpy(client->local_key, TEST_LOCAL_KEY, sizeof(client->local_key) - 1); + snprintf(client->mqtt_url, sizeof(client->mqtt_url), "%s", TEST_MQTT_URL); + client->cacert = g_cacert; + client->message_callback = test_message_callback; + iot_ai_ctrl_set_callback(client, test_ai_ctrl_callback, NULL); + reset_callback_state(); + reset_ai_ctrl_callback_state(); + + if (iot_client_message_connect(client) != OPRT_OK) goto out; + (void)wait_for_callback(client, 20); + reset_callback_state(); + + if (iot_client_message_publish(client, (const uint8_t *)envelope, + strlen(envelope)) != OPRT_OK) { + goto out; + } + for (int i = 0; i < 30 && ai_ctrl_cb_called == 0; i++) { + if (iot_client_message_process(client, 50) != OPRT_OK) goto out; + } + + if (ai_ctrl_cb_called != 1) { + printf(" AI control callback count=%d\n", ai_ctrl_cb_called); + } else if (strcmp(ai_ctrl_cb_type, "asrInterrupt") != 0) { + printf(" AI control type=%s\n", ai_ctrl_cb_type); + } else if (!strstr(ai_ctrl_cb_data, "\"eventId\":\"evt-mqtt\"")) { + printf(" AI control data=%s\n", ai_ctrl_cb_data); + } else if (cb_called != 0) { + printf(" raw callback received consumed control message\n"); + } else { + result = 0; + } + +out: + iot_client_message_disconnect(client); + client->cacert = NULL; + pal->free(client); + return result; +} + /* ---------- main ---------- */ int main(void) @@ -1103,6 +1181,7 @@ int main(void) /* Success tests */ RUN_TEST(test_raw_message); RUN_TEST(test_encrypted_message); + RUN_TEST(test_encrypted_ai_control_message); /* Failure tests */ RUN_TEST(test_invalid_format_message); diff --git a/modules/rtc-tcp-client/CONTEXT.md b/modules/rtc-tcp-client/CONTEXT.md index 9ef2d42..f203c09 100644 --- a/modules/rtc-tcp-client/CONTEXT.md +++ b/modules/rtc-tcp-client/CONTEXT.md @@ -89,6 +89,14 @@ uplink open. The current cloud no longer sends `TAI_EVT_SERVER_VAD`; a device mu an inbound ChatBreak as the turn boundary. _Avoid_: cancel, stop, abort. +**Server-initiated interrupt**: +An application concern delivered independently from the media Connection: either an inbound +ChatBreak Event on this context or an authenticated MQTT protocol-9000 AI control notice from +the IoT Client. Applications route both paths into one playback policy; this module does not +receive or inject the MQTT notice. The independent path remains available while receive +backpressure intentionally stops this Connection. +_Avoid_: MQTT Event, TAI injection, acknowledgement. + **Server VAD**: A server-sent Event (`TAI_EVT_SERVER_VAD`) signalling that voice-activity detection found the end of the user's speech in audio mode. **Legacy**: the current cloud signals the From 6b3b2bfcb6d9ab36043a1439f1b3244cb5e7193f Mon Sep 17 00:00:00 2001 From: wangsn Date: Wed, 9 Sep 2026 19:54:56 +0800 Subject: [PATCH 02/10] feat(rtc-tcp-client): add TCP backpressure via on_flow_control hook The worker calls the app's on_flow_control hook before each recv and between buffered frames; when it returns 0 the read is skipped so the lwIP receive window closes and stalls the sender. Buffered frames resume without requiring new network data. Add worker_yield (zero-event tcp_poll) so single-core targets (ESP32-C3) cannot starve the IDLE task and trip the task watchdog under a TTS flood. New knobs: TAI_FLOW_CONTROL_POLL_MS and TAI_WORKER_YIELD_MS. Integration test covers per-packet pause, resume without new bytes, and disconnect while paused. --- CHANGELOG.md | 2 + .../include/tai_config_defaults.h | 10 ++++ modules/rtc-tcp-client/include/tuya_ai.h | 12 +++++ modules/rtc-tcp-client/src/tai_client.c | 32 +++++++++++- modules/rtc-tcp-client/src/tai_internal.h | 8 ++- .../rtc-tcp-client/test/test_integration.c | 52 +++++++++++++++++++ 6 files changed, 113 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc0844c..a6c5f90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- rtc-tcp-client — enforce receive backpressure between buffered Frames and drain reads; resume buffered data without another network read (PR pending). + - tuya-ble — accept incoming Trsmitr versions >= 2 for app Pairing compatibility while keeping TX at version 4 (#39). - Docs — the pair-by-ble callback sample no longer claims credentials are never logged; the demo's DEBUG protocol log prints the credential JSON on purpose, for device bring-up. diff --git a/modules/rtc-tcp-client/include/tai_config_defaults.h b/modules/rtc-tcp-client/include/tai_config_defaults.h index 57f1e09..6fbf281 100644 --- a/modules/rtc-tcp-client/include/tai_config_defaults.h +++ b/modules/rtc-tcp-client/include/tai_config_defaults.h @@ -130,4 +130,14 @@ #define AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS 2000U #endif +/* Delay between admission checks while on_flow_control pauses all receives. */ +#ifndef AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS +#define AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS 50U +#endif + +/* CPU yield between receive-worker drain passes under sustained traffic. */ +#ifndef AGENTIC_KIT_TAI_WORKER_YIELD_MS +#define AGENTIC_KIT_TAI_WORKER_YIELD_MS 10U +#endif + #endif /* AGENTIC_KIT_TAI_CONFIG_DEFAULTS_H */ diff --git a/modules/rtc-tcp-client/include/tuya_ai.h b/modules/rtc-tcp-client/include/tuya_ai.h index 6d0451c..dd1a29b 100644 --- a/modules/rtc-tcp-client/include/tuya_ai.h +++ b/modules/rtc-tcp-client/include/tuya_ai.h @@ -328,6 +328,18 @@ typedef struct tai_config { void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *user_data); void *user_data; + /* Optional TCP receive backpressure. Called on the worker thread before + * each receive and between complete buffered Frames, but never during the + * synchronous connect handshake. Return nonzero to admit another Frame or + * read; return 0 to pause parsing and skip the read, allowing the TCP receive + * window to close. This pauses all inbound traffic, including ChatBreak, + * text, Pong, and EOF detection. The hook must return promptly. + * + * Admission is at wire Frame boundaries. One Audio Packet may emit several + * codec-frame callbacks, so applications must also bound their queue inside + * on_audio. NULL means receive continuously. */ + int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); + } tai_config_t; /* ========================================================================= diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index 67517fc..7dbcab0 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -341,6 +341,7 @@ tai_ctx_t *tai_ctx_init(void *mem, const tai_config_t *cfg) ctx->on_image = cfg->on_image; ctx->on_event = cfg->on_event; ctx->on_disconnect = cfg->on_disconnect; + ctx->on_flow_control = cfg->on_flow_control; ctx->user_data = cfg->user_data; ctx->ping_interval_ms = cfg->ping_interval_ms ? cfg->ping_interval_ms : 60000U; @@ -723,6 +724,13 @@ static int tai_process_rx(tai_ctx_t *ctx) * by dropping bytes/frames, so we RETURN the cause and let the worker tear * the connection down (the app reconnects). */ while (ctx->rx_len >= 5) { + /* One receive may contain several complete Frames. Stop before the next + * one when the consumer fills while dispatching the previous Frame. */ + if (!ctx->connecting && ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data)) { + break; + } + /* Detect version from first byte */ int ver = tai_frame_detect_version(ctx->rx_buf[0]); if (ver < 0) { @@ -1238,6 +1246,12 @@ static void *worker_thread(void *arg) tai_ctx_t *ctx = (tai_ctx_t *)arg; TAI_LOGD(ctx->pal, TAG, "worker: started"); + /* Before pal_t gained a sleep callback, a zero-event poll was the portable + * timed yield. The hardening follow-up replaces this with pal->sleep_ms. */ + void *yield_tcp = ctx->tls ? tls_get_tcp_handle(ctx->tls) : ctx->raw_tcp; +#define worker_yield(ms) \ + do { if (yield_tcp) ctx->pal->tcp_poll(yield_tcp, 0, (ms)); } while (0) + /* The worker owns the disconnect decision: lower layers RETURN a fatal * cause, transport faults are detected here, and we fire on_disconnect once * on exit. f_reason==0xFF means a clean stop (tai_disconnect / request). */ @@ -1293,8 +1307,17 @@ static void *worker_thread(void *arg) * ~AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS instead of waiting out a whole ping interval. */ if (wait_ms > AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS) wait_ms = AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS; + /* Pausing skips the read so the receive window can close. The worker + * continues housekeeping and checks admission on a bounded cadence. */ + if (ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data)) { + worker_yield(AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS); + continue; + } + uint64_t drain_start = ctx->pal->time_ms(); - int n = tai_recv_data(ctx, wait_ms); + /* Resume complete buffered Frames before waiting for more network data. */ + int n = ctx->rx_len ? (int)ctx->rx_len : tai_recv_data(ctx, wait_ms); while (n > 0 && ctx->running) { int fatal = tai_process_rx(ctx); if (fatal != TAI_OK) { @@ -1312,6 +1335,11 @@ static void *worker_thread(void *arg) * the next pass. */ if (ctx->pal->time_ms() - drain_start > AGENTIC_KIT_TAI_DRAIN_BUDGET_MS) break; + if (ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data)) { + break; + } + worker_yield(AGENTIC_KIT_TAI_WORKER_YIELD_MS); n = tai_recv_data(ctx, 0); /* drain remainder non-blocking */ } if (f_reason != 0xFF) break; /* fail-fast / CONNECTION_CLOSE during drain */ @@ -1331,6 +1359,7 @@ static void *worker_thread(void *arg) /* n > 0: budget yield (running still set) -> loop, keep housekeeping. * n == TAI_ERR_AGAIN: wait timed out (ping due) or drain finished. * running cleared with no fatal: a clean tai_request_disconnect. */ + worker_yield(AGENTIC_KIT_TAI_WORKER_YIELD_MS); } /* Single-point disconnect: fire on_disconnect exactly once if the worker @@ -1344,5 +1373,6 @@ static void *worker_thread(void *arg) } TAI_LOGD(ctx->pal, TAG, "worker: exiting"); +#undef worker_yield return NULL; } diff --git a/modules/rtc-tcp-client/src/tai_internal.h b/modules/rtc-tcp-client/src/tai_internal.h index ad223af..1f96242 100644 --- a/modules/rtc-tcp-client/src/tai_internal.h +++ b/modules/rtc-tcp-client/src/tai_internal.h @@ -72,8 +72,9 @@ * Buffer sizes — defaults & docs: include/tai_config_defaults.h * (AGENTIC_KIT_TAI_MAX_FRAGMENT_PAYLOAD, AGENTIC_KIT_TAI_FRAG_BUF_SIZE, AGENTIC_KIT_TAI_TX_HDR_BUF_SIZE, * AGENTIC_KIT_TAI_FRAME_COALESCE_LIMIT, AGENTIC_KIT_TAI_TX_CTRL_BUF_SIZE, AGENTIC_KIT_TAI_MAX_ATTRS, - * AGENTIC_KIT_TAI_DRAIN_BUDGET_MS, AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS; reduce for - * memory-constrained targets, e.g. ESP32 without PSRAM.) + * AGENTIC_KIT_TAI_DRAIN_BUDGET_MS, AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS, + * AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS, AGENTIC_KIT_TAI_WORKER_YIELD_MS; + * reduce for memory-constrained targets, e.g. ESP32 without PSRAM.) * ========================================================================= */ /* RX sliding-window buffer. Sized to EXACTLY one maximum wire frame — @@ -241,6 +242,9 @@ struct tai_ctx { void (*on_disconnect)(tai_ctx_t *, const tai_disconnect_msg_t *, void *); void *user_data; + /* Optional TCP receive backpressure (see tuya_ai.h). */ + int (*on_flow_control)(tai_ctx_t *, void *); + /* RX linear buffer (sliding-window: bytes always at buf[0]) */ uint8_t rx_buf[TAI_RX_BUF_SIZE]; size_t rx_len; diff --git a/modules/rtc-tcp-client/test/test_integration.c b/modules/rtc-tcp-client/test/test_integration.c index 4380963..1d1d864 100644 --- a/modules/rtc-tcp-client/test/test_integration.c +++ b/modules/rtc-tcp-client/test/test_integration.c @@ -2022,6 +2022,57 @@ static void test_disconnect_latency(void) tai_ctx_deinit(ctx); } +/* Pause after each delivered Packet, including Packets buffered by one recv. + * Resume without new network bytes to prove rx_buf work cannot be stranded. */ +static int flow_allowed_calls; + +static int test_flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)ctx; + (void)user_data; + pthread_mutex_lock(&g_st.mtx); + int ready = g_st.audio_calls < flow_allowed_calls; + pthread_mutex_unlock(&g_st.mtx); + return ready; +} + +static void test_receive_backpressure(void) +{ + SECTION("receive_backpressure"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + tai_ctx_t *ctx = setup_ctx(ctx_mem); + CHECK(ctx != NULL); + flow_allowed_calls = 0; + ctx->on_flow_control = test_flow_control; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + + uint8_t audio[800]; + memset(audio, 0x5a, sizeof(audio)); + for (int i = 0; i < 40; i++) { + CHECK(server_send_audio(ctx, + i == 0 ? TAI_STREAM_START : TAI_STREAM_MIDDLE, + i == 0 ? "111 1 16 16000" : NULL, + audio, sizeof(audio), (uint16_t)(100 + i)) > 0); + } + + for (int allowed = 1; allowed <= 40; allowed++) { + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = allowed; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(g_st.audio_calls >= allowed, 1000)); + sleep_ms(10); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.audio_calls, allowed); + pthread_mutex_unlock(&g_st.mtx); + } + + tai_disconnect(ctx); + CHECK_EQ_INT(g_st.audio_calls, 40); + CHECK_EQ_INT(g_st.audio_bytes, 40 * sizeof(audio)); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); +} + /* ========================================================================= * main * ========================================================================= */ @@ -2034,6 +2085,7 @@ int main(void) test_log_env_default(); pthread_mutex_init(&g_st.mtx, NULL); + test_receive_backpressure(); test_text_query(); test_audio_roundtrip(); test_image_query(); From b616a3ed3d1c8166bfc526dec3fc4fa0118856cc Mon Sep 17 00:00:00 2001 From: wangsn Date: Sat, 12 Sep 2026 20:14:02 +0800 Subject: [PATCH 03/10] fix(rtc-tcp-client): harden receive backpressure and add pal_t.sleep_ms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A paused worker could still be disconnected by the liveness deadline, EOF could drop buffered complete Frames, a partial frame fell into a busy poll, and the drain rechecked admission after the read instead of before it. Pauses now suspend receive liveness (resume grants a fresh ping_timeout_ms budget), buffered Frames drain before the next read or EOF check, and partial input returns to bounded blocking receive. Add mandatory pal_t.sleep_ms for backpressure waits and CPU yields, replacing the zero-event tcp_poll trick (POSIX/FreeRTOS nanosleep / vTaskDelay with tick-rounding chunking). pal_is_valid requires it, so custom PAL ports must supply it and all consumers must rebuild — pal_t grows. Worker drain-pass yields require received or buffered input. Idle receive timeouts return directly to housekeeping and the next blocking receive, avoiding the extra AGENTIC_KIT_TAI_WORKER_YIELD_MS delay. New integration coverage: no reads while paused, buffered Frame delivered before EOF, partial header/body blocking recv, liveness across a long pause, and PAL sleep behavior on loopback and POSIX. Verified against 15/15 ctest suites (tai_integration_tests 24 cases passing). Co-Authored-By: Claude Code Co-Authored-By: OpenAI Codex --- CHANGELOG.md | 3 +- docs-site/docs/reference/rtc-tcp-client.md | 15 +- .../current/reference/rtc-tcp-client.md | 15 +- modules/rtc-tcp-client/CONTEXT.md | 17 +- modules/rtc-tcp-client/include/tuya_ai.h | 5 +- modules/rtc-tcp-client/src/tai_client.c | 83 ++++--- .../rtc-tcp-client/test/tai_pal_loopback.c | 7 +- modules/rtc-tcp-client/test/test_core.c | 1 + .../rtc-tcp-client/test/test_integration.c | 206 +++++++++++++++++- pal/pal.h | 12 +- pal/pal_freertos.c | 15 ++ pal/pal_posix.c | 8 + 12 files changed, 341 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c5f90..76b23aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- rtc-tcp-client — enforce receive backpressure between buffered Frames and drain reads; resume buffered data without another network read (PR pending). +- rtc-tcp-client — enforce receive backpressure before reads and between Frames; resume buffered Frames first with a fresh liveness budget and timed CPU yields (PR pending). + - Custom PAL ports must supply the new mandatory `pal_t.sleep_ms` callback (IoT and TAI); rebuild all consumers because `pal_t` grows. `tcp_poll` timeout semantics are unchanged. - tuya-ble — accept incoming Trsmitr versions >= 2 for app Pairing compatibility while keeping TX at version 4 (#39). - Docs — the pair-by-ble callback sample no longer claims credentials are never logged; the diff --git a/docs-site/docs/reference/rtc-tcp-client.md b/docs-site/docs/reference/rtc-tcp-client.md index 2746456..cbc97bc 100644 --- a/docs-site/docs/reference/rtc-tcp-client.md +++ b/docs-site/docs/reference/rtc-tcp-client.md @@ -178,7 +178,7 @@ sidebar_position: 1 | 字段 | 类型 | 说明 | |------|------|------| | `ping_interval_ms` | `uint32_t` | Ping 间隔（0 = 默认 60000ms） | -| `ping_timeout_ms` | `uint32_t` | Ping 超时（0 = 默认 90000ms） | +| `ping_timeout_ms` | `uint32_t` | 接收存活超时，任意入站数据均刷新（0 = 默认 90000ms）；主动背压暂停期间不计超时，恢复时获得完整的新预算 | | `connect_timeout_ms` | `uint32_t` | 连接超时（0 = 默认 5000ms）。分别约束 `tai_connect` 的两个串行等待阶段：先是连接建立（TCP 建连 + TLS 握手，共用一份预算），再是服务端 SessionNew 应答。任一阶段超时即判定连接失败，因此 `tai_connect` 最坏耗时约为该值的 2 倍。 | ### 3.7 测试配置 {#37-测试配置} @@ -205,6 +205,7 @@ sidebar_position: 1 | `on_image` | function pointer | 图像数据回调（云端生成的图片） | | `on_event` | function pointer | 事件回调（MCP、打断、VAD 等） | | `on_disconnect` | function pointer | 断连回调 | +| `on_flow_control` | `int (*)(tai_ctx_t *, void *)` | 可选接收背压钩子；非零允许接收，0 暂停读取与解析，NULL 不启用背压 | | `user_data` | `void *` | 透传到所有回调 | **回调签名：** @@ -217,8 +218,20 @@ void (*on_text) (tai_ctx_t *ctx, const tai_text_msg_t *msg, void *use void (*on_image) (tai_ctx_t *ctx, const tai_image_msg_t *msg, void *user_data); void (*on_event) (tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data); void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *user_data); +int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); ``` +#### 接收背压（`on_flow_control`） + +- worker 在每次读取前及完整 Frame 之间调用钩子，握手阶段不调用。钩子必须非阻塞；应用负责同步共享的队列状态。 +- 返回 0 会暂停读取和解析，所有入站流量都会停滞，包括 ChatBreak、ASR 文本、Pong 和 EOF 检测。恢复后先处理已缓冲的完整 Frame，再读取；不完整输入回到有上限的阻塞接收。 +- 主动暂停挂起接收存活超时；恢复时获得新的完整 `ping_timeout_ms` 预算。Ping 和停止请求仍执行，Ping 发送失败仍断连。 +- 背压以线协议 Frame 为边界。一个 Audio Packet 可能同步触发多个编解码帧回调，`on_audio` 仍须限制自身入队量。 + +默认每 `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS`（50 ms）重查背压，并以 `AGENTIC_KIT_TAI_WORKER_YIELD_MS`（10 ms）在持续流量下让出 CPU。 + +`pal_t` 新增必填的 `sleep_ms` 回调。它必须非忙等地休眠至少请求的毫秒数，0 为 no-op，且不能依赖 socket 可读性。所有自定义 PAL 和使用方必须用新头文件重新编译。 + ### 接收消息结构体 {#接收消息结构体} `tai_audio_msg_t`（音频回调）： diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md index 6850265..edd3f2e 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md @@ -178,7 +178,7 @@ Populate this structure before calling `tai_ctx_init`. All pointer fields must r | Field | Type | Description | |------|------|------| | `ping_interval_ms` | `uint32_t` | Ping interval (0 = default 60000ms) | -| `ping_timeout_ms` | `uint32_t` | Ping timeout (0 = default 90000ms) | +| `ping_timeout_ms` | `uint32_t` | Receive-liveness timeout refreshed by any inbound data (0 = default 90000ms); intentional backpressure suspends it and resume grants a fresh full budget | | `connect_timeout_ms` | `uint32_t` | Connection timeout (0 = default 5000ms). It separately constrains the two sequential wait stages of `tai_connect`: first Connection establishment (TCP connection + TLS handshake, sharing one budget), then the server's SessionNew response. A timeout in either stage means the Connection failed, so the worst-case duration of `tai_connect` is approximately twice this value. | ### 3.7 Test Configuration {#37-测试配置} @@ -205,6 +205,7 @@ All callbacks are invoked on the background receive thread. | `on_image` | function pointer | Image data callback (images generated by the cloud) | | `on_event` | function pointer | Event callback (MCP commands, Chat break, VAD, and so on) | | `on_disconnect` | function pointer | Disconnect callback | +| `on_flow_control` | `int (*)(tai_ctx_t *, void *)` | Optional receive-backpressure hook; nonzero admits receive, 0 pauses reads and parsing, NULL disables backpressure | | `user_data` | `void *` | Passed through to all callbacks | **Callback signatures:** @@ -217,8 +218,20 @@ void (*on_text) (tai_ctx_t *ctx, const tai_text_msg_t *msg, void *use void (*on_image) (tai_ctx_t *ctx, const tai_image_msg_t *msg, void *user_data); void (*on_event) (tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data); void (*on_disconnect)(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void *user_data); +int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); ``` +#### Receive Backpressure (`on_flow_control`) + +- The worker calls the hook before each read and between complete Frames, but not during the connect handshake. The hook must not block; the application synchronizes shared queue state. +- Returning 0 pauses both reads and parsing, stalling all inbound traffic including ChatBreak, ASR text, Pong, and EOF detection. Resume processes complete buffered Frames before reading again; partial input returns to bounded blocking receive. +- An intentional pause suspends receive-liveness timeout accounting, and resume grants a fresh `ping_timeout_ms` budget. Ping and stop requests remain active; Ping send failure still disconnects. +- Admission is at wire-Frame boundaries. One Audio Packet can synchronously emit several codec-frame callbacks, so `on_audio` must still bound its own queue. + +Backpressure is rechecked every `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS` (50 ms by default), and sustained receive traffic yields CPU for `AGENTIC_KIT_TAI_WORKER_YIELD_MS` (10 ms by default). + +`pal_t` adds a mandatory `sleep_ms` callback. It must sleep without busy-waiting for at least the requested milliseconds, treat 0 as a no-op, and not depend on socket readability. Recompile all custom PALs and consumers against the updated header. + ### Received Message Structures {#接收消息结构体} `tai_audio_msg_t` (audio callback): diff --git a/modules/rtc-tcp-client/CONTEXT.md b/modules/rtc-tcp-client/CONTEXT.md index f203c09..0786107 100644 --- a/modules/rtc-tcp-client/CONTEXT.md +++ b/modules/rtc-tcp-client/CONTEXT.md @@ -77,9 +77,18 @@ The liveness exchange the background thread runs: it sends a Ping every `ping_in (default 60 s) and treats the Connection as dead if no inbound traffic — a Pong *or any* received data — arrives within `ping_timeout_ms` (default 90 s), then fires `on_disconnect`. Counting any receive, not just Pong, keeps a long downstream stream from tripping a spurious -timeout. +timeout. Intentional receive backpressure suspends only receive liveness; resuming grants a +fresh timeout budget without counting as received traffic. Pings and requested stop remain +active; a Ping send failure still disconnects, and the peer may enforce its own timeout. _Avoid_: heartbeat, poll. +**Receive backpressure**: +Application-controlled admission that pauses all inbound traffic, including buffered Frames, +ChatBreak, ASR text, Pong and EOF detection, rather than only media delivery. Admission is at +wire Frame boundaries, not codec-frame boundaries: one dispatched Packet can deliver multiple +codec frames, so the application must also bound admission in its audio callback. +_Avoid_: audio mute, selective pause, codec-frame flow control. + **Chat break**: A client-sent Event (`TAI_EVT_CHAT_BREAK`) that interrupts the server's in-progress response. Sent standalone, not part of an Event's normal lifecycle. The server also sends @@ -202,8 +211,10 @@ The worker loops: check the liveness deadline, send a Ping when due, then block `tai_recv_data` until bytes arrive or the next Ping falls due, then drain. The drain is time-bounded (`AGENTIC_KIT_TAI_DRAIN_BUDGET_MS`, default 150 ms) so a sustained downstream flood cannot starve the Ping / liveness / shutdown checks — leftover bytes wait for the next pass; and any -successful receive refreshes the liveness clock. Bytes accumulate in a sliding receive buffer; -EOF or a transport error makes the worker fire `on_disconnect`. +successful receive refreshes the liveness clock. Receive backpressure pauses both reads and +parsing; on resume, buffered complete Frames precede the next read or EOF detection, while +partial input returns to bounded blocking reception. Bytes accumulate in a sliding receive +buffer; detected EOF or a transport error makes the worker fire `on_disconnect`. `tai_process_rx` peels complete Frames off the front of that buffer: diff --git a/modules/rtc-tcp-client/include/tuya_ai.h b/modules/rtc-tcp-client/include/tuya_ai.h index dd1a29b..4c61660 100644 --- a/modules/rtc-tcp-client/include/tuya_ai.h +++ b/modules/rtc-tcp-client/include/tuya_ai.h @@ -333,7 +333,10 @@ typedef struct tai_config { * synchronous connect handshake. Return nonzero to admit another Frame or * read; return 0 to pause parsing and skip the read, allowing the TCP receive * window to close. This pauses all inbound traffic, including ChatBreak, - * text, Pong, and EOF detection. The hook must return promptly. + * text, Pong, and EOF detection. Receive-liveness timeout accounting is + * suspended during the pause; reopening admission grants a fresh + * ping_timeout_ms budget. Pings and requested shutdown remain active, and a + * Ping send failure still disconnects. The hook must return promptly. * * Admission is at wire Frame boundaries. One Audio Packet may emit several * codec-frame callbacks, so applications must also bound their queue inside diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index 7dbcab0..e06b4a1 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -404,7 +404,7 @@ void tai_ctx_deinit(tai_ctx_t *ctx) /* Forward declarations (defined below; used by the confirmed-connect wait) */ static void *worker_thread(void *arg); static int tai_recv_data(tai_ctx_t *ctx, uint32_t timeout_ms); -static int tai_process_rx(tai_ctx_t *ctx); +static int tai_process_rx(tai_ctx_t *ctx, int *paused); int tai_connect(tai_ctx_t *ctx) { @@ -537,7 +537,7 @@ int tai_connect(tai_ctx_t *ctx) tai_disconnect(ctx); return TAI_ERR_NET; } if (n > 0) { - int fatal = tai_process_rx(ctx); + int fatal = tai_process_rx(ctx, NULL); if (fatal != TAI_OK) { TAI_LOGE(ctx->pal, TAG, "connect: protocol error before ack (%d)", fatal); tai_disconnect(ctx); return TAI_ERR_PROTO; @@ -715,10 +715,14 @@ static int tai_recv_data(tai_ctx_t *ctx, uint32_t timeout_ms) * Returns a fail-fast cause for the worker: TAI_OK (no fatal; processed zero or * more frames), a TAI_PROTO_ERR_* detail on a protocol error, or * TAI_RX_PEER_CLOSE|code on a server CONNECTION_CLOSE. Never touches the - * connection lifecycle itself — the worker owns that. + * connection lifecycle itself — the worker owns that. Sets @p paused when + * admission closes between complete Frames; NULL disables flow control during + * the synchronous connect handshake. * ========================================================================= */ -static int tai_process_rx(tai_ctx_t *ctx) +static int tai_process_rx(tai_ctx_t *ctx, int *paused) { + if (paused) *paused = 0; + /* Process all complete frames sitting in rx_buf. Any structural error is * fail-fast: on a reliable, ordered TLS stream a desync cannot be recovered * by dropping bytes/frames, so we RETURN the cause and let the worker tear @@ -726,8 +730,9 @@ static int tai_process_rx(tai_ctx_t *ctx) while (ctx->rx_len >= 5) { /* One receive may contain several complete Frames. Stop before the next * one when the consumer fills while dispatching the previous Frame. */ - if (!ctx->connecting && ctx->on_flow_control && + if (paused && ctx->on_flow_control && !ctx->on_flow_control(ctx, ctx->user_data)) { + *paused = 1; break; } @@ -1246,11 +1251,9 @@ static void *worker_thread(void *arg) tai_ctx_t *ctx = (tai_ctx_t *)arg; TAI_LOGD(ctx->pal, TAG, "worker: started"); - /* Before pal_t gained a sleep callback, a zero-event poll was the portable - * timed yield. The hardening follow-up replaces this with pal->sleep_ms. */ - void *yield_tcp = ctx->tls ? tls_get_tcp_handle(ctx->tls) : ctx->raw_tcp; -#define worker_yield(ms) \ - do { if (yield_tcp) ctx->pal->tcp_poll(yield_tcp, 0, (ms)); } while (0) + int paused = 0; + int resume_buffered = 0; + uint64_t last_resume_ms = 0; /* The worker owns the disconnect decision: lower layers RETURN a fatal * cause, transport faults are detected here, and we fire on_disconnect once @@ -1260,14 +1263,21 @@ static void *worker_thread(void *arg) uint16_t f_code = 0; while (ctx->running) { + int was_paused = paused; + paused = ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data); + if (!ctx->running) break; + uint64_t now = ctx->pal->time_ms(); + if (was_paused && !paused) last_resume_ms = now; - /* Liveness timeout: any inbound traffic counts as alive, not just - * PONGs. A busy downstream stream proves the link is up even while a - * ping is overdue, so we never kill an actively-receiving connection. */ + /* Any inbound traffic proves liveness. An intentional receive pause + * suspends this deadline; reopening admission grants a fresh budget + * without pretending any bytes were received. */ uint64_t last_alive = (ctx->last_rx_ms > ctx->last_pong_ms) ? ctx->last_rx_ms : ctx->last_pong_ms; - if (now - last_alive > ctx->ping_timeout_ms) { + if (last_resume_ms > last_alive) last_alive = last_resume_ms; + if (!paused && now - last_alive > ctx->ping_timeout_ms) { TAI_LOGW(ctx->pal, TAG, "worker: liveness timeout (%llu ms idle)", (unsigned long long)(now - last_alive)); f_reason = TAI_DISCONNECT_TRANSPORT; @@ -1307,19 +1317,24 @@ static void *worker_thread(void *arg) * ~AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS instead of waiting out a whole ping interval. */ if (wait_ms > AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS) wait_ms = AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS; - /* Pausing skips the read so the receive window can close. The worker - * continues housekeeping and checks admission on a bounded cadence. */ - if (ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data)) { - worker_yield(AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS); + /* Pausing skips the read so the receive window can close. Keep Ping and + * shutdown housekeeping active while admission is closed. */ + if (paused) { + ctx->pal->sleep_ms(AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS); continue; } uint64_t drain_start = ctx->pal->time_ms(); - /* Resume complete buffered Frames before waiting for more network data. */ - int n = ctx->rx_len ? (int)ctx->rx_len : tai_recv_data(ctx, wait_ms); + /* Only a parser pause leaves complete work behind. Partial input must + * return to a bounded blocking receive rather than spin. */ + int n = resume_buffered ? (int)ctx->rx_len : tai_recv_data(ctx, wait_ms); + resume_buffered = 0; + /* Set when this pass drained real input; gates the end-of-pass yield + * below so an idle pass (blocking wait timed out, no data) does not + * gain fixed latency on top of it. */ + int did_work = (n > 0); while (n > 0 && ctx->running) { - int fatal = tai_process_rx(ctx); + int fatal = tai_process_rx(ctx, &paused); if (fatal != TAI_OK) { if (fatal & TAI_RX_PEER_CLOSE) { f_reason = TAI_DISCONNECT_CONNECTION_CLOSE; @@ -1330,16 +1345,22 @@ static void *worker_thread(void *arg) } break; } + if (paused) { + /* Do not read past the buffered Frame that closed admission. */ + resume_buffered = 1; + break; + } + if (!ctx->running) break; /* Bound the greedy drain so periodic ping / liveness / shutdown * checks run even under a sustained flood; leftover bytes wait for * the next pass. */ if (ctx->pal->time_ms() - drain_start > AGENTIC_KIT_TAI_DRAIN_BUDGET_MS) break; - if (ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data)) { - break; - } - worker_yield(AGENTIC_KIT_TAI_WORKER_YIELD_MS); + ctx->pal->sleep_ms(AGENTIC_KIT_TAI_WORKER_YIELD_MS); + /* Admission is checked immediately before the next drain read. */ + paused = ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data); + if (paused || !ctx->running) break; n = tai_recv_data(ctx, 0); /* drain remainder non-blocking */ } if (f_reason != 0xFF) break; /* fail-fast / CONNECTION_CLOSE during drain */ @@ -1358,8 +1379,11 @@ static void *worker_thread(void *arg) } /* n > 0: budget yield (running still set) -> loop, keep housekeeping. * n == TAI_ERR_AGAIN: wait timed out (ping due) or drain finished. - * running cleared with no fatal: a clean tai_request_disconnect. */ - worker_yield(AGENTIC_KIT_TAI_WORKER_YIELD_MS); + * running cleared with no fatal: a clean tai_request_disconnect. + * The yield applies only between drain passes that did work; an idle + * pass must not add fixed latency on top of its blocking wait. */ + if (did_work) + ctx->pal->sleep_ms(AGENTIC_KIT_TAI_WORKER_YIELD_MS); } /* Single-point disconnect: fire on_disconnect exactly once if the worker @@ -1373,6 +1397,5 @@ static void *worker_thread(void *arg) } TAI_LOGD(ctx->pal, TAG, "worker: exiting"); -#undef worker_yield return NULL; } diff --git a/modules/rtc-tcp-client/test/tai_pal_loopback.c b/modules/rtc-tcp-client/test/tai_pal_loopback.c index 521c1d5..573f870 100644 --- a/modules/rtc-tcp-client/test/tai_pal_loopback.c +++ b/modules/rtc-tcp-client/test/tai_pal_loopback.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "log.h" #include "../src/tai_internal.h" /* frame/packet codec + key derivation for the handshake mock */ @@ -375,13 +376,14 @@ static int lb_tcp_send(void *tcp, const uint8_t *buf, size_t len, return (int)len; /* PAL contract: >0 = bytes written */ } -/* Sleep helper for timeout emulation. */ +/* Real CPU sleep for timeout emulation; never advances the virtual clock. */ static void lb_sleep_ms(uint32_t ms) { + if (ms == 0) return; struct timespec ts; ts.tv_sec = ms / 1000; ts.tv_nsec = (long)(ms % 1000) * 1000000L; - nanosleep(&ts, NULL); + while (nanosleep(&ts, &ts) < 0 && errno == EINTR) { } } static uint64_t lb_time_ms(void); /* defined below; used for a real-time recv timeout */ @@ -513,6 +515,7 @@ static const pal_t g_loopback_pal = { .mutex_destroy = lb_mutex_destroy, .thread_create = lb_thread_create, .thread_join = lb_thread_join, + .sleep_ms = lb_sleep_ms, }; const pal_t *tai_pal_loopback(void) diff --git a/modules/rtc-tcp-client/test/test_core.c b/modules/rtc-tcp-client/test/test_core.c index b80af0f..475710e 100644 --- a/modules/rtc-tcp-client/test/test_core.c +++ b/modules/rtc-tcp-client/test/test_core.c @@ -49,6 +49,7 @@ static const pal_t g_stub_pal = { .mutex_destroy = stub_mutex_destroy, .thread_create = NULL, .thread_join = NULL, + .sleep_ms = NULL, }; /* ------------------------------------------------------------------------- diff --git a/modules/rtc-tcp-client/test/test_integration.c b/modules/rtc-tcp-client/test/test_integration.c index 1d1d864..8e752d8 100644 --- a/modules/rtc-tcp-client/test/test_integration.c +++ b/modules/rtc-tcp-client/test/test_integration.c @@ -18,11 +18,16 @@ #include #include #include +#include +#include +#include #include "../src/tai_internal.h" #include "tai_pal_loopback.h" #include "test_log.h" +extern const pal_t *tai_pal_posix(void); + /* ========================================================================= * Test harness * ========================================================================= */ @@ -96,6 +101,8 @@ typedef struct { uint16_t disconnect_code; uint8_t disconnect_reason; uint8_t disconnect_detail; + int text_calls_at_disconnect; + uint64_t disconnect_ms; char last_text_event_id[64]; char last_event_event_id[64]; @@ -128,6 +135,8 @@ static void st_reset(void) g_st.disconnect_code = 0; g_st.disconnect_reason = 0xFF; g_st.disconnect_detail = 0xFF; + g_st.text_calls_at_disconnect = 0; + g_st.disconnect_ms = 0; g_st.last_text_event_id[0] = '\0'; g_st.last_event_event_id[0] = '\0'; pthread_mutex_unlock(&g_st.mtx); @@ -211,6 +220,8 @@ static void on_disconnect(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, void g_st.disconnect_code = msg->close_code; g_st.disconnect_reason = msg->reason; g_st.disconnect_detail = msg->detail; + g_st.text_calls_at_disconnect = g_st.text_calls; + g_st.disconnect_ms = ctx->pal->time_ms(); pthread_mutex_unlock(&g_st.mtx); } @@ -510,7 +521,7 @@ static int build_event_app(tai_ctx_t *ctx, uint16_t evt_type, /* ========================================================================= * Common setup/teardown * ========================================================================= */ -static tai_ctx_t *setup_ctx(void *mem) +static tai_ctx_t *setup_ctx_config(void *mem, const tai_config_t *options) { tai_loopback_reset(); tai_loopback_seed_random(42); @@ -535,6 +546,12 @@ static tai_ctx_t *setup_ctx(void *mem) cfg.on_image = on_image; cfg.on_event = on_event; cfg.on_disconnect = on_disconnect; + if (options) { + if (options->pal) cfg.pal = options->pal; + cfg.on_flow_control = options->on_flow_control; + cfg.ping_interval_ms = options->ping_interval_ms; + cfg.ping_timeout_ms = options->ping_timeout_ms; + } /* The loopback completes the handshake (signs the SessionNew ack with the * key derived from this local_key), so confirmed-connect succeeds. */ @@ -543,6 +560,11 @@ static tai_ctx_t *setup_ctx(void *mem) return tai_ctx_init(mem, &cfg); } +static tai_ctx_t *setup_ctx(void *mem) +{ + return setup_ctx_config(mem, NULL); +} + /* ========================================================================= * Test 1: text_query happy path * ========================================================================= */ @@ -2022,16 +2044,63 @@ static void test_disconnect_latency(void) tai_ctx_deinit(ctx); } +static const pal_t *observed_base_pal; +static pal_t observed_pal; +static size_t observed_recv_calls; +static size_t observed_sleep_calls; +static size_t observed_empty_polls; + +static int observed_recv(void *tcp, uint8_t *buf, size_t len, + uint32_t timeout_ms) +{ + pthread_mutex_lock(&g_st.mtx); + observed_recv_calls++; + pthread_mutex_unlock(&g_st.mtx); + return observed_base_pal->tcp_recv(tcp, buf, len, timeout_ms); +} + +static int observed_poll(void *tcp, int events, uint32_t timeout_ms) +{ + pthread_mutex_lock(&g_st.mtx); + if (events == 0) observed_empty_polls++; + pthread_mutex_unlock(&g_st.mtx); + return observed_base_pal->tcp_poll(tcp, events, timeout_ms); +} + +static void observed_sleep(uint32_t ms) +{ + pthread_mutex_lock(&g_st.mtx); + observed_sleep_calls++; + pthread_mutex_unlock(&g_st.mtx); + observed_base_pal->sleep_ms(ms); +} + +static const pal_t *observe_pal(void) +{ + observed_base_pal = tai_pal_loopback(); + observed_pal = *observed_base_pal; + observed_pal.tcp_recv = observed_recv; + observed_pal.tcp_poll = observed_poll; + observed_pal.sleep_ms = observed_sleep; + observed_recv_calls = 0; + observed_sleep_calls = 0; + observed_empty_polls = 0; + return &observed_pal; +} + /* Pause after each delivered Packet, including Packets buffered by one recv. * Resume without new network bytes to prove rx_buf work cannot be stranded. */ static int flow_allowed_calls; +static int flow_paused_queries; +static uint64_t flow_resume_ms; static int test_flow_control(tai_ctx_t *ctx, void *user_data) { - (void)ctx; (void)user_data; pthread_mutex_lock(&g_st.mtx); int ready = g_st.audio_calls < flow_allowed_calls; + if (!ready) flow_paused_queries++; + else if (flow_resume_ms == 0) flow_resume_ms = ctx->pal->time_ms(); pthread_mutex_unlock(&g_st.mtx); return ready; } @@ -2040,12 +2109,22 @@ static void test_receive_backpressure(void) { SECTION("receive_backpressure"); static uint8_t ctx_mem[sizeof(struct tai_ctx)]; - tai_ctx_t *ctx = setup_ctx(ctx_mem); - CHECK(ctx != NULL); flow_allowed_calls = 0; - ctx->on_flow_control = test_flow_control; + flow_paused_queries = 0; + flow_resume_ms = 0; + tai_config_t options = { + .pal = observe_pal(), + .on_flow_control = test_flow_control, + }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(flow_paused_queries > 0, 1000)); + pthread_mutex_lock(&g_st.mtx); + size_t paused_recv_calls = observed_recv_calls; + pthread_mutex_unlock(&g_st.mtx); uint8_t audio[800]; memset(audio, 0x5a, sizeof(audio)); for (int i = 0; i < 40; i++) { @@ -2054,25 +2133,138 @@ static void test_receive_backpressure(void) i == 0 ? "111 1 16 16000" : NULL, audio, sizeof(audio), (uint16_t)(100 + i)) > 0); } + sleep_ms(100); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(observed_recv_calls, paused_recv_calls); + pthread_mutex_unlock(&g_st.mtx); for (int allowed = 1; allowed <= 40; allowed++) { pthread_mutex_lock(&g_st.mtx); flow_allowed_calls = allowed; pthread_mutex_unlock(&g_st.mtx); CHECK(WAIT_FOR(g_st.audio_calls >= allowed, 1000)); - sleep_ms(10); + sleep_ms(30); pthread_mutex_lock(&g_st.mtx); CHECK_EQ_INT(g_st.audio_calls, allowed); pthread_mutex_unlock(&g_st.mtx); } tai_disconnect(ctx); + CHECK(observed_sleep_calls > 0); + CHECK_EQ_INT(observed_empty_polls, 0); CHECK_EQ_INT(g_st.audio_calls, 40); CHECK_EQ_INT(g_st.audio_bytes, 40 * sizeof(audio)); CHECK_EQ_INT(g_st.disconnect_calls, 0); tai_ctx_deinit(ctx); } +static void test_long_receive_pause_liveness(void) +{ + SECTION("long_receive_pause_liveness"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + const uint32_t timeout_ms = 400; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + tai_config_t options = { + .pal = observe_pal(), + .on_flow_control = test_flow_control, + .ping_interval_ms = 100, + .ping_timeout_ms = timeout_ms, + }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(flow_paused_queries > 0, 1000)); + + sleep_ms(timeout_ms + 250); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + flow_allowed_calls = 100; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(flow_resume_ms != 0, 1000)); + sleep_ms(100); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + pthread_mutex_unlock(&g_st.mtx); + + uint8_t audio[40]; + memset(audio, 0x6b, sizeof(audio)); + uint64_t sent_ms = ctx->pal->time_ms(); + CHECK(server_send_audio(ctx, TAI_STREAM_START, + "111 1 16 16000 0 16000 20 40", + audio, sizeof(audio), 10) > 0); + CHECK(WAIT_FOR(g_st.audio_calls == 1, 1000)); + CHECK(WAIT_FOR(g_st.disconnect_calls > 0, timeout_ms + 1500)); + tai_disconnect(ctx); + CHECK_EQ_INT(g_st.disconnect_reason, TAI_DISCONNECT_TRANSPORT); + CHECK_EQ_INT(g_st.disconnect_detail, TAI_TRANSPORT_PING_TIMEOUT); + CHECK(g_st.disconnect_ms >= sent_ms + timeout_ms); + tai_ctx_deinit(ctx); +} + +static void check_pal_sleep(const pal_t *pal, void *tcp) +{ + uint8_t byte = 0; + CHECK_EQ_INT(pal->tcp_poll(tcp, 1, 1000), 1); + uint64_t start = pal->time_ms(); + pal->sleep_ms(120); + uint64_t elapsed = pal->time_ms() - start; + CHECK(elapsed >= 120); + CHECK(elapsed < 2120); + CHECK_EQ_INT(pal->tcp_poll(tcp, 1, 0), 1); + CHECK_EQ_INT(pal->tcp_recv(tcp, &byte, 1, 0), 1); + CHECK_EQ_INT(byte, 0x5a); + start = pal->time_ms(); + pal->sleep_ms(0); + CHECK(pal->time_ms() - start < 1000); +} + +static void test_pal_sleep(void) +{ + SECTION("pal_sleep_required"); + pal_t incomplete = *tai_pal_loopback(); + CHECK(pal_is_valid(&incomplete)); + incomplete.sleep_ms = NULL; + CHECK(!pal_is_valid(&incomplete)); + + SECTION("pal_sleep_loopback"); + tai_loopback_reset(); + const pal_t *pal = tai_pal_loopback(); + void *tcp = pal->tcp_connect("loopback.test", 443, 1000); + const uint8_t byte = 0x5a; + tai_loopback_push_recv(&byte, 1); + check_pal_sleep(pal, tcp); + pal->tcp_close(tcp); + + SECTION("pal_sleep_posix"); + int listener = socket(AF_INET, SOCK_STREAM, 0); + CHECK(listener >= 0); + if (listener < 0) return; + struct sockaddr_in addr = {0}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + CHECK_EQ_INT(bind(listener, (struct sockaddr *)&addr, sizeof(addr)), 0); + socklen_t addr_len = sizeof(addr); + CHECK_EQ_INT(getsockname(listener, (struct sockaddr *)&addr, &addr_len), 0); + CHECK_EQ_INT(listen(listener, 1), 0); + pal = tai_pal_posix(); + tcp = pal->tcp_connect("127.0.0.1", ntohs(addr.sin_port), 1000); + CHECK(tcp != NULL); + if (tcp) { + int peer = accept(listener, NULL, NULL); + CHECK(peer >= 0); + if (peer >= 0) { + CHECK_EQ_INT(send(peer, &byte, 1, 0), 1); + check_pal_sleep(pal, tcp); + close(peer); + } + pal->tcp_close(tcp); + } + close(listener); +} + /* ========================================================================= * main * ========================================================================= */ @@ -2086,6 +2278,8 @@ int main(void) pthread_mutex_init(&g_st.mtx, NULL); test_receive_backpressure(); + test_long_receive_pause_liveness(); + test_pal_sleep(); test_text_query(); test_audio_roundtrip(); test_image_query(); diff --git a/pal/pal.h b/pal/pal.h index 562027c..39e7055 100644 --- a/pal/pal.h +++ b/pal/pal.h @@ -50,6 +50,8 @@ extern "C" { * designated-initializer tables plus pal_is_valid() below: * pal/pal_posix.c, pal/pal_freertos.c, * modules/rtc-tcp-client/test/tai_pal_loopback.c, and .../test/test_core.c + * (test_core intentionally leaves networking/thread/sleep callbacks NULL because + * it never passes that table through pal_is_valid()). * The host build compiles three of them -- pal_freertos.c is in no CMake * target here, only the ESP-IDF component -- and C99 zero-fills whatever a * designated initializer omits, with no -Wmissing-field-initializers @@ -126,6 +128,13 @@ typedef struct pal { int (*thread_create)(void **handle, void *(*func)(void *), void *arg); int (*thread_join)(void *handle); + /* --- Sleep ----------------------------------------------------------- + * Suspend the calling thread without busy-waiting for at least ms + * milliseconds; 0 is a no-op. Scheduler delays and tick rounding may + * extend the sleep. No socket or socket-readiness dependency. + */ + void (*sleep_ms)(uint32_t ms); + } pal_t; /* ------------------------------------------------------------------------- @@ -146,7 +155,8 @@ static inline bool pal_is_valid(const pal_t *p) return p && p->tcp_connect && p->tcp_send && p->tcp_recv && p->tcp_close && p->tcp_poll && p->time_ms && p->malloc && p->free && p->mutex_create && p->mutex_lock && p->mutex_unlock - && p->mutex_destroy && p->thread_create && p->thread_join; + && p->mutex_destroy && p->thread_create && p->thread_join + && p->sleep_ms; } #ifdef __cplusplus diff --git a/pal/pal_freertos.c b/pal/pal_freertos.c index 5057de7..2b2ce21 100644 --- a/pal/pal_freertos.c +++ b/pal/pal_freertos.c @@ -14,6 +14,7 @@ * configSUPPORT_DYNAMIC_ALLOCATION 1 (pvPortMalloc, dynamic semaphores/tasks) * INCLUDE_vTaskSuspend 1 (used by thread-join shim) * INCLUDE_vTaskDelete 1 (joiner deletes the worker task) + * INCLUDE_vTaskDelay 1 (sleep_ms blocks without busy-waiting) * * Also recommended: a FreeRTOS heap implementation that supports vPortFree * (heap_3.c / heap_4.c / heap_5.c -- NOT heap_1.c). @@ -280,6 +281,19 @@ static uint64_t pal_time_ms(void) return (uint64_t)xTaskGetTickCount() * (uint64_t)portTICK_PERIOD_MS; } +static void pal_sleep_ms(uint32_t ms) +{ + if (ms == 0) return; + + uint64_t ticks = ((uint64_t)ms * configTICK_RATE_HZ + 999U) / 1000U; + const uint64_t max_chunk = (uint64_t)portMAX_DELAY - 1U; + while (ticks > 0) { + uint64_t chunk = ticks > max_chunk ? max_chunk : ticks; + vTaskDelay((TickType_t)(chunk + 1U)); + ticks -= chunk; + } +} + /* ------------------------------------------------------------------------- * Memory * ------------------------------------------------------------------------- */ @@ -392,6 +406,7 @@ static const pal_t g_freertos_pal = { .mutex_destroy = pal_mutex_destroy, .thread_create = pal_thread_create, .thread_join = pal_thread_join, + .sleep_ms = pal_sleep_ms, }; const pal_t *tai_pal_freertos(void) diff --git a/pal/pal_posix.c b/pal/pal_posix.c index 0409cb0..fcc5bfa 100644 --- a/pal/pal_posix.c +++ b/pal/pal_posix.c @@ -242,6 +242,13 @@ static uint64_t pal_time_ms(void) return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)(ts.tv_nsec / 1000000); } +static void pal_sleep_ms(uint32_t ms) +{ + if (ms == 0) return; + struct timespec delay = { ms / 1000, (long)(ms % 1000) * 1000000L }; + while (nanosleep(&delay, &delay) < 0 && errno == EINTR) { } +} + /* ------------------------------------------------------------------------- * Memory * ------------------------------------------------------------------------- */ @@ -308,6 +315,7 @@ static const pal_t g_posix_pal = { .mutex_destroy = pal_mutex_destroy, .thread_create = pal_thread_create, .thread_join = pal_thread_join, + .sleep_ms = pal_sleep_ms, }; const pal_t *tai_pal_posix(void) From fcef421e5045b3118b8731ff3ce3a024eb827db3 Mon Sep 17 00:00:00 2001 From: wangsn Date: Mon, 21 Sep 2026 20:26:06 +0800 Subject: [PATCH 04/10] docs(examples): demonstrate independent MQTT interruption Show one application-owned MQTT pump alongside pressured TAI media so server interrupts can flush stale playback without corrupting the TCP stream. Verified against 19/19 ctest suites, the bilingual docs build, and a manually linked mqtt_interrupt_demo. Co-Authored-By: OpenAI Codex --- CHANGELOG.md | 3 + docs-site/docs/guides/vad-and-interrupt.md | 6 + docs-site/docs/reference/iot-client.md | 14 + .../current/guides/vad-and-interrupt.md | 6 + .../current/reference/iot-client.md | 14 + examples/posix/CMakeLists.txt | 3 + .../ai/rtc-tcp-client/mqtt_interrupt_demo.c | 296 ++++++++++++++++++ 7 files changed, 342 insertions(+) create mode 100644 examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c diff --git a/CHANGELOG.md b/CHANGELOG.md index 76b23aa..43b5448 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- iot-client — authenticated MQTT protocol-9000 AI control notices can be delivered independently of RTC TCP receive backpressure (PR pending). + - Register `ai_ctrl_callback_t` with `iot_ai_ctrl_set_callback()` and keep one application thread pumping MQTT. + - docs-site — English edition of the full documentation site, published at `/en/` with Simplified Chinese retained at `/`. - All 29 docs are mirrored under `docs-site/i18n/en/`, with a locale selector in both the landing-page navbar and the custom docs topbar, localized navbar/footer/sidebar catalogs, and English SVG schematics under `current/images/`. - Heading anchors use explicit IDs shared across locales, and `npm run check:i18n` gates path, image, and anchor parity as part of `npm run build`. diff --git a/docs-site/docs/guides/vad-and-interrupt.md b/docs-site/docs/guides/vad-and-interrupt.md index 06e63b7..d7c785b 100644 --- a/docs-site/docs/guides/vad-and-interrupt.md +++ b/docs-site/docs/guides/vad-and-interrupt.md @@ -142,6 +142,12 @@ tai_config_t cfg = { ### RTC TCP Client {#rtc-tcp-client} +:::note MQTT 独立控制路径 +设备可同时保持 IoT MQTT 与 RTC TCP Connection。注册 `iot_ai_ctrl_set_callback()` 后，protocol-9000 `asrInterrupt` 不受 RTC TCP 接收背压影响。应用应让一个线程独占 `iot_client_process()` / publish / reconnect，并把 MQTT callback 与 `TAI_EVT_CHAT_BREAK` 汇入同一线程安全的播放策略。 + +收到 MQTT 打断时，先把对应 Event 标记为过期并清空其播放队列，再恢复 RTC 接收，让 worker 继续校验和排空旧媒体；不要直接丢弃 TCP 字节，否则会破坏 Frame 边界。旧 Event 的后续 `on_audio` 必须被应用丢弃，直到新 Event 开始。服务端打断不要求调用 `tai_chat_break()`，也不要结束或重开云端 VAD 上行流。 +::: + **接收服务端打断（`TAI_EVT_CHAT_BREAK`，type=4）：** `TAI_EVT_CHAT_BREAK` 有双重身份：用户在 AI 回复中插话时它是打断信号；在云端 VAD 模式下它同时也是**回合结束信号**（云端检测到用户停止说话后下发，当前云端不再下发 `TAI_EVT_SERVER_VAD`）。两种情况下的设备处理相同： diff --git a/docs-site/docs/reference/iot-client.md b/docs-site/docs/reference/iot-client.md index 2150838..3d8bead 100644 --- a/docs-site/docs/reference/iot-client.md +++ b/docs-site/docs/reference/iot-client.md @@ -441,6 +441,20 @@ int iot_client_publish(iot_client_t *client, const uint8_t *data, size_t data_le --- +### `iot_ai_ctrl_set_callback` {#iot_ai_ctrl_set_callback} + +```c +int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, + void *user_data); +``` + +注册经过 P2.3 解密和认证的 MQTT protocol-9000 AI 控制通知。回调在调用 `iot_client_process()` 的线程上触发，`type` 和 `json_data` 只在回调期间有效。回调应仅更新有界状态或通知应用线程，不应在其中断开/销毁 IoT client。 + +若必须接收订阅建立期间立即到达的通知，初始化时设置 `mqtt_disable_auto_connect=true`，先注册回调，再调用 `iot_client_connect()`。传 NULL callback 可注销。RTC TCP 接收背压不会阻塞该 MQTT 路径；应用应把 MQTT 通知和 TAI ChatBreak 汇入同一播放策略，并按 Event 标识丢弃被中断 Event 的后续媒体。服务端通知不是 `tai_chat_break()` 的应答，也不应自动结束云端 VAD 上行流。 + +--- + ### `iot_get_qrcode_info` {#iot_get_qrcode_info} ```c diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md index 0a5915d..2379f0d 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md @@ -144,6 +144,12 @@ When the user speaks again while the AI is responding, the current response need ### RTC TCP Client {#rtc-tcp-client} +:::note Independent MQTT control path +A device can keep the IoT MQTT and RTC TCP Connection active together. After `iot_ai_ctrl_set_callback()` is registered, a protocol-9000 `asrInterrupt` is independent of RTC TCP receive backpressure. One application thread should own `iot_client_process()`, publish, and reconnect, while the MQTT callback and `TAI_EVT_CHAT_BREAK` feed the same thread-safe playback policy. + +On an MQTT interrupt, first mark the correlated Event stale and flush its playback queue, then release RTC receive pressure so the worker can continue authenticating and draining old media. Never discard raw TCP bytes, which would corrupt Frame boundaries. The application must reject subsequent `on_audio` callbacks for the stale Event until a new Event starts. A server notice does not require `tai_chat_break()`, and it must not end or reopen a Server-VAD uplink. +::: + **Receive a server-initiated chat break (`TAI_EVT_CHAT_BREAK`, type=4):** `TAI_EVT_CHAT_BREAK` has two roles: it signals an interruption when the user speaks during the AI response; in server-VAD mode, it is also the **turn-end signal** (sent when the cloud detects that the user has stopped speaking; the current cloud no longer sends `TAI_EVT_SERVER_VAD`). Device-side handling is the same in both cases: diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md index a617232..f7f909f 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md @@ -440,6 +440,20 @@ Publishes an encrypted message to `smart/device/out/{deviceid}`. --- +### `iot_ai_ctrl_set_callback` {#iot_ai_ctrl_set_callback} + +```c +int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, + void *user_data); +``` + +Registers for P2.3-decrypted and authenticated MQTT protocol-9000 AI control notices. The callback runs on the thread calling `iot_client_process()`, and `type` plus `json_data` remain valid only during the callback. Keep it bounded to updating state or notifying an application thread; do not disconnect or destroy the IoT client there. + +If notices sent during subscription setup must be observable, initialize with `mqtt_disable_auto_connect=true`, register the callback, and then call `iot_client_connect()`. Pass a NULL callback to deregister. RTC TCP receive backpressure does not stall this MQTT path. The application should merge MQTT notices and TAI ChatBreak into one playback policy and discard subsequent media belonging to the interrupted Event. A server notice is not an acknowledgment request for `tai_chat_break()` and must not automatically end a Server-VAD uplink. + +--- + ### `iot_get_qrcode_info` {#iot_get_qrcode_info} ```c diff --git a/examples/posix/CMakeLists.txt b/examples/posix/CMakeLists.txt index f32c40f..22f057c 100644 --- a/examples/posix/CMakeLists.txt +++ b/examples/posix/CMakeLists.txt @@ -154,6 +154,9 @@ target_link_libraries(mcp_demo PRIVATE tuya_ai_tcp tuya_iot_client agentic_kit_p add_executable(agent_trigger_demo "${CMAKE_CURRENT_SOURCE_DIR}/ai/rtc-tcp-client/agent_trigger_demo.c") target_link_libraries(agent_trigger_demo PRIVATE tuya_ai_tcp tuya_iot_client agentic_kit_pal) +add_executable(mqtt_interrupt_demo "${CMAKE_CURRENT_SOURCE_DIR}/ai/rtc-tcp-client/mqtt_interrupt_demo.c") +target_link_libraries(mqtt_interrupt_demo PRIVATE tuya_ai_tcp tuya_iot_client agentic_kit_pal) + find_library(OPUS_LIBRARY opus) find_path(OPUS_INCLUDE_DIR opus/opus.h) if(OPUS_LIBRARY AND OPUS_INCLUDE_DIR) diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c new file mode 100644 index 0000000..6dfd9f8 --- /dev/null +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -0,0 +1,296 @@ +/* + * mqtt_interrupt_demo.c -- Independent MQTT control plus pressured TAI media. + * + * This demo uses a bounded synthetic playback queue. MQTT is pumped by one + * application-owned thread while TAI's worker independently receives media. + * An asrInterrupt flushes queued playback, marks that Event stale, and reopens + * TAI receive admission so remaining stale media can be drained and discarded. + */ + +#include +#include +#include +#include + +#include "demo_json.h" +#include "iot_client.h" +#include "tuya_ai.h" + +extern const pal_t *tai_pal_posix(void); + +#define DEFAULT_DEVID "6cd370251e8be96de8vwoe" +#define DEFAULT_SECRET_KEY "[SPT;N:b@)wPzK/)" +#define DEFAULT_LOCAL_KEY "#d[<4y*N.vE]RAAG" +#define PLAYBACK_CAPACITY 4 + +#define WAIT_STEP_MS 50U +#define WAIT_LIMIT_MS 60000U + +typedef struct { + pthread_mutex_t mutex; + iot_client_t *iot; + tai_ctx_t *tai; + int mqtt_running; + int mqtt_failed; + int done; + int playback_running; + size_t queued_audio; + int drop_until_new_event; + char stale_event_id[64]; +} demo_state_t; + +static void set_stale_event(demo_state_t *state, const char *event_id) +{ + pthread_mutex_lock(&state->mutex); + state->queued_audio = 0; + if (event_id && event_id[0]) { + snprintf(state->stale_event_id, sizeof(state->stale_event_id), + "%s", event_id); + } else { + state->drop_until_new_event = 1; + state->stale_event_id[0] = '\0'; + } + pthread_mutex_unlock(&state->mutex); +} + +static void on_ai_control(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + demo_state_t *state = (demo_state_t *)user_data; + if (strcmp(type, "asrInterrupt") != 0) return; + + char event_id[64] = {0}; + static const char key[] = "\"eventId\":\""; + const char *id = NULL; + for (size_t i = 0; i + sizeof(key) - 1 <= data_len; i++) { + if (memcmp(json_data + i, key, sizeof(key) - 1) == 0) { + id = json_data + i + sizeof(key) - 1; + break; + } + } + if (id) { + size_t available = data_len - (size_t)(id - json_data); + const char *end = memchr(id, '"', available); + if (end) { + size_t n = (size_t)(end - id); + if (n >= sizeof(event_id)) n = sizeof(event_id) - 1; + memcpy(event_id, id, n); + event_id[n] = '\0'; + } + } + set_stale_event(state, event_id); + printf("\n[MQTT control] asrInterrupt event=%s; playback flushed\n", + event_id[0] ? event_id : "(unscoped)"); +} + +static int on_flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + pthread_mutex_lock(&state->mutex); + int admit = state->queued_audio < PLAYBACK_CAPACITY; + pthread_mutex_unlock(&state->mutex); + return admit; +} + +static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + pthread_mutex_lock(&state->mutex); + if (state->drop_until_new_event && msg->stream_flag == TAI_STREAM_START) { + state->drop_until_new_event = 0; + } + if (state->stale_event_id[0] && msg->event_id && msg->event_id[0] && + strcmp(state->stale_event_id, msg->event_id) != 0) { + state->stale_event_id[0] = '\0'; + } + int stale = state->drop_until_new_event || + (state->stale_event_id[0] && msg->event_id && + strcmp(state->stale_event_id, msg->event_id) == 0); + if (!stale && state->queued_audio < PLAYBACK_CAPACITY) { + state->queued_audio++; + } + pthread_mutex_unlock(&state->mutex); +} + +static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + if (msg->event_type == TAI_EVT_CHAT_BREAK) { + set_stale_event(state, msg->event_id); + printf("\n[TAI Event] ChatBreak event=%s; playback flushed\n", + msg->event_id && msg->event_id[0] ? msg->event_id : "(unscoped)"); + } else if (msg->event_type == TAI_EVT_END) { + pthread_mutex_lock(&state->mutex); + state->done = 1; + pthread_mutex_unlock(&state->mutex); + } +} + +static void on_text(tai_ctx_t *ctx, const tai_text_msg_t *msg, void *user_data) +{ + (void)ctx; + (void)user_data; + fwrite(msg->text, 1, msg->len, stdout); + fflush(stdout); +} + +static void on_disconnect(tai_ctx_t *ctx, const tai_disconnect_msg_t *msg, + void *user_data) +{ + (void)ctx; + demo_state_t *state = (demo_state_t *)user_data; + fprintf(stderr, "\n[TAI disconnected: reason=%u detail=%u]\n", + (unsigned)msg->reason, (unsigned)msg->detail); + pthread_mutex_lock(&state->mutex); + state->done = 1; + pthread_mutex_unlock(&state->mutex); +} + +static void *mqtt_owner(void *arg) +{ + demo_state_t *state = (demo_state_t *)arg; + while (1) { + pthread_mutex_lock(&state->mutex); + int running = state->mqtt_running; + pthread_mutex_unlock(&state->mutex); + if (!running) break; + if (iot_client_process(state->iot, WAIT_STEP_MS) != OPRT_OK) { + pthread_mutex_lock(&state->mutex); + state->mqtt_failed = 1; + state->mqtt_running = 0; + pthread_mutex_unlock(&state->mutex); + break; + } + } + return NULL; +} + +static void *playback_consumer(void *arg) +{ + demo_state_t *state = (demo_state_t *)arg; + while (1) { + state->iot->pal->sleep_ms(100); + pthread_mutex_lock(&state->mutex); + if (!state->playback_running) { + pthread_mutex_unlock(&state->mutex); + break; + } + if (state->queued_audio > 0) state->queued_audio--; + pthread_mutex_unlock(&state->mutex); + } + return NULL; +} + +int main(int argc, char **argv) +{ + const char *devid = argc > 1 ? argv[1] : DEFAULT_DEVID; + const char *secret_key = argc > 2 ? argv[2] : DEFAULT_SECRET_KEY; + const char *local_key = argc > 3 ? argv[3] : DEFAULT_LOCAL_KEY; + const pal_t *pal = tai_pal_posix(); + demo_state_t state; + memset(&state, 0, sizeof(state)); + pthread_mutex_init(&state.mutex, NULL); + + if (iot_init_default() != OPRT_OK) return 1; + iot_client_config_t iot_cfg = { + .region = AY, + .env = PROD, + .mqtt_disable_auto_connect = true, + }; + if (demo_copy_field(iot_cfg.devid, sizeof(iot_cfg.devid), devid, "devid") || + demo_copy_field(iot_cfg.secret_key, sizeof(iot_cfg.secret_key), + secret_key, "secret_key") || + demo_copy_field(iot_cfg.local_key, sizeof(iot_cfg.local_key), + local_key, "local_key")) { + return 1; + } + state.iot = iot_client_init(&iot_cfg); + if (!state.iot) return 1; + + char *token = (char *)calloc(1, 4096); + tai_conn_params_t params; + if (!token || iot_client_get_session_token(state.iot, NULL, token, 4096) || + parse_token(token, ¶ms)) { + fprintf(stderr, "failed to obtain/parse TAI session token\n"); + free(token); + iot_client_deinit(state.iot); + return 1; + } + if (params.biz_code == 0) params.biz_code = 65537; + if (params.biz_tag == 0) params.biz_tag = 119; + + iot_ai_ctrl_set_callback(state.iot, on_ai_control, &state); + if (iot_client_connect(state.iot) != OPRT_OK) { + fprintf(stderr, "MQTT connect failed\n"); + free(token); + iot_client_deinit(state.iot); + return 1; + } + + tai_config_t tai_cfg = { + .host = params.host, + .port = params.port, + .tls_sni = params.tls_sni, + .device_id = params.derived_client_id, + .local_key = local_key, + .client_type = TAI_CLIENT_DEVICE, + .protocol_version = TAI_VER_21, + .sign_level = TAI_SIGN_HMAC_SHA256, + .biz_code = (uint32_t)params.biz_code, + .biz_tag = (uint64_t)params.biz_tag, + .agent_token = params.agent_token, + .pal = pal, + .on_audio = on_audio, + .on_text = on_text, + .on_event = on_event, + .on_disconnect = on_disconnect, + .on_flow_control = on_flow_control, + .user_data = &state, + }; + void *tai_mem = pal->malloc(tai_ctx_size()); + state.tai = tai_mem ? tai_ctx_init(tai_mem, &tai_cfg) : NULL; + if (!state.tai || tai_connect(state.tai) != TAI_OK) { + fprintf(stderr, "TAI connect failed\n"); + if (state.tai) tai_ctx_deinit(state.tai); + pal->free(tai_mem); + free(token); + iot_client_deinit(state.iot); + return 1; + } + + pthread_t mqtt_thread; + pthread_t playback_thread; + state.mqtt_running = 1; + state.playback_running = 1; + pthread_create(&mqtt_thread, NULL, mqtt_owner, &state); + pthread_create(&playback_thread, NULL, playback_consumer, &state); + + const char *question = "Please give a detailed explanation of TCP backpressure."; + int send_rc = tai_send_text(state.tai, question, strlen(question)); + uint32_t waited = 0; + while (send_rc == TAI_OK && waited < WAIT_LIMIT_MS) { + pal->sleep_ms(WAIT_STEP_MS); + pthread_mutex_lock(&state.mutex); + int done = state.done || state.mqtt_failed; + pthread_mutex_unlock(&state.mutex); + if (done) break; + waited += WAIT_STEP_MS; + } + + pthread_mutex_lock(&state.mutex); + state.mqtt_running = 0; + state.playback_running = 0; + pthread_mutex_unlock(&state.mutex); + pthread_join(mqtt_thread, NULL); + pthread_join(playback_thread, NULL); + tai_disconnect(state.tai); + tai_ctx_deinit(state.tai); + pal->free(tai_mem); + free(token); + iot_client_deinit(state.iot); + pthread_mutex_destroy(&state.mutex); + return send_rc == TAI_OK && !state.mqtt_failed ? 0 : 1; +} From e88cd7a134c7a1cbbaddb2e7cac985003b810574 Mon Sep 17 00:00:00 2001 From: wangsn Date: Mon, 21 Sep 2026 20:31:24 +0800 Subject: [PATCH 05/10] test(iot-client): cover MQTT interruption during TAI backpressure Prove authenticated control remains deliverable while the media Connection is paused, stale Event audio is drained without refilling playback, and the next Event is accepted. Verified against 20/20 ctest suites and the focused ASan/UBSan target. Co-Authored-By: OpenAI Codex --- CMakeLists.txt | 25 ++ .../iot-client/test/iot_tai_control_test.c | 260 ++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 modules/iot-client/test/iot_tai_control_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 21b3926..70a1188 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -444,6 +444,31 @@ if(AGENTIC_KIT_BUILD_TESTS AND AGENTIC_KIT_ENABLE_PROJECT_TESTS) agentic_kit_add_iot_test(iot_ota_test "${IOT_CLIENT_TESTS_DIR}/iot_ota_test.c") agentic_kit_add_iot_test(iot_ota_verify_test "${IOT_CLIENT_TESTS_DIR}/iot_ota_verify_test.c") + add_executable(iot_tai_control_test + ${AI_TCP_SOURCES} + "${AI_TCP_TESTS_DIR}/tai_pal_loopback.c" + "${IOT_CLIENT_TESTS_DIR}/iot_tai_control_test.c" + "${IOT_CLIENT_TESTS_DIR}/test_log.c" + ) + target_include_directories(iot_tai_control_test PRIVATE + "${AI_TCP_DIR}/include" + "${AI_TCP_DIR}/src" + "${AI_TCP_TESTS_DIR}" + "${IOT_CLIENT_DIR}/include" + "${IOT_CLIENT_DIR}/src" + "${IOT_CLIENT_TESTS_DIR}" + "${IOT_CLIENT_TESTS_DIR}/log_config" + "${cjson_source_dir}" + "${PAL_DIR}" + "${COMMON_DIR}" + ) + target_link_libraries(iot_tai_control_test PRIVATE + tuya_iot_client_testlog + agentic_kit_pal + Threads::Threads + ) + add_test(NAME iot_tai_control_test COMMAND iot_tai_control_test) + # The ble sources compile into the executable (not the tuya_ble # library) so they pick up the test sink's log remap from # tuya_iot_client_testlog's include path -- that is what lets diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c new file mode 100644 index 0000000..bb272e3 --- /dev/null +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -0,0 +1,260 @@ +/* + * Cross-module regression: MQTT control remains deliverable while TAI receive + * backpressure stops the media Connection, then stale media drains without + * refilling playback and a new Event remains playable. + */ + +#include +#include +#include +#include + +#include "iot_ai_ctrl.h" +#include "iot_client.h" +#include "tai_internal.h" +#include "tai_pal_loopback.h" + +static int failures; + +#define CHECK(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #expr); \ + failures++; \ + } \ + } while (0) + +static void sleep_ms(uint32_t ms) +{ + struct timespec ts = { ms / 1000U, (long)(ms % 1000U) * 1000000L }; + nanosleep(&ts, NULL); +} + +typedef struct { + pthread_mutex_t mutex; + size_t queued_audio; + int audio_callbacks; + int accepted_old_audio; + int accepted_new_audio; + int chat_breaks; + int mqtt_interrupts; + char stale_event_id[64]; +} app_state_t; + +static app_state_t state; + +#define WAIT_FOR(expr, timeout_ms) \ + ({ \ + int found = 0; \ + for (uint32_t waited = 0; waited < (timeout_ms); waited += 5) { \ + pthread_mutex_lock(&state.mutex); \ + found = (expr); \ + pthread_mutex_unlock(&state.mutex); \ + if (found) break; \ + sleep_ms(5); \ + } \ + found; \ + }) + +static int flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)ctx; + app_state_t *s = (app_state_t *)user_data; + pthread_mutex_lock(&s->mutex); + int admit = s->queued_audio == 0; + pthread_mutex_unlock(&s->mutex); + return admit; +} + +static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, + void *user_data) +{ + (void)ctx; + app_state_t *s = (app_state_t *)user_data; + pthread_mutex_lock(&s->mutex); + s->audio_callbacks++; + int stale = s->stale_event_id[0] && msg->event_id && + strcmp(s->stale_event_id, msg->event_id) == 0; + if (!stale && s->queued_audio == 0) { + s->queued_audio = 1; + if (msg->event_id && strcmp(msg->event_id, "old-event") == 0) { + s->accepted_old_audio++; + } else if (msg->event_id && strcmp(msg->event_id, "new-event") == 0) { + s->accepted_new_audio++; + } + } + pthread_mutex_unlock(&s->mutex); +} + +static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, + void *user_data) +{ + (void)ctx; + app_state_t *s = (app_state_t *)user_data; + pthread_mutex_lock(&s->mutex); + if (msg->event_type == TAI_EVT_CHAT_BREAK) { + s->chat_breaks++; + s->queued_audio = 0; + if (msg->event_id && msg->event_id[0]) { + snprintf(s->stale_event_id, sizeof(s->stale_event_id), + "%s", msg->event_id); + } + } else if (msg->event_type == TAI_EVT_START && + s->stale_event_id[0] && msg->event_id && msg->event_id[0] && + strcmp(s->stale_event_id, msg->event_id) != 0) { + s->stale_event_id[0] = '\0'; + } + pthread_mutex_unlock(&s->mutex); +} + +static void on_ai_control(const char *type, const char *json_data, + size_t data_len, void *user_data) +{ + app_state_t *s = (app_state_t *)user_data; + if (strcmp(type, "asrInterrupt") != 0) return; + + static const char expected[] = "\"eventId\":\"old-event\""; + int matched = 0; + for (size_t i = 0; i + sizeof(expected) - 1 <= data_len; i++) { + if (memcmp(json_data + i, expected, sizeof(expected) - 1) == 0) { + matched = 1; + break; + } + } + if (!matched) return; + + pthread_mutex_lock(&s->mutex); + s->mqtt_interrupts++; + s->queued_audio = 0; + snprintf(s->stale_event_id, sizeof(s->stale_event_id), "%s", "old-event"); + pthread_mutex_unlock(&s->mutex); +} + +static int server_push(tai_ctx_t *ctx, uint8_t packet_type, + const tai_attr_t *attrs, int attr_count, + const uint8_t *payload, size_t payload_len, + uint16_t sequence) +{ + uint8_t packet[512]; + uint8_t frame[600]; + int packet_len = tai_packet_encode(TAI_VER_21, packet_type, + attrs, attr_count, + payload, payload_len, + packet, sizeof(packet)); + if (packet_len <= 0) return packet_len; + int frame_len = tai_frame_encode(TAI_FRAG_NONE, sequence, + packet, (size_t)packet_len, + ctx->sign_key, 32, ctx->pal, + frame, sizeof(frame)); + if (frame_len > 0) tai_loopback_push_recv(frame, (size_t)frame_len); + return frame_len; +} + +static int server_push_event(tai_ctx_t *ctx, uint16_t event_type, + const char *event_id, uint16_t sequence) +{ + tai_attr_t attrs[2] = { + tai_attr_strv(TAI_ATTR_SESSION_ID, ctx->session_id), + tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + }; + uint8_t payload[4]; + int payload_len = tai_pack_event(TAI_VER_21, event_type, + NULL, 0, payload, sizeof(payload)); + if (payload_len <= 0) return payload_len; + return server_push(ctx, TAI_PKT_EVENT, attrs, 2, + payload, (size_t)payload_len, sequence); +} + +static int server_push_audio(tai_ctx_t *ctx, const char *event_id, + uint8_t fill, uint16_t sequence) +{ + uint8_t payload[48]; + int header_len = tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, + ctx->pal->time_ms(), + payload, sizeof(payload)); + if (header_len <= 0) return header_len; + memset(payload + header_len, fill, 40); + + tai_attr_t attrs[2] = { + tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, + "111 1 16 16000 0 16000 20 40"), + }; + return server_push(ctx, TAI_PKT_AUDIO, attrs, 2, + payload, (size_t)header_len + 40, sequence); +} + +int main(void) +{ + const pal_t *pal = tai_pal_loopback(); + pthread_mutex_init(&state.mutex, NULL); + tai_loopback_reset(); + tai_loopback_set_local_key("test-local-key-16"); + CHECK(iot_init(pal) == OPRT_OK); + + iot_client_t iot; + memset(&iot, 0, sizeof(iot)); + iot.pal = pal; + CHECK(iot_ai_ctrl_set_callback(&iot, on_ai_control, &state) == OPRT_OK); + + tai_config_t config = { + .host = "loopback.test", + .port = 443, + .device_id = "test-device", + .local_key = "test-local-key-16", + .protocol_version = TAI_VER_21, + .client_type = TAI_CLIENT_DEVICE, + .sign_level = TAI_SIGN_HMAC_SHA256, + .disable_tls = 1, + .pal = pal, + .on_audio = on_audio, + .on_event = on_event, + .on_flow_control = flow_control, + .user_data = &state, + }; + static uint8_t context_memory[sizeof(struct tai_ctx)]; + tai_ctx_t *ctx = tai_ctx_init(context_memory, &config); + CHECK(ctx != NULL); + if (!ctx) return 1; + CHECK(tai_connect(ctx) == TAI_OK); + + uint8_t sent[2048]; + (void)tai_loopback_pop_sent(sent, sizeof(sent)); + uint16_t sequence = 10; + CHECK(server_push_event(ctx, TAI_EVT_START, "old-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "old-event", 0x11, sequence++) > 0); + CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); + + CHECK(server_push_audio(ctx, "old-event", 0x22, sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, + "old-event", sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_END, "old-event", sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_START, "new-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "new-event", 0x33, sequence++) > 0); + sleep_ms(100); + CHECK(state.audio_callbacks == 1); + CHECK(state.chat_breaks == 0); + + static const char control[] = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\"," + "\"data\":{\"eventId\":\"old-event\"}}}}"; + CHECK(iot_ai_ctrl_dispatch(&iot, (const uint8_t *)control, + sizeof(control) - 1)); + CHECK(state.mqtt_interrupts == 1); + CHECK(state.chat_breaks == 0); + + CHECK(WAIT_FOR(state.accepted_new_audio == 1, 2000)); + CHECK(state.audio_callbacks == 3); + CHECK(state.accepted_old_audio == 1); + CHECK(state.accepted_new_audio == 1); + CHECK(state.chat_breaks == 1); + CHECK(state.queued_audio == 1); + + tai_disconnect(ctx); + tai_ctx_deinit(ctx); + pthread_mutex_destroy(&state.mutex); + printf("iot_tai_control_test: %s\n", failures ? "FAIL" : "PASS"); + return failures ? 1 : 0; +} From 87bcd6962803811e2b2d981ff10dbd18e789ca67 Mon Sep 17 00:00:00 2001 From: wangsn Date: Mon, 21 Sep 2026 21:13:14 +0800 Subject: [PATCH 06/10] fix(rtc-tcp-client): preserve callback threading under backpressure Keep coalesced post-ack media buffered for the receive worker, ignore delayed interrupts for obsolete Events, document the cross-module ownership decisions, and add the new network-free suites to leak CI. Verified with focused TAI/control regressions. Co-Authored-By: OpenAI Codex --- .gitlab-ci.yml | 5 ++- CONTEXT-MAP.md | 4 +- .../ai/rtc-tcp-client/mqtt_interrupt_demo.c | 24 +++++++++-- ...ai-control-remains-application-composed.md | 22 ++++++++++ .../iot-client/test/iot_tai_control_test.c | 40 ++++++++++++++----- ...eive-backpressure-pauses-the-connection.md | 22 ++++++++++ modules/rtc-tcp-client/src/tai_client.c | 5 +++ 7 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md create mode 100644 modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a44c4e6..19280ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,13 +89,14 @@ test:memory-leak: # Leak-check the pure (no-subprocess) binaries, grouped by module: # rtc-tcp-client : tai_unit_tests, tai_log_level_zero_tests, # tai_log_module_zero_tests, tai_integration_tests - # iot-client : iot_cipher_test + # iot-client : iot_cipher_test, iot_ai_ctrl_test, + # iot_tai_control_test # tuya-ble : tuya_ble_test # rtc-client is a prebuilt closed lib (headers + libstm.a) with no offline # test, so there is nothing to leak-check for it here. The mock-driven iot # tests are covered by the `test` job; under valgrind their Python # handshakes would time out. - for t in tai_unit_tests tai_log_level_zero_tests tai_log_module_zero_tests tai_integration_tests iot_cipher_test tuya_ble_test; do + for t in tai_unit_tests tai_log_level_zero_tests tai_log_module_zero_tests tai_integration_tests iot_cipher_test iot_ai_ctrl_test iot_tai_control_test tuya_ble_test; do echo "==================== valgrind: ${t} ====================" "$VG" "./${CMAKE_BUILD_DIR}/${t}" 2>&1 | tee "valgrind-${t}.log" rc=${PIPESTATUS[0]} diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index 1ca0c91..0e0d35e 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -21,7 +21,9 @@ and decisions. Applications may also keep MQTT active as an independent control path: authenticated protocol-9000 notices reach `ai_ctrl_callback_t` even while receive backpressure stalls the RTC Connection. The application composes the - callbacks; neither client owns or calls the other. + callbacks; neither client owns or calls the other. Decision records: + `modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md` + and `modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md`. - **Tuya BLE → IoT Client**: BLE provisioning runs first and hands the device its WiFi credentials (incl. the pairing token); once on WiFi, the IoT Client uses them to activate against the cloud. diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c index 6dfd9f8..d99caa4 100644 --- a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -36,17 +36,25 @@ typedef struct { int playback_running; size_t queued_audio; int drop_until_new_event; + char current_event_id[64]; char stale_event_id[64]; } demo_state_t; static void set_stale_event(demo_state_t *state, const char *event_id) { pthread_mutex_lock(&state->mutex); - state->queued_audio = 0; if (event_id && event_id[0]) { + /* A delayed duplicate for an older Event must not flush current audio. */ + if (state->current_event_id[0] && + strcmp(state->current_event_id, event_id) != 0) { + pthread_mutex_unlock(&state->mutex); + return; + } + state->queued_audio = 0; snprintf(state->stale_event_id, sizeof(state->stale_event_id), "%s", event_id); } else { + state->queued_audio = 0; state->drop_until_new_event = 1; state->stale_event_id[0] = '\0'; } @@ -118,9 +126,19 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data { (void)ctx; demo_state_t *state = (demo_state_t *)user_data; - if (msg->event_type == TAI_EVT_CHAT_BREAK) { + if (msg->event_type == TAI_EVT_START && msg->event_id && msg->event_id[0]) { + pthread_mutex_lock(&state->mutex); + snprintf(state->current_event_id, sizeof(state->current_event_id), + "%s", msg->event_id); + if (state->stale_event_id[0] && + strcmp(state->stale_event_id, msg->event_id) != 0) { + state->stale_event_id[0] = '\0'; + } + state->drop_until_new_event = 0; + pthread_mutex_unlock(&state->mutex); + } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { set_stale_event(state, msg->event_id); - printf("\n[TAI Event] ChatBreak event=%s; playback flushed\n", + printf("\n[TAI Event] ChatBreak event=%s\n", msg->event_id && msg->event_id[0] ? msg->event_id : "(unscoped)"); } else if (msg->event_type == TAI_EVT_END) { pthread_mutex_lock(&state->mutex); diff --git a/modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md b/modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md new file mode 100644 index 0000000..d7091ce --- /dev/null +++ b/modules/iot-client/docs/adr/0002-mqtt-ai-control-remains-application-composed.md @@ -0,0 +1,22 @@ +# MQTT AI control remains application-composed + +The IoT Client consumes authenticated MQTT protocol-9000 AI control notices and +passes their event type and data to an application callback. It does not depend +on the RTC TCP Client, inject TAI Events, stop playback, or send an automatic +acknowledgement. The application keeps one owner for MQTT process/publish and +combines this callback with RTC receive callbacks in its own synchronized state. + +We chose a separate MQTT control path because intentional RTC receive +backpressure stalls every Frame on that Connection, including a later ChatBreak. +Keeping the modules independent preserves their existing threading models and +lets an interrupt reach the application without waiting behind media bytes. + +## Consequences + +- The application must keep MQTT connected and call `iot_client_process()` while + a TAI Session is active. +- Callback data is borrowed and the callback must return promptly. +- Event correlation, duplicate suppression, queue flushing, and rejection of + stale media are application responsibilities. +- A server notice does not imply calling `tai_chat_break()` or ending a + server-VAD uplink. diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c index bb272e3..64ac53b 100644 --- a/modules/iot-client/test/iot_tai_control_test.c +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -38,6 +38,7 @@ typedef struct { int accepted_new_audio; int chat_breaks; int mqtt_interrupts; + char current_event_id[64]; char stale_event_id[64]; } app_state_t; @@ -92,17 +93,23 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, (void)ctx; app_state_t *s = (app_state_t *)user_data; pthread_mutex_lock(&s->mutex); - if (msg->event_type == TAI_EVT_CHAT_BREAK) { + if (msg->event_type == TAI_EVT_START && msg->event_id && msg->event_id[0]) { + snprintf(s->current_event_id, sizeof(s->current_event_id), + "%s", msg->event_id); + if (s->stale_event_id[0] && + strcmp(s->stale_event_id, msg->event_id) != 0) { + s->stale_event_id[0] = '\0'; + } + } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { s->chat_breaks++; - s->queued_audio = 0; - if (msg->event_id && msg->event_id[0]) { - snprintf(s->stale_event_id, sizeof(s->stale_event_id), - "%s", msg->event_id); + if (!msg->event_id || !msg->event_id[0] || !s->current_event_id[0] || + strcmp(s->current_event_id, msg->event_id) == 0) { + s->queued_audio = 0; + if (msg->event_id && msg->event_id[0]) { + snprintf(s->stale_event_id, sizeof(s->stale_event_id), + "%s", msg->event_id); + } } - } else if (msg->event_type == TAI_EVT_START && - s->stale_event_id[0] && msg->event_id && msg->event_id[0] && - strcmp(s->stale_event_id, msg->event_id) != 0) { - s->stale_event_id[0] = '\0'; } pthread_mutex_unlock(&s->mutex); } @@ -125,8 +132,12 @@ static void on_ai_control(const char *type, const char *json_data, pthread_mutex_lock(&s->mutex); s->mqtt_interrupts++; - s->queued_audio = 0; - snprintf(s->stale_event_id, sizeof(s->stale_event_id), "%s", "old-event"); + if (!s->current_event_id[0] || + strcmp(s->current_event_id, "old-event") == 0) { + s->queued_audio = 0; + snprintf(s->stale_event_id, sizeof(s->stale_event_id), + "%s", "old-event"); + } pthread_mutex_unlock(&s->mutex); } @@ -252,6 +263,13 @@ int main(void) CHECK(state.chat_breaks == 1); CHECK(state.queued_audio == 1); + /* Delayed duplicate for the old Event must not flush the new playback. */ + CHECK(iot_ai_ctrl_dispatch(&iot, (const uint8_t *)control, + sizeof(control) - 1)); + CHECK(state.mqtt_interrupts == 2); + CHECK(state.queued_audio == 1); + CHECK(state.accepted_new_audio == 1); + tai_disconnect(ctx); tai_ctx_deinit(ctx); pthread_mutex_destroy(&state.mutex); diff --git a/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md b/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md new file mode 100644 index 0000000..2ad3ba9 --- /dev/null +++ b/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md @@ -0,0 +1,22 @@ +# Receive backpressure pauses the Connection at Frame boundaries + +The RTC TCP Client exposes an optional `on_flow_control` admission hook. When it +returns zero, the worker stops parsing complete buffered Frames and stops reading +the Connection, allowing the TCP receive window to close. This pauses all inbound +traffic rather than selecting only audio. The worker still runs Ping, shutdown, +and bounded admission checks through `pal_t.sleep_ms`. + +We chose Frame-boundary admission because receive and reassembly buffers are +worker-owned borrowed storage. Deferring individual callback payloads would need +a second bounded queue and new lifetime rules. The application must still bound +its audio callback because one Audio Packet can emit multiple codec frames. + +## Consequences + +- ChatBreak, text, Pong, and EOF detection are delayed while admission is closed; + an independent MQTT control path can carry urgent application notices. +- Intentional pauses suspend receive-liveness accounting. Resume starts a fresh + `ping_timeout_ms` budget without pretending traffic arrived. +- Complete buffered Frames are processed before another read after resume; + incomplete input returns to bounded blocking receive. +- `pal_t.sleep_ms` is mandatory, so custom PALs and consumers must rebuild. diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index e06b4a1..876c0d6 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -829,6 +829,11 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) /* Consume frame from rx_buf */ memmove(ctx->rx_buf, ctx->rx_buf + needed, ctx->rx_len - needed); ctx->rx_len -= needed; + + /* During tai_connect(), stop immediately after the acknowledgement. + * Coalesced media stays buffered for the worker, preserving the public + * contract that application receive callbacks run on that thread. */ + if (ctx->connecting && ctx->session_ack >= 0) break; } return TAI_OK; From f3966d4150c018b518a86afbf05dddc06dc573ec Mon Sep 17 00:00:00 2001 From: wangsn Date: Tue, 22 Sep 2026 00:53:41 +0800 Subject: [PATCH 07/10] test(iot-client): exercise encrypted MQTT interruption ordering Route the combined regression through a TLS MQTT echo while TAI media uses the loopback peer, then cover authentication rejection, MQTT-before-media ordering, delayed TCP/MQTT duplicates, reconnect, and shutdown while pressured. Because the suite spawns mock subprocesses, keep it out of the Valgrind no-subprocess list. Verified against the focused TAI and combined regressions. Co-Authored-By: OpenAI Codex --- .gitlab-ci.yml | 5 +- CHANGELOG.md | 5 - CMakeLists.txt | 5 + .../ai/rtc-tcp-client/mqtt_interrupt_demo.c | 4 + .../iot-client/test/iot_tai_control_test.c | 283 ++++++++++++++++-- modules/rtc-tcp-client/src/tai_client.c | 2 +- .../rtc-tcp-client/test/tai_pal_loopback.c | 39 ++- .../rtc-tcp-client/test/tai_pal_loopback.h | 2 + .../rtc-tcp-client/test/test_integration.c | 20 ++ 9 files changed, 331 insertions(+), 34 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19280ae..0632b77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,14 +89,13 @@ test:memory-leak: # Leak-check the pure (no-subprocess) binaries, grouped by module: # rtc-tcp-client : tai_unit_tests, tai_log_level_zero_tests, # tai_log_module_zero_tests, tai_integration_tests - # iot-client : iot_cipher_test, iot_ai_ctrl_test, - # iot_tai_control_test + # iot-client : iot_cipher_test, iot_ai_ctrl_test # tuya-ble : tuya_ble_test # rtc-client is a prebuilt closed lib (headers + libstm.a) with no offline # test, so there is nothing to leak-check for it here. The mock-driven iot # tests are covered by the `test` job; under valgrind their Python # handshakes would time out. - for t in tai_unit_tests tai_log_level_zero_tests tai_log_module_zero_tests tai_integration_tests iot_cipher_test iot_ai_ctrl_test iot_tai_control_test tuya_ble_test; do + for t in tai_unit_tests tai_log_level_zero_tests tai_log_module_zero_tests tai_integration_tests iot_cipher_test iot_ai_ctrl_test tuya_ble_test; do echo "==================== valgrind: ${t} ====================" "$VG" "./${CMAKE_BUILD_DIR}/${t}" 2>&1 | tee "valgrind-${t}.log" rc=${PIPESTATUS[0]} diff --git a/CHANGELOG.md b/CHANGELOG.md index 43b5448..d8e9938 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- iot-client — authenticated MQTT protocol-9000 AI control notices can be delivered independently of RTC TCP receive backpressure (PR pending). - - Register `ai_ctrl_callback_t` with `iot_ai_ctrl_set_callback()` and keep one application thread pumping MQTT. - docs-site — English edition of the full documentation site, published at `/en/` with Simplified Chinese retained at `/`. - All 29 docs are mirrored under `docs-site/i18n/en/`, with a locale selector in both the landing-page navbar and the custom docs topbar, localized navbar/footer/sidebar catalogs, and English SVG schematics under `current/images/`. @@ -50,9 +48,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- rtc-tcp-client — enforce receive backpressure before reads and between Frames; resume buffered Frames first with a fresh liveness budget and timed CPU yields (PR pending). - - Custom PAL ports must supply the new mandatory `pal_t.sleep_ms` callback (IoT and TAI); rebuild all consumers because `pal_t` grows. `tcp_poll` timeout semantics are unchanged. - - tuya-ble — accept incoming Trsmitr versions >= 2 for app Pairing compatibility while keeping TX at version 4 (#39). - Docs — the pair-by-ble callback sample no longer claims credentials are never logged; the demo's DEBUG protocol log prints the credential JSON on purpose, for device bring-up. diff --git a/CMakeLists.txt b/CMakeLists.txt index 70a1188..6eaa8aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -467,6 +467,11 @@ if(AGENTIC_KIT_BUILD_TESTS AND AGENTIC_KIT_ENABLE_PROJECT_TESTS) agentic_kit_pal Threads::Threads ) + target_compile_definitions(iot_tai_control_test PRIVATE + PYTHON3_EXEC="${Python3_EXECUTABLE}" + TEST_CONFIG_DIR="${IOT_CLIENT_TESTS_DIR}/config" + MESSAGE_MOCK_PATH="${IOT_CLIENT_TESTS_DIR}/mock/message_mock.py" + ) add_test(NAME iot_tai_control_test COMMAND iot_tai_control_test) # The ble sources compile into the executable (not the tuya_ble diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c index d99caa4..ff83750 100644 --- a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -142,6 +142,10 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data msg->event_id && msg->event_id[0] ? msg->event_id : "(unscoped)"); } else if (msg->event_type == TAI_EVT_END) { pthread_mutex_lock(&state->mutex); + if (msg->event_id && state->current_event_id[0] && + strcmp(state->current_event_id, msg->event_id) == 0) { + state->current_event_id[0] = '\0'; + } state->done = 1; pthread_mutex_unlock(&state->mutex); } diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c index 64ac53b..2cbdd8c 100644 --- a/modules/iot-client/test/iot_tai_control_test.c +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -4,17 +4,90 @@ * refilling playback and a new Event remains playable. */ +#include +#include #include +#include #include +#include #include +#include +#include #include +#include -#include "iot_ai_ctrl.h" #include "iot_client.h" +#include "iot_client_message.h" #include "tai_internal.h" #include "tai_pal_loopback.h" +#define TEST_DEVID "test_device_msg_001" +#define TEST_SECRET_KEY "abcdef1234567890" +#define TEST_LOCAL_KEY "0123456789abcdef" +#define TEST_MQTT_URL "mqtts://127.0.0.1:11885" +#define TEST_MQTT_PORT 11885 +#define WRONG_KEY_PORT 11887 + +extern const pal_t *tai_pal_posix(void); + static int failures; +static const pal_t *posix_pal; +static const pal_t *loopback_pal; +static pal_t routing_pal; + +static int is_loopback_handle(void *tcp) +{ + return tcp == (void *)0x1; +} + +static void *routing_tcp_connect(const char *host, uint16_t port, + uint32_t timeout_ms) +{ + if (host && strcmp(host, "loopback.test") == 0) { + return loopback_pal->tcp_connect(host, port, timeout_ms); + } + return posix_pal->tcp_connect(host, port, timeout_ms); +} + +static int routing_tcp_send(void *tcp, const uint8_t *buf, size_t len, + uint32_t timeout_ms) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + return owner->tcp_send(tcp, buf, len, timeout_ms); +} + +static int routing_tcp_recv(void *tcp, uint8_t *buf, size_t len, + uint32_t timeout_ms) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + return owner->tcp_recv(tcp, buf, len, timeout_ms); +} + +static void routing_tcp_close(void *tcp) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + owner->tcp_close(tcp); +} + +static int routing_tcp_poll(void *tcp, int events, uint32_t timeout_ms) +{ + const pal_t *owner = is_loopback_handle(tcp) ? loopback_pal : posix_pal; + return owner->tcp_poll(tcp, events, timeout_ms); +} + +static const pal_t *make_routing_pal(void) +{ + posix_pal = tai_pal_posix(); + loopback_pal = tai_pal_loopback(); + routing_pal = *posix_pal; + routing_pal.tcp_connect = routing_tcp_connect; + routing_pal.tcp_send = routing_tcp_send; + routing_pal.tcp_recv = routing_tcp_recv; + routing_pal.tcp_close = routing_tcp_close; + routing_pal.tcp_poll = routing_tcp_poll; + return &routing_pal; +} + #define CHECK(expr) \ do { \ @@ -38,11 +111,83 @@ typedef struct { int accepted_new_audio; int chat_breaks; int mqtt_interrupts; + pthread_t mqtt_callback_thread; char current_event_id[64]; char stale_event_id[64]; } app_state_t; static app_state_t state; +static pid_t mqtt_mock_pid = -1; +static pid_t wrong_key_mock_pid = -1; + +static char *load_file(const pal_t *pal, const char *path) +{ + FILE *file = fopen(path, "rb"); + if (!file) return NULL; + fseek(file, 0, SEEK_END); + long len = ftell(file); + fseek(file, 0, SEEK_SET); + char *data = (char *)pal->malloc((size_t)len + 1); + if (data && fread(data, 1, (size_t)len, file) == (size_t)len) { + data[len] = '\0'; + } else { + pal->free(data); + data = NULL; + } + fclose(file); + return data; +} + +static int wait_for_port(uint16_t port) +{ + for (int attempt = 0; attempt < 100; attempt++) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + struct sockaddr_in address = {0}; + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + int result = connect(fd, (struct sockaddr *)&address, sizeof(address)); + close(fd); + if (result == 0) return 0; + sleep_ms(20); + } + return -1; +} + +static int start_mqtt_mock(void) +{ + mqtt_mock_pid = fork(); + if (mqtt_mock_pid == 0) { + execlp(PYTHON3_EXEC, PYTHON3_EXEC, MESSAGE_MOCK_PATH, NULL); + _exit(1); + } + if (mqtt_mock_pid < 0) return -1; + + wrong_key_mock_pid = fork(); + if (wrong_key_mock_pid == 0) { + setenv("MESSAGE_MOCK_TYPE", "wrong_key_encrypted", 1); + setenv("MESSAGE_MOCK_PORT", "11887", 1); + execlp(PYTHON3_EXEC, PYTHON3_EXEC, MESSAGE_MOCK_PATH, NULL); + _exit(1); + } + if (wrong_key_mock_pid < 0) return -1; + return wait_for_port(TEST_MQTT_PORT) || wait_for_port(WRONG_KEY_PORT); +} + +static void stop_mqtt_mock(void) +{ + if (wrong_key_mock_pid > 0) { + kill(wrong_key_mock_pid, SIGTERM); + waitpid(wrong_key_mock_pid, NULL, 0); + wrong_key_mock_pid = -1; + } + if (mqtt_mock_pid > 0) { + kill(mqtt_mock_pid, SIGTERM); + waitpid(mqtt_mock_pid, NULL, 0); + mqtt_mock_pid = -1; + } +} #define WAIT_FOR(expr, timeout_ms) \ ({ \ @@ -74,6 +219,10 @@ static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, app_state_t *s = (app_state_t *)user_data; pthread_mutex_lock(&s->mutex); s->audio_callbacks++; + if (s->stale_event_id[0] && msg->event_id && msg->event_id[0] && + strcmp(s->stale_event_id, msg->event_id) != 0) { + s->stale_event_id[0] = '\0'; + } int stale = s->stale_event_id[0] && msg->event_id && strcmp(s->stale_event_id, msg->event_id) == 0; if (!stale && s->queued_audio == 0) { @@ -100,6 +249,10 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, strcmp(s->stale_event_id, msg->event_id) != 0) { s->stale_event_id[0] = '\0'; } + } else if (msg->event_type == TAI_EVT_END && msg->event_id && msg->event_id[0] && + s->current_event_id[0] && + strcmp(s->current_event_id, msg->event_id) == 0) { + s->current_event_id[0] = '\0'; } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { s->chat_breaks++; if (!msg->event_id || !msg->event_id[0] || !s->current_event_id[0] || @@ -120,23 +273,30 @@ static void on_ai_control(const char *type, const char *json_data, app_state_t *s = (app_state_t *)user_data; if (strcmp(type, "asrInterrupt") != 0) return; - static const char expected[] = "\"eventId\":\"old-event\""; - int matched = 0; - for (size_t i = 0; i + sizeof(expected) - 1 <= data_len; i++) { - if (memcmp(json_data + i, expected, sizeof(expected) - 1) == 0) { - matched = 1; - break; + static const char prefix[] = "\"eventId\":\""; + char event_id[64] = {0}; + for (size_t i = 0; i + sizeof(prefix) - 1 <= data_len; i++) { + if (memcmp(json_data + i, prefix, sizeof(prefix) - 1) != 0) continue; + const char *value = json_data + i + sizeof(prefix) - 1; + size_t available = data_len - (size_t)(value - json_data); + const char *end = memchr(value, '"', available); + if (end) { + size_t len = (size_t)(end - value); + if (len >= sizeof(event_id)) len = sizeof(event_id) - 1; + memcpy(event_id, value, len); } + break; } - if (!matched) return; + if (!event_id[0]) return; pthread_mutex_lock(&s->mutex); + s->mqtt_callback_thread = pthread_self(); s->mqtt_interrupts++; if (!s->current_event_id[0] || - strcmp(s->current_event_id, "old-event") == 0) { + strcmp(s->current_event_id, event_id) == 0) { s->queued_audio = 0; snprintf(s->stale_event_id, sizeof(s->stale_event_id), - "%s", "old-event"); + "%s", event_id); } pthread_mutex_unlock(&s->mutex); } @@ -198,16 +358,39 @@ static int server_push_audio(tai_ctx_t *ctx, const char *event_id, int main(void) { - const pal_t *pal = tai_pal_loopback(); + const pal_t *pal = make_routing_pal(); pthread_mutex_init(&state.mutex, NULL); - tai_loopback_reset(); - tai_loopback_set_local_key("test-local-key-16"); CHECK(iot_init(pal) == OPRT_OK); + CHECK(start_mqtt_mock() == 0); + char *cacert = load_file(pal, TEST_CONFIG_DIR "/root_cert.pem"); + CHECK(cacert != NULL); iot_client_t iot; memset(&iot, 0, sizeof(iot)); iot.pal = pal; + snprintf(iot.devid, sizeof(iot.devid), "%s", TEST_DEVID); + snprintf(iot.secret_key, sizeof(iot.secret_key), "%s", TEST_SECRET_KEY); + snprintf(iot.local_key, sizeof(iot.local_key), "%s", TEST_LOCAL_KEY); + snprintf(iot.mqtt_url, sizeof(iot.mqtt_url), "%s", TEST_MQTT_URL); + iot.cacert = cacert; CHECK(iot_ai_ctrl_set_callback(&iot, on_ai_control, &state) == OPRT_OK); + CHECK(iot_client_message_connect(&iot) == OPRT_OK); + /* Consume the mock's initial raw message before the combined scenario. */ + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + + /* An authenticated-looking payload encrypted with another local key must + * never reach the control callback. */ + iot_client_t wrong_key_iot = iot; + wrong_key_iot.mqtt = NULL; + snprintf(wrong_key_iot.mqtt_url, sizeof(wrong_key_iot.mqtt_url), + "mqtts://127.0.0.1:%u", WRONG_KEY_PORT); + CHECK(iot_client_message_connect(&wrong_key_iot) == OPRT_OK); + CHECK(iot_client_message_process(&wrong_key_iot, 50) == OPRT_OK); + CHECK(state.mqtt_interrupts == 0); + iot_client_message_disconnect(&wrong_key_iot); + + tai_loopback_reset(); + tai_loopback_set_local_key("test-local-key-16"); tai_config_t config = { .host = "loopback.test", @@ -233,6 +416,27 @@ int main(void) uint8_t sent[2048]; (void)tai_loopback_pop_sent(sent, sizeof(sent)); uint16_t sequence = 10; + static const char early_control[] = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\"," + "\"data\":{\"eventId\":\"early-event\"}}}}"; + CHECK(iot_client_message_publish(&iot, (const uint8_t *)early_control, + sizeof(early_control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && state.mqtt_interrupts == 0; attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + CHECK(state.mqtt_interrupts == 1); + CHECK(state.queued_audio == 0); + /* MQTT-before-audio ordering: stale early-event state is discarded when the + * next audio packet identifies a different Event. */ + CHECK(server_push_audio(ctx, "old-event", 0x01, sequence++) > 0); + CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); + CHECK(state.queued_audio == 1); + + state.queued_audio = 0; + state.accepted_old_audio = 0; + state.audio_callbacks = 0; + CHECK(server_push_event(ctx, TAI_EVT_END, "early-event", sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_START, "old-event", sequence++) > 0); CHECK(server_push_audio(ctx, "old-event", 0x11, sequence++) > 0); CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); @@ -243,6 +447,9 @@ int main(void) CHECK(server_push_event(ctx, TAI_EVT_END, "old-event", sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_START, "new-event", sequence++) > 0); CHECK(server_push_audio(ctx, "new-event", 0x33, sequence++) > 0); + /* Keep admission closed while the delayed duplicate waits on TCP. */ + CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, + "old-event", sequence++) > 0); sleep_ms(100); CHECK(state.audio_callbacks == 1); CHECK(state.chat_breaks == 0); @@ -251,9 +458,14 @@ int main(void) "{\"protocol\":9000,\"data\":{\"data\":{" "\"type\":\"asrInterrupt\"," "\"data\":{\"eventId\":\"old-event\"}}}}"; - CHECK(iot_ai_ctrl_dispatch(&iot, (const uint8_t *)control, - sizeof(control) - 1)); - CHECK(state.mqtt_interrupts == 1); + pthread_t mqtt_owner = pthread_self(); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && state.mqtt_interrupts < 2; attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + CHECK(state.mqtt_interrupts == 2); + CHECK(pthread_equal(mqtt_owner, state.mqtt_callback_thread)); CHECK(state.chat_breaks == 0); CHECK(WAIT_FOR(state.accepted_new_audio == 1, 2000)); @@ -263,15 +475,46 @@ int main(void) CHECK(state.chat_breaks == 1); CHECK(state.queued_audio == 1); - /* Delayed duplicate for the old Event must not flush the new playback. */ - CHECK(iot_ai_ctrl_dispatch(&iot, (const uint8_t *)control, - sizeof(control) - 1)); - CHECK(state.mqtt_interrupts == 2); + /* Release pressure by consuming new audio; the queued duplicate is ignored. */ + state.queued_audio = 0; + CHECK(WAIT_FOR(state.chat_breaks == 2, 2000)); + CHECK(state.queued_audio == 0); + CHECK(state.accepted_new_audio == 1); + + /* Delayed duplicate on MQTT must also not flush the new playback. */ + state.queued_audio = 1; + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && state.mqtt_interrupts < 3; attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + CHECK(state.mqtt_interrupts == 3); CHECK(state.queued_audio == 1); CHECK(state.accepted_new_audio == 1); + /* MQTT loss removes only the independent control path. TAI stays paused on + * the full playback queue; reconnect restores authenticated delivery. */ + iot_client_message_disconnect(&iot); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_UNINITIALIZED); + CHECK(state.queued_audio == 1); + CHECK(iot_client_message_connect(&iot) == OPRT_OK); + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, + sizeof(control) - 1) == OPRT_OK); + for (int attempt = 0; attempt < 50 && state.mqtt_interrupts < 4; attempt++) { + CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); + } + CHECK(state.mqtt_interrupts == 4); + CHECK(state.queued_audio == 1); + + /* Shutdown while receive admission is still closed must not hang. */ tai_disconnect(ctx); tai_ctx_deinit(ctx); + iot_client_message_disconnect(&iot); + iot.cacert = NULL; + pal->free(cacert); + stop_mqtt_mock(); pthread_mutex_destroy(&state.mutex); printf("iot_tai_control_test: %s\n", failures ? "FAIL" : "PASS"); return failures ? 1 : 0; diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index 876c0d6..8ce549d 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -1257,7 +1257,7 @@ static void *worker_thread(void *arg) TAI_LOGD(ctx->pal, TAG, "worker: started"); int paused = 0; - int resume_buffered = 0; + int resume_buffered = ctx->rx_len > 0; uint64_t last_resume_ms = 0; /* The worker owns the disconnect decision: lower layers RETURN a fatal diff --git a/modules/rtc-tcp-client/test/tai_pal_loopback.c b/modules/rtc-tcp-client/test/tai_pal_loopback.c index 573f870..a4a398f 100644 --- a/modules/rtc-tcp-client/test/tai_pal_loopback.c +++ b/modules/rtc-tcp-client/test/tai_pal_loopback.c @@ -14,6 +14,10 @@ #include "log.h" #include "../src/tai_internal.h" /* frame/packet codec + key derivation for the handshake mock */ +extern const char *test_reserved_audio_event_id; +extern uint8_t test_reserved_audio_body[40]; +extern size_t test_reserved_audio_body_len; + /* ========================================================================= * Byte FIFO: single-producer, single-consumer, mutex-protected. * ========================================================================= */ @@ -349,11 +353,36 @@ static void lb_hs_feed(const uint8_t *buf, size_t len) ack_na ? ack_attrs : NULL, ack_na, (const uint8_t *)"", 0, app, sizeof(app)); if (alen > 0) { - uint8_t frame[128]; - int flen = tai_frame_encode(TAI_FRAG_NONE, 1, app, (size_t)alen, - sign_key, sig_len, tai_pal_loopback(), - frame, sizeof(frame)); - if (flen > 0) lb_fifo_push(&g_rx, frame, (size_t)flen); + uint8_t batch[384]; + int batch_len = tai_frame_encode(TAI_FRAG_NONE, 1, app, (size_t)alen, + sign_key, sig_len, tai_pal_loopback(), + batch, sizeof(batch)); + if (batch_len > 0 && g_hs.mode == TAI_LB_HS_ACK_WITH_TEXT) { + uint8_t text_payload[32]; + int text_header_len = tai_pack_text_hdr(TAI_VER_21, + TAI_DATA_ID_TEXT_DOWN, + TAI_STREAM_ONE_SHOT, 1, + text_payload, + sizeof(text_payload)); + static const char text[] = "coalesced"; + if (text_header_len > 0 && + (size_t)text_header_len + sizeof(text) - 1 <= sizeof(text_payload)) { + memcpy(text_payload + text_header_len, text, sizeof(text) - 1); + uint8_t text_app[64]; + int text_app_len = tai_packet_encode( + TAI_VER_21, TAI_PKT_TEXT, NULL, 0, + text_payload, (size_t)text_header_len + sizeof(text) - 1, + text_app, sizeof(text_app)); + if (text_app_len > 0) { + int text_frame_len = tai_frame_encode( + TAI_FRAG_NONE, 2, text_app, (size_t)text_app_len, + sign_key, sig_len, tai_pal_loopback(), + batch + batch_len, sizeof(batch) - (size_t)batch_len); + if (text_frame_len > 0) batch_len += text_frame_len; + } + } + } + if (batch_len > 0) lb_fifo_push(&g_rx, batch, (size_t)batch_len); } g_hs.done = 1; } diff --git a/modules/rtc-tcp-client/test/tai_pal_loopback.h b/modules/rtc-tcp-client/test/tai_pal_loopback.h index 6cc87b2..f20297c 100644 --- a/modules/rtc-tcp-client/test/tai_pal_loopback.h +++ b/modules/rtc-tcp-client/test/tai_pal_loopback.h @@ -76,6 +76,8 @@ void tai_loopback_close_connection(void); #define TAI_LB_HS_AUTH_OK 4 /* confirm via an AuthenticateResponse (pkt 3) with * connection-status-code 200, the way the production * server does, instead of a SessionNew ack. */ +#define TAI_LB_HS_ACK_WITH_TEXT 5 /* push SessionNew ack and a signed Text Frame in + * one receive batch; Text must reach the worker. */ void tai_loopback_set_local_key(const char *local_key); void tai_loopback_set_handshake_mode(int mode); diff --git a/modules/rtc-tcp-client/test/test_integration.c b/modules/rtc-tcp-client/test/test_integration.c index 8e752d8..d872365 100644 --- a/modules/rtc-tcp-client/test/test_integration.c +++ b/modules/rtc-tcp-client/test/test_integration.c @@ -109,6 +109,8 @@ typedef struct { } test_state_t; static test_state_t g_st; +static pthread_t g_connect_thread; +static pthread_t g_text_callback_thread; static void st_reset(void) { @@ -146,6 +148,7 @@ static void on_text(tai_ctx_t *ctx, const tai_text_msg_t *msg, void *ud) { (void)ctx; (void)ud; pthread_mutex_lock(&g_st.mtx); + g_text_callback_thread = pthread_self(); g_st.text_calls++; g_st.last_text_flag = msg->stream_flag; if (msg->event_id) { @@ -1947,6 +1950,23 @@ static void test_confirmed_connect(void) tai_ctx_deinit(ctx); } + /* Coalesced ack + application data: tai_connect consumes only the ack and + * leaves Text buffered for the receive worker. The callback must not run on + * the connecting thread even when both Frames arrived in one recv. */ + { + tai_ctx_t *ctx = setup_ctx(ctx_mem); + CHECK(ctx != NULL); + g_connect_thread = pthread_self(); + memset(&g_text_callback_thread, 0, sizeof(g_text_callback_thread)); + tai_loopback_set_handshake_mode(TAI_LB_HS_ACK_WITH_TEXT); + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(g_st.text_calls == 1, 1000)); + CHECK(strcmp(g_st.text_buf, "coalesced") == 0); + CHECK(!pthread_equal(g_connect_thread, g_text_callback_thread)); + tai_disconnect(ctx); + tai_ctx_deinit(ctx); + } + /* AuthenticateResponse ack: the production server confirms the connect with * an AuthenticateResponse (pkt 3) carrying connection-status-code 200, NOT a * SessionNew ack. tai_connect must complete. (Regression: pkt 3 used to fall From 8fa4b32d22aee6757ad9f05b40e9646a18c68d86 Mon Sep 17 00:00:00 2001 From: wangsn Date: Tue, 22 Sep 2026 13:03:22 +0800 Subject: [PATCH 08/10] fix(rtc-tcp-client,iot-client): harden backpressure and interruption ordering Audio Packets paused mid-body now retain a pending-delivery cursor instead of silently dropping codec frames; reopening admission resumes from the first unadmitted byte, and the pinned Frame is slid only after the remainder drains. The slide boundary is verified against the Frame's own length field, so a metadata/storage mismatch fails fast instead of desyncing the stream. The MQTT control demo now handles early interrupts (before an Event's TCP START, fail-closed unscoped interrupts, and stale Event ENDs without terminating the Session. iot_ai_ctrl no longer rewrites a scoped interrupt to an unscoped empty payload when payload serialization fails. The combined regression is fully synchronized (mutex-guarded assertions), covers same-event MQTT-before-START ordering, and records interruption ordering inside the callback instead of inferring it post-hoc. Verified: 20/20 tests pass serially and under ASan/UBSan (7338 assertions in tai_integration_tests, including byte-exact multi-Packet backpressure with per-codec-frame admission). Co-Authored-By: MightyCode EOF ) --- CHANGELOG.md | 7 + .../ai/rtc-tcp-client/mqtt_interrupt_demo.c | 101 ++++++---- modules/iot-client/src/iot_ai_ctrl.c | 18 +- modules/iot-client/test/iot_ai_ctrl_test.c | 56 ++++++ .../iot-client/test/iot_tai_control_test.c | 94 +++++---- modules/rtc-tcp-client/src/tai_client.c | 66 ++++++- modules/rtc-tcp-client/src/tai_internal.h | 32 +++ modules/rtc-tcp-client/src/tai_protocol.c | 114 +++++++++-- .../rtc-tcp-client/test/test_integration.c | 186 ++++++++++++++++-- 9 files changed, 570 insertions(+), 104 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e9938..e6d2a8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- iot-client — MQTT protocol-9000 AI control channel for server-initiated `asrInterrupt` notices, delivered via `ai_ctrl_callback_t` independently of the RTC TCP Connection. +- rtc-tcp-client — `on_flow_control` admission hook for TCP receive backpressure; pauses all inbound Frames when the application's audio queue is full, including codec-frame callbacks mid-Packet. + +### Changed + +- **BREAKING** pal — `pal_t` gains a mandatory `sleep_ms` member; custom PALs must supply it and all consumers must rebuild. +- rtc-tcp-client — audio Packets paused mid-body now retain a pending-delivery cursor instead of silently dropping codec frames; reopening admission resumes from the first unadmitted byte. - docs-site — English edition of the full documentation site, published at `/en/` with Simplified Chinese retained at `/`. - All 29 docs are mirrored under `docs-site/i18n/en/`, with a locale selector in both the landing-page navbar and the custom docs topbar, localized navbar/footer/sidebar catalogs, and English SVG schematics under `current/images/`. diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c index ff83750..733ec39 100644 --- a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -14,7 +14,7 @@ #include "demo_json.h" #include "iot_client.h" -#include "tuya_ai.h" +#include "tai_internal.h" extern const pal_t *tai_pal_posix(void); @@ -22,6 +22,7 @@ extern const pal_t *tai_pal_posix(void); #define DEFAULT_SECRET_KEY "[SPT;N:b@)wPzK/)" #define DEFAULT_LOCAL_KEY "#d[<4y*N.vE]RAAG" #define PLAYBACK_CAPACITY 4 +#define MAX_EVENT_ID 64 #define WAIT_STEP_MS 50U #define WAIT_LIMIT_MS 60000U @@ -35,28 +36,64 @@ typedef struct { int done; int playback_running; size_t queued_audio; - int drop_until_new_event; - char current_event_id[64]; - char stale_event_id[64]; + int drop_unscoped_audio; + char current_event_id[MAX_EVENT_ID]; + char recent_event_ids[4][MAX_EVENT_ID]; + size_t recent_count; } demo_state_t; +static int event_id_seen(const demo_state_t *state, const char *event_id) +{ + for (size_t i = 0; i < state->recent_count; i++) { + if (strcmp(state->recent_event_ids[i], event_id) == 0) return 1; + } + return 0; +} + +/* An event id remains unusable after playback for it has been interrupted. + * Once it disappears from the bounded current/recent record, a new TCP Event + * with that same id is ambiguous and must not silently clear newer playback. */ +static int mark_event_stale(demo_state_t *state, const char *event_id) +{ + if (!event_id_seen(state, event_id)) { + if (state->recent_count == 4) { + memmove(state->recent_event_ids, state->recent_event_ids + 1, + 3 * sizeof(state->recent_event_ids[0])); + state->recent_count = 3; + } + snprintf(state->recent_event_ids[state->recent_count++], + sizeof(state->recent_event_ids[0]), "%s", event_id); + } + return 1; +} + +static void forget_uninterrupted_event(demo_state_t *state, const char *event_id) +{ + for (size_t i = 0; i < state->recent_count; i++) { + if (strcmp(state->recent_event_ids[i], event_id) != 0) continue; + memmove(state->recent_event_ids + i, state->recent_event_ids + i + 1, + (state->recent_count - i - 1) * sizeof(state->recent_event_ids[0])); + state->recent_count--; + return; + } +} + static void set_stale_event(demo_state_t *state, const char *event_id) { pthread_mutex_lock(&state->mutex); + state->queued_audio = 0; if (event_id && event_id[0]) { - /* A delayed duplicate for an older Event must not flush current audio. */ - if (state->current_event_id[0] && - strcmp(state->current_event_id, event_id) != 0) { - pthread_mutex_unlock(&state->mutex); - return; - } - state->queued_audio = 0; - snprintf(state->stale_event_id, sizeof(state->stale_event_id), - "%s", event_id); + mark_event_stale(state, event_id); + /* If pressure paused this Event inside a Packet, flush only its retained + * remainder; the worker then drains later wire Frames without letting + * already accepted bytes replay or stale callbacks refill playback. */ + if (state->tai) tai_discard_pending_audio(state->tai); } else { - state->queued_audio = 0; - state->drop_until_new_event = 1; - state->stale_event_id[0] = '\0'; + /* Missing/ambiguous correlation is fail-closed: discard audio until + * the TCP side identifies the next Event. A repeated wire START inside + * one multi-frame Packet must not reopen playback. */ + state->drop_unscoped_audio = 1; + if (state->tai) tai_discard_pending_audio(state->tai); } pthread_mutex_unlock(&state->mutex); } @@ -67,7 +104,7 @@ static void on_ai_control(const char *type, const char *json_data, demo_state_t *state = (demo_state_t *)user_data; if (strcmp(type, "asrInterrupt") != 0) return; - char event_id[64] = {0}; + char event_id[MAX_EVENT_ID] = {0}; static const char key[] = "\"eventId\":\""; const char *id = NULL; for (size_t i = 0; i + sizeof(key) - 1 <= data_len; i++) { @@ -106,19 +143,10 @@ static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *user_data (void)ctx; demo_state_t *state = (demo_state_t *)user_data; pthread_mutex_lock(&state->mutex); - if (state->drop_until_new_event && msg->stream_flag == TAI_STREAM_START) { - state->drop_until_new_event = 0; - } - if (state->stale_event_id[0] && msg->event_id && msg->event_id[0] && - strcmp(state->stale_event_id, msg->event_id) != 0) { - state->stale_event_id[0] = '\0'; - } - int stale = state->drop_until_new_event || - (state->stale_event_id[0] && msg->event_id && - strcmp(state->stale_event_id, msg->event_id) == 0); - if (!stale && state->queued_audio < PLAYBACK_CAPACITY) { - state->queued_audio++; - } + int stale = state->drop_unscoped_audio || + !msg->event_id || !msg->event_id[0] || + event_id_seen(state, msg->event_id); + if (!stale) state->queued_audio++; pthread_mutex_unlock(&state->mutex); } @@ -128,13 +156,11 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data demo_state_t *state = (demo_state_t *)user_data; if (msg->event_type == TAI_EVT_START && msg->event_id && msg->event_id[0]) { pthread_mutex_lock(&state->mutex); - snprintf(state->current_event_id, sizeof(state->current_event_id), - "%s", msg->event_id); - if (state->stale_event_id[0] && - strcmp(state->stale_event_id, msg->event_id) != 0) { - state->stale_event_id[0] = '\0'; + if (!event_id_seen(state, msg->event_id)) { + snprintf(state->current_event_id, sizeof(state->current_event_id), + "%s", msg->event_id); + state->drop_unscoped_audio = 0; } - state->drop_until_new_event = 0; pthread_mutex_unlock(&state->mutex); } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { set_stale_event(state, msg->event_id); @@ -145,8 +171,9 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data if (msg->event_id && state->current_event_id[0] && strcmp(state->current_event_id, msg->event_id) == 0) { state->current_event_id[0] = '\0'; + forget_uninterrupted_event(state, msg->event_id); + state->done = 1; } - state->done = 1; pthread_mutex_unlock(&state->mutex); } } diff --git a/modules/iot-client/src/iot_ai_ctrl.c b/modules/iot-client/src/iot_ai_ctrl.c index 1b7280d..ce8b0f9 100644 --- a/modules/iot-client/src/iot_ai_ctrl.c +++ b/modules/iot-client/src/iot_ai_ctrl.c @@ -35,9 +35,21 @@ bool iot_ai_ctrl_dispatch(iot_client_t *client, return false; } - char *payload_json = payload ? cJSON_PrintUnformatted(payload) : NULL; - const char *payload_bytes = payload_json ? payload_json : ""; - size_t payload_len = payload_json ? strlen(payload_json) : 0; + char *payload_json = NULL; + const char *payload_bytes = ""; + size_t payload_len = 0; + if (payload) { + payload_json = cJSON_PrintUnformatted(payload); + if (!payload_json) { + /* Do not turn a scoped notice into an unscoped empty payload under + * allocation pressure. Leave the authenticated message unconsumed + * so the application can apply its own fallback policy. */ + cJSON_Delete(root); + return false; + } + payload_bytes = payload_json; + payload_len = strlen(payload_json); + } client->ai_ctrl_callback(type->valuestring, payload_bytes, payload_len, client->ai_ctrl_user_data); diff --git a/modules/iot-client/test/iot_ai_ctrl_test.c b/modules/iot-client/test/iot_ai_ctrl_test.c index 121d913..13dda9e 100644 --- a/modules/iot-client/test/iot_ai_ctrl_test.c +++ b/modules/iot-client/test/iot_ai_ctrl_test.c @@ -12,6 +12,7 @@ #include "iot_ai_ctrl.h" #include "iot_client.h" #include "iot_internal.h" +#include "cJSON.h" static int tests_run; static int tests_passed; @@ -269,6 +270,60 @@ static int test_protocol_must_be_exact_integer(void) return failed ? -1 : 0; } +static int cjson_allocation_count; +static int cjson_fail_after; + +static void *counting_malloc(size_t size) +{ + cjson_allocation_count++; + if (cjson_fail_after >= 0 && cjson_allocation_count > cjson_fail_after) + return NULL; + return malloc(size); +} + +static int test_payload_print_allocation_failure_is_unconsumed(void) +{ + const pal_t *pal = get_default_pal(); + iot_client_t *client = make_client(pal); + const char *envelope = + "{\"protocol\":9000,\"data\":{\"data\":{" + "\"type\":\"asrInterrupt\",\"data\":{\"eventId\":\"evt-1\"}}}}"; + cJSON_Hooks counting_hooks = { counting_malloc, pal->free }; + cJSON_Hooks restore_hooks = { pal->malloc, pal->free }; + int result = -1; + + if (!client) return -1; + reset_callback(); + iot_ai_ctrl_set_callback(client, control_callback, NULL); + + cJSON_InitHooks(&counting_hooks); + cjson_allocation_count = 0; + cjson_fail_after = -1; + cJSON *root = cJSON_ParseWithLength(envelope, strlen(envelope)); + if (root) cJSON_Delete(root); + if (cjson_allocation_count <= 0) goto out; + + /* Sweep one failing allocation index at a time. The successful run proves + * every earlier run failed before the callback; no scoped payload was + * rewritten to an empty string. */ + result = -1; + for (cjson_fail_after = 0; cjson_fail_after < 32; cjson_fail_after++) { + cjson_allocation_count = 0; + int consumed = iot_ai_ctrl_dispatch(client, (const uint8_t *)envelope, + strlen(envelope)); + if (consumed) { + result = callback_count == 1 ? 0 : -1; + break; + } + if (callback_count != 0) break; + } + +out: + cJSON_InitHooks(&restore_hooks); + destroy_client(client); + return result; +} + int main(void) { const pal_t *pal = get_default_pal(); @@ -288,6 +343,7 @@ int main(void) RUN_TEST(test_deregister); RUN_TEST(test_set_callback_null_client); RUN_TEST(test_protocol_must_be_exact_integer); + RUN_TEST(test_payload_print_allocation_failure_is_unconsumed); printf("\n========== Results: %d/%d passed ==========\n", tests_passed, tests_run); diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c index 2cbdd8c..20ae1c9 100644 --- a/modules/iot-client/test/iot_tai_control_test.c +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -109,8 +109,10 @@ typedef struct { int audio_callbacks; int accepted_old_audio; int accepted_new_audio; + int accepted_early_audio; int chat_breaks; int mqtt_interrupts; + int interrupted_before_tcp_drain; pthread_t mqtt_callback_thread; char current_event_id[64]; char stale_event_id[64]; @@ -202,6 +204,16 @@ static void stop_mqtt_mock(void) found; \ }) +static int state_check(int expression) +{ + pthread_mutex_lock(&state.mutex); + int result = expression; + pthread_mutex_unlock(&state.mutex); + return result; +} + +#define STATE_CHECK(expr) CHECK(state_check(expr)) + static int flow_control(tai_ctx_t *ctx, void *user_data) { (void)ctx; @@ -231,6 +243,8 @@ static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, s->accepted_old_audio++; } else if (msg->event_id && strcmp(msg->event_id, "new-event") == 0) { s->accepted_new_audio++; + } else if (msg->event_id && strcmp(msg->event_id, "early-event") == 0) { + s->accepted_early_audio++; } } pthread_mutex_unlock(&s->mutex); @@ -292,6 +306,8 @@ static void on_ai_control(const char *type, const char *json_data, pthread_mutex_lock(&s->mutex); s->mqtt_callback_thread = pthread_self(); s->mqtt_interrupts++; + if (s->mqtt_interrupts == 2 && s->chat_breaks == 0) + s->interrupted_before_tcp_drain = 1; if (!s->current_event_id[0] || strcmp(s->current_event_id, event_id) == 0) { s->queued_audio = 0; @@ -386,7 +402,7 @@ int main(void) "mqtts://127.0.0.1:%u", WRONG_KEY_PORT); CHECK(iot_client_message_connect(&wrong_key_iot) == OPRT_OK); CHECK(iot_client_message_process(&wrong_key_iot, 50) == OPRT_OK); - CHECK(state.mqtt_interrupts == 0); + STATE_CHECK(state.mqtt_interrupts == 0); iot_client_message_disconnect(&wrong_key_iot); tai_loopback_reset(); @@ -422,24 +438,27 @@ int main(void) "\"data\":{\"eventId\":\"early-event\"}}}}"; CHECK(iot_client_message_publish(&iot, (const uint8_t *)early_control, sizeof(early_control) - 1) == OPRT_OK); - for (int attempt = 0; attempt < 50 && state.mqtt_interrupts == 0; attempt++) { + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 1, 5); attempt++) { CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } - CHECK(state.mqtt_interrupts == 1); - CHECK(state.queued_audio == 0); - /* MQTT-before-audio ordering: stale early-event state is discarded when the - * next audio packet identifies a different Event. */ - CHECK(server_push_audio(ctx, "old-event", 0x01, sequence++) > 0); + STATE_CHECK(state.mqtt_interrupts == 1); + STATE_CHECK(state.queued_audio == 0); + /* MQTT-before-audio ordering: early-event is cancelled before its START; + * its later audio must drain without entering playback. */ + CHECK(server_push_event(ctx, TAI_EVT_START, "early-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "early-event", 0x01, sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_END, "early-event", sequence++) > 0); + CHECK(server_push_event(ctx, TAI_EVT_START, "old-event", sequence++) > 0); + CHECK(server_push_audio(ctx, "old-event", 0x02, sequence++) > 0); CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); - CHECK(state.queued_audio == 1); + STATE_CHECK(state.accepted_early_audio == 0); + STATE_CHECK(state.queued_audio == 1); + pthread_mutex_lock(&state.mutex); state.queued_audio = 0; - state.accepted_old_audio = 0; - state.audio_callbacks = 0; - CHECK(server_push_event(ctx, TAI_EVT_END, "early-event", sequence++) > 0); - CHECK(server_push_event(ctx, TAI_EVT_START, "old-event", sequence++) > 0); + pthread_mutex_unlock(&state.mutex); CHECK(server_push_audio(ctx, "old-event", 0x11, sequence++) > 0); - CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); + CHECK(WAIT_FOR(state.accepted_old_audio == 2, 1000)); CHECK(server_push_audio(ctx, "old-event", 0x22, sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, @@ -451,8 +470,8 @@ int main(void) CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, "old-event", sequence++) > 0); sleep_ms(100); - CHECK(state.audio_callbacks == 1); - CHECK(state.chat_breaks == 0); + STATE_CHECK(state.audio_callbacks == 3); + STATE_CHECK(state.chat_breaks == 0); static const char control[] = "{\"protocol\":9000,\"data\":{\"data\":{" @@ -461,52 +480,59 @@ int main(void) pthread_t mqtt_owner = pthread_self(); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_OK); - for (int attempt = 0; attempt < 50 && state.mqtt_interrupts < 2; attempt++) { + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 2, 5); attempt++) { CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } - CHECK(state.mqtt_interrupts == 2); - CHECK(pthread_equal(mqtt_owner, state.mqtt_callback_thread)); - CHECK(state.chat_breaks == 0); + STATE_CHECK(state.mqtt_interrupts == 2); + pthread_mutex_lock(&state.mutex); + int same_thread = pthread_equal(mqtt_owner, state.mqtt_callback_thread); + pthread_mutex_unlock(&state.mutex); + CHECK(same_thread); + STATE_CHECK(state.interrupted_before_tcp_drain == 1); CHECK(WAIT_FOR(state.accepted_new_audio == 1, 2000)); - CHECK(state.audio_callbacks == 3); - CHECK(state.accepted_old_audio == 1); - CHECK(state.accepted_new_audio == 1); - CHECK(state.chat_breaks == 1); - CHECK(state.queued_audio == 1); + STATE_CHECK(state.audio_callbacks == 5); + STATE_CHECK(state.accepted_old_audio == 2); + STATE_CHECK(state.accepted_new_audio == 1); + STATE_CHECK(state.chat_breaks == 1); + STATE_CHECK(state.queued_audio == 1); /* Release pressure by consuming new audio; the queued duplicate is ignored. */ + pthread_mutex_lock(&state.mutex); state.queued_audio = 0; + pthread_mutex_unlock(&state.mutex); CHECK(WAIT_FOR(state.chat_breaks == 2, 2000)); - CHECK(state.queued_audio == 0); - CHECK(state.accepted_new_audio == 1); + STATE_CHECK(state.queued_audio == 0); + STATE_CHECK(state.accepted_new_audio == 1); /* Delayed duplicate on MQTT must also not flush the new playback. */ + pthread_mutex_lock(&state.mutex); state.queued_audio = 1; + pthread_mutex_unlock(&state.mutex); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_OK); - for (int attempt = 0; attempt < 50 && state.mqtt_interrupts < 3; attempt++) { + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 3, 5); attempt++) { CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } - CHECK(state.mqtt_interrupts == 3); - CHECK(state.queued_audio == 1); - CHECK(state.accepted_new_audio == 1); + STATE_CHECK(state.mqtt_interrupts == 3); + STATE_CHECK(state.queued_audio == 1); + STATE_CHECK(state.accepted_new_audio == 1); /* MQTT loss removes only the independent control path. TAI stays paused on * the full playback queue; reconnect restores authenticated delivery. */ iot_client_message_disconnect(&iot); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_UNINITIALIZED); - CHECK(state.queued_audio == 1); + STATE_CHECK(state.queued_audio == 1); CHECK(iot_client_message_connect(&iot) == OPRT_OK); CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_OK); - for (int attempt = 0; attempt < 50 && state.mqtt_interrupts < 4; attempt++) { + for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 4, 5); attempt++) { CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } - CHECK(state.mqtt_interrupts == 4); - CHECK(state.queued_audio == 1); + STATE_CHECK(state.mqtt_interrupts == 4); + STATE_CHECK(state.queued_audio == 1); /* Shutdown while receive admission is still closed must not hang. */ tai_disconnect(ctx); diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index 8ce549d..5a07b1a 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -640,6 +640,12 @@ void tai_disconnect(tai_ctx_t *ctx) ctx->rx_len = 0; ctx->frag_len = 0; ctx->frag_state = 0; + ctx->rx_pending_valid = 0; /* teardown drops a paused Packet remainder */ + ctx->rx_pending_discard = 0; + ctx->rx_pending_offset = 0; + ctx->rx_pending_len = 0; + ctx->rx_pending_body = NULL; + ctx->rx_pending_event_id[0] = '\0'; ctx->connecting = 0; /* clear in case a connect aborted mid-handshake */ ctx->disconnect_emitted = 0; /* re-arm the single-point on_disconnect */ ctx->rx_event_id[0] = '\0'; /* clear latched turn id so a reconnect starts clean */ @@ -736,6 +742,44 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) break; } + /* A codec-frame callback may have filled the app queue mid-Packet. + * Re-dispatching the Frame here would replay accepted bytes; drain only + * its explicit pending remainder instead. Once it drains, slide the + * pinned Frame out from rx_buf so the next loop starts at the Frame + * behind it. */ + if (ctx->rx_pending_valid) { + ctx->rx_pending_valid = tai_proto_drain_pending_audio( + ctx, ctx->rx_pending_from_frag); + if (ctx->rx_pending_valid) { + if (paused) *paused = 1; + break; + } + if (ctx->rx_pending_body_off && !ctx->rx_pending_from_frag) { + /* body_off−1 = byte offset of the body's first byte; +body_len + * reaches the end of the app payload; +sig_len reaches the end + * of the wire Frame. Verify against the Frame's own length + * field: they must agree, so a mismatch means the pending + * metadata was captured for the wrong storage. */ + size_t pending_end = ctx->rx_pending_body_off - 1 + + ctx->rx_pending_len + ctx->sig_len; + size_t wire_len = tai_frame_total_size(ctx->rx_buf, ctx->rx_len); + if (pending_end != wire_len) { + TAI_LOGE(ctx->pal, TAG, + "pending audio slide mismatch: end=%zu wire=%zu", + pending_end, wire_len); + return TAI_PROTO_ERR_FRAME_DECODE; + } + if (pending_end <= ctx->rx_len) { + memmove(ctx->rx_buf, ctx->rx_buf + pending_end, + ctx->rx_len - pending_end); + ctx->rx_len -= pending_end; + } + ctx->rx_pending_body = NULL; + ctx->rx_pending_body_off = 0; + } + continue; + } + /* Detect version from first byte */ int ver = tai_frame_detect_version(ctx->rx_buf[0]); if (ver < 0) { @@ -821,11 +865,22 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) } if (complete && app_bytes && app_len > 0) { + ctx->rx_dispatch_from_frag = (frag_flag == TAI_FRAG_LAST); int fatal = process_app_packet(ctx, app_bytes, app_len); + ctx->rx_dispatch_from_frag = 0; if (fatal != TAI_OK) return fatal; /* PROTOCOL detail or TAI_RX_PEER_CLOSE|code */ } + /* A mid-Packet pause pins this Frame's front; the paused body pointer + * therefore remains valid until the remainder drains. Once it drains, + * the pending-drain path above already slid the Frame, so this code + * runs only for the no-pending fast path. */ + if (ctx->rx_pending_valid) { + if (paused) *paused = 1; + break; + } + /* Consume frame from rx_buf */ memmove(ctx->rx_buf, ctx->rx_buf + needed, ctx->rx_len - needed); ctx->rx_len -= needed; @@ -1330,9 +1385,14 @@ static void *worker_thread(void *arg) } uint64_t drain_start = ctx->pal->time_ms(); - /* Only a parser pause leaves complete work behind. Partial input must - * return to a bounded blocking receive rather than spin. */ - int n = resume_buffered ? (int)ctx->rx_len : tai_recv_data(ctx, wait_ms); + /* A pending codec-frame remainder is not extra inbound bytes. Enter the + * drain directly so reopening admission resumes it without requiring a + * new receive; a pending rx_buf remainder also keeps the Frame pinned, + * making rx_len a valid resume marker. Partial input returns to a + * bounded blocking receive rather than spinning. */ + int n = ctx->rx_pending_valid ? 1 + : resume_buffered ? (int)ctx->rx_len + : tai_recv_data(ctx, wait_ms); resume_buffered = 0; /* Set when this pass drained real input; gates the end-of-pass yield * below so an idle pass (blocking wait timed out, no data) does not diff --git a/modules/rtc-tcp-client/src/tai_internal.h b/modules/rtc-tcp-client/src/tai_internal.h index 1f96242..b5cab43 100644 --- a/modules/rtc-tcp-client/src/tai_internal.h +++ b/modules/rtc-tcp-client/src/tai_internal.h @@ -215,6 +215,9 @@ struct tai_ctx { uint16_t rx_audio_frame_duration; /* ms */ uint16_t rx_audio_frame_size; /* bytes per Opus packet */ uint8_t rx_audio_codec; /* TAI_AUDIO_* from audio-params f[0]; 0=unknown */ + uint8_t rx_pending_audio_codec; + uint32_t rx_pending_audio_rate; + uint16_t rx_pending_frame_duration; /* Received turn id (attr 61) backing callback msg->event_id; "" if none */ char rx_event_id[64]; @@ -245,6 +248,23 @@ struct tai_ctx { /* Optional TCP receive backpressure (see tuya_ai.h). */ int (*on_flow_control)(tai_ctx_t *, void *); + /* A codec-frame Packet paused inside dispatch. The bytes and Event id stay + * worker-owned; pending_offset advances only after an admitted callback. + * Teardown deliberately discards any remainder. */ + uint8_t rx_pending_valid; + uint8_t rx_pending_flag; + uint8_t rx_pending_discard; + uint8_t rx_pending_from_frag; + uint8_t rx_dispatch_from_frag; + uint16_t rx_pending_data_id; + uint32_t rx_pending_frame_ms; + uint64_t rx_pending_ts_ms; + size_t rx_pending_offset; + size_t rx_pending_len; + size_t rx_pending_body_off; + const uint8_t *rx_pending_body; + char rx_pending_event_id[64]; + /* RX linear buffer (sliding-window: bytes always at buf[0]) */ uint8_t rx_buf[TAI_RX_BUF_SIZE]; size_t rx_len; @@ -384,6 +404,10 @@ int tai_unpack_media_hdr(const uint8_t *buf, size_t len, void tai_emit_disconnect(tai_ctx_t *ctx, uint8_t reason, uint8_t detail, uint16_t close_code); +/* Internal storage-origin hint for a dispatched Packet: 0 = rx_buf, 1 = frag_buf. + * Only tai_process_rx sets it around process_app_packet; it is not public API. */ +#define TAI_DISPATCH_STORAGE_FRAG 1 + /* Fail-fast cause, RETURNED up the receive chain (dispatch -> process_app_packet * -> tai_process_rx) to the worker, which owns the run loop and fires a single * on_disconnect. Lower layers never touch connection-lifecycle state: @@ -504,6 +528,14 @@ int tai_proto_dispatch (tai_ctx_t *ctx, uint8_t pkt_type, const tai_attr_t *attrs, int attr_count, const uint8_t *payload, size_t payload_len); +/* Drain one audio Packet paused inside callback delivery. Returns 1 while + * delivery remains paused, 0 once the Packet (or a deliberately discarded + * remainder) is finished. Callers only use it for a packet whose bytes are + * still pinned in the worker's rx_buf / frag_buf. */ +int tai_proto_drain_pending_audio(tai_ctx_t *ctx, int from_frag); +/* Mark the current pending remainder obsolete without delivering it. Used by + * an application interruption from another thread before reopening admission. */ +void tai_discard_pending_audio(tai_ctx_t *ctx); /* Internal sequence helper */ static inline uint16_t tai_next_seq(tai_ctx_t *ctx) { diff --git a/modules/rtc-tcp-client/src/tai_protocol.c b/modules/rtc-tcp-client/src/tai_protocol.c index e0c0f64..df2909e 100644 --- a/modules/rtc-tcp-client/src/tai_protocol.c +++ b/modules/rtc-tcp-client/src/tai_protocol.c @@ -605,27 +605,112 @@ static void parse_audio_params_once(tai_ctx_t *ctx, } /* Split an audio body into CBR Opus frames of rx_audio_frame_size, emitting a - * final short remainder. Whole frames are emitted zero-copy from `body` (valid - * until the frame is consumed from rx_buf). fs==0 (PCM / unknown) delivers the - * body whole. */ + * final short remainder. When flow control closes mid-body, metadata plus the + * first unadmitted offset are retained and the caller pins this Packet's wire + * storage. Reopening admission resumes from that offset; admitted bytes are + * never replayed. fs==0 (PCM / unknown) delivers the body whole once admitted, + * so an impossible-to-admit callback cannot spin the worker. */ static void media_audio_body(tai_ctx_t *ctx, const uint8_t *body, size_t body_len, - uint8_t stream_flag, uint16_t data_id, uint64_t ts_ms) + uint8_t stream_flag, uint16_t data_id, + uint64_t ts_ms, int from_frag) { if (!ctx->on_audio) return; uint16_t fs = ctx->rx_audio_frame_size; - if (fs == 0) { - if (body_len > 0) - emit_audio(ctx, body, body_len, stream_flag, data_id, ts_ms); - return; + size_t offset = 0; + while (offset < body_len) { + if (ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data)) { + size_t n = strlen(ctx->rx_event_id); + if (n >= sizeof(ctx->rx_pending_event_id)) + n = sizeof(ctx->rx_pending_event_id) - 1; + + memcpy(ctx->rx_pending_event_id, ctx->rx_event_id, n); + ctx->rx_pending_event_id[n] = '\0'; + ctx->rx_pending_valid = 1; + ctx->rx_pending_flag = stream_flag; + ctx->rx_pending_discard = 0; + ctx->rx_pending_data_id = data_id; + ctx->rx_pending_frame_ms = fs; + ctx->rx_pending_audio_codec = ctx->rx_audio_codec; + ctx->rx_pending_audio_rate = ctx->rx_audio_sample_rate; + ctx->rx_pending_frame_duration = ctx->rx_audio_frame_duration; + ctx->rx_pending_ts_ms = ts_ms; + ctx->rx_pending_offset = offset; + ctx->rx_pending_len = body_len; + ctx->rx_pending_body = body; + ctx->rx_pending_body_off = from_frag + ? 0 : (size_t)(body - ctx->rx_buf) + 1; + ctx->rx_pending_from_frag = (uint8_t)from_frag; + fprintf(stderr, "CAP off=%zu len=%zu event=%s\n", + offset, body_len, ctx->rx_pending_event_id); + return; + } + size_t len = fs == 0 || body_len - offset < fs ? body_len - offset : fs; + emit_audio(ctx, body + offset, len, stream_flag, data_id, ts_ms); + offset += len; + } +} + +int tai_proto_drain_pending_audio(tai_ctx_t *ctx, int from_frag) +{ + if (!ctx->rx_pending_valid) return 0; + if ((from_frag != 0) != (ctx->rx_pending_from_frag != 0)) { + TAI_LOGE(ctx->pal, TAG, "pending audio storage mismatch"); + return 1; /* keep the connection alive rather than resyncing blindly */ } - while (body_len >= fs) { - emit_audio(ctx, body, fs, stream_flag, data_id, ts_ms); - body += fs; body_len -= fs; + /* The caller pins the storage containing this Packet: the unslid rx_buf + * Frame for FRAG_NONE, or frag_buf until its next reassembly begins. */ + const uint8_t *body = ctx->rx_pending_body; + if (!body) return 1; + if (ctx->rx_pending_offset > ctx->rx_pending_len) return 0; + + while (ctx->rx_pending_offset < ctx->rx_pending_len) { + if (ctx->rx_pending_discard) { + ctx->rx_pending_valid = 0; + ctx->rx_pending_discard = 0; + return 0; + } + if (ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data)) { + return 1; + } + + size_t remaining = ctx->rx_pending_len - ctx->rx_pending_offset; + size_t len = ctx->rx_pending_frame_ms == 0 || + remaining < ctx->rx_pending_frame_ms + ? remaining : ctx->rx_pending_frame_ms; + char saved_event[sizeof(ctx->rx_event_id)]; + uint8_t saved_codec = ctx->rx_audio_codec; + uint32_t saved_rate = ctx->rx_audio_sample_rate; + uint16_t saved_duration = ctx->rx_audio_frame_duration; + memcpy(saved_event, ctx->rx_event_id, sizeof(saved_event)); + memcpy(ctx->rx_event_id, ctx->rx_pending_event_id, + sizeof(ctx->rx_event_id)); + ctx->rx_audio_codec = ctx->rx_pending_audio_codec; + ctx->rx_audio_sample_rate = ctx->rx_pending_audio_rate; + ctx->rx_audio_frame_duration = ctx->rx_pending_frame_duration; + + emit_audio(ctx, body + ctx->rx_pending_offset, len, + ctx->rx_pending_flag, ctx->rx_pending_data_id, + ctx->rx_pending_ts_ms); + memcpy(ctx->rx_event_id, saved_event, sizeof(saved_event)); + ctx->rx_audio_codec = saved_codec; + ctx->rx_audio_sample_rate = saved_rate; + ctx->rx_audio_frame_duration = saved_duration; + ctx->rx_pending_offset += len; } - if (body_len > 0) - emit_audio(ctx, body, body_len, stream_flag, data_id, ts_ms); + + ctx->rx_pending_valid = 0; + ctx->rx_pending_body = NULL; + return 0; +} + +void tai_discard_pending_audio(tai_ctx_t *ctx) +{ + if (!ctx) return; + if (ctx->rx_pending_valid) ctx->rx_pending_discard = 1; } /* AUDIO packet payload: [data_id:2][48-bit stream_flag|ts_ms][opus frames…]. */ @@ -654,7 +739,8 @@ static int media_audio(tai_ctx_t *ctx, parse_audio_params_once(ctx, attrs, attr_count); latch_event_id(ctx, attrs, attr_count); - media_audio_body(ctx, payload + 8, payload_len - 8, stream_flag, data_id, ts_ms); + media_audio_body(ctx, payload + 8, payload_len - 8, stream_flag, data_id, + ts_ms, ctx->rx_dispatch_from_frag); return TAI_OK; } diff --git a/modules/rtc-tcp-client/test/test_integration.c b/modules/rtc-tcp-client/test/test_integration.c index d872365..d8157e2 100644 --- a/modules/rtc-tcp-client/test/test_integration.c +++ b/modules/rtc-tcp-client/test/test_integration.c @@ -81,7 +81,7 @@ typedef struct { size_t audio_bytes; uint32_t audio_sample_rate; uint16_t audio_frame_duration; - size_t audio_frame_lens[256]; /* per-call msg->len (carry-alignment check) */ + size_t audio_frame_lens[160]; /* per-call msg->len (carry-alignment check) */ uint8_t audio_concat[8192]; /* concatenation of all delivered frames */ size_t audio_concat_len; @@ -2067,16 +2067,33 @@ static void test_disconnect_latency(void) static const pal_t *observed_base_pal; static pal_t observed_pal; static size_t observed_recv_calls; +static size_t observed_recv_bytes; static size_t observed_sleep_calls; static size_t observed_empty_polls; +static size_t observed_after_bytes; +static size_t observed_blocking_after_bytes; +static size_t observed_nonblocking_after_bytes; +static uint32_t observed_max_timeout_after_bytes; static int observed_recv(void *tcp, uint8_t *buf, size_t len, uint32_t timeout_ms) { pthread_mutex_lock(&g_st.mtx); observed_recv_calls++; + if (observed_after_bytes && observed_recv_bytes >= observed_after_bytes) { + if (timeout_ms) observed_blocking_after_bytes++; + else observed_nonblocking_after_bytes++; + if (timeout_ms > observed_max_timeout_after_bytes) + observed_max_timeout_after_bytes = timeout_ms; + } pthread_mutex_unlock(&g_st.mtx); - return observed_base_pal->tcp_recv(tcp, buf, len, timeout_ms); + + int n = observed_base_pal->tcp_recv(tcp, buf, len, timeout_ms); + + pthread_mutex_lock(&g_st.mtx); + if (n > 0) observed_recv_bytes += (size_t)n; + pthread_mutex_unlock(&g_st.mtx); + return n; } static int observed_poll(void *tcp, int events, uint32_t timeout_ms) @@ -2103,13 +2120,19 @@ static const pal_t *observe_pal(void) observed_pal.tcp_poll = observed_poll; observed_pal.sleep_ms = observed_sleep; observed_recv_calls = 0; + observed_recv_bytes = 0; observed_sleep_calls = 0; observed_empty_polls = 0; + observed_after_bytes = 0; + observed_blocking_after_bytes = 0; + observed_nonblocking_after_bytes = 0; + observed_max_timeout_after_bytes = 0; return &observed_pal; } -/* Pause after each delivered Packet, including Packets buffered by one recv. - * Resume without new network bytes to prove rx_buf work cannot be stranded. */ +/* Pause after each delivered codec frame, including frames produced by one + * Packet. Resume without new network bytes to prove no pending callback or + * buffered Frame is stranded. */ static int flow_allowed_calls; static int flow_paused_queries; static uint64_t flow_resume_ms; @@ -2125,6 +2148,12 @@ static int test_flow_control(tai_ctx_t *ctx, void *user_data) return ready; } +static size_t audio_frame_len_at(int index) +{ + return index < 160 ? g_st.audio_frame_lens[index] + : g_st.audio_frame_lens[159]; +} + static void test_receive_backpressure(void) { SECTION("receive_backpressure"); @@ -2145,20 +2174,25 @@ static void test_receive_backpressure(void) pthread_mutex_lock(&g_st.mtx); size_t paused_recv_calls = observed_recv_calls; pthread_mutex_unlock(&g_st.mtx); - uint8_t audio[800]; - memset(audio, 0x5a, sizeof(audio)); - for (int i = 0; i < 40; i++) { + + static uint8_t audio[8][800]; + /* Distinct constant per Packet: a wrong slide lands in another Packet's + * bytes and fails the packet-id check below regardless of position. */ + for (size_t packet = 0; packet < 8; packet++) { + memset(audio[packet], (int)(0x10 + packet), sizeof(audio[packet])); CHECK(server_send_audio(ctx, - i == 0 ? TAI_STREAM_START : TAI_STREAM_MIDDLE, - i == 0 ? "111 1 16 16000" : NULL, - audio, sizeof(audio), (uint16_t)(100 + i)) > 0); + packet == 0 ? TAI_STREAM_START : TAI_STREAM_MIDDLE, + packet == 0 ? "111 1 16 16000 0 16000 20 40" : NULL, + audio[packet], sizeof(audio[packet]), + (uint16_t)(100 + packet)) > 0); } sleep_ms(100); pthread_mutex_lock(&g_st.mtx); CHECK_EQ_INT(observed_recv_calls, paused_recv_calls); + CHECK_EQ_INT(g_st.audio_calls, 0); pthread_mutex_unlock(&g_st.mtx); - for (int allowed = 1; allowed <= 40; allowed++) { + for (int allowed = 1; allowed <= 8 * 20; allowed++) { pthread_mutex_lock(&g_st.mtx); flow_allowed_calls = allowed; pthread_mutex_unlock(&g_st.mtx); @@ -2172,12 +2206,136 @@ static void test_receive_backpressure(void) tai_disconnect(ctx); CHECK(observed_sleep_calls > 0); CHECK_EQ_INT(observed_empty_polls, 0); - CHECK_EQ_INT(g_st.audio_calls, 40); - CHECK_EQ_INT(g_st.audio_bytes, 40 * sizeof(audio)); + CHECK_EQ_INT(g_st.audio_calls, 160); + CHECK_EQ_INT(g_st.audio_bytes, 8 * sizeof(audio[0])); + for (int frame = 0; frame < 160; frame++) + CHECK_EQ_INT(audio_frame_len_at(frame), 40); + /* Byte-exact across ALL Packets, not just the paused first one: each + * 800-byte Packet carries its own index in every byte, so any mis-slide + * shows up as a wrong packet id. */ + for (size_t i = 0; i < g_st.audio_concat_len; i++) { + size_t packet = i / sizeof(audio[0]); + if (packet >= 8) break; + CHECK_EQ_INT(g_st.audio_concat[i], audio[packet][0]); + } CHECK_EQ_INT(g_st.disconnect_calls, 0); tai_ctx_deinit(ctx); } +static int eof_released, eof_hook_phase, eof_buffered_b; +static size_t eof_paused_queries, eof_frame_b_len; +static uint8_t eof_frame_b[256]; + +static int buffered_eof_flow_control(tai_ctx_t *ctx, void *user_data) +{ + (void)user_data; + pthread_mutex_lock(&g_st.mtx); + int ready = eof_released; + if (!ready) eof_paused_queries++; + else if (g_st.text_calls == 1 && eof_hook_phase == 0) { + /* This callback runs on the receive worker: inspecting its buffer here + * is safe, unlike sampling rx_len from the test thread. */ + eof_buffered_b = ctx->rx_len == eof_frame_b_len && + memcmp(ctx->rx_buf, eof_frame_b, eof_frame_b_len) == 0; + eof_hook_phase = 1; + tai_loopback_close_connection(); + ready = 0; + } else if (eof_hook_phase == 1) { + eof_hook_phase = 2; /* admission reopens on the very next query */ + } + pthread_mutex_unlock(&g_st.mtx); + return ready; +} + +static void test_buffered_frame_before_eof(void) +{ + SECTION("buffered_frame_before_eof"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + eof_released = eof_hook_phase = eof_buffered_b = 0; + eof_paused_queries = 0; + tai_config_t options = { .pal = observe_pal(), + .on_flow_control = buffered_eof_flow_control }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(eof_paused_queries > 0, 1000)); + + uint8_t app[128], frames[512]; + int alen = build_text_app(ctx, TAI_STREAM_ONE_SHOT, 1, "A", 1, + app, sizeof(app)); + CHECK(alen > 0); + int a = alen > 0 ? tai_frame_encode(TAI_FRAG_NONE, 10, app, (size_t)alen, + ctx->sign_key, 32, ctx->pal, frames, sizeof(frames)) : -1; + alen = build_text_app(ctx, TAI_STREAM_ONE_SHOT, 2, "B", 1, + app, sizeof(app)); + CHECK(alen > 0); + int b = alen > 0 ? tai_frame_encode(TAI_FRAG_NONE, 11, app, (size_t)alen, + ctx->sign_key, 32, ctx->pal, eof_frame_b, sizeof(eof_frame_b)) : -1; + CHECK(a > 0 && b > 0); + if (a <= 0 || b <= 0) { tai_disconnect(ctx); tai_ctx_deinit(ctx); return; } + memcpy(frames + a, eof_frame_b, (size_t)b); + pthread_mutex_lock(&g_st.mtx); + eof_frame_b_len = (size_t)b; + size_t before_bytes = observed_recv_bytes; + tai_loopback_push_recv(frames, (size_t)(a + b)); + eof_released = 1; + pthread_mutex_unlock(&g_st.mtx); + + CHECK(WAIT_FOR(g_st.disconnect_calls > 0, 2000)); + tai_disconnect(ctx); + CHECK_EQ_INT(eof_hook_phase, 2); + CHECK(eof_buffered_b); + CHECK_EQ_INT(observed_recv_bytes - before_bytes, a + b); + CHECK_EQ_INT(g_st.text_calls_at_disconnect, 2); + CHECK(strcmp(g_st.text_buf, "AB") == 0); + CHECK_EQ_INT(g_st.disconnect_calls, 1); + CHECK_EQ_INT(g_st.disconnect_reason, TAI_DISCONNECT_TRANSPORT); + CHECK_EQ_INT(g_st.disconnect_detail, TAI_TRANSPORT_EOF); + tai_ctx_deinit(ctx); +} + +static void test_partial_frame_blocking_recv(void) +{ + const size_t cuts[] = { 2, 8 }; /* incomplete 5-byte header / incomplete body */ + for (size_t i = 0; i < sizeof(cuts) / sizeof(cuts[0]); i++) { + SECTION(i == 0 ? "partial_header_blocking_recv" : "partial_body_blocking_recv"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + tai_config_t options = { .pal = observe_pal(), + .ping_interval_ms = 1000000 }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + tai_loopback_set_recv_cap_ms(AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS + 5000U); + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + uint8_t app[128], frame[256]; + int alen = build_text_app(ctx, TAI_STREAM_ONE_SHOT, 1, + "partial", 7, app, sizeof(app)); + CHECK(alen > 0); + int flen = alen > 0 ? tai_frame_encode(TAI_FRAG_NONE, 10, app, (size_t)alen, + ctx->sign_key, 32, ctx->pal, frame, sizeof(frame)) : -1; + CHECK(flen > (int)cuts[i]); + if (flen <= (int)cuts[i]) { tai_disconnect(ctx); tai_ctx_deinit(ctx); continue; } + pthread_mutex_lock(&g_st.mtx); + observed_after_bytes = observed_recv_bytes + cuts[i]; + pthread_mutex_unlock(&g_st.mtx); + tai_loopback_push_recv(frame, cuts[i]); + CHECK(WAIT_FOR(observed_recv_bytes >= observed_after_bytes, 1000)); + CHECK(WAIT_FOR(observed_blocking_after_bytes > 0, 1000)); + pthread_mutex_lock(&g_st.mtx); + CHECK(observed_nonblocking_after_bytes <= 4); + CHECK(observed_max_timeout_after_bytes <= AGENTIC_KIT_TAI_WORKER_POLL_CAP_MS); + CHECK_EQ_INT(g_st.text_calls, 0); + pthread_mutex_unlock(&g_st.mtx); + tai_loopback_push_recv(frame + cuts[i], (size_t)flen - cuts[i]); + CHECK(WAIT_FOR(g_st.text_calls == 1, 1000)); + tai_disconnect(ctx); + CHECK(strcmp(g_st.text_buf, "partial") == 0); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); + } +} + static void test_long_receive_pause_liveness(void) { SECTION("long_receive_pause_liveness"); @@ -2298,6 +2456,8 @@ int main(void) pthread_mutex_init(&g_st.mtx, NULL); test_receive_backpressure(); + test_buffered_frame_before_eof(); + test_partial_frame_blocking_recv(); test_long_receive_pause_liveness(); test_pal_sleep(); test_text_query(); From 3d9d2b6dec6b41cabafdec35cf03d528165c1b0b Mon Sep 17 00:00:00 2001 From: wangsn Date: Tue, 22 Sep 2026 17:59:56 +0800 Subject: [PATCH 09/10] fix(examples): keep interruption filtering app-owned Use only the public TAI interface in the MQTT interruption demo and drain paused audio through its mutex-protected stale Event filter, avoiding an unsafe cross-thread SDK state mutation. Add deterministic coverage for paused, racing, queued, unscoped, and fresh Event orderings.\n\nVerified: 20/20 ctest suites pass serially; mqtt_interrupt_demo builds successfully. --- CMakeLists.txt | 1 + .../ai/rtc-tcp-client/mqtt_interrupt_demo.c | 9 +- .../iot-client/test/iot_tai_control_test.c | 2 + .../test/mqtt_interrupt_demo_test.c | 157 ++++++++++++++++++ 4 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 modules/iot-client/test/mqtt_interrupt_demo_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 6eaa8aa..fc56e36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -448,6 +448,7 @@ if(AGENTIC_KIT_BUILD_TESTS AND AGENTIC_KIT_ENABLE_PROJECT_TESTS) ${AI_TCP_SOURCES} "${AI_TCP_TESTS_DIR}/tai_pal_loopback.c" "${IOT_CLIENT_TESTS_DIR}/iot_tai_control_test.c" + "${IOT_CLIENT_TESTS_DIR}/mqtt_interrupt_demo_test.c" "${IOT_CLIENT_TESTS_DIR}/test_log.c" ) target_include_directories(iot_tai_control_test PRIVATE diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c index 733ec39..7cdab01 100644 --- a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -14,7 +14,7 @@ #include "demo_json.h" #include "iot_client.h" -#include "tai_internal.h" +#include "tuya_ai.h" extern const pal_t *tai_pal_posix(void); @@ -84,16 +84,13 @@ static void set_stale_event(demo_state_t *state, const char *event_id) state->queued_audio = 0; if (event_id && event_id[0]) { mark_event_stale(state, event_id); - /* If pressure paused this Event inside a Packet, flush only its retained - * remainder; the worker then drains later wire Frames without letting - * already accepted bytes replay or stale callbacks refill playback. */ - if (state->tai) tai_discard_pending_audio(state->tai); + /* Clearing playback reopens admission. The worker drains retained audio + * through on_audio(), which rejects this stale Event under the mutex. */ } else { /* Missing/ambiguous correlation is fail-closed: discard audio until * the TCP side identifies the next Event. A repeated wire START inside * one multi-frame Packet must not reopen playback. */ state->drop_unscoped_audio = 1; - if (state->tai) tai_discard_pending_audio(state->tai); } pthread_mutex_unlock(&state->mutex); } diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c index 20ae1c9..d95b1ea 100644 --- a/modules/iot-client/test/iot_tai_control_test.c +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -29,6 +29,7 @@ #define WRONG_KEY_PORT 11887 extern const pal_t *tai_pal_posix(void); +int mqtt_interrupt_demo_tests(void); static int failures; static const pal_t *posix_pal; @@ -377,6 +378,7 @@ int main(void) const pal_t *pal = make_routing_pal(); pthread_mutex_init(&state.mutex, NULL); CHECK(iot_init(pal) == OPRT_OK); + failures += mqtt_interrupt_demo_tests(); CHECK(start_mqtt_mock() == 0); char *cacert = load_file(pal, TEST_CONFIG_DIR "/root_cert.pem"); diff --git a/modules/iot-client/test/mqtt_interrupt_demo_test.c b/modules/iot-client/test/mqtt_interrupt_demo_test.c new file mode 100644 index 0000000..e02b4a8 --- /dev/null +++ b/modules/iot-client/test/mqtt_interrupt_demo_test.c @@ -0,0 +1,157 @@ +/* Include the demo so regressions exercise its actual static callbacks rather + * than a test-only copy of the interruption/filtering logic. Never run its main. */ +#define main mqtt_interrupt_demo_main +#include "../../../examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c" +#undef main + +#include "tai_internal.h" + +#define DEMO_CHECK(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #expr); \ + failures++; \ + } \ + } while (0) + +static int demo_test_start(tai_ctx_t *ctx, const char *event_id) +{ + uint8_t payload[4]; + tai_attr_t attr = tai_attr_strv(TAI_ATTR_EVENT_ID, event_id); + int len = tai_pack_event(TAI_VER_21, TAI_EVT_START, NULL, 0, + payload, sizeof(payload)); + if (len <= 0) return TAI_ERR_PROTO; + return tai_proto_dispatch(ctx, TAI_PKT_EVENT, &attr, 1, + payload, (size_t)len); +} + +static int demo_test_audio(tai_ctx_t *ctx, size_t frames) +{ + /* Keep the paused body's storage pinned in rx_buf, just like the worker. */ + int len = tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, 123, + ctx->rx_buf, sizeof(ctx->rx_buf)); + if (len <= 0) return TAI_ERR_PROTO; + size_t body_len = frames * 40; + if (body_len > sizeof(ctx->rx_buf) - (size_t)len) return TAI_ERR_MEM; + memset(ctx->rx_buf + len, 0x5a, body_len); + tai_attr_t attr = tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, + "111 1 16 16000 0 16000 20 40"); + return tai_proto_dispatch(ctx, TAI_PKT_AUDIO, &attr, 1, + ctx->rx_buf, (size_t)len + body_len); +} + +int mqtt_interrupt_demo_tests(void) +{ + int failures = 0; + const pal_t *pal = tai_pal_posix(); + demo_state_t state = {0}; + if (pthread_mutex_init(&state.mutex, NULL) != 0) { + fprintf(stderr, "FAIL mqtt_interrupt_demo_tests: mutex init\n"); + return 1; + } + tai_config_t config = { + .pal = pal, + .protocol_version = TAI_VER_21, + .on_audio = on_audio, + .on_event = on_event, + .on_flow_control = on_flow_control, + .user_data = &state, + }; + void *memory = pal->malloc(tai_ctx_size()); + state.tai = memory ? tai_ctx_init(memory, &config) : NULL; + DEMO_CHECK(state.tai != NULL); + if (!state.tai) goto cleanup; + tai_ctx_t *ctx = state.tai; + + /* Mid-Packet pause: accepted frames fill playback, two remain retained. + * Interruption must not mutate the worker-owned pending state. */ + DEMO_CHECK(demo_test_start(ctx, "paused-event") == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, PLAYBACK_CAPACITY + 2) == TAI_OK); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + DEMO_CHECK(ctx->rx_pending_valid == 1); + DEMO_CHECK(ctx->rx_pending_offset == PLAYBACK_CAPACITY * 40); + DEMO_CHECK(ctx->rx_pending_len == (PLAYBACK_CAPACITY + 2) * 40); + DEMO_CHECK(strcmp(ctx->rx_pending_event_id, "paused-event") == 0); + DEMO_CHECK(on_flow_control(ctx, &state) == 0); + DEMO_CHECK(tai_proto_drain_pending_audio(ctx, 0) == 1); + DEMO_CHECK(ctx->rx_pending_offset == PLAYBACK_CAPACITY * 40); + static const char paused_interrupt[] = "{\"eventId\":\"paused-event\"}"; + on_ai_control("asrInterrupt", paused_interrupt, + sizeof(paused_interrupt) - 1, &state); + DEMO_CHECK(state.queued_audio == 0); + DEMO_CHECK(ctx->rx_pending_valid == 1); + DEMO_CHECK(ctx->rx_pending_discard == 0); + DEMO_CHECK(ctx->rx_pending_offset == PLAYBACK_CAPACITY * 40); + DEMO_CHECK(on_flow_control(ctx, &state) == 1); + DEMO_CHECK(tai_proto_drain_pending_audio(ctx, 0) == 0); + DEMO_CHECK(ctx->rx_pending_valid == 0); + DEMO_CHECK(ctx->rx_pending_offset == ctx->rx_pending_len); + DEMO_CHECK(ctx->rx_pending_body == NULL); + DEMO_CHECK(state.queued_audio == 0); + + /* Deterministically order the race: admission, interrupt, audio callback. */ + DEMO_CHECK(demo_test_start(ctx, "racing-event") == TAI_OK); + tai_audio_msg_t audio = { + .event_id = "racing-event", + .stream_flag = TAI_STREAM_START, + .data = (const uint8_t *)"opus", + .len = 4, + }; + DEMO_CHECK(on_flow_control(ctx, &state) == 1); + static const char racing_interrupt[] = "{\"eventId\":\"racing-event\"}"; + on_ai_control("asrInterrupt", racing_interrupt, + sizeof(racing_interrupt) - 1, &state); + on_audio(ctx, &audio, &state); + DEMO_CHECK(state.queued_audio == 0); + + /* The opposite ordering accepts audio first, then flushes that queue. */ + DEMO_CHECK(demo_test_start(ctx, "queued-event") == TAI_OK); + audio.event_id = "queued-event"; + DEMO_CHECK(on_flow_control(ctx, &state) == 1); + on_audio(ctx, &audio, &state); + DEMO_CHECK(state.queued_audio == 1); + static const char queued_interrupt[] = "{\"eventId\":\"queued-event\"}"; + on_ai_control("asrInterrupt", queued_interrupt, + sizeof(queued_interrupt) - 1, &state); + DEMO_CHECK(state.queued_audio == 0); + on_audio(ctx, &audio, &state); + DEMO_CHECK(state.queued_audio == 0); + + /* A distinct non-stale Event remains playable after scoped interruptions. */ + DEMO_CHECK(demo_test_start(ctx, "fresh-event") == TAI_OK); + DEMO_CHECK(strcmp(state.current_event_id, "fresh-event") == 0); + DEMO_CHECK(demo_test_audio(ctx, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 1); + DEMO_CHECK(ctx->rx_pending_valid == 0); + + /* Missing correlation also drains retained START frames without reopening + * playback; only the next identified Event restores admission to playback. */ + DEMO_CHECK(demo_test_audio(ctx, PLAYBACK_CAPACITY + 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + DEMO_CHECK(ctx->rx_pending_valid == 1); + DEMO_CHECK(ctx->rx_pending_offset == (PLAYBACK_CAPACITY - 1) * 40); + on_ai_control("asrInterrupt", "{}", 2, &state); + DEMO_CHECK(state.queued_audio == 0); + DEMO_CHECK(state.drop_unscoped_audio == 1); + DEMO_CHECK(ctx->rx_pending_valid == 1); + DEMO_CHECK(ctx->rx_pending_discard == 0); + DEMO_CHECK(tai_proto_drain_pending_audio(ctx, 0) == 0); + DEMO_CHECK(ctx->rx_pending_valid == 0); + DEMO_CHECK(ctx->rx_pending_offset == ctx->rx_pending_len); + DEMO_CHECK(state.queued_audio == 0); + DEMO_CHECK(state.drop_unscoped_audio == 1); + DEMO_CHECK(demo_test_start(ctx, "after-unscoped-event") == TAI_OK); + DEMO_CHECK(state.drop_unscoped_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 1); + +cleanup: + if (state.tai) tai_ctx_deinit(state.tai); + pal->free(memory); + DEMO_CHECK(pthread_mutex_destroy(&state.mutex) == 0); + printf("mqtt_interrupt_demo_tests: %d failures\n", failures); + return failures; +} + +#undef DEMO_CHECK From 65ce503942015f52f9b48db7864eabc3052611aa Mon Sep 17 00:00:00 2001 From: wangsn Date: Tue, 22 Sep 2026 23:05:44 +0800 Subject: [PATCH 10/10] feat(rtc-tcp-client,iot-client): filter interruptions by server time Interruption filtering compared event IDs, which cannot order a stream against an interruption that arrives out of band. Both notice paths already carry a server time -- the MQTT asrInterrupt in its own payload, the TCP ChatBreak in the UserData attribute (attr 111) -- and the audio media header already carries the same clock, so the two can be compared directly. tai_event_msg_t now exposes the borrowed user_data attribute so an application can read it without the SDK parsing JSON. Measured on real traffic (audio_chat_demo, real speech barge-in) the ChatBreak cut-off lands strictly between the interrupted stream's START and the next stream's START: 1790088542969 <= 1790088545108 < 1790088557463, all one epoch-millisecond timeline that matched the device wall clock to the second. The demo now discards on that comparison and fails closed onto the in-flight stream when a notice carries no usable time, instead of ignoring the interruption. The paused-Packet cursor drops the fields that only duplicated state the pause cannot change: no metadata copies, no application-side discard call, and the storage-origin bookkeeping collapses to the pinned Frame's wire length. A wire length that cannot be consumed exactly once fails fast with TAI_PROTO_ERR_FRAME_DECODE rather than sliding the receive buffer out of range (Bus error without the guard, pinned by pending_audio_bad_wire_len). Share one admission query across all receive checkpoints and document when handshake media can invoke the hook. Clarify AI-control payload re-serialization and raw-callback fallback, export its setter with IOT_API, and correct the development SDK version. Document pacing knobs, post-ack media delivery, and the demo's shared-test fixture contract; remove unused loopback extern declarations and complete the Unreleased entries for #42. Verified: full serial ctest 20/20; ASan/UBSan tai_integration_tests, iot_tai_control_test, iot_ai_ctrl_test 3/3; docs-site i18n parity 30/30. Co-Authored-By: MightyCode Co-Authored-By: Claude Code Co-Authored-By: OpenAI Codex --- CHANGELOG.md | 15 +- docs-site/docs/guides/vad-and-interrupt.md | 26 +- docs-site/docs/reference/iot-client.md | 2 +- docs-site/docs/reference/rtc-tcp-client.md | 12 +- .../current/guides/vad-and-interrupt.md | 28 +- .../current/reference/iot-client.md | 2 +- .../current/reference/rtc-tcp-client.md | 12 +- .../ai/rtc-tcp-client/mqtt_interrupt_demo.c | 179 +++++----- modules/iot-client/include/iot_client.h | 6 + modules/iot-client/src/iot_ai_ctrl.c | 4 +- modules/iot-client/src/iot_internal.h | 3 +- .../iot-client/test/iot_tai_control_test.c | 161 ++++----- .../test/mqtt_interrupt_demo_test.c | 317 +++++++++++++----- modules/rtc-tcp-client/CONTEXT.md | 21 +- ...eive-backpressure-pauses-the-connection.md | 33 +- modules/rtc-tcp-client/include/tuya_ai.h | 21 +- modules/rtc-tcp-client/src/tai_client.c | 77 ++--- modules/rtc-tcp-client/src/tai_internal.h | 51 ++- modules/rtc-tcp-client/src/tai_protocol.c | 135 ++------ .../rtc-tcp-client/test/tai_pal_loopback.c | 4 - .../rtc-tcp-client/test/test_integration.c | 248 +++++++++++++- 21 files changed, 870 insertions(+), 487 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6d2a8a..43924bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- iot-client — MQTT protocol-9000 AI control channel for server-initiated `asrInterrupt` notices, delivered via `ai_ctrl_callback_t` independently of the RTC TCP Connection. -- rtc-tcp-client — `on_flow_control` admission hook for TCP receive backpressure; pauses all inbound Frames when the application's audio queue is full, including codec-frame callbacks mid-Packet. +- iot-client — MQTT protocol-9000 AI control channel for server-initiated `asrInterrupt` notices, delivered via `ai_ctrl_callback_t` independently of the RTC TCP Connection (#42). + - The notice payload carries `eventId` and the server time `time`; the time is the interruption cutoff. +- rtc-tcp-client — `on_flow_control` admission hook for TCP receive backpressure; pauses all inbound Frames when the application's audio queue is full, including codec-frame callbacks mid-Packet (#42). + - New knobs: `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS` (admission-check period while paused, default 50 ms) and `AGENTIC_KIT_TAI_WORKER_YIELD_MS` (worker drain-pass yield under sustained traffic, default 10 ms). +- rtc-tcp-client — `tai_event_msg_t` gains borrowed `user_data` / `user_data_len` (attr 111), so an interruption notice can be read without the SDK parsing JSON (#42). + - `TAI_EVT_CHAT_BREAK` carries `{"breakAttributes":{"time":""}}`; compare that time with the `timestamp_ms` latched from audio START to discard an interrupted stream. A notice whose time is missing or unusable fails closed onto the in-flight stream. ### Changed -- **BREAKING** pal — `pal_t` gains a mandatory `sleep_ms` member; custom PALs must supply it and all consumers must rebuild. -- rtc-tcp-client — audio Packets paused mid-body now retain a pending-delivery cursor instead of silently dropping codec frames; reopening admission resumes from the first unadmitted byte. +- **BREAKING** pal — `pal_t` gains a mandatory `sleep_ms` member; custom PALs must supply it and all consumers must rebuild (#42). +- rtc-tcp-client — audio Packets paused mid-body now retain a pending-delivery cursor instead of silently dropping codec frames; reopening admission resumes from the first unadmitted byte (#42). + - A pinned Packet whose recorded wire length cannot be consumed exactly once fails fast with `TAI_PROTO_ERR_FRAME_DECODE` instead of sliding the receive buffer out of range. + - A header-only START/ONE_SHOT now reaches `on_audio` with `len == 0`, so its server-side `timestamp_ms` can be latched for interruption filtering. +- rtc-tcp-client — `tai_connect()` returns once the session acknowledgement is handled; media the server coalesced into the handshake stays buffered and is first delivered by the receive worker after connect returns (#42). - docs-site — English edition of the full documentation site, published at `/en/` with Simplified Chinese retained at `/`. - All 29 docs are mirrored under `docs-site/i18n/en/`, with a locale selector in both the landing-page navbar and the custom docs topbar, localized navbar/footer/sidebar catalogs, and English SVG schematics under `current/images/`. diff --git a/docs-site/docs/guides/vad-and-interrupt.md b/docs-site/docs/guides/vad-and-interrupt.md index d7c785b..517fcbb 100644 --- a/docs-site/docs/guides/vad-and-interrupt.md +++ b/docs-site/docs/guides/vad-and-interrupt.md @@ -145,7 +145,7 @@ tai_config_t cfg = { :::note MQTT 独立控制路径 设备可同时保持 IoT MQTT 与 RTC TCP Connection。注册 `iot_ai_ctrl_set_callback()` 后，protocol-9000 `asrInterrupt` 不受 RTC TCP 接收背压影响。应用应让一个线程独占 `iot_client_process()` / publish / reconnect，并把 MQTT callback 与 `TAI_EVT_CHAT_BREAK` 汇入同一线程安全的播放策略。 -收到 MQTT 打断时，先把对应 Event 标记为过期并清空其播放队列，再恢复 RTC 接收，让 worker 继续校验和排空旧媒体；不要直接丢弃 TCP 字节，否则会破坏 Frame 边界。旧 Event 的后续 `on_audio` 必须被应用丢弃，直到新 Event 开始。服务端打断不要求调用 `tai_chat_break()`，也不要结束或重开云端 VAD 上行流。 +打断的判定依据是**服务端时间**，不是 eventId：MQTT `asrInterrupt` 读 `data.time`，TCP `TAI_EVT_CHAT_BREAK` 读 attr 111 中的 `breakAttributes.time`（不是事件负载）。把最大的时间记为截止时间，清空播放队列，再恢复 RTC 接收，让 worker 继续校验和排空旧媒体；不要直接丢弃 TCP 字节，否则会破坏 Frame 边界。`on_audio` 中锁定 START 的 `msg->timestamp_ms`，凡不晚于截止时间的流一律丢弃。服务端打断不要求调用 `tai_chat_break()`，也不要结束或重开云端 VAD 上行流。 ::: **接收服务端打断（`TAI_EVT_CHAT_BREAK`，type=4）：** @@ -158,18 +158,34 @@ void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *ud) if (msg->event_type == TAI_EVT_CHAT_BREAK) { // 1. 停止 TTS 播放 audio_player_stop(); - // 2. 清空播放缓冲区（丢弃本轮在途 TTS，直到下一个 TAI_STREAM_START） + // 2. 从 attr 111 取服务端打断时间，推进截止时间（取最大值） + uint64_t cutoff = parse_break_time(msg->user_data, msg->user_data_len); + if (cutoff) audio_cutoff_ms = cutoff > audio_cutoff_ms ? cutoff : audio_cutoff_ms; + // 3. 清空播放缓冲区；只有晚于截止时间的流才会重新入队 audio_buffer_flush(); - // 3. 忽略本轮后续回调 - set_ignore_current_response(true); // 不要停止麦克风采音，不要调用 tai_send_audio_end()， // 也不要调用 tai_send_audio_start() 重开上行流—— // 云端 VAD 模式下上行 Event 一直保持打开。 - // 若该打断的 eventId 没有对应的本地下行缓存，记录日志并忽略即可。 + // 若 attr 111 缺失或时间无效，按失败关闭处理：把当前流的 START + // 当作截止时间，丢弃在途流，而不是当作无事发生继续播放。 } } ``` +`on_audio` 侧锁定 START 的服务端时间，并据此过滤： + +```c +void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *ud) +{ + if (msg->stream_flag == TAI_STREAM_START || + msg->stream_flag == TAI_STREAM_ONE_SHOT) + stream_start_ms = msg->timestamp_ms; // 服务端时间，非本地时间 + // MIDDLE/END 不带新时间，沿用本轮流锁定的起始时间 + if (msg->len && stream_start_ms > audio_cutoff_ms) + audio_buffer_push(msg->data, msg->len); +} +``` + **发送客户端打断：** ```c diff --git a/docs-site/docs/reference/iot-client.md b/docs-site/docs/reference/iot-client.md index 3d8bead..285e1ea 100644 --- a/docs-site/docs/reference/iot-client.md +++ b/docs-site/docs/reference/iot-client.md @@ -451,7 +451,7 @@ int iot_ai_ctrl_set_callback(iot_client_t *client, 注册经过 P2.3 解密和认证的 MQTT protocol-9000 AI 控制通知。回调在调用 `iot_client_process()` 的线程上触发，`type` 和 `json_data` 只在回调期间有效。回调应仅更新有界状态或通知应用线程，不应在其中断开/销毁 IoT client。 -若必须接收订阅建立期间立即到达的通知，初始化时设置 `mqtt_disable_auto_connect=true`，先注册回调，再调用 `iot_client_connect()`。传 NULL callback 可注销。RTC TCP 接收背压不会阻塞该 MQTT 路径；应用应把 MQTT 通知和 TAI ChatBreak 汇入同一播放策略，并按 Event 标识丢弃被中断 Event 的后续媒体。服务端通知不是 `tai_chat_break()` 的应答，也不应自动结束云端 VAD 上行流。 +若必须接收订阅建立期间立即到达的通知，初始化时设置 `mqtt_disable_auto_connect=true`，先注册回调，再调用 `iot_client_connect()`。传 NULL callback 可注销。RTC TCP 接收背压不会阻塞该 MQTT 路径；应用应把 MQTT 通知和 TAI ChatBreak 汇入同一播放策略。`asrInterrupt` 的负载携带 `eventId` 与服务端时间 `time`（字符串毫秒），后者是打断判定的截止时间；把它与 `on_audio` 中锁定的 `msg->timestamp_ms` 比较来丢弃过期媒体，而不是按 eventId 判断。服务端通知不是 `tai_chat_break()` 的应答，也不应自动结束云端 VAD 上行流。 --- diff --git a/docs-site/docs/reference/rtc-tcp-client.md b/docs-site/docs/reference/rtc-tcp-client.md index cbc97bc..737666b 100644 --- a/docs-site/docs/reference/rtc-tcp-client.md +++ b/docs-site/docs/reference/rtc-tcp-client.md @@ -223,10 +223,10 @@ int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); #### 接收背压（`on_flow_control`） -- worker 在每次读取前及完整 Frame 之间调用钩子，握手阶段不调用。钩子必须非阻塞；应用负责同步共享的队列状态。 +- worker 在每次读取前及完整 Frame 之间调用钩子。符合规范的服务端不会在握手阶段触发它（应答先于媒体下发）；仅当服务端把媒体排在应答之前才可能在握手期被调用。钩子必须非阻塞；应用负责同步共享的队列状态。 - 返回 0 会暂停读取和解析，所有入站流量都会停滞，包括 ChatBreak、ASR 文本、Pong 和 EOF 检测。恢复后先处理已缓冲的完整 Frame，再读取；不完整输入回到有上限的阻塞接收。 - 主动暂停挂起接收存活超时；恢复时获得新的完整 `ping_timeout_ms` 预算。Ping 和停止请求仍执行，Ping 发送失败仍断连。 -- 背压以线协议 Frame 为边界。一个 Audio Packet 可能同步触发多个编解码帧回调，`on_audio` 仍须限制自身入队量。 +- 背压也在 Audio Packet 内的每个编解码帧回调前检查。中途暂停会以零拷贝方式保留该 Packet 的剩余字节，恢复时先交付剩余帧、再处理后续 Packet。应用应在 `on_audio` 中按服务端时间（`msg->timestamp_ms`）丢弃过期音频，而不是改写 SDK 接收状态。 默认每 `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS`（50 ms）重查背压，并以 `AGENTIC_KIT_TAI_WORKER_YIELD_MS`（10 ms）在持续流量下让出 CPU。 @@ -246,7 +246,7 @@ int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); | `stream_flag` | `uint8_t` | `TAI_STREAM_*`（取自媒体头） | | `data_id` | `uint16_t` | 数据 ID：`AUDIO_DOWN`(2) / `AUDIO_AUX`(7) | | `event_id` | `const char *` | turn id（借用）；无则为 `""` | -| `timestamp_ms` | `uint64_t` | 流起始时间戳（媒体头） | +| `timestamp_ms` | `uint64_t` | 服务端媒体头时间戳，**非**本地时间；过滤流时以 START 的值锁定，与打断时间比较 | `tai_text_msg_t`（文本回调）： @@ -285,6 +285,12 @@ int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); | `data` | `const uint8_t *` | 事件负载（通常为 JSON） | | `len` | `size_t` | 负载字节数 | | `event_id` | `const char *` | attr 61（借用）；无则为 `""` | +| `user_data` | `const uint8_t *` | attr 111（借用），**非** NUL 结尾；无则为 NULL | +| `user_data_len` | `size_t` | `user_data` 字节数；与事件负载分离，SDK 不解析其 JSON | + +:::note ChatBreak 打断时间 +`TAI_EVT_CHAT_BREAK` 的服务端时间不在事件负载里，而在 attr 111 中：`{"breakAttributes":{"time":""}}`，值为服务端 epoch 毫秒，与 `on_audio` 的 `timestamp_ms` 同钟同单位。读取 `msg->user_data`、解析 `breakAttributes.time`，再与 `on_audio` 中锁定的 `timestamp_ms` 比较，即可判定该流是否已过期。MQTT 侧的 `asrInterrupt` 走另一条路径：时间在其自身负载的 `time` 字段，且是同一个服务端时间值。 +::: `tai_disconnect_msg_t`（断连回调）： diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md index 2379f0d..f303707 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/guides/vad-and-interrupt.md @@ -147,7 +147,7 @@ When the user speaks again while the AI is responding, the current response need :::note Independent MQTT control path A device can keep the IoT MQTT and RTC TCP Connection active together. After `iot_ai_ctrl_set_callback()` is registered, a protocol-9000 `asrInterrupt` is independent of RTC TCP receive backpressure. One application thread should own `iot_client_process()`, publish, and reconnect, while the MQTT callback and `TAI_EVT_CHAT_BREAK` feed the same thread-safe playback policy. -On an MQTT interrupt, first mark the correlated Event stale and flush its playback queue, then release RTC receive pressure so the worker can continue authenticating and draining old media. Never discard raw TCP bytes, which would corrupt Frame boundaries. The application must reject subsequent `on_audio` callbacks for the stale Event until a new Event starts. A server notice does not require `tai_chat_break()`, and it must not end or reopen a Server-VAD uplink. +An interruption is decided by **server time**, not by event ID: MQTT `asrInterrupt` carries `data.time`, and TCP `TAI_EVT_CHAT_BREAK` carries `breakAttributes.time` inside attr 111 (not in the event payload). Keep the greatest value as the cutoff, flush the playback queue, then release RTC receive pressure so the worker can keep authenticating and draining old media -- never discard raw TCP bytes, which would corrupt Frame boundaries. In `on_audio`, latch START's `msg->timestamp_ms` and drop every stream at or before the cutoff. A server notice does not require `tai_chat_break()`, and it must not end or reopen a Server-VAD uplink. ::: **Receive a server-initiated chat break (`TAI_EVT_CHAT_BREAK`, type=4):** @@ -160,20 +160,34 @@ void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *ud) if (msg->event_type == TAI_EVT_CHAT_BREAK) { // 1. Stop TTS playback audio_player_stop(); - // 2. Clear the playback buffer (discard this turn's in-flight TTS - // until the next TAI_STREAM_START) + // 2. Read the server interruption time from attr 111 and advance the cutoff + uint64_t cutoff = parse_break_time(msg->user_data, msg->user_data_len); + if (cutoff) audio_cutoff_ms = cutoff > audio_cutoff_ms ? cutoff : audio_cutoff_ms; + // 3. Clear the playback buffer; only streams newer than the cutoff requeue audio_buffer_flush(); - // 3. Ignore subsequent callbacks for this turn - set_ignore_current_response(true); // Do not stop microphone capture or call tai_send_audio_end(). // Do not call tai_send_audio_start() to reopen the uplink stream either: // in server-VAD mode, the uplink Event remains open throughout. - // If this chat break's eventId has no corresponding local downlink buffer, - // just log and ignore it. + // If attr 111 is missing or its time is invalid, fail closed: treat the + // in-flight stream's START as the cutoff instead of letting it play on. } } ``` +`on_audio` latches START's server time and filters on it: + +```c +void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *ud) +{ + if (msg->stream_flag == TAI_STREAM_START || + msg->stream_flag == TAI_STREAM_ONE_SHOT) + stream_start_ms = msg->timestamp_ms; // server time, not local time + // MIDDLE/END carry no new time; they reuse this turn's latched start + if (msg->len && stream_start_ms > audio_cutoff_ms) + audio_buffer_push(msg->data, msg->len); +} +``` + **Send a client-initiated chat break:** ```c diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md index f7f909f..5345bf4 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/iot-client.md @@ -450,7 +450,7 @@ int iot_ai_ctrl_set_callback(iot_client_t *client, Registers for P2.3-decrypted and authenticated MQTT protocol-9000 AI control notices. The callback runs on the thread calling `iot_client_process()`, and `type` plus `json_data` remain valid only during the callback. Keep it bounded to updating state or notifying an application thread; do not disconnect or destroy the IoT client there. -If notices sent during subscription setup must be observable, initialize with `mqtt_disable_auto_connect=true`, register the callback, and then call `iot_client_connect()`. Pass a NULL callback to deregister. RTC TCP receive backpressure does not stall this MQTT path. The application should merge MQTT notices and TAI ChatBreak into one playback policy and discard subsequent media belonging to the interrupted Event. A server notice is not an acknowledgment request for `tai_chat_break()` and must not automatically end a Server-VAD uplink. +If notices sent during subscription setup must be observable, initialize with `mqtt_disable_auto_connect=true`, register the callback, and then call `iot_client_connect()`. Pass a NULL callback to deregister. RTC TCP receive backpressure does not stall this MQTT path. The application should merge MQTT notices and TAI ChatBreak into one playback policy. An `asrInterrupt` payload carries an `eventId` and a server time `time` (a string of milliseconds); that time is the cutoff for interruption filtering -- compare it with the `msg->timestamp_ms` latched in `on_audio` to discard obsolete media, rather than deciding by event ID. A server notice is not an acknowledgment request for `tai_chat_break()` and must not automatically end a Server-VAD uplink. --- diff --git a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md index edd3f2e..6989467 100644 --- a/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md +++ b/docs-site/i18n/en/docusaurus-plugin-content-docs/current/reference/rtc-tcp-client.md @@ -223,10 +223,10 @@ int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); #### Receive Backpressure (`on_flow_control`) -- The worker calls the hook before each read and between complete Frames, but not during the connect handshake. The hook must not block; the application synchronizes shared queue state. +- The worker calls the hook before each read and between complete Frames. A conforming server never consults it during the connect handshake (the ack precedes media); only a server coalescing media ahead of the ack could. The hook must not block; the application synchronizes shared queue state. - Returning 0 pauses both reads and parsing, stalling all inbound traffic including ChatBreak, ASR text, Pong, and EOF detection. Resume processes complete buffered Frames before reading again; partial input returns to bounded blocking receive. - An intentional pause suspends receive-liveness timeout accounting, and resume grants a fresh `ping_timeout_ms` budget. Ping and stop requests remain active; Ping send failure still disconnects. -- Admission is at wire-Frame boundaries. One Audio Packet can synchronously emit several codec-frame callbacks, so `on_audio` must still bound its own queue. +- Admission is also checked before each codec-frame callback within an Audio Packet. A mid-Packet pause retains that Packet's remaining bytes without copying; resume delivers the remaining frames before any later Packet. Applications discard obsolete audio inside `on_audio` by comparing the server timestamp (`msg->timestamp_ms`), not by mutating SDK receive state. Backpressure is rechecked every `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS` (50 ms by default), and sustained receive traffic yields CPU for `AGENTIC_KIT_TAI_WORKER_YIELD_MS` (10 ms by default). @@ -246,7 +246,7 @@ Backpressure is rechecked every `AGENTIC_KIT_TAI_FLOW_CONTROL_POLL_MS` (50 ms by | `stream_flag` | `uint8_t` | `TAI_STREAM_*` (from the media header) | | `data_id` | `uint16_t` | Data ID: `AUDIO_DOWN`(2) / `AUDIO_AUX`(7) | | `event_id` | `const char *` | Turn ID (borrowed); `""` if absent | -| `timestamp_ms` | `uint64_t` | Stream start timestamp (media header) | +| `timestamp_ms` | `uint64_t` | Server media-header timestamp, **not** local time; latch START's value to filter a stream against an interruption time | `tai_text_msg_t` (text callback): @@ -285,6 +285,12 @@ Received images arrive as a stream of chunks: START (or ONE_SHOT) carries the fi | `data` | `const uint8_t *` | Event payload (usually JSON) | | `len` | `size_t` | Payload length in bytes | | `event_id` | `const char *` | attr 61 (borrowed); `""` if absent | +| `user_data` | `const uint8_t *` | attr 111 (borrowed), **not** NUL-terminated; NULL if absent | +| `user_data_len` | `size_t` | `user_data` length in bytes; separate from the event payload, and the SDK does not parse its JSON | + +:::note ChatBreak interruption time +The server time for `TAI_EVT_CHAT_BREAK` is not in the event payload but in attr 111: `{"breakAttributes":{"time":""}}`, a server epoch in milliseconds — the same clock and unit as `on_audio`'s `timestamp_ms`. Read `msg->user_data`, parse `breakAttributes.time`, and compare it with the `timestamp_ms` latched in `on_audio` to decide whether a stream is obsolete. The MQTT `asrInterrupt` arrives by a different path: its time is the `time` field of its own payload, and it is the same server time value. +::: `tai_disconnect_msg_t` (disconnect callback): diff --git a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c index 7cdab01..81e507e 100644 --- a/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c +++ b/examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c @@ -3,15 +3,23 @@ * * This demo uses a bounded synthetic playback queue. MQTT is pumped by one * application-owned thread while TAI's worker independently receives media. - * An asrInterrupt flushes queued playback, marks that Event stale, and reopens - * TAI receive admission so remaining stale media can be drained and discarded. + * Interruptions discard audio whose stream started at or before the server's + * cutoff time, reopening admission so retained stale media can be drained. + * + * Test fixture: iot_tai_control_test.c and mqtt_interrupt_demo_test.c include + * this file whole with `main` #defined away, so regressions exercise these + * actual callbacks. Every file-scope symbol must therefore stay static -- the + * same test binary links two copies of this file, and a non-static symbol + * collides at link time. */ +#include #include #include #include #include +#include "cJSON.h" #include "demo_json.h" #include "iot_client.h" #include "tuya_ai.h" @@ -36,93 +44,91 @@ typedef struct { int done; int playback_running; size_t queued_audio; - int drop_unscoped_audio; + uint64_t audio_cutoff_ms; + uint64_t stream_start_ms; + uint64_t queued_start_ms[PLAYBACK_CAPACITY]; char current_event_id[MAX_EVENT_ID]; - char recent_event_ids[4][MAX_EVENT_ID]; - size_t recent_count; + int current_event_has_audio; } demo_state_t; -static int event_id_seen(const demo_state_t *state, const char *event_id) -{ - for (size_t i = 0; i < state->recent_count; i++) { - if (strcmp(state->recent_event_ids[i], event_id) == 0) return 1; - } - return 0; -} +#define MAX_AUDIO_TIME_MS ((UINT64_C(1) << 42) - 1) -/* An event id remains unusable after playback for it has been interrupted. - * Once it disappears from the bounded current/recent record, a new TCP Event - * with that same id is ambiguous and must not silently clear newer playback. */ -static int mark_event_stale(demo_state_t *state, const char *event_id) +static uint64_t interrupt_time(const char *data, size_t len, int chat_break) { - if (!event_id_seen(state, event_id)) { - if (state->recent_count == 4) { - memmove(state->recent_event_ids, state->recent_event_ids + 1, - 3 * sizeof(state->recent_event_ids[0])); - state->recent_count = 3; + if (!data || !len) return 0; + /* cJSON strings have no length: reject encoded NUL rather than silently + * treating a timestamp with a NUL suffix as a shorter decimal string. */ + for (size_t i = 0; i < len; i++) { + if (data[i] == '\0') return 0; + if (data[i] == '\\' && i + 1 < len) { + if (len - i >= 6 && memcmp(data + i, "\\u0000", 6) == 0) return 0; + i++; } - snprintf(state->recent_event_ids[state->recent_count++], - sizeof(state->recent_event_ids[0]), "%s", event_id); } - return 1; -} - -static void forget_uninterrupted_event(demo_state_t *state, const char *event_id) -{ - for (size_t i = 0; i < state->recent_count; i++) { - if (strcmp(state->recent_event_ids[i], event_id) != 0) continue; - memmove(state->recent_event_ids + i, state->recent_event_ids + i + 1, - (state->recent_count - i - 1) * sizeof(state->recent_event_ids[0])); - state->recent_count--; - return; + const char *end = NULL; + cJSON *root = cJSON_ParseWithLengthOpts(data, len, &end, 0); + if (!root) return 0; + while (end < data + len && + (*end == ' ' || *end == '\t' || *end == '\r' || *end == '\n')) end++; + cJSON *object = chat_break + ? cJSON_GetObjectItemCaseSensitive(root, "breakAttributes") : root; + cJSON *time = cJSON_GetObjectItemCaseSensitive(object, "time"); + uint64_t result = 0; + if (end == data + len && cJSON_IsObject(object)) { + if (cJSON_IsString(time)) { + const char *p = time->valuestring; + for (; *p; p++) { + if (*p < '0' || *p > '9' || + result > (MAX_AUDIO_TIME_MS - (unsigned)(*p - '0')) / 10) { + result = 0; + break; + } + result = result * 10 + (unsigned)(*p - '0'); + } + } else if (cJSON_IsNumber(time) && time->valuedouble > 0 && + time->valuedouble <= (double)MAX_AUDIO_TIME_MS) { + uint64_t value = (uint64_t)time->valuedouble; + if ((double)value == time->valuedouble) result = value; + } } + cJSON_Delete(root); + return result; } -static void set_stale_event(demo_state_t *state, const char *event_id) +static void interrupt_playback(demo_state_t *state, const char *data, + size_t len, int chat_break) { + uint64_t cutoff = interrupt_time(data, len, chat_break); + const char *source = chat_break ? "TAI ChatBreak" : "MQTT asrInterrupt"; pthread_mutex_lock(&state->mutex); - state->queued_audio = 0; - if (event_id && event_id[0]) { - mark_event_stale(state, event_id); - /* Clearing playback reopens admission. The worker drains retained audio - * through on_audio(), which rejects this stale Event under the mutex. */ - } else { - /* Missing/ambiguous correlation is fail-closed: discard audio until - * the TCP side identifies the next Event. A repeated wire START inside - * one multi-frame Packet must not reopen playback. */ - state->drop_unscoped_audio = 1; + if (cutoff) { + if (cutoff > state->audio_cutoff_ms) state->audio_cutoff_ms = cutoff; + } else if (state->stream_start_ms > state->audio_cutoff_ms) { + /* Fail closed. An interruption we cannot place on the server timeline + * must not leave the reply playing over the user, so drop the stream + * currently in flight: its own START becomes the cutoff (a START equal + * to the cutoff is discarded), and later streams are still admitted. */ + state->audio_cutoff_ms = state->stream_start_ms; + } + size_t kept = 0; + for (size_t i = 0; i < state->queued_audio; i++) { + if (state->queued_start_ms[i] > state->audio_cutoff_ms) + state->queued_start_ms[kept++] = state->queued_start_ms[i]; } + state->queued_audio = kept; pthread_mutex_unlock(&state->mutex); + if (cutoff) + printf("\n[%s] interruption time=%" PRIu64 "\n", source, cutoff); + else + printf("\n[%s] no usable interruption time; dropped the in-flight stream\n", + source); } static void on_ai_control(const char *type, const char *json_data, size_t data_len, void *user_data) { - demo_state_t *state = (demo_state_t *)user_data; if (strcmp(type, "asrInterrupt") != 0) return; - - char event_id[MAX_EVENT_ID] = {0}; - static const char key[] = "\"eventId\":\""; - const char *id = NULL; - for (size_t i = 0; i + sizeof(key) - 1 <= data_len; i++) { - if (memcmp(json_data + i, key, sizeof(key) - 1) == 0) { - id = json_data + i + sizeof(key) - 1; - break; - } - } - if (id) { - size_t available = data_len - (size_t)(id - json_data); - const char *end = memchr(id, '"', available); - if (end) { - size_t n = (size_t)(end - id); - if (n >= sizeof(event_id)) n = sizeof(event_id) - 1; - memcpy(event_id, id, n); - event_id[n] = '\0'; - } - } - set_stale_event(state, event_id); - printf("\n[MQTT control] asrInterrupt event=%s; playback flushed\n", - event_id[0] ? event_id : "(unscoped)"); + interrupt_playback((demo_state_t *)user_data, json_data, data_len, 0); } static int on_flow_control(tai_ctx_t *ctx, void *user_data) @@ -140,10 +146,13 @@ static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *user_data (void)ctx; demo_state_t *state = (demo_state_t *)user_data; pthread_mutex_lock(&state->mutex); - int stale = state->drop_unscoped_audio || - !msg->event_id || !msg->event_id[0] || - event_id_seen(state, msg->event_id); - if (!stale) state->queued_audio++; + if (msg->stream_flag == TAI_STREAM_START || + msg->stream_flag == TAI_STREAM_ONE_SHOT) + state->stream_start_ms = msg->timestamp_ms; + state->current_event_has_audio = 1; + if (msg->len && state->stream_start_ms > state->audio_cutoff_ms && + state->queued_audio < PLAYBACK_CAPACITY) + state->queued_start_ms[state->queued_audio++] = state->stream_start_ms; pthread_mutex_unlock(&state->mutex); } @@ -153,23 +162,21 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *user_data demo_state_t *state = (demo_state_t *)user_data; if (msg->event_type == TAI_EVT_START && msg->event_id && msg->event_id[0]) { pthread_mutex_lock(&state->mutex); - if (!event_id_seen(state, msg->event_id)) { - snprintf(state->current_event_id, sizeof(state->current_event_id), - "%s", msg->event_id); - state->drop_unscoped_audio = 0; - } + snprintf(state->current_event_id, sizeof(state->current_event_id), + "%s", msg->event_id); + state->current_event_has_audio = 0; pthread_mutex_unlock(&state->mutex); } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { - set_stale_event(state, msg->event_id); - printf("\n[TAI Event] ChatBreak event=%s\n", - msg->event_id && msg->event_id[0] ? msg->event_id : "(unscoped)"); + interrupt_playback(state, (const char *)msg->user_data, + msg->user_data_len, 1); } else if (msg->event_type == TAI_EVT_END) { pthread_mutex_lock(&state->mutex); if (msg->event_id && state->current_event_id[0] && strcmp(state->current_event_id, msg->event_id) == 0) { state->current_event_id[0] = '\0'; - forget_uninterrupted_event(state, msg->event_id); - state->done = 1; + if (!state->current_event_has_audio || !state->audio_cutoff_ms || + state->stream_start_ms > state->audio_cutoff_ms) + state->done = 1; } pthread_mutex_unlock(&state->mutex); } @@ -224,7 +231,11 @@ static void *playback_consumer(void *arg) pthread_mutex_unlock(&state->mutex); break; } - if (state->queued_audio > 0) state->queued_audio--; + if (state->queued_audio > 0) { + state->queued_audio--; + memmove(state->queued_start_ms, state->queued_start_ms + 1, + state->queued_audio * sizeof(state->queued_start_ms[0])); + } pthread_mutex_unlock(&state->mutex); } return NULL; diff --git a/modules/iot-client/include/iot_client.h b/modules/iot-client/include/iot_client.h index 079e32c..7a818c3 100644 --- a/modules/iot-client/include/iot_client.h +++ b/modules/iot-client/include/iot_client.h @@ -118,8 +118,14 @@ typedef void (*iot_message_callback_t)(const char *topic, size_t topic_len, * borrowed and remain valid only for the callback duration. Keep the callback * non-blocking and copy anything retained after it returns. * + * A protocol-9000 notice this layer cannot scope -- a non-object data or + * data.data, a non-string type, or a re-serialization allocation failure -- + * is left unconsumed and continues to the raw message callback. + * * @param type Control event type, such as "asrInterrupt". * @param json_data Serialized event-specific data; not necessarily an object. + * Re-serialized from the parsed notice with cJSON (unformatted), + * so its byte layout can differ from the original wire bytes. * @param data_len Length of @p json_data, excluding any terminating NUL. * @param user_data Opaque pointer supplied at registration. */ diff --git a/modules/iot-client/src/iot_ai_ctrl.c b/modules/iot-client/src/iot_ai_ctrl.c index ce8b0f9..cfcb442 100644 --- a/modules/iot-client/src/iot_ai_ctrl.c +++ b/modules/iot-client/src/iot_ai_ctrl.c @@ -59,8 +59,8 @@ bool iot_ai_ctrl_dispatch(iot_client_t *client, return true; } -int iot_ai_ctrl_set_callback(iot_client_t *client, - ai_ctrl_callback_t callback, void *user_data) +IOT_API int iot_ai_ctrl_set_callback(iot_client_t *client, + ai_ctrl_callback_t callback, void *user_data) { if (!client) return OPRT_INVALID_PARAMETER; diff --git a/modules/iot-client/src/iot_internal.h b/modules/iot-client/src/iot_internal.h index 92538fc..89714bc 100644 --- a/modules/iot-client/src/iot_internal.h +++ b/modules/iot-client/src/iot_internal.h @@ -39,7 +39,8 @@ #ifndef IOT_SDK_BV #define IOT_SDK_BV "2.0" #endif -#define SDK_VERSION "agentic-kit_0.5.0" + +#define SDK_VERSION "agentic-kit_0.5.0-dev" /* ========================================================================= * iot-client: service endpoints & protocol constants (fixed by the Tuya diff --git a/modules/iot-client/test/iot_tai_control_test.c b/modules/iot-client/test/iot_tai_control_test.c index d95b1ea..5ed15ef 100644 --- a/modules/iot-client/test/iot_tai_control_test.c +++ b/modules/iot-client/test/iot_tai_control_test.c @@ -21,6 +21,14 @@ #include "tai_internal.h" #include "tai_pal_loopback.h" +/* Include the demo whole (its main renamed away) so the regression drives its + * actual static callbacks. mqtt_interrupt_demo_test.c includes the same demo + * in this one binary -- every file-scope symbol in it must stay static, or + * the two copies collide at link time. */ +#define main combined_demo_main +#include "../../../examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c" +#undef main + #define TEST_DEVID "test_device_msg_001" #define TEST_SECRET_KEY "abcdef1234567890" #define TEST_LOCAL_KEY "0123456789abcdef" @@ -28,6 +36,10 @@ #define TEST_MQTT_PORT 11885 #define WRONG_KEY_PORT 11887 +#define EARLY_AUDIO_MS UINT64_C(1700000000000) +#define OLD_AUDIO_MS UINT64_C(1700000002000) +#define NEW_AUDIO_MS UINT64_C(1700000004000) + extern const pal_t *tai_pal_posix(void); int mqtt_interrupt_demo_tests(void); @@ -105,8 +117,9 @@ static void sleep_ms(uint32_t ms) } typedef struct { + /* This mutex serializes counters and wrappers; callbacks also take playback.mutex. */ pthread_mutex_t mutex; - size_t queued_audio; + demo_state_t playback; int audio_callbacks; int accepted_old_audio; int accepted_new_audio; @@ -115,8 +128,6 @@ typedef struct { int mqtt_interrupts; int interrupted_before_tcp_drain; pthread_t mqtt_callback_thread; - char current_event_id[64]; - char stale_event_id[64]; } app_state_t; static app_state_t state; @@ -205,41 +216,33 @@ static void stop_mqtt_mock(void) found; \ }) -static int state_check(int expression) -{ - pthread_mutex_lock(&state.mutex); - int result = expression; - pthread_mutex_unlock(&state.mutex); - return result; -} - -#define STATE_CHECK(expr) CHECK(state_check(expr)) +#define STATE_CHECK(expr) \ + do { \ + pthread_mutex_lock(&state.mutex); \ + CHECK(expr); \ + pthread_mutex_unlock(&state.mutex); \ + } while (0) static int flow_control(tai_ctx_t *ctx, void *user_data) { (void)ctx; app_state_t *s = (app_state_t *)user_data; pthread_mutex_lock(&s->mutex); - int admit = s->queued_audio == 0; + /* One queued frame is enough to pressure the real worker in this test. */ + int admit = s->playback.queued_audio == 0; pthread_mutex_unlock(&s->mutex); return admit; } -static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, - void *user_data) +static void combined_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, + void *user_data) { - (void)ctx; app_state_t *s = (app_state_t *)user_data; pthread_mutex_lock(&s->mutex); s->audio_callbacks++; - if (s->stale_event_id[0] && msg->event_id && msg->event_id[0] && - strcmp(s->stale_event_id, msg->event_id) != 0) { - s->stale_event_id[0] = '\0'; - } - int stale = s->stale_event_id[0] && msg->event_id && - strcmp(s->stale_event_id, msg->event_id) == 0; - if (!stale && s->queued_audio == 0) { - s->queued_audio = 1; + size_t before = s->playback.queued_audio; + on_audio(ctx, msg, &s->playback); + if (s->playback.queued_audio > before) { if (msg->event_id && strcmp(msg->event_id, "old-event") == 0) { s->accepted_old_audio++; } else if (msg->event_id && strcmp(msg->event_id, "new-event") == 0) { @@ -251,70 +254,27 @@ static void on_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, pthread_mutex_unlock(&s->mutex); } -static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, - void *user_data) +static void combined_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, + void *user_data) { - (void)ctx; app_state_t *s = (app_state_t *)user_data; pthread_mutex_lock(&s->mutex); - if (msg->event_type == TAI_EVT_START && msg->event_id && msg->event_id[0]) { - snprintf(s->current_event_id, sizeof(s->current_event_id), - "%s", msg->event_id); - if (s->stale_event_id[0] && - strcmp(s->stale_event_id, msg->event_id) != 0) { - s->stale_event_id[0] = '\0'; - } - } else if (msg->event_type == TAI_EVT_END && msg->event_id && msg->event_id[0] && - s->current_event_id[0] && - strcmp(s->current_event_id, msg->event_id) == 0) { - s->current_event_id[0] = '\0'; - } else if (msg->event_type == TAI_EVT_CHAT_BREAK) { - s->chat_breaks++; - if (!msg->event_id || !msg->event_id[0] || !s->current_event_id[0] || - strcmp(s->current_event_id, msg->event_id) == 0) { - s->queued_audio = 0; - if (msg->event_id && msg->event_id[0]) { - snprintf(s->stale_event_id, sizeof(s->stale_event_id), - "%s", msg->event_id); - } - } - } + if (msg->event_type == TAI_EVT_CHAT_BREAK) s->chat_breaks++; + on_event(ctx, msg, &s->playback); pthread_mutex_unlock(&s->mutex); } -static void on_ai_control(const char *type, const char *json_data, - size_t data_len, void *user_data) +static void combined_control(const char *type, const char *json_data, + size_t data_len, void *user_data) { app_state_t *s = (app_state_t *)user_data; if (strcmp(type, "asrInterrupt") != 0) return; - - static const char prefix[] = "\"eventId\":\""; - char event_id[64] = {0}; - for (size_t i = 0; i + sizeof(prefix) - 1 <= data_len; i++) { - if (memcmp(json_data + i, prefix, sizeof(prefix) - 1) != 0) continue; - const char *value = json_data + i + sizeof(prefix) - 1; - size_t available = data_len - (size_t)(value - json_data); - const char *end = memchr(value, '"', available); - if (end) { - size_t len = (size_t)(end - value); - if (len >= sizeof(event_id)) len = sizeof(event_id) - 1; - memcpy(event_id, value, len); - } - break; - } - if (!event_id[0]) return; - pthread_mutex_lock(&s->mutex); s->mqtt_callback_thread = pthread_self(); s->mqtt_interrupts++; if (s->mqtt_interrupts == 2 && s->chat_breaks == 0) s->interrupted_before_tcp_drain = 1; - if (!s->current_event_id[0] || - strcmp(s->current_event_id, event_id) == 0) { - s->queued_audio = 0; - snprintf(s->stale_event_id, sizeof(s->stale_event_id), - "%s", event_id); - } + on_ai_control(type, json_data, data_len, &s->playback); pthread_mutex_unlock(&s->mutex); } @@ -341,25 +301,28 @@ static int server_push(tai_ctx_t *ctx, uint8_t packet_type, static int server_push_event(tai_ctx_t *ctx, uint16_t event_type, const char *event_id, uint16_t sequence) { - tai_attr_t attrs[2] = { + tai_attr_t attrs[3] = { tai_attr_strv(TAI_ATTR_SESSION_ID, ctx->session_id), tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + tai_attr_strv(TAI_ATTR_USER_DATA, + "{\"breakAttributes\":{\"time\":\"1700000003000\"}}"), }; uint8_t payload[4]; int payload_len = tai_pack_event(TAI_VER_21, event_type, NULL, 0, payload, sizeof(payload)); if (payload_len <= 0) return payload_len; - return server_push(ctx, TAI_PKT_EVENT, attrs, 2, + return server_push(ctx, TAI_PKT_EVENT, attrs, + event_type == TAI_EVT_CHAT_BREAK ? 3 : 2, payload, (size_t)payload_len, sequence); } static int server_push_audio(tai_ctx_t *ctx, const char *event_id, - uint8_t fill, uint16_t sequence) + uint64_t timestamp_ms, uint8_t fill, uint16_t sequence) { uint8_t payload[48]; int header_len = tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, TAI_STREAM_START, - ctx->pal->time_ms(), + timestamp_ms, payload, sizeof(payload)); if (header_len <= 0) return header_len; memset(payload + header_len, fill, 40); @@ -377,6 +340,7 @@ int main(void) { const pal_t *pal = make_routing_pal(); pthread_mutex_init(&state.mutex, NULL); + pthread_mutex_init(&state.playback.mutex, NULL); CHECK(iot_init(pal) == OPRT_OK); failures += mqtt_interrupt_demo_tests(); CHECK(start_mqtt_mock() == 0); @@ -391,7 +355,7 @@ int main(void) snprintf(iot.local_key, sizeof(iot.local_key), "%s", TEST_LOCAL_KEY); snprintf(iot.mqtt_url, sizeof(iot.mqtt_url), "%s", TEST_MQTT_URL); iot.cacert = cacert; - CHECK(iot_ai_ctrl_set_callback(&iot, on_ai_control, &state) == OPRT_OK); + CHECK(iot_ai_ctrl_set_callback(&iot, combined_control, &state) == OPRT_OK); CHECK(iot_client_message_connect(&iot) == OPRT_OK); /* Consume the mock's initial raw message before the combined scenario. */ CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); @@ -420,8 +384,8 @@ int main(void) .sign_level = TAI_SIGN_HMAC_SHA256, .disable_tls = 1, .pal = pal, - .on_audio = on_audio, - .on_event = on_event, + .on_audio = combined_audio, + .on_event = combined_event, .on_flow_control = flow_control, .user_data = &state, }; @@ -437,37 +401,37 @@ int main(void) static const char early_control[] = "{\"protocol\":9000,\"data\":{\"data\":{" "\"type\":\"asrInterrupt\"," - "\"data\":{\"eventId\":\"early-event\"}}}}"; + "\"data\":{\"time\":\"1700000001000\"}}}}"; CHECK(iot_client_message_publish(&iot, (const uint8_t *)early_control, sizeof(early_control) - 1) == OPRT_OK); for (int attempt = 0; attempt < 50 && !WAIT_FOR(state.mqtt_interrupts >= 1, 5); attempt++) { CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } STATE_CHECK(state.mqtt_interrupts == 1); - STATE_CHECK(state.queued_audio == 0); + STATE_CHECK(state.playback.queued_audio == 0); /* MQTT-before-audio ordering: early-event is cancelled before its START; * its later audio must drain without entering playback. */ CHECK(server_push_event(ctx, TAI_EVT_START, "early-event", sequence++) > 0); - CHECK(server_push_audio(ctx, "early-event", 0x01, sequence++) > 0); + CHECK(server_push_audio(ctx, "early-event", EARLY_AUDIO_MS, 0x01, sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_END, "early-event", sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_START, "old-event", sequence++) > 0); - CHECK(server_push_audio(ctx, "old-event", 0x02, sequence++) > 0); + CHECK(server_push_audio(ctx, "old-event", OLD_AUDIO_MS, 0x02, sequence++) > 0); CHECK(WAIT_FOR(state.accepted_old_audio == 1, 1000)); STATE_CHECK(state.accepted_early_audio == 0); - STATE_CHECK(state.queued_audio == 1); + STATE_CHECK(state.playback.queued_audio == 1); pthread_mutex_lock(&state.mutex); - state.queued_audio = 0; + state.playback.queued_audio = 0; pthread_mutex_unlock(&state.mutex); - CHECK(server_push_audio(ctx, "old-event", 0x11, sequence++) > 0); + CHECK(server_push_audio(ctx, "old-event", OLD_AUDIO_MS, 0x11, sequence++) > 0); CHECK(WAIT_FOR(state.accepted_old_audio == 2, 1000)); - CHECK(server_push_audio(ctx, "old-event", 0x22, sequence++) > 0); + CHECK(server_push_audio(ctx, "old-event", OLD_AUDIO_MS, 0x22, sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, "old-event", sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_END, "old-event", sequence++) > 0); CHECK(server_push_event(ctx, TAI_EVT_START, "new-event", sequence++) > 0); - CHECK(server_push_audio(ctx, "new-event", 0x33, sequence++) > 0); + CHECK(server_push_audio(ctx, "new-event", NEW_AUDIO_MS, 0x33, sequence++) > 0); /* Keep admission closed while the delayed duplicate waits on TCP. */ CHECK(server_push_event(ctx, TAI_EVT_CHAT_BREAK, "old-event", sequence++) > 0); @@ -478,7 +442,7 @@ int main(void) static const char control[] = "{\"protocol\":9000,\"data\":{\"data\":{" "\"type\":\"asrInterrupt\"," - "\"data\":{\"eventId\":\"old-event\"}}}}"; + "\"data\":{\"time\":\"1700000003000\"}}}}"; pthread_t mqtt_owner = pthread_self(); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_OK); @@ -497,19 +461,19 @@ int main(void) STATE_CHECK(state.accepted_old_audio == 2); STATE_CHECK(state.accepted_new_audio == 1); STATE_CHECK(state.chat_breaks == 1); - STATE_CHECK(state.queued_audio == 1); + STATE_CHECK(state.playback.queued_audio == 1); /* Release pressure by consuming new audio; the queued duplicate is ignored. */ pthread_mutex_lock(&state.mutex); - state.queued_audio = 0; + state.playback.queued_audio = 0; pthread_mutex_unlock(&state.mutex); CHECK(WAIT_FOR(state.chat_breaks == 2, 2000)); - STATE_CHECK(state.queued_audio == 0); + STATE_CHECK(state.playback.queued_audio == 0); STATE_CHECK(state.accepted_new_audio == 1); /* Delayed duplicate on MQTT must also not flush the new playback. */ pthread_mutex_lock(&state.mutex); - state.queued_audio = 1; + state.playback.queued_audio = 1; pthread_mutex_unlock(&state.mutex); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_OK); @@ -517,7 +481,7 @@ int main(void) CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } STATE_CHECK(state.mqtt_interrupts == 3); - STATE_CHECK(state.queued_audio == 1); + STATE_CHECK(state.playback.queued_audio == 1); STATE_CHECK(state.accepted_new_audio == 1); /* MQTT loss removes only the independent control path. TAI stays paused on @@ -525,7 +489,7 @@ int main(void) iot_client_message_disconnect(&iot); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, sizeof(control) - 1) == OPRT_UNINITIALIZED); - STATE_CHECK(state.queued_audio == 1); + STATE_CHECK(state.playback.queued_audio == 1); CHECK(iot_client_message_connect(&iot) == OPRT_OK); CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); CHECK(iot_client_message_publish(&iot, (const uint8_t *)control, @@ -534,7 +498,7 @@ int main(void) CHECK(iot_client_message_process(&iot, 50) == OPRT_OK); } STATE_CHECK(state.mqtt_interrupts == 4); - STATE_CHECK(state.queued_audio == 1); + STATE_CHECK(state.playback.queued_audio == 1); /* Shutdown while receive admission is still closed must not hang. */ tai_disconnect(ctx); @@ -543,6 +507,7 @@ int main(void) iot.cacert = NULL; pal->free(cacert); stop_mqtt_mock(); + pthread_mutex_destroy(&state.playback.mutex); pthread_mutex_destroy(&state.mutex); printf("iot_tai_control_test: %s\n", failures ? "FAIL" : "PASS"); return failures ? 1 : 0; diff --git a/modules/iot-client/test/mqtt_interrupt_demo_test.c b/modules/iot-client/test/mqtt_interrupt_demo_test.c index e02b4a8..ab1a14a 100644 --- a/modules/iot-client/test/mqtt_interrupt_demo_test.c +++ b/modules/iot-client/test/mqtt_interrupt_demo_test.c @@ -1,5 +1,8 @@ /* Include the demo so regressions exercise its actual static callbacks rather - * than a test-only copy of the interruption/filtering logic. Never run its main. */ + * than a test-only copy of the interruption/filtering logic. Never run its + * main. iot_tai_control_test.c includes the same demo in this one binary -- + * every file-scope symbol in it must stay static, or the two copies collide + * at link time. */ #define main mqtt_interrupt_demo_main #include "../../../examples/posix/ai/rtc-tcp-client/mqtt_interrupt_demo.c" #undef main @@ -14,33 +17,70 @@ } \ } while (0) -static int demo_test_start(tai_ctx_t *ctx, const char *event_id) +#define EPOCH_MS UINT64_C(1700000000000) + +static int demo_test_event(tai_ctx_t *ctx, uint16_t type, const char *event_id, + const char *user_data, const char *body) { - uint8_t payload[4]; - tai_attr_t attr = tai_attr_strv(TAI_ATTR_EVENT_ID, event_id); - int len = tai_pack_event(TAI_VER_21, TAI_EVT_START, NULL, 0, - payload, sizeof(payload)); + uint8_t payload[256]; + tai_attr_t attrs[2] = { + tai_attr_strv(TAI_ATTR_EVENT_ID, event_id), + tai_attr_strv(TAI_ATTR_USER_DATA, user_data ? user_data : ""), + }; + int len = tai_pack_event(TAI_VER_21, type, (const uint8_t *)body, + body ? strlen(body) : 0, payload, sizeof(payload)); if (len <= 0) return TAI_ERR_PROTO; - return tai_proto_dispatch(ctx, TAI_PKT_EVENT, &attr, 1, + return tai_proto_dispatch(ctx, TAI_PKT_EVENT, attrs, user_data ? 2 : 1, payload, (size_t)len); } -static int demo_test_audio(tai_ctx_t *ctx, size_t frames) +static int demo_test_audio(tai_ctx_t *ctx, uint8_t flag, uint64_t timestamp, + size_t frames) { /* Keep the paused body's storage pinned in rx_buf, just like the worker. */ int len = tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, - TAI_STREAM_START, 123, - ctx->rx_buf, sizeof(ctx->rx_buf)); + flag, timestamp, ctx->rx_buf, sizeof(ctx->rx_buf)); if (len <= 0) return TAI_ERR_PROTO; size_t body_len = frames * 40; if (body_len > sizeof(ctx->rx_buf) - (size_t)len) return TAI_ERR_MEM; - memset(ctx->rx_buf + len, 0x5a, body_len); + for (size_t i = 0; i < frames; i++) + memset(ctx->rx_buf + len + i * 40, (int)i + 1, 40); tai_attr_t attr = tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, "111 1 16 16000 0 16000 20 40"); return tai_proto_dispatch(ctx, TAI_PKT_AUDIO, &attr, 1, ctx->rx_buf, (size_t)len + body_len); } +static void demo_test_interrupt(demo_state_t *state, uint64_t timestamp) +{ + char json[64]; + int len = snprintf(json, sizeof(json), "{\"time\":\"%" PRIu64 "\"}", timestamp); + on_ai_control("asrInterrupt", json, (size_t)len, state); +} + +static void *demo_racing_audio(void *arg) +{ + demo_state_t *state = arg; + tai_audio_msg_t audio = { + .stream_flag = TAI_STREAM_START, + .data = (const uint8_t *)"opus", + .len = 4, + }; + for (uint64_t i = 1; i <= 1000; i++) { + audio.timestamp_ms = EPOCH_MS + i; + if (on_flow_control(NULL, state)) on_audio(NULL, &audio, state); + } + return NULL; +} + +static void *demo_racing_interrupt(void *arg) +{ + demo_state_t *state = arg; + static const char cutoff[] = "{\"time\":\"1700000001000\"}"; + on_ai_control("asrInterrupt", cutoff, sizeof(cutoff) - 1, state); + return NULL; +} + int mqtt_interrupt_demo_tests(void) { int failures = 0; @@ -64,87 +104,214 @@ int mqtt_interrupt_demo_tests(void) if (!state.tai) goto cleanup; tai_ctx_t *ctx = state.tai; - /* Mid-Packet pause: accepted frames fill playback, two remain retained. - * Interruption must not mutate the worker-owned pending state. */ - DEMO_CHECK(demo_test_start(ctx, "paused-event") == TAI_OK); - DEMO_CHECK(demo_test_audio(ctx, PLAYBACK_CAPACITY + 2) == TAI_OK); - DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); - DEMO_CHECK(ctx->rx_pending_valid == 1); - DEMO_CHECK(ctx->rx_pending_offset == PLAYBACK_CAPACITY * 40); - DEMO_CHECK(ctx->rx_pending_len == (PLAYBACK_CAPACITY + 2) * 40); - DEMO_CHECK(strcmp(ctx->rx_pending_event_id, "paused-event") == 0); - DEMO_CHECK(on_flow_control(ctx, &state) == 0); - DEMO_CHECK(tai_proto_drain_pending_audio(ctx, 0) == 1); - DEMO_CHECK(ctx->rx_pending_offset == PLAYBACK_CAPACITY * 40); - static const char paused_interrupt[] = "{\"eventId\":\"paused-event\"}"; - on_ai_control("asrInterrupt", paused_interrupt, - sizeof(paused_interrupt) - 1, &state); - DEMO_CHECK(state.queued_audio == 0); - DEMO_CHECK(ctx->rx_pending_valid == 1); - DEMO_CHECK(ctx->rx_pending_discard == 0); - DEMO_CHECK(ctx->rx_pending_offset == PLAYBACK_CAPACITY * 40); - DEMO_CHECK(on_flow_control(ctx, &state) == 1); - DEMO_CHECK(tai_proto_drain_pending_audio(ctx, 0) == 0); - DEMO_CHECK(ctx->rx_pending_valid == 0); - DEMO_CHECK(ctx->rx_pending_offset == ctx->rx_pending_len); - DEMO_CHECK(ctx->rx_pending_body == NULL); - DEMO_CHECK(state.queued_audio == 0); + /* The two Event slices are independent. A plausible payload is never a + * fallback for a missing or malformed Attribute 111: an unusable time + * fails closed onto the in-flight stream's own START (EPOCH_MS), never + * onto the payload's timestamp (+900). */ + const char *plausible_body = + "{\"breakAttributes\":{\"time\":\"1700000000900\"}}"; + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS, 1) == TAI_OK); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", NULL, + plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", "{}", + plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000300\\u0000junk\"}}", + plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000100\"}}", plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 100 && state.queued_audio == 0); + + /* No Event ID is required for audio, and stale START cannot revive playback. + * Later per-Packet timestamps do not change the stream's classification. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS, 2) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_MIDDLE, EPOCH_MS + 900, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_END, EPOCH_MS + 1000, 1) == TAI_OK); + DEMO_CHECK(state.stream_start_ms == EPOCH_MS && state.queued_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_ONE_SHOT, EPOCH_MS + 200, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 1); + + static const char *invalid[] = { + "{}", "null", "[]", "{\"time\":null}", "{\"time\":true}", + "{\"time\":{}}", "{\"time\":\"\"}", "{\"time\":\"0\"}", + "{\"time\":0}", "{\"time\":-1}", "{\"time\":1.5}", + "{\"time\":\"-1\"}", "{\"time\":\"+1700000000300\"}", + "{\"time\":\"1700000000300x\"}", "{\"time\":\" 1700000000300\"}", + "{\"time\":\"1700000000300.0\"}", "{\"time\":\"1e12\"}", + "{\"time\":\"4398046511104\"}", "{\"time\":4398046511104}", + "{\"time\":\"18446744073709551616000\"}", + "{\"time\":\"1700000000300\\u0000junk\"}", + "{\"time\":\"1700000000300\"", "{\"time\":\"1700000000300\"}garbage", + }; + for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) { + on_ai_control("asrInterrupt", invalid[i], strlen(invalid[i]), &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 0); + } + on_ai_control("asrInterrupt", NULL, 0, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 0); + static const char bounded[] = "{\"time\":\"1700000000300\"}ignored"; + on_ai_control("asrInterrupt", bounded, sizeof(bounded) - 10, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 0); + on_ai_control("not-an-interrupt", bounded, sizeof(bounded) - 8, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200); + + /* Duplicates and older notices cannot flush fresh audio. A newer cutoff + * still below the stream start is applied but must preserve it. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 250, 1) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 1); + demo_test_interrupt(&state, EPOCH_MS + 100); + demo_test_interrupt(&state, EPOCH_MS + 50); + static const char numeric[] = "{\"time\":1700000000150} \n"; + on_ai_control("asrInterrupt", numeric, sizeof(numeric) - 1, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 200 && state.queued_audio == 1); + DEMO_CHECK(state.queued_start_ms[0] == EPOCH_MS + 250); + static const char newer[] = "{\"time\":\"1700000000225\"}"; + on_ai_control("asrInterrupt", newer, sizeof(newer) - 1, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 225 && state.queued_audio == 1); + DEMO_CHECK(state.queued_start_ms[0] == EPOCH_MS + 250); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000175\"}}", plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 225 && state.queued_audio == 1); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_CHAT_BREAK, "unrelated", + "{\"breakAttributes\":{\"time\":\"1700000000100\"}}", plausible_body) == TAI_OK); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 225 && state.queued_audio == 1); + + /* A bounded, non-NUL-terminated view is accepted, ignoring adjacent bytes. */ + on_ai_control("asrInterrupt", bounded, sizeof(bounded) - 8, &state); + DEMO_CHECK(state.audio_cutoff_ms == EPOCH_MS + 300 && state.queued_audio == 0); - /* Deterministically order the race: admission, interrupt, audio callback. */ - DEMO_CHECK(demo_test_start(ctx, "racing-event") == TAI_OK); + /* Deterministically exercise both callback/admission race orderings. */ tai_audio_msg_t audio = { - .event_id = "racing-event", .stream_flag = TAI_STREAM_START, + .timestamp_ms = EPOCH_MS + 400, .data = (const uint8_t *)"opus", .len = 4, }; DEMO_CHECK(on_flow_control(ctx, &state) == 1); - static const char racing_interrupt[] = "{\"eventId\":\"racing-event\"}"; - on_ai_control("asrInterrupt", racing_interrupt, - sizeof(racing_interrupt) - 1, &state); + demo_test_interrupt(&state, EPOCH_MS + 400); on_audio(ctx, &audio, &state); DEMO_CHECK(state.queued_audio == 0); - - /* The opposite ordering accepts audio first, then flushes that queue. */ - DEMO_CHECK(demo_test_start(ctx, "queued-event") == TAI_OK); - audio.event_id = "queued-event"; + audio.timestamp_ms = EPOCH_MS + 500; DEMO_CHECK(on_flow_control(ctx, &state) == 1); on_audio(ctx, &audio, &state); DEMO_CHECK(state.queued_audio == 1); - static const char queued_interrupt[] = "{\"eventId\":\"queued-event\"}"; - on_ai_control("asrInterrupt", queued_interrupt, - sizeof(queued_interrupt) - 1, &state); - DEMO_CHECK(state.queued_audio == 0); - on_audio(ctx, &audio, &state); + demo_test_interrupt(&state, EPOCH_MS + 500); DEMO_CHECK(state.queued_audio == 0); - /* A distinct non-stale Event remains playable after scoped interruptions. */ - DEMO_CHECK(demo_test_start(ctx, "fresh-event") == TAI_OK); - DEMO_CHECK(strcmp(state.current_event_id, "fresh-event") == 0); - DEMO_CHECK(demo_test_audio(ctx, 1) == TAI_OK); - DEMO_CHECK(state.queued_audio == 1); - DEMO_CHECK(ctx->rx_pending_valid == 0); - - /* Missing correlation also drains retained START frames without reopening - * playback; only the next identified Event restores admission to playback. */ - DEMO_CHECK(demo_test_audio(ctx, PLAYBACK_CAPACITY + 1) == TAI_OK); + /* Mixed queues retain each item's original stream start, not merely the + * latest stream's timestamp. Preserve order while removing stale items. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 600, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 900, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 700, 1) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_ONE_SHOT, EPOCH_MS + 1000, 1) == TAI_OK); DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); - DEMO_CHECK(ctx->rx_pending_valid == 1); - DEMO_CHECK(ctx->rx_pending_offset == (PLAYBACK_CAPACITY - 1) * 40); - on_ai_control("asrInterrupt", "{}", 2, &state); + demo_test_interrupt(&state, EPOCH_MS + 700); + DEMO_CHECK(state.queued_audio == 2); + DEMO_CHECK(state.queued_start_ms[0] == EPOCH_MS + 900); + DEMO_CHECK(state.queued_start_ms[1] == EPOCH_MS + 1000); + demo_test_interrupt(&state, EPOCH_MS + 1000); DEMO_CHECK(state.queued_audio == 0); - DEMO_CHECK(state.drop_unscoped_audio == 1); - DEMO_CHECK(ctx->rx_pending_valid == 1); - DEMO_CHECK(ctx->rx_pending_discard == 0); - DEMO_CHECK(tai_proto_drain_pending_audio(ctx, 0) == 0); - DEMO_CHECK(ctx->rx_pending_valid == 0); - DEMO_CHECK(ctx->rx_pending_offset == ctx->rx_pending_len); + + /* Ordinary pause/resume loses no codec frames; repeated START callbacks, + * including resumed ones, all retain the same server timestamp. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1100, + PLAYBACK_CAPACITY + 2) == TAI_OK); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + DEMO_CHECK(ctx->rx_pending_len == 80); + DEMO_CHECK(ctx->rx_pending_body == ctx->rx_buf + 8 + PLAYBACK_CAPACITY * 40); + DEMO_CHECK(ctx->rx_pending_ts_ms == EPOCH_MS + 1100); + for (size_t i = 0; i < state.queued_audio; i++) + DEMO_CHECK(state.queued_start_ms[i] == EPOCH_MS + 1100); + const uint8_t *cursor = ctx->rx_pending_body; + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 1); + DEMO_CHECK(ctx->rx_pending_body == cursor && ctx->rx_pending_len == 80); + /* Consume one frame, resume exactly one, then consume the full queue. */ + state.queued_audio--; + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 1); + DEMO_CHECK(state.queued_audio == PLAYBACK_CAPACITY); + DEMO_CHECK(ctx->rx_pending_body == cursor + 40 && ctx->rx_pending_len == 40); + DEMO_CHECK(state.queued_start_ms[PLAYBACK_CAPACITY - 1] == EPOCH_MS + 1100); + state.queued_audio = 0; + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 0); + DEMO_CHECK(ctx->rx_pending_body == NULL && ctx->rx_pending_len == 0); + DEMO_CHECK(state.queued_audio == 1 && state.queued_start_ms[0] == EPOCH_MS + 1100); + state.queued_audio = 0; + + /* Interrupt a paused START without touching any worker-owned pending state. + * The worker then drains the retained frames through the real callback. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1200, + PLAYBACK_CAPACITY + 2) == TAI_OK); + cursor = ctx->rx_pending_body; + demo_test_interrupt(&state, EPOCH_MS + 1200); + DEMO_CHECK(state.queued_audio == 0 && on_flow_control(ctx, &state) == 1); + DEMO_CHECK(ctx->rx_pending_body == cursor && ctx->rx_pending_len == 80); + DEMO_CHECK(ctx->rx_pending_ts_ms == EPOCH_MS + 1200); + DEMO_CHECK(ctx->rx_pending_flag == TAI_STREAM_START); + DEMO_CHECK(tai_proto_drain_pending_audio(ctx) == 0); + DEMO_CHECK(ctx->rx_pending_body == NULL && ctx->rx_pending_len == 0); + DEMO_CHECK(state.queued_audio == 0 && state.stream_start_ms == EPOCH_MS + 1200); + + /* Empty START/ONE_SHOT must latch through the real dispatcher but must not + * enter playback. MIDDLE/END use that latch, not their own timestamps. */ + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1300, 0) == TAI_OK); + DEMO_CHECK(state.stream_start_ms == EPOCH_MS + 1300 && state.queued_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_MIDDLE, EPOCH_MS, 1) == TAI_OK); + DEMO_CHECK(state.queued_audio == 1 && state.queued_start_ms[0] == EPOCH_MS + 1300); + demo_test_interrupt(&state, EPOCH_MS + 1300); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_ONE_SHOT, EPOCH_MS + 1250, 0) == TAI_OK); + DEMO_CHECK(state.stream_start_ms == EPOCH_MS + 1250 && state.queued_audio == 0); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_END, EPOCH_MS + 2000, 1) == TAI_OK); DEMO_CHECK(state.queued_audio == 0); - DEMO_CHECK(state.drop_unscoped_audio == 1); - DEMO_CHECK(demo_test_start(ctx, "after-unscoped-event") == TAI_OK); - DEMO_CHECK(state.drop_unscoped_audio == 0); - DEMO_CHECK(demo_test_audio(ctx, 1) == TAI_OK); - DEMO_CHECK(state.queued_audio == 1); + + /* END for interrupted or previous Events cannot terminate the demo. */ + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_START, "old", NULL, NULL) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1200, 1) == TAI_OK); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "old", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_START, "new", NULL, NULL) == TAI_OK); + DEMO_CHECK(demo_test_audio(ctx, TAI_STREAM_START, EPOCH_MS + 1400, 1) == TAI_OK); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "old", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 0 && strcmp(state.current_event_id, "new") == 0); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "new", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 1); + + /* A later text-only Event must not inherit the interrupted audio latch. */ + demo_test_interrupt(&state, EPOCH_MS + 1400); + state.done = 0; + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_START, "text-only", NULL, NULL) == TAI_OK); + tai_text_msg_t text = { .text = "", .len = 0 }; + on_text(ctx, &text, &state); + DEMO_CHECK(demo_test_event(ctx, TAI_EVT_END, "text-only", NULL, NULL) == TAI_OK); + DEMO_CHECK(state.done == 1); + + /* The inclusive 42-bit upper bound is valid, without narrowing to 32 bits. */ + demo_test_interrupt(&state, MAX_AUDIO_TIME_MS); + DEMO_CHECK(state.audio_cutoff_ms == MAX_AUDIO_TIME_MS && state.queued_audio == 0); + + /* No outer test mutex serializes these production callbacks: either order + * must leave all <= cutoff audio discarded using only the playback mutex. */ + demo_state_t racing = {0}; + int mutex_rc = pthread_mutex_init(&racing.mutex, NULL); + DEMO_CHECK(mutex_rc == 0); + if (mutex_rc == 0) { + for (int round = 0; round < 16; round++) { + racing.audio_cutoff_ms = 0; + racing.stream_start_ms = 0; + racing.queued_audio = 0; + pthread_t audio_thread, control_thread; + int audio_rc = pthread_create(&audio_thread, NULL, demo_racing_audio, &racing); + int control_rc = pthread_create(&control_thread, NULL, demo_racing_interrupt, &racing); + DEMO_CHECK(audio_rc == 0 && control_rc == 0); + if (audio_rc == 0) pthread_join(audio_thread, NULL); + if (control_rc == 0) pthread_join(control_thread, NULL); + DEMO_CHECK(racing.audio_cutoff_ms == EPOCH_MS + 1000); + DEMO_CHECK(racing.queued_audio == 0); + } + DEMO_CHECK(pthread_mutex_destroy(&racing.mutex) == 0); + } cleanup: if (state.tai) tai_ctx_deinit(state.tai); diff --git a/modules/rtc-tcp-client/CONTEXT.md b/modules/rtc-tcp-client/CONTEXT.md index 0786107..25c0efd 100644 --- a/modules/rtc-tcp-client/CONTEXT.md +++ b/modules/rtc-tcp-client/CONTEXT.md @@ -84,9 +84,12 @@ _Avoid_: heartbeat, poll. **Receive backpressure**: Application-controlled admission that pauses all inbound traffic, including buffered Frames, -ChatBreak, ASR text, Pong and EOF detection, rather than only media delivery. Admission is at -wire Frame boundaries, not codec-frame boundaries: one dispatched Packet can deliver multiple -codec frames, so the application must also bound admission in its audio callback. +ChatBreak, ASR text, Pong and EOF detection, rather than only media delivery. Admission is +checked between complete Frames and before each codec-frame callback within an Audio Packet: a +mid-Packet pause keeps the Packet's remaining bytes where they are (zero-copy, worker-owned) +and delivers them before any later Packet once admission reopens. Interruption filtering stays +in the application and is time-based — the server media timestamp against the interruption +time. _Avoid_: audio mute, selective pause, codec-frame flow control. **Chat break**: @@ -101,7 +104,10 @@ _Avoid_: cancel, stop, abort. **Server-initiated interrupt**: An application concern delivered independently from the media Connection: either an inbound ChatBreak Event on this context or an authenticated MQTT protocol-9000 AI control notice from -the IoT Client. Applications route both paths into one playback policy; this module does not +the IoT Client. Both carry a server time naming the interruption — the MQTT notice in its own +payload, the ChatBreak in the UserData attribute (attr 111, `breakAttributes.time`) — and the +application filters downlink audio by comparing that time against the media timestamp of the +stream's START. Applications route both paths into one playback policy; this module does not receive or inject the MQTT notice. The independent path remains available while receive backpressure intentionally stops this Connection. _Avoid_: MQTT Event, TAI injection, acknowledgement. @@ -212,9 +218,10 @@ The worker loops: check the liveness deadline, send a Ping when due, then block time-bounded (`AGENTIC_KIT_TAI_DRAIN_BUDGET_MS`, default 150 ms) so a sustained downstream flood cannot starve the Ping / liveness / shutdown checks — leftover bytes wait for the next pass; and any successful receive refreshes the liveness clock. Receive backpressure pauses both reads and -parsing; on resume, buffered complete Frames precede the next read or EOF detection, while -partial input returns to bounded blocking reception. Bytes accumulate in a sliding receive -buffer; detected EOF or a transport error makes the worker fire `on_disconnect`. +parsing; on resume, a paused Packet's remaining codec frames drain first, then buffered complete +Frames precede the next read or EOF detection, while partial input returns to bounded blocking +reception. Bytes accumulate in a sliding receive buffer; detected EOF or a transport error makes +the worker fire `on_disconnect`. `tai_process_rx` peels complete Frames off the front of that buffer: diff --git a/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md b/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md index 2ad3ba9..2bb7cc4 100644 --- a/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md +++ b/modules/rtc-tcp-client/docs/adr/0001-receive-backpressure-pauses-the-connection.md @@ -1,15 +1,22 @@ -# Receive backpressure pauses the Connection at Frame boundaries +# Receive backpressure pauses the Connection and retains a partial Packet in place The RTC TCP Client exposes an optional `on_flow_control` admission hook. When it -returns zero, the worker stops parsing complete buffered Frames and stops reading -the Connection, allowing the TCP receive window to close. This pauses all inbound -traffic rather than selecting only audio. The worker still runs Ping, shutdown, -and bounded admission checks through `pal_t.sleep_ms`. +returns zero, the worker stops parsing and stops reading the Connection, allowing +the TCP receive window to close. This pauses all inbound traffic rather than +selecting only audio. The worker still runs Ping, shutdown, and bounded admission +checks through `pal_t.sleep_ms`. -We chose Frame-boundary admission because receive and reassembly buffers are -worker-owned borrowed storage. Deferring individual callback payloads would need -a second bounded queue and new lifetime rules. The application must still bound -its audio callback because one Audio Packet can emit multiple codec frames. +Admission is checked between complete Frames and before each codec-frame callback +within an Audio Packet. Checking only at Frame boundaries would force the +application either to accept a whole Packet or to lose its remainder: the queue +that fills is the application's, and one Audio Packet can emit several codec frames. + +A mid-Packet pause keeps the Packet's bytes where they already are rather than +copying them into a second queue. The worker records the remaining body pointer, its +length, and the wire length of the pinned Frame; only the worker advances that +cursor, and no other Packet dispatches until it is exhausted — so `rx_audio_*` and +`rx_event_id` still describe the paused Packet and need not be duplicated. Teardown +discards a remainder. ## Consequences @@ -17,6 +24,12 @@ its audio callback because one Audio Packet can emit multiple codec frames. an independent MQTT control path can carry urgent application notices. - Intentional pauses suspend receive-liveness accounting. Resume starts a fresh `ping_timeout_ms` budget without pretending traffic arrived. -- Complete buffered Frames are processed before another read after resume; +- Complete buffered Frames are processed before another read after resume, and a + paused Packet's remaining frames are delivered before any later Packet; incomplete input returns to bounded blocking receive. +- A header-only START/ONE_SHOT reaches `on_audio` with `len == 0` without an + admission check, so its server-side `timestamp_ms` can be latched for filtering. +- Interruption filtering is application-owned and time-based: `on_audio` compares + the server timestamp against the interruption time from the MQTT notice or from + ChatBreak's `user_data` (attr 111). The SDK exposes no receive state to mutate. - `pal_t.sleep_ms` is mandatory, so custom PALs and consumers must rebuild. diff --git a/modules/rtc-tcp-client/include/tuya_ai.h b/modules/rtc-tcp-client/include/tuya_ai.h index 4c61660..199e27a 100644 --- a/modules/rtc-tcp-client/include/tuya_ai.h +++ b/modules/rtc-tcp-client/include/tuya_ai.h @@ -165,14 +165,15 @@ typedef struct tai_ctx tai_ctx_t; /* --- Audio --------------------------------------------------------------- */ typedef struct tai_audio_msg { const uint8_t *data; /* Opus frame / PCM bytes; callback-lifetime */ - size_t len; + size_t len; /* may be 0 for a header-only START/ONE_SHOT */ uint8_t codec; /* TAI_AUDIO_OPUS / TAI_AUDIO_PCM / 0=unknown */ uint32_t sample_rate; /* Hz, 0 if unknown */ uint16_t frame_duration; /* ms per Opus frame */ uint8_t stream_flag; /* TAI_STREAM_* (from the media header) */ uint16_t data_id; /* Data ID: AUDIO_DOWN(2) / AUDIO_AUX(7) */ const char *event_id; /* turn id, borrowed; "" if none */ - uint64_t timestamp_ms; /* stream-start ts (media header) */ + uint64_t timestamp_ms; /* server media-header timestamp, not local time; + * latch START's value for stream filtering */ uint8_t _reserved[8]; } tai_audio_msg_t; @@ -212,6 +213,8 @@ typedef struct tai_event_msg { const uint8_t *data; /* event payload (often JSON); callback-life */ size_t len; const char *event_id; /* attr 61, borrowed; "" if absent */ + const uint8_t *user_data; /* attr 111, borrowed, NOT NUL-terminated; NULL if absent */ + size_t user_data_len; /* separate from event payload; SDK does not parse JSON */ uint8_t _reserved[8]; } tai_event_msg_t; @@ -329,8 +332,10 @@ typedef struct tai_config { void *user_data; /* Optional TCP receive backpressure. Called on the worker thread before - * each receive and between complete buffered Frames, but never during the - * synchronous connect handshake. Return nonzero to admit another Frame or + * each receive and between complete buffered Frames. A conforming server + * never consults it during the synchronous connect handshake (the ack + * precedes media); only a server coalescing media ahead of the ack could. + * Return nonzero to admit another Frame or * read; return 0 to pause parsing and skip the read, allowing the TCP receive * window to close. This pauses all inbound traffic, including ChatBreak, * text, Pong, and EOF detection. Receive-liveness timeout accounting is @@ -338,9 +343,11 @@ typedef struct tai_config { * ping_timeout_ms budget. Pings and requested shutdown remain active, and a * Ping send failure still disconnects. The hook must return promptly. * - * Admission is at wire Frame boundaries. One Audio Packet may emit several - * codec-frame callbacks, so applications must also bound their queue inside - * on_audio. NULL means receive continuously. */ + * Admission is also checked before each codec-frame callback within an Audio + * Packet. A pause retains its remaining bytes without copying; resuming + * delivers them before any later Packet. Applications filter obsolete audio + * in on_audio using their own synchronized interruption state, not by mutating + * SDK receive state. NULL means receive continuously. */ int (*on_flow_control)(tai_ctx_t *ctx, void *user_data); } tai_config_t; diff --git a/modules/rtc-tcp-client/src/tai_client.c b/modules/rtc-tcp-client/src/tai_client.c index 5a07b1a..4d9e216 100644 --- a/modules/rtc-tcp-client/src/tai_client.c +++ b/modules/rtc-tcp-client/src/tai_client.c @@ -640,12 +640,9 @@ void tai_disconnect(tai_ctx_t *ctx) ctx->rx_len = 0; ctx->frag_len = 0; ctx->frag_state = 0; - ctx->rx_pending_valid = 0; /* teardown drops a paused Packet remainder */ - ctx->rx_pending_discard = 0; - ctx->rx_pending_offset = 0; - ctx->rx_pending_len = 0; - ctx->rx_pending_body = NULL; - ctx->rx_pending_event_id[0] = '\0'; + ctx->rx_pending_len = 0; /* teardown drops a paused Packet remainder */ + ctx->rx_pending_wire_len = 0; + ctx->rx_pending_body = NULL; ctx->connecting = 0; /* clear in case a connect aborted mid-handshake */ ctx->disconnect_emitted = 0; /* re-arm the single-point on_disconnect */ ctx->rx_event_id[0] = '\0'; /* clear latched turn id so a reconnect starts clean */ @@ -735,9 +732,10 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) * the connection down (the app reconnects). */ while (ctx->rx_len >= 5) { /* One receive may contain several complete Frames. Stop before the next - * one when the consumer fills while dispatching the previous Frame. */ - if (paused && ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data)) { + * one when the consumer fills while dispatching the previous Frame. + * The leading paused gate (not the helper) is what keeps this + * checkpoint dark during the connect handshake (paused == NULL). */ + if (paused && tai_rx_admission_paused(ctx)) { *paused = 1; break; } @@ -747,36 +745,25 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) * its explicit pending remainder instead. Once it drains, slide the * pinned Frame out from rx_buf so the next loop starts at the Frame * behind it. */ - if (ctx->rx_pending_valid) { - ctx->rx_pending_valid = tai_proto_drain_pending_audio( - ctx, ctx->rx_pending_from_frag); - if (ctx->rx_pending_valid) { + if (ctx->rx_pending_len) { + if (tai_proto_drain_pending_audio(ctx)) { if (paused) *paused = 1; break; } - if (ctx->rx_pending_body_off && !ctx->rx_pending_from_frag) { - /* body_off−1 = byte offset of the body's first byte; +body_len - * reaches the end of the app payload; +sig_len reaches the end - * of the wire Frame. Verify against the Frame's own length - * field: they must agree, so a mismatch means the pending - * metadata was captured for the wrong storage. */ - size_t pending_end = ctx->rx_pending_body_off - 1 + - ctx->rx_pending_len + ctx->sig_len; - size_t wire_len = tai_frame_total_size(ctx->rx_buf, ctx->rx_len); - if (pending_end != wire_len) { - TAI_LOGE(ctx->pal, TAG, - "pending audio slide mismatch: end=%zu wire=%zu", - pending_end, wire_len); - return TAI_PROTO_ERR_FRAME_DECODE; - } - if (pending_end <= ctx->rx_len) { - memmove(ctx->rx_buf, ctx->rx_buf + pending_end, - ctx->rx_len - pending_end); - ctx->rx_len -= pending_end; - } - ctx->rx_pending_body = NULL; - ctx->rx_pending_body_off = 0; + /* The pinned Frame must be consumed exactly once. A zero or + * oversized length would underflow rx_len (a huge memmove) or + * leave the Frame in place to be dispatched again, so fail fast + * instead of silently desyncing the stream. */ + size_t wire_len = ctx->rx_pending_wire_len; + if (wire_len < 5 || wire_len > ctx->rx_len) { + TAI_LOGE(ctx->pal, TAG, + "pending slide out of range: wire=%zu rx=%zu", + wire_len, ctx->rx_len); + return TAI_PROTO_ERR_FRAME_DECODE; } + memmove(ctx->rx_buf, ctx->rx_buf + wire_len, ctx->rx_len - wire_len); + ctx->rx_len -= wire_len; + ctx->rx_pending_wire_len = 0; continue; } @@ -865,9 +852,7 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) } if (complete && app_bytes && app_len > 0) { - ctx->rx_dispatch_from_frag = (frag_flag == TAI_FRAG_LAST); int fatal = process_app_packet(ctx, app_bytes, app_len); - ctx->rx_dispatch_from_frag = 0; if (fatal != TAI_OK) return fatal; /* PROTOCOL detail or TAI_RX_PEER_CLOSE|code */ } @@ -876,7 +861,8 @@ static int tai_process_rx(tai_ctx_t *ctx, int *paused) * therefore remains valid until the remainder drains. Once it drains, * the pending-drain path above already slid the Frame, so this code * runs only for the no-pending fast path. */ - if (ctx->rx_pending_valid) { + if (ctx->rx_pending_len) { + ctx->rx_pending_wire_len = needed; if (paused) *paused = 1; break; } @@ -1299,6 +1285,13 @@ int tai_send_mcp_response(tai_ctx_t *ctx, const char *json_rpc_response) * Drives recv + dispatch in a loop, sends periodic pings, detects pong * timeouts. Auto-started by tai_connect(), stopped by tai_disconnect(). * + * Receive backpressure: admission (tai_rx_admission_paused(), i.e. the + * on_flow_control hook) is queried at four checkpoints — pass top, before + * each drain read, between buffered Frames (tai_process_rx), and before each + * codec-frame callback (tai_protocol.c). See tuya_ai.h on_flow_control and + * ADR 0001 for the contract; the paused branches below each consume one + * facet of that state. + * * Locking: the mbedTLS read/write mutex now lives inside the shared TLS module * (tls_write / tls_read, granularity = a single ssl_* call), so the worker no * longer needs to wrap recv+dispatch in ctx_lock. rx_buf / frag_buf are touched @@ -1324,8 +1317,7 @@ static void *worker_thread(void *arg) while (ctx->running) { int was_paused = paused; - paused = ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data); + paused = tai_rx_admission_paused(ctx); if (!ctx->running) break; uint64_t now = ctx->pal->time_ms(); @@ -1390,7 +1382,7 @@ static void *worker_thread(void *arg) * new receive; a pending rx_buf remainder also keeps the Frame pinned, * making rx_len a valid resume marker. Partial input returns to a * bounded blocking receive rather than spinning. */ - int n = ctx->rx_pending_valid ? 1 + int n = ctx->rx_pending_len ? 1 : resume_buffered ? (int)ctx->rx_len : tai_recv_data(ctx, wait_ms); resume_buffered = 0; @@ -1423,8 +1415,7 @@ static void *worker_thread(void *arg) break; ctx->pal->sleep_ms(AGENTIC_KIT_TAI_WORKER_YIELD_MS); /* Admission is checked immediately before the next drain read. */ - paused = ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data); + paused = tai_rx_admission_paused(ctx); if (paused || !ctx->running) break; n = tai_recv_data(ctx, 0); /* drain remainder non-blocking */ } diff --git a/modules/rtc-tcp-client/src/tai_internal.h b/modules/rtc-tcp-client/src/tai_internal.h index b5cab43..062a90d 100644 --- a/modules/rtc-tcp-client/src/tai_internal.h +++ b/modules/rtc-tcp-client/src/tai_internal.h @@ -215,9 +215,6 @@ struct tai_ctx { uint16_t rx_audio_frame_duration; /* ms */ uint16_t rx_audio_frame_size; /* bytes per Opus packet */ uint8_t rx_audio_codec; /* TAI_AUDIO_* from audio-params f[0]; 0=unknown */ - uint8_t rx_pending_audio_codec; - uint32_t rx_pending_audio_rate; - uint16_t rx_pending_frame_duration; /* Received turn id (attr 61) backing callback msg->event_id; "" if none */ char rx_event_id[64]; @@ -248,22 +245,15 @@ struct tai_ctx { /* Optional TCP receive backpressure (see tuya_ai.h). */ int (*on_flow_control)(tai_ctx_t *, void *); - /* A codec-frame Packet paused inside dispatch. The bytes and Event id stay - * worker-owned; pending_offset advances only after an admitted callback. - * Teardown deliberately discards any remainder. */ - uint8_t rx_pending_valid; + /* Worker-owned audio cursor: nonzero remaining length means pending. + * No other Packet dispatches until it drains, so rx_audio_* / rx_event_id + * stay valid. Pin the current wire Frame even for reassembled Packets. */ + const uint8_t *rx_pending_body; + size_t rx_pending_len; + size_t rx_pending_wire_len; uint8_t rx_pending_flag; - uint8_t rx_pending_discard; - uint8_t rx_pending_from_frag; - uint8_t rx_dispatch_from_frag; uint16_t rx_pending_data_id; - uint32_t rx_pending_frame_ms; uint64_t rx_pending_ts_ms; - size_t rx_pending_offset; - size_t rx_pending_len; - size_t rx_pending_body_off; - const uint8_t *rx_pending_body; - char rx_pending_event_id[64]; /* RX linear buffer (sliding-window: bytes always at buf[0]) */ uint8_t rx_buf[TAI_RX_BUF_SIZE]; @@ -404,10 +394,6 @@ int tai_unpack_media_hdr(const uint8_t *buf, size_t len, void tai_emit_disconnect(tai_ctx_t *ctx, uint8_t reason, uint8_t detail, uint16_t close_code); -/* Internal storage-origin hint for a dispatched Packet: 0 = rx_buf, 1 = frag_buf. - * Only tai_process_rx sets it around process_app_packet; it is not public API. */ -#define TAI_DISPATCH_STORAGE_FRAG 1 - /* Fail-fast cause, RETURNED up the receive chain (dispatch -> process_app_packet * -> tai_process_rx) to the worker, which owns the run loop and fires a single * on_disconnect. Lower layers never touch connection-lifecycle state: @@ -528,14 +514,23 @@ int tai_proto_dispatch (tai_ctx_t *ctx, uint8_t pkt_type, const tai_attr_t *attrs, int attr_count, const uint8_t *payload, size_t payload_len); -/* Drain one audio Packet paused inside callback delivery. Returns 1 while - * delivery remains paused, 0 once the Packet (or a deliberately discarded - * remainder) is finished. Callers only use it for a packet whose bytes are - * still pinned in the worker's rx_buf / frag_buf. */ -int tai_proto_drain_pending_audio(tai_ctx_t *ctx, int from_frag); -/* Mark the current pending remainder obsolete without delivering it. Used by - * an application interruption from another thread before reopening admission. */ -void tai_discard_pending_audio(tai_ctx_t *ctx); +/* Worker-only: returns 1 while delivery remains paused, 0 once finished. + * The Packet's bytes stay pinned in rx_buf / frag_buf until it drains. */ +int tai_proto_drain_pending_audio(tai_ctx_t *ctx); + +/* Admission query — the single judgment behind receive backpressure, shared + * by the four checkpoints (worker pass top, before each drain read, between + * buffered Frames, before each codec-frame callback). Nonzero = this + * checkpoint must pause; 0 = admit. A NULL hook never pauses and is never + * invoked. Raw query only: the paused-pointer gate that disables flow + * control during the synchronous connect handshake stays at the + * tai_process_rx call site. Runs on the receive path — the hook must return + * promptly and is called at most once per check. */ +static inline int tai_rx_admission_paused(tai_ctx_t *ctx) +{ + return ctx->on_flow_control && + !ctx->on_flow_control(ctx, ctx->user_data); +} /* Internal sequence helper */ static inline uint16_t tai_next_seq(tai_ctx_t *ctx) { diff --git a/modules/rtc-tcp-client/src/tai_protocol.c b/modules/rtc-tcp-client/src/tai_protocol.c index df2909e..960929c 100644 --- a/modules/rtc-tcp-client/src/tai_protocol.c +++ b/modules/rtc-tcp-client/src/tai_protocol.c @@ -522,7 +522,8 @@ static void emit_text(tai_ctx_t *ctx, const char *text, size_t len, } static void emit_event(tai_ctx_t *ctx, uint16_t event_type, - const uint8_t *data, size_t len) + const uint8_t *data, size_t len, + const tai_attr_t *user_data) { if (!ctx->on_event) return; tai_event_msg_t m = {0}; @@ -530,6 +531,10 @@ static void emit_event(tai_ctx_t *ctx, uint16_t event_type, m.data = data; m.len = len; m.event_id = ctx->rx_event_id; + if (user_data) { + m.user_data = user_data->value; + m.user_data_len = user_data->len; + } ctx->on_event(ctx, &m, ctx->user_data); } @@ -604,115 +609,29 @@ static void parse_audio_params_once(tai_ctx_t *ctx, } } -/* Split an audio body into CBR Opus frames of rx_audio_frame_size, emitting a - * final short remainder. When flow control closes mid-body, metadata plus the - * first unadmitted offset are retained and the caller pins this Packet's wire - * storage. Reopening admission resumes from that offset; admitted bytes are - * never replayed. fs==0 (PCM / unknown) delivers the body whole once admitted, - * so an impossible-to-admit callback cannot spin the worker. */ -static void media_audio_body(tai_ctx_t *ctx, const uint8_t *body, size_t body_len, - uint8_t stream_flag, uint16_t data_id, - uint64_t ts_ms, int from_frag) +/* Split CBR Opus into codec frames, including a final short remainder. + * A zero frame size (PCM / unknown) delivers the remaining body whole. + * Only the worker advances this zero-copy cursor; the caller pins storage + * and does not dispatch another Packet until the cursor is exhausted. */ +int tai_proto_drain_pending_audio(tai_ctx_t *ctx) { - if (!ctx->on_audio) return; - uint16_t fs = ctx->rx_audio_frame_size; - size_t offset = 0; - while (offset < body_len) { - if (ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data)) { - size_t n = strlen(ctx->rx_event_id); - if (n >= sizeof(ctx->rx_pending_event_id)) - n = sizeof(ctx->rx_pending_event_id) - 1; - - memcpy(ctx->rx_pending_event_id, ctx->rx_event_id, n); - ctx->rx_pending_event_id[n] = '\0'; - ctx->rx_pending_valid = 1; - ctx->rx_pending_flag = stream_flag; - ctx->rx_pending_discard = 0; - ctx->rx_pending_data_id = data_id; - ctx->rx_pending_frame_ms = fs; - ctx->rx_pending_audio_codec = ctx->rx_audio_codec; - ctx->rx_pending_audio_rate = ctx->rx_audio_sample_rate; - ctx->rx_pending_frame_duration = ctx->rx_audio_frame_duration; - ctx->rx_pending_ts_ms = ts_ms; - ctx->rx_pending_offset = offset; - ctx->rx_pending_len = body_len; - ctx->rx_pending_body = body; - ctx->rx_pending_body_off = from_frag - ? 0 : (size_t)(body - ctx->rx_buf) + 1; - ctx->rx_pending_from_frag = (uint8_t)from_frag; - fprintf(stderr, "CAP off=%zu len=%zu event=%s\n", - offset, body_len, ctx->rx_pending_event_id); - return; - } - size_t len = fs == 0 || body_len - offset < fs ? body_len - offset : fs; - emit_audio(ctx, body + offset, len, stream_flag, data_id, ts_ms); - offset += len; - } -} - -int tai_proto_drain_pending_audio(tai_ctx_t *ctx, int from_frag) -{ - if (!ctx->rx_pending_valid) return 0; - if ((from_frag != 0) != (ctx->rx_pending_from_frag != 0)) { - TAI_LOGE(ctx->pal, TAG, "pending audio storage mismatch"); - return 1; /* keep the connection alive rather than resyncing blindly */ - } - - /* The caller pins the storage containing this Packet: the unslid rx_buf - * Frame for FRAG_NONE, or frag_buf until its next reassembly begins. */ - const uint8_t *body = ctx->rx_pending_body; - if (!body) return 1; - if (ctx->rx_pending_offset > ctx->rx_pending_len) return 0; - - while (ctx->rx_pending_offset < ctx->rx_pending_len) { - if (ctx->rx_pending_discard) { - ctx->rx_pending_valid = 0; - ctx->rx_pending_discard = 0; - return 0; - } - if (ctx->on_flow_control && - !ctx->on_flow_control(ctx, ctx->user_data)) { + while (ctx->rx_pending_len) { + if (tai_rx_admission_paused(ctx)) { return 1; } - - size_t remaining = ctx->rx_pending_len - ctx->rx_pending_offset; - size_t len = ctx->rx_pending_frame_ms == 0 || - remaining < ctx->rx_pending_frame_ms - ? remaining : ctx->rx_pending_frame_ms; - char saved_event[sizeof(ctx->rx_event_id)]; - uint8_t saved_codec = ctx->rx_audio_codec; - uint32_t saved_rate = ctx->rx_audio_sample_rate; - uint16_t saved_duration = ctx->rx_audio_frame_duration; - memcpy(saved_event, ctx->rx_event_id, sizeof(saved_event)); - memcpy(ctx->rx_event_id, ctx->rx_pending_event_id, - sizeof(ctx->rx_event_id)); - ctx->rx_audio_codec = ctx->rx_pending_audio_codec; - ctx->rx_audio_sample_rate = ctx->rx_pending_audio_rate; - ctx->rx_audio_frame_duration = ctx->rx_pending_frame_duration; - - emit_audio(ctx, body + ctx->rx_pending_offset, len, + size_t len = fs == 0 || ctx->rx_pending_len < fs + ? ctx->rx_pending_len : fs; + emit_audio(ctx, ctx->rx_pending_body, len, ctx->rx_pending_flag, ctx->rx_pending_data_id, ctx->rx_pending_ts_ms); - memcpy(ctx->rx_event_id, saved_event, sizeof(saved_event)); - ctx->rx_audio_codec = saved_codec; - ctx->rx_audio_sample_rate = saved_rate; - ctx->rx_audio_frame_duration = saved_duration; - ctx->rx_pending_offset += len; + ctx->rx_pending_body += len; + ctx->rx_pending_len -= len; } - - ctx->rx_pending_valid = 0; ctx->rx_pending_body = NULL; return 0; } -void tai_discard_pending_audio(tai_ctx_t *ctx) -{ - if (!ctx) return; - if (ctx->rx_pending_valid) ctx->rx_pending_discard = 1; -} - /* AUDIO packet payload: [data_id:2][48-bit stream_flag|ts_ms][opus frames…]. */ static int media_audio(tai_ctx_t *ctx, const tai_attr_t *attrs, int attr_count, @@ -739,8 +658,19 @@ static int media_audio(tai_ctx_t *ctx, parse_audio_params_once(ctx, attrs, attr_count); latch_event_id(ctx, attrs, attr_count); - media_audio_body(ctx, payload + 8, payload_len - 8, stream_flag, data_id, - ts_ms, ctx->rx_dispatch_from_frag); + /* Header-only STARTs still establish the server-time playback boundary. */ + if (payload_len == 8 && + (stream_flag == TAI_STREAM_START || stream_flag == TAI_STREAM_ONE_SHOT)) { + emit_audio(ctx, payload + 8, 0, stream_flag, data_id, ts_ms); + } + if (ctx->on_audio) { + ctx->rx_pending_body = payload + 8; + ctx->rx_pending_len = payload_len - 8; + ctx->rx_pending_flag = stream_flag; + ctx->rx_pending_data_id = data_id; + ctx->rx_pending_ts_ms = ts_ms; + tai_proto_drain_pending_audio(ctx); + } return TAI_OK; } @@ -852,7 +782,8 @@ int tai_proto_dispatch(tai_ctx_t *ctx, } latch_event_id(ctx, attrs, attr_count); - emit_event(ctx, evt_type, evt_data, evt_data_len); + emit_event(ctx, evt_type, evt_data, evt_data_len, + tai_attr_find(attrs, attr_count, TAI_ATTR_USER_DATA)); if (evt_type == TAI_EVT_END) ctx->rx_event_id[0] = '\0'; /* turn over: clear after firing END */ break; diff --git a/modules/rtc-tcp-client/test/tai_pal_loopback.c b/modules/rtc-tcp-client/test/tai_pal_loopback.c index a4a398f..59cb3cb 100644 --- a/modules/rtc-tcp-client/test/tai_pal_loopback.c +++ b/modules/rtc-tcp-client/test/tai_pal_loopback.c @@ -14,10 +14,6 @@ #include "log.h" #include "../src/tai_internal.h" /* frame/packet codec + key derivation for the handshake mock */ -extern const char *test_reserved_audio_event_id; -extern uint8_t test_reserved_audio_body[40]; -extern size_t test_reserved_audio_body_len; - /* ========================================================================= * Byte FIFO: single-producer, single-consumer, mutex-protected. * ========================================================================= */ diff --git a/modules/rtc-tcp-client/test/test_integration.c b/modules/rtc-tcp-client/test/test_integration.c index d8157e2..3cf0844 100644 --- a/modules/rtc-tcp-client/test/test_integration.c +++ b/modules/rtc-tcp-client/test/test_integration.c @@ -96,6 +96,10 @@ typedef struct { int event_count; uint8_t event_payload[4096]; /* last event's data (reassembly check) */ size_t event_payload_len; + uint8_t event_user_data[256]; + size_t event_user_data_len; + int event_user_data_present; + int event_user_data_borrowed; int disconnect_calls; uint16_t disconnect_code; @@ -133,6 +137,9 @@ static void st_reset(void) g_st.event_calls = 0; g_st.event_count = 0; g_st.event_payload_len = 0; + g_st.event_user_data_len = 0; + g_st.event_user_data_present = 0; + g_st.event_user_data_borrowed = 0; g_st.disconnect_calls = 0; g_st.disconnect_code = 0; g_st.disconnect_reason = 0xFF; @@ -202,9 +209,19 @@ static void on_event(tai_ctx_t *ctx, const tai_event_msg_t *msg, void *ud) g_st.event_calls++; if (g_st.event_count < (int)(sizeof(g_st.event_types)/sizeof(g_st.event_types[0]))) g_st.event_types[g_st.event_count++] = msg->event_type; - if (msg->data && msg->len <= sizeof(g_st.event_payload)) { + g_st.event_payload_len = msg->len; + if (msg->data && msg->len <= sizeof(g_st.event_payload)) memcpy(g_st.event_payload, msg->data, msg->len); - g_st.event_payload_len = msg->len; + g_st.event_user_data_len = msg->user_data_len; + g_st.event_user_data_present = msg->user_data != NULL; + if (msg->user_data && msg->user_data_len <= sizeof(g_st.event_user_data)) { + memcpy(g_st.event_user_data, msg->user_data, msg->user_data_len); + uintptr_t p = (uintptr_t)msg->user_data; + g_st.event_user_data_borrowed = + (p >= (uintptr_t)ctx->rx_buf && + p + msg->user_data_len <= (uintptr_t)ctx->rx_buf + sizeof(ctx->rx_buf)) || + (p >= (uintptr_t)ctx->frag_buf && + p + msg->user_data_len <= (uintptr_t)ctx->frag_buf + sizeof(ctx->frag_buf)); } if (msg->event_id) { size_t n = strlen(msg->event_id); @@ -1008,6 +1025,52 @@ static void test_request_disconnect_from_callback(void) * turn's packets that carry no attr 61 (inherited, not reset to ""), and is * cleared after EVT_END. Distinguishes §4 from §1's reset-if-absent behaviour. * ========================================================================= */ +static void test_event_user_data(void) +{ + SECTION("event_user_data"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + tai_ctx_t *ctx = setup_ctx(ctx_mem); + CHECK(ctx != NULL); + if (!ctx) return; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + const uint8_t user_data[] = "{\"serverTime\":123456789}"; + const uint8_t payload[] = { 'p', 0, 'x' }; + tai_attr_t attr = tai_attr_bytesv(TAI_ATTR_USER_DATA, user_data, sizeof(user_data) - 1); + uint16_t seq = 100; + for (int round = 0; round < 4; round++) { + uint8_t evt[64], app[256]; + size_t len = round == 3 ? 0 : sizeof(payload); + int elen = tai_pack_event(TAI_VER_21, TAI_EVT_CHAT_BREAK, + payload, len, evt, sizeof(evt)); + CHECK(elen > 0); + if (round == 1) { + int alen = tai_packet_encode(TAI_VER_21, TAI_PKT_EVENT, &attr, 1, + evt, (size_t)elen, app, sizeof(app)); + CHECK(alen > 10); + if (alen <= 10) break; + size_t cuts[] = { 10, (size_t)alen }; + server_send_app_fragmented(ctx, app, (size_t)alen, cuts, 2, &seq); + } else { + CHECK(server_send(ctx, TAI_PKT_EVENT, round == 2 ? NULL : &attr, + round == 2 ? 0 : 1, evt, (size_t)elen, seq++) > 0); + } + CHECK(WAIT_FOR(g_st.event_calls == round + 1, 1000)); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.event_payload_len, len); + CHECK(memcmp(g_st.event_payload, payload, len) == 0); + CHECK_EQ_INT(g_st.event_user_data_present, round != 2); + CHECK_EQ_INT(g_st.event_user_data_len, round == 2 ? 0 : sizeof(user_data) - 1); + if (round != 2) { + CHECK(g_st.event_user_data_borrowed); + CHECK(memcmp(g_st.event_user_data, user_data, sizeof(user_data) - 1) == 0); + } + pthread_mutex_unlock(&g_st.mtx); + } + tai_disconnect(ctx); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); +} + static void test_event_id_latch(void) { SECTION("event_id_latch"); @@ -2222,6 +2285,184 @@ static void test_receive_backpressure(void) tai_ctx_deinit(ctx); } +static size_t pending_remaining, pending_wire_len; +static uintptr_t pending_cursor; +static int pending_fragmented, pending_metadata_ok, pending_audio_ok; + +static int pending_flow_control(tai_ctx_t *ctx, void *user_data) +{ + int ready = test_flow_control(ctx, user_data); + pthread_mutex_lock(&g_st.mtx); + if (!ready && ctx->rx_pending_wire_len) { + pending_remaining = ctx->rx_pending_len; + pending_wire_len = ctx->rx_pending_wire_len; + pending_cursor = (uintptr_t)ctx->rx_pending_body; + pending_metadata_ok = ctx->rx_audio_codec == TAI_AUDIO_OPUS && + ctx->rx_audio_sample_rate == 16000 && ctx->rx_audio_frame_duration == 20 && + ctx->rx_audio_frame_size == 40 && strcmp(ctx->rx_event_id, "pending-event") == 0; + } + pthread_mutex_unlock(&g_st.mtx); + return ready; +} + +static void pending_audio(tai_ctx_t *ctx, const tai_audio_msg_t *msg, void *ud) +{ + const uint8_t *storage = pending_fragmented ? ctx->frag_buf : ctx->rx_buf; + uintptr_t p = (uintptr_t)msg->data; + size_t size = pending_fragmented ? sizeof(ctx->frag_buf) : sizeof(ctx->rx_buf); + pthread_mutex_lock(&g_st.mtx); + pending_audio_ok &= p >= (uintptr_t)storage && p + msg->len <= (uintptr_t)storage + size && + msg->codec == TAI_AUDIO_OPUS && msg->sample_rate == 16000 && + msg->frame_duration == 20 && msg->stream_flag == TAI_STREAM_START && + msg->data_id == TAI_DATA_ID_AUDIO_DOWN && msg->timestamp_ms == 123456789 && + strcmp(msg->event_id, "pending-event") == 0; + pthread_mutex_unlock(&g_st.mtx); + on_audio(ctx, msg, ud); +} + +/* A pinned Frame whose recorded wire length cannot be trusted must fail fast: + * an oversized value would underflow rx_len into a huge memmove, and a zero one + * would leave the Frame in place to be dispatched a second time. */ +static void test_pending_bad_wire_len(void) +{ + SECTION("pending_audio_bad_wire_len"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + pending_remaining = pending_wire_len = pending_cursor = 0; + pending_fragmented = 0; + pending_metadata_ok = 0; + pending_audio_ok = 1; + tai_config_t options = { .pal = observe_pal(), + .on_flow_control = pending_flow_control }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + ctx->on_audio = pending_audio; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + + uint8_t payload[8 + 130], app[256]; + CHECK_EQ_INT(tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, 123456789, payload, sizeof(payload)), 8); + for (size_t i = 8; i < sizeof(payload); i++) payload[i] = (uint8_t)i; + tai_attr_t attrs[] = { + tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, "111 1 16 16000 0 16000 20 40"), + tai_attr_strv(TAI_ATTR_EVENT_ID, "pending-event"), + }; + int alen = tai_packet_encode(TAI_VER_21, TAI_PKT_AUDIO, attrs, 2, + payload, sizeof(payload), app, sizeof(app)); + CHECK(alen > 100); + if (alen <= 100) { tai_disconnect(ctx); tai_ctx_deinit(ctx); return; } + + /* Admit one codec frame, then close admission: the Packet pauses mid-body. */ + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = 1; + pthread_mutex_unlock(&g_st.mtx); + uint16_t seq = 200; + CHECK(server_send(ctx, TAI_PKT_AUDIO, attrs, 2, payload, sizeof(payload), + seq++) > 0); + CHECK(WAIT_FOR(pending_wire_len > 0, 1000)); + + /* Corrupt the recorded wire length, then let delivery run to completion. */ + pthread_mutex_lock(&g_st.mtx); + ctx->rx_pending_wire_len = ctx->rx_len + 1; + flow_allowed_calls = 100; + pthread_mutex_unlock(&g_st.mtx); + + CHECK(WAIT_FOR(g_st.disconnect_calls >= 1, 1000)); + CHECK_EQ_INT(g_st.disconnect_reason, TAI_DISCONNECT_PROTOCOL); + CHECK_EQ_INT(g_st.disconnect_detail, TAI_PROTO_ERR_FRAME_DECODE); + tai_disconnect(ctx); + tai_ctx_deinit(ctx); +} + +static void test_pending_audio(void) +{ + for (int mode = 0; mode < 3; mode++) { + SECTION(mode == 0 ? "pending_audio_multiple_pauses" : + mode == 1 ? "pending_audio_fragmented" : "pending_audio_shutdown"); + static uint8_t ctx_mem[sizeof(struct tai_ctx)]; + flow_allowed_calls = 0; + flow_paused_queries = 0; + flow_resume_ms = 0; + pending_remaining = pending_wire_len = pending_cursor = 0; + pending_fragmented = mode != 0; + pending_metadata_ok = 0; + pending_audio_ok = 1; + tai_config_t options = { .pal = observe_pal(), .on_flow_control = pending_flow_control }; + tai_ctx_t *ctx = setup_ctx_config(ctx_mem, &options); + CHECK(ctx != NULL); + if (!ctx) return; + ctx->on_audio = pending_audio; + CHECK_EQ_INT(tai_connect(ctx), TAI_OK); + CHECK(WAIT_FOR(flow_paused_queries > 0, 1000)); + uint8_t payload[8 + 130], app[256]; + CHECK_EQ_INT(tai_pack_media_hdr(TAI_VER_21, TAI_DATA_ID_AUDIO_DOWN, + TAI_STREAM_START, 123456789, payload, sizeof(payload)), 8); + for (size_t i = 8; i < sizeof(payload); i++) payload[i] = (uint8_t)i; + tai_attr_t attrs[] = { + tai_attr_strv(TAI_ATTR_AUDIO_PARAMS, "111 1 16 16000 0 16000 20 40"), + tai_attr_strv(TAI_ATTR_EVENT_ID, "pending-event"), + }; + int alen = tai_packet_encode(TAI_VER_21, TAI_PKT_AUDIO, attrs, 2, + payload, sizeof(payload), app, sizeof(app)); + CHECK(alen > 100); + if (alen <= 100) { tai_disconnect(ctx); tai_ctx_deinit(ctx); return; } + uint16_t seq = 100; + size_t expected_wire_len; + if (pending_fragmented) { + size_t cuts[] = { 50, 100, (size_t)alen }; + server_send_app_fragmented(ctx, app, (size_t)alen, cuts, 3, &seq); + expected_wire_len = (size_t)alen - 100 + 37; + } else { + expected_wire_len = server_send(ctx, TAI_PKT_AUDIO, attrs, 2, + payload, sizeof(payload), seq++); + } + CHECK(server_send_event_id(ctx, TAI_EVT_END, "later-event", seq++) > 0); + uintptr_t first_cursor = 0; + for (int allowed = 1; allowed <= (mode == 2 ? 1 : 3); allowed++) { + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = allowed; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(pending_remaining == 130 - 40 * (size_t)allowed, 1000)); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(g_st.audio_calls, allowed); + CHECK_EQ_INT(g_st.event_calls, 0); + CHECK_EQ_INT(pending_wire_len, expected_wire_len); + CHECK(pending_metadata_ok); + if (allowed == 1) first_cursor = pending_cursor; + CHECK_EQ_INT(pending_cursor - first_cursor, 40 * (allowed - 1)); + size_t reads = observed_recv_calls; + pthread_mutex_unlock(&g_st.mtx); + sleep_ms(50); + pthread_mutex_lock(&g_st.mtx); + CHECK_EQ_INT(observed_recv_calls, reads); + pthread_mutex_unlock(&g_st.mtx); + } + if (mode != 2) { + pthread_mutex_lock(&g_st.mtx); + flow_allowed_calls = 100; + pthread_mutex_unlock(&g_st.mtx); + CHECK(WAIT_FOR(g_st.event_calls == 1, 1000)); + } + tai_disconnect(ctx); + CHECK(pending_audio_ok); + CHECK_EQ_INT(g_st.audio_calls, mode == 2 ? 1 : 4); + CHECK_EQ_INT(g_st.audio_concat_len, mode == 2 ? 40 : 130); + CHECK(memcmp(g_st.audio_concat, payload + 8, g_st.audio_concat_len) == 0); + if (mode != 2) { + CHECK_EQ_INT(g_st.audio_frame_lens[3], 10); + CHECK(strcmp(g_st.last_event_event_id, "later-event") == 0); + } + CHECK_EQ_INT(ctx->rx_pending_len, 0); + CHECK_EQ_INT(ctx->rx_pending_wire_len, 0); + CHECK(ctx->rx_pending_body == NULL); + CHECK_EQ_INT(g_st.disconnect_calls, 0); + tai_ctx_deinit(ctx); + } +} + static int eof_released, eof_hook_phase, eof_buffered_b; static size_t eof_paused_queries, eof_frame_b_len; static uint8_t eof_frame_b[256]; @@ -2455,6 +2696,9 @@ int main(void) test_log_env_default(); pthread_mutex_init(&g_st.mtx, NULL); + test_event_user_data(); + test_pending_audio(); + test_pending_bad_wire_len(); test_receive_backpressure(); test_buffered_frame_before_eof(); test_partial_frame_blocking_recv();