From 75136c98b2d07544c5d0378ecfd06fdf2143d02b Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 13:55:11 +0800 Subject: [PATCH 1/6] DEV-1291 DEV-1275 HIL: inStream and connectReady scenarios inStream (DEV-1291): while streaming, start SD logging, stop it, and read the battery and the status every 40 ms for 400 ms. Android writes those reads straight out, so some land in the ~100 ms the unit pauses its data packets while it closes its SD file, and their replies come with no data packet before them. Then stop and start logging back to back, so the queued start goes out in the pause. It fails when: - a battery read goes unanswered, or fewer status replies than reads are decoded; - logging doesn't stop or restart; - a command times out; - there's a parser error. Loss near the reads is reported, not failed: a Shimmer3 skips a sample there now and then on its own. connectReady (DEV-1275): connect to the unit idle, then again while it logs. It fails when the unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before NOTIFICATION_SHIMMER_FULLY_INITIALIZED, which ends the connect sequence. The HIL's own connect waits for isInitialised(), which on Classic comes only at the end, so the existing scenarios never saw the early report. The runner runs both by default, and the README lists them. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 179 +++++++++++++++++- .../hiltest/ShimmerSession.java | 98 +++++++++- tools/hil/README.md | 6 +- tools/hil/run-android-hil.ps1 | 4 +- 4 files changed, 280 insertions(+), 7 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index 2cb2b9f6..63c8ea57 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -37,7 +37,7 @@ * type Shimmer3 (default) or Shimmer3R; over BLE it picks the device class * rate sampling rate in Hz, default 256 * seconds stream_rest duration, default 30 - * cycles mid_stream_set / reconnect_loop repetitions, default 3 / 5 + * cycles mid_stream_set / reconnect_loop / in_stream repetitions, default 3 / 5 / 3 * crc link CRC to ask for: off, 1 or 2 (default 2; the driver may settle on less) * maxLossPercent default 1.0 * accelTolerance m/s^2 either side of 9.81 at rest, default 1.0 @@ -332,6 +332,183 @@ public void midStreamSet() throws Exception { finish(); } + /** Reads sent after each SD logging stop in inStream, alternately battery and status */ + private static final int READS_PER_BURST = 10; + + /** Between those reads: the unit pauses for about 100 ms, so two or three land in the pause */ + private static final int READ_INTERVAL_MS = 40; + + /** What the driver logs when a command gets no reply in time: "Command: [0x95] GET_VBATT_COMMAND timeout" */ + private static final String[] COMMAND_TIMEOUTS = {"_COMMAND timeout", "UNKNOWN timeout"}; + + /** + * In-stream replies while the unit pauses its data packets (DEV-1291). After STOP_LOGGING_ONLY the + * unit sends no data packets for about 100 ms while it closes its SD file, and answers a command + * sent then at once, with no data packet before the reply. Android's Classic parser threw such a + * reply away. Each round starts SD logging, then stops it and reads the battery and the status + * every 40 ms for 400 ms: Android writes those reads straight out, so some land in the pause. Then + * each restart stops logging and starts it again back to back, so the queued START_LOGGING_ONLY goes + * out in the pause and its bare ACK comes first. Every battery read must be answered, logging must + * restart, and nothing may log a parser error or a command timeout. Needs an SD card in the unit. + */ + @Test + public void inStream() throws Exception { + begin("in_stream"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 3); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = session.rateOnDevice(); + checkRate(deviceRate, "stream"); + String marker = session.logMarker("in_stream"); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(2000); + List sdMoments = new ArrayList<>(); + List readMoments = new ArrayList<>(); + int batteryReads = 0; + int statusReads = 0; + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "round " + i; + session.event(what); + if (!startSdLoggingWhileStreaming(what, sdMoments)) { + break; + } + Thread.sleep(1000); + sdMoments.add(session.nowMs()); + report.check(session.stopSdLogging(), what + ": stopSDLogging did not complete (see the events)"); + for (int k = 0; k < READS_PER_BURST && session.hung == null; k++) { + readMoments.add(session.nowMs()); + if (k % 2 == 0) { + batteryReads += session.readBattery() ? 1 : 0; + } else { + statusReads += session.readStatus() ? 1 : 0; + } + Thread.sleep(READ_INTERVAL_MS); + } + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING && session.queuedInstructions() == 0, 5000), + what + ": SD logging did not stop (state " + session.state() + ", " + session.queuedInstructions() + " instructions queued)"); + Thread.sleep(1000); + } + if (session.hung == null && startSdLoggingWhileStreaming("restarts", sdMoments)) { + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "restart " + i; + session.event(what); + Thread.sleep(1000); + sdMoments.add(session.nowMs()); + report.check(session.stopSdLogging() && session.startSdLogging(), + what + ": stopSDLogging or startSDLogging did not complete (see the events)"); + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING && session.queuedInstructions() == 0, 5000), + what + ": logging did not restart after a stop and a start back to back (state " + session.state() + + ", " + session.queuedInstructions() + " instructions queued)"); + } + Thread.sleep(1000); + sdMoments.add(session.nowMs()); + report.check(session.stopSdLogging() && session.waitFor(() -> session.state() == BT_STATE.STREAMING, 5000), + "SD logging did not stop at the end (state " + session.state() + ")"); + } + Thread.sleep(1000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + + int batteryReplies = session.logLinesSince(marker, "IN STREAM: VBATT_RESPONSE"); + int statusReplies = session.logLinesSince(marker, "IN STREAM: STATUS_RESPONSE"); + int repliesInAPause = session.logLinesSince(marker, "with no data packet before it"); + int timeouts = session.logLinesSince(marker, COMMAND_TIMEOUTS); + int parserErrors = session.parserErrorsSince(marker); + report.put("batteryReads", batteryReads); + report.put("batteryReplies", batteryReplies); + report.put("statusReads", statusReads); + report.put("statusReplies", statusReplies); + report.put("repliesInAPause", repliesInAPause); + report.put("commandTimeouts", timeouts); + report.put("parserErrors", parserErrors); + session.event(String.format(Locale.ROOT, "battery %d reads, %d replies; status %d reads, %d replies (and pushes); %d with no data packet before them", + batteryReads, batteryReplies, statusReads, statusReplies, repliesInAPause)); + report.check(batteryReplies == batteryReads, batteryReads + " battery reads but " + batteryReplies + " replies decoded"); + report.check(statusReplies >= statusReads, statusReads + " status reads but " + statusReplies + " status replies decoded"); + report.check(timeouts == 0, timeouts + " command timeouts logged"); + report.check(parserErrors == 0, parserErrors + " streaming parser errors logged (\"Packet syncing problem\")"); + + // Loss near SD logging start/stop is the unit's own (see midStreamSet). A Shimmer3 also skips a + // sample now and then around a battery read, with no parser error, so loss near the reads is + // reported, not failed. + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + int[] attributed = a.attribute(new double[][] {toArray(sdMoments), toArray(readMoments)}, 1000); + report.put("missingNearSdLoggingStartStop", attributed[0]); + report.put("missingNearReads", attributed[1]); + int elsewhere = a.missing - attributed[0] - attributed[1]; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from SD logging start/stop and the reads (%d), limit %.2f%%", + elsewherePercent, elsewhere, maxLoss)); + report.check(a.received > 0, "stream: no packets received"); + report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, "stream"); + checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Starts SD logging while streaming and waits until it has started */ + private boolean startSdLoggingWhileStreaming(String what, List sdMoments) throws InterruptedException { + sdMoments.add(session.nowMs()); + boolean started = session.startSdLogging() + && session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING && session.queuedInstructions() == 0, 5000); + report.check(started, what + ": SD logging did not start while streaming (state " + session.state() + "; is there an SD card?)"); + return started; + } + + /** + * The unit is reported ready only once its connect sequence has ended (DEV-1275), which + * NOTIFICATION_SHIMMER_FULLY_INITIALIZED marks: CONNECTED, or SDLOGGING for a unit that is logging + * to its SD card. The sequence reads the status part way through, before the calibration dump and + * the inquiry, and that reply used to report the unit ready early, so an app's next command went + * out in among the rest of the sequence. Connects to the unit idle, then again while it logs. + * Needs an SD card in the unit. + */ + @Test + public void connectReady() throws Exception { + begin("connect_ready"); + if (connectAndCheckReady("idle", BT_STATE.CONNECTED)) { + boolean logging = session.startSdLogging() + && session.waitFor(() -> session.state() == BT_STATE.SDLOGGING && session.queuedInstructions() == 0, 5000); + report.check(logging, "SD logging did not start (state " + session.state() + "; is there an SD card?)"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + if (logging && session.hung == null) { + Thread.sleep(2000); + connectAndCheckReady("logging", BT_STATE.SDLOGGING); + // Leave the unit as it was found, whatever happened above + if (session.device() != null && session.device().isSDLogging()) { + report.check(session.stopSdLogging() && session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 5000), + "SD logging did not stop (state " + session.state() + ")"); + } + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + + /** Connects and checks the unit was first reported ready, in readyState, at the end of the connect sequence */ + private boolean connectAndCheckReady(String what, BT_STATE readyState) throws InterruptedException { + boolean ready = session.connect(30000, readyState) + && session.waitFor(() -> !Double.isNaN(session.fullyInitialisedAtMs()), 2000); + report.check(ready, what + ": did not connect, end in " + readyState + " and send FULLY_INITIALIZED within 30 s (state " + session.state() + ")"); + session.event(what + ": " + session.readyTimeline()); + report.put(what + "ReadyEarlyMs", session.readyEarlyMs()); + if (ready) { + report.check(session.readyEarlyMs() <= 250, String.format(Locale.ROOT, + "%s: reported ready %.0f ms before the connect sequence ended (%s)", what, session.readyEarlyMs(), session.readyTimeline())); + } + return ready; + } + private static double[] toArray(List values) { double[] out = new double[values.size()]; for (int i = 0; i < out.length; i++) { diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java index 47c0647d..2ac1770f 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -63,6 +63,18 @@ interface Condition { private volatile int stateMessages; private final java.util.Set otherMessagesSeen = new java.util.HashSet<>(); + /** + * Since the last connect: when the app was first told the unit is ready (a CONNECTED or + * SDLOGGING state message), when the API's state first showed it (polled), and when the API sent + * NOTIFICATION_SHIMMER_FULLY_INITIALIZED, which marks the end of the connect sequence. NaN until + * then. + */ + private volatile double readyMessageAtMs = Double.NaN; + private volatile double readyStateAtMs = Double.NaN; + private volatile double fullyInitialisedAtMs = Double.NaN; + private volatile BT_STATE readyMessageState; + private volatile int queuedAtReadyState = -1; + ShimmerSession(Context context, String mac, String transport, String deviceType) throws Exception { this.mac = mac; this.transport = transport; @@ -123,7 +135,17 @@ private void onMessage(Message msg) { lastState = state; stateMessages++; event("state " + state); + if (isReady(state) && Double.isNaN(readyMessageAtMs)) { + readyMessageAtMs = nowMs(); + readyMessageState = state; + } + } + } else if (msg.what == ShimmerBluetooth.MSG_IDENTIFIER_NOTIFICATION_MESSAGE && msg.obj instanceof CallbackObject + && ((CallbackObject) msg.obj).mIndicator == ShimmerBluetooth.NOTIFICATION_SHIMMER_FULLY_INITIALIZED) { + if (Double.isNaN(fullyInitialisedAtMs)) { + fullyInitialisedAtMs = nowMs(); } + event("fully initialised"); } else if (msg.what == ShimmerBluetooth.MSG_IDENTIFIER_PROGRESS_REPORT_PER_DEVICE && msg.obj instanceof BluetoothProgressReportPerDevice) { // The manager's own verdict on an operation such as configureShimmer. @@ -263,6 +285,20 @@ private void dumpThreads() { * configuration. Classic needs the unit paired with the phone; BLE does not. */ boolean connect(long timeoutMs) throws InterruptedException { + return connect(timeoutMs, BT_STATE.CONNECTED); + } + + /** + * As connect(long), for a unit that ends its connect in readyState: SDLOGGING for one that is + * logging to its SD card. It also waits for NOTIFICATION_SHIMMER_FULLY_INITIALIZED, and records + * when the unit was first reported ready (readyEarlyMs()). + */ + boolean connect(long timeoutMs, final BT_STATE readyState) throws InterruptedException { + readyMessageAtMs = Double.NaN; + readyStateAtMs = Double.NaN; + fullyInitialisedAtMs = Double.NaN; + readyMessageState = null; + queuedAtReadyState = -1; final boolean ble = "ble".equals(transport); if (!call("connect over " + transport, 10000, () -> manager.connectShimmerThroughBTAddress(mac, deviceType, ble ? ShimmerBluetoothManagerAndroid.BT_TYPE.BLE : ShimmerBluetoothManagerAndroid.BT_TYPE.BT_CLASSIC))) { @@ -270,7 +306,12 @@ boolean connect(long timeoutMs) throws InterruptedException { } boolean ready = waitFor(() -> { ShimmerBluetooth d = device(); - return d != null && d.isInitialised() && d.getBluetoothRadioState() == BT_STATE.CONNECTED; + if (d != null && isReady(d.getBluetoothRadioState()) && Double.isNaN(readyStateAtMs)) { + readyStateAtMs = nowMs(); + queuedAtReadyState = queuedInstructions(); + event("state first shows " + d.getBluetoothRadioState() + " with " + queuedAtReadyState + " instructions queued"); + } + return d != null && d.isInitialised() && d.getBluetoothRadioState() == readyState; }, timeoutMs); ShimmerBluetooth d = device(); if (ready && ble && d != null) { @@ -281,6 +322,50 @@ boolean connect(long timeoutMs) throws InterruptedException { return ready && d != null; } + private static boolean isReady(BT_STATE state) { + return state == BT_STATE.CONNECTED || state == BT_STATE.SDLOGGING; + } + + /** When the last connect sent NOTIFICATION_SHIMMER_FULLY_INITIALIZED, or NaN if it has not */ + double fullyInitialisedAtMs() { + return fullyInitialisedAtMs; + } + + /** + * How long before NOTIFICATION_SHIMMER_FULLY_INITIALIZED the last connect first reported the unit + * ready, by state message or in the API's state, whichever came first. NaN if either is missing. + */ + double readyEarlyMs() { + double first = Double.isNaN(readyMessageAtMs) ? readyStateAtMs + : Double.isNaN(readyStateAtMs) ? readyMessageAtMs : Math.min(readyMessageAtMs, readyStateAtMs); + return fullyInitialisedAtMs - first; + } + + /** What readyEarlyMs() is made of, for the report */ + String readyTimeline() { + return String.format(Locale.ROOT, "first ready message %s at %.0f ms, state first ready at %.0f ms with %d instructions queued, fully initialised at %.0f ms", + readyMessageState, readyMessageAtMs, readyStateAtMs, queuedAtReadyState, fullyInitialisedAtMs); + } + + /** + * The API's instruction queue length, or -1 without a device. The queue is protected, so an app + * can't see it; the test reads it by reflection. + */ + int queuedInstructions() { + ShimmerBluetooth d = device(); + if (d == null) { + return -1; + } + try { + java.lang.reflect.Method m = ShimmerBluetooth.class.getDeclaredMethod("getListofInstructions"); + m.setAccessible(true); + return ((List) m.invoke(d)).size(); + } catch (Exception e) { + event("could not read the instruction queue: " + e); + return -1; + } + } + /** * Accel only (the low-noise accel on Shimmer3, the LSM6DSV's on Shimmer3R) at rateHz, written * the way the examples do: a clone, AssembleShimmerConfig, then configureShimmer. @@ -363,6 +448,10 @@ boolean readBattery() { return call("readBattery", 10000, () -> requireDevice().readBattery()); } + boolean readStatus() { + return call("readStatusLogAndStream", 10000, () -> requireDevice().readStatusLogAndStream()); + } + /** * The API's current battery details. Both the Classic and the BLE parser replace the object with * a new one for each battery reply they decode, so a different object means a reply arrived. @@ -392,6 +481,11 @@ String logMarker(String label) { /** Parser errors this process logged after the marker. Apps may read their own process's log. */ int parserErrorsSince(String marker) { + return logLinesSince(marker, PARSER_ERRORS); + } + + /** Lines this process logged after the marker that contain any of the texts, or -1 if the log can't be read */ + int logLinesSince(String marker, String... texts) { int count = 0; try { java.lang.Process p = Runtime.getRuntime().exec(new String[] {"logcat", "-d", "--pid=" + Process.myPid()}); @@ -401,7 +495,7 @@ int parserErrorsSince(String marker) { if (line.contains(marker)) { afterMarker = true; } else if (afterMarker) { - for (String e : PARSER_ERRORS) { + for (String e : texts) { if (line.contains(e)) { count++; } diff --git a/tools/hil/README.md b/tools/hil/README.md index 9aa46d80..878b8d5b 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -55,6 +55,8 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas | `streamRest` | Connects, writes an accel-only config at `-Rate` (default 256 Hz), streams for `-Seconds` (default 30) with the unit lying still, then stops and disconnects. | It can't connect or configure; more than `-MaxLossPercent` (1 %) of packets are lost; a timestamp doesn't increase, or leaps more than a second ahead of when it arrived (a bad unwrap, not loss); the timestamps give no rate, or one more than 1 % off the configured rate; the stream goes silent for over a second before the test stops it; accel is more than 1 m/s² from 9.81; or the API logs a "Packet syncing problem". | | `midStreamSet` | While streaming, starts and stops SD logging and reads the battery, three times. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop (the DEV-1148 symptom with a link CRC on); a packet is lost around a battery read, or more than `-MaxLossPercent` away from the SD transitions; the timestamps fail the same rate and silence checks as `streamRest`; or there's a parser error. Packets missing around SD logging start/stop are reported, not failed: the unit itself pauses while it opens its file, which is for the firmware HIL bench to judge. Needs an SD card in the unit. | | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times. | Any cycle fails to connect within 30 s, loses packets, fails the rate and silence checks, or logs a parser error. | +| `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | +| `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | Every scenario also fails if an API call doesn't return within 10 s. The report then holds every thread's stack, which a phone without root can't otherwise give, and the scenario stops. If the whole test process still never finishes, the runner force-stops it after `-ScenarioTimeoutSeconds` (600) and records TIMEOUT. @@ -82,7 +84,7 @@ since Android uses those unchanged. | Java driver | the newest `API_v` release | `-DriverVersion 0.11.8_beta` | | Units | every unit in the rig file | `-Unit s3r-15bc` | | Transports | every transport each unit lists | `-Transport ble` | -| Scenarios | all three | `-Scenario midStreamSet` | +| Scenarios | all five | `-Scenario midStreamSet` | | Link CRC | ask for 2 bytes (the driver may settle on 1 or off) | `-Crc off`, `-Crc 1` | | Pause between runs | 10 s | `-SettleSeconds 60` | | Record the result for the release gate | no | `-Report` (full run on a clean, pushed commit only) | @@ -103,7 +105,7 @@ since Android uses those unchanged. 3. **Prepare each unit.** - **SD sync and Bluetooth disable off.** Either one keeps Bluetooth off (DEV-1228). Check in Consensys. - - **SD card in,** for `midStreamSet`. + - **SD card in,** for `midStreamSet`, `inStream` and `connectReady`. - **Lying still** during `streamRest`. 4. **List the units:** copy `rig-android.example.json` to `rig-android.json` and fill in each name, MAC, `type` (`Shimmer3` or `Shimmer3R`) and `transports` (`["classic"]`, `["ble"]` or both; diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index d060245c..22f4fd53 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -26,8 +26,8 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady'), [string[]]$Unit, [ValidateSet('classic', 'ble')] [string[]]$Transport, From a7f524db060d8791602fbda3d51aa3fe134c2d5a Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 14:10:10 +0800 Subject: [PATCH 2/6] DEV-1291 DEV-1275 HIL: read the instruction queue without the device's lock The IO thread takes the queue's lock and then the device's (DEV-1198). Polling the queue through the synchronized getListofInstructions() made the IO thread wait on the device's lock while it held the queue's, which widened that deadlock's window. Read the field directly instead. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hiltest/ShimmerSession.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java index 2ac1770f..0c3e8f92 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -349,7 +349,9 @@ String readyTimeline() { /** * The API's instruction queue length, or -1 without a device. The queue is protected, so an app - * can't see it; the test reads it by reflection. + * can't see it; the test reads the field by reflection. Not through getListofInstructions(), which + * takes the device's lock: the IO thread takes the queue's lock and then the device's + * (DEV-1198), so holding the device's lock from here would widen that deadlock's window. */ int queuedInstructions() { ShimmerBluetooth d = device(); @@ -357,9 +359,9 @@ int queuedInstructions() { return -1; } try { - java.lang.reflect.Method m = ShimmerBluetooth.class.getDeclaredMethod("getListofInstructions"); - m.setAccessible(true); - return ((List) m.invoke(d)).size(); + java.lang.reflect.Field f = ShimmerBluetooth.class.getDeclaredField("mListofInstructions"); + f.setAccessible(true); + return ((List) f.get(d)).size(); } catch (Exception e) { event("could not read the instruction queue: " + e); return -1; From 89d8bf03659da7a6219ca4f00947a941ea42a5ab Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 14:19:26 +0800 Subject: [PATCH 3/6] DEV-1291 DEV-1275 HIL: never leave a unit logging; dump threads on a stalled connect connectReady's second connect, to the unit while it logs, can fail. The unit was then left logging, and every later run failed to connect to it as idle. It now connects again to stop logging, and resetUnit does the same by hand. A connect that doesn't finish now dumps every thread, as a hung call does. The connect call itself returns at once, so nothing showed a stalled connect sequence, such as the DEV-1198 lock-order deadlock. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 37 ++++++++++++++++--- .../hiltest/ShimmerSession.java | 16 +++++--- tools/hil/README.md | 2 + tools/hil/run-android-hil.ps1 | 2 +- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index 63c8ea57..df83ed5e 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -483,18 +483,45 @@ public void connectReady() throws Exception { report.check(session.disconnect(10000), "did not disconnect within 10 s"); if (logging && session.hung == null) { Thread.sleep(2000); - connectAndCheckReady("logging", BT_STATE.SDLOGGING); - // Leave the unit as it was found, whatever happened above - if (session.device() != null && session.device().isSDLogging()) { - report.check(session.stopSdLogging() && session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 5000), - "SD logging did not stop (state " + session.state() + ")"); + if (!connectAndCheckReady("logging", BT_STATE.SDLOGGING) && session.hung == null) { + // Leave the unit as it was found, not logging: connect again to stop it + session.disconnect(10000); + Thread.sleep(2000); + session.connect(30000, BT_STATE.CONNECTED, BT_STATE.SDLOGGING); } + stopSdLoggingIfLogging(); report.check(session.disconnect(10000), "did not disconnect within 10 s"); } } finish(); } + private void stopSdLoggingIfLogging() throws InterruptedException { + ShimmerBluetooth d = session.device(); + if (session.hung == null && d != null && d.isSDLogging()) { + report.check(session.stopSdLogging() && session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 5000), + "SD logging did not stop, so the unit was left logging (state " + session.state() + ")"); + } + } + + /** + * Not a check: brings the unit back to idle after a run that left it logging to its SD card. It + * connects, stops SD logging if the unit logs, and disconnects. Run it on its own, with + * -Scenario resetUnit; it isn't part of a default run. + */ + @Test + public void resetUnit() throws Exception { + begin("reset_unit"); + boolean connected = session.connect(30000, BT_STATE.CONNECTED, BT_STATE.SDLOGGING); + report.check(connected, "did not connect within 30 s (state " + session.state() + ")"); + if (connected) { + session.event("connected: " + session.state()); + stopSdLoggingIfLogging(); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + /** Connects and checks the unit was first reported ready, in readyState, at the end of the connect sequence */ private boolean connectAndCheckReady(String what, BT_STATE readyState) throws InterruptedException { boolean ready = session.connect(30000, readyState) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java index 0c3e8f92..5df539ac 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -289,11 +289,11 @@ boolean connect(long timeoutMs) throws InterruptedException { } /** - * As connect(long), for a unit that ends its connect in readyState: SDLOGGING for one that is - * logging to its SD card. It also waits for NOTIFICATION_SHIMMER_FULLY_INITIALIZED, and records - * when the unit was first reported ready (readyEarlyMs()). + * As connect(long), for a unit that ends its connect in one of readyStates: SDLOGGING for one that + * is logging to its SD card. It records when the unit was first reported ready (readyEarlyMs()). */ - boolean connect(long timeoutMs, final BT_STATE readyState) throws InterruptedException { + boolean connect(long timeoutMs, final BT_STATE... readyStates) throws InterruptedException { + final java.util.List accepted = java.util.Arrays.asList(readyStates); readyMessageAtMs = Double.NaN; readyStateAtMs = Double.NaN; fullyInitialisedAtMs = Double.NaN; @@ -311,8 +311,14 @@ boolean connect(long timeoutMs, final BT_STATE readyState) throws InterruptedExc queuedAtReadyState = queuedInstructions(); event("state first shows " + d.getBluetoothRadioState() + " with " + queuedAtReadyState + " instructions queued"); } - return d != null && d.isInitialised() && d.getBluetoothRadioState() == readyState; + return d != null && d.isInitialised() && accepted.contains(d.getBluetoothRadioState()); }, timeoutMs); + if (!ready) { + // The connect call returned, so nothing else dumps the threads: a stalled connect sequence + // (such as the DEV-1198 lock-order deadlock) shows here + event("connect did not finish within " + timeoutMs + " ms (state " + state() + "); thread dump follows"); + dumpThreads(); + } ShimmerBluetooth d = device(); if (ready && ble && d != null) { // Over BLE the driver logs parser errors with consolePrintLn, which only prints in verbose diff --git a/tools/hil/README.md b/tools/hil/README.md index 878b8d5b..36ad4cfe 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -58,6 +58,8 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas | `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | | `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | +`-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects. + Every scenario also fails if an API call doesn't return within 10 s. The report then holds every thread's stack, which a phone without root can't otherwise give, and the scenario stops. If the whole test process still never finishes, the runner force-stops it after `-ScenarioTimeoutSeconds` (600) and records TIMEOUT. The packet checks use the device's own timestamps, so a loss is a gap in the device's sequence, not diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index 22f4fd53..956f7ca3 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -26,7 +26,7 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady')] + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'resetUnit')] [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady'), [string[]]$Unit, [ValidateSet('classic', 'ble')] From 78dad604e13322447b93b2d847d12225617e4011 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 14:30:11 +0800 Subject: [PATCH 4/6] DEV-1291 DEV-1275 HIL: count inStream's log a round at a time; jumps at an SD start are the unit's The phone's log buffer is small, and with the Bluetooth snoop log on it rolled past inStream's start marker within a run. Every count then read 0, so 15 decoded battery replies were reported as none. inStream now places a marker per round and counts right after it. If a marker is gone anyway, it fails saying the lines could not be counted, rather than counting none. A timestamp jump within 3 s after an SD logging start or stop is now reported rather than failed, just as the loss there is. The captured bytes show a Shimmer3 doing this itself. After a 1.5 s pause while it opened its file, it sent two packets it had held, and a millisecond later the next one it sampled, 385 periods on. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 90 +++++++++++++++---- .../hiltest/ShimmerSession.java | 39 +++++++- 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index df83ed5e..7f0c23a7 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -360,10 +360,12 @@ public void inStream() throws Exception { if (connectAndConfigure(rate)) { double deviceRate = session.rateOnDevice(); checkRate(deviceRate, "stream"); - String marker = session.logMarker("in_stream"); + LogTally tally = new LogTally(); + String marker = session.logMarker("in_stream start"); session.startRecording(); report.check(session.startStreaming(10000), "streaming did not start within 10 s"); Thread.sleep(2000); + tally.add(marker); List sdMoments = new ArrayList<>(); List readMoments = new ArrayList<>(); int batteryReads = 0; @@ -371,6 +373,7 @@ public void inStream() throws Exception { for (int i = 1; i <= cycles && session.hung == null; i++) { String what = "round " + i; session.event(what); + marker = session.logMarker("in_stream " + what); if (!startSdLoggingWhileStreaming(what, sdMoments)) { break; } @@ -389,12 +392,16 @@ public void inStream() throws Exception { report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING && session.queuedInstructions() == 0, 5000), what + ": SD logging did not stop (state " + session.state() + ", " + session.queuedInstructions() + " instructions queued)"); Thread.sleep(1000); + tally.add(marker); } + marker = session.logMarker("in_stream restarts"); if (session.hung == null && startSdLoggingWhileStreaming("restarts", sdMoments)) { for (int i = 1; i <= cycles && session.hung == null; i++) { String what = "restart " + i; session.event(what); Thread.sleep(1000); + tally.add(marker); + marker = session.logMarker("in_stream " + what); sdMoments.add(session.nowMs()); report.check(session.stopSdLogging() && session.startSdLogging(), what + ": stopSDLogging or startSDLogging did not complete (see the events)"); @@ -411,25 +418,22 @@ public void inStream() throws Exception { double stoppedAtMs = session.nowMs(); List samples = session.stopRecording(); report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + tally.add(marker); - int batteryReplies = session.logLinesSince(marker, "IN STREAM: VBATT_RESPONSE"); - int statusReplies = session.logLinesSince(marker, "IN STREAM: STATUS_RESPONSE"); - int repliesInAPause = session.logLinesSince(marker, "with no data packet before it"); - int timeouts = session.logLinesSince(marker, COMMAND_TIMEOUTS); - int parserErrors = session.parserErrorsSince(marker); report.put("batteryReads", batteryReads); - report.put("batteryReplies", batteryReplies); + report.put("batteryReplies", tally.batteryReplies); report.put("statusReads", statusReads); - report.put("statusReplies", statusReplies); - report.put("repliesInAPause", repliesInAPause); - report.put("commandTimeouts", timeouts); - report.put("parserErrors", parserErrors); + report.put("statusReplies", tally.statusReplies); + report.put("repliesInAPause", tally.repliesInAPause); + report.put("commandTimeouts", tally.timeouts); + report.put("parserErrors", tally.parserErrors); session.event(String.format(Locale.ROOT, "battery %d reads, %d replies; status %d reads, %d replies (and pushes); %d with no data packet before them", - batteryReads, batteryReplies, statusReads, statusReplies, repliesInAPause)); - report.check(batteryReplies == batteryReads, batteryReads + " battery reads but " + batteryReplies + " replies decoded"); - report.check(statusReplies >= statusReads, statusReads + " status reads but " + statusReplies + " status replies decoded"); - report.check(timeouts == 0, timeouts + " command timeouts logged"); - report.check(parserErrors == 0, parserErrors + " streaming parser errors logged (\"Packet syncing problem\")"); + batteryReads, tally.batteryReplies, statusReads, tally.statusReplies, tally.repliesInAPause)); + report.check(tally.complete, "the phone's log rolled past a round's start, so its replies and errors could not all be counted (see the events)"); + report.check(tally.batteryReplies == batteryReads, batteryReads + " battery reads but " + tally.batteryReplies + " replies decoded"); + report.check(tally.statusReplies >= statusReads, statusReads + " status reads but " + tally.statusReplies + " status replies decoded"); + report.check(tally.timeouts == 0, tally.timeouts + " command timeouts logged"); + report.check(tally.parserErrors == 0, tally.parserErrors + " streaming parser errors logged (\"Packet syncing problem\")"); // Loss near SD logging start/stop is the unit's own (see midStreamSet). A Shimmer3 also skips a // sample now and then around a battery read, with no parser error, so loss near the reads is @@ -449,13 +453,65 @@ public void inStream() throws Exception { elsewherePercent, elsewhere, maxLoss)); report.check(a.received > 0, "stream: no packets received"); report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); - checkJumps(a, "stream"); + checkJumpsAwayFrom(a, sdMoments, "stream"); checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); report.check(session.disconnect(10000), "did not disconnect within 10 s"); } finish(); } + /** inStream's in-stream replies, pause replies, timeouts and parser errors, counted a round at a time */ + private final class LogTally { + int batteryReplies; + int statusReplies; + int repliesInAPause; + int timeouts; + int parserErrors; + /** False once a round's lines could not be counted */ + boolean complete = true; + + /** Adds the lines logged since the marker. Called soon after it, before the phone's small log buffer rolls past it. */ + void add(String marker) { + int[] c = session.logCountsSince(marker, new String[] {"IN STREAM: VBATT_RESPONSE"}, new String[] {"IN STREAM: STATUS_RESPONSE"}, + new String[] {"with no data packet before it"}, COMMAND_TIMEOUTS, ShimmerSession.PARSER_ERRORS); + if (c == null) { + complete = false; + return; + } + batteryReplies += c[0]; + statusReplies += c[1]; + repliesInAPause += c[2]; + timeouts += c[3]; + parserErrors += c[4]; + } + } + + /** + * As checkJumps(), except that a jump within 3 s after an SD logging start or stop is reported, not + * failed. The unit can pause for over a second while it opens its file, then send a packet or two it + * held from before the pause and, a millisecond later, the next one it sampled: a timestamp leap + * with no pause in arrival, but the unit's own (the bytes captured on a Shimmer3 show it). + */ + private void checkJumpsAwayFrom(StreamAnalysis a, List sdMoments, String what) { + int nearSd = 0; + for (StreamAnalysis.Gap j : a.timestampJumps) { + boolean near = false; + for (double moment : sdMoments) { + near |= j.afterReceivedMs >= moment && j.afterReceivedMs - moment <= 3000; + } + if (near) { + nearSd++; + session.event(String.format(Locale.ROOT, "%s: timestamp jumped %.1f s ahead at %.0f ms, within 3 s of an SD logging start or stop (the unit's own; not failed)", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs)); + } else { + report.check(false, String.format(Locale.ROOT, + "%s: timestamp jumped %.1f s ahead at %.0f ms (device time %.0f ms) while packets kept arriving", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs, j.afterTimestampMs)); + } + } + report.put("timestampJumpsNearSdLoggingStartStop", nearSd); + } + /** Starts SD logging while streaming and waits until it has started */ private boolean startSdLoggingWhileStreaming(String what, List sdMoments) throws InterruptedException { sdMoments.add(session.nowMs()); diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java index 5df539ac..3c231eb4 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -41,7 +41,7 @@ final class ShimmerSession implements AutoCloseable { static final String TAG = "ShimmerHil"; /** What the API logs (Log.d, tag "Shimmer") when its streaming parser loses its place. */ - private static final String[] PARSER_ERRORS = {"Packet syncing problem", "Unknown parsing error while streaming"}; + static final String[] PARSER_ERRORS = {"Packet syncing problem", "Unknown parsing error while streaming"}; interface Condition { boolean holds(); @@ -518,6 +518,43 @@ int logLinesSince(String marker, String... texts) { return count; } + /** + * For each set of texts, the lines this process logged after the marker that contain any of them, + * or null if they can't be counted. Unlike logLinesSince(), that includes a log that no longer + * holds the marker: the phone's log buffer is small, and with the Bluetooth snoop log on it can + * roll past a marker within seconds, which would read as no lines at all. + */ + int[] logCountsSince(String marker, String[]... textSets) { + int[] counts = new int[textSets.length]; + boolean afterMarker = false; + try { + java.lang.Process p = Runtime.getRuntime().exec(new String[] {"logcat", "-d", "--pid=" + Process.myPid()}); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + for (String line; (line = r.readLine()) != null; ) { + if (line.contains(marker)) { + afterMarker = true; + } else if (afterMarker) { + for (int k = 0; k < textSets.length; k++) { + for (String e : textSets[k]) { + if (line.contains(e)) { + counts[k]++; + } + } + } + } + } + p.waitFor(); + } catch (Exception e) { + event("could not read logcat: " + e); + return null; + } + if (!afterMarker) { + event("the phone's log no longer holds \"" + marker + "\", so the lines after it can't be counted"); + return null; + } + return counts; + } + @Override public void close() throws Exception { try { From 79953480882e79e94a842a28117f6584c62a411d Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 20:44:35 +0800 Subject: [PATCH 5/6] DEV-1266 Android HIL: queueStress, to reproduce the lock-order deadlock The driver guards its instruction queue with two locks, the device and the queue. Up to 0.11.10_beta its IO thread took them queue first when it fetched the next instruction, and every call that queues one takes them device first, so the two could deadlock and every later call hung. On Android it hung 5 of 12 inStream runs on driver 0.11.7_beta, but inStream needs SD logging and catches it only by chance. queueStress makes the window wide on purpose, as the Java HIL's scenario of the same name does. With the unit connected and idle, timers on, four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Idle, because while streaming Android writes these reads straight out, unqueued, and they never take the queue's lock. It needs no SD card, so it also runs on a docked or charging unit. It fails when a read doesn't return within 10 s or no read returns for 10 s, with every thread's stack in the events; when a read throws; or when the queue doesn't drain within 15 s. The runner runs it by default and the README lists it. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 134 ++++++++++++++++++ .../hiltest/ShimmerSession.java | 2 +- tools/hil/README.md | 1 + tools/hil/run-android-hil.ps1 | 4 +- 4 files changed, 138 insertions(+), 3 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index 7f0c23a7..9ccb679b 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -6,6 +6,7 @@ import android.content.Context; import android.os.Build; import android.os.Bundle; +import android.os.SystemClock; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; @@ -26,6 +27,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReference; /** * Hardware-in-the-loop scenarios against one real Shimmer3 or Shimmer3R, paired with the phone over @@ -40,6 +45,7 @@ * cycles mid_stream_set / reconnect_loop / in_stream repetitions, default 3 / 5 / 3 * crc link CRC to ask for: off, 1 or 2 (default 2; the driver may settle on less) * maxLossPercent default 1.0 + * queueStressSeconds how long queue_stress queues reads, default 60 * accelTolerance m/s^2 either side of 9.81 at rest, default 1.0 * * @@ -600,6 +606,134 @@ private static double[] toArray(List values) { return out; } + /** queueStress: the app threads, the most instructions they keep queued, and how long a stuck call or queue takes to count as one */ + private static final int QUEUE_STRESS_THREADS = 4; + private static final int QUEUE_STRESS_MAX_QUEUED = 8; + private static final long QUEUE_STRESS_STALL_MS = 10000; + + /** + * DEV-1266: the driver guards its instruction queue with two locks, the device and the queue. Its + * IO thread took them queue first when it fetched the next instruction, and every call that queues + * one takes them device first, so the two could deadlock and every later call hung (5 of 12 + * inStream runs on driver 0.11.7). With the unit connected and idle, timers on as in an app, four + * threads queue battery and status reads in tight loops for queueStressSeconds (default 60), + * keeping the queue under 8, so a call is nearly always taking the locks while the IO thread + * fetches. Idle, because while streaming Android writes both reads straight out, unqueued. No SD + * card is needed. + */ + @Test + public void queueStress() throws Exception { + begin("queue_stress"); + int seconds = arg("queueStressSeconds", 60); + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + final ShimmerBluetooth d = connected ? session.device() : null; + if (d != null) { + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + final AtomicBoolean stop = new AtomicBoolean(); + final AtomicLong calls = new AtomicLong(); + // When each thread's current call began (elapsedRealtime), or 0 between calls + final AtomicLongArray inCallSince = new AtomicLongArray(QUEUE_STRESS_THREADS); + final AtomicReference thrown = new AtomicReference<>(); + List threads = new ArrayList<>(); + for (int i = 0; i < QUEUE_STRESS_THREADS; i++) { + final int k = i; + Thread t = new Thread(() -> { + try { + while (!stop.get()) { + if (session.queuedInstructions() >= QUEUE_STRESS_MAX_QUEUED) { + Thread.sleep(1); + continue; + } + inCallSince.set(k, SystemClock.elapsedRealtime()); + if ((calls.get() + k) % 2 == 0) { + d.readBattery(); + } else { + d.readStatusLogAndStream(); + } + inCallSince.set(k, 0); + calls.incrementAndGet(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable e) { + thrown.compareAndSet(null, e); + stop.set(true); + } + }, "QueueStress" + i); + t.setDaemon(true); // a deadlocked thread must not keep the test process alive + threads.add(t); + } + session.event("queue stress: " + QUEUE_STRESS_THREADS + " threads for " + seconds + " s"); + Object battery = session.batteryDetails(); + int batteryReplies = 0; + int maxQueued = 0; + long lastCalls = 0; + long lastProgress = SystemClock.elapsedRealtime(); + long end = lastProgress + seconds * 1000L; + String stall = null; + for (Thread t : threads) { + t.start(); + } + while (SystemClock.elapsedRealtime() < end && stall == null && !stop.get()) { + Thread.sleep(100); + long now = SystemClock.elapsedRealtime(); + Object b = session.batteryDetails(); + if (b != null && b != battery) { + battery = b; + batteryReplies++; + } + int queued = session.queuedInstructions(); + maxQueued = Math.max(maxQueued, queued); + long c = calls.get(); + if (c != lastCalls) { + lastCalls = c; + lastProgress = now; + } + for (int i = 0; i < QUEUE_STRESS_THREADS && stall == null; i++) { + long since = inCallSince.get(i); + if (since != 0 && now - since > QUEUE_STRESS_STALL_MS) { + stall = String.format(Locale.ROOT, "a read queued by QueueStress%d has not returned for %.0f s", i, (now - since) / 1000.0); + } + } + if (stall == null && now - lastProgress > QUEUE_STRESS_STALL_MS) { + stall = String.format(Locale.ROOT, "no read returned for %.0f s, with %d instructions queued", (now - lastProgress) / 1000.0, queued); + } + } + stop.set(true); + report.put("calls", calls.get()); + report.put("batteryReplies", batteryReplies); + report.put("maxQueued", maxQueued); + session.event(String.format(Locale.ROOT, "queue stress: %d reads queued, %d battery replies seen, at most %d instructions queued", + calls.get(), batteryReplies, maxQueued)); + if (stall != null) { + session.hung = "readBattery/readStatusLogAndStream (queueStress)"; + session.event(stall + "; thread dump follows"); + session.dumpThreads(); + report.check(false, stall + " (the calls and the IO thread are stuck; DEV-1266's deadlock if the dump shows them waiting on each other)"); + } else { + for (Thread t : threads) { + t.join(2000); + } + report.check(thrown.get() == null, "a read threw " + thrown.get()); + report.check(calls.get() > 0, "no read was queued"); + boolean drained = session.waitFor(() -> session.queuedInstructions() == 0, 15000); + report.check(drained, "the queue did not drain within 15 s after the reads stopped (" + + session.queuedInstructions() + " left; the IO thread is stuck, see the thread dump in the events)"); + if (!drained) { + session.event("the queue did not drain; thread dump follows"); + session.dumpThreads(); + } + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + /** Connect, stream briefly and disconnect, repeatedly: the reconnect path apps hit every day. */ @Test public void reconnectLoop() throws Exception { diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java index 3c231eb4..c6b66d9b 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -270,7 +270,7 @@ boolean call(String what, long timeoutMs, Call call) { } } - private void dumpThreads() { + void dumpThreads() { for (java.util.Map.Entry t : Thread.getAllStackTraces().entrySet()) { StringBuilder b = new StringBuilder(" thread \"" + t.getKey().getName() + "\" " + t.getKey().getState()); for (StackTraceElement frame : t.getValue()) { diff --git a/tools/hil/README.md b/tools/hil/README.md index 36ad4cfe..10ab0870 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -57,6 +57,7 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times. | Any cycle fails to connect within 30 s, loses packets, fails the rate and silence checks, or logs a parser error. | | `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | | `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | +| `queueStress` | Connected and idle, with the API's timers on as in an app: four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Each queued read takes the device's lock and then the queue's while the IO thread fetches the next instruction, the window DEV-1266's lock-order deadlock needs. Idle, because while streaming Android writes these reads straight out, unqueued. | A read doesn't return within 10 s, or no read returns for 10 s (the report then holds every thread's stack); a read throws; or the queue doesn't drain within 15 s after the reads stop. No SD card needed. On driver 0.11.7_beta, which has the deadlock, this is the scenario to reproduce it. | `-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects. diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index 956f7ca3..b588e703 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -26,8 +26,8 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'resetUnit')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'resetUnit')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress'), [string[]]$Unit, [ValidateSet('classic', 'ble')] [string[]]$Transport, From b60c9afee53ea067c3f0247b09fdd17681f8adbf Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 21:25:36 +0800 Subject: [PATCH 6/6] DEV-1306 Android HIL: samplingRates, configuring straight after a stop Up to driver 0.11.10_beta, a configuration written straight after stopStreaming() could lose its first chunk, the one holding the sampling rate: operationPrepare() cleared the instruction list while the STOP was still in flight, and the stop's reply then retired the next instruction in its place. The unit kept its old rate. The Java HIL's samplingRates found it on a Shimmer3 and a Shimmer3R. Android configures through the same driver code over both transports, and had no scenario for it. samplingRates, as in the Java and shimmer-logandstream HILs, writes 51.2, 256 and 512 Hz in turn and streams 5 s at each, then writes the unit's own rate back; every configure but the first comes straight after a stop. Unlike the Java HIL's configure, the Android one can finish on the manager's SUCCESS without looking at the rate, so the scenario checks that the rate read back is the one written. No SD card is needed. The runner runs it by default and the README lists it. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 75 +++++++++++++++++++ tools/hil/README.md | 1 + tools/hil/run-android-hil.ps1 | 4 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index 9ccb679b..24bf7030 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -606,6 +606,81 @@ private static double[] toArray(List values) { return out; } + /** The rates samplingRates writes in turn, as the Java and shimmer-logandstream HILs' scenarios of the same name do */ + private static final double[] SAMPLING_RATES_HZ = {51.2, 256, 512}; + + /** + * Writes each rate in SAMPLING_RATES_HZ in turn, the way the examples configure, and streams 5 s + * at each; then writes the unit's own rate back. Every configure but the first comes straight + * after a stopStreaming(), as an app that changes the rate does. Up to driver 0.11.10_beta the + * stop's reply then retired the configuration's first chunk, the one holding the sampling rate, + * and the unit kept its old rate (DEV-1306). No SD card is needed. + */ + @Test + public void samplingRates() throws Exception { + begin("sampling_rates"); + double maxLoss = arg("maxLossPercent", 1.0); + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + ShimmerBluetooth d = connected ? session.device() : null; + if (d != null) { + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + double ownRate = d.getSamplingRateShimmer(); + report.put("ownRateHz", ownRate); + List streams = new ArrayList<>(); + for (double rate : SAMPLING_RATES_HZ) { + if (session.hung != null) { + break; + } + String what = String.format(Locale.ROOT, "%.1f Hz", rate); + if (!configureRate(rate, what)) { + continue; + } + double deviceRate = session.rateOnDevice(); + String marker = session.logMarker("sampling_rates " + what); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + streams.add(what + ": " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + int parserErrors = session.parserErrorsSince(marker); + report.check(parserErrors == 0, what + ": " + parserErrors + " streaming parser errors logged"); + } + report.put("streams", new org.json.JSONArray(streams)); + if (session.hung == null) { + configureRate(ownRate, "the unit's own rate"); + } + report.put("rateOnDeviceHz", session.rateOnDevice()); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** + * Writes rateHz and checks the unit took it. The configure ends by reading the configuration + * back, so the API's rate is then the unit's. configureAccelOnly can finish on the manager's + * SUCCESS alone, without looking at the rate, so the rate is checked here. + */ + private boolean configureRate(double rateHz, String what) throws InterruptedException { + boolean configured = session.configureAccelOnly(rateHz, 20000); + double onDevice = session.rateOnDevice(); + report.check(configured, String.format(Locale.ROOT, "%s: the configuration did not complete within 20 s (rate on device %.2f Hz, state %s)", + what, onDevice, session.state())); + boolean took = Math.abs(onDevice - rateHz) < 0.5; + report.check(!configured || took, String.format(Locale.ROOT, + "%s: the unit kept %.2f Hz after %.2f Hz was written (straight after a stop, that is DEV-1306)", what, onDevice, rateHz)); + return configured && took; + } + /** queueStress: the app threads, the most instructions they keep queued, and how long a stuck call or queue takes to count as one */ private static final int QUEUE_STRESS_THREADS = 4; private static final int QUEUE_STRESS_MAX_QUEUED = 8; diff --git a/tools/hil/README.md b/tools/hil/README.md index 10ab0870..5db379ff 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -58,6 +58,7 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas | `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | | `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | | `queueStress` | Connected and idle, with the API's timers on as in an app: four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Each queued read takes the device's lock and then the queue's while the IO thread fetches the next instruction, the window DEV-1266's lock-order deadlock needs. Idle, because while streaming Android writes these reads straight out, unqueued. | A read doesn't return within 10 s, or no read returns for 10 s (the report then holds every thread's stack); a read throws; or the queue doesn't drain within 15 s after the reads stop. No SD card needed. On driver 0.11.7_beta, which has the deadlock, this is the scenario to reproduce it. | +| `samplingRates` | Writes 51.2, 256 and 512 Hz in turn, the way the examples configure, and streams 5 s at each; then writes the unit's own rate back. Every configure but the first comes straight after a `stopStreaming()`, as an app that changes the rate does: up to driver 0.11.10_beta the stop's reply then retired the configuration's first chunk, the one holding the sampling rate (DEV-1306). | A configuration doesn't complete within 20 s, or the rate read back from the unit isn't the one written; or a stream fails `streamRest`'s loss, timestamp, rate and silence checks, or logs a parser error. No SD card needed. | `-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects. diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index b588e703..3aab2cb3 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -26,8 +26,8 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'resetUnit')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates', 'resetUnit')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates'), [string[]]$Unit, [ValidateSet('classic', 'ble')] [string[]]$Transport,