From 79953480882e79e94a842a28117f6584c62a411d Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 20:44:35 +0800 Subject: [PATCH 1/2] DEV-1266 Android HIL: queueStress, to reproduce the lock-order deadlock The driver guards its instruction queue with two locks, the device and the queue. Up to 0.11.10_beta its IO thread took them queue first when it fetched the next instruction, and every call that queues one takes them device first, so the two could deadlock and every later call hung. On Android it hung 5 of 12 inStream runs on driver 0.11.7_beta, but inStream needs SD logging and catches it only by chance. queueStress makes the window wide on purpose, as the Java HIL's scenario of the same name does. With the unit connected and idle, timers on, four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Idle, because while streaming Android writes these reads straight out, unqueued, and they never take the queue's lock. It needs no SD card, so it also runs on a docked or charging unit. It fails when a read doesn't return within 10 s or no read returns for 10 s, with every thread's stack in the events; when a read throws; or when the queue doesn't drain within 15 s. The runner runs it by default and the README lists it. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 134 ++++++++++++++++++ .../hiltest/ShimmerSession.java | 2 +- tools/hil/README.md | 1 + tools/hil/run-android-hil.ps1 | 4 +- 4 files changed, 138 insertions(+), 3 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index 7f0c23a..9ccb679 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -6,6 +6,7 @@ import android.content.Context; import android.os.Build; import android.os.Bundle; +import android.os.SystemClock; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; @@ -26,6 +27,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReference; /** * Hardware-in-the-loop scenarios against one real Shimmer3 or Shimmer3R, paired with the phone over @@ -40,6 +45,7 @@ * cycles mid_stream_set / reconnect_loop / in_stream repetitions, default 3 / 5 / 3 * crc link CRC to ask for: off, 1 or 2 (default 2; the driver may settle on less) * maxLossPercent default 1.0 + * queueStressSeconds how long queue_stress queues reads, default 60 * accelTolerance m/s^2 either side of 9.81 at rest, default 1.0 * * @@ -600,6 +606,134 @@ private static double[] toArray(List values) { return out; } + /** queueStress: the app threads, the most instructions they keep queued, and how long a stuck call or queue takes to count as one */ + private static final int QUEUE_STRESS_THREADS = 4; + private static final int QUEUE_STRESS_MAX_QUEUED = 8; + private static final long QUEUE_STRESS_STALL_MS = 10000; + + /** + * DEV-1266: the driver guards its instruction queue with two locks, the device and the queue. Its + * IO thread took them queue first when it fetched the next instruction, and every call that queues + * one takes them device first, so the two could deadlock and every later call hung (5 of 12 + * inStream runs on driver 0.11.7). With the unit connected and idle, timers on as in an app, four + * threads queue battery and status reads in tight loops for queueStressSeconds (default 60), + * keeping the queue under 8, so a call is nearly always taking the locks while the IO thread + * fetches. Idle, because while streaming Android writes both reads straight out, unqueued. No SD + * card is needed. + */ + @Test + public void queueStress() throws Exception { + begin("queue_stress"); + int seconds = arg("queueStressSeconds", 60); + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + final ShimmerBluetooth d = connected ? session.device() : null; + if (d != null) { + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + final AtomicBoolean stop = new AtomicBoolean(); + final AtomicLong calls = new AtomicLong(); + // When each thread's current call began (elapsedRealtime), or 0 between calls + final AtomicLongArray inCallSince = new AtomicLongArray(QUEUE_STRESS_THREADS); + final AtomicReference thrown = new AtomicReference<>(); + List threads = new ArrayList<>(); + for (int i = 0; i < QUEUE_STRESS_THREADS; i++) { + final int k = i; + Thread t = new Thread(() -> { + try { + while (!stop.get()) { + if (session.queuedInstructions() >= QUEUE_STRESS_MAX_QUEUED) { + Thread.sleep(1); + continue; + } + inCallSince.set(k, SystemClock.elapsedRealtime()); + if ((calls.get() + k) % 2 == 0) { + d.readBattery(); + } else { + d.readStatusLogAndStream(); + } + inCallSince.set(k, 0); + calls.incrementAndGet(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable e) { + thrown.compareAndSet(null, e); + stop.set(true); + } + }, "QueueStress" + i); + t.setDaemon(true); // a deadlocked thread must not keep the test process alive + threads.add(t); + } + session.event("queue stress: " + QUEUE_STRESS_THREADS + " threads for " + seconds + " s"); + Object battery = session.batteryDetails(); + int batteryReplies = 0; + int maxQueued = 0; + long lastCalls = 0; + long lastProgress = SystemClock.elapsedRealtime(); + long end = lastProgress + seconds * 1000L; + String stall = null; + for (Thread t : threads) { + t.start(); + } + while (SystemClock.elapsedRealtime() < end && stall == null && !stop.get()) { + Thread.sleep(100); + long now = SystemClock.elapsedRealtime(); + Object b = session.batteryDetails(); + if (b != null && b != battery) { + battery = b; + batteryReplies++; + } + int queued = session.queuedInstructions(); + maxQueued = Math.max(maxQueued, queued); + long c = calls.get(); + if (c != lastCalls) { + lastCalls = c; + lastProgress = now; + } + for (int i = 0; i < QUEUE_STRESS_THREADS && stall == null; i++) { + long since = inCallSince.get(i); + if (since != 0 && now - since > QUEUE_STRESS_STALL_MS) { + stall = String.format(Locale.ROOT, "a read queued by QueueStress%d has not returned for %.0f s", i, (now - since) / 1000.0); + } + } + if (stall == null && now - lastProgress > QUEUE_STRESS_STALL_MS) { + stall = String.format(Locale.ROOT, "no read returned for %.0f s, with %d instructions queued", (now - lastProgress) / 1000.0, queued); + } + } + stop.set(true); + report.put("calls", calls.get()); + report.put("batteryReplies", batteryReplies); + report.put("maxQueued", maxQueued); + session.event(String.format(Locale.ROOT, "queue stress: %d reads queued, %d battery replies seen, at most %d instructions queued", + calls.get(), batteryReplies, maxQueued)); + if (stall != null) { + session.hung = "readBattery/readStatusLogAndStream (queueStress)"; + session.event(stall + "; thread dump follows"); + session.dumpThreads(); + report.check(false, stall + " (the calls and the IO thread are stuck; DEV-1266's deadlock if the dump shows them waiting on each other)"); + } else { + for (Thread t : threads) { + t.join(2000); + } + report.check(thrown.get() == null, "a read threw " + thrown.get()); + report.check(calls.get() > 0, "no read was queued"); + boolean drained = session.waitFor(() -> session.queuedInstructions() == 0, 15000); + report.check(drained, "the queue did not drain within 15 s after the reads stopped (" + + session.queuedInstructions() + " left; the IO thread is stuck, see the thread dump in the events)"); + if (!drained) { + session.event("the queue did not drain; thread dump follows"); + session.dumpThreads(); + } + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + /** Connect, stream briefly and disconnect, repeatedly: the reconnect path apps hit every day. */ @Test public void reconnectLoop() throws Exception { diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java index 3c231eb..c6b66d9 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -270,7 +270,7 @@ boolean call(String what, long timeoutMs, Call call) { } } - private void dumpThreads() { + void dumpThreads() { for (java.util.Map.Entry t : Thread.getAllStackTraces().entrySet()) { StringBuilder b = new StringBuilder(" thread \"" + t.getKey().getName() + "\" " + t.getKey().getState()); for (StackTraceElement frame : t.getValue()) { diff --git a/tools/hil/README.md b/tools/hil/README.md index 36ad4cf..10ab087 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -57,6 +57,7 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times. | Any cycle fails to connect within 30 s, loses packets, fails the rate and silence checks, or logs a parser error. | | `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | | `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | +| `queueStress` | Connected and idle, with the API's timers on as in an app: four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Each queued read takes the device's lock and then the queue's while the IO thread fetches the next instruction, the window DEV-1266's lock-order deadlock needs. Idle, because while streaming Android writes these reads straight out, unqueued. | A read doesn't return within 10 s, or no read returns for 10 s (the report then holds every thread's stack); a read throws; or the queue doesn't drain within 15 s after the reads stop. No SD card needed. On driver 0.11.7_beta, which has the deadlock, this is the scenario to reproduce it. | `-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects. diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index 956f7ca..b588e70 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -26,8 +26,8 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'resetUnit')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'resetUnit')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress'), [string[]]$Unit, [ValidateSet('classic', 'ble')] [string[]]$Transport, From b60c9afee53ea067c3f0247b09fdd17681f8adbf Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 21:25:36 +0800 Subject: [PATCH 2/2] DEV-1306 Android HIL: samplingRates, configuring straight after a stop Up to driver 0.11.10_beta, a configuration written straight after stopStreaming() could lose its first chunk, the one holding the sampling rate: operationPrepare() cleared the instruction list while the STOP was still in flight, and the stop's reply then retired the next instruction in its place. The unit kept its old rate. The Java HIL's samplingRates found it on a Shimmer3 and a Shimmer3R. Android configures through the same driver code over both transports, and had no scenario for it. samplingRates, as in the Java and shimmer-logandstream HILs, writes 51.2, 256 and 512 Hz in turn and streams 5 s at each, then writes the unit's own rate back; every configure but the first comes straight after a stop. Unlike the Java HIL's configure, the Android one can finish on the manager's SUCCESS without looking at the rate, so the scenario checks that the rate read back is the one written. No SD card is needed. The runner runs it by default and the README lists it. Co-Authored-By: Claude Opus 5.5 --- .../hiltest/HilScenariosTest.java | 75 +++++++++++++++++++ tools/hil/README.md | 1 + tools/hil/run-android-hil.ps1 | 4 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java index 9ccb679..24bf703 100644 --- a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -606,6 +606,81 @@ private static double[] toArray(List values) { return out; } + /** The rates samplingRates writes in turn, as the Java and shimmer-logandstream HILs' scenarios of the same name do */ + private static final double[] SAMPLING_RATES_HZ = {51.2, 256, 512}; + + /** + * Writes each rate in SAMPLING_RATES_HZ in turn, the way the examples configure, and streams 5 s + * at each; then writes the unit's own rate back. Every configure but the first comes straight + * after a stopStreaming(), as an app that changes the rate does. Up to driver 0.11.10_beta the + * stop's reply then retired the configuration's first chunk, the one holding the sampling rate, + * and the unit kept its old rate (DEV-1306). No SD card is needed. + */ + @Test + public void samplingRates() throws Exception { + begin("sampling_rates"); + double maxLoss = arg("maxLossPercent", 1.0); + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + ShimmerBluetooth d = connected ? session.device() : null; + if (d != null) { + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + double ownRate = d.getSamplingRateShimmer(); + report.put("ownRateHz", ownRate); + List streams = new ArrayList<>(); + for (double rate : SAMPLING_RATES_HZ) { + if (session.hung != null) { + break; + } + String what = String.format(Locale.ROOT, "%.1f Hz", rate); + if (!configureRate(rate, what)) { + continue; + } + double deviceRate = session.rateOnDevice(); + String marker = session.logMarker("sampling_rates " + what); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + streams.add(what + ": " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + int parserErrors = session.parserErrorsSince(marker); + report.check(parserErrors == 0, what + ": " + parserErrors + " streaming parser errors logged"); + } + report.put("streams", new org.json.JSONArray(streams)); + if (session.hung == null) { + configureRate(ownRate, "the unit's own rate"); + } + report.put("rateOnDeviceHz", session.rateOnDevice()); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** + * Writes rateHz and checks the unit took it. The configure ends by reading the configuration + * back, so the API's rate is then the unit's. configureAccelOnly can finish on the manager's + * SUCCESS alone, without looking at the rate, so the rate is checked here. + */ + private boolean configureRate(double rateHz, String what) throws InterruptedException { + boolean configured = session.configureAccelOnly(rateHz, 20000); + double onDevice = session.rateOnDevice(); + report.check(configured, String.format(Locale.ROOT, "%s: the configuration did not complete within 20 s (rate on device %.2f Hz, state %s)", + what, onDevice, session.state())); + boolean took = Math.abs(onDevice - rateHz) < 0.5; + report.check(!configured || took, String.format(Locale.ROOT, + "%s: the unit kept %.2f Hz after %.2f Hz was written (straight after a stop, that is DEV-1306)", what, onDevice, rateHz)); + return configured && took; + } + /** queueStress: the app threads, the most instructions they keep queued, and how long a stuck call or queue takes to count as one */ private static final int QUEUE_STRESS_THREADS = 4; private static final int QUEUE_STRESS_MAX_QUEUED = 8; diff --git a/tools/hil/README.md b/tools/hil/README.md index 10ab087..5db379f 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -58,6 +58,7 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas | `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | | `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | | `queueStress` | Connected and idle, with the API's timers on as in an app: four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Each queued read takes the device's lock and then the queue's while the IO thread fetches the next instruction, the window DEV-1266's lock-order deadlock needs. Idle, because while streaming Android writes these reads straight out, unqueued. | A read doesn't return within 10 s, or no read returns for 10 s (the report then holds every thread's stack); a read throws; or the queue doesn't drain within 15 s after the reads stop. No SD card needed. On driver 0.11.7_beta, which has the deadlock, this is the scenario to reproduce it. | +| `samplingRates` | Writes 51.2, 256 and 512 Hz in turn, the way the examples configure, and streams 5 s at each; then writes the unit's own rate back. Every configure but the first comes straight after a `stopStreaming()`, as an app that changes the rate does: up to driver 0.11.10_beta the stop's reply then retired the configuration's first chunk, the one holding the sampling rate (DEV-1306). | A configuration doesn't complete within 20 s, or the rate read back from the unit isn't the one written; or a stream fails `streamRest`'s loss, timestamp, rate and silence checks, or logs a parser error. No SD card needed. | `-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects. diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index b588e70..3aab2cb 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -26,8 +26,8 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'resetUnit')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates', 'resetUnit')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates'), [string[]]$Unit, [ValidateSet('classic', 'ble')] [string[]]$Transport,