From 02b5380383cb4d23489fb28718eee19f6163d56e Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 12:29:55 +0800 Subject: [PATCH 1/2] DEV-1198 Dump all threads when a simulator connect times out The lns0_16_11_w_sd_bt_streaming connect hangs on Linux CI and not on Windows, so the CI log is the only evidence of where the IO thread is stuck. Print every thread's full stack, the monitors it holds, the lock it waits on and any JVM-detected deadlock before failing. Co-Authored-By: Claude Opus 5.5 --- .../ByteCommunicationTestUtils.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java index 5722ce6bf..dc8213484 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java @@ -2,6 +2,12 @@ import static org.junit.Assert.fail; +import java.lang.management.LockInfo; +import java.lang.management.ManagementFactory; +import java.lang.management.MonitorInfo; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; + import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; import com.shimmerresearch.pcDriver.ShimmerPC; @@ -23,6 +29,9 @@ public static void connectAndWaitUntilInitialised(ShimmerPC device) { long deadline = System.currentTimeMillis() + CONNECT_TIMEOUT_MS; while (!(device.isInitialised() && device.getBluetoothRadioState()==BT_STATE.CONNECTED)) { if (System.currentTimeMillis() > deadline) { + // A hang here has so far only reproduced on CI, so print where every thread is + // before failing - the log is the only evidence there is. + System.out.println(dumpAllThreads()); fail("Device did not finish initialising within " + CONNECT_TIMEOUT_MS + " ms (state: " + device.getBluetoothRadioState() + ")"); } try { @@ -34,4 +43,48 @@ public static void connectAndWaitUntilInitialised(ShimmerPC device) { } } + /** Every thread's full stack with the monitors it holds and the lock it waits on, plus any + * deadlock the JVM can see. ThreadInfo.toString() stops at 8 frames, too few for the driver. */ + public static String dumpAllThreads() { + ThreadMXBean mxBean = ManagementFactory.getThreadMXBean(); + StringBuilder sb = new StringBuilder("==== Thread dump ====\n"); + + long[] deadlocked = mxBean.findDeadlockedThreads(); + if (deadlocked != null) { + sb.append("DEADLOCK between thread ids:"); + for (long id : deadlocked) { + sb.append(' ').append(id); + } + sb.append('\n'); + } + + for (ThreadInfo ti : mxBean.dumpAllThreads(true, true)) { + sb.append('"').append(ti.getThreadName()).append("\" id=").append(ti.getThreadId()) + .append(' ').append(ti.getThreadState()); + if (ti.getLockName() != null) { + sb.append(" on ").append(ti.getLockName()); + } + if (ti.getLockOwnerName() != null) { + sb.append(" owned by \"").append(ti.getLockOwnerName()).append("\" id=").append(ti.getLockOwnerId()); + } + sb.append('\n'); + + StackTraceElement[] stack = ti.getStackTrace(); + MonitorInfo[] monitors = ti.getLockedMonitors(); + for (int i = 0; i < stack.length; i++) { + sb.append("\tat ").append(stack[i]).append('\n'); + for (MonitorInfo mi : monitors) { + if (mi.getLockedStackDepth() == i) { + sb.append("\t- locked ").append(mi).append('\n'); + } + } + } + for (LockInfo li : ti.getLockedSynchronizers()) { + sb.append("\t- locked synchronizer ").append(li).append('\n'); + } + sb.append('\n'); + } + return sb.append("==== End of thread dump ====").toString(); + } + } From ffb9647a2fedeab736f96556349e11f224fa3082 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 13:54:39 +0800 Subject: [PATCH 2/2] DEV-1198 Stop the simulator tests deadlocking on the read-status timer The connect sequence unlocks the instruction stack and then starts the read-status timer with no delay, so the timer's first readStatusTask (device lock, then instruction-list lock) runs just as the IOThread takes the first post-connect instruction in getInstruction() (list lock, then device lock). When they meet, both block for good and the connect never finishes. The thread dump from the previous commit shows that cycle on Linux CI in lns0_16_11_w_sd_bt_streaming, and locally in API_0000X_ByteCommunicationShimmer3. The driver is deliberately left unchanged. The simulator tests now build their ShimmerPC through ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer(), which overrides startTimerReadStatus() to do nothing, so nothing else touches the instruction list while the IOThread connects. The tests lose only the 5 s status poll; the connect sequence still reads the status once. Co-Authored-By: Claude Opus 5.5 --- .../API_0000X_ByteCommunicationShimmer3.java | 2 +- .../API_0000X_ByteCommunicationShimmer3R.java | 2 +- ...teCommunicationShimmer3_CRC_Streaming.java | 2 +- ...X_ByteCommunicationShimmer3_Streaming.java | 2 +- ...onShimmer3_Streaming_Contiguous_Check.java | 2 +- ...mmer3_Streaming_Contiguous_Check_Fail.java | 2 +- ...3_Streaming_Contiguous_Check_Overflow.java | 2 +- ...icationShimmer3_Streaming_Throw_Bytes.java | 2 +- ...ationShimmer3_Streaming_Throw_Bytes_2.java | 2 +- ...X_ByteCommunicationShimmer3lns0_16_11.java | 2 +- ...nShimmer3lns0_16_11_w_sd_bt_streaming.java | 2 +- .../ByteCommunicationTestUtils.java | 20 +++++++++++++++++++ 12 files changed, 31 insertions(+), 11 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java index ef2a5083c..45ac4ec9d 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java @@ -37,7 +37,7 @@ public class API_0000X_ByteCommunicationShimmer3 extends BasicProcessWithCallBac @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java index 87369358d..5dd1e90b9 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java @@ -54,7 +54,7 @@ public class API_0000X_ByteCommunicationShimmer3R extends BasicProcessWithCallBa @Before public void setUp() { mByteCommunicationSimulatorS3R = new ByteCommunicationSimulatorS3R("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3R); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java index af7afaf6f..7cc1efbfe 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_CRC_Streaming extends BasicProc @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_instream_crc("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java index 0752fbfe3..07adcec3e 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming extends BasicProcessW @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java index 0c31a771b..430e7cb18 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check exte @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_timestamp_contiguous_check("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java index be362947a..5884c764e 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java @@ -37,7 +37,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_timestamp_contiguous_check_fail("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java index 42dbba835..d8d93e0f2 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Over @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_timestamp_contiguous_check_overflow("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java index 0eeab0c01..84b8bbd8a 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes extends B @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_with_improved_throw_bytes("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java index 6cf3777df..da4b56f44 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2 extends @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_with_improved_throw_bytes_2("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java index 8d2d07628..054f13afe 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11 extends BasicProcessW ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11("COM99")); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java index a84b082b3..d264c594a 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming ext ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sd_bt_streaming("COM99")); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java index dc8213484..69d9f203c 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java @@ -18,6 +18,26 @@ public class ByteCommunicationTestUtils { * but a loaded CI runner can stall it for several. Only a genuine hang should hit this. */ public static final long CONNECT_TIMEOUT_MS = 30000; + /** + * A ShimmerPC for the simulator tests whose periodic read-status timer never starts. + *

+ * The connect sequence unlocks the instruction stack and then starts that timer with no delay, + * so its first readStatusTask (which locks the device, then the instruction list) runs just as + * the IOThread takes the first post-connect instruction in ShimmerBluetooth.getInstruction() + * (which locks the list, then the device). When the two meet they deadlock and the connect never + * finishes. That hung these tests on Linux CI every time (DEV-1198). The driver is deliberately + * left as it is; the tests only lose the 5 s status poll, and the connect sequence still reads + * the status once. + */ + public static ShimmerPC newShimmerPCWithoutStatusTimer(String comPort) { + return new ShimmerPC(comPort) { + @Override + public void startTimerReadStatus() { + // Deliberately empty - see above + } + }; + } + /** * Connects and blocks until the device has finished its post-connect config read (infomem, * calibration, etc.), i.e. isInitialised() is set and the state has moved to CONNECTED.