From 3f010ecfb7b87da3a3c5ce6ba81b52bb0c6f484b Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 10:43:58 +0800 Subject: [PATCH 1/2] DEV-1310: deepClone() no longer copies the connection's queues ShimmerPC.deepClone() copies a device by serialising it, which takes no lock. On a connected device the timers and the IO thread kept changing the instruction list (and, while streaming, the PC timestamp list and packet queue) during the copy, so deepClone() could throw ConcurrentModificationException and the configure built on it failed. A clone is a copy of the configuration, not of the connection: those three queues are now transient and readObject() gives the copy empty ones. The Java HIL's HilShimmerPC had the same problem with its own log, which the driver's threads append to on every message; its counters are now transient too. Tests: API_00034 clones while a thread queues and retires instructions, and a simulator-connected device while an app reads battery and status; HilShimmerPCTest clones while the driver logs. All three fail on master. Co-Authored-By: Claude Opus 5.5 --- .../bluetooth/ShimmerBluetooth.java | 23 +++- .../API_00034_DeepCloneWhileBusyTest.java | 121 ++++++++++++++++++ .../com/shimmerresearch/hil/HilShimmerPC.java | 19 ++- .../shimmerresearch/hil/HilShimmerPCTest.java | 50 ++++++++ 4 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index 00998a855..0b7b49c01 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -76,6 +76,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.ObjectInputStream; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -200,8 +201,12 @@ public String toString() { private boolean mUseLegacyDelayToDelayForResponse = false; - ArrayBlockingQueue mABQPacketByeArray = new ArrayBlockingQueue(10000); - public List mListofPCTimeStamps = new ArrayList(); + // The connection's queues are transient: the IO thread and the timers change them while a + // connected device is serialised by deepClone(), which threw ConcurrentModificationException + // (DEV-1310). readObject() gives a copy empty ones. + private static final int PACKET_QUEUE_CAPACITY = 10000; + transient ArrayBlockingQueue mABQPacketByeArray = new ArrayBlockingQueue(PACKET_QUEUE_CAPACITY); + public transient List mListofPCTimeStamps = new ArrayList(); protected boolean mIamAlive = false; protected abstract void connect(String address,String bluetoothLibrary); @@ -233,7 +238,7 @@ public String toString() { private SystemTimestampPlot systemTimestampPlot = new SystemTimestampPlot(); - private List mListofInstructions = new ArrayList(); + private transient List mListofInstructions = new ArrayList(); private final int ACK_TIMER_DURATION = 2; // Duration to wait for an ack packet (seconds) protected boolean mDummyRead = false, mDummyReadStarted = false; @Deprecated @@ -6072,6 +6077,18 @@ public void enableCheckifAlive(boolean set){ //endregion + /** + * A copy made by serialisation (deepClone()) is of the configuration, not of the connection, + * so it starts with none of the connection's queues. They are transient (DEV-1310), and a + * deserialised object runs no field initialisers. + */ + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + mABQPacketByeArray = new ArrayBlockingQueue(PACKET_QUEUE_CAPACITY); + mListofPCTimeStamps = new ArrayList(); + mListofInstructions = new ArrayList(); + } + /** * @return the mListofInstructions */ diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java new file mode 100644 index 000000000..5bc4ea94f --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java @@ -0,0 +1,121 @@ +package com.shimmerresearch.bluetooth; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.After; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.pcDriver.ShimmerPC; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationSimulatorS3; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; + +/** + * DEV-1310: ShimmerPC.deepClone() copies a device by serialising it, and Consensys's + * Bluetooth manager configures a connected device through such a copy. Serialisation + * takes no lock, so on a connected device the timers and the IO thread went on changing + * its instruction list while it was copied: the status timer queues GET_STATUS every + * 5 s, the battery timer queues a battery read, and the IO thread removes each + * instruction it sends. A change mid-copy threw ConcurrentModificationException out of + * deepClone(), and the configure failed. The Java HIL saw it about once in 30 configures. + *

+ * A clone is a copy of the configuration, not of the connection, so it now starts with + * none of the connection's queues. + */ +public class API_00034_DeepCloneWhileBusyTest { + + private static final int CLONES = 200; + + private ShimmerPC mDevice; + + @After + public void tearDown() { + if(mDevice!=null){ + mDevice.disconnectNoException(); + } + } + + /** + * Another thread queues an instruction the way a timer does (readBattery() takes the + * device, then the list) and retires one the way the IO thread does, while this one + * copies the device. A long queue keeps the copy of the list busy for long enough that + * nearly every clone overlaps a change. + */ + @Test + public void clonesWhileATimerQueuesInstructions() throws Exception { + ShimmerPC device = new ShimmerPC("COM99"); + for(int i=0;i<2000;i++){ + device.readBattery(); + } + AtomicBoolean stop = new AtomicBoolean(); + Thread timer = daemon("timer", () -> { + while(!stop.get()){ + device.readBattery(); + device.removeInstruction(0); + } + }); + timer.start(); + try { + for(int i=1;i<=CLONES;i++){ + ShimmerPC clone = device.deepClone(); + assertNotNull("clone " + i, clone); + assertTrue("clone " + i + " carries no instructions", clone.getListofInstructions().isEmpty()); + } + } finally { + stop.set(true); + timer.join(5000); + } + assertFalse("the queue was not copied away from the device", device.getListofInstructions().isEmpty()); + } + + /** + * The product as it runs: a device connected to the simulator with its status timer on, + * an app thread reading the battery and status as Consensys does, and clones taken all + * the while. Each clone must be usable to configure from. + */ + @Test + public void clonesAConnectedDeviceWhileItWorksThroughItsQueue() throws Exception { + mDevice = new ShimmerPC("COM99"); + mDevice.setTestRadio(new ByteCommunicationSimulatorS3("COM99")); + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + assertEquals(BT_STATE.CONNECTED, mDevice.getBluetoothRadioState()); + + AtomicBoolean stop = new AtomicBoolean(); + AtomicReference appFailure = new AtomicReference<>(); + Thread app = daemon("app", () -> { + try { + while(!stop.get()){ + if(mDevice.getListofInstructions().size()<8){ + mDevice.readBattery(); + mDevice.readStatusLogAndStream(); + } + Thread.yield(); + } + } catch (Throwable t) { + appFailure.set(t); + } + }); + app.start(); + try { + for(int i=1;i<=CLONES;i++){ + ShimmerPC clone = mDevice.deepClone(); + assertNotNull("clone " + i, clone); + assertTrue("clone " + i + " carries no instructions", clone.getListofInstructions().isEmpty()); + assertEquals("clone " + i + " has the device's rate", mDevice.getSamplingRateShimmer(), clone.getSamplingRateShimmer(), 0); + } + } finally { + stop.set(true); + app.join(5000); + } + assertNull("the app thread failed", appFailure.get()); + } + + private Thread daemon(String name, Runnable runnable) { + Thread thread = new Thread(runnable, name); + thread.setDaemon(true); + return thread; + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java index e18b6cdec..8dfa48413 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java @@ -1,5 +1,7 @@ package com.shimmerresearch.hil; +import java.io.IOException; +import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -23,18 +25,29 @@ class HilShimmerPC extends ShimmerPC { private static final int MAX_LOG_LINES = 50000; private final long t0; - private final Map inStreamReplies = new TreeMap<>(); - private final List log = new ArrayList<>(); + // What the session counted is transient: the driver's threads add to it all the time, and + // configureAccelOnly copies this device by serialising it (deepClone()), which threw + // ConcurrentModificationException when a line was logged mid-copy (DEV-1310). + private transient Map inStreamReplies = new TreeMap<>(); + private transient List log = new ArrayList<>(); private volatile int statusesDecoded; private volatile int batteryReadings; /** When each battery reply was decoded, in ms on the session's clock (HilSession.nowMs()). */ - private final List batteryReplyTimesMs = new ArrayList<>(); + private transient List batteryReplyTimesMs = new ArrayList<>(); HilShimmerPC(String comPort, long t0Nanos) { super(comPort); this.t0 = t0Nanos; } + /** A copy starts counting from nothing; a deserialised object runs no field initialisers. */ + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + inStreamReplies = new TreeMap<>(); + log = new ArrayList<>(); + batteryReplyTimesMs = new ArrayList<>(); + } + @Override protected void processInstreamResponse(byte inStreamResponseCommand, boolean shouldClearCrcFromBuffer) { synchronized (inStreamReplies) { diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java new file mode 100644 index 000000000..171946892 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java @@ -0,0 +1,50 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Test; + +import com.shimmerresearch.pcDriver.ShimmerPC; + +/** + * configureAccelOnly copies the device the way Consensys does (deepClone()) while the driver's + * threads keep logging into it. The copy must not trip over the log (DEV-1310). + */ +public class HilShimmerPCTest { + + @Test + public void clonesWhileTheDriverLogs() throws Exception { + HilShimmerPC device = new HilShimmerPC("COM99", System.nanoTime()); + for (int i = 0; i < 5000; i++) { + device.consolePrintLn("line " + i); + } + AtomicBoolean stop = new AtomicBoolean(); + // A line a millisecond or so: often enough to land in most copies, slowly enough to stay + // under the log's cap (MAX_LOG_LINES), past which it no longer changes + Thread driver = new Thread(() -> { + while (!stop.get()) { + device.consolePrintLn("GET_STATUS_COMMAND Instruction added"); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + return; + } + } + }, "driver"); + driver.setDaemon(true); + driver.start(); + try { + for (int i = 1; i <= 100; i++) { + ShimmerPC clone = device.deepClone(); + assertNotNull("clone " + i, clone); + assertEquals("clone " + i + " starts its own log", 0, ((HilShimmerPC) clone).logSize()); + } + } finally { + stop.set(true); + driver.join(5000); + } + } +} From 78ad1e3f93d0529d5d148f281f62f58a618d91ab Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 11:45:18 +0800 Subject: [PATCH 2/2] DEV-1310: test the streaming queues a clone no longer copies On a Shimmer3 streaming at 256 Hz, about one copy in a hundred threw ConcurrentModificationException in mListofPCTimeStamps, which the IO thread changes for every packet. And the processing thread's packet queue holds packets that can't be serialised, so deepClone() returned null whenever one was waiting. Both tests fail on master. Co-Authored-By: Claude Opus 5.5 --- .../API_00034_DeepCloneWhileBusyTest.java | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java index 5bc4ea94f..678bce6a1 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java @@ -9,6 +9,7 @@ import org.junit.Test; import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.driver.ShimmerObject; import com.shimmerresearch.pcDriver.ShimmerPC; import com.shimmerresearch.shimmer3.communication.ByteCommunicationSimulatorS3; import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; @@ -20,7 +21,8 @@ * its instruction list while it was copied: the status timer queues GET_STATUS every * 5 s, the battery timer queues a battery read, and the IO thread removes each * instruction it sends. A change mid-copy threw ConcurrentModificationException out of - * deepClone(), and the configure failed. The Java HIL saw it about once in 30 configures. + * deepClone(), and the configure failed. While streaming the same happened with the PC + * timestamp list, and a packet waiting for the processing thread made deepClone() return null. *

* A clone is a copy of the configuration, not of the connection, so it now starts with * none of the connection's queues. @@ -71,6 +73,52 @@ public void clonesWhileATimerQueuesInstructions() throws Exception { assertFalse("the queue was not copied away from the device", device.getListofInstructions().isEmpty()); } + /** + * While streaming, the IO thread notes the PC time as each packet starts and drops it once the + * packet is parsed. On a Shimmer3 streaming at 256 Hz that raced about one clone in a hundred. + */ + @Test + public void clonesWhileTheIoThreadTimestampsPackets() throws Exception { + ShimmerPC device = new ShimmerPC("COM99"); + for(int i=0;i<2000;i++){ + device.mListofPCTimeStamps.add(System.currentTimeMillis()); + } + AtomicBoolean stop = new AtomicBoolean(); + Thread ioThread = daemon("IO thread", () -> { + while(!stop.get()){ + device.mListofPCTimeStamps.add(System.currentTimeMillis()); + device.mListofPCTimeStamps.remove(0); + } + }); + ioThread.start(); + try { + for(int i=1;i<=CLONES;i++){ + ShimmerPC clone = device.deepClone(); + assertNotNull("clone " + i, clone); + assertTrue("clone " + i + " carries no PC timestamps", clone.mListofPCTimeStamps.isEmpty()); + } + } finally { + stop.set(true); + ioThread.join(5000); + } + } + + /** + * With the processing thread on, parsed packets wait in a queue for it. Its packets can't be + * serialised, so deepClone() returned null whenever one was waiting. + */ + @Test + public void clonesWithPacketsWaitingForTheProcessingThread() { + ShimmerPC device = new ShimmerPC("COM99"); + // The queue is package-private to ShimmerBluetooth, so reach it through that type + ShimmerBluetooth original = device; + original.mABQPacketByeArray.add(new RawBytePacketWithPCTimeStamp(new byte[] {ShimmerObject.DATA_PACKET, 0x01, 0x02}, System.currentTimeMillis())); + ShimmerBluetooth clone = device.deepClone(); + assertNotNull("a device with a packet waiting can be copied", clone); + assertTrue("the copy has no packets waiting", clone.mABQPacketByeArray.isEmpty()); + assertEquals("the device keeps its packet", 1, original.mABQPacketByeArray.size()); + } + /** * The product as it runs: a device connected to the simulator with its status timer on, * an app thread reading the battery and status as Consensys does, and clones taken all