From 3340f9bd5b00ec1d1f7e49448e1368d6b45d1b34 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 10:33:18 +0000 Subject: [PATCH 01/41] DEV-895 Stop per-device reader threads from busy-spinning when idle ShimmerBluetooth.IOThread and LiteProtocol.IOThread ran tight while(!stop) loops with no sleep, calling processNextInstruction()/bytesAvailableToBeRead() as fast as possible, pegging a CPU core per connected device even when idle. Track whether an iteration did useful work (instruction processed, actively streaming, or bytes read) and only sleep briefly (2ms) when it didn't - latency of the active/streaming paths is unchanged since sleep is skipped whenever there's real work to do. ShimmerBluetooth.ProcessingThread spun on mABQPacketByeArray.isEmpty(); since the queue is an ArrayBlockingQueue, swap to a blocking poll(100ms) instead so the thread parks while idle rather than busy-checking emptiness. Also replace two teardown busy-waits with bounded Thread.join() calls so closing a connection doesn't spin a core while waiting for the reader threads to exit: ShimmerPC.closeConnection() (mIOThread + mPThread) and LiteProtocol.stopIoThread() (mIOThread). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JjgNkS6wtPkQUdK3KdHVdN --- .../bluetooth/ShimmerBluetooth.java | 60 +++++++++++++------ .../comms/radioProtocol/LiteProtocol.java | 51 ++++++++++++---- .../shimmerresearch/pcDriver/ShimmerPC.java | 16 ++++- 3 files changed, 94 insertions(+), 33 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index db2beb4d1..b3e1ae8f9 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -581,19 +581,26 @@ public class ProcessingThread extends Thread { int count=0; public void run() { while (!stop) { - if(!mABQPacketByeArray.isEmpty()){ + RawBytePacketWithPCTimeStamp rbp = null; + try { + // Blocks (with a timeout) instead of busy-spinning on isEmpty() while idle. + // The timeout ensures the stop flag is still checked periodically. + rbp = mABQPacketByeArray.poll(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if(rbp!=null){ count++; if(count%1000==0){ consolePrintLn("Queue Size: " + mABQPacketByeArray.size()); printLogDataForDebugging("Queue Size: " + mABQPacketByeArray.size() + "\n"); } - RawBytePacketWithPCTimeStamp rbp = mABQPacketByeArray.remove(); // buildAndSendMsg(rbp.mDataArray, FW_TYPE_BT, false, rbp.mSystemTimeStamp); buildAndSendMsg(rbp.mDataArray, COMMUNICATION_TYPE.BLUETOOTH, false, rbp.mSystemTimeStamp); } - } - } - } + } + } + } //region --------- BLUETOOH STACK --------- @@ -603,9 +610,11 @@ public class IOThread extends Thread { public void run() { while(!stop) { + boolean didWork = false; //In-Shimmer Test here if(InShimmerTest) { if (bytesAvailableToBeRead()) { + didWork = true; byte[] data = readBytes(availableBytes()); if (data!=null) { if (data.length>0) { @@ -619,29 +628,43 @@ public void run() { else { if (mDummyRead) { performDummyRead(); + didWork = true; } - + // Process Instruction on stack. is an instruction running? if not proceed if(!isInstructionStackLock()){ - processNextInstruction(); + if(processNextInstruction()){ + didWork = true; + } } - + if(mIsStreaming){ processWhileStreaming(); + didWork = true; } else if(bytesAvailableToBeRead()){ + didWork = true; if(mWaitForAck) { processNotStreamingWaitForAck(); - } + } else if(mWaitForResponse) { processNotStreamingWaitForResp(); - } - + } + processBytesAvailableAndInstreamSupported(); - + + } + } + + if(!didWork){ + // Nothing useful happened this iteration (idle, not streaming, no pending + // instructions/bytes) - avoid busy-spinning and pegging a CPU core. + try { + Thread.sleep(2); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } - } } @@ -674,10 +697,11 @@ private void performDummyRead() { mDummyReadStarted = false; } - private void processNextInstruction() { + /** @return true if an instruction was found and processed (i.e. useful work was done) */ + private boolean processNextInstruction() { // check instruction stack, are there any other instructions left to be executed? //checkAndRemoveFirstInstructionIfNull(); - + byte[] insBytes = getInstruction(); if (insBytes!=null){ mCurrentCommand=insBytes[0]; @@ -754,14 +778,16 @@ else if((mCurrentCommand==GET_FW_VERSION_COMMAND) mTransactionCompleted=false; + return true; } else { if (!mIsStreaming && !bytesAvailableToBeRead()){ threadSleep(50); } + return false; } } - - /** Process ACK from a GET or SET command while not streaming */ + + /** Process ACK from a GET or SET command while not streaming */ private void processNotStreamingWaitForAck() { //JC TEST:: IMPORTANT TO REMOVE // This is to simulate packet loss /* diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index dc1634651..04ec411e7 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -255,6 +255,13 @@ private void startIoThread() { private void stopIoThread() { if(mIOThread!=null){ mIOThread.stop=true; + // Bounded wait for the thread to actually exit its loop before dropping the + // reference, without risking blocking indefinitely. + try { + mIOThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } mIOThread = null; } } @@ -266,36 +273,52 @@ public class IOThread extends Thread { public synchronized void run() { while (!stop) { + boolean didWork = false; try { // Process Instruction on stack. is an instruction running? if not proceed if(!isInstructionStackLock()){ - processNextInstruction(); + if(processNextInstruction()){ + didWork = true; + } } - + if(isStreaming()){ processWhileStreaming(); + didWork = true; } else if(bytesAvailableToBeRead()){ + didWork = true; if(mWaitForAck) { processNotStreamingWaitForAck(); - } + } else if(mWaitForResponse) { processNotStreamingWaitForResp(); - } - + } + processBytesAvailableAndInstreamSupported(); } } catch (ShimmerException dE) { // stop=true; - + killConnection(dE); // e.printStackTrace(); //TODO send event up the ladder } - } - } - - private void processNextInstruction() throws ShimmerException { + + if(!didWork){ + // Nothing useful happened this iteration (idle, not streaming, no pending + // instructions/bytes) - avoid busy-spinning and pegging a CPU core. + try { + Thread.sleep(2); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + + /** @return true if an instruction was found and processed (i.e. useful work was done) */ + private boolean processNextInstruction() throws ShimmerException { // check instruction stack, are there any other instructions left to be executed? if(!getListofInstructions().isEmpty()) { if(getListofInstructions().get(0)==null) { @@ -387,16 +410,18 @@ else if((mCurrentCommand==InstructionsGet.GET_FW_VERSION_COMMAND_VALUE) } }*/ } - - + + mTransactionCompleted=false; + return true; } - } + } else { if (!isStreaming() && !bytesAvailableToBeRead()){ threadSleep(50); } } + return false; } private void processWhileStreaming() throws ShimmerException { diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index eda410989..d6e0170fa 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -608,13 +608,23 @@ private void closeConnection(){ if (mIOThread != null) { mIOThread.stop = true; - // Closing serial port before before thread is finished stopping throws an error so waiting here - while(mIOThread != null && mIOThread.isAlive()); + // Closing serial port before before thread is finished stopping throws an error so waiting here. + // Bounded join instead of an empty-body busy-wait, so this can't block indefinitely. + try { + mIOThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } mIOThread = null; - + if(mUseProcessingThread){ mPThread.stop = true; + try { + mPThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } mPThread = null; } } From 050410c4f9b21875f9db51f5e7c0e385c3a2297e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 11:28:42 +0000 Subject: [PATCH 02/41] DEV-895 review fixes: self-join guards, un-synchronize LiteProtocol run(), interrupt-exit loops Review findings on the idle-sleep changeset: - LiteProtocol.IOThread.run() was declared synchronized; Thread.join() locks the same Thread instance, so external stopIoThread() callers blocked on monitor entry with no timeout until the loop exited, defeating the bounded join. run() is no longer synchronized. - stopIoThread()/closeConnection() can be reached synchronously from the reader thread itself (killConnection from IOThread's catch; connectionLost on serial errors), where join() can never succeed and just burns the full 2s timeout. Skip the join when running on that thread. - Reader loops now also exit on thread interrupt: previously an interrupt without stop=true would make poll()/sleep() throw immediately every iteration - a latent busy-spin. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JjgNkS6wtPkQUdK3KdHVdN --- .../bluetooth/ShimmerBluetooth.java | 4 ++-- .../comms/radioProtocol/LiteProtocol.java | 23 +++++++++++++------ .../shimmerresearch/pcDriver/ShimmerPC.java | 13 +++++++---- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index b3e1ae8f9..6df67e805 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -580,7 +580,7 @@ public class ProcessingThread extends Thread { public boolean stop = false; int count=0; public void run() { - while (!stop) { + while (!stop && !Thread.currentThread().isInterrupted()) { RawBytePacketWithPCTimeStamp rbp = null; try { // Blocks (with a timeout) instead of busy-spinning on isEmpty() while idle. @@ -609,7 +609,7 @@ public class IOThread extends Thread { public boolean stop = false; public void run() { - while(!stop) { + while(!stop && !Thread.currentThread().isInterrupted()) { boolean didWork = false; //In-Shimmer Test here if(InShimmerTest) { diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index 04ec411e7..5103754ec 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -256,11 +256,15 @@ private void stopIoThread() { if(mIOThread!=null){ mIOThread.stop=true; // Bounded wait for the thread to actually exit its loop before dropping the - // reference, without risking blocking indefinitely. - try { - mIOThread.join(2000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // reference. Guard against the self-join case: killConnection() can reach here + // synchronously from within IOThread.run()'s catch block (error-driven teardown), + // and a thread joining itself would just burn the full timeout. + if(Thread.currentThread() != mIOThread){ + try { + mIOThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } mIOThread = null; } @@ -271,8 +275,13 @@ public class IOThread extends Thread { byte[] byteBuffer = {0}; public boolean stop = false; - public synchronized void run() { - while (!stop) { + /* + * Deliberately NOT synchronized: Thread.join() locks this same Thread + * instance, so a synchronized run() would block external stopIoThread() + * callers on monitor entry (with no timeout) until the loop exits. + */ + public void run() { + while (!stop && !Thread.currentThread().isInterrupted()) { boolean didWork = false; try { // Process Instruction on stack. is an instruction running? if not proceed diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index d6e0170fa..194a33ba6 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -610,10 +610,15 @@ private void closeConnection(){ // Closing serial port before before thread is finished stopping throws an error so waiting here. // Bounded join instead of an empty-body busy-wait, so this can't block indefinitely. - try { - mIOThread.join(2000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // Skip the join when closeConnection() is reached from the IOThread itself + // (connectionLost() fires on it when the serial port errors mid-read/write) - + // a self-join can never succeed and would just burn the full timeout. + if(Thread.currentThread() != mIOThread){ + try { + mIOThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } mIOThread = null; From d3054979e641664c2cd93db8b5252bffa6208671 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 06:46:46 +0000 Subject: [PATCH 03/41] DEV-895 Avoid stacking idle sleep on processNextInstruction backoff processNextInstruction()'s no-instruction branch already waits 50ms (threadSleep(50)) when idle (no instruction pending, not streaming, no bytes available). The new outer idle-sleep in IOThread.run() added a further Thread.sleep(2) whenever didWork stayed false, so the fully idle case waited 52ms instead of 50ms. Fix: have that branch report true (i.e. "waited") instead of false, so run()'s outer idle-sleep is skipped for the iteration that already paced itself. This is the minimal change - it keeps the pre-existing 50ms idle cadence exactly as before this PR, rather than replacing it with the new 2ms cadence (a bigger behavioural change) or leaving the two sleeps stacked. The 50ms sleep only fires when there is no instruction on the stack at all, so it has no bearing on pacing retries for an in-flight instruction - removing it in favour of the 2ms sleep was rejected because that would make idle polling 25x more frequent than before. Applied identically to ShimmerBluetooth.java and LiteProtocol.java, with matching javadoc updates on processNextInstruction(). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015q29AEJZbwa7aVYeAj61Py --- .../shimmerresearch/bluetooth/ShimmerBluetooth.java | 11 ++++++++++- .../comms/radioProtocol/LiteProtocol.java | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index 044183e85..0dea4087c 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -697,7 +697,12 @@ private void performDummyRead() { mDummyReadStarted = false; } - /** @return true if an instruction was found and processed (i.e. useful work was done) */ + /** + * @return true if an instruction was found and processed (i.e. useful work was done), + * or if this call already idle-paced the loop with its own threadSleep(50) below - + * in that case reporting true here stops run()'s outer idle-sleep from stacking an + * additional wait on top of this one. + */ private boolean processNextInstruction() { // check instruction stack, are there any other instructions left to be executed? //checkAndRemoveFirstInstructionIfNull(); @@ -781,7 +786,11 @@ else if((mCurrentCommand==GET_FW_VERSION_COMMAND) return true; } else { if (!mIsStreaming && !bytesAvailableToBeRead()){ + // No instruction pending and nothing else to do - wait here rather than + // busy-spinning. Report "did work" (true) so the caller's own idle sleep + // in run() doesn't stack an extra wait on top of this one. threadSleep(50); + return true; } return false; } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index 5103754ec..54b6af0ca 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -326,7 +326,12 @@ else if(mWaitForResponse) { } } - /** @return true if an instruction was found and processed (i.e. useful work was done) */ + /** + * @return true if an instruction was found and processed (i.e. useful work was done), + * or if this call already idle-paced the loop with its own threadSleep(50) below - + * in that case reporting true here stops run()'s outer idle-sleep from stacking an + * additional wait on top of this one. + */ private boolean processNextInstruction() throws ShimmerException { // check instruction stack, are there any other instructions left to be executed? if(!getListofInstructions().isEmpty()) { @@ -427,7 +432,11 @@ else if((mCurrentCommand==InstructionsGet.GET_FW_VERSION_COMMAND_VALUE) } else { if (!isStreaming() && !bytesAvailableToBeRead()){ + // No instruction pending and nothing else to do - wait here rather than + // busy-spinning. Report "did work" (true) so the caller's own idle sleep + // in run() doesn't stack an extra wait on top of this one. threadSleep(50); + return true; } } return false; From 30462c83dc9c2f51f2b235d090c7efe0be46165e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 06:59:07 +0000 Subject: [PATCH 04/41] DEV-895 Harden bounded teardown joins: null-safe snapshots, guard ProcessingThread self-join, volatile stop Round-2 review findings on the bounded-join teardown from the previous commit: - ShimmerPC.closeConnection() and ShimmerGRPC.closeConnection() are unsynchronized and can run concurrently (connectionLost() on the IOThread itself vs. a user-triggered disconnect() on another thread). Both dereferenced mIOThread/mPThread after only a non-atomic null check, so a concurrent caller nulling the field out mid-method could turn the passed check into an NPE - swallowed by the outer catch(Exception), skipping purgePort/closePort and leaking the COM port. Fixed by snapshotting mIOThread/mPThread into locals once and operating on the snapshot throughout. - Added the same self-join guard used for mIOThread to mPThread in both classes: buildAndSendMsg() runs on the ProcessingThread and dispatches to app listeners, so a listener calling disconnect() would otherwise self-join and burn the full 2s timeout. - After each bounded join(2000), if the thread is still alive, log a warning and call interrupt() so the thread's own interrupt checks can finish tearing it down, instead of silently closing the port under a still-live reader. Applied to mIOThread/mPThread in ShimmerPC.closeConnection() and ShimmerGRPC.closeConnection(), and to mIOThread in LiteProtocol.stopIoThread(). - ShimmerGRPC.closeConnection() previously did an unbounded join() with no self-join guard; converted to the same bounded snapshot+guard+alive-check pattern, keeping its existing interrupt() calls. - Made the `stop` flags volatile (ShimmerBluetooth.IOThread, ShimmerBluetooth.ProcessingThread, LiteProtocol.IOThread) so a writer on one thread is visibly seen by the reader thread without relying on incidental synchronization elsewhere. - Softened the processNextInstruction() idle-branch comment in ShimmerBluetooth.java: getInstruction() returning null doesn't only mean "no instruction pending" - it also happens right after removing a leading null placeholder, with a real instruction possibly next in the list. No behavioural change, comment only. Re-verified: streaming path in both IOThread.run() loops still never sleeps, every idle path still sleeps (no spin), and closeConnection() still reaches purgePort/closePort on every path. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015q29AEJZbwa7aVYeAj61Py --- .../bluetooth/ShimmerBluetooth.java | 15 ++++-- .../comms/radioProtocol/LiteProtocol.java | 9 +++- .../shimmerresearch/pcDriver/ShimmerGRPC.java | 53 +++++++++++++++---- .../shimmerresearch/pcDriver/ShimmerPC.java | 47 ++++++++++++---- 4 files changed, 99 insertions(+), 25 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index 0dea4087c..d52c1bfad 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -577,7 +577,7 @@ public ShimmerBluetooth(String userAssignedName, double samplingRate, int enable } public class ProcessingThread extends Thread { - public boolean stop = false; + public volatile boolean stop = false; int count=0; public void run() { while (!stop && !Thread.currentThread().isInterrupted()) { @@ -606,7 +606,7 @@ public void run() { public class IOThread extends Thread { protected byte[] byteBuffer = {0}; - public boolean stop = false; + public volatile boolean stop = false; public void run() { while(!stop && !Thread.currentThread().isInterrupted()) { @@ -785,10 +785,15 @@ else if((mCurrentCommand==GET_FW_VERSION_COMMAND) mTransactionCompleted=false; return true; } else { + // getInstruction() returned null here - either the instruction stack is + // genuinely empty, or its leading entry was a null placeholder that + // getInstruction() just removed (a real instruction may still be next in + // the list and will be picked up on the following iteration). Either way, + // if there's also nothing else to do this iteration, wait here rather than + // busy-spinning. if (!mIsStreaming && !bytesAvailableToBeRead()){ - // No instruction pending and nothing else to do - wait here rather than - // busy-spinning. Report "did work" (true) so the caller's own idle sleep - // in run() doesn't stack an extra wait on top of this one. + // Report "did work" (true) so the caller's own idle sleep in run() + // doesn't stack an extra wait on top of this one. threadSleep(50); return true; } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index 54b6af0ca..112f2c97f 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -265,6 +265,13 @@ private void stopIoThread() { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + if (mIOThread.isAlive()) { + // Didn't terminate within the bounded join - interrupt it so the + // thread's own interrupt checks can finish tearing it down, rather + // than silently proceeding under a live reader. + printLogDataForDebugging("Warning: IOThread did not terminate within join timeout; interrupting"); + mIOThread.interrupt(); + } } mIOThread = null; } @@ -273,7 +280,7 @@ private void stopIoThread() { public class IOThread extends Thread { byte[] byteBuffer = {0}; - public boolean stop = false; + public volatile boolean stop = false; /* * Deliberately NOT synchronized: Thread.join() locks this same Thread diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java index f9dc48001..9542f4ee7 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java @@ -541,16 +541,51 @@ public void disconnect() throws ShimmerException { private void closeConnection() { try { - if (mIOThread != null) { - mIOThread.stop = true; - mIOThread.interrupt(); - mIOThread.join(); // Wait until the thread terminates + // closeConnection() is unsynchronized and can be entered concurrently - once from + // connectionLost() running on the IOThread itself, and once from a user-triggered + // disconnect() on another thread. Snapshot the fields into locals before touching + // them so a concurrent caller nulling the field out mid-method can't turn a passed + // null-check into an NPE further down (see ShimmerPC.closeConnection() for the + // same fix). + IOThread ioThread = mIOThread; + if (ioThread != null) { + ioThread.stop = true; + ioThread.interrupt(); + // Skip the join when closeConnection() is reached from the IOThread itself - + // a self-join can never succeed and would just burn the full timeout. + if (Thread.currentThread() != ioThread) { + try { + ioThread.join(2000); // Bounded wait so this can't block indefinitely + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (ioThread.isAlive()) { + // Didn't terminate within the bounded join - interrupt again so the + // thread's own interrupt checks can finish tearing it down, rather + // than silently proceeding to close the port under a live reader. + consolePrintLn("Warning: IOThread did not terminate within join timeout; interrupting"); + ioThread.interrupt(); + } + } mIOThread = null; - + if (mUseProcessingThread) { - mPThread.stop = true; - mPThread.interrupt(); - mPThread.join(); + ProcessingThread pThread = mPThread; + if (pThread != null) { + pThread.stop = true; + pThread.interrupt(); + if (Thread.currentThread() != pThread) { + try { + pThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pThread.isAlive()) { + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout; interrupting"); + pThread.interrupt(); + } + } + } mPThread = null; } } @@ -561,7 +596,7 @@ private void closeConnection() { } catch (Exception ex) { consolePrintException(ex.getMessage(), ex.getStackTrace()); setBluetoothRadioState(BT_STATE.DISCONNECTED); - } + } } //Need to override here because ShimmerDevice class uses a different map diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index 194a33ba6..04a040030 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -605,30 +605,57 @@ protected void connectionLost() { private void closeConnection(){ try { - if (mIOThread != null) { - mIOThread.stop = true; - + // closeConnection() is unsynchronized and can be entered concurrently - + // once from connectionLost() running on the IOThread itself, and once from + // a user-triggered disconnect() on another thread. Snapshot the fields into + // locals before touching them so a concurrent caller nulling the field out + // mid-method can't turn a passed null-check into an NPE further down (which + // would otherwise be swallowed by the catch below and skip purgePort/closePort, + // leaking the COM port). + IOThread ioThread = mIOThread; + if (ioThread != null) { + ioThread.stop = true; + // Closing serial port before before thread is finished stopping throws an error so waiting here. // Bounded join instead of an empty-body busy-wait, so this can't block indefinitely. // Skip the join when closeConnection() is reached from the IOThread itself // (connectionLost() fires on it when the serial port errors mid-read/write) - // a self-join can never succeed and would just burn the full timeout. - if(Thread.currentThread() != mIOThread){ + if(Thread.currentThread() != ioThread){ try { - mIOThread.join(2000); + ioThread.join(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + if (ioThread.isAlive()) { + // Didn't terminate within the bounded join - interrupt it so the + // thread's own interrupt checks can finish tearing it down, rather + // than silently proceeding to close the port under a live reader. + consolePrintLn("Warning: IOThread did not terminate within join timeout; interrupting"); + ioThread.interrupt(); + } } mIOThread = null; if(mUseProcessingThread){ - mPThread.stop = true; - try { - mPThread.join(2000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + ProcessingThread pThread = mPThread; + if (pThread != null) { + pThread.stop = true; + // buildAndSendMsg() runs on the ProcessingThread and dispatches to app + // listeners, so a listener calling disconnect() would self-join here - + // guard the same way as the IOThread above. + if (Thread.currentThread() != pThread) { + try { + pThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pThread.isAlive()) { + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout; interrupting"); + pThread.interrupt(); + } + } } mPThread = null; } From 254f490697cbf26f7e0be10345f6bf752e74ec22 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 07:28:19 +0000 Subject: [PATCH 05/41] DEV-895 Address review comments: interrupt-preserving idle sleep, interrupt before join, tidy-ups Copilot review findings on PR #279: - ShimmerBluetooth.java and LiteProtocol.java: the idle-branch sleep in processNextInstruction() used threadSleep(50), whose shared implementation swallows InterruptedException without restoring the interrupt flag - defeating run()'s !Thread.currentThread().isInterrupted() exit check. Replaced with a local try/Thread.sleep(50)/catch that re-sets the flag via Thread.currentThread().interrupt(). Left every other threadSleep() call in both files untouched. - ShimmerBluetooth.java IOThread.run(): fixed inconsistent indentation in the InShimmerTest branch (didWork assignment and the readBytes block were not nested under the bytesAvailableToBeRead() guard). Indentation only - control flow verified unchanged. - ShimmerPC.java: fixed a duplicated "before before" in a comment. - ShimmerPC.closeConnection(), ShimmerGRPC.closeConnection(), and LiteProtocol.stopIoThread(): interrupt() is now called before the bounded join(2000) (still only in the non-self-join branch) instead of only after a failed join. Since run() now exits on Thread.currentThread().isInterrupted() and any idle Thread.sleep()/ queue poll() wakes immediately on interrupt, this makes shutdown faster and more deterministic. The post-join "still alive" branch now just logs a warning instead of interrupting a second time. - ShimmerGRPC.closeConnection(): moved the pre-existing interrupt() calls (for both mIOThread and mPThread) inside the Thread.currentThread() != t guard - previously they ran unconditionally and would mark the *current* thread interrupted when closeConnection() runs on the IOThread/ProcessingThread itself (the self-close case), which is neither needed nor correct. Safety check for interrupting a thread blocked in a serial read (requested before making the interrupt-before-join change): ShimmerPC reads via ByteCommunicationJSSC, which calls JSSC's SerialPort.readBytes(count, timeout) - a native, timeout-bounded JNI call that is not interruptible and never throws InterruptedException; Thread.interrupt() on a thread blocked there only sets the interrupt flag, observed once the call returns (by timeout or data) at the top of run() or in a subsequent Thread.sleep()/queue.poll(). bytesAvailableToBeRead()/readBytes()/ writeBytes() only catch SerialPortException/SerialPortTimeoutException/ NullPointerException, none of which interrupt() can trigger, so no new connectionLost() -> closeConnection() recursion is introduced. ShimmerGRPC reads via a ThreadSafeByteFifoBuffer whose read() does declare InterruptedException and is already caught locally (printed and treated as "no data"), so it is safe by construction. No path was found where the interrupt flag causes a new failure mode; the change was made in all three teardown sites as requested. Re-verified: both IOThread.run() loops still never sleep on the streaming path, no path can spin without sleeping, and closeConnection() still reaches purgePort/closePort on every path. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015q29AEJZbwa7aVYeAj61Py --- .../bluetooth/ShimmerBluetooth.java | 25 ++++++++++++------- .../comms/radioProtocol/LiteProtocol.java | 23 ++++++++++++----- .../shimmerresearch/pcDriver/ShimmerGRPC.java | 23 +++++++++-------- .../shimmerresearch/pcDriver/ShimmerPC.java | 21 ++++++++++------ 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index d52c1bfad..1bb68ec4f 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -614,16 +614,16 @@ public void run() { //In-Shimmer Test here if(InShimmerTest) { if (bytesAvailableToBeRead()) { - didWork = true; - byte[] data = readBytes(availableBytes()); - if (data!=null) { - if (data.length>0) { - if (mTestByteListener != null) { - mTestByteListener.eventNewBytesReceived(data); - } + didWork = true; + byte[] data = readBytes(availableBytes()); + if (data!=null) { + if (data.length>0) { + if (mTestByteListener != null) { + mTestByteListener.eventNewBytesReceived(data); + } + } } } - } } else { if (mDummyRead) { @@ -794,7 +794,14 @@ else if((mCurrentCommand==GET_FW_VERSION_COMMAND) if (!mIsStreaming && !bytesAvailableToBeRead()){ // Report "did work" (true) so the caller's own idle sleep in run() // doesn't stack an extra wait on top of this one. - threadSleep(50); + // Not threadSleep(): that helper swallows InterruptedException without + // restoring the interrupt flag, which would defeat run()'s + // !Thread.currentThread().isInterrupted() loop-exit check. + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } return true; } return false; diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index 112f2c97f..5b33cbe5c 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -260,17 +260,21 @@ private void stopIoThread() { // synchronously from within IOThread.run()'s catch block (error-driven teardown), // and a thread joining itself would just burn the full timeout. if(Thread.currentThread() != mIOThread){ + // Interrupt before joining: run() now exits promptly on + // Thread.currentThread().isInterrupted(), and any idle Thread.sleep()/ + // queue poll() it's waiting in will wake immediately - making shutdown + // faster and more deterministic than waiting out the bounded join first. + mIOThread.interrupt(); try { mIOThread.join(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (mIOThread.isAlive()) { - // Didn't terminate within the bounded join - interrupt it so the - // thread's own interrupt checks can finish tearing it down, rather - // than silently proceeding under a live reader. - printLogDataForDebugging("Warning: IOThread did not terminate within join timeout; interrupting"); - mIOThread.interrupt(); + // Still not terminated after the interrupt + bounded join - most likely + // blocked in a non-interruptible native serial read. Just warn; a second + // interrupt() here wouldn't unblock it either. + printLogDataForDebugging("Warning: IOThread did not terminate within join timeout; it may be blocked in a non-interruptible read"); } } mIOThread = null; @@ -442,7 +446,14 @@ else if((mCurrentCommand==InstructionsGet.GET_FW_VERSION_COMMAND_VALUE) // No instruction pending and nothing else to do - wait here rather than // busy-spinning. Report "did work" (true) so the caller's own idle sleep // in run() doesn't stack an extra wait on top of this one. - threadSleep(50); + // Not threadSleep(): that helper swallows InterruptedException without + // restoring the interrupt flag, which would defeat run()'s + // !Thread.currentThread().isInterrupted() loop-exit check. + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } return true; } } diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java index 9542f4ee7..9c3bced9f 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java @@ -550,21 +550,24 @@ private void closeConnection() { IOThread ioThread = mIOThread; if (ioThread != null) { ioThread.stop = true; - ioThread.interrupt(); // Skip the join when closeConnection() is reached from the IOThread itself - - // a self-join can never succeed and would just burn the full timeout. + // a self-join can never succeed and would just burn the full timeout. The + // interrupt() call moves inside this guard too: calling it unconditionally + // would mark the *current* thread interrupted in the self-close case. if (Thread.currentThread() != ioThread) { + // Interrupt before joining: run() now exits promptly on + // Thread.currentThread().isInterrupted(), and any idle Thread.sleep() + // it's waiting in will wake immediately - making shutdown faster and + // more deterministic than waiting out the bounded join first. + ioThread.interrupt(); try { ioThread.join(2000); // Bounded wait so this can't block indefinitely } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (ioThread.isAlive()) { - // Didn't terminate within the bounded join - interrupt again so the - // thread's own interrupt checks can finish tearing it down, rather - // than silently proceeding to close the port under a live reader. - consolePrintLn("Warning: IOThread did not terminate within join timeout; interrupting"); - ioThread.interrupt(); + // Still not terminated after the interrupt + bounded join. + consolePrintLn("Warning: IOThread did not terminate within join timeout"); } } mIOThread = null; @@ -573,16 +576,16 @@ private void closeConnection() { ProcessingThread pThread = mPThread; if (pThread != null) { pThread.stop = true; - pThread.interrupt(); if (Thread.currentThread() != pThread) { + // Interrupt before joining - see the IOThread teardown above. + pThread.interrupt(); try { pThread.join(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (pThread.isAlive()) { - consolePrintLn("Warning: ProcessingThread did not terminate within join timeout; interrupting"); - pThread.interrupt(); + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); } } } diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index 04a040030..da872e9fd 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -616,23 +616,27 @@ private void closeConnection(){ if (ioThread != null) { ioThread.stop = true; - // Closing serial port before before thread is finished stopping throws an error so waiting here. + // Closing serial port before thread is finished stopping throws an error so waiting here. // Bounded join instead of an empty-body busy-wait, so this can't block indefinitely. // Skip the join when closeConnection() is reached from the IOThread itself // (connectionLost() fires on it when the serial port errors mid-read/write) - // a self-join can never succeed and would just burn the full timeout. if(Thread.currentThread() != ioThread){ + // Interrupt before joining: run() now exits promptly on + // Thread.currentThread().isInterrupted(), and any idle Thread.sleep() + // it's waiting in will wake immediately - making shutdown faster and + // more deterministic than waiting out the bounded join first. + ioThread.interrupt(); try { ioThread.join(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (ioThread.isAlive()) { - // Didn't terminate within the bounded join - interrupt it so the - // thread's own interrupt checks can finish tearing it down, rather - // than silently proceeding to close the port under a live reader. - consolePrintLn("Warning: IOThread did not terminate within join timeout; interrupting"); - ioThread.interrupt(); + // Still not terminated after the interrupt + bounded join - most + // likely blocked in a non-interruptible native serial read. Just + // warn; a second interrupt() here wouldn't unblock it either. + consolePrintLn("Warning: IOThread did not terminate within join timeout; it may be blocked in a non-interruptible read"); } } @@ -646,14 +650,15 @@ private void closeConnection(){ // listeners, so a listener calling disconnect() would self-join here - // guard the same way as the IOThread above. if (Thread.currentThread() != pThread) { + // Interrupt before joining - see the IOThread teardown above. + pThread.interrupt(); try { pThread.join(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (pThread.isAlive()) { - consolePrintLn("Warning: ProcessingThread did not terminate within join timeout; interrupting"); - pThread.interrupt(); + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); } } } From e36c8919cef1e5867939b9e0713727a1ab5227f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 08:30:27 +0000 Subject: [PATCH 06/41] DEV-895: harden IOThread/ProcessingThread teardown per PR #279 review - LiteProtocol.stopIoThread(): snapshot mIOThread into a local and use it for stop/interrupt/join/isAlive, so a concurrent teardown that nulls the field after the null check can no longer cause an NPE. Only clear the field if it still points at the thread that was stopped, so a newly started IOThread is not clobbered. - ShimmerPC/ShimmerGRPC.closeConnection(): move the ProcessingThread (mPThread) teardown out of the `if (ioThread != null)` block into its own sibling block, so it still runs when another caller has already nulled mIOThread while the ProcessingThread is alive. The logic of each block is otherwise unchanged. The merge of origin/master is in the preceding merge commit. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FkxGgMBTsQEZ36RNU3khML --- .../comms/radioProtocol/LiteProtocol.java | 21 ++++++---- .../shimmerresearch/pcDriver/ShimmerGRPC.java | 35 ++++++++-------- .../shimmerresearch/pcDriver/ShimmerPC.java | 41 ++++++++++--------- 3 files changed, 53 insertions(+), 44 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index 5b33cbe5c..228934875 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -253,31 +253,38 @@ private void startIoThread() { } private void stopIoThread() { - if(mIOThread!=null){ - mIOThread.stop=true; + // Snapshot the field: a concurrent teardown may null mIOThread between the + // null check and the interrupt/join/isAlive calls below. + IOThread ioThread = mIOThread; + if(ioThread!=null){ + ioThread.stop=true; // Bounded wait for the thread to actually exit its loop before dropping the // reference. Guard against the self-join case: killConnection() can reach here // synchronously from within IOThread.run()'s catch block (error-driven teardown), // and a thread joining itself would just burn the full timeout. - if(Thread.currentThread() != mIOThread){ + if(Thread.currentThread() != ioThread){ // Interrupt before joining: run() now exits promptly on // Thread.currentThread().isInterrupted(), and any idle Thread.sleep()/ // queue poll() it's waiting in will wake immediately - making shutdown // faster and more deterministic than waiting out the bounded join first. - mIOThread.interrupt(); + ioThread.interrupt(); try { - mIOThread.join(2000); + ioThread.join(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - if (mIOThread.isAlive()) { + if (ioThread.isAlive()) { // Still not terminated after the interrupt + bounded join - most likely // blocked in a non-interruptible native serial read. Just warn; a second // interrupt() here wouldn't unblock it either. printLogDataForDebugging("Warning: IOThread did not terminate within join timeout; it may be blocked in a non-interruptible read"); } } - mIOThread = null; + // Only clear the field if it still refers to the thread we stopped, so a + // newly started IOThread is not clobbered. + if (mIOThread == ioThread) { + mIOThread = null; + } } } diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java index 9c3bced9f..c146588b2 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java @@ -571,26 +571,27 @@ private void closeConnection() { } } mIOThread = null; - - if (mUseProcessingThread) { - ProcessingThread pThread = mPThread; - if (pThread != null) { - pThread.stop = true; - if (Thread.currentThread() != pThread) { - // Interrupt before joining - see the IOThread teardown above. - pThread.interrupt(); - try { - pThread.join(2000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - if (pThread.isAlive()) { - consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); - } + } + // Not nested under the IOThread block: a concurrent caller may already have + // nulled mIOThread while the ProcessingThread is still running. + if (mUseProcessingThread) { + ProcessingThread pThread = mPThread; + if (pThread != null) { + pThread.stop = true; + if (Thread.currentThread() != pThread) { + // Interrupt before joining - see the IOThread teardown above. + pThread.interrupt(); + try { + pThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pThread.isAlive()) { + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); } } - mPThread = null; } + mPThread = null; } mIsStreaming = false; mIsInitialised = false; diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index da872e9fd..353262f0c 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -641,29 +641,30 @@ private void closeConnection(){ } mIOThread = null; - - if(mUseProcessingThread){ - ProcessingThread pThread = mPThread; - if (pThread != null) { - pThread.stop = true; - // buildAndSendMsg() runs on the ProcessingThread and dispatches to app - // listeners, so a listener calling disconnect() would self-join here - - // guard the same way as the IOThread above. - if (Thread.currentThread() != pThread) { - // Interrupt before joining - see the IOThread teardown above. - pThread.interrupt(); - try { - pThread.join(2000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - if (pThread.isAlive()) { - consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); - } + } + // Not nested under the IOThread block: a concurrent caller may already have + // nulled mIOThread while the ProcessingThread is still running. + if(mUseProcessingThread){ + ProcessingThread pThread = mPThread; + if (pThread != null) { + pThread.stop = true; + // buildAndSendMsg() runs on the ProcessingThread and dispatches to app + // listeners, so a listener calling disconnect() would self-join here - + // guard the same way as the IOThread above. + if (Thread.currentThread() != pThread) { + // Interrupt before joining - see the IOThread teardown above. + pThread.interrupt(); + try { + pThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pThread.isAlive()) { + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); } } - mPThread = null; } + mPThread = null; } mIsStreaming = false; mIsInitialised = false; From 8e9e63f9cac937450baad5cc98220a76bc0abe2b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 09:00:10 +0000 Subject: [PATCH 07/41] DEV-895: keep a user disconnect from turning into CONNECTION_LOST F1: closeConnection() closes the serial port after a bounded 2 s join even if the IOThread is still blocked in a non-interruptible jssc call. When that call then failed on the closed port (or NPE'd on the nulled mByteCommunication), the ShimmerPC I/O helpers called connectionLost(), setting CONNECTION_LOST after a deliberate disconnect, which AbstractConsensysApi answers with an auto-reconnect in NeuroLynQ. The catch sites in bytesAvailableToBeRead/availableBytes/writeBytes/readBytes, and ShimmerGRPC.writeBytes, now skip connectionLost() when running on an IOThread whose stop flag is set. Both closeConnection()s also clear mIOThread/mPThread only if they still hold the thread that was stopped, as LiteProtocol.stopIoThread() already does, so a late closeConnection() cannot clobber a reconnect's new threads. F2: LiteProtocol.IOThread.run() no longer calls killConnection() when stop is already set: the teardown interrupt surfacing from txBytes as ERR_WRITING_DATA caused a spurious error event and a re-entrant disconnect. Test: API_00028_IoThreadTeardownTest (ShimmerDriverPC) pins that idle IOThread/ProcessingThread use <300 ms CPU per second, that disconnect() stops both within 1500 ms, and that a write blocked past the join and then failing on the closed port leaves the device DISCONNECTED (fails without F1). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FkxGgMBTsQEZ36RNU3khML --- .../comms/radioProtocol/LiteProtocol.java | 6 +- .../shimmerresearch/pcDriver/ShimmerGRPC.java | 23 ++- .../shimmerresearch/pcDriver/ShimmerPC.java | 39 +++- .../API_00028_IoThreadTeardownTest.java | 193 ++++++++++++++++++ 4 files changed, 250 insertions(+), 11 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index 228934875..8ab6f3a79 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -327,7 +327,11 @@ else if(mWaitForResponse) { } catch (ShimmerException dE) { // stop=true; - killConnection(dE); + // DEV-895: once stop is set, an error here is the teardown interrupt surfacing + // (e.g. as ERR_WRITING_DATA from txBytes), not a real link failure. + if(!stop){ + killConnection(dE); + } // e.printStackTrace(); //TODO send event up the ladder } diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java index c146588b2..0f251cc73 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java @@ -290,10 +290,21 @@ protected void writeBytes(byte[] data) { // Remote gRPC service is not available, so disconnect Shimmer device sre.printStackTrace(); System.out.println("ERROR: Lost connection to GRPC Server"); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } } } + /** DEV-895: true when running on an IOThread whose stop flag is set, i.e. the failure is the + * expected consequence of closeConnection() tearing down under an IOThread that outlived the + * bounded join. Reporting CONNECTION_LOST then would turn a user disconnect into an + * auto-reconnect downstream. */ + private boolean isOnStoppedIoThread(){ + Thread t = Thread.currentThread(); + return (t instanceof IOThread) && ((IOThread)t).stop; + } + @Override protected void stop() { // TODO Auto-generated method stub @@ -570,7 +581,11 @@ private void closeConnection() { consolePrintLn("Warning: IOThread did not terminate within join timeout"); } } - mIOThread = null; + // Only clear the field if it still refers to the thread we stopped, so a + // late closeConnection() cannot clobber a reconnect's new IOThread. + if (mIOThread == ioThread) { + mIOThread = null; + } } // Not nested under the IOThread block: a concurrent caller may already have // nulled mIOThread while the ProcessingThread is still running. @@ -590,8 +605,10 @@ private void closeConnection() { consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); } } + if (mPThread == pThread) { + mPThread = null; + } } - mPThread = null; } mIsStreaming = false; mIsInitialised = false; diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index 353262f0c..7c5ee004d 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -434,7 +434,9 @@ public boolean bytesAvailableToBeRead() { } catch (SerialPortException | NullPointerException ex) { consolePrintException(ex.getMessage(), ex.getStackTrace()); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } // e.printStackTrace(); } return false; @@ -450,7 +452,9 @@ public int availableBytes(){ } } catch (SerialPortException | NullPointerException ex) { consolePrintException(ex.getMessage(), ex.getStackTrace()); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } return 0; } } @@ -467,7 +471,9 @@ public void writeBytes(byte[] data) { } catch (SerialPortException | NullPointerException ex) { consolePrintLn("Tried to writeBytes but port is closed"); consolePrintException(ex.getMessage(), ex.getStackTrace()); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } } } @@ -491,7 +497,9 @@ protected byte[] readBytes(int numberOfBytes) { } } } catch (SerialPortException | NullPointerException e) { - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } consolePrintLn("Tried to readBytes but serial port error"); consolePrintException(e.getMessage(), e.getStackTrace()); // e.printStackTrace(); @@ -505,7 +513,9 @@ protected byte[] readBytes(int numberOfBytes) { // ShimmerBluetooth if(mBluetoothRadioState==BT_STATE.CONNECTING || mBluetoothRadioState==BT_STATE.CONFIGURING){ - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } } } return null; @@ -595,6 +605,15 @@ public void disconnectNoException() { } } + /** DEV-895: true when running on an IOThread whose stop flag is set, i.e. the failure is the + * expected consequence of closeConnection() closing the port under a reader that outlived the + * bounded join. Reporting CONNECTION_LOST then would turn a user disconnect into an + * auto-reconnect downstream. */ + private boolean isOnStoppedIoThread(){ + Thread t = Thread.currentThread(); + return (t instanceof IOThread) && ((IOThread)t).stop; + } + @Override protected void connectionLost() { disconnectNoException(); @@ -640,7 +659,11 @@ private void closeConnection(){ } } - mIOThread = null; + // Only clear the field if it still refers to the thread we stopped, so a + // late closeConnection() cannot clobber a reconnect's new IOThread. + if (mIOThread == ioThread) { + mIOThread = null; + } } // Not nested under the IOThread block: a concurrent caller may already have // nulled mIOThread while the ProcessingThread is still running. @@ -663,8 +686,10 @@ private void closeConnection(){ consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); } } + if (mPThread == pThread) { + mPThread = null; + } } - mPThread = null; } mIsStreaming = false; mIsInitialised = false; diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java new file mode 100644 index 000000000..18f44c653 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java @@ -0,0 +1,193 @@ +package com.shimmerresearch.bluetooth; + +import static org.junit.Assert.*; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Assume; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.pcDriver.ShimmerPC; +import com.shimmerresearch.shimmer3.communication.ByteCommunication; +import com.shimmerresearch.verisense.communication.ByteCommunicationListener; + +import jssc.SerialPortException; +import jssc.SerialPortTimeoutException; + +/** + * DEV-895: the IOThread and ProcessingThread must sleep rather than spin when + * idle, must stop promptly on disconnect(), and a disconnect() must stay a + * disconnect. closeConnection() only waits a bounded 2 s for the IOThread, so + * one blocked in a non-interruptible serial call outlives it, and the port is + * closed under it. When that call then fails, the I/O helpers used to report + * CONNECTION_LOST, which NeuroLynQ answers with an auto-reconnect. + *

+ * No hardware: the threads are started as connect() starts them, over an + * in-memory radio. It lives in this package because mIOThread and mPThread + * are protected members of ShimmerBluetooth. + */ +public class API_00028_IoThreadTeardownTest { + + /** Thread-safe in-memory radio. Its writes can block, ignoring interrupts, as a native jssc call does. */ + private static class BlockingRadio implements ByteCommunication { + final ConcurrentLinkedDeque rx = new ConcurrentLinkedDeque(); + final long writeBlockMs; + final CountDownLatch writeStarted = new CountDownLatch(1); + volatile boolean open = true; + + BlockingRadio(long writeBlockMs) { + this.writeBlockMs = writeBlockMs; + } + + @Override + public boolean writeBytes(byte[] buffer) throws SerialPortException { + writeStarted.countDown(); + if(writeBlockMs>0){ + long end = System.currentTimeMillis()+writeBlockMs; + boolean interrupted = false; + long left; + while((left = end-System.currentTimeMillis())>0){ + try { + Thread.sleep(left); + } catch (InterruptedException e) { + interrupted = true; + } + } + if(interrupted){ + Thread.currentThread().interrupt(); + } + if(!open){ + throw new SerialPortException("TEST", "writeBytes", "Port closed during write"); + } + } + return true; + } + + @Override + public byte[] readBytes(int byteCount, int timeout) throws SerialPortTimeoutException, SerialPortException { + if(!open){ + throw new SerialPortException("TEST", "readBytes", "Port not opened"); + } + if(rx.size()0){ + Thread.sleep(left); + } + assertEquals("still disconnected, not CONNECTION_LOST", BT_STATE.DISCONNECTED, device.getBluetoothRadioState()); + } +} From 4f3c438ca3911918ddc0661947aece0c96c91485 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 10:39:34 +0000 Subject: [PATCH 08/41] DEV-896: frame-clock-driven live X viewport, 1 px default stroke Live time-axis plots moved their X window from the data thread after every sample (a PlotCustomFeature calling setXAxisRangeBasedOnXDuration(), which also allocated a new RangePolicyFixedViewport 256 times a second). The window's right edge was the newest sample's X, so it advanced in Bluetooth bursts of 10-30 samples, and Chart2D's 50 ms repaint timer showed each burst as a jump - worst at UHD, where a jump covers twice the pixels. BasicPlotManagerPC now has an opt-in frame clock (setLiveViewportFrameClockEnabled): one javax.swing.Timer per plot manager, FRAME_INTERVAL_MS = 16 (33 is the documented fallback if 60 fps is too heavy on the EDT). Each tick, on the EDT, advances the window's right edge by the real time elapsed since the previous tick and repaints the chart directly, so each frame shows exactly one even step. While it runs, Chart2D's own repaint timer is backed off to 250 ms (restored on stop) so it does not add a second, unsynchronised stream of paints. Latency design: the right edge is a playout clock that trails "newest X + time since that sample arrived" by VIEWPORT_LATENCY_MS = 200, which absorbs the bursts and the IOThread -> callback -> plot-thread hops. It is steered toward that target by 5% of the error per tick, each correction capped at 10% of the tick's elapsed time, snaps when more than 1 s off, never runs past the newest sample (a stall freezes the window, as before) and never scrolls backwards. It is deliberately not anchored to System.currentTimeMillis(): System_Timestamp_Plot is the device clock offset once onto the PC epoch at the first packet, so it carries that packet's latency and drifts with the device crystal, and the wall clock can be stepped by NTP; System.nanoTime() plus the newest sample needs no common epoch. Points newer than the right edge are clipped by jchart2d, not drawn into the margin. Lifecycle (DEV-895/DEV-717): the clock starts when live time-axis samples arrive into a showing chart, and stops itself when none has arrived for 2 s, the plot is paused, the chart is not showing, the X axis is not the time axis, or the data was cleared; removeAllSignals() (the frame-close path) stops it immediately. On stop the window is left ending at the newest sample. A stopped Swing Timer is not referenced by the TimerQueue, so it does not pin the frame. The tick takes only the chart monitor, never the list monitors, so the list -> chart lock order is kept. mCurrentXValue is now volatile; the data thread's per-sample cost is a nanoTime() read and a volatile write. setXAxisRangeBasedOnXDuration() and its sampling-rate variant are unchanged for explicit callers (ConsensysGQ's right-aligned plots, PlotEventMarkers); frequency/FFT and value X axes never run the clock. VISIBLE CHANGE: DEFAULT_LINE_THICKNESS is now 1 (was 2), so live traces are thinner. A full paint of 2560 points into a 3600x1800 chart measured 9.2 ms at 2 px against 2.3 ms at 1 px. UtilChart2D's separate constant is left at 2: only QuickPlot uses it, not live traces. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FkxGgMBTsQEZ36RNU3khML --- .../guiUtilities/plot/BasicPlotManagerPC.java | 312 +++++++++++++++++- 1 file changed, 310 insertions(+), 2 deletions(-) diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java index e7b8881a0..9d9357f35 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java @@ -55,6 +55,10 @@ import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.swing.SwingUtilities; + import org.apache.commons.collections.buffer.CircularFifoBuffer; import com.shimmerresearch.driver.FormatCluster; @@ -82,9 +86,16 @@ public class BasicPlotManagerPC extends AbstractPlotManager { public HashMap mMapofHalfWindowSize = new HashMap(); - public static float DEFAULT_LINE_THICKNESS=2; + /** DEV-896: 1 px (was 2). jchart2d draws one drawLine per point with antialiasing off; a 2 px + * stroke makes Java2D rasterise each segment as a filled shape instead of a plain line: one full + * paint of 2560 points into a 3600x1800 chart measured 9.2 ms at 2 px against 2.3 ms at 1 px. + * Live traces are visibly thinner. */ + public static float DEFAULT_LINE_THICKNESS=1; - protected double mCurrentXValue = 0; + /** X value (for the time axis: System_Timestamp_Plot, epoch ms) of the most recently plotted + * sample. Written by the data thread in filterDataAndPlot(); volatile because the live-viewport + * frame clock reads it on the EDT (DEV-896). */ + protected volatile double mCurrentXValue = 0; protected boolean mIsPlotPaused = false; public boolean mIsLegendLabelsPainted = true; public boolean mIsScaleLabelsPainted = true; @@ -412,6 +423,7 @@ public Chart2D getChart(){ * @param chart the Chart to be cleared */ public void removeAllSignals(){ + stopLiveViewportFrameClock(); //DEV-896: also the frame-close path (InternalFrameWithPlotManager.frameClosed()) mCurrentXValue=0; super.removeAllSignals(); if (mChart!=null){ @@ -539,6 +551,10 @@ public void setXAxisRange(double minX,double maxY){ x.setRangePolicy(new RangePolicyFixedViewport(new Range(minX, maxY))); } + /** Anchors the X window's right edge to the newest plotted sample. Allocates a new range policy + * per call, so it is for explicit, occasional callers (window change, pause, static contents). + * Live time-axis plots should use the frame clock instead of calling this per sample - see + * {@link #setLiveViewportFrameClockEnabled(boolean)} (DEV-896). */ public void setXAxisRangeBasedOnXDuration(){ setXAxisRange(mCurrentXValue-(mXAxisTimeDuration*1000), mCurrentXValue); } @@ -553,6 +569,289 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl setXAxisRange(minTime, mCurrentXValue); } + //---------------------- DEV-896: live X viewport frame clock START -----------------------// + // + // Live time-axis plots used to move the X window from the data thread, once per sample: the + // window's right edge was the newest sample's X, so it advanced in whatever bursts the data + // arrived in (Bluetooth delivers 10-30 samples at once), and Chart2D's own 50 ms repaint timer + // then showed those bursts as 20 fps jumps. Here one Swing Timer per plot manager moves the + // window instead, on the EDT, by the real time elapsed since the previous frame, and repaints + // the chart right after, so every frame shows exactly one even step. + // + // The window trails the newest data by VIEWPORT_LATENCY_MS (a playout delay). It is NOT + // anchored to System.currentTimeMillis(): the time axis plots System_Timestamp_Plot, which is + // the device's own clock offset once, at the first packet, onto the PC epoch (see + // SystemTimestampPlot / SensorShimmerClock). So it is epoch ms, but it carries that first + // packet's latency and drifts with the device crystal (20 ppm is 72 ms per hour), and the PC + // clock can be stepped by NTP. Instead the right edge is a playout clock that advances with + // System.nanoTime() and is steered gently toward "newest X + time since it arrived - latency", + // which needs no common epoch, follows drift, and cannot jump with the wall clock. + // + // Threading: the data thread only writes mCurrentXValue and mLastLiveSampleNanos (volatile) + // and, when the clock is not running, posts one start request to the EDT. Everything else runs + // on the EDT. The tick never takes the mListofPropertiestoPlot or mListofTraces monitors; it + // only takes the chart monitor (through jchart2d's property-change and repaint-flag methods), + // so it cannot invert the list -> chart lock order that filterDataAndPlot() relies on. + // + // Lifecycle (DEV-895/DEV-717: an idle app must not tick at 60 fps): the timer runs only while + // live time-axis samples are arriving into a chart that is showing. It stops itself on the + // first tick at which no sample has arrived for FRAME_CLOCK_IDLE_STOP_MS, the plot is paused, + // the chart is not showing (tab switched, frame iconified or closed), the X axis is not the + // time axis, or the data was cleared (mCurrentXValue back to 0, which removeAllSignals() and + // clearAllDataBuffer() do). removeAllSignals() - which InternalFrameWithPlotManager.frameClosed() + // calls - also stops it immediately. A stopped javax.swing.Timer is no longer referenced by + // Swing's TimerQueue, so it cannot pin this manager, its chart or the owning frame. + + /** Frame clock period: one X-window update and one chart repaint per tick, ~60 fps. If 60 fps + * proves too heavy for the EDT at UHD with many plots open, 33 (~30 fps) is the fallback; the + * motion stays time-correct at any rate because each tick advances by the real elapsed time. */ + public static final int FRAME_INTERVAL_MS = 16; + /** Playout delay: the live window's right edge trails "newest sample's X + time since it + * arrived" by this much. It absorbs Bluetooth bursts and the IOThread -> callback deque -> + * plot-thread hops, so data scrolls in evenly instead of in jumps; a sample becomes visible + * roughly this long after it was plotted. */ + public static final double VIEWPORT_LATENCY_MS = 200; + /** The frame clock stops once no live sample has arrived for this long. */ + public static final long FRAME_CLOCK_IDLE_STOP_MS = 2000; + /** Extra history a live trace needs while the frame clock runs. A trace sized to exactly the + * window would leave a blank strip of up to ~VIEWPORT_LATENCY_MS at the window's left edge, + * because the window now trails the newest sample. See getLiveViewportTraceMarginInSeconds(). */ + public static final double LIVE_VIEWPORT_TRACE_MARGIN_MS = 2*VIEWPORT_LATENCY_MS; + /** Beyond this distance from its target the playout clock snaps instead of slewing: on the + * first tick, after a long stall, or when the X values jump (new session, clock reset). */ + private static final double FRAME_CLOCK_RESYNC_MS = 1000; + /** Share of the remaining error the playout clock corrects per tick (~0.3 s time constant). */ + private static final double FRAME_CLOCK_CORRECTION_GAIN = 0.05; + /** Largest correction per tick, as a share of that tick's elapsed time: a frame's X step stays + * within +-10% of real time while the clock converges. */ + private static final double FRAME_CLOCK_MAX_SLEW = 0.1; + /** Chart2D's own repaint timer delay while the frame clock repaints the chart itself. Kept long + * so that timer does not add a second, unsynchronised stream of paints; restored on stop. */ + private static final int FRAME_CLOCK_CHART_PAINT_LATENCY_MS = 250; + /** At most one start request per this interval from the data thread. */ + private static final long FRAME_CLOCK_START_REQUEST_MIN_INTERVAL_NS = 100L*1000*1000; + + private volatile boolean mIsLiveViewportFrameClockEnabled = false; + /** System.nanoTime() when the data thread last moved mCurrentXValue. Data thread writes. */ + private volatile long mLastLiveSampleNanos = 0; + private volatile javax.swing.Timer mFrameClockTimer = null; + /** Written on the EDT only; read by the data thread to decide whether to request a start. */ + private volatile boolean mIsFrameClockRunning = false; + private final AtomicBoolean mIsFrameClockStartPending = new AtomicBoolean(false); + /** Data thread only. */ + private long mLastFrameClockStartRequestNanos = System.nanoTime() - FRAME_CLOCK_START_REQUEST_MIN_INTERVAL_NS; + //EDT-only state below + private double mFrameClockViewportEndX = Double.NaN; + private long mLastFrameClockTickNanos = 0; + private Chart2D mFrameClockChart = null; + private int mFrameClockSavedPaintLatency = -1; + /** Installed on the X axis once, then only its Range is swapped per tick (no per-frame policy + * allocation). Replaced only if something else installed a different policy in between. */ + private RangePolicyFixedViewport mFrameClockRangePolicy = null; + private Range mFrameClockSpareRange = null; + + /** Lets a frame clock, rather than each incoming sample, move this plot's X window while the X + * axis is the time axis. Enable it before signals are added, so callers that size traces from + * {@link #getLiveViewportTraceMarginInSeconds()} see the margin. Non-time X axes (frequency/FFT, + * value) are unaffected: the clock never runs for them. */ + public void setLiveViewportFrameClockEnabled(boolean enabled){ + mIsLiveViewportFrameClockEnabled = enabled; + if(!enabled){ + stopLiveViewportFrameClock(); + } + } + + public boolean isLiveViewportFrameClockEnabled(){ + return mIsLiveViewportFrameClockEnabled; + } + + public boolean isLiveViewportFrameClockRunning(){ + return mIsFrameClockRunning; + } + + /** Extra seconds of history a live trace should hold beyond the X window, 0 when the frame + * clock is disabled. */ + public double getLiveViewportTraceMarginInSeconds(){ + return mIsLiveViewportFrameClockEnabled ? (LIVE_VIEWPORT_TRACE_MARGIN_MS/1000.0) : 0; + } + + /** Stops the frame clock now; it restarts by itself when live samples arrive again. Safe from + * any thread: off the EDT the timer is stopped directly (javax.swing.Timer.stop() is + * thread-safe) and the rest of the teardown, which touches the chart, is posted to the EDT. */ + public void stopLiveViewportFrameClock(){ + if(SwingUtilities.isEventDispatchThread()){ + stopFrameClockOnEdt(false); + } + else{ + javax.swing.Timer timer = mFrameClockTimer; + if(timer!=null){ + timer.stop(); + } + SwingUtilities.invokeLater(new Runnable(){ + @Override + public void run() { + stopFrameClockOnEdt(false); + } + }); + } + } + + /** Data thread, once per sample that moved mCurrentXValue on a time X axis. Cheap on purpose: + * a nanoTime() read, a volatile write and, only while the clock is stopped, a throttled check. */ + private void noteLiveSampleForFrameClock(){ + long nowNanos = System.nanoTime(); + mLastLiveSampleNanos = nowNanos; + if(!mIsFrameClockRunning + && (nowNanos-mLastFrameClockStartRequestNanos) >= FRAME_CLOCK_START_REQUEST_MIN_INTERVAL_NS){ + Chart2D chart = mChart; + //isShowing() off the EDT is a benign racy read: at worst one request is skipped or wasted. + //Checking it here keeps a hidden plot from posting a start request per sample. + if(chart!=null && chart.isShowing() && mIsFrameClockStartPending.compareAndSet(false, true)){ + mLastFrameClockStartRequestNanos = nowNanos; + SwingUtilities.invokeLater(new Runnable(){ + @Override + public void run() { + startFrameClockOnEdt(); + } + }); + } + } + } + + private boolean isFrameClockWanted(){ + return mIsLiveViewportFrameClockEnabled && isXAxisTime() && !mIsPlotPaused && mCurrentXValue!=0; + } + + /** EDT. */ + private void startFrameClockOnEdt(){ + mIsFrameClockStartPending.set(false); + Chart2D chart = mChart; + if(mIsFrameClockRunning || !isFrameClockWanted() || chart==null || !chart.isShowing()){ + return; + } + if(mFrameClockTimer==null){ + javax.swing.Timer timer = new javax.swing.Timer(FRAME_INTERVAL_MS, new java.awt.event.ActionListener(){ + @Override + public void actionPerformed(java.awt.event.ActionEvent e) { + onFrameClockTick(); + } + }); + timer.setCoalesce(true); + mFrameClockTimer = timer; + } + mFrameClockViewportEndX = Double.NaN; + mFrameClockChart = chart; + mFrameClockSavedPaintLatency = chart.getMinPaintLatency(); + chart.setMinPaintLatency(FRAME_CLOCK_CHART_PAINT_LATENCY_MS); + mIsFrameClockRunning = true; + mFrameClockTimer.start(); + } + + /** EDT. + * @param snapToNewestSample leave the window ending at the newest sample, as the old + * sample-driven path did, so nothing received stays hidden behind the playout delay. */ + private void stopFrameClockOnEdt(boolean snapToNewestSample){ + javax.swing.Timer timer = mFrameClockTimer; + if(timer!=null){ + timer.stop(); + } + boolean wasRunning = mIsFrameClockRunning; + mIsFrameClockRunning = false; + mFrameClockViewportEndX = Double.NaN; + Chart2D chart = mFrameClockChart; + mFrameClockChart = null; + if(chart!=null && mFrameClockSavedPaintLatency>0){ + chart.setMinPaintLatency(mFrameClockSavedPaintLatency); + } + mFrameClockSavedPaintLatency = -1; + + double latestX = mCurrentXValue; + if(wasRunning && snapToNewestSample && chart!=null && chart==mChart && latestX!=0){ + applyLiveViewport(chart, latestX-(mXAxisTimeDuration*1000), latestX); + chart.repaint(); + } + } + + /** EDT, once per FRAME_INTERVAL_MS while the clock runs. */ + private void onFrameClockTick(){ + if(!mIsFrameClockRunning){ + return; //an event already queued when the timer was stopped + } + Chart2D chart = mChart; + if(chart==null || chart!=mFrameClockChart || !mIsLiveViewportFrameClockEnabled || !isXAxisTime() || !chart.isShowing()){ + stopFrameClockOnEdt(true); + return; + } + //Read the arrival time before X: the data thread writes X first, so a fresh arrival time + //implies a fresh X. + long lastSampleNanos = mLastLiveSampleNanos; + double latestX = mCurrentXValue; + if(latestX==0){ + stopFrameClockOnEdt(false); //cleared: leave the axis to whoever cleared it + return; + } + long nowNanos = System.nanoTime(); + if(mIsPlotPaused || (nowNanos-lastSampleNanos) > FRAME_CLOCK_IDLE_STOP_MS*1000*1000){ + stopFrameClockOnEdt(true); + return; + } + double end = computeLiveViewportEnd(nowNanos, latestX, lastSampleNanos); + applyLiveViewport(chart, end-(mXAxisTimeDuration*1000), end); + //This tick paints the chart itself, now, so each frame shows exactly one step. Clearing + //the flag stops Chart2D's own timer from scheduling a second paint for the same changes. + chart.setRequestedRepaint(false); + chart.repaint(); + } + + /** EDT (or a test driving it directly). Advances the playout clock to nowNanos and returns the + * X window's new right edge. + * @param latestX X of the newest plotted sample + * @param lastSampleNanos System.nanoTime() when that sample was plotted */ + double computeLiveViewportEnd(long nowNanos, double latestX, long lastSampleNanos){ + double sinceSampleMs = (nowNanos-lastSampleNanos)/1e6; + double target = latestX + sinceSampleMs - VIEWPORT_LATENCY_MS; + double previousEnd = mFrameClockViewportEndX; + double end; + if(Double.isNaN(previousEnd) || Math.abs(target-previousEnd) > FRAME_CLOCK_RESYNC_MS){ + end = Math.min(target, latestX); + } + else{ + double frameMs = Math.max(0, (nowNanos-mLastFrameClockTickNanos)/1e6); + double predicted = previousEnd + frameMs; + double maxCorrection = frameMs*FRAME_CLOCK_MAX_SLEW; + double correction = (target-predicted)*FRAME_CLOCK_CORRECTION_GAIN; + correction = Math.max(-maxCorrection, Math.min(maxCorrection, correction)); + end = predicted + correction; + //Never run ahead of the newest sample (a stall freezes the window, as before), and + //never scroll backwards (with several devices in one plot mCurrentXValue alternates + //between their slightly different timestamps). + end = Math.max(previousEnd, Math.min(end, latestX)); + } + mFrameClockViewportEndX = end; + mLastFrameClockTickNanos = nowNanos; + return end; + } + + /** EDT. Sets the X window without allocating a range policy per frame: the policy installed + * on the axis is kept and only its Range swapped (ARangePolicy.setRange() fires the property + * change that marks the chart for repaint). Protected so a test harness can observe frames. */ + protected void applyLiveViewport(Chart2D chart, double minX, double maxX){ + IAxis axisX = chart.getAxisX(); + if(mFrameClockRangePolicy==null || axisX.getRangePolicy()!=mFrameClockRangePolicy){ + mFrameClockRangePolicy = new RangePolicyFixedViewport(new Range(minX, maxX)); + mFrameClockSpareRange = new Range(minX, maxX); + axisX.setRangePolicy(mFrameClockRangePolicy); + return; + } + Range spare = mFrameClockSpareRange; + mFrameClockSpareRange = mFrameClockRangePolicy.getRange(); + spare.setMin(minX); + spare.setMax(maxX); + mFrameClockRangePolicy.setRange(spare); + } + + //---------------------- DEV-896: live X viewport frame clock END -----------------------// + public void setYAxisRange(double miny,double maxy){ IAxis yAxisLeft = mChart.getAxisY(); yAxisLeft.setRangePolicy(new RangePolicyFixedViewport(new Range(miny, maxy))); @@ -2173,6 +2472,8 @@ public void filterDataAndPlot(ObjectCluster ojc) throws Exception { //DEV-896: stash rather than propagate, so the deferred work still gets replayed on the //"Trace does not exist" path (it used to run inline, before the batching). Exception pendingException = null; + //DEV-896: whether this sample moved mCurrentXValue, i.e. is live data for the frame clock. + boolean isCurrentXValueUpdated = false; try { while (entries.hasNext()) { synchronized(chartMonitor){ @@ -2260,6 +2561,7 @@ public void filterDataAndPlot(ObjectCluster ojc) throws Exception { } mCurrentXValue = xData; + isCurrentXValueUpdated = true; //DEV-896: record instead of printing/updating Swing here - see DeferredPlotAction. //The trace size is read now, under the monitor, so the deferred debug line @@ -2347,6 +2649,12 @@ else if(isXAxisFrequency){ if(isDummyPointAddedToFillTrace) { isFirstPointOnFillTrace = false; } + + //DEV-896: the frame clock, not this per-sample path, moves the X window of a live + //time-axis plot (see setLiveViewportFrameClockEnabled()). + if(isCurrentXValueUpdated && isXAxisTime && mIsLiveViewportFrameClockEnabled){ + noteLiveSampleForFrameClock(); + } } } // mChart.getAxisX().setRange(new Range(mCurrentXValue-(mXAxisTimeDuraton*1000),mCurrentXValue)); From 5aeba04aaf34785d407c74580b1650df429766c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 11:07:56 +0000 Subject: [PATCH 09/41] DEV-896: frame clock review fixes - catch-up, per-plot stroke, multi-device X Follow-up to 2e22633 after review. - Catch-up: below the 1 s snap threshold the playout clock could only speed up by 10% of real time, so a 600-1100 ms stall or a 500 ms forward X jump left the newest data hidden for 3-8 s, and the lag could outgrow the 0.4 s trace margin. The correction is now asymmetric: slowing down and errors under 50 ms keep the 10% cap and 5% gain; from 50 ms to 200 ms behind, the cap ramps linearly to 100% (2x real-time scrolling) and the gain to 15%. Simulated recovery to <260 ms hidden: 300/600/900/1100 ms stalls 68/376/ 668/836 ms, +500/+900 ms X jumps 576/976 ms; no backward steps, no overshoot, max step 32 ms per 16 ms tick. The 1 s snap is unchanged. - Stroke: revert DEFAULT_LINE_THICKNESS to 2, so reports, GQ/NeuroLynQ, PlotEventMarkers, PanelCalibrateShimmer and every other plot keep 2 px. New per-manager setTraceLineThickness(float), defaulting to DEFAULT_LINE_THICKNESS, is used by createNormalTrace() and by setTraceLineStyle() for CONTINUOUS/INDIVIDUAL_POINTS (which previously re-applied the static). The comment now states what jchart2d does (TracePainterPolyline, i.e. Graphics.drawPolyline) and the measured cost of a 2560-point UHD paint: ~0.4 vs ~3.7 ms for 1 vs 2 px at identity transform, ~3.5 vs ~4.0 ms at uiScale 2. - First frame: startFrameClockOnEdt() now runs one tick synchronously, so a chart shown again does not paint its stale window for the first frame. - Multi-device plots: mCurrentXValue alternates between the devices' timestamps, so with a >=1 s offset the snap branch made the window jump back and forth, and with smaller offsets it followed the slower device. The clock now uses mFrameClockNewestX (and the arrival time of that X), kept by the data thread as max(X); it restarts from a lower X only on a genuine backward jump: more than 1 s behind with no sample near the newest for 500 ms (without that confirmation a 1300 ms device offset still flipped it on every sample). Reset to 0 with mCurrentXValue in removeAllSignals() and clearAllDataBuffer(). Idle detection still uses any device's last sample. mCurrentXValue itself is unchanged. Simulated 0-1300 ms offsets: zero backward steps, hidden lag vs the faster device mean 172 ms (144-201); a -5 s jump re-syncs with one snap. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FkxGgMBTsQEZ36RNU3khML --- .../guiUtilities/plot/BasicPlotManagerPC.java | 133 ++++++++++++++---- 1 file changed, 104 insertions(+), 29 deletions(-) diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java index 9d9357f35..69d859fea 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java @@ -86,11 +86,13 @@ public class BasicPlotManagerPC extends AbstractPlotManager { public HashMap mMapofHalfWindowSize = new HashMap(); - /** DEV-896: 1 px (was 2). jchart2d draws one drawLine per point with antialiasing off; a 2 px - * stroke makes Java2D rasterise each segment as a filled shape instead of a plain line: one full - * paint of 2560 points into a 3600x1800 chart measured 9.2 ms at 2 px against 2.3 ms at 1 px. - * Live traces are visibly thinner. */ - public static float DEFAULT_LINE_THICKNESS=1; + /** Default stroke width for continuous traces. A plot manager can override it for the traces it + * creates with {@link #setTraceLineThickness(float)}. */ + public static float DEFAULT_LINE_THICKNESS=2; + + /** DEV-896: stroke width this plot manager gives continuous traces, DEFAULT_LINE_THICKNESS + * unless {@link #setTraceLineThickness(float)} was called. */ + private float mTraceLineThickness = DEFAULT_LINE_THICKNESS; /** X value (for the time axis: System_Timestamp_Plot, epoch ms) of the most recently plotted * sample. Written by the data thread in filterDataAndPlot(); volatile because the live-viewport @@ -357,10 +359,25 @@ private ITrace2D addBarTrace(Chart2D chart, int plotMaxSize) { return trace; } + /** DEV-896: stroke width for continuous traces this plot manager creates from now on, and for + * traces later set to CONTINUOUS/INDIVIDUAL_POINTS via setTraceLineStyle(). Existing traces + * keep their stroke, so call it before signals are added. + *

Cost: jchart2d paints a trace with TracePainterPolyline, i.e. Graphics.drawPolyline, + * antialiasing off. One paint of 2560 points at UHD measured ~0.4 ms at 1 px against ~3.7 ms at + * 2 px at identity transform (the installed DPI-unaware exe, LauncherConsensysInternal's + * uiScale=1.0), and ~3.5 ms against ~4.0 ms at uiScale 2, where even 1 px is scaled. */ + public void setTraceLineThickness(float thickness){ + mTraceLineThickness = thickness; + } + + public float getTraceLineThickness(){ + return mTraceLineThickness; + } + private ITrace2D createNormalTrace(int plotMaxSize) { Trace2DLtd trace = new Trace2DLtdMonotonicX(plotMaxSize); //DEV-896: monotonic-X trace avoids O(n) minX rescans per sample BasicStroke stroke = ((BasicStroke)trace.getStroke()); - BasicStroke newStroke = new BasicStroke(DEFAULT_LINE_THICKNESS,stroke.getEndCap(),stroke.getLineJoin(),stroke.getMiterLimit(),stroke.getDashArray(),stroke.getDashPhase()); + BasicStroke newStroke = new BasicStroke(mTraceLineThickness,stroke.getEndCap(),stroke.getLineJoin(),stroke.getMiterLimit(),stroke.getDashArray(),stroke.getDashPhase()); trace.setStroke(newStroke); return trace; } @@ -425,6 +442,7 @@ public Chart2D getChart(){ public void removeAllSignals(){ stopLiveViewportFrameClock(); //DEV-896: also the frame-close path (InternalFrameWithPlotManager.frameClosed()) mCurrentXValue=0; + mFrameClockNewestX=0; //DEV-896 super.removeAllSignals(); if (mChart!=null){ try { @@ -587,7 +605,7 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl // System.nanoTime() and is steered gently toward "newest X + time since it arrived - latency", // which needs no common epoch, follows drift, and cannot jump with the wall clock. // - // Threading: the data thread only writes mCurrentXValue and mLastLiveSampleNanos (volatile) + // Threading: the data thread only writes mCurrentXValue and the frame-clock volatiles // and, when the clock is not running, posts one start request to the EDT. Everything else runs // on the EDT. The tick never takes the mListofPropertiestoPlot or mListofTraces monitors; it // only takes the chart monitor (through jchart2d's property-change and repaint-flag methods), @@ -597,7 +615,7 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl // live time-axis samples are arriving into a chart that is showing. It stops itself on the // first tick at which no sample has arrived for FRAME_CLOCK_IDLE_STOP_MS, the plot is paused, // the chart is not showing (tab switched, frame iconified or closed), the X axis is not the - // time axis, or the data was cleared (mCurrentXValue back to 0, which removeAllSignals() and + // time axis, or the data was cleared (mFrameClockNewestX back to 0, which removeAllSignals() and // clearAllDataBuffer() do). removeAllSignals() - which InternalFrameWithPlotManager.frameClosed() // calls - also stops it immediately. A stopped javax.swing.Timer is no longer referenced by // Swing's TimerQueue, so it cannot pin this manager, its chart or the owning frame. @@ -622,9 +640,21 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl private static final double FRAME_CLOCK_RESYNC_MS = 1000; /** Share of the remaining error the playout clock corrects per tick (~0.3 s time constant). */ private static final double FRAME_CLOCK_CORRECTION_GAIN = 0.05; - /** Largest correction per tick, as a share of that tick's elapsed time: a frame's X step stays - * within +-10% of real time while the clock converges. */ + /** Largest correction per tick, as a share of that tick's elapsed time, for small errors and + * for slowing down: in normal running a frame's X step stays within +-10% of real time. */ private static final double FRAME_CLOCK_MAX_SLEW = 0.1; + /** Largest speed-up per tick, as a share of the tick's elapsed time, once the window is behind + * its target by FRAME_CLOCK_CATCH_UP_FULL_MS or more (after a stall or a forward X jump below + * the snap threshold): the window then scrolls at up to 2x real time, so it recovers in about + * a second instead of the several seconds the 10% cap would take, and before the lag outgrows + * the trace margin. */ + private static final double FRAME_CLOCK_CATCH_UP_SLEW = 1.0; + /** Behind by less than this, only FRAME_CLOCK_MAX_SLEW applies: covers normal burst jitter. */ + private static final double FRAME_CLOCK_CATCH_UP_START_MS = 50; + /** Behind by this much or more, the full FRAME_CLOCK_CATCH_UP_SLEW applies; linear in between. */ + private static final double FRAME_CLOCK_CATCH_UP_FULL_MS = VIEWPORT_LATENCY_MS; + /** Share of the remaining error corrected per tick while catching up. */ + private static final double FRAME_CLOCK_CATCH_UP_GAIN = 0.15; /** Chart2D's own repaint timer delay while the frame clock repaints the chart itself. Kept long * so that timer does not add a second, unsynchronised stream of paints; restored on stop. */ private static final int FRAME_CLOCK_CHART_PAINT_LATENCY_MS = 250; @@ -632,8 +662,24 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl private static final long FRAME_CLOCK_START_REQUEST_MIN_INTERVAL_NS = 100L*1000*1000; private volatile boolean mIsLiveViewportFrameClockEnabled = false; - /** System.nanoTime() when the data thread last moved mCurrentXValue. Data thread writes. */ + /** System.nanoTime() when the data thread last moved mCurrentXValue (any device). Used for + * idle detection. Data thread writes. */ private volatile long mLastLiveSampleNanos = 0; + /** Newest X seen by the frame clock, 0 = none since the last clear. Unlike mCurrentXValue, + * which alternates between devices' timestamps in a multi-device plot, this only moves + * forward, except on a genuine backward jump (reconnect, device reset): X more than + * FRAME_CLOCK_RESYNC_MS behind it, with nothing closer arriving for + * FRAME_CLOCK_BACKWARD_JUMP_CONFIRM_MS. Data thread writes (before mFrameClockNewestXNanos). */ + private volatile double mFrameClockNewestX = 0; + /** System.nanoTime() when mFrameClockNewestX was last moved. */ + private volatile long mFrameClockNewestXNanos = 0; + /** Data thread only: System.nanoTime() of the last sample within FRAME_CLOCK_RESYNC_MS of + * mFrameClockNewestX. */ + private long mFrameClockLastNearNewestNanos = 0; + /** A sample far behind the newest X only counts as a backward jump once no sample near the + * newest X has arrived for this long. Without it, two devices whose timestamps differ by more + * than FRAME_CLOCK_RESYNC_MS would reset the newest X on every sample of the slower one. */ + private static final long FRAME_CLOCK_BACKWARD_JUMP_CONFIRM_MS = 500; private volatile javax.swing.Timer mFrameClockTimer = null; /** Written on the EDT only; read by the data thread to decide whether to request a start. */ private volatile boolean mIsFrameClockRunning = false; @@ -697,10 +743,12 @@ public void run() { } /** Data thread, once per sample that moved mCurrentXValue on a time X axis. Cheap on purpose: - * a nanoTime() read, a volatile write and, only while the clock is stopped, a throttled check. */ - private void noteLiveSampleForFrameClock(){ + * a nanoTime() read, a few volatile accesses and, only while the clock is stopped, a throttled + * check. */ + private void noteLiveSampleForFrameClock(double xData){ long nowNanos = System.nanoTime(); mLastLiveSampleNanos = nowNanos; + updateFrameClockNewestX(xData, nowNanos); if(!mIsFrameClockRunning && (nowNanos-mLastFrameClockStartRequestNanos) >= FRAME_CLOCK_START_REQUEST_MIN_INTERVAL_NS){ Chart2D chart = mChart; @@ -718,8 +766,24 @@ public void run() { } } + /** Data thread (or a test). Keeps mFrameClockNewestX = max(X seen), restarting from xData on a + * genuine backward jump (see mFrameClockNewestX). */ + void updateFrameClockNewestX(double xData, long nowNanos){ + double newest = mFrameClockNewestX; + boolean isNearNewest = xData>=newest-FRAME_CLOCK_RESYNC_MS; + if(newest==0 || xData>newest + || (!isNearNewest && (nowNanos-mFrameClockLastNearNewestNanos) > FRAME_CLOCK_BACKWARD_JUMP_CONFIRM_MS*1000*1000)){ + mFrameClockNewestX = xData; + mFrameClockNewestXNanos = nowNanos; + mFrameClockLastNearNewestNanos = nowNanos; + } + else if(isNearNewest){ + mFrameClockLastNearNewestNanos = nowNanos; + } + } + private boolean isFrameClockWanted(){ - return mIsLiveViewportFrameClockEnabled && isXAxisTime() && !mIsPlotPaused && mCurrentXValue!=0; + return mIsLiveViewportFrameClockEnabled && isXAxisTime() && !mIsPlotPaused && mFrameClockNewestX!=0; } /** EDT. */ @@ -745,6 +809,9 @@ public void actionPerformed(java.awt.event.ActionEvent e) { chart.setMinPaintLatency(FRAME_CLOCK_CHART_PAINT_LATENCY_MS); mIsFrameClockRunning = true; mFrameClockTimer.start(); + //Apply a current window now rather than one timer interval later, so a chart that was just + //shown again does not paint its stale window for the first frames. + onFrameClockTick(); } /** EDT. @@ -765,7 +832,7 @@ private void stopFrameClockOnEdt(boolean snapToNewestSample){ } mFrameClockSavedPaintLatency = -1; - double latestX = mCurrentXValue; + double latestX = mFrameClockNewestX; if(wasRunning && snapToNewestSample && chart!=null && chart==mChart && latestX!=0){ applyLiveViewport(chart, latestX-(mXAxisTimeDuration*1000), latestX); chart.repaint(); @@ -784,18 +851,18 @@ private void onFrameClockTick(){ } //Read the arrival time before X: the data thread writes X first, so a fresh arrival time //implies a fresh X. - long lastSampleNanos = mLastLiveSampleNanos; - double latestX = mCurrentXValue; - if(latestX==0){ + long newestXNanos = mFrameClockNewestXNanos; + double newestX = mFrameClockNewestX; + if(newestX==0){ stopFrameClockOnEdt(false); //cleared: leave the axis to whoever cleared it return; } long nowNanos = System.nanoTime(); - if(mIsPlotPaused || (nowNanos-lastSampleNanos) > FRAME_CLOCK_IDLE_STOP_MS*1000*1000){ + if(mIsPlotPaused || (nowNanos-mLastLiveSampleNanos) > FRAME_CLOCK_IDLE_STOP_MS*1000*1000){ stopFrameClockOnEdt(true); return; } - double end = computeLiveViewportEnd(nowNanos, latestX, lastSampleNanos); + double end = computeLiveViewportEnd(nowNanos, newestX, newestXNanos); applyLiveViewport(chart, end-(mXAxisTimeDuration*1000), end); //This tick paints the chart itself, now, so each frame shows exactly one step. Clearing //the flag stops Chart2D's own timer from scheduling a second paint for the same changes. @@ -805,8 +872,8 @@ private void onFrameClockTick(){ /** EDT (or a test driving it directly). Advances the playout clock to nowNanos and returns the * X window's new right edge. - * @param latestX X of the newest plotted sample - * @param lastSampleNanos System.nanoTime() when that sample was plotted */ + * @param latestX newest X plotted (mFrameClockNewestX) + * @param lastSampleNanos System.nanoTime() when that X was plotted */ double computeLiveViewportEnd(long nowNanos, double latestX, long lastSampleNanos){ double sinceSampleMs = (nowNanos-lastSampleNanos)/1e6; double target = latestX + sinceSampleMs - VIEWPORT_LATENCY_MS; @@ -818,13 +885,18 @@ private void onFrameClockTick(){ else{ double frameMs = Math.max(0, (nowNanos-mLastFrameClockTickNanos)/1e6); double predicted = previousEnd + frameMs; - double maxCorrection = frameMs*FRAME_CLOCK_MAX_SLEW; - double correction = (target-predicted)*FRAME_CLOCK_CORRECTION_GAIN; - correction = Math.max(-maxCorrection, Math.min(maxCorrection, correction)); + double error = target-predicted; + //Asymmetric: slowing down (error<0) and small speed-ups stay within 10% of real time; + //a window that has fallen well behind may speed up to 2x so it recovers in ~1 s. + double catchUp = Math.max(0, Math.min(1, (error-FRAME_CLOCK_CATCH_UP_START_MS) + /(FRAME_CLOCK_CATCH_UP_FULL_MS-FRAME_CLOCK_CATCH_UP_START_MS))); + double gain = FRAME_CLOCK_CORRECTION_GAIN + catchUp*(FRAME_CLOCK_CATCH_UP_GAIN-FRAME_CLOCK_CORRECTION_GAIN); + double maxSpeedUp = frameMs*(FRAME_CLOCK_MAX_SLEW + catchUp*(FRAME_CLOCK_CATCH_UP_SLEW-FRAME_CLOCK_MAX_SLEW)); + double maxSlowDown = frameMs*FRAME_CLOCK_MAX_SLEW; + double correction = Math.max(-maxSlowDown, Math.min(maxSpeedUp, error*gain)); end = predicted + correction; //Never run ahead of the newest sample (a stall freezes the window, as before), and - //never scroll backwards (with several devices in one plot mCurrentXValue alternates - //between their slightly different timestamps). + //never scroll backwards. end = Math.max(previousEnd, Math.min(end, latestX)); } mFrameClockViewportEndX = end; @@ -1239,7 +1311,7 @@ public void setTraceLineStyle(ITrace2D trace, PLOT_LINE_STYLE selectedLineStyle) || selectedLineStyle==PLOT_LINE_STYLE.INDIVIDUAL_POINTS){ strokeNew = new BasicStroke( // strokeOld.getLineWidth(), - DEFAULT_LINE_THICKNESS, + mTraceLineThickness, //DEV-896: was DEFAULT_LINE_THICKNESS strokeOld.getEndCap(), strokeOld.getLineJoin(), strokeOld.getMiterLimit(), @@ -1819,6 +1891,7 @@ public synchronized void clearAllDataBuffer(){ } setXAxisDuration(mXAxisTimeDuration); mCurrentXValue=0; + mFrameClockNewestX=0; //DEV-896 isFirstPointOnFillTrace=true; } @@ -2474,6 +2547,7 @@ public void filterDataAndPlot(ObjectCluster ojc) throws Exception { Exception pendingException = null; //DEV-896: whether this sample moved mCurrentXValue, i.e. is live data for the frame clock. boolean isCurrentXValueUpdated = false; + double xDataForFrameClock = 0; try { while (entries.hasNext()) { synchronized(chartMonitor){ @@ -2562,6 +2636,7 @@ public void filterDataAndPlot(ObjectCluster ojc) throws Exception { mCurrentXValue = xData; isCurrentXValueUpdated = true; + xDataForFrameClock = xData; //DEV-896: record instead of printing/updating Swing here - see DeferredPlotAction. //The trace size is read now, under the monitor, so the deferred debug line @@ -2653,7 +2728,7 @@ else if(isXAxisFrequency){ //DEV-896: the frame clock, not this per-sample path, moves the X window of a live //time-axis plot (see setLiveViewportFrameClockEnabled()). if(isCurrentXValueUpdated && isXAxisTime && mIsLiveViewportFrameClockEnabled){ - noteLiveSampleForFrameClock(); + noteLiveSampleForFrameClock(xDataForFrameClock); } } } From 03106bfae94edbd08e127bcc805dcd49d92aef0f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 11:47:44 +0000 Subject: [PATCH 10/41] DEV-896: frame clock anchored fallback for playback and sparse data Round-2 review follow-up to 5aeba04. The playout clock assumes X advances ~1 ms per wall-clock ms and that new X arrives well within VIEWPORT_LATENCY_MS. Two real cases break that: - Consensys DB playback runs recorded data through the same live plots; PlaybackSession's timer period is truncated, so X runs ~1.3x at "x1" and 2-4x+ at x2/x4. The 2x catch-up cap could not keep up, the 1 s resync snapped forward 17-82 times per 25 s and a blank strip opened at the left. - Low-rate series (the 2 Hz fusion-response plot, any device < 5 Hz) froze at the newest-sample clamp, then caught up at 2x (40% frozen ticks at 2 Hz). computeLiveViewportEnd() now estimates, on the EDT, the X rate (dNewestX/dArrival) and the spacing between new X values over windows of >= 500 ms of arrivals (so a stall followed by buffered data still reads as ~1x), EWMA-smoothed (alpha 0.5). If the rate is > 1.5 or < 0.67, or the spacing > 1.2 x VIEWPORT_LATENCY_MS, in both the window and the EWMA for two windows in a row, it switches to anchored mode: the right edge is the newest X, as before DEV-896 but still one update and one paint per tick. When anchored for fast X, each tick may advance at most 1.5x the estimated rate, so two playback deliveries landing in one 16 ms frame are spread over the following frames instead of drawing as one double step. It switches back when the rate is within 0.83-1.2 and the spacing < 0.75 x latency for two windows, then holds the window still until the playout target catches up (~110-130 ms) instead of stepping back. The estimate resets when the clock stops. The spacing threshold is 1.2x the latency, not 0.75x: a 5 Hz series still scrolls evenly in playout mode (0% frozen) and would only get jumpier anchored. Sims (25 s each): playback 2.6/3.9/5.2x go from 17/50/82 forward snaps to 0, with lag behind the newest X <= 37 ms (trace margin 400). At 1/2/3/4 Hz the longest frozen run is one sample interval, and the only fast ticks are the anchored jumps themselves. The stall, X-jump and two-device tables from the last round are unchanged. Also: - onFrameClockTick() stops without snapping when the X axis is no longer the time axis, so a value/frequency axis is not given a time window. - The clock also stops when the chart's top-level Frame is iconified (an iconified JFrame's components still report isShowing()), and is not started while it is. - New API_00029_LiveViewportFrameClockTest (Swing-free, synthetic nanoTime): steady stream, stall catch-up, long EDT tick, far-behind second device, genuine backward jump, X rate 4x (no step > 100 ms, lag within the trace margin), 2 Hz data (no frozen run longer than one sample interval). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FkxGgMBTsQEZ36RNU3khML --- .../guiUtilities/plot/BasicPlotManagerPC.java | 161 +++++++++++++++- .../API_00029_LiveViewportFrameClockTest.java | 182 ++++++++++++++++++ 2 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/guiUtilities/plot/API_00029_LiveViewportFrameClockTest.java diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java index 69d859fea..b36fb7505 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java @@ -655,6 +655,42 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl private static final double FRAME_CLOCK_CATCH_UP_FULL_MS = VIEWPORT_LATENCY_MS; /** Share of the remaining error corrected per tick while catching up. */ private static final double FRAME_CLOCK_CATCH_UP_GAIN = 0.15; + + // Anchored fallback. The playout clock assumes X advances about 1 ms per wall-clock ms and + // that new X arrives well within VIEWPORT_LATENCY_MS. Neither holds for Consensys DB playback + // (PlaybackSession replays at ~1.3x at "x1", because its timer period is truncated, and 2-4x+ + // at x2/x4) or for low-rate series (e.g. the 2 Hz fusion-response plot): the clock would then + // snap forward repeatedly, or freeze and rubber-band. So, from the arrivals of the newest X, + // the clock estimates the X rate and the spacing between new X values over windows of at + // least FRAME_CLOCK_RATE_WINDOW_MS, and when either is out of range - in the window itself and + // in its EWMA - for FRAME_CLOCK_MODE_SWITCH_WINDOWS windows in a row it switches to anchored mode: each tick the + // window's right edge is simply the newest X (the old behaviour, but still one update and one + // paint per tick). It switches back, with hysteresis, the same way. + /** Minimum span of newest-X arrivals per rate/spacing estimate. */ + private static final long FRAME_CLOCK_RATE_WINDOW_MS = 500; + /** EWMA weight of each new window's rate and spacing. */ + private static final double FRAME_CLOCK_RATE_EWMA_ALPHA = 0.5; + /** Consecutive windows a mode switch must be wanted for (a one-off X jump or a stall changes + * one window only). */ + private static final int FRAME_CLOCK_MODE_SWITCH_WINDOWS = 2; + /** Anchor when the X rate (X ms per wall-clock ms) is above / below these. */ + private static final double FRAME_CLOCK_ANCHOR_RATE_HIGH = 1.5; + private static final double FRAME_CLOCK_ANCHOR_RATE_LOW = 0.67; + /** Leave anchored mode only when the rate is back between these. */ + private static final double FRAME_CLOCK_UNANCHOR_RATE_HIGH = 1.2; + private static final double FRAME_CLOCK_UNANCHOR_RATE_LOW = 0.83; + /** Anchor when new X values arrive further apart than this. Above VIEWPORT_LATENCY_MS the + * playout clock rubber-bands (freezes at the newest sample, then catches up); at or just below + * it, e.g. a 5 Hz series, it still scrolls evenly, so the threshold is 1.2x the latency rather + * than below it. */ + private static final double FRAME_CLOCK_ANCHOR_SPACING_MS = 1.2*VIEWPORT_LATENCY_MS; + /** Leave anchored mode only when new X values arrive closer together than this. */ + private static final double FRAME_CLOCK_UNANCHOR_SPACING_MS = 0.75*VIEWPORT_LATENCY_MS; + /** Anchored because X runs fast (playback): the right edge may advance at most this many times + * the estimated X rate per tick, so two deliveries landing in one tick (e.g. a 15.6 ms playback + * timer against 16 ms frames) are spread over the next frames instead of showing as one double + * step. Anchored for sparse data at ~1x rate, the edge jumps straight to the newest X. */ + private static final double FRAME_CLOCK_ANCHORED_MAX_SPEED_FACTOR = 1.5; /** Chart2D's own repaint timer delay while the frame clock repaints the chart itself. Kept long * so that timer does not add a second, unsynchronised stream of paints; restored on stop. */ private static final int FRAME_CLOCK_CHART_PAINT_LATENCY_MS = 250; @@ -688,6 +724,19 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl private long mLastFrameClockStartRequestNanos = System.nanoTime() - FRAME_CLOCK_START_REQUEST_MIN_INTERVAL_NS; //EDT-only state below private double mFrameClockViewportEndX = Double.NaN; + /** Anchored fallback state (see FRAME_CLOCK_ANCHOR_*). */ + private boolean mIsFrameClockAnchored = false; + /** Just left anchored mode: hold the window still until the playout target catches up with it, + * instead of stepping back by up to VIEWPORT_LATENCY_MS. */ + private boolean mIsFrameClockHoldingAfterAnchor = false; + private boolean mIsFrameClockRateBaseSet = false; + private double mFrameClockRateBaseX = 0; + private long mFrameClockRateBaseNanos = 0; + private long mFrameClockRateLastSeenNanos = 0; + private int mFrameClockRateUpdatesInWindow = 0; + private double mFrameClockRateEwma = Double.NaN; + private double mFrameClockSpacingEwmaMs = Double.NaN; + private int mFrameClockModeSwitchVotes = 0; private long mLastFrameClockTickNanos = 0; private Chart2D mFrameClockChart = null; private int mFrameClockSavedPaintLatency = -1; @@ -754,7 +803,7 @@ private void noteLiveSampleForFrameClock(double xData){ Chart2D chart = mChart; //isShowing() off the EDT is a benign racy read: at worst one request is skipped or wasted. //Checking it here keeps a hidden plot from posting a start request per sample. - if(chart!=null && chart.isShowing() && mIsFrameClockStartPending.compareAndSet(false, true)){ + if(chart!=null && isChartDisplayed(chart) && mIsFrameClockStartPending.compareAndSet(false, true)){ mLastFrameClockStartRequestNanos = nowNanos; SwingUtilities.invokeLater(new Runnable(){ @Override @@ -782,6 +831,16 @@ else if(isNearNewest){ } } + /** Showing, and its top-level frame not iconified (an iconified JFrame's components still + * report isShowing()). Off the EDT this is a benign racy read. */ + private static boolean isChartDisplayed(Chart2D chart){ + if(!chart.isShowing()){ + return false; + } + java.awt.Window window = SwingUtilities.getWindowAncestor(chart); + return !(window instanceof java.awt.Frame) || (((java.awt.Frame)window).getExtendedState() & java.awt.Frame.ICONIFIED)==0; + } + private boolean isFrameClockWanted(){ return mIsLiveViewportFrameClockEnabled && isXAxisTime() && !mIsPlotPaused && mFrameClockNewestX!=0; } @@ -790,7 +849,7 @@ private boolean isFrameClockWanted(){ private void startFrameClockOnEdt(){ mIsFrameClockStartPending.set(false); Chart2D chart = mChart; - if(mIsFrameClockRunning || !isFrameClockWanted() || chart==null || !chart.isShowing()){ + if(mIsFrameClockRunning || !isFrameClockWanted() || chart==null || !isChartDisplayed(chart)){ return; } if(mFrameClockTimer==null){ @@ -825,6 +884,7 @@ private void stopFrameClockOnEdt(boolean snapToNewestSample){ boolean wasRunning = mIsFrameClockRunning; mIsFrameClockRunning = false; mFrameClockViewportEndX = Double.NaN; + resetFrameClockRateEstimate(); Chart2D chart = mFrameClockChart; mFrameClockChart = null; if(chart!=null && mFrameClockSavedPaintLatency>0){ @@ -845,7 +905,11 @@ private void onFrameClockTick(){ return; //an event already queued when the timer was stopped } Chart2D chart = mChart; - if(chart==null || chart!=mFrameClockChart || !mIsLiveViewportFrameClockEnabled || !isXAxisTime() || !chart.isShowing()){ + if(!isXAxisTime()){ + stopFrameClockOnEdt(false); //do not put a time window on a value/frequency axis + return; + } + if(chart==null || chart!=mFrameClockChart || !mIsLiveViewportFrameClockEnabled || !isChartDisplayed(chart)){ stopFrameClockOnEdt(true); return; } @@ -875,14 +939,30 @@ private void onFrameClockTick(){ * @param latestX newest X plotted (mFrameClockNewestX) * @param lastSampleNanos System.nanoTime() when that X was plotted */ double computeLiveViewportEnd(long nowNanos, double latestX, long lastSampleNanos){ + updateFrameClockRateEstimate(latestX, lastSampleNanos); double sinceSampleMs = (nowNanos-lastSampleNanos)/1e6; double target = latestX + sinceSampleMs - VIEWPORT_LATENCY_MS; double previousEnd = mFrameClockViewportEndX; double end; - if(Double.isNaN(previousEnd) || Math.abs(target-previousEnd) > FRAME_CLOCK_RESYNC_MS){ + if(mIsFrameClockAnchored){ + //X does not advance like wall-clock time, or arrives too sparsely: follow it directly. + end = latestX; + if(!Double.isNaN(previousEnd) && mFrameClockRateEwma>FRAME_CLOCK_UNANCHOR_RATE_HIGH + && latestX>=previousEnd && latestX-previousEnd<=FRAME_CLOCK_RESYNC_MS){ + double frameMs = Math.max(0, (nowNanos-mLastFrameClockTickNanos)/1e6); + end = Math.min(latestX, previousEnd + frameMs*mFrameClockRateEwma*FRAME_CLOCK_ANCHORED_MAX_SPEED_FACTOR); + } + mIsFrameClockHoldingAfterAnchor = true; + } + else if(Double.isNaN(previousEnd) || Math.abs(target-previousEnd) > FRAME_CLOCK_RESYNC_MS){ end = Math.min(target, latestX); + mIsFrameClockHoldingAfterAnchor = false; + } + else if(mIsFrameClockHoldingAfterAnchor && target=FRAME_CLOCK_MODE_SWITCH_WINDOWS){ + mIsFrameClockAnchored = !mIsFrameClockAnchored; + mFrameClockModeSwitchVotes = 0; + } + } + + private static boolean isFrameClockRateAbnormal(double rate, double spacingMs){ + return rate>FRAME_CLOCK_ANCHOR_RATE_HIGH || rateFRAME_CLOCK_ANCHOR_SPACING_MS; + } + + private static boolean isFrameClockRateNormal(double rate, double spacingMs){ + return rate>=FRAME_CLOCK_UNANCHOR_RATE_LOW && rate<=FRAME_CLOCK_UNANCHOR_RATE_HIGH && spacingMs(), 3000, new Chart2D()); + fNewestX = BasicPlotManagerPC.class.getDeclaredField("mFrameClockNewestX"); + fNewestXNanos = BasicPlotManagerPC.class.getDeclaredField("mFrameClockNewestXNanos"); + fNewestX.setAccessible(true); + fNewestXNanos.setAccessible(true); + } + + private static long ns(double ms){ return T0_NS + (long)(ms*1e6); } + + private double newestX() throws Exception { return fNewestX.getDouble(pm); } + + private double tick(double tMs) throws Exception { + return pm.computeLiveViewportEnd(ns(tMs), fNewestX.getDouble(pm), fNewestXNanos.getLong(pm)); + } + + /** 256 Hz, delivered in bursts every burstMs, X = arrival time - 20 ms; deliveries in + * [stallFrom, stallTo) are held back and arrive together at stallTo. */ + private void deliver(double fromMs, double toMs, double burstMs, double stallFrom, double stallTo){ + for(double bt=Math.floor(fromMs/burstMs)*burstMs+burstMs; bt<=toMs; bt+=burstMs){ + if(bt>=stallFrom && bt=stallTo && bt-burstMs2000){ + double step = end-prevEnd; + assertTrue("never backwards", step >= 0); + assertEquals("one tick = one tick of X, within the 10% slew", TICK_MS, step, TICK_MS*0.11); + double hidden = newestX()-end; + assertTrue("never ahead of the newest sample", hidden >= 0); + assertTrue("trails by about VIEWPORT_LATENCY_MS: "+hidden, + hidden <= BasicPlotManagerPC.VIEWPORT_LATENCY_MS + 60); + assertTrue("lag stays inside the trace margin", + hidden <= BasicPlotManagerPC.LIVE_VIEWPORT_TRACE_MARGIN_MS); + } + prevEnd = end; + } + } + + @Test + public void dataStall_freezesWithoutBackwardSteps_thenCatchesUpWithinASecond() throws Exception { + double prevEnd = Double.NaN, recoveredAt = -1; + double stallFrom = 10000, stallTo = 10600; + for(double t=TICK_MS; t<16000; t+=TICK_MS){ + deliver(t-TICK_MS, t, 60, stallFrom, stallTo); + double end = tick(t); + if(!Double.isNaN(prevEnd)){ + assertTrue("never backwards", end >= prevEnd); + assertTrue("at most 2x real time while catching up (no snap below 1 s)", + t < 2000 || end-prevEnd <= 2*TICK_MS + 1e-6); + } + assertTrue("never ahead of the newest sample", end <= newestX()); + if(t>stallTo && recoveredAt<0 && newestX()-end < BasicPlotManagerPC.VIEWPORT_LATENCY_MS+60){ + recoveredAt = t; + } + prevEnd = end; + } + assertTrue("caught up within 1 s of the stall ending: "+recoveredAt, recoveredAt>0 && recoveredAt-stallTo <= 1000); + } + + @Test + public void longEdtStall_stepIsElapsedTimeNotDoubled() throws Exception { + double t = 0, prevT = 0, prevEnd = Double.NaN; + while(t < 12000){ + double next = t + ((t>=10000 && t<10000+TICK_MS) ? 300 : TICK_MS); //one 300 ms EDT hiccup + deliver(t, next, 60, -1, -1); + t = next; + double end = tick(t); + if(t>2000){ + //slew branch: step <= elapsed + min(elapsed, 0.15 * error), error <= FRAME_CLOCK_RESYNC_MS + assertTrue("step "+(end-prevEnd)+" after "+(t-prevT)+" ms", end-prevEnd <= (t-prevT) + 150); + } + prevEnd = end; prevT = t; + } + } + + @Test + public void secondDeviceFarBehind_doesNotPullNewestBack() throws Exception { + for(double t=0; t<5000; t+=10){ + pm.updateFrameClockNewestX(X0+t, ns(t)); + pm.updateFrameClockNewestX(X0+t-1500, ns(t+1)); //device B, 1.5 s behind + assertEquals(X0+t, newestX(), 0); + } + } + + @Test + public void genuineBackwardJump_resetsAfterConfirmation_singleSnap() throws Exception { + double prevEnd = Double.NaN; int backwardSteps = 0; + for(double t=TICK_MS; t<20000; t+=TICK_MS){ + double x = X0 + t - 30 + (t>=10000 ? -60_000 : 0); //device clock reset by a minute + pm.updateFrameClockNewestX(x, ns(t-2)); + double end = tick(t); + if(!Double.isNaN(prevEnd) && end3000){ + double step = end-prevEnd; + assertTrue("never backwards", step >= 0); + assertTrue("no single-tick step > 100 ms: "+step+" at "+t, step <= 100); + double lag = newestX()-end; + assertTrue("lag "+lag+" within the trace margin", lag <= BasicPlotManagerPC.LIVE_VIEWPORT_TRACE_MARGIN_MS); + } + prevEnd = end; + } + } + + /** A 2 Hz series (e.g. the fusion-response plot) must not rubber-band: the window may wait for + * the next sample, but never stays frozen for longer than one sample interval. */ + @Test + public void twoHzData_noFrozenRunLongerThanOneSampleInterval() throws Exception { + double intervalMs = 500, nextSample = 0, prevEnd = Double.NaN, frozenSince = -1, longestFrozen = 0; + for(double t=TICK_MS; t<30000; t+=TICK_MS){ + while(nextSample<=t){ + pm.updateFrameClockNewestX(X0+nextSample-20, ns(nextSample)); + nextSample += intervalMs; + } + double end = tick(t); + if(t>5000){ + assertTrue("never backwards", end >= prevEnd); + if(end==prevEnd){ + if(frozenSince<0) frozenSince = t-TICK_MS; + longestFrozen = Math.max(longestFrozen, t-frozenSince); + } + else{ + frozenSince = -1; + } + } + prevEnd = end; + } + assertTrue("longest frozen run "+longestFrozen+" ms", longestFrozen <= intervalMs); + } +} From b6bf436e08eb7eb50d0a5ed33d8b0aa03f2a5464 Mon Sep 17 00:00:00 2001 From: Mas Azalya Date: Wed, 7 Oct 2026 14:35:30 +0800 Subject: [PATCH 11/41] Wait for initialisation instead of a fixed sleep in byte-communication tests The simulator connect tests called connect() and then slept a fixed 3-5 s before asserting on the device. isConnected() turns true before the post-connect config read (infomem, calibration) runs, so on a loaded CI runner the read could miss the window and assertions such as getTrialDurationEstimatedInSecs() failed - seen in Shimmer-Advance-API run 37567210311, which passed on rerun. Add ByteCommunicationTestUtils.connectAndWaitUntilInitialised(), which waits until isInitialised() and the CONNECTED state (30 s ceiling, fails with a message on timeout), and use it for all 22 connect waits. The TaskCompletionSources it replaces were never completed, so they were only ever sleeps. Also disconnect the device in @After so earlier tests' IO threads and status timers no longer run on into later tests. Co-Authored-By: Claude Opus 5.5 --- .../API_0000X_ByteCommunicationShimmer3.java | 29 ++---- .../API_0000X_ByteCommunicationShimmer3R.java | 95 +++---------------- ...teCommunicationShimmer3_CRC_Streaming.java | 20 ++-- ...X_ByteCommunicationShimmer3_Streaming.java | 20 ++-- ...onShimmer3_Streaming_Contiguous_Check.java | 20 ++-- ...mmer3_Streaming_Contiguous_Check_Fail.java | 20 ++-- ...3_Streaming_Contiguous_Check_Overflow.java | 20 ++-- ...icationShimmer3_Streaming_Throw_Bytes.java | 20 ++-- ...ationShimmer3_Streaming_Throw_Bytes_2.java | 20 ++-- ...X_ByteCommunicationShimmer3lns0_16_11.java | 18 ++-- ...nShimmer3lns0_16_11_w_sd_bt_streaming.java | 18 ++-- ...unicationShimmer3lns0_16_11_w_sd_only.java | 18 ++-- ...ationShimmer3lns0_16_11_w_sync_master.java | 18 ++-- ...cationShimmer3lns0_16_11_w_sync_slave.java | 18 ++-- .../ByteCommunicationTestUtils.java | 37 ++++++++ 15 files changed, 150 insertions(+), 241 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java index 3a4de3a82..ef2a5083c 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -30,7 +31,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3 extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -41,20 +41,16 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); //Just to give time to connect to finish - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); @@ -88,16 +84,7 @@ public void test001_testConnectandDisconnect() { @Test public void test002_testConnectandDisconnect_NewBMPSupported() { mByteCommunicationSimulatorS3.setIsNewBMPSupported(true); - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - boolean result = mWaitTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java index f04fa4111..87369358d 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -49,7 +50,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3R extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; ByteCommunicationSimulatorS3R mByteCommunicationSimulatorS3R; @Before public void setUp() { @@ -58,19 +58,15 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3R); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS);//Just to give time to connect to finish - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); @@ -118,16 +114,7 @@ public void test001_testConnectandDisconnect() { @Test public void test002_ConnectandTestBMP390() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); @@ -159,16 +146,7 @@ public void test002_ConnectandTestBMP390() { @Test public void test003_ConnectandTestCalibParamRead() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); } @@ -214,16 +192,7 @@ public void test003_ConnectandTestCalibParamRead() { @Test public void test004_ConnectandTestDefaultLNAccelAndGyroCalibParam() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); } @@ -266,16 +235,7 @@ public void test004_ConnectandTestDefaultLNAccelAndGyroCalibParam() { @Test public void test005_ConnectandTestDefaultWRAccelCalibParam() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); } @@ -306,16 +266,7 @@ public void test005_ConnectandTestDefaultWRAccelCalibParam() { @Test public void test006_ConnectandTestDefaultMagCalibParam() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); } @@ -335,16 +286,7 @@ public void test006_ConnectandTestDefaultMagCalibParam() { @Test public void test007_ConnectandTestDefaultHighGAccelCalibParam() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); } @@ -363,16 +305,7 @@ public void test007_ConnectandTestDefaultHighGAccelCalibParam() { @Test public void test008_ConnectandTestDefaultAltMagCalibParam() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java index 568d693c2..af7afaf6f 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -29,7 +30,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_CRC_Streaming extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -40,23 +40,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java index fab8ec632..0752fbfe3 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -29,7 +30,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_Streaming extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -40,23 +40,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java index 74bb8a4cb..0c31a771b 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -29,7 +30,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -40,23 +40,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java index 4bdc92711..be362947a 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -30,7 +31,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; public static int CONTIGUOUS_TIMESTAMP_TICKS_LIMIT = (10*32768); @@ -41,23 +41,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java index a4f46feec..42dbba835 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -29,7 +30,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -40,23 +40,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java index 4b89df292..0eeab0c01 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -29,7 +30,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -40,23 +40,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java index a5ed4bd21..6cf3777df 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -29,7 +30,6 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2 extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mCalibrationTask; TaskCompletionSource mStreamingTask; ByteCommunicationSimulatorS3 mByteCommunicationSimulatorS3; @@ -40,23 +40,19 @@ public void setUp() { mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } ArrayList mListOJC; @Test public void test001_testStreaming() { mListOJC = new ArrayList(); mByteCommunicationSimulatorS3.setIsNewBMPSupported(false); - mCalibrationTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mCalibrationTask = new TaskCompletionSource<>(); - mStreamingTask = new TaskCompletionSource(); - try { - boolean result = mCalibrationTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + mStreamingTask = new TaskCompletionSource(); try { mDevice.startStreaming(); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java index 352fb6060..8d2d07628 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,26 +29,21 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3lns0_16_11 extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; @Before public void setUp() { mDevice = new ShimmerPC("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11("COM99")); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java index 26851dbe5..a84b082b3 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,26 +29,21 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; @Before public void setUp() { mDevice = new ShimmerPC("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sd_bt_streaming("COM99")); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(5, TimeUnit.SECONDS); //Just to give time to connect to finish - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java index 37fba3b78..7eb4ec253 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,26 +29,21 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; @Before public void setUp() { mDevice = new ShimmerPC("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sd_only("COM99")); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS);//Just to give time to connect to finish - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java index d849f02df..6b0695dbf 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,26 +29,21 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; @Before public void setUp() { mDevice = new ShimmerPC("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sync_master("COM99")); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS);//Just to give time to connect to finish - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java index 9ef38ea74..2b20f77a4 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java @@ -1,4 +1,5 @@ package com.shimmerresearch.shimmer3.communication; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,26 +29,21 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) // Test methods will be run in alphabetical order public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave extends BasicProcessWithCallBack{ ShimmerPC mDevice; - TaskCompletionSource mWaitTask; @Before public void setUp() { mDevice = new ShimmerPC("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sync_slave("COM99")); setWaitForData(mDevice); } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } @Test public void test001_testConnectandDisconnect() { - mWaitTask = new TaskCompletionSource(); - mDevice.connect("",""); - - mWaitTask = new TaskCompletionSource<>(); - try { - boolean result = mWaitTask.getTask().waitForCompletion(3, TimeUnit.SECONDS);//Just to give time to connect to finish - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); if (!mDevice.isConnected()) { assert(false); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java new file mode 100644 index 000000000..5722ce6bf --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java @@ -0,0 +1,37 @@ +package com.shimmerresearch.shimmer3.communication; + +import static org.junit.Assert.fail; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.pcDriver.ShimmerPC; + +/** Shared helpers for the byte-communication simulator tests. */ +public class ByteCommunicationTestUtils { + + /** Generous on purpose: a connect to the simulator normally finishes in well under a second, + * but a loaded CI runner can stall it for several. Only a genuine hang should hit this. */ + public static final long CONNECT_TIMEOUT_MS = 30000; + + /** + * Connects and blocks until the device has finished its post-connect config read (infomem, + * calibration, etc.), i.e. isInitialised() is set and the state has moved to CONNECTED. + * isConnected() alone is not enough - it turns true before that read has run, so a fixed + * sleep made these tests flaky on slow runners. + */ + public static void connectAndWaitUntilInitialised(ShimmerPC device) { + device.connect("",""); + long deadline = System.currentTimeMillis() + CONNECT_TIMEOUT_MS; + while (!(device.isInitialised() && device.getBluetoothRadioState()==BT_STATE.CONNECTED)) { + if (System.currentTimeMillis() > deadline) { + fail("Device did not finish initialising within " + CONNECT_TIMEOUT_MS + " ms (state: " + device.getBluetoothRadioState() + ")"); + } + try { + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Interrupted while waiting for the device to initialise"); + } + } + } + +} From dd35d042aff4133a0ba69ca15aad0b9fbff788d7 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 8 Oct 2026 11:09:31 +0800 Subject: [PATCH 12/41] DEV-1163 Add ShimmerObject.setLSM6DSVAccelRange Lets the Shimmer3R SD log parser set the low-noise accel (LSM6DSV) range from the file header, as setLSM6DSVGyroRange does for the gyro. mSensorLSM6DSV is private, and ShimmerObject.setAccelRange sets the wide-range accel on Shimmer3R, so there was no way to reach it. Co-Authored-By: Claude Opus 5.5 --- .../java/com/shimmerresearch/driver/ShimmerObject.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java index 26d39129f..2679bc312 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java @@ -9334,6 +9334,13 @@ protected void setLSM6DSVGyroRange(int i){ } } + /** Shimmer3R low-noise accel range: 0 = +/-2g, 1 = +/-4g, 2 = +/-8g, 3 = +/-16g */ + protected void setLSM6DSVAccelRange(int i){ + if(mSensorLSM6DSV!=null){ + mSensorLSM6DSV.setLSM6DSVAccelRange(i); + } + } + /** * @param mMPU9150MPLSamplingRate the mMPU9150MPLSamplingRate to set */ From c09344c4024253f995200151b2db7a0f41afbd38 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 12:11:25 +0800 Subject: [PATCH 13/41] DEV-1115 Replace Process.onExit() so the MATLAB jar runs on Java 8 Process.onExit() is Java 9+. ShimmerJavaClass.jar is built as Java 8 bytecode for MATLAB, and source/target 1.8 compiled this call without complaint, so starting the BLE gRPC server from MATLAB on Java 8 would throw NoSuchMethodError. The jar currently in Shimmer-MATLAB-ID/libs (built 2025-11-24) contains the call. Wait on the process in a daemon thread instead, which logs the same exit message. Found by building the jar with --release 8. Co-Authored-By: Claude Opus 5.5 --- .../grpc/GrpcBLERadioByteTools.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/grpc/GrpcBLERadioByteTools.java b/ShimmerDriver/src/main/java/com/shimmerresearch/grpc/GrpcBLERadioByteTools.java index 254d1ce60..360d54199 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/grpc/GrpcBLERadioByteTools.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/grpc/GrpcBLERadioByteTools.java @@ -153,11 +153,20 @@ public int startServer() throws Exception { e.printStackTrace(); } }); - runningProcess.onExit().thenAccept(p -> { - int code = p.exitValue(); - System.err.println("[BLEGrpcServer] exited: code=" + code + - (code >= 128 ? " (likely signal " + (code - 128) + ")" : "")); - }); + // Not Process.onExit(): that is Java 9+, and MATLAB loads this class on Java 8 via + // ShimmerJavaClass.jar, where it throws NoSuchMethodError. Daemon, like onExit()'s pool thread. + final Process process = runningProcess; + Thread exitWatcher = new Thread(() -> { + try { + int code = process.waitFor(); + System.err.println("[BLEGrpcServer] exited: code=" + code + + (code >= 128 ? " (likely signal " + (code - 128) + ")" : "")); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }, "BLEGrpcServer-exit"); + exitWatcher.setDaemon(true); + exitWatcher.start(); processThread.start(); return port; From 44f9f98c3f01fb0fbf3b4950741077bf49b7cbf9 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 12:11:25 +0800 Subject: [PATCH 14/41] DEV-1115 Add one-click Release Java API workflow Actions -> "Release Java API" releases the version already in the build files (bumped by the usual uprev PR; master is protected): - tests and builds as gradle.yml does - publishes shimmerdriver, shimmerbluetoothmanager and shimmerdriverpc in one gradlew publish, so they cannot drift apart again - tags API_v and creates the GitHub Release with the jars, not marked Latest - builds ShimmerJavaClass.jar as Java 8 bytecode on its own runner and opens a PR replacing Shimmer-MATLAB-ID/libs/ShimmerJavaClass.jar (needs the MATLAB_REPO_TOKEN secret) ShimmerDriverPC gains -PmatlabJava8, which compiles all three projects with --release 8 in place of hand-editing three build files to 1.8. Publishing URLs move from ShimmerEngineering, now a user account, to ShimmerResearch. AGENTS.md documents the release steps. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 291 +++++++++++++++++++++++++++ AGENTS.md | 17 ++ ShimmerBluetoothManager/build.gradle | 2 +- ShimmerDriver/build.gradle | 2 +- ShimmerDriverPC/build.gradle | 20 +- ShimmerLSL/build.gradle | 2 +- ShimmerPCBasicExamples/build.gradle | 2 +- 7 files changed, 325 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..0d8c2778e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,291 @@ +# One-click release of the Java API: GitHub Packages, a GitHub Release, and the MATLAB jar. +# +# How to release +# 1. Merge the usual uprev PR, which bumps the publishing `version` in ShimmerDriver, +# ShimmerBluetoothManager and ShimmerDriverPC build.gradle. master is protected, so this +# workflow cannot commit the bump itself. +# 2. Actions -> "Release Java API" -> Run workflow (on master). +# 3. Review and merge the ShimmerJavaClass.jar PR it opens in Shimmer-MATLAB-ID. +# +# One run does, in order, and creates nothing public until the build and tests have passed: +# build reads the version from the three build.gradle files and fails if they disagree; +# fails if API_v is already tagged; tests ShimmerDriver and builds +# ShimmerDriverPC exactly as gradle.yml does; then publishes shimmerdriver, +# shimmerbluetoothmanager and shimmerdriverpc together with one `gradlew publish` +# (DEV-1115: publishing them one at a time by hand left shimmerbluetoothmanager +# 0.11.8_beta unpublished) +# matlab-jar builds ShimmerJavaClass.jar as Java 8 bytecode (`shadowJar -PmatlabJava8`), on a +# separate runner so its Java 8 classes cannot end up in the published jars +# release tags API_v on the commit it built and creates the GitHub Release, with +# notes from the previous API_v release and the jars attached. It is not marked +# "Latest", which stays with the Consensys releases in this repo. +# update-matlab opens a PR in Shimmer-MATLAB-ID replacing libs/ShimmerJavaClass.jar. Needs the +# MATLAB_REPO_TOKEN secret: a fine-grained token for ShimmerResearch/Shimmer-MATLAB-ID +# with Contents and Pull requests read/write. +# +# A failed job can be re-run on its own with "Re-run failed jobs"; the jobs before it are not +# repeated. A failed publish can leave some of the three versions in GitHub Packages: delete those +# versions under the repo's Packages before re-running, or bump the version. +# +# dry_run runs build and matlab-jar without publishing, from any branch. The workflow also +# dry-runs on any PR that changes this file. + +name: Release Java API + +on: + workflow_dispatch: + inputs: + dry_run: + description: 'Build and test only; publish nothing' + type: boolean + default: false + pull_request: + paths: + - '.github/workflows/release.yml' + +concurrency: + group: release-java-api + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + DRY_RUN: ${{ github.event_name != 'workflow_dispatch' || inputs.dry_run }} + outputs: + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} + previous_tag: ${{ steps.version.outputs.previous_tag }} + steps: + - name: Release from master only + if: env.DRY_RUN != 'true' && github.ref != 'refs/heads/master' + run: | + echo "::error::A release must run on master, not ${GITHUB_REF_NAME}. Tick dry_run to try another branch." + exit 1 + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # tags, to refuse a version that is already released + + - name: Read the version from the build files + id: version + env: + GH_TOKEN: ${{ github.token }} + run: | + VERSION= + for p in ShimmerDriver ShimmerBluetoothManager ShimmerDriverPC; do + v=$(sed -nE "s/^[[:space:]]*version = '([^']+)'.*/\1/p" "$p/build.gradle") + echo "$p: ${v:-}" + if [ "$(printf '%s\n' "$v" | grep -c .)" -ne 1 ]; then + echo "::error file=$p/build.gradle::Expected exactly one publishing version = '...' line" + exit 1 + fi + if [ -n "$VERSION" ] && [ "$v" != "$VERSION" ]; then + echo "::error file=$p/build.gradle::$p is at $v but ShimmerDriver is at $VERSION. The uprev must bump all three." + exit 1 + fi + VERSION=$v + done + TAG="API_v$VERSION" + if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then + echo "::error::$TAG already exists. Bump the version with an uprev PR first." + exit 1 + fi + PREVIOUS_TAG=$(gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("API_v"))] | sort_by(.createdAt) | last | .tagName // ""') + { + echo "version=$VERSION" + echo "tag=$TAG" + echo "previous_tag=$PREVIOUS_TAG" + } >> "$GITHUB_OUTPUT" + echo "Releasing $VERSION as $TAG (notes since ${PREVIOUS_TAG:-the first commit})" + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + # Same pinned action as gradle.yml, and likewise no gradle-version: build through the + # committed wrapper. + - name: Setup Gradle + uses: gradle/actions/setup-gradle@ec92e829475ac0c2315ea8f9eced72db85bb337a # v3.0.0 + + - name: Test ShimmerDriver + working-directory: ShimmerDriver + run: ./gradlew test + + # ShimmerDriverPC includes :ShimmerDriver and :ShimmerBluetoothManager as subprojects. + - name: Build ShimmerDriverPC + working-directory: ShimmerDriverPC + run: ./gradlew build + + # `publish` runs in every project of the ShimmerDriverPC build, so all three artifacts go + # out in one invocation. The build files read these credentials from USERNAME and TOKEN. + - name: Publish to GitHub Packages + if: env.DRY_RUN != 'true' + working-directory: ShimmerDriverPC + env: + USERNAME: ${{ github.actor }} + TOKEN: ${{ github.token }} + run: ./gradlew publish + + - name: Collect the release jars + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + mkdir -p release-assets + cp ShimmerDriver/build/libs/ShimmerDriver.jar "release-assets/shimmerdriver-$VERSION.jar" + cp ShimmerBluetoothManager/build/libs/ShimmerBluetoothManager.jar "release-assets/shimmerbluetoothmanager-$VERSION.jar" + cp ShimmerDriverPC/build/libs/ShimmerDriverPC.jar "release-assets/shimmerdriverpc-$VERSION.jar" + ls -l release-assets + + - uses: actions/upload-artifact@v4 + with: + name: release-jars + path: release-assets/ + if-no-files-found: error + + matlab-jar: + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@ec92e829475ac0c2315ea8f9eced72db85bb337a # v3.0.0 + + - name: Build ShimmerJavaClass.jar for MATLAB (Java 8 bytecode) + working-directory: ShimmerDriverPC + run: ./gradlew shadowJar -PmatlabJava8 + + - name: Check the jar is Java 8 bytecode + run: | + jar=ShimmerDriverPC/build/libs/ShimmerJavaClass.jar + for c in com/shimmerresearch/tools/matlab/ShimmerJavaClass.class \ + com/shimmerresearch/driver/ShimmerObject.class \ + com/shimmerresearch/managers/bluetoothManager/ShimmerBluetoothManager.class; do + major=$(unzip -p "$jar" "$c" | head -c 8 | od -An -tu1 | awk '{print $7 * 256 + $8}') + echo "$c: class file major version $major" + if [ "$major" != 52 ]; then + echo "::error::$c is class version $major, not 52 (Java 8); MATLAB on Java 8 cannot load it." + exit 1 + fi + done + mkdir -p matlab-jar + cp "$jar" matlab-jar/ShimmerJavaClass.jar + ls -l matlab-jar + + - uses: actions/upload-artifact@v4 + with: + name: matlab-jar + path: matlab-jar/ + if-no-files-found: error + + release: + needs: [build, matlab-jar] + if: github.event_name == 'workflow_dispatch' && !inputs.dry_run + runs-on: ubuntu-latest + permissions: + contents: write # create the tag and the release + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.build.outputs.version }} + TAG: ${{ needs.build.outputs.tag }} + PREVIOUS_TAG: ${{ needs.build.outputs.previous_tag }} + steps: + - uses: actions/download-artifact@v4 + with: + name: release-jars + path: release-assets + + - uses: actions/download-artifact@v4 + with: + name: matlab-jar + path: matlab-jar + + - name: Create the tag and GitHub Release + run: | + cp matlab-jar/ShimmerJavaClass.jar "release-assets/ShimmerJavaClass-$VERSION.jar" + gh release create "$TAG" release-assets/* \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "Shimmer Java API Rev $VERSION" \ + --generate-notes ${PREVIOUS_TAG:+--notes-start-tag "$PREVIOUS_TAG"} \ + --latest=false + { + echo "### Released Java API $VERSION" + echo "- Release: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$TAG" + echo "- Packages: com.shimmerresearch:shimmerdriver, shimmerbluetoothmanager and shimmerdriverpc at $VERSION" + echo "- Next: bump the shimmerdriver and shimmerbluetoothmanager pins in ShimmerAndroidAPI to $VERSION, then run its Release workflow." + } >> "$GITHUB_STEP_SUMMARY" + + update-matlab: + needs: [build, release] + runs-on: ubuntu-latest + permissions: {} # writes to Shimmer-MATLAB-ID with MATLAB_REPO_TOKEN, not this repo's token + env: + VERSION: ${{ needs.build.outputs.version }} + TAG: ${{ needs.build.outputs.tag }} + MATLAB_REPO: ShimmerResearch/Shimmer-MATLAB-ID + BRANCH: ShimmerJavaClass_${{ needs.build.outputs.version }} + HAS_TOKEN: ${{ secrets.MATLAB_REPO_TOKEN != '' }} + steps: + - name: Check for MATLAB_REPO_TOKEN + if: env.HAS_TOKEN != 'true' + run: | + echo "::error::Secret MATLAB_REPO_TOKEN is not set, so the Shimmer-MATLAB-ID PR cannot be opened. Add a fine-grained token for $MATLAB_REPO (Contents and Pull requests: read/write) as a repo secret, then re-run this job." + exit 1 + + - uses: actions/checkout@v4 + with: + repository: ${{ env.MATLAB_REPO }} + token: ${{ secrets.MATLAB_REPO_TOKEN }} + path: matlab + + - uses: actions/download-artifact@v4 + with: + name: matlab-jar + path: matlab-jar + + - name: Open the PR updating libs/ShimmerJavaClass.jar + env: + GH_TOKEN: ${{ secrets.MATLAB_REPO_TOKEN }} + working-directory: matlab + run: | + existing=$(gh pr list --repo "$MATLAB_REPO" --head "$BRANCH" --state open --json url --jq '.[0].url // ""') + if [ -n "$existing" ]; then + echo "PR already open: $existing" + echo "- MATLAB PR: $existing" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + cp ../matlab-jar/ShimmerJavaClass.jar libs/ShimmerJavaClass.jar + if git diff --quiet -- libs/ShimmerJavaClass.jar; then + echo "libs/ShimmerJavaClass.jar is already this build; no PR needed." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$BRANCH" + git add libs/ShimmerJavaClass.jar + git commit -m "Update ShimmerJavaClass.jar to Shimmer Java API $VERSION" \ + -m "Built by $GITHUB_REPOSITORY's Release workflow from $TAG ($GITHUB_SHA), as Java 8 bytecode." + git push --force origin "HEAD:refs/heads/$BRANCH" + url=$(gh pr create --repo "$MATLAB_REPO" --base master --head "$BRANCH" \ + --title "Update ShimmerJavaClass.jar to Shimmer Java API $VERSION" \ + --body "Replaces \`libs/ShimmerJavaClass.jar\` with the build from [$TAG]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$TAG) (\`$GITHUB_SHA\`), compiled as Java 8 bytecode with \`gradlew shadowJar -PmatlabJava8\`. + + Opened by the [Release Java API run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID).") + echo "Opened $url" + echo "- MATLAB PR: $url" >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index 5e47628f6..7e26a89c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,23 @@ all: they are legacy Eclipse ADT Android projects. Test results land in `**/build/test-results/test/*.xml`. +## Releasing +Releases are cut by the **Release Java API** workflow (`release.yml`), not from a developer machine: +1. Merge an uprev PR that bumps the publishing `version` in `ShimmerDriver`, `ShimmerBluetoothManager` + and `ShimmerDriverPC` `build.gradle` — all three, to the same value; the workflow refuses a mismatch. +2. Run the workflow on `master`. It tests and builds as CI does, publishes all three artifacts to + GitHub Packages in one `gradlew publish`, tags `API_v`, and creates the GitHub Release + with the jars attached (not marked Latest — that stays with Consensys). +3. It also builds `ShimmerJavaClass.jar` for MATLAB and opens a PR replacing + `Shimmer-MATLAB-ID/libs/ShimmerJavaClass.jar` (needs the `MATLAB_REPO_TOKEN` secret). Review and + merge it. +4. Bump the driver pins in `ShimmerAndroidAPI` to the new version, then run its own Release workflow. + +`ShimmerJavaClass.jar` must be Java 8 bytecode, because MATLAB may load it on Java 8. Build it locally +with `cd ShimmerDriverPC && ./gradlew shadowJar -PmatlabJava8`, which compiles all three projects with +`--release 8` — that also rejects any API newer than Java 8, so `Process.onExit()`, `var` and the like +fail the build here instead of throwing `NoSuchMethodError` inside MATLAB. + ## Subprojects | Project | Role | |---|---| diff --git a/ShimmerBluetoothManager/build.gradle b/ShimmerBluetoothManager/build.gradle index b5ed8183e..72fd006ae 100644 --- a/ShimmerBluetoothManager/build.gradle +++ b/ShimmerBluetoothManager/build.gradle @@ -40,7 +40,7 @@ publishing { repositories { maven { name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/ShimmerEngineering/Shimmer-Java-Android-API") + url = uri("https://maven.pkg.github.com/ShimmerResearch/Shimmer-Java-Android-API") credentials { /* Create gradle.properties file in GRADLE_USER_HOME/.gradle/ (e.g. C:/Users/YourUsername/.gradle/) with the two lines listed below. Fill in your diff --git a/ShimmerDriver/build.gradle b/ShimmerDriver/build.gradle index 8cce51299..47840827c 100644 --- a/ShimmerDriver/build.gradle +++ b/ShimmerDriver/build.gradle @@ -63,7 +63,7 @@ publishing { repositories { maven { name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/ShimmerEngineering/Shimmer-Java-Android-API") + url = uri("https://maven.pkg.github.com/ShimmerResearch/Shimmer-Java-Android-API") credentials { /* Create gradle.properties file in GRADLE_USER_HOME/.gradle/ (e.g. C:/Users/YourUsername/.gradle/) with the two lines listed below. Fill in your diff --git a/ShimmerDriverPC/build.gradle b/ShimmerDriverPC/build.gradle index 00ee312ac..dd821701a 100644 --- a/ShimmerDriverPC/build.gradle +++ b/ShimmerDriverPC/build.gradle @@ -57,7 +57,7 @@ publishing { repositories { maven { name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/ShimmerEngineering/Shimmer-Java-Android-API") + url = uri("https://maven.pkg.github.com/ShimmerResearch/Shimmer-Java-Android-API") credentials { /* Create gradle.properties file in GRADLE_USER_HOME/.gradle/ (e.g. C:/Users/YourUsername/.gradle/) with the two lines listed below. Fill in your @@ -110,12 +110,18 @@ dependencies { implementation("org.bouncycastle:bcprov-jdk15on:1.61") } -// Shadow JAR task to create a runnable JAR with dependencies -/* -Change to using the following below for ShimmerDriverPC/ShimmerDriver/ShimmerBluetoothManager -sourceCompatibility = '1.8' // Ensure code is compiled with Java 8 -targetCompatibility = '1.8' // Ensure bytecode is compatible with Java 8 -*/ +// Shadow JAR task to create a runnable JAR with dependencies. +// ShimmerJavaClass.jar is loaded by MATLAB, which may run it on Java 8, so build it as Java 8 +// bytecode with: ./gradlew shadowJar -PmatlabJava8 +// This applies to the ShimmerDriver and ShimmerBluetoothManager subprojects too, so their build files +// no longer need editing to 1.8 by hand. --release 8 also fails the compile if the code uses an API +// newer than Java 8, which sourceCompatibility = '1.8' alone would let through to fail in MATLAB. +// The "Release Java API" workflow builds the jar this way and opens a PR into Shimmer-MATLAB-ID/libs. +if (hasProperty('matlabJava8')) { + allprojects { + tasks.withType(JavaCompile).configureEach { options.release = 8 } + } +} shadowJar { archiveBaseName.set('ShimmerJavaClass') // Use archiveBaseName instead of baseName archiveClassifier.set('') // Remove classifier so it’s named shimmerjavaclass.jar diff --git a/ShimmerLSL/build.gradle b/ShimmerLSL/build.gradle index 4227dc2c0..419de8816 100644 --- a/ShimmerLSL/build.gradle +++ b/ShimmerLSL/build.gradle @@ -39,7 +39,7 @@ publishing { repositories { maven { name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/ShimmerEngineering/Shimmer-Java-Android-API") + url = uri("https://maven.pkg.github.com/ShimmerResearch/Shimmer-Java-Android-API") credentials { /* Create gradle.properties file in GRADLE_USER_HOME/.gradle/ (e.g. C:/Users/YourUsername/.gradle/) with the two lines listed below. Fill in your diff --git a/ShimmerPCBasicExamples/build.gradle b/ShimmerPCBasicExamples/build.gradle index e3b8eb351..213a64eab 100644 --- a/ShimmerPCBasicExamples/build.gradle +++ b/ShimmerPCBasicExamples/build.gradle @@ -22,7 +22,7 @@ repositories { } maven { name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/ShimmerEngineering/Shimmer-Java-Android-API") + url = uri("https://maven.pkg.github.com/ShimmerResearch/Shimmer-Java-Android-API") credentials { /* Create gradle.properties file in GRADLE_USER_HOME/.gradle/ (e.g. C:/Users/YourUsername/.gradle/) with the two lines listed below. Fill in your From 1bd3a654f4554961c526f251571ffdd83faab37a Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 12:12:53 +0800 Subject: [PATCH 15/41] DEV-1115 Let a dry run continue past an already-released version The version in the build files is normally one that has already been released until the next uprev, so stopping there made every PR dry run end before the build it is meant to test. Dry runs now warn; a real release still refuses. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d8c2778e..a58e518ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,8 +91,12 @@ jobs: done TAG="API_v$VERSION" if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then - echo "::error::$TAG already exists. Bump the version with an uprev PR first." - exit 1 + if [ "$DRY_RUN" = true ]; then + echo "::warning::$TAG already exists, so a real release would stop here. Dry run continues." + else + echo "::error::$TAG already exists. Bump the version with an uprev PR first." + exit 1 + fi fi PREVIOUS_TAG=$(gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName,createdAt \ --jq '[.[] | select(.tagName | startswith("API_v"))] | sort_by(.createdAt) | last | .tagName // ""') From 02b5380383cb4d23489fb28718eee19f6163d56e Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 12:29:55 +0800 Subject: [PATCH 16/41] DEV-1198 Dump all threads when a simulator connect times out The lns0_16_11_w_sd_bt_streaming connect hangs on Linux CI and not on Windows, so the CI log is the only evidence of where the IO thread is stuck. Print every thread's full stack, the monitors it holds, the lock it waits on and any JVM-detected deadlock before failing. Co-Authored-By: Claude Opus 5.5 --- .../ByteCommunicationTestUtils.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java index 5722ce6bf..dc8213484 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java @@ -2,6 +2,12 @@ import static org.junit.Assert.fail; +import java.lang.management.LockInfo; +import java.lang.management.ManagementFactory; +import java.lang.management.MonitorInfo; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; + import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; import com.shimmerresearch.pcDriver.ShimmerPC; @@ -23,6 +29,9 @@ public static void connectAndWaitUntilInitialised(ShimmerPC device) { long deadline = System.currentTimeMillis() + CONNECT_TIMEOUT_MS; while (!(device.isInitialised() && device.getBluetoothRadioState()==BT_STATE.CONNECTED)) { if (System.currentTimeMillis() > deadline) { + // A hang here has so far only reproduced on CI, so print where every thread is + // before failing - the log is the only evidence there is. + System.out.println(dumpAllThreads()); fail("Device did not finish initialising within " + CONNECT_TIMEOUT_MS + " ms (state: " + device.getBluetoothRadioState() + ")"); } try { @@ -34,4 +43,48 @@ public static void connectAndWaitUntilInitialised(ShimmerPC device) { } } + /** Every thread's full stack with the monitors it holds and the lock it waits on, plus any + * deadlock the JVM can see. ThreadInfo.toString() stops at 8 frames, too few for the driver. */ + public static String dumpAllThreads() { + ThreadMXBean mxBean = ManagementFactory.getThreadMXBean(); + StringBuilder sb = new StringBuilder("==== Thread dump ====\n"); + + long[] deadlocked = mxBean.findDeadlockedThreads(); + if (deadlocked != null) { + sb.append("DEADLOCK between thread ids:"); + for (long id : deadlocked) { + sb.append(' ').append(id); + } + sb.append('\n'); + } + + for (ThreadInfo ti : mxBean.dumpAllThreads(true, true)) { + sb.append('"').append(ti.getThreadName()).append("\" id=").append(ti.getThreadId()) + .append(' ').append(ti.getThreadState()); + if (ti.getLockName() != null) { + sb.append(" on ").append(ti.getLockName()); + } + if (ti.getLockOwnerName() != null) { + sb.append(" owned by \"").append(ti.getLockOwnerName()).append("\" id=").append(ti.getLockOwnerId()); + } + sb.append('\n'); + + StackTraceElement[] stack = ti.getStackTrace(); + MonitorInfo[] monitors = ti.getLockedMonitors(); + for (int i = 0; i < stack.length; i++) { + sb.append("\tat ").append(stack[i]).append('\n'); + for (MonitorInfo mi : monitors) { + if (mi.getLockedStackDepth() == i) { + sb.append("\t- locked ").append(mi).append('\n'); + } + } + } + for (LockInfo li : ti.getLockedSynchronizers()) { + sb.append("\t- locked synchronizer ").append(li).append('\n'); + } + sb.append('\n'); + } + return sb.append("==== End of thread dump ====").toString(); + } + } From ffb9647a2fedeab736f96556349e11f224fa3082 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 13:54:39 +0800 Subject: [PATCH 17/41] DEV-1198 Stop the simulator tests deadlocking on the read-status timer The connect sequence unlocks the instruction stack and then starts the read-status timer with no delay, so the timer's first readStatusTask (device lock, then instruction-list lock) runs just as the IOThread takes the first post-connect instruction in getInstruction() (list lock, then device lock). When they meet, both block for good and the connect never finishes. The thread dump from the previous commit shows that cycle on Linux CI in lns0_16_11_w_sd_bt_streaming, and locally in API_0000X_ByteCommunicationShimmer3. The driver is deliberately left unchanged. The simulator tests now build their ShimmerPC through ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer(), which overrides startTimerReadStatus() to do nothing, so nothing else touches the instruction list while the IOThread connects. The tests lose only the 5 s status poll; the connect sequence still reads the status once. Co-Authored-By: Claude Opus 5.5 --- .../API_0000X_ByteCommunicationShimmer3.java | 2 +- .../API_0000X_ByteCommunicationShimmer3R.java | 2 +- ...teCommunicationShimmer3_CRC_Streaming.java | 2 +- ...X_ByteCommunicationShimmer3_Streaming.java | 2 +- ...onShimmer3_Streaming_Contiguous_Check.java | 2 +- ...mmer3_Streaming_Contiguous_Check_Fail.java | 2 +- ...3_Streaming_Contiguous_Check_Overflow.java | 2 +- ...icationShimmer3_Streaming_Throw_Bytes.java | 2 +- ...ationShimmer3_Streaming_Throw_Bytes_2.java | 2 +- ...X_ByteCommunicationShimmer3lns0_16_11.java | 2 +- ...nShimmer3lns0_16_11_w_sd_bt_streaming.java | 2 +- .../ByteCommunicationTestUtils.java | 20 +++++++++++++++++++ 12 files changed, 31 insertions(+), 11 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java index ef2a5083c..45ac4ec9d 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3.java @@ -37,7 +37,7 @@ public class API_0000X_ByteCommunicationShimmer3 extends BasicProcessWithCallBac @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java index 87369358d..5dd1e90b9 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3R.java @@ -54,7 +54,7 @@ public class API_0000X_ByteCommunicationShimmer3R extends BasicProcessWithCallBa @Before public void setUp() { mByteCommunicationSimulatorS3R = new ByteCommunicationSimulatorS3R("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3R); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java index af7afaf6f..7cc1efbfe 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_CRC_Streaming.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_CRC_Streaming extends BasicProc @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_instream_crc("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java index 0752fbfe3..07adcec3e 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming extends BasicProcessW @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java index 0c31a771b..430e7cb18 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check exte @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_timestamp_contiguous_check("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java index be362947a..5884c764e 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail.java @@ -37,7 +37,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Fail @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_timestamp_contiguous_check_fail("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java index 42dbba835..d8d93e0f2 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Overflow.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Contiguous_Check_Over @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_timestamp_contiguous_check_overflow("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java index 0eeab0c01..84b8bbd8a 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes extends B @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_with_improved_throw_bytes("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java index 6cf3777df..da4b56f44 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2.java @@ -36,7 +36,7 @@ public class API_0000X_ByteCommunicationShimmer3_Streaming_Throw_Bytes_2 extends @Before public void setUp() { mByteCommunicationSimulatorS3 = new ByteCommunicationSimulatorS3_streaming_with_improved_throw_bytes_2("COM99"); - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(mByteCommunicationSimulatorS3); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java index 8d2d07628..054f13afe 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11 extends BasicProcessW ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11("COM99")); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java index a84b082b3..d264c594a 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_bt_streaming ext ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sd_bt_streaming("COM99")); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java index dc8213484..69d9f203c 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationTestUtils.java @@ -18,6 +18,26 @@ public class ByteCommunicationTestUtils { * but a loaded CI runner can stall it for several. Only a genuine hang should hit this. */ public static final long CONNECT_TIMEOUT_MS = 30000; + /** + * A ShimmerPC for the simulator tests whose periodic read-status timer never starts. + *

+ * The connect sequence unlocks the instruction stack and then starts that timer with no delay, + * so its first readStatusTask (which locks the device, then the instruction list) runs just as + * the IOThread takes the first post-connect instruction in ShimmerBluetooth.getInstruction() + * (which locks the list, then the device). When the two meet they deadlock and the connect never + * finishes. That hung these tests on Linux CI every time (DEV-1198). The driver is deliberately + * left as it is; the tests only lose the 5 s status poll, and the connect sequence still reads + * the status once. + */ + public static ShimmerPC newShimmerPCWithoutStatusTimer(String comPort) { + return new ShimmerPC(comPort) { + @Override + public void startTimerReadStatus() { + // Deliberately empty - see above + } + }; + } + /** * Connects and blocks until the device has finished its post-connect config read (infomem, * calibration, etc.), i.e. isInitialised() is set and the state has moved to CONNECTED. From 934c25ed2a4ed5c83f72d914c63b17356cdc38f5 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 15 Oct 2026 13:37:08 +0800 Subject: [PATCH 18/41] DEV-1198 Re-enable the three ShimmerDriverPC simulator test classes API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only, _w_sync_master and _w_sync_slave were excluded from the test task in Feb 2026 as failing on CI. Their CI logs have expired, so the cause cannot be confirmed, but it fits the read-status timer deadlock the previous commit avoids: these classes assert values from the post-connect config read (SD log sync, Bluetooth disabled, sync nodes, trial duration), which a stalled connect never reaches. They now build their device through newShimmerPCWithoutStatusTimer() like the other simulator classes, and run again. Co-Authored-By: Claude Opus 5.5 --- ShimmerDriverPC/build.gradle | 3 --- ...PI_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java | 2 +- ...000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java | 2 +- ...0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ShimmerDriverPC/build.gradle b/ShimmerDriverPC/build.gradle index 00ee312ac..fb8ce399e 100644 --- a/ShimmerDriverPC/build.gradle +++ b/ShimmerDriverPC/build.gradle @@ -32,9 +32,6 @@ repositories { test { // this test requires a physical Shimmer2r to run exclude '**/API_00001_ShimmerPC_GeneralBluetoothShimmer2R.class' - exclude '**/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.class' - exclude '**/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.class' - exclude '**/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.class' } publishing { diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java index 7eb4ec253..4b40b8ba2 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sd_only extends Bas ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sd_only("COM99")); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java index 6b0695dbf..725719a8e 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_master extends ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sync_master("COM99")); setWaitForData(mDevice); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java index 2b20f77a4..75aff1e45 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave.java @@ -31,7 +31,7 @@ public class API_0000X_ByteCommunicationShimmer3lns0_16_11_w_sync_slave extends ShimmerPC mDevice; @Before public void setUp() { - mDevice = new ShimmerPC("COM99"); + mDevice = ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer("COM99"); mDevice.setTestRadio(new ByteCommunicationSimulatorS3LNS0_16_11_w_sync_slave("COM99")); setWaitForData(mDevice); } From d5f5b44e9e9110492a26bbefc459c4b2160e25bc Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 16 Oct 2026 10:45:37 +0800 Subject: [PATCH 19/41] DEV-1115 Pick the release version in the Run form, not an uprev PR The version no longer lives in the source. The three publishing blocks read -PreleaseVersion (falling back to 'local'), and Release Java API passes it: the version typed in the Run form, or by default the next patch after the last API_v release (0.11.8_beta -> 0.11.9_beta). The API_v tags record what was released, so a release needs no uprev PR and nothing has to be committed to the protected master. Checked locally: with -PreleaseVersion=0.11.9_beta all three POMs and their dependencies on each other are 0.11.9_beta; without it, 'local'. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 64 ++++++++++++++++------------ AGENTS.md | 20 +++++---- ShimmerBluetoothManager/build.gradle | 2 +- ShimmerDriver/build.gradle | 2 +- ShimmerDriverPC/build.gradle | 2 +- 5 files changed, 52 insertions(+), 38 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a58e518ac..a34d69d8a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,16 @@ # One-click release of the Java API: GitHub Packages, a GitHub Release, and the MATLAB jar. # # How to release -# 1. Merge the usual uprev PR, which bumps the publishing `version` in ShimmerDriver, -# ShimmerBluetoothManager and ShimmerDriverPC build.gradle. master is protected, so this -# workflow cannot commit the bump itself. -# 2. Actions -> "Release Java API" -> Run workflow (on master). -# 3. Review and merge the ShimmerJavaClass.jar PR it opens in Shimmer-MATLAB-ID. +# 1. Actions -> "Release Java API" -> Run workflow (on master). Leave `version` blank for the next +# patch after the last API_v release (0.11.8_beta -> 0.11.9_beta), or type one. +# 2. Review and merge the ShimmerJavaClass.jar PR it opens in Shimmer-MATLAB-ID. +# 3. Run "Release Android API" in ShimmerAndroidAPI, which builds against this version by default. +# +# The version is not kept in the source. The build files read it from -PreleaseVersion, which this +# workflow passes, and say 'local' otherwise; the API_v tags record what was released. # # One run does, in order, and creates nothing public until the build and tests have passed: -# build reads the version from the three build.gradle files and fails if they disagree; -# fails if API_v is already tagged; tests ShimmerDriver and builds +# build picks the version; fails if API_v is already tagged; tests ShimmerDriver and builds # ShimmerDriverPC exactly as gradle.yml does; then publishes shimmerdriver, # shimmerbluetoothmanager and shimmerdriverpc together with one `gradlew publish` # (DEV-1115: publishing them one at a time by hand left shimmerbluetoothmanager @@ -27,7 +28,8 @@ # repeated. A failed publish can leave some of the three versions in GitHub Packages: delete those # versions under the repo's Packages before re-running, or bump the version. # -# dry_run runs build and matlab-jar without publishing, from any branch. The workflow also +# dry_run runs build and matlab-jar without publishing, from any branch; a blank version still picks +# the next one, so the run shows what a real release would be called. The workflow also # dry-runs on any PR that changes this file. name: Release Java API @@ -35,6 +37,10 @@ name: Release Java API on: workflow_dispatch: inputs: + version: + description: 'Version, e.g. 0.12.0_beta. Blank = next patch after the last release' + type: string + required: false dry_run: description: 'Build and test only; publish nothing' type: boolean @@ -70,36 +76,38 @@ jobs: with: fetch-depth: 0 # tags, to refuse a version that is already released - - name: Read the version from the build files + - name: Choose the version id: version env: GH_TOKEN: ${{ github.token }} + REQUESTED: ${{ inputs.version }} run: | - VERSION= - for p in ShimmerDriver ShimmerBluetoothManager ShimmerDriverPC; do - v=$(sed -nE "s/^[[:space:]]*version = '([^']+)'.*/\1/p" "$p/build.gradle") - echo "$p: ${v:-}" - if [ "$(printf '%s\n' "$v" | grep -c .)" -ne 1 ]; then - echo "::error file=$p/build.gradle::Expected exactly one publishing version = '...' line" - exit 1 - fi - if [ -n "$VERSION" ] && [ "$v" != "$VERSION" ]; then - echo "::error file=$p/build.gradle::$p is at $v but ShimmerDriver is at $VERSION. The uprev must bump all three." + PREVIOUS_TAG=$(gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("API_v"))] | sort_by(.createdAt) | last | .tagName // ""') + if [ -n "$REQUESTED" ]; then + VERSION=$REQUESTED + else + # Next patch after the last release, keeping its suffix: 0.11.8_beta -> 0.11.9_beta. + last=${PREVIOUS_TAG#API_v} + if ! [[ "$last" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(.*)$ ]]; then + echo "::error::Cannot work out the next version from '${PREVIOUS_TAG:-no previous release}'. Type one in the version box." exit 1 fi - VERSION=$v - done + VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}.$((BASH_REMATCH[3] + 1))${BASH_REMATCH[4]}" + fi + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+[A-Za-z0-9_.-]*$ ]]; then + echo "::error::'$VERSION' is not a version like 0.11.9_beta." + exit 1 + fi TAG="API_v$VERSION" if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then if [ "$DRY_RUN" = true ]; then echo "::warning::$TAG already exists, so a real release would stop here. Dry run continues." else - echo "::error::$TAG already exists. Bump the version with an uprev PR first." + echo "::error::$TAG already exists. Leave the version blank for the next one, or type another." exit 1 fi fi - PREVIOUS_TAG=$(gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName,createdAt \ - --jq '[.[] | select(.tagName | startswith("API_v"))] | sort_by(.createdAt) | last | .tagName // ""') { echo "version=$VERSION" echo "tag=$TAG" @@ -128,14 +136,16 @@ jobs: run: ./gradlew build # `publish` runs in every project of the ShimmerDriverPC build, so all three artifacts go - # out in one invocation. The build files read these credentials from USERNAME and TOKEN. + # out in one invocation, and -PreleaseVersion versions all three (and their POM + # dependencies on each other). The build files read the credentials from USERNAME and TOKEN. - name: Publish to GitHub Packages if: env.DRY_RUN != 'true' working-directory: ShimmerDriverPC env: USERNAME: ${{ github.actor }} TOKEN: ${{ github.token }} - run: ./gradlew publish + VERSION: ${{ steps.version.outputs.version }} + run: ./gradlew publish -PreleaseVersion="$VERSION" - name: Collect the release jars env: @@ -232,7 +242,7 @@ jobs: echo "### Released Java API $VERSION" echo "- Release: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$TAG" echo "- Packages: com.shimmerresearch:shimmerdriver, shimmerbluetoothmanager and shimmerdriverpc at $VERSION" - echo "- Next: bump the shimmerdriver and shimmerbluetoothmanager pins in ShimmerAndroidAPI to $VERSION, then run its Release workflow." + echo "- Next: merge the MATLAB PR, then run Release Android API in ShimmerAndroidAPI; it builds against $VERSION unless you pick another driver version." } >> "$GITHUB_STEP_SUMMARY" update-matlab: diff --git a/AGENTS.md b/AGENTS.md index 7e26a89c1..987a978b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,16 +19,20 @@ all: they are legacy Eclipse ADT Android projects. Test results land in `**/build/test-results/test/*.xml`. ## Releasing -Releases are cut by the **Release Java API** workflow (`release.yml`), not from a developer machine: -1. Merge an uprev PR that bumps the publishing `version` in `ShimmerDriver`, `ShimmerBluetoothManager` - and `ShimmerDriverPC` `build.gradle` — all three, to the same value; the workflow refuses a mismatch. -2. Run the workflow on `master`. It tests and builds as CI does, publishes all three artifacts to - GitHub Packages in one `gradlew publish`, tags `API_v`, and creates the GitHub Release - with the jars attached (not marked Latest — that stays with Consensys). -3. It also builds `ShimmerJavaClass.jar` for MATLAB and opens a PR replacing +Releases are cut by the **Release Java API** workflow (`release.yml`), not from a developer machine, +and need no uprev PR: +1. Run the workflow on `master`. Leave `version` blank for the next patch after the last `API_v` + release, or type one. It tests and builds as CI does, publishes all three artifacts to GitHub + Packages in one `gradlew publish`, tags `API_v`, and creates the GitHub Release with the + jars attached (not marked Latest — that stays with Consensys). +2. It also builds `ShimmerJavaClass.jar` for MATLAB and opens a PR replacing `Shimmer-MATLAB-ID/libs/ShimmerJavaClass.jar` (needs the `MATLAB_REPO_TOKEN` secret). Review and merge it. -4. Bump the driver pins in `ShimmerAndroidAPI` to the new version, then run its own Release workflow. +3. Run `ShimmerAndroidAPI`'s own Release workflow, which builds against the newest Java release unless + told otherwise. + +The version is not stored in the source: the three publishing blocks read `-PreleaseVersion`, which +the workflow passes, and fall back to `local`. The `API_v` tags are the record of what was released. `ShimmerJavaClass.jar` must be Java 8 bytecode, because MATLAB may load it on Java 8. Build it locally with `cd ShimmerDriverPC && ./gradlew shadowJar -PmatlabJava8`, which compiles all three projects with diff --git a/ShimmerBluetoothManager/build.gradle b/ShimmerBluetoothManager/build.gradle index 72fd006ae..c4961ecda 100644 --- a/ShimmerBluetoothManager/build.gradle +++ b/ShimmerBluetoothManager/build.gradle @@ -27,7 +27,7 @@ publishing { groupId = 'com.shimmerresearch' // Replace with your package's group/organization name artifactId = 'shimmerbluetoothmanager' // Replace with the name of your package - version = '0.11.8_beta' // Replace with your package version + version = project.findProperty('releaseVersion') ?: 'local' // set by the Release Java API workflow (-PreleaseVersion) // Jar publication //( diff --git a/ShimmerDriver/build.gradle b/ShimmerDriver/build.gradle index 47840827c..84d17c6ea 100644 --- a/ShimmerDriver/build.gradle +++ b/ShimmerDriver/build.gradle @@ -49,7 +49,7 @@ publishing { groupId = 'com.shimmerresearch' // Replace with your package's group/organization name artifactId = 'shimmerdriver' // Replace with the name of your package - version = '0.11.8_beta' // Replace with your package version + version = project.findProperty('releaseVersion') ?: 'local' // set by the Release Java API workflow (-PreleaseVersion) // Jar publication //( diff --git a/ShimmerDriverPC/build.gradle b/ShimmerDriverPC/build.gradle index dd821701a..3b4ad7258 100644 --- a/ShimmerDriverPC/build.gradle +++ b/ShimmerDriverPC/build.gradle @@ -44,7 +44,7 @@ publishing { groupId = 'com.shimmerresearch' // Replace with your package's group/organization name artifactId = 'shimmerdriverpc' // Replace with the name of your package - version = '0.11.8_beta' // Replace with your package version + version = project.findProperty('releaseVersion') ?: 'local' // set by the Release Java API workflow (-PreleaseVersion) // Jar publication //( From 17515618d414b9e7f12cd8660b37c4ff5ee0c25d Mon Sep 17 00:00:00 2001 From: Mas Azalya Date: Fri, 16 Oct 2026 12:14:40 +0800 Subject: [PATCH 20/41] DEV-1115 Fix publish of shimmerdriverpc and refuse a 'local' publish publish failed on shimmerdriverpc with "multiple artifacts with the identical extension and classifier ('jar', 'null')": the Shadow plugin adds ShimmerJavaClass.jar (empty classifier) to components.java, and Gradle 8.14 (DEV-1013) rejects the clash. The PR dry run skips publish, so the first real release would have failed there after publishing the other two. Skip the shadow variant, which publishes the plain jar with its dependencies in the POM, as 0.11.7_beta was. Without -PreleaseVersion the version is 'local', and a hand-run `gradlew publish` would push that to GitHub Packages, where it can never be overwritten. Publishing to a remote repository now refuses 'local'; publishToMavenLocal is unaffected. Add job timeouts so a hung Gradle daemon cannot hold a runner for the default 6 hours. Checked locally (JDK 11): publishToMavenLocal -PreleaseVersion=0.0.0_test now succeeds, all three POMs and their cross-dependencies read 0.0.0_test, and shimmerdriverpc is the 140 KB thin jar. Publishing to a file repository without -PreleaseVersion fails for all three. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 4 ++++ ShimmerBluetoothManager/build.gradle | 11 +++++++++++ ShimmerDriver/build.gradle | 11 +++++++++++ ShimmerDriverPC/build.gradle | 16 ++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a34d69d8a..903707753 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 # a hung Gradle daemon otherwise holds the runner for 6 hours permissions: contents: read packages: write @@ -166,6 +167,7 @@ jobs: matlab-jar: needs: build runs-on: ubuntu-latest + timeout-minutes: 30 # a hung Gradle daemon otherwise holds the runner for 6 hours permissions: contents: read steps: @@ -211,6 +213,7 @@ jobs: needs: [build, matlab-jar] if: github.event_name == 'workflow_dispatch' && !inputs.dry_run runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: write # create the tag and the release env: @@ -248,6 +251,7 @@ jobs: update-matlab: needs: [build, release] runs-on: ubuntu-latest + timeout-minutes: 10 permissions: {} # writes to Shimmer-MATLAB-ID with MATLAB_REPO_TOKEN, not this repo's token env: VERSION: ${{ needs.build.outputs.version }} diff --git a/ShimmerBluetoothManager/build.gradle b/ShimmerBluetoothManager/build.gradle index c4961ecda..6bf3cb637 100644 --- a/ShimmerBluetoothManager/build.gradle +++ b/ShimmerBluetoothManager/build.gradle @@ -68,3 +68,14 @@ dependencies { // 'test.useTestNG()' to your build script. testImplementation 'junit:junit:4.12' } + +// Without -PreleaseVersion the version is 'local', and GitHub Packages never lets a version be +// overwritten. Releases go through the Release Java API workflow; publishToMavenLocal is unaffected. +tasks.withType(PublishToMavenRepository).configureEach { + doFirst { + if (publication.version == 'local') { + throw new GradleException("Refusing to publish ${publication.artifactId} as version 'local'. " + + "Release with the Release Java API workflow, or pass -PreleaseVersion.") + } + } +} diff --git a/ShimmerDriver/build.gradle b/ShimmerDriver/build.gradle index 84d17c6ea..8d62a7215 100644 --- a/ShimmerDriver/build.gradle +++ b/ShimmerDriver/build.gradle @@ -173,3 +173,14 @@ tasks.withType(Test).configureEach { } } + +// Without -PreleaseVersion the version is 'local', and GitHub Packages never lets a version be +// overwritten. Releases go through the Release Java API workflow; publishToMavenLocal is unaffected. +tasks.withType(PublishToMavenRepository).configureEach { + doFirst { + if (publication.version == 'local') { + throw new GradleException("Refusing to publish ${publication.artifactId} as version 'local'. " + + "Release with the Release Java API workflow, or pass -PreleaseVersion.") + } + } +} diff --git a/ShimmerDriverPC/build.gradle b/ShimmerDriverPC/build.gradle index 3b4ad7258..623db5df3 100644 --- a/ShimmerDriverPC/build.gradle +++ b/ShimmerDriverPC/build.gradle @@ -41,6 +41,11 @@ publishing { publications { mavenJava(MavenPublication) { from components.java + // The Shadow plugin adds ShimmerJavaClass.jar (no classifier) to components.java, where it + // collides with the plain jar and fails publish on Gradle 8. Publish the plain jar with its + // dependencies in the POM, as releases up to 0.11.7_beta did; the fat jar goes to MATLAB + // and the GitHub Release instead. + components.java.withVariantsFromConfiguration(configurations.shadowRuntimeElements) { skip() } groupId = 'com.shimmerresearch' // Replace with your package's group/organization name artifactId = 'shimmerdriverpc' // Replace with the name of your package @@ -132,3 +137,14 @@ shadowJar { mergeServiceFiles() // Merges service files (if any) } + +// Without -PreleaseVersion the version is 'local', and GitHub Packages never lets a version be +// overwritten. Releases go through the Release Java API workflow; publishToMavenLocal is unaffected. +tasks.withType(PublishToMavenRepository).configureEach { + doFirst { + if (publication.version == 'local') { + throw new GradleException("Refusing to publish ${publication.artifactId} as version 'local'. " + + "Release with the Release Java API workflow, or pass -PreleaseVersion.") + } + } +} From 77aaa3b9236b2449f09c1e6e5c3819d9d24bd9d0 Mon Sep 17 00:00:00 2001 From: JongChern Date: Sun, 18 Oct 2026 13:26:11 +0800 Subject: [PATCH 21/41] DEV-1115 Keep the packages:write token out of .git/config actions/checkout persists github.token in .git/config by default, so the Gradle build in the build job, and any plugin or dependency it runs, could read a token with packages:write. Neither build nor matlab-jar needs git credentials after checkout, so turn persistence off there. update-matlab keeps it, because its git push uses it. Raised in review on #324. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 903707753..2437a14d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,6 +76,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # tags, to refuse a version that is already released + # Keep this job's packages:write token out of .git/config, where the Gradle build (and any + # plugin or dependency it runs) could read it. Nothing here needs git credentials after + # checkout: publish gets the token explicitly, and gh uses GH_TOKEN. + persist-credentials: false - name: Choose the version id: version @@ -172,6 +176,8 @@ jobs: contents: read steps: - uses: actions/checkout@v4 + with: + persist-credentials: false # the Gradle build needs no git credentials - name: Set up JDK 11 uses: actions/setup-java@v4 @@ -266,6 +272,8 @@ jobs: echo "::error::Secret MATLAB_REPO_TOKEN is not set, so the Shimmer-MATLAB-ID PR cannot be opened. Add a fine-grained token for $MATLAB_REPO (Contents and Pull requests: read/write) as a repo secret, then re-run this job." exit 1 + # Credentials stay persisted here on purpose: the git push below uses them. Nothing else runs + # in this job. - uses: actions/checkout@v4 with: repository: ${{ env.MATLAB_REPO }} From 9a4055690d58462905cde64464ff817625245baa Mon Sep 17 00:00:00 2001 From: JongChern Date: Sun, 18 Oct 2026 13:31:13 +0800 Subject: [PATCH 22/41] DEV-1115 Publish from a dispatch-only job; build job read-only The build job ran on PRs with packages:write. Any code in a job can recover that job's token from the runner, so a PR dry run should not hold a write token at all. The build job is now read-only, and a new publish job, which runs only on a real release, is the one place with packages:write. It publishes and uploads the jars it published, so the GitHub Release carries the same files as GitHub Packages. The release job now waits for it. Raised in review on #324. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 49 +++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2437a14d9..487853510 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,10 +11,11 @@ # # One run does, in order, and creates nothing public until the build and tests have passed: # build picks the version; fails if API_v is already tagged; tests ShimmerDriver and builds -# ShimmerDriverPC exactly as gradle.yml does; then publishes shimmerdriver, +# ShimmerDriverPC exactly as gradle.yml does. Read-only: it runs on PRs too. +# publish real releases only, and the only job with packages:write. Publishes shimmerdriver, # shimmerbluetoothmanager and shimmerdriverpc together with one `gradlew publish` # (DEV-1115: publishing them one at a time by hand left shimmerbluetoothmanager -# 0.11.8_beta unpublished) +# 0.11.8_beta unpublished), and keeps the jars it published for the release # matlab-jar builds ShimmerJavaClass.jar as Java 8 bytecode (`shadowJar -PmatlabJava8`), on a # separate runner so its Java 8 classes cannot end up in the published jars # release tags API_v on the commit it built and creates the GitHub Release, with @@ -57,9 +58,10 @@ jobs: build: runs-on: ubuntu-latest timeout-minutes: 30 # a hung Gradle daemon otherwise holds the runner for 6 hours + # Read-only on purpose: this job also runs on PRs, and any code in a job can recover that + # job's token. Only the publish job, which runs on real releases alone, can write packages. permissions: contents: read - packages: write env: DRY_RUN: ${{ github.event_name != 'workflow_dispatch' || inputs.dry_run }} outputs: @@ -76,9 +78,8 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # tags, to refuse a version that is already released - # Keep this job's packages:write token out of .git/config, where the Gradle build (and any - # plugin or dependency it runs) could read it. Nothing here needs git credentials after - # checkout: publish gets the token explicitly, and gh uses GH_TOKEN. + # Keep the token out of .git/config, where the Gradle build (and any plugin or dependency + # it runs) could read it. Nothing here needs git credentials after checkout; gh uses GH_TOKEN. persist-credentials: false - name: Choose the version @@ -140,21 +141,43 @@ jobs: working-directory: ShimmerDriverPC run: ./gradlew build + publish: + needs: build + if: github.event_name == 'workflow_dispatch' && !inputs.dry_run + runs-on: ubuntu-latest + timeout-minutes: 30 # a hung Gradle daemon otherwise holds the runner for 6 hours + permissions: + contents: read + packages: write + env: + VERSION: ${{ needs.build.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false # publish gets its token explicitly, below + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@ec92e829475ac0c2315ea8f9eced72db85bb337a # v3.0.0 + # `publish` runs in every project of the ShimmerDriverPC build, so all three artifacts go # out in one invocation, and -PreleaseVersion versions all three (and their POM - # dependencies on each other). The build files read the credentials from USERNAME and TOKEN. + # dependencies on each other). It builds the jars but does not re-run the tests, which the + # build job ran on this same commit. The build files read the credentials from USERNAME and TOKEN. - name: Publish to GitHub Packages - if: env.DRY_RUN != 'true' working-directory: ShimmerDriverPC env: USERNAME: ${{ github.actor }} TOKEN: ${{ github.token }} - VERSION: ${{ steps.version.outputs.version }} run: ./gradlew publish -PreleaseVersion="$VERSION" - - name: Collect the release jars - env: - VERSION: ${{ steps.version.outputs.version }} + # The jars just published, so the GitHub Release carries the same files as GitHub Packages. + - name: Collect the published jars run: | mkdir -p release-assets cp ShimmerDriver/build/libs/ShimmerDriver.jar "release-assets/shimmerdriver-$VERSION.jar" @@ -216,7 +239,7 @@ jobs: if-no-files-found: error release: - needs: [build, matlab-jar] + needs: [build, publish, matlab-jar] if: github.event_name == 'workflow_dispatch' && !inputs.dry_run runs-on: ubuntu-latest timeout-minutes: 10 From b819f4c6075bef8b0ae93e8e191dce7f65de90fc Mon Sep 17 00:00:00 2001 From: JongChern Date: Sun, 18 Oct 2026 13:38:07 +0800 Subject: [PATCH 23/41] DEV-1115 Check every class in the MATLAB jar is Java 8 The check sampled three of our own classes, but ShimmerJavaClass.jar also shades every dependency, and one built for a later Java would break MATLAB on Java 8 just the same. Check every class instead (about 32,500, read once with Python in ~2 s), skipping module-info and META-INF/versions entries, which Java 8 never loads, and still require the main class to be present. Tested on a fresh -PmatlabJava8 build (32,510 classes, all Java 8 or older, passes), and on jars missing the main class or carrying a Java 11 dependency class (both fail, naming the class). Raised in review on #324. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 44 ++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 487853510..b21194945 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -215,19 +215,41 @@ jobs: working-directory: ShimmerDriverPC run: ./gradlew shadowJar -PmatlabJava8 - - name: Check the jar is Java 8 bytecode + # Every class in the jar, dependencies included, since a dependency built for a later Java + # breaks MATLAB just as surely as our own code would (~32,500 classes, about 2 s). + - name: Check every class in the jar is Java 8 bytecode run: | jar=ShimmerDriverPC/build/libs/ShimmerJavaClass.jar - for c in com/shimmerresearch/tools/matlab/ShimmerJavaClass.class \ - com/shimmerresearch/driver/ShimmerObject.class \ - com/shimmerresearch/managers/bluetoothManager/ShimmerBluetoothManager.class; do - major=$(unzip -p "$jar" "$c" | head -c 8 | od -An -tu1 | awk '{print $7 * 256 + $8}') - echo "$c: class file major version $major" - if [ "$major" != 52 ]; then - echo "::error::$c is class version $major, not 52 (Java 8); MATLAB on Java 8 cannot load it." - exit 1 - fi - done + python3 - "$jar" <<'EOF' + import struct, sys, zipfile + + MAIN = "com/shimmerresearch/tools/matlab/ShimmerJavaClass.class" + jar = sys.argv[1] + checked, bad = 0, [] + with zipfile.ZipFile(jar) as z: + names = z.namelist() + if MAIN not in names: + print(f"::error::{jar} does not contain {MAIN}.") + sys.exit(1) + for name in names: + # Java 8 never loads module descriptors or multi-release entries for later Javas. + if (not name.endswith(".class") or name.startswith("META-INF/versions/") + or name.endswith("module-info.class")): + continue + with z.open(name) as f: + head = f.read(8) + checked += 1 + if len(head) < 8 or head[:4] != b"\xca\xfe\xba\xbe": + bad.append((name, "not a class file")) + elif struct.unpack(">H", head[6:8])[0] > 52: + bad.append((name, f"class version {struct.unpack('>H', head[6:8])[0]}")) + print(f"Checked {checked} classes; {len(bad)} newer than Java 8 (class version 52).") + for name, why in bad[:20]: + print(f"::error::{name}: {why}. MATLAB on Java 8 cannot load it.") + if len(bad) > 20: + print(f"::error::...and {len(bad) - 20} more.") + sys.exit(1 if bad else 0) + EOF mkdir -p matlab-jar cp "$jar" matlab-jar/ShimmerJavaClass.jar ls -l matlab-jar From 7663cdd06dcaf620a065b668eea90b28c7f2b959 Mon Sep 17 00:00:00 2001 From: JongChern Date: Sun, 18 Oct 2026 14:40:19 +0800 Subject: [PATCH 24/41] DEV-1115 Pin the actions/* steps to commit SHAs Matches ShimmerAndroidAPI#124, which pinned checkout and setup-java in review. Each pin is the commit its v4 tag points at today (checkout v4.4.0, setup-java v4.9.1, upload-artifact v4.6.2, download-artifact v4.3.0), so nothing that runs changes; setup-gradle was already pinned. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b21194945..83c66e2f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,7 +75,7 @@ jobs: echo "::error::A release must run on master, not ${GITHUB_REF_NAME}. Tick dry_run to try another branch." exit 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 # tags, to refuse a version that is already released # Keep the token out of .git/config, where the Gradle build (and any plugin or dependency @@ -122,7 +122,7 @@ jobs: echo "Releasing $VERSION as $TAG (notes since ${PREVIOUS_TAG:-the first commit})" - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 with: java-version: '11' distribution: 'temurin' @@ -152,12 +152,12 @@ jobs: env: VERSION: ${{ needs.build.outputs.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false # publish gets its token explicitly, below - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 with: java-version: '11' distribution: 'temurin' @@ -185,7 +185,7 @@ jobs: cp ShimmerDriverPC/build/libs/ShimmerDriverPC.jar "release-assets/shimmerdriverpc-$VERSION.jar" ls -l release-assets - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: release-jars path: release-assets/ @@ -198,12 +198,12 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false # the Gradle build needs no git credentials - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 with: java-version: '11' distribution: 'temurin' @@ -254,7 +254,7 @@ jobs: cp "$jar" matlab-jar/ShimmerJavaClass.jar ls -l matlab-jar - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: matlab-jar path: matlab-jar/ @@ -273,12 +273,12 @@ jobs: TAG: ${{ needs.build.outputs.tag }} PREVIOUS_TAG: ${{ needs.build.outputs.previous_tag }} steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-jars path: release-assets - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: matlab-jar path: matlab-jar @@ -319,13 +319,13 @@ jobs: # Credentials stay persisted here on purpose: the git push below uses them. Nothing else runs # in this job. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ env.MATLAB_REPO }} token: ${{ secrets.MATLAB_REPO_TOKEN }} path: matlab - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: matlab-jar path: matlab-jar From 12be52c6bf986101af68ca5e5219f8dc71644fcb Mon Sep 17 00:00:00 2001 From: JongChern Date: Wed, 21 Oct 2026 12:52:16 +0800 Subject: [PATCH 25/41] DEV-1262 Complete the command in flight only on its own in-stream reply While streaming, the branch of processPacket() for an in-stream response (FF 8A ...) cancelled the ACK timer, cleared the wait flags, removed the head of the instruction queue and unlocked it, whatever the response was. The firmware also sends a status response unprompted (dock, undock, the user button, a low-battery stop). So such a push removed a SET command still waiting for its ACK, or a queued instruction not yet sent, and the next ACK was matched to the wrong instruction: for startSDLogging(), processAckFromSetCommand(SET_RWC) against START_LOGGING_ONLY's one-byte instruction. The branch now reads the response opcode first and completes the command in flight only if the opcode is that command's expected response (mBtGetCommandMap), as the protocol says (log-and-stream-common docs/SHIMMER3_BT_COMMUNICATION_PROTOCOL.md section 5.4, rule 2). Any other response is still processed, but leaves the queue, the wait flags and the ACK timer alone. processInstreamResponse(boolean) still reads the opcode itself and hands it to the new processInstreamResponse(byte, boolean), which holds the unchanged parsing, so overrides and other callers keep working. API_00030 (9 tests, CRC off/1/2) covers a status push while a SET command awaits its ACK, while a GET command awaits its reply, and with nothing in flight. All 9 fail without the fix. ShimmerDriver and ShimmerDriverPC test suites pass. The Android half is ShimmerResearch/ShimmerAndroidAPI#122 (DEV-1149). Co-Authored-By: Claude Opus 5.5 --- .../bluetooth/ShimmerBluetooth.java | 172 ++++++----- ...30_StreamingInstreamResponseQueueTest.java | 277 ++++++++++++++++++ 2 files changed, 378 insertions(+), 71 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00030_StreamingInstreamResponseQueueTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index abdbf73ed..df8801f60 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -1052,22 +1052,34 @@ else if(isSupportedInStreamCmds() && bufferTemp[getPacketSizeWithCrc()+2]==INSTR printLogDataForDebugging("COMMAND TXed and ACK RECEIVED IN STREAM"); printLogDataForDebugging("INS CMD RESP"); - //Firstly handle the in-stream response - stopTimerCheckForAckOrResp(); //cancel the ack timer - mWaitForResponse=false; - mWaitForAck=false; + // The firmware also sends a status response unprompted (on a dock or + // undock, the user button, a low-battery stop), so this completes the + // command in flight only if the response is that command's reply + byte[] inStreamResponseCommand = readBytes(1, INSTREAM_CMD_RESPONSE); + boolean isReplyToCommandInFlight = inStreamResponseCommand!=null + && isReplyToCommandInFlight(inStreamResponseCommand[0]); + if(isReplyToCommandInFlight){ + stopTimerCheckForAckOrResp(); //cancel the ack timer + mWaitForResponse=false; + mWaitForAck=false; + } - processInstreamResponse(true); + //Firstly handle the in-stream response + if(inStreamResponseCommand!=null){ + processInstreamResponse(inStreamResponseCommand[0], true); + } - // Need to remove here because it is an - // in-stream response while streaming so not - // handled elsewhere - if(getListofInstructions().size()>0){ - removeInstruction(0); + if(isReplyToCommandInFlight){ + // Need to remove here because it is an + // in-stream response while streaming so not + // handled elsewhere + if(getListofInstructions().size()>0){ + removeInstruction(0); + } + + mTransactionCompleted=true; + setInstructionStackLock(false); } - - mTransactionCompleted=true; - setInstructionStackLock(false); //Then process the Data packet processDataPacket(bufferTemp); @@ -1203,76 +1215,94 @@ else if(currentCommand==GET_INFOMEM_COMMAND || currentCommand==GET_CALIB_DUMP_CO } } + /** + * Whether an in-stream response is the reply to the command in flight. The + * protocol matches a reply to its command by the response opcode, never by the + * ACK (log-and-stream-common docs/SHIMMER3_BT_COMMUNICATION_PROTOCOL.md + * section 5.4, rule 2). + */ + private boolean isReplyToCommandInFlight(byte inStreamResponseCommand) { + if(!mWaitForAck && !mWaitForResponse){ + return false; + } + BtCommandDetails commandDetails = mBtGetCommandMap.get(mCurrentCommand); + return commandDetails!=null && commandDetails.expectedResponse==inStreamResponseCommand; + } + /** process responses to in-stream response */ protected void processInstreamResponse(boolean shouldClearCrcFromBuffer) { - boolean responseWasParsed = true; - byte[] inStreamResponseCommandBuffer = readBytes(1, INSTREAM_CMD_RESPONSE); if(inStreamResponseCommandBuffer!=null){ - byte inStreamResponseCommand = inStreamResponseCommandBuffer[0]; - consolePrintLn("In-stream received = " + btCommandToString(inStreamResponseCommand)); + processInstreamResponse(inStreamResponseCommandBuffer[0], shouldClearCrcFromBuffer); + } + } - if(inStreamResponseCommand==DIR_RESPONSE){ - byte[] responseData = readBytes(1, inStreamResponseCommand); - if(responseData!=null){ - int directoryNameLength = responseData[0]; - byte[] bufferDirectoryName = new byte[directoryNameLength]; - bufferDirectoryName = readBytes(directoryNameLength, inStreamResponseCommand); - if(bufferDirectoryName!=null){ - String tempDirectory = new String(bufferDirectoryName); - mDirectoryName = tempDirectory; - printLogDataForDebugging("Directory Name = " + mDirectoryName); - } + /** process an in-stream response whose response command has already been read */ + protected void processInstreamResponse(byte inStreamResponseCommand, boolean shouldClearCrcFromBuffer) { + boolean responseWasParsed = true; + + consolePrintLn("In-stream received = " + btCommandToString(inStreamResponseCommand)); + + if(inStreamResponseCommand==DIR_RESPONSE){ + byte[] responseData = readBytes(1, inStreamResponseCommand); + if(responseData!=null){ + int directoryNameLength = responseData[0]; + byte[] bufferDirectoryName = new byte[directoryNameLength]; + bufferDirectoryName = readBytes(directoryNameLength, inStreamResponseCommand); + if(bufferDirectoryName!=null){ + String tempDirectory = new String(bufferDirectoryName); + mDirectoryName = tempDirectory; + printLogDataForDebugging("Directory Name = " + mDirectoryName); } } - else if(inStreamResponseCommand==STATUS_RESPONSE){ - int statusBytesToRead = 0; - if(isSupportedUSBPluggedInStatus()) { - statusBytesToRead = 2; - }else { - statusBytesToRead = 1; - } - byte[] responseData = readBytes(statusBytesToRead, inStreamResponseCommand); - if(responseData!=null){ - parseStatusByte(responseData); - - if(!isSupportedRtcStateInStatus()){ - if(!mIsSensing && !isInitialised()){ - writeRealTimeClock(); - } - } else { - //New case to make sure RTC is set if it hasn't been already - if(!isSDLogging() && (!isInitialised() || !mIsRtcSet)){ - writeRealTimeClock(); - } + } + else if(inStreamResponseCommand==STATUS_RESPONSE){ + int statusBytesToRead = 0; + if(isSupportedUSBPluggedInStatus()) { + statusBytesToRead = 2; + }else { + statusBytesToRead = 1; + } + byte[] responseData = readBytes(statusBytesToRead, inStreamResponseCommand); + if(responseData!=null){ + parseStatusByte(responseData); + + if(!isSupportedRtcStateInStatus()){ + if(!mIsSensing && !isInitialised()){ + writeRealTimeClock(); + } + } else { + //New case to make sure RTC is set if it hasn't been already + if(!isSDLogging() && (!isInitialised() || !mIsRtcSet)){ + writeRealTimeClock(); } - - eventLogAndStreamStatusChanged(mCurrentCommand); - } - } - else if(inStreamResponseCommand==VBATT_RESPONSE) { - byte[] responseData = readBytes(3, inStreamResponseCommand); - if(responseData!=null){ - ShimmerBattStatusDetails battStatusDetails = new ShimmerBattStatusDetails(((responseData[1]&0xFF)<<8)+(responseData[0]&0xFF),responseData[2]); - setBattStatusDetails(battStatusDetails); - printLogDataForDebugging("Battery Status:" - + "\tVoltage=" + battStatusDetails.getBattVoltageParsed() - + "\tCharging status=" + battStatusDetails.getChargingStatusParsed() - + "\tBatt %=" + battStatusDetails.getEstimatedChargePercentageParsed()); } + + eventLogAndStreamStatusChanged(mCurrentCommand); } - else - { - responseWasParsed = false; - } - - if (shouldClearCrcFromBuffer && mBtCommsCrcModeCurrent != BT_CRC_MODE.OFF && responseWasParsed) { - // The driver doesn't currently support checking of CRC bytes on any response - // other then data packets but we still need to clear them from the buffer if - // CRC is enabled. - clearCrcBytesFromBuffer(inStreamResponseCommand); + } + else if(inStreamResponseCommand==VBATT_RESPONSE) { + byte[] responseData = readBytes(3, inStreamResponseCommand); + if(responseData!=null){ + ShimmerBattStatusDetails battStatusDetails = new ShimmerBattStatusDetails(((responseData[1]&0xFF)<<8)+(responseData[0]&0xFF),responseData[2]); + setBattStatusDetails(battStatusDetails); + printLogDataForDebugging("Battery Status:" + + "\tVoltage=" + battStatusDetails.getBattVoltageParsed() + + "\tCharging status=" + battStatusDetails.getChargingStatusParsed() + + "\tBatt %=" + battStatusDetails.getEstimatedChargePercentageParsed()); } } + else + { + responseWasParsed = false; + } + + if (shouldClearCrcFromBuffer && mBtCommsCrcModeCurrent != BT_CRC_MODE.OFF && responseWasParsed) { + // The driver doesn't currently support checking of CRC bytes on any response + // other then data packets but we still need to clear them from the buffer if + // CRC is enabled. + clearCrcBytesFromBuffer(inStreamResponseCommand); + } } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00030_StreamingInstreamResponseQueueTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00030_StreamingInstreamResponseQueueTest.java new file mode 100644 index 000000000..1aba75222 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00030_StreamingInstreamResponseQueueTest.java @@ -0,0 +1,277 @@ +package com.shimmerresearch.bluetooth; + +import static org.junit.Assert.*; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; + +import org.junit.After; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_CRC_MODE; +import com.shimmerresearch.comms.wiredProtocol.ShimmerCrc; +import com.shimmerresearch.driver.ShimmerObject; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails.FW_ID; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID; +import com.shimmerresearch.driverUtilities.ShimmerVerObject; +import com.shimmerresearch.pcDriver.ShimmerPC; +import com.shimmerresearch.shimmer3.communication.ByteCommunication; +import com.shimmerresearch.verisense.communication.ByteCommunicationListener; + +import jssc.SerialPortTimeoutException; + +/** + * DEV-1262: while streaming, the branch of processPacket() for an in-stream + * response (FF 8A ...) completed the command in flight whatever the response + * was: it cancelled the ACK timer, cleared the wait flags, removed the head of + * the instruction queue and unlocked it. The firmware also sends a status + * response unprompted (dock, undock, the user button, a low-battery stop), so + * such a push removed a SET command that was still waiting for its ACK, or a + * queued instruction not yet sent, and the next ACK was matched to the wrong + * instruction. The protocol matches a response to the command in flight by + * its opcode (log-and-stream-common docs/SHIMMER3_BT_COMMUNICATION_PROTOCOL.md + * section 5.4, rule 2). + *

+ * No hardware: the parser reads from an in-memory byte queue, one byte at a + * time as the IO thread does, as in API_00027. + */ +public class API_00030_StreamingInstreamResponseQueueTest { + + /** Bytes after the DATA_PACKET byte: a 3-byte timestamp and three 2-byte channels */ + private static final int PACKET_SIZE = 9; + + /** Docked, sensing, RTC set and streaming; not SD logging */ + private static final byte STATUS_DOCKED_AND_STREAMING = 0x17; + + private static class QueueRadio implements ByteCommunication { + final Deque rx = new ArrayDeque(); + + void queue(byte... bytes) { + for(byte b:bytes){ + rx.add(b); + } + } + + @Override + public byte[] readBytes(int byteCount, int timeout) throws SerialPortTimeoutException { + if(rx.size() Date: Wed, 21 Oct 2026 16:01:05 +0800 Subject: [PATCH 26/41] DEV-1269 Java HIL v1: the driver on real Shimmer3 and Shimmer3R over Classic, and a release gate Hardware-in-the-loop tests of ShimmerDriver/ShimmerDriverPC on a PC with paired units, through ShimmerPC with its timers on, as Consensys and MATLAB run it. - JavaHilScenariosTest (hilTest task, skipped without -Phil.port, left out of the normal test task): streamRest, midStreamSet, reconnectLoop and inStream. inStream requests each in-stream reply (status, directory, battery) and then overlaps SET_RWC with status and battery requests, the DEV-1262 case. Every API call runs under a watchdog that dumps every thread with its locks. - HilShimmerPC, a test-only subclass, counts in-stream replies by type, decoded statuses and battery readings, and captures every driver log line, without changing what the driver does. The configure step does not write the calibration dump back, so runs leave the units' stored calibration alone. - StreamAnalysis is the Android HIL's class (same rules on both); its tests run in CI. - tools/hil/run-java-hil.ps1 finds each unit's COM port from its MAC, builds once, runs each scenario in its own Gradle run under a time limit, checks every report came back, and with -Report posts the commit status java-hil (full default run, clean pushed commit only). - Release Java API stops unless java-hil passed on the commit, or hil_waiver gives a Jira key; the release notes record which. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 52 +- .gitignore | 4 + README.md | 6 + ShimmerDriverPC/build.gradle | 22 + .../com/shimmerresearch/hil/HilReport.java | 161 +++++++ .../com/shimmerresearch/hil/HilSession.java | 276 +++++++++++ .../com/shimmerresearch/hil/HilShimmerPC.java | 170 +++++++ .../hil/JavaHilScenariosTest.java | 453 ++++++++++++++++++ .../shimmerresearch/hil/StreamAnalysis.java | 196 ++++++++ .../hil/StreamAnalysisTest.java | 102 ++++ tools/hil/README.md | 121 +++++ tools/hil/rig-java.example.json | 6 + tools/hil/run-java-hil.ps1 | 234 +++++++++ 13 files changed, 1802 insertions(+), 1 deletion(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java create mode 100644 tools/hil/README.md create mode 100644 tools/hil/rig-java.example.json create mode 100644 tools/hil/run-java-hil.ps1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83c66e2f8..7f1dcf178 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,9 @@ # One-click release of the Java API: GitHub Packages, a GitHub Release, and the MATLAB jar. # # How to release +# 0. On the bench, at the commit to release: tools/hil/run-java-hil.ps1 -Report (tools/hil/README.md). +# It posts the commit status "java-hil"; the build job stops unless that passed, or hil_waiver +# gives a Jira key (the waiver goes into the release notes). Dry runs only warn. # 1. Actions -> "Release Java API" -> Run workflow (on master). Leave `version` blank for the next # patch after the last API_v release (0.11.8_beta -> 0.11.9_beta), or type one. # 2. Review and merge the ShimmerJavaClass.jar PR it opens in Shimmer-MATLAB-ID. @@ -46,6 +49,10 @@ on: description: 'Build and test only; publish nothing' type: boolean default: false + hil_waiver: + description: 'Jira key to release without a passing hardware test (HIL), e.g. DEV-1234. Blank = require one' + type: string + required: false pull_request: paths: - '.github/workflows/release.yml' @@ -62,12 +69,14 @@ jobs: # job's token. Only the publish job, which runs on real releases alone, can write packages. permissions: contents: read + statuses: read # the hardware test result the bench posts on the commit env: DRY_RUN: ${{ github.event_name != 'workflow_dispatch' || inputs.dry_run }} outputs: version: ${{ steps.version.outputs.version }} tag: ${{ steps.version.outputs.tag }} previous_tag: ${{ steps.version.outputs.previous_tag }} + hil: ${{ steps.hil.outputs.hil }} steps: - name: Release from master only if: env.DRY_RUN != 'true' && github.ref != 'refs/heads/master' @@ -121,6 +130,35 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Releasing $VERSION as $TAG (notes since ${PREVIOUS_TAG:-the first commit})" + - name: Check the hardware test (HIL) result + id: hil + env: + GH_TOKEN: ${{ github.token }} + WAIVER: ${{ inputs.hil_waiver }} + run: | + context=java-hil + if [ -n "$WAIVER" ] && ! [[ "$WAIVER" =~ ^[A-Z][A-Z0-9]+-[0-9]+$ ]]; then + echo "::error::hil_waiver must be a Jira key such as DEV-1234, not '$WAIVER'." + exit 1 + fi + # Newest first: the latest result for this context on this exact commit decides. + line=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/statuses" --paginate \ + --jq '.[] | [.context, .state, .description] | @tsv' | awk -F'\t' -v c="$context" '$1 == c { print; exit }') + state=$(printf '%s' "$line" | cut -f2) + description=$(printf '%s' "$line" | cut -f3) + if [ "$state" = success ]; then + echo "Hardware test passed on $GITHUB_SHA: $description" + echo "hil=Tested on hardware before release: $description." >> "$GITHUB_OUTPUT" + elif [ -n "$WAIVER" ]; then + echo "::warning::No passing '$context' on $GITHUB_SHA (found: ${state:-none}); released under waiver $WAIVER." + echo "hil=Released without a passing hardware test (HIL), waived under $WAIVER." >> "$GITHUB_OUTPUT" + elif [ "$DRY_RUN" = true ]; then + echo "::warning::No passing '$context' on $GITHUB_SHA (found: ${state:-none}). A real release would stop here." + else + echo "::error::No passing '$context' on $GITHUB_SHA (found: ${state:-none}${description:+: $description}). On the bench, at this commit, run: tools/hil/run-java-hil.ps1 -Report. Then run this workflow again, or give a Jira key in hil_waiver." + exit 1 + fi + - name: Set up JDK 11 uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 with: @@ -272,6 +310,7 @@ jobs: VERSION: ${{ needs.build.outputs.version }} TAG: ${{ needs.build.outputs.tag }} PREVIOUS_TAG: ${{ needs.build.outputs.previous_tag }} + HIL: ${{ needs.build.outputs.hil }} steps: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: @@ -286,16 +325,27 @@ jobs: - name: Create the tag and GitHub Release run: | cp matlab-jar/ShimmerJavaClass.jar "release-assets/ShimmerJavaClass-$VERSION.jar" + # The generated notes, with the hardware test result (or its waiver) in front. + args=(-f tag_name="$TAG" -f target_commitish="$GITHUB_SHA") + if [ -n "$PREVIOUS_TAG" ]; then args+=(-f previous_tag_name="$PREVIOUS_TAG"); fi + { + if [ -n "$HIL" ]; then + echo "$HIL" + echo + fi + gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" "${args[@]}" --jq .body + } > notes.md gh release create "$TAG" release-assets/* \ --repo "$GITHUB_REPOSITORY" \ --target "$GITHUB_SHA" \ --title "Shimmer Java API Rev $VERSION" \ - --generate-notes ${PREVIOUS_TAG:+--notes-start-tag "$PREVIOUS_TAG"} \ + --notes-file notes.md \ --latest=false { echo "### Released Java API $VERSION" echo "- Release: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$TAG" echo "- Packages: com.shimmerresearch:shimmerdriver, shimmerbluetoothmanager and shimmerdriverpc at $VERSION" + echo "- ${HIL:-No hardware test result}" echo "- Next: merge the MATLAB PR, then run Release Android API in ShimmerAndroidAPI; it builds against $VERSION unless you pick another driver version." } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 19c5b1390..a480c7ae5 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,7 @@ ShimmerAndroidInstrumentDriver/gen/com/shimmerresearch/androidinstrumentdriver/B ShimmerAndroidInstrumentDriver/gen/com/shimmerresearch/androidinstrumentdriver/R.java gradle.properties github.properties + +# Java HIL (tools/hil): the local rig file and run output +tools/hil/rig-java.json +tools/hil/out/ diff --git a/README.md b/README.md index ab212feab..708d6f538 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ Best examples to start with are For help with using on Ubuntu please check the [wiki](https://github.com/ShimmerEngineering/Shimmer-Java-Android-API/wiki). +## Hardware tests + +[`tools/hil`](tools/hil/README.md) runs the driver on real Shimmer3 and Shimmer3R units over +Bluetooth Classic: streaming, SD logging and in-stream commands while streaming, and reconnects, with +the driver's timers on. **Release Java API** requires a passing run on the commit it releases. + Rev 3.00 - Change to Handler MSGs diff --git a/ShimmerDriverPC/build.gradle b/ShimmerDriverPC/build.gradle index b60226f77..b452317e4 100644 --- a/ShimmerDriverPC/build.gradle +++ b/ShimmerDriverPC/build.gradle @@ -32,6 +32,26 @@ repositories { test { // this test requires a physical Shimmer2r to run exclude '**/API_00001_ShimmerPC_GeneralBluetoothShimmer2R.class' + // the hardware-in-the-loop scenarios need a unit; hilTest runs them (tools/hil/run-java-hil.ps1) + exclude '**/hil/JavaHilScenariosTest.class' +} + +// Hardware-in-the-loop scenarios on a real Shimmer over a paired Bluetooth serial port, run by +// tools/hil/run-java-hil.ps1 (see tools/hil/README.md). Each -Phil.= reaches the tests +// as the system property hil.. +tasks.register('hilTest', Test) { + description = 'Runs the hardware-in-the-loop scenarios on a real unit (tools/hil/run-java-hil.ps1)' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + include '**/hil/JavaHilScenariosTest.class' + dependsOn 'testClasses' + systemProperties project.properties.findAll { k, v -> k.startsWith('hil.') } + outputs.upToDateWhen { false } + testLogging { + events 'passed', 'skipped', 'failed' + showStandardStreams = true + exceptionFormat 'full' + } } publishing { @@ -103,6 +123,8 @@ dependencies { // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add // 'test.useTestNG()' to your build script. testImplementation 'junit:junit:4.12' + // the HIL reports (src/test/java/com/shimmerresearch/hil) + testImplementation 'com.google.code.gson:gson:2.10.1' implementation 'com.parse.bolts:bolts-tasks:1.4.0' // https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java new file mode 100644 index 000000000..683586284 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java @@ -0,0 +1,161 @@ +package com.shimmerresearch.hil; + +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * Everything one scenario run found: a JSON report, a CSV of the samples it judged and the + * driver's own log, written to the folder the runner passes (hil.outDir). + */ +final class HilReport { + + private final String scenario; + private final String unit; + private final JsonObject json = new JsonObject(); + private final List failures = new ArrayList<>(); + private List samples; + + HilReport(String scenario, String unit, String port, String mac, String type, Map args) { + this.scenario = scenario; + this.unit = unit; + json.addProperty("scenario", scenario); + json.addProperty("unit", unit); + json.addProperty("port", port); + json.addProperty("mac", mac); + json.addProperty("type", type); + json.addProperty("started", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ROOT).format(new Date())); + JsonObject a = new JsonObject(); + args.forEach(a::addProperty); + json.add("args", a); + } + + /** JSON has no NaN or infinity, and a failing run is the likeliest to produce one: store null. */ + static JsonElement num(double v) { + return Double.isNaN(v) || Double.isInfinite(v) ? JsonNull.INSTANCE : new JsonPrimitive(v); + } + + void put(String key, Object value) { + if (value == null) { + json.add(key, JsonNull.INSTANCE); + } else if (value instanceof Double || value instanceof Float) { + json.add(key, num(((Number) value).doubleValue())); + } else if (value instanceof Number) { + json.addProperty(key, (Number) value); + } else if (value instanceof Boolean) { + json.addProperty(key, (Boolean) value); + } else if (value instanceof Map) { + JsonObject o = new JsonObject(); + ((Map) value).forEach((k, v) -> o.addProperty(String.valueOf(k), v instanceof Number ? (Number) v : null)); + json.add(key, o); + } else if (value instanceof List) { + JsonArray arr = new JsonArray(); + for (Object o : (List) value) { + arr.add(o instanceof Number ? new JsonPrimitive((Number) o) : new JsonPrimitive(String.valueOf(o))); + } + json.add(key, arr); + } else { + json.addProperty(key, String.valueOf(value)); + } + } + + void putStream(String key, StreamAnalysis a) { + JsonObject o = new JsonObject(); + o.addProperty("received", a.received); + o.addProperty("missing", a.missing); + o.add("lossPercent", num(a.lossPercent())); + o.addProperty("nonIncreasing", a.nonIncreasing); + o.addProperty("timestampJumps", a.timestampJumps.size()); + o.add("effectiveRateHz", num(a.effectiveRateHz)); + o.add("accelMagnitudeMean", num(a.accelMagnitudeMean)); + o.add("accelMagnitudeStd", num(a.accelMagnitudeStd)); + o.addProperty("gaps", a.gaps.size()); + json.add(key, o); + } + + /** Every gap and timestamp jump, with when it arrived, to match against what the test was doing. */ + void putGaps(StreamAnalysis a) { + JsonArray gaps = new JsonArray(); + for (StreamAnalysis.Gap g : a.gaps) { + JsonObject o = new JsonObject(); + o.addProperty("afterReceivedMs", Math.round(g.afterReceivedMs)); + o.add("afterTimestampMs", num(g.afterTimestampMs)); + o.addProperty("missing", g.missing); + gaps.add(o); + } + json.add("gapList", gaps); + JsonArray jumps = new JsonArray(); + for (StreamAnalysis.Gap g : a.timestampJumps) { + JsonObject o = new JsonObject(); + o.addProperty("afterReceivedMs", Math.round(g.afterReceivedMs)); + o.add("afterTimestampMs", num(g.afterTimestampMs)); + o.addProperty("periodsAhead", g.missing + 1); + jumps.add(o); + } + json.add("timestampJumpList", jumps); + } + + void samples(List samples) { + this.samples = samples; + } + + /** Records a failed check; the test fails at the end with every one of them. */ + void check(boolean ok, String failure) { + if (!ok) { + failures.add(failure); + System.out.println(scenario + " " + unit + " FAIL: " + failure); + } + } + + List failures() { + return failures; + } + + /** Writes _.json, .csv (samples, if any) and .driver.log into outDir. */ + void write(File outDir, List events, List driverLog) { + try { + json.addProperty("passed", failures.isEmpty()); + JsonArray f = new JsonArray(); + failures.forEach(f::add); + json.add("failures", f); + JsonArray e = new JsonArray(); + events.forEach(e::add); + json.add("events", e); + if (!outDir.exists() && !outDir.mkdirs()) { + throw new IOException("cannot create " + outDir); + } + String base = scenario + "_" + unit.replaceAll("[^\\w.-]", "_"); + try (Writer w = Files.newBufferedWriter(new File(outDir, base + ".json").toPath(), StandardCharsets.UTF_8)) { + new GsonBuilder().setPrettyPrinting().create().toJson(json, w); + } + if (samples != null) { + try (Writer w = Files.newBufferedWriter(new File(outDir, base + ".csv").toPath(), StandardCharsets.UTF_8)) { + w.write("received_ms,timestamp_ms,accel_x,accel_y,accel_z\n"); + for (StreamAnalysis.Sample s : samples) { + w.write(String.format(Locale.ROOT, "%.0f,%.3f,%.4f,%.4f,%.4f\n", + s.receivedMs, s.timestampMs, s.accelX, s.accelY, s.accelZ)); + } + } + } + Files.write(new File(outDir, base + ".driver.log").toPath(), driverLog, StandardCharsets.UTF_8); + System.out.println("report written: " + new File(outDir, base + ".json")); + } catch (IOException ex) { + System.out.println("could not write report: " + ex); + } + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java new file mode 100644 index 000000000..453d829d1 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java @@ -0,0 +1,276 @@ +package com.shimmerresearch.hil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.driver.BasicProcessWithCallBack; +import com.shimmerresearch.driver.CallbackObject; +import com.shimmerresearch.driver.Configuration; +import com.shimmerresearch.driver.FormatCluster; +import com.shimmerresearch.driver.ObjectCluster; +import com.shimmerresearch.driver.ShimmerDevice; +import com.shimmerresearch.driver.ShimmerMsg; +import com.shimmerresearch.driverUtilities.AssembleShimmerConfig; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; + +/** + * One unit on one serial port, driven through ShimmerPC as an app drives it. Every API call runs + * under a watchdog: if one does not return, the session records which, dumps every thread (with + * the locks each holds and waits on) and refuses further calls, so a deadlock ends the scenario + * with evidence instead of hanging the run. + */ +class HilSession extends BasicProcessWithCallBack implements AutoCloseable { + + interface Call { + void run() throws Exception; + } + + interface Condition { + boolean holds(); + } + + final String port; + final String mac; + final String type; + final HilShimmerPC device; + private final long t0 = System.nanoTime(); + private final List events = new ArrayList<>(); + private final List samples = new ArrayList<>(); + private final Set otherMessagesSeen = new HashSet<>(); + private final ExecutorService calls = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "HilCall"); + t.setDaemon(true); + return t; + }); + private volatile boolean recording; + private volatile BT_STATE lastState; + /** The call that never returned, if one did not. */ + volatile String hung; + /** Connects made in this session, for the report: each is a chance for the connect-time race. */ + int connects; + + HilSession(String port, String mac, String type) { + this.port = port; + this.mac = mac; + this.type = type; + this.device = new HilShimmerPC(port, t0); + setWaitForData(device); + } + + double nowMs() { + return (System.nanoTime() - t0) / 1e6; + } + + void event(String text) { + String line = String.format(Locale.ROOT, "%9.0f %s", nowMs(), text); + synchronized (events) { + events.add(line); + } + System.out.println(port + " " + line); + } + + List events() { + synchronized (events) { + return new ArrayList<>(events); + } + } + + @Override + protected void processMsgFromCallback(ShimmerMsg msg) { + if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_STATE_CHANGE && msg.mB instanceof CallbackObject) { + BT_STATE state = ((CallbackObject) msg.mB).mState; + if (state != null) { + lastState = state; + event("state " + state); + } + } else if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_DATA_PACKET && msg.mB instanceof ObjectCluster) { + if (recording) { + recordPacket((ObjectCluster) msg.mB); + } + } else if (otherMessagesSeen.add(msg.mIdentifier)) { + event("message " + msg.mIdentifier + (msg.mB == null ? "" : " " + msg.mB.getClass().getSimpleName())); + } + } + + private void recordPacket(ObjectCluster oc) { + StreamAnalysis.Sample s = new StreamAnalysis.Sample(nowMs(), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.TIMESTAMP), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_X), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Y), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Z)); + synchronized (samples) { + samples.add(s); + } + } + + private static double cal(ObjectCluster oc, String channel) { + Collection formats = oc.getCollectionOfFormatClusters(channel); + FormatCluster c = (FormatCluster) ObjectCluster.returnFormatCluster(formats, "CAL"); + return c == null ? Double.NaN : c.mData; + } + + BT_STATE state() { + BT_STATE s = device.getBluetoothRadioState(); + return s != null ? s : lastState; + } + + boolean waitFor(Condition condition, long timeoutMs) throws InterruptedException { + long deadline = System.nanoTime() + timeoutMs * 1_000_000L; + while (System.nanoTime() < deadline) { + if (condition.holds()) { + return true; + } + Thread.sleep(20); + } + return condition.holds(); + } + + /** + * Runs an API call under the watchdog. After a hung call nothing more is run: the driver is in + * an unknown state, and queuing calls behind the stuck one only adds stuck threads. + */ + boolean call(String what, long timeoutMs, Call call) { + if (hung != null) { + event("skipped " + what + ": " + hung + " never returned"); + return false; + } + event(what); + Future f = calls.submit(() -> { + call.run(); + return null; + }); + try { + f.get(timeoutMs, TimeUnit.MILLISECONDS); + return true; + } catch (TimeoutException e) { + hung = what; + event(what + " did not return within " + timeoutMs + " ms; every thread:\n" + + ByteCommunicationTestUtils.dumpAllThreads()); + return false; + } catch (java.util.concurrent.ExecutionException e) { + event(what + " threw " + e.getCause()); + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** Connects and waits until the driver has read the unit's configuration. */ + boolean connect(long timeoutMs) throws InterruptedException { + connects++; + if (!call("connect on " + port, 10000, () -> device.connect(port, ""))) { + return false; + } + boolean ready = waitFor(() -> device.isInitialised() && device.getBluetoothRadioState() == BT_STATE.CONNECTED, timeoutMs); + event("connected: timers " + device.timers()); + return ready; + } + + /** + * Accel only (the low-noise accel on Shimmer3, the LSM6DSV's on Shimmer3R) at rateHz, written the + * way Consensys's Bluetooth manager does it: a clone, AssembleShimmerConfig, then + * operationPrepare, configureFromClone and operationStart(CONFIGURING) on the connected device, + * except that the calibration is not written back. + */ + boolean configureAccelOnly(double rateHz, long timeoutMs) throws InterruptedException { + ShimmerDevice clone = device.deepClone(); + clone.disableAllSensors(); + if (clone.getHardwareVersion() == ShimmerVerDetails.HW_ID.SHIMMER_3) { + clone.setSensorEnabledState(Configuration.Shimmer3.SENSOR_ID.SHIMMER_ANALOG_ACCEL, true); + } else { + clone.setSensorEnabledState(Configuration.Shimmer3.SENSOR_ID.SHIMMER_LSM6DSV_ACCEL_LN, true); + } + clone.setShimmerAndSensorsSamplingRate(rateHz); + AssembleShimmerConfig.generateSingleShimmerConfig(clone, Configuration.COMMUNICATION_TYPE.BLUETOOTH); + // Leave the unit's stored calibration alone. configureFromClone otherwise writes the clone's + // calibration dump back (AA-246), which would rewrite the bench units' calibration on every run. + ((ShimmerBluetooth) clone).setWriteCalibrationDumpWhenConfiguringForClone(false); + int logMark = device.logSize(); + boolean sent = call(String.format(Locale.ROOT, "configure accel only at %.1f Hz", rateHz), 10000, () -> { + device.operationPrepare(); + device.configureFromClone(clone); + device.operationStart(BT_STATE.CONFIGURING); + }); + if (!sent) { + return false; + } + // The operation ends by reading the configuration back, finishing with an INQUIRY. Wait for that + // reply (logged after this configure began), an empty queue and the requested rate read back: + // state and rate alone can already hold before the write starts, if the unit kept them. + boolean done = waitFor(() -> device.countSince(logMark, "Inquiry Response Received") > 0 + && device.queuedInstructions() == 0 && state() == BT_STATE.CONNECTED + && Math.abs(device.getSamplingRateShimmer() - rateHz) < 0.5, timeoutMs); + event(String.format(Locale.ROOT, "configure %s: rate on device %.2f Hz, timers %s", done ? "finished" : "timed out", + device.getSamplingRateShimmer(), device.timers())); + return done; + } + + void startRecording() { + synchronized (samples) { + samples.clear(); + } + recording = true; + } + + List stopRecording() { + recording = false; + synchronized (samples) { + return new ArrayList<>(samples); + } + } + + boolean startStreaming(long timeoutMs) throws InterruptedException { + return call("startStreaming", 10000, device::startStreaming) && waitFor(device::isStreaming, timeoutMs); + } + + boolean stopStreaming(long timeoutMs) throws InterruptedException { + event("while streaming: timers " + device.timers()); + return call("stopStreaming", 10000, device::stopStreaming) && waitFor(() -> !device.isStreaming(), timeoutMs); + } + + boolean disconnect(long timeoutMs) throws InterruptedException { + return call("disconnect", 10000, device::disconnect) && waitFor(() -> { + BT_STATE s = state(); + return s == BT_STATE.DISCONNECTED || s == BT_STATE.CONNECTION_LOST; + }, timeoutMs); + } + + /** Instructions waiting to be sent, read as the driver's own accessors read them. */ + int queuedInstructions() { + return device.queuedInstructions(); + } + + @Override + public void close() { + try { + // After a hung call leave the unit alone; the runner's timeout ends the JVM. + if (hung == null) { + BT_STATE s = state(); + if (s != null && s != BT_STATE.DISCONNECTED && s != BT_STATE.CONNECTION_LOST) { + if (device.isStreaming()) { + stopStreaming(5000); + } + disconnect(10000); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + removeSetWaitForData(device); + calls.shutdownNow(); + } + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java new file mode 100644 index 000000000..932a18460 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java @@ -0,0 +1,170 @@ +package com.shimmerresearch.hil; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; + +import com.shimmerresearch.driverUtilities.ShimmerBattStatusDetails; +import com.shimmerresearch.pcDriver.ShimmerPC; + +/** + * A ShimmerPC that behaves exactly as an app's does (timers on, the real serial port) but counts + * what the tests need to see: each in-stream reply by type, each status the driver decoded, each + * battery reading, and every line the driver logs, whether or not verbose mode is on. Nothing here + * changes what the driver does. + */ +class HilShimmerPC extends ShimmerPC { + + private static final long serialVersionUID = 1L; + + /** Lines kept per session; enough for minutes of commands, bounded if something loops. */ + private static final int MAX_LOG_LINES = 50000; + + private final long t0; + private final Map inStreamReplies = new TreeMap<>(); + private final List log = new ArrayList<>(); + private volatile int statusesDecoded; + private volatile int batteryReadings; + + HilShimmerPC(String comPort, long t0Nanos) { + super(comPort); + this.t0 = t0Nanos; + } + + @Override + protected void processInstreamResponse(byte inStreamResponseCommand, boolean shouldClearCrcFromBuffer) { + synchronized (inStreamReplies) { + inStreamReplies.merge(replyName(inStreamResponseCommand), 1, Integer::sum); + } + super.processInstreamResponse(inStreamResponseCommand, shouldClearCrcFromBuffer); + } + + @Override + protected void eventLogAndStreamStatusChanged(byte currentCommand) { + statusesDecoded++; + super.eventLogAndStreamStatusChanged(currentCommand); + } + + @Override + public void setBattStatusDetails(ShimmerBattStatusDetails s) { + batteryReadings++; + super.setBattStatusDetails(s); + } + + @Override + public void consolePrintLn(String message) { + record(message); + super.consolePrintLn(message); + } + + @Override + public void consolePrintErrLn(String message) { + record("ERR " + message); + super.consolePrintErrLn(message); + } + + private void record(String message) { + String line = String.format(Locale.ROOT, "%9.0f %s", (System.nanoTime() - t0) / 1e6, message); + synchronized (log) { + if (log.size() < MAX_LOG_LINES) { + log.add(line); + } + } + } + + static String replyName(byte b) { + switch (b) { + case STATUS_RESPONSE: + return "STATUS"; + case DIR_RESPONSE: + return "DIR"; + case VBATT_RESPONSE: + return "VBATT"; + default: + return String.format(Locale.ROOT, "0x%02X", b & 0xFF); + } + } + + int inStreamReplies(String type) { + synchronized (inStreamReplies) { + return inStreamReplies.getOrDefault(type, 0); + } + } + + Map inStreamReplies() { + synchronized (inStreamReplies) { + return new TreeMap<>(inStreamReplies); + } + } + + /** Which of the driver's periodic timers exist right now: an app runs with all three. */ + String timers() { + List t = new ArrayList<>(); + if (mTimerReadStatus != null) { + t.add("status"); + } + if (mTimerCheckAlive != null) { + t.add("alive"); + } + if (mTimerReadBattStatus != null) { + t.add("battery"); + } + return t.isEmpty() ? "none" : String.join("+", t); + } + + /** Instructions waiting to be sent, read through the driver's own (synchronized) accessor. */ + int queuedInstructions() { + return getListofInstructions().size(); + } + + int statusesDecoded() { + return statusesDecoded; + } + + int batteryReadings() { + return batteryReadings; + } + + int logSize() { + synchronized (log) { + return log.size(); + } + } + + List log() { + synchronized (log) { + return new ArrayList<>(log); + } + } + + /** Lines logged since index from that match the pattern anywhere. */ + int countSince(int from, java.util.regex.Pattern pattern) { + int n = 0; + synchronized (log) { + for (int i = Math.max(0, from); i < log.size(); i++) { + if (pattern.matcher(log.get(i)).find()) { + n++; + } + } + } + return n; + } + + /** Lines logged since index from that contain any of the given texts. */ + int countSince(int from, String... texts) { + int n = 0; + synchronized (log) { + for (int i = Math.max(0, from); i < log.size(); i++) { + for (String t : texts) { + if (log.get(i).contains(t)) { + n++; + break; + } + } + } + } + return n; + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java new file mode 100644 index 000000000..7e6d2ef67 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -0,0 +1,453 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_CRC_MODE; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; + +/** + * Hardware-in-the-loop scenarios for the Java driver: a real Shimmer3 or Shimmer3R on a paired + * Bluetooth serial port, driven through ShimmerPC with its timers on, as Consensys and MATLAB run + * it. Run by tools/hil/run-java-hil.ps1 through the hilTest task, one scenario per JVM. Without + * hil.port every test is skipped, so the class never runs (or fails) without hardware. + * + *

+ *   hil.port            COM port of the unit (required)
+ *   hil.mac, hil.unit   for the report
+ *   hil.type            Shimmer3 (default) or Shimmer3R
+ *   hil.rate            sampling rate in Hz, default 256
+ *   hil.seconds         streamRest duration, default 30
+ *   hil.cycles          midStreamSet / reconnectLoop / inStream repetitions, default 3 / 5 / 3
+ *   hil.overlaps        inStream overlap rounds, default 5
+ *   hil.crc             off, 1 or 2; default: the driver's own default
+ *   hil.maxLossPercent  default 1.0
+ *   hil.outDir          where reports go, default build/hil
+ * 
+ */ +public class JavaHilScenariosTest { + + private static final double GRAVITY = 9.81; + /** At 256 Hz a second without a packet is ~256 packets the gap count cannot see. */ + private static final double MAX_SILENCE_AT_END_MS = 1000; + /** The ACK/response timer's own line when a command gets no reply: "Command: timeout". */ + private static final java.util.regex.Pattern COMMAND_TIMEOUT = java.util.regex.Pattern.compile("Command:\t\\S+ timeout"); + + private HilSession session; + private HilReport report; + private HilShimmerPC device; + private int logStart; + + private static String arg(String name, String fallback) { + String v = System.getProperty("hil." + name); + return v == null || v.isEmpty() ? fallback : v; + } + + private static double arg(String name, double fallback) { + return Double.parseDouble(arg(name, Double.toString(fallback))); + } + + private static int arg(String name, int fallback) { + return Integer.parseInt(arg(name, Integer.toString(fallback))); + } + + private void begin(String scenario) { + String port = arg("port", null); + assumeTrue("No unit given: pass -Phil.port=COMx (tools/hil/run-java-hil.ps1 does)", port != null); + String crc = arg("crc", null); + if (crc != null) { + ShimmerBluetooth.setDefaultBtCrcModeToUseIfFwSupported( + "off".equals(crc) ? BT_CRC_MODE.OFF : "1".equals(crc) ? BT_CRC_MODE.ONE_BYTE_CRC : BT_CRC_MODE.TWO_BYTE_CRC); + } + Map args = new LinkedHashMap<>(); + System.getProperties().stringPropertyNames().stream().filter(k -> k.startsWith("hil.")).sorted() + .forEach(k -> args.put(k.substring(4), System.getProperty(k))); + String mac = arg("mac", ""); + String type = arg("type", "Shimmer3"); + report = new HilReport(scenario, arg("unit", port), port, mac, type, args); + session = new HilSession(port, mac, type); + device = session.device; + logStart = device.logSize(); + } + + @After + public void end() { + if (session == null) { + return; + } + putDriverCounts(); + session.close(); + report.write(new File(arg("outDir", "build/hil")), session.events(), device.log()); + } + + private void finish() { + if (session.hung != null) { + report.put("hungCall", session.hung); + report.check(false, session.hung + " never returned: the API call blocked (thread dump in the report's events)"); + } + if (!report.failures().isEmpty()) { + Assert.fail(String.join("; ", report.failures())); + } + } + + /** What the driver did in this session: replies by type, decodes, timer requests, connects. */ + private void putDriverCounts() { + report.put("connects", session.connects); + report.put("inStreamReplies", device.inStreamReplies()); + report.put("statusesDecoded", device.statusesDecoded()); + report.put("batteryReadings", device.batteryReadings()); + report.put("statusRequests", device.countSince(logStart, "GET_STATUS_COMMAND Instruction added")); + report.put("commandTimeouts", device.countSince(logStart, COMMAND_TIMEOUT)); + } + + // --- shared checks ------------------------------------------------------------------------- + + private boolean connectAndConfigure(double rateHz) throws Exception { + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + if (!connected) { + return false; + } + report.put("hardware", device.getHardwareVersionParsed()); + report.put("firmware", device.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(device.getCurrentBtCommsCrcMode())); + boolean configured = session.configureAccelOnly(rateHz, 20000); + report.put("rateOnDeviceHz", device.getSamplingRateShimmer()); + report.check(configured, "configuration did not complete within 20 s (state " + session.state() + ")"); + return configured; + } + + private void checkRate(double deviceRateHz, String what) { + report.check(!Double.isNaN(deviceRateHz) && deviceRateHz > 0, + what + ": the API has no sampling rate for the unit (" + deviceRateHz + " Hz)"); + } + + private void checkStream(StreamAnalysis a, List samples, double stoppedAtMs, + double maxLossPercent, double deviceRateHz, String what) { + report.check(a.received > 0, what + ": no packets received"); + report.check(a.lossPercent() <= maxLossPercent, String.format(Locale.ROOT, + "%s: %.3f%% packets lost (%d of %d), limit %.2f%%", what, a.lossPercent(), a.missing, + a.received + a.missing, maxLossPercent)); + report.check(a.nonIncreasing == 0, what + ": " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, what); + checkTimeline(a, samples, stoppedAtMs, deviceRateHz, what); + } + + /** A device timestamp that leapt ahead of the arrival time is wrong data (DEV-1023 class), never loss. */ + private void checkJumps(StreamAnalysis a, String what) { + for (StreamAnalysis.Gap j : a.timestampJumps) { + report.check(false, String.format(Locale.ROOT, + "%s: timestamp jumped %.1f s ahead at %.0f ms (device time %.0f ms) while packets kept arriving", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs, j.afterTimestampMs)); + } + } + + /** + * Loss is counted between packets received, so two things it cannot see are checked here: the + * timestamps must give a rate within 1 % of the configured one, and packets must have kept + * arriving until recording stopped. + */ + private void checkTimeline(StreamAnalysis a, List samples, double stoppedAtMs, + double deviceRateHz, String what) { + if (Double.isNaN(a.effectiveRateHz) || Double.isInfinite(a.effectiveRateHz)) { + report.check(false, what + ": no usable device timestamps to measure the rate from (" + a.received + " packets)"); + } else { + report.check(Math.abs(a.effectiveRateHz - deviceRateHz) <= 0.01 * deviceRateHz, String.format(Locale.ROOT, + "%s: effective rate %.2f Hz, device set to %.2f Hz", what, a.effectiveRateHz, deviceRateHz)); + } + if (!samples.isEmpty()) { + double silentMs = stoppedAtMs - samples.get(samples.size() - 1).receivedMs; + report.check(silentMs <= MAX_SILENCE_AT_END_MS, String.format(Locale.ROOT, + "%s: no packet in the last %.1f s before recording stopped (the stream went silent)", what, silentMs / 1000.0)); + } + } + + /** Parser errors and command timeouts the driver logged since index from. */ + private void checkDriverLog(int from, String what) { + int parserErrors = device.countSince(from, "Unknown parsing error", "Unknown packet error", "Throw Bytes"); + report.check(parserErrors == 0, what + ": " + parserErrors + " parser errors logged by the driver (see the .driver.log)"); + int timeouts = device.countSince(from, COMMAND_TIMEOUT); + report.check(timeouts == 0, what + ": " + timeouts + " command timeouts logged by the driver (see the .driver.log)"); + } + + private static double[] toArray(List values) { + double[] out = new double[values.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = values.get(i); + } + return out; + } + + // --- scenarios ----------------------------------------------------------------------------- + + /** Connect, configure, stream at rest, stop, disconnect. */ + @Test + public void streamRest() throws Exception { + begin("stream_rest"); + double rate = arg("rate", 256.0); + int seconds = arg("seconds", 30); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(seconds * 1000L); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); + if (!Double.isNaN(a.accelMagnitudeMean)) { + report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= 1.0, String.format(Locale.ROOT, + "accel at rest %.3f m/s^2, expected %.2f±1.00 (is the unit lying still?)", a.accelMagnitudeMean, GRAVITY)); + } + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** + * Commands while streaming: start and stop SD logging (SET_RWC and the logging commands, with + * their bare ACKs) and read the battery, each waited on. Needs an SD card in the unit. + */ + @Test + public void midStreamSet() throws Exception { + begin("mid_stream_set"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 3); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(3000); + List sdMoments = new ArrayList<>(); + List batteryMoments = new ArrayList<>(); + for (int i = 1; i <= cycles && session.hung == null; i++) { + session.event("cycle " + i); + sdMoments.add(session.nowMs()); + boolean started = session.call("startSDLogging", 10000, device::startSDLogging); + report.check(started, "cycle " + i + ": startSDLogging did not complete: it threw, hung or was skipped (see the events)"); + if (started) { + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING, 5000), + "cycle " + i + ": SD logging did not start while streaming (state " + session.state() + "; is there an SD card?)"); + } + Thread.sleep(2000); + sdMoments.add(session.nowMs()); + boolean stopped = session.call("stopSDLogging", 10000, device::stopSDLogging); + report.check(stopped, "cycle " + i + ": stopSDLogging did not complete: it threw, hung or was skipped (see the events)"); + if (stopped) { + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING, 5000), + "cycle " + i + ": SD logging did not stop (state " + session.state() + ")"); + } + Thread.sleep(1000); + batteryMoments.add(session.nowMs()); + int readingsBefore = device.batteryReadings(); + boolean asked = session.call("readBattery", 10000, device::readBattery); + report.check(asked, "cycle " + i + ": readBattery did not complete: it threw, hung or was skipped (see the events)"); + if (asked) { + report.check(session.waitFor(() -> device.batteryReadings() > readingsBefore, 5000), + "cycle " + i + ": no battery reply decoded within 5 s of readBattery"); + } + Thread.sleep(2000); + } + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + // Loss around SD logging start/stop is reported, not failed: the unit itself pauses while it + // opens or closes its file. Loss anywhere else is the driver's. + int[] attributed = a.attribute(new double[][] {toArray(sdMoments), toArray(batteryMoments)}, 1000); + report.put("missingNearSdLoggingStartStop", attributed[0]); + report.put("missingNearBatteryReads", attributed[1]); + if (attributed[0] > 0) { + session.event(attributed[0] + " packets not received within 1 s of SD logging start/stop (firmware side; not failed)"); + } + report.check(attributed[1] == 0, attributed[1] + " packets lost within 1 s of a battery read while streaming"); + int elsewhere = a.missing - attributed[0]; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from SD logging start/stop (%d), limit %.2f%%", elsewherePercent, elsewhere, maxLoss)); + report.check(a.received > 0, "stream: no packets received"); + report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, "stream"); + checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Connect, stream for 5 s and disconnect, repeatedly. Every connect starts the status timer. */ + @Test + public void reconnectLoop() throws Exception { + begin("reconnect_loop"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 5); + double maxLoss = arg("maxLossPercent", 1.0); + List connectTimes = new ArrayList<>(); + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "cycle " + i; + long start = System.currentTimeMillis(); + boolean ready = i == 1 ? connectAndConfigure(rate) : session.connect(30000); + connectTimes.add(System.currentTimeMillis() - start); + report.check(ready, what + ": did not connect within 30 s (state " + session.state() + ")"); + if (!ready) { + break; + } + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, what); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + checkDriverLog(logMark, what); + report.check(session.disconnect(10000), what + ": did not disconnect within 10 s"); + Thread.sleep(2000); + } + report.put("connectMs", connectTimes); + finish(); + } + + /** + * In-stream commands while streaming. First each reply on its own: status (0x8A 0x71), directory + * (0x8A 0x88) and battery (0x8A 0x94), each checked to be decoded. Then the overlap: SET_RWC + * followed at once by status and battery requests, so a reply arrives while another command + * waits for its ACK (DEV-1262). Every command must complete, with its own reply, and none may + * time out. Packets the unit drops right after an RTC write are reported; timestamp faults fail. + */ + @Test + public void inStream() throws Exception { + begin("in_stream"); + double rate = arg("rate", 256.0); + int rounds = arg("cycles", 3); + int overlaps = arg("overlaps", 5); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(2000); + for (int i = 1; i <= rounds && session.hung == null; i++) { + String what = "round " + i; + int statusBefore = device.statusesDecoded(); + expectReply(what, "status", "STATUS", device::readStatusLogAndStream, () -> device.statusesDecoded() > statusBefore); + expectReply(what, "directory", "DIR", device::readDirectoryName, + () -> device.getDirectoryName() != null && !device.getDirectoryName().isEmpty()); + int batteryBefore = device.batteryReadings(); + expectReply(what, "battery", "VBATT", device::readBattery, () -> device.batteryReadings() > batteryBefore); + Thread.sleep(500); + } + List rtcWrites = new ArrayList<>(); + for (int i = 1; i <= overlaps && session.hung == null; i++) { + String what = "overlap " + i; + rtcWrites.add(session.nowMs()); + int statusReplies = device.inStreamReplies("STATUS"); + int batteryReplies = device.inStreamReplies("VBATT"); + int statusBefore = device.statusesDecoded(); + int batteryBefore = device.batteryReadings(); + int timeoutsBefore = device.countSince(logMark, COMMAND_TIMEOUT); + boolean sent = session.call(what + ": SET_RWC, GET_STATUS and GET_VBATT back to back", 10000, () -> { + device.writeRealTimeClock(); + device.readStatusLogAndStream(); + device.readBattery(); + }); + report.check(sent, what + ": the three calls did not complete (see the events)"); + if (sent) { + boolean done = session.waitFor(() -> device.inStreamReplies("STATUS") > statusReplies + && device.inStreamReplies("VBATT") > batteryReplies + && session.queuedInstructions() == 0, 5000); + report.check(done, String.format(Locale.ROOT, + "%s: not every command completed within 5 s (status replies +%d, battery replies +%d, %d still queued)", + what, device.inStreamReplies("STATUS") - statusReplies, device.inStreamReplies("VBATT") - batteryReplies, + session.queuedInstructions())); + report.check(device.statusesDecoded() > statusBefore, what + ": the status reply was not decoded"); + report.check(device.batteryReadings() > batteryBefore, what + ": the battery reply was not decoded"); + int timeouts = device.countSince(logMark, COMMAND_TIMEOUT) - timeoutsBefore; + report.check(timeouts == 0, what + ": " + timeouts + " command timeouts (a reply matched to the wrong command?)"); + } + Thread.sleep(1000); + } + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + report.check(session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 3000), + "after streaming the unit is " + session.state() + ", not CONNECTED"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + // Loss right after an RTC write is reported, not failed: the unit itself drops packets when + // its clock is set while streaming (firmware side, as at SD logging start in midStreamSet). + // Timestamps that go backwards or leap ahead still fail: the API hands them to apps. + int nearRtc = a.attribute(new double[][] {toArray(rtcWrites)}, 1000)[0]; + report.put("missingNearRtcWrites", nearRtc); + if (nearRtc > 0) { + session.event(nearRtc + " packets not received within 1 s of an RTC write (firmware side; not failed)"); + } + int elsewhere = a.missing - nearRtc; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from the RTC writes (%d), limit %.2f%%", elsewherePercent, elsewhere, maxLoss)); + report.check(a.received > 0, "stream: no packets received"); + report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, "stream"); + checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Sends one in-stream request and checks its reply both arrived and was decoded within 3 s. */ + private void expectReply(String round, String what, String replyType, HilSession.Call request, + HilSession.Condition decoded) throws InterruptedException { + int before = device.inStreamReplies(replyType); + if (!session.call(round + ": request " + what, 10000, request)) { + report.check(false, round + ": the " + what + " request did not complete (see the events)"); + return; + } + boolean arrived = session.waitFor(() -> device.inStreamReplies(replyType) > before, 3000); + report.check(arrived, round + ": no in-stream " + what + " reply within 3 s"); + if (arrived) { + report.check(session.waitFor(decoded, 1000), round + ": the " + what + " reply arrived but was not decoded"); + } + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java new file mode 100644 index 000000000..85154a618 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java @@ -0,0 +1,196 @@ +package com.shimmerresearch.hil; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** + * Judges a recorded stream from the device timestamps alone. The checks mirror the firmware HIL + * harness in log-and-stream-common Test/hil (DEV-1165): packet loss and non-increasing timestamps + * from the driver's already-unwrapped timestamps, the effective sample rate, and accel at rest. + * Pure Java, so it is unit-tested on the JVM (StreamAnalysisTest, in the normal test task). + *

+ * The same class as the Android HIL's (ShimmerAndroidAPI, shimmerHilTest), so both judge a stream + * by the same rules; a change here belongs there too. + */ +public final class StreamAnalysis { + + /** One received packet. Times in ms; accel in m/s^2, NaN when the channel is absent. */ + public static final class Sample { + public final double receivedMs; + public final double timestampMs; + public final double accelX, accelY, accelZ; + + public Sample(double receivedMs, double timestampMs, double accelX, double accelY, double accelZ) { + this.receivedMs = receivedMs; + this.timestampMs = timestampMs; + this.accelX = accelX; + this.accelY = accelY; + this.accelZ = accelZ; + } + + double accelMagnitude() { + return Math.sqrt(accelX * accelX + accelY * accelY + accelZ * accelZ); + } + } + + /** A run of packets missing between two received ones. */ + public static final class Gap { + public final double afterTimestampMs; + public final double afterReceivedMs; + public final int missing; + + Gap(double afterTimestampMs, double afterReceivedMs, int missing) { + this.afterTimestampMs = afterTimestampMs; + this.afterReceivedMs = afterReceivedMs; + this.missing = missing; + } + } + + public final int received; + public final int missing; + public final int nonIncreasing; + /** + * Steps where the device timestamp leapt far ahead of the arrival time: not lost packets but a + * bad timestamp, such as a 24-bit wrap unwrapped wrongly (512 s at a 32768 Hz clock, DEV-1023). + */ + public final List timestampJumps; + public final double expectedPeriodMs; + public final double effectiveRateHz; + public final double accelMagnitudeMean; + public final double accelMagnitudeStd; + public final List gaps; + + private StreamAnalysis(int received, int missing, int nonIncreasing, List timestampJumps, double expectedPeriodMs, + double effectiveRateHz, double accelMagnitudeMean, double accelMagnitudeStd, List gaps) { + this.received = received; + this.missing = missing; + this.nonIncreasing = nonIncreasing; + this.timestampJumps = timestampJumps; + this.expectedPeriodMs = expectedPeriodMs; + this.effectiveRateHz = effectiveRateHz; + this.accelMagnitudeMean = accelMagnitudeMean; + this.accelMagnitudeStd = accelMagnitudeStd; + this.gaps = gaps; + } + + /** + * @param samples in arrival order + * @param rateHz the sampling rate the device was configured with + */ + public static StreamAnalysis analyse(List samples, double rateHz) { + double period = 1000.0 / rateHz; + int missing = 0; + int nonIncreasing = 0; + // The effective rate counts sample periods over device time, from ordinary steps only: a + // jump or a step backwards says nothing about the rate. + double periods = 0, spanMs = 0; + List gaps = new ArrayList<>(); + List jumps = new ArrayList<>(); + for (int i = 1; i < samples.size(); i++) { + double step = samples.get(i).timestampMs - samples.get(i - 1).timestampMs; + double arrivalStep = samples.get(i).receivedMs - samples.get(i - 1).receivedMs; + if (step <= 0) { + nonIncreasing++; + } else if (step > 1.5 * period) { + int lost = (int) Math.round(step / period) - 1; + Gap g = new Gap(samples.get(i - 1).timestampMs, samples.get(i - 1).receivedMs, lost); + // Packets really lost leave a pause in arrival about as long as the gap. A timestamp that + // leaps a second or more beyond the arrival time is a bad timestamp, not lost data. + if (step - arrivalStep > JUMP_MS) { + jumps.add(g); + } else { + missing += lost; + gaps.add(g); + periods += lost + 1; + spanMs += step; + } + } else { + periods += 1; + spanMs += step; + } + } + double effectiveRate = spanMs > 0 ? periods * 1000.0 / spanMs : Double.NaN; + double sum = 0, sumSq = 0; + int n = 0; + for (Sample s : samples) { + double m = s.accelMagnitude(); + if (!Double.isNaN(m)) { + sum += m; + sumSq += m * m; + n++; + } + } + double mean = n > 0 ? sum / n : Double.NaN; + double std = n > 1 ? Math.sqrt(Math.max(0, (sumSq - n * mean * mean) / (n - 1))) : Double.NaN; + return new StreamAnalysis(samples.size(), missing, nonIncreasing, jumps, period, effectiveRate, mean, std, gaps); + } + + /** How far ahead of the arrival time a timestamp must leap to count as a jump, not loss. */ + static final double JUMP_MS = 1000; + + /** + * Charges each gap to the kind of command nearest to it (by arrival time), if one is within + * windowMs. Commands that follow each other closely therefore never share a loss. + * + * @param momentsByKind moments (ms) of each kind of command + * @return packets lost per kind, in the same order + */ + public int[] attribute(double[][] momentsByKind, double windowMs) { + int[] lost = new int[momentsByKind.length]; + for (Gap g : gaps) { + int nearestKind = -1; + double nearest = Double.MAX_VALUE; + for (int k = 0; k < momentsByKind.length; k++) { + for (double t : momentsByKind[k]) { + double d = Math.abs(g.afterReceivedMs - t); + if (d < nearest) { + nearest = d; + nearestKind = k; + } + } + } + if (nearestKind >= 0 && nearest <= windowMs) { + lost[nearestKind] += g.missing; + } + } + return lost; + } + + /** Packets lost within windowMs (by arrival time) of any of the given moments. */ + public int missingNear(double[] momentsMs, double windowMs) { + int total = 0; + for (Gap g : gaps) { + for (double t : momentsMs) { + if (Math.abs(g.afterReceivedMs - t) <= windowMs) { + total += g.missing; + break; + } + } + } + return total; + } + + public double lossPercent() { + int expected = received + missing; + return expected == 0 ? 0 : 100.0 * missing / expected; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, + "received=%d missing=%d (%.3f%%) nonIncreasing=%d timestampJumps=%d effectiveRate=%.2fHz accel=%.3f±%.3f m/s^2 gaps=%s", + received, missing, lossPercent(), nonIncreasing, timestampJumps.size(), effectiveRateHz, + accelMagnitudeMean, accelMagnitudeStd, gapsSummary()); + } + + private String gapsSummary() { + int shown = Math.min(gaps.size(), 5); + String[] parts = new String[shown]; + for (int i = 0; i < shown; i++) { + parts[i] = String.format(Locale.ROOT, "%d@%.0fms", gaps.get(i).missing, gaps.get(i).afterReceivedMs); + } + return Arrays.toString(parts) + (gaps.size() > shown ? "+" + (gaps.size() - shown) : ""); + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java new file mode 100644 index 000000000..472366319 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java @@ -0,0 +1,102 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +/** The HIL verdicts are only as good as this arithmetic, so it is checked without a phone. */ +public class StreamAnalysisTest { + + private static List stream(double rateHz, int count, int... droppedIndices) { + List samples = new ArrayList<>(); + double period = 1000.0 / rateHz; + outer: + for (int i = 0; i < count; i++) { + for (int d : droppedIndices) { + if (d == i) { + continue outer; + } + } + samples.add(new StreamAnalysis.Sample(i * period, 1000 + i * period, 0, 0, 9.81)); + } + return samples; + } + + @Test + public void cleanStreamHasNoLoss() { + StreamAnalysis a = StreamAnalysis.analyse(stream(256, 2560), 256); + assertEquals(2560, a.received); + assertEquals(0, a.missing); + assertEquals(0, a.nonIncreasing); + assertEquals(256, a.effectiveRateHz, 0.01); + assertEquals(9.81, a.accelMagnitudeMean, 1e-9); + } + + @Test + public void countsEachDroppedPacket() { + StreamAnalysis a = StreamAnalysis.analyse(stream(256, 1000, 100, 500, 501, 502), 256); + assertEquals(996, a.received); + assertEquals(4, a.missing); + assertEquals(2, a.gaps.size()); + assertEquals(256, a.effectiveRateHz, 0.01); + } + + @Test + public void attributesLossToTheMomentItHappened() { + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + // Packet 300 is missing; packet 299 arrived at 2990 ms. + assertEquals(1, a.missingNear(new double[] {3000}, 500)); + assertEquals(0, a.missingNear(new double[] {9000}, 500)); + } + + @Test + public void chargesEachGapToTheNearestCommandOnly() { + // 100 Hz; packet 300 (arrives ~3000 ms) is missing. An SD stop at 2900 ms and a battery read + // at 3920 ms are both within 1 s of it: the SD stop is nearer, so it alone is charged. + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + int[] lost = a.attribute(new double[][] {{2900}, {3920}}, 1000); + assertEquals(1, lost[0]); + assertEquals(0, lost[1]); + } + + @Test + public void chargesNothingOutsideTheWindow() { + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + int[] lost = a.attribute(new double[][] {{5000}, {8000}}, 1000); + assertEquals(0, lost[0] + lost[1]); + } + + @Test + public void tellsABadTimestampFromLostPackets() { + // What a Shimmer3R did over BLE at an SD logging start: one packet stamped a 24-bit wrap + // (512 s) ahead while packets kept arriving on time, then the next one back on track. + List s = stream(256, 1000); + StreamAnalysis.Sample bad = s.get(500); + s.set(500, new StreamAnalysis.Sample(bad.receivedMs, bad.timestampMs + 512000, 0, 0, 9.81)); + StreamAnalysis a = StreamAnalysis.analyse(s, 256); + assertEquals(1, a.timestampJumps.size()); + assertEquals(0, a.missing); + assertEquals(1, a.nonIncreasing); + assertEquals(256, a.effectiveRateHz, 0.5); + } + + @Test + public void flagsTimestampsThatGoBackwards() { + List s = stream(100, 10); + s.add(5, new StreamAnalysis.Sample(45, 1000, 0, 0, 9.81)); + StreamAnalysis a = StreamAnalysis.analyse(s, 100); + assertEquals(1, a.nonIncreasing); + } + + @Test + public void toleratesJitterBelowHalfAPeriod() { + List s = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + s.add(new StreamAnalysis.Sample(i * 10, 1000 + i * 10 + (i % 2 == 0 ? 0 : 4), 0, 0, 9.81)); + } + assertEquals(0, StreamAnalysis.analyse(s, 100).missing); + } +} diff --git a/tools/hil/README.md b/tools/hil/README.md new file mode 100644 index 000000000..d8af1ae6e --- /dev/null +++ b/tools/hil/README.md @@ -0,0 +1,121 @@ +# Java HIL: the Java driver on real Shimmers + +Hardware-in-the-loop tests of the Java driver (`ShimmerDriver` with `ShimmerDriverPC`). A PC connects +to real Shimmer3 and Shimmer3R units over their paired Bluetooth Classic serial ports, through the +same API Consensys and MATLAB use (`ShimmerPC`), **with the driver's timers on** as an app runs it. +One command builds the tests, runs every scenario on every unit, and prints pass or fail. + +```powershell +.\tools\hil\run-java-hil.ps1 +``` + +It exits 0 when everything passed, 1 when anything failed, errored, timed out or was skipped, and 2 +when the bench isn't set up. + +The CI tests run against byte simulators, and since DEV-1198 (#325, #326) with the status timer off. +This is where the driver meets real firmware, a real radio and its own timers. + +## Before every Java API release (the gate) + +**Release Java API** won't publish a commit that hasn't passed this on hardware. Each round: + +1. On the bench, on `master` with a clean working tree and every unit in the rig, run: + ```powershell + git checkout master; git pull + .\tools\hil\run-java-hil.ps1 -Report + ``` + With `-Report`, the result is posted to GitHub as the commit status **`java-hil`** on that commit: + passed or failed, with each unit's firmware. It refuses to report anything but the full gate run: + - every scenario, on at least one Shimmer3 and one Shimmer3R; + - the default rate, duration, link CRC and loss limit, and a fresh build (no `-SkipBuild`); + - a clean working tree, pushed. +2. Run **Release Java API**. It looks up that status for the commit it's releasing and stops before + building anything unless it passed. The release notes say what was tested. + +If anything merges after step 1, the commit changes, so test again. In an emergency, put a Jira key in +**hil_waiver** to release without a pass; the release notes record the waiver. Dry runs only warn. + +## What it tests + +| Scenario | What it does | Fails when | +|---|---|---| +| `streamRest` | Connects, writes an accel-only config at `-Rate` (default 256 Hz) the way Consensys's Bluetooth manager does, streams for `-Seconds` (default 30) with the unit lying still, then stops and disconnects. | It can't connect or configure; more than `-MaxLossPercent` (1 %) of packets are lost; a timestamp doesn't increase, or leaps more than a second ahead of when it arrived (a bad unwrap, not loss); the timestamps give no rate, or one more than 1 % off the configured rate; the stream goes silent for over a second before the test stops it; accel is more than 1 m/s² from 9.81; or the driver logs a parser error or a command timeout. | +| `midStreamSet` | While streaming, starts and stops SD logging and reads the battery, three times. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop; a packet is lost around a battery read, or more than `-MaxLossPercent` away from the SD transitions; the timestamps fail the checks above; or the driver logs a parser error or a command timeout. Packets missing around SD logging start/stop are reported, not failed: the unit itself pauses while it opens its file. Needs an SD card in the unit. | +| `reconnectLoop` | Connects, streams for 5 s and disconnects, five times on the same `ShimmerPC`. Every connect starts the driver's status timer, which is when the DEV-1266 lock inversion is most likely. | Any cycle fails to connect within 30 s, loses packets, fails the timestamp checks, or the driver logs a parser error or command timeout. | +| `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps: `SET_RWC`, a status request and a battery request back to back, so replies arrive while another command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, the queue empty); the driver logs a command timeout; or the stream fails the checks above. Packets missing within 1 s of an RTC write are reported, not failed (the unit itself drops them; see DEV-1245), but timestamp faults fail. | + +Every API call runs under a watchdog. If one doesn't return within 10 s, the scenario stops, and its +events hold every thread's stack with the locks each holds and waits on, plus any deadlock the JVM +detects (DEV-1266). If the whole run still doesn't finish, the runner stops Gradle after +`-ScenarioTimeoutSeconds` (900) and records TIMEOUT. + +Each report counts what the session covered: connects, in-stream replies by type, statuses and +battery readings decoded, the status timer's requests, and command timeouts. Its events also say +which driver timers were running after connect, after configure and while streaming. + +The stream checks are the Android HIL's (`StreamAnalysis`, the same class in ShimmerAndroidAPI's +`shimmerHilTest`), so the two judge a stream by the same rules. `StreamAnalysisTest` runs in CI. + +### What it does not test + +- BLE: on the PC it goes through the gRPC BLE server (`ShimmerGRPC`); not in v1. +- Status the firmware pushes on its own (dock, undock, button press), which needs someone at the + bench; the managed USB hub in `bench-tools/hub` could do that later. +- The SD and file parsers, and the Android API's own Classic parser (the Android HIL covers that). + +## Choosing what to test + +| | Default | Option | +|---|---|---| +| Units | every unit in the rig file | `-Unit s3r-86f8` | +| Scenarios | all four | `-Scenario inStream` | +| Sampling rate, stream duration | 256 Hz, 30 s | `-Rate 512`, `-Seconds 60` | +| Link CRC | the driver's default (1 byte) | `-Crc off`, `-Crc 2` | +| Pause between runs | 10 s | `-SettleSeconds 30` | +| Record the result for the release gate | no | `-Report` (full run on a clean, pushed commit only) | + +## One-time setup + +1. **Pair each unit with the PC** in Windows Settings → Bluetooth (PIN `1234`). Windows creates an + outgoing serial port for it; the script finds that port from the unit's MAC. +2. **Prepare each unit:** SD sync and Bluetooth disable off (either keeps Bluetooth off, DEV-1228), an + SD card in for `midStreamSet`, lying still during `streamRest`. +3. **List the units:** copy `rig-java.example.json` to `rig-java.json` and fill in each name, MAC + and `type` (`Shimmer3` or `Shimmer3R`). Add `comPort` only to override the port the script finds. + That file is gitignored. +4. **The checkout's folder must be named `Shimmer-Java-Android-API`**, because + `ShimmerDriverPC/settings.gradle` finds its sibling projects by that name. In a worktree, make a + junction with that name and run the script through it: + ```powershell + New-Item -ItemType Junction -Path C:\wt\Shimmer-Java-Android-API -Target C:\wt\ + C:\wt\Shimmer-Java-Android-API\tools\hil\run-java-hil.ps1 + ``` +5. **JDK 11 or newer.** The script uses `JAVA_HOME` if it qualifies, otherwise the first one it finds + under Amazon Corretto, Eclipse Adoptium or Java in Program Files. + +A Shimmer holds one Bluetooth link at a time. Don't run this while Consensys, the Android HIL's phone +or anything else is connected to the same unit. + +## Output + +The run's folder, `tools/hil/out//`, holds: +- `summary.json` and the table the script prints; +- `reports/`, per scenario and unit: + - `_.json`: versions, the CRC mode in use, loss, rate, accel, the counts above, a + timeline of events and the failures; + - `.csv`: every sample; + - `.driver.log`: every line the driver logged, verbose or not; +- `*.console.txt` and `*.junit.xml`: Gradle's output and the JUnit result for each run; +- `build.log`. + +## Running one scenario by hand + +```powershell +cd C:\wt\Shimmer-Java-Android-API\ShimmerDriverPC +.\gradlew :hilTest --no-watch-fs --tests com.shimmerresearch.hil.JavaHilScenariosTest.inStream "-Phil.port=COM36" "-Phil.type=Shimmer3R" "-Phil.unit=s3r-86f8" +``` + +Every `-Phil.` reaches the tests as `hil.`; the names are listed in +`JavaHilScenariosTest`. Without `hil.port` every scenario is skipped, and the normal `test` task +leaves them out. `--no-watch-fs` matters when you edit through a worktree's real path but build +through the junction: Gradle's file watcher misses those edits and would run stale classes. diff --git a/tools/hil/rig-java.example.json b/tools/hil/rig-java.example.json new file mode 100644 index 000000000..8d62516d9 --- /dev/null +++ b/tools/hil/rig-java.example.json @@ -0,0 +1,6 @@ +{ + "units": [ + { "name": "s3-xxxx", "mac": "00:06:66:XX:XX:XX", "type": "Shimmer3" }, + { "name": "s3r-xxxx", "mac": "E8:EB:1B:XX:XX:XX", "type": "Shimmer3R" } + ] +} diff --git a/tools/hil/run-java-hil.ps1 b/tools/hil/run-java-hil.ps1 new file mode 100644 index 000000000..1e0278b37 --- /dev/null +++ b/tools/hil/run-java-hil.ps1 @@ -0,0 +1,234 @@ +<# +.SYNOPSIS +Runs the Java driver's hardware-in-the-loop (HIL) scenarios on every Shimmer in a rig file, over +the PC's paired Bluetooth serial ports, and prints a pass/fail table. Exits 1 if anything did not pass. + +.DESCRIPTION +One-time setup is in tools/hil/README.md: pair each unit with this PC, list it in +tools/hil/rig-java.json (copy rig-java.example.json), and check out the repository in a folder named +Shimmer-Java-Android-API (ShimmerDriverPC's settings.gradle finds its siblings by that name). + +.EXAMPLE +.\tools\hil\run-java-hil.ps1 +Every scenario on every unit. + +.EXAMPLE +.\tools\hil\run-java-hil.ps1 -Unit s3r-86f8 -Scenario inStream + +.EXAMPLE +.\tools\hil\run-java-hil.ps1 -Report +Before "Release Java API": the full run on this commit, recorded on GitHub as the commit status +"java-hil". The release refuses to publish a commit without a passing one. +#> +param( + [string]$Rig = (Join-Path $PSScriptRoot 'rig-java.json'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream'), + [string[]]$Unit, + [double]$Rate = 256, + [int]$Seconds = 30, + # Link CRC to ask for; by default the driver's own default, as an app gets it. + [ValidateSet('off', '1', '2')] + [string]$Crc, + [double]$MaxLossPercent = 1.0, + [switch]$SkipBuild, + [switch]$Report, + [int]$ScenarioTimeoutSeconds = 900, + # Pause between runs, so the unit has dropped the last link before the next connect. + [int]$SettleSeconds = 10 +) + +$ErrorActionPreference = 'Continue' +$repo = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path +$gradleRoot = Join-Path $repo 'ShimmerDriverPC' +$testClass = 'com.shimmerresearch.hil.JavaHilScenariosTest' + +function Fail([string]$message) { + Write-Host "ERROR: $message" -ForegroundColor Red + exit 2 +} + +# --- Rig ----------------------------------------------------------------------------------------- +if (-not (Test-Path $Rig)) { Fail "No rig file at $Rig. Copy rig-java.example.json to rig-java.json and list your units (README)." } +$rigData = Get-Content $Rig -Raw | ConvertFrom-Json +$units = @($rigData.units) +if ($Unit) { $units = @($units | Where-Object { $Unit -contains $_.name }) } +if ($units.Count -eq 0) { Fail "No units to test (rig: $Rig, -Unit: $Unit)." } + +# Each unit's outgoing Bluetooth serial port, from its MAC, unless the rig names one. +$ports = @{} +Get-CimInstance Win32_PnPEntity -Filter "PNPClass='Ports'" -ErrorAction SilentlyContinue | + Where-Object { $_.DeviceID -match 'BTHENUM' } | ForEach-Object { + $m = [regex]::Match($_.DeviceID, '&([0-9A-F]{12})_') + $p = [regex]::Match($_.Name, 'COM\d+') + if ($m.Success -and $p.Success -and $m.Groups[1].Value -ne '000000000000') { $ports[$m.Groups[1].Value] = $p.Value } + } +foreach ($u in $units) { + if (-not $u.type) { Fail "Unit $($u.name): give its type (Shimmer3 or Shimmer3R)." } + if (-not $u.comPort) { + $key = ($u.mac -replace '[:-]', '').ToUpper() + if (-not $ports.ContainsKey($key)) { Fail "Unit $($u.name) ($($u.mac)) is not paired with this PC: no Bluetooth serial port has its MAC. Pair it (README) or give comPort." } + $u | Add-Member -NotePropertyName comPort -NotePropertyValue $ports[$key] -Force + } +} + +# The release gate trusts this status, so only report a run that tested exactly this commit, with +# the gate's own settings: every scenario, both device types, defaults, a fresh build, clean tree. +$commit = $null +$githubRepo = $null +if ($Report) { + if ($PSBoundParameters.ContainsKey('Scenario') -or $Unit) { Fail '-Report needs every scenario on every unit in the rig; drop -Scenario and -Unit.' } + $shaping = @('Rate', 'Seconds', 'Crc', 'MaxLossPercent', 'SkipBuild') | Where-Object { $PSBoundParameters.ContainsKey($_) } + if ($shaping) { Fail "-Report runs with the default settings and a fresh build; drop $((@($shaping) | ForEach-Object { "-$_" }) -join ', ')." } + $types = @($units | ForEach-Object { $_.type }) + if (-not ($types -contains 'Shimmer3') -or -not ($types -contains 'Shimmer3R')) { + Fail "-Report needs at least one Shimmer3 and one Shimmer3R in the rig (types found: $($types -join ', '))." + } + $dirty = (git -C $repo status --porcelain) -join "`n" + if ($dirty) { Fail "-Report needs a clean working tree, so the tested code is exactly the commit:`n$dirty" } + $commit = ((git -C $repo rev-parse HEAD) -join '').Trim() + $remote = ((git -C $repo remote get-url origin) -join '').Trim() + $githubRepo = ([regex]::Match($remote, 'github\.com[:/](.+?)(\.git)?$')).Groups[1].Value + if (-not $githubRepo) { Fail "Cannot tell the GitHub repository from origin ($remote)." } + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { Fail '-Report posts the result with the GitHub CLI (gh); install it and log in.' } + gh api "repos/$githubRepo/commits/$commit" --silent 2>$null + if ($LASTEXITCODE -ne 0) { Fail "Commit $commit is not on GitHub ($githubRepo); push it first." } +} + +# --- Build --------------------------------------------------------------------------------------- +# ShimmerDriverPC's settings.gradle finds ShimmerDriver at ../../Shimmer-Java-Android-API/ShimmerDriver. +if ((Split-Path $repo -Leaf) -ne 'Shimmer-Java-Android-API') { + Fail ("ShimmerDriverPC only builds from a folder named Shimmer-Java-Android-API (its settings.gradle finds its siblings that way), " + + "and this checkout is $repo. Make a junction and run from it:`n" + + " New-Item -ItemType Junction -Path '$(Join-Path (Split-Path $repo -Parent) 'Shimmer-Java-Android-API')' -Target '$repo'") +} +function JavaMajor([string]$javaHome) { + $release = Join-Path $javaHome 'release' + if (-not (Test-Path $release)) { return 0 } + $m = [regex]::Match((Get-Content $release -Raw), 'JAVA_VERSION="(?:1\.)?(\d+)') + if ($m.Success) { return [int]$m.Groups[1].Value } else { return 0 } +} +if (-not $env:JAVA_HOME -or (JavaMajor $env:JAVA_HOME) -lt 11) { + $jdk = @(Get-ChildItem 'C:\Program Files\Amazon Corretto', 'C:\Program Files\Eclipse Adoptium', 'C:\Program Files\Java' -Directory -ErrorAction SilentlyContinue | + Where-Object { (JavaMajor $_.FullName) -ge 11 } | Sort-Object { JavaMajor $_.FullName }) | Select-Object -First 1 + if (-not $jdk) { Fail 'No JDK 11 or newer found; set JAVA_HOME.' } + $env:JAVA_HOME = $jdk.FullName +} + +$outDir = Join-Path $PSScriptRoot ('out\' + (Get-Date -Format 'yyyyMMdd-HHmmss')) +$reportDir = Join-Path $outDir 'reports' +New-Item -ItemType Directory -Force $reportDir | Out-Null +$sha = ((git -C $repo rev-parse --short HEAD) -join '').Trim() +Write-Host "Java HIL: driver source $sha, JDK $(JavaMajor $env:JAVA_HOME) ($env:JAVA_HOME)" +Write-Host "Results: $outDir" + +# --no-watch-fs: Gradle's file watcher can miss edits made through the real path of a junction, +# and would then run stale test classes. +$gradle = Join-Path $gradleRoot 'gradlew.bat' +if (-not $SkipBuild) { + Write-Host 'Building...' + Push-Location $gradleRoot + & $gradle ':testClasses' --no-watch-fs --console=plain *> (Join-Path $outDir 'build.log') + $buildExit = $LASTEXITCODE + Pop-Location + if ($buildExit -ne 0) { Fail "Build failed; see $(Join-Path $outDir 'build.log')" } +} + +# --- Run ----------------------------------------------------------------------------------------- +function Invoke-Scenario([string[]]$gradleArgs, [int]$timeoutSeconds, [string]$logFile) { + $p = Start-Process -FilePath $gradle -ArgumentList $gradleArgs -WorkingDirectory $gradleRoot -NoNewWindow -PassThru ` + -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err" + if (-not $p.WaitForExit($timeoutSeconds * 1000)) { + # A test JVM stuck in a deadlock outlives the Gradle client; stopping the daemons ends it. + & $gradle --stop *> $null + $p.Kill() + return $true + } + return $false +} + +$results = @() +$runs = 0 +foreach ($u in $units) { + foreach ($s in $Scenario) { + if ($runs++ -gt 0) { Start-Sleep -Seconds $SettleSeconds } + $label = "$($u.name) $s" + Write-Host "Running $label on $($u.comPort)..." -NoNewline + $gradleArgs = @(':hilTest', '--no-watch-fs', '--console=plain', '--tests', "$testClass.$s", + "-Phil.port=$($u.comPort)", "-Phil.mac=$($u.mac)", "-Phil.unit=$($u.name)", "-Phil.type=$($u.type)", + "-Phil.rate=$Rate", "-Phil.seconds=$Seconds", "-Phil.maxLossPercent=$MaxLossPercent", "-Phil.outDir=$reportDir") + if ($Crc) { $gradleArgs += "-Phil.crc=$Crc" } + $safe = "$($u.name)_$s" -replace '[^\w.-]', '_' + $log = Join-Path $outDir "$safe.console.txt" + $timedOut = Invoke-Scenario $gradleArgs $ScenarioTimeoutSeconds $log + + $verdict = 'ERROR' + $detail = '' + $xml = Join-Path $gradleRoot "build\test-results\hilTest\TEST-$testClass.xml" + if ($timedOut) { + $verdict = 'TIMEOUT' + $detail = "did not finish within $ScenarioTimeoutSeconds s; stopped (see $safe.console.txt)" + } + elseif (Test-Path $xml) { + Copy-Item $xml (Join-Path $outDir "$safe.junit.xml") + $case = ([xml](Get-Content $xml -Raw -Encoding UTF8)).testsuite.testcase | Where-Object { $_.name -eq $s } | Select-Object -First 1 + if (-not $case) { $detail = 'the scenario did not run (see the console)' } + elseif ($case.failure) { $verdict = 'FAIL'; $detail = (($case.failure.message -split "`n")[0]) -replace '^java\.lang\.AssertionError: ', '' } + elseif ($case.skipped) { $verdict = 'SKIP'; $detail = 'skipped' } + else { $verdict = 'PASS' } + Remove-Item $xml + } + else { + $detail = ((Get-Content $log, "$log.err" -ErrorAction SilentlyContinue | Select-String 'error|FAILURE|wrong' | Select-Object -First 2) -join ' ').Trim() + } + Write-Host " $verdict" -ForegroundColor $(if ($verdict -eq 'PASS') { 'Green' } elseif ($verdict -eq 'SKIP') { 'Yellow' } else { 'Red' }) + $results += [pscustomobject]@{ Unit = $u.name; Port = $u.comPort; Scenario = $s; Result = $verdict; Detail = $detail } + } +} + +# --- Collect ------------------------------------------------------------------------------------- +# Every run that reached a verdict wrote a report; the gate's status rests on them. +foreach ($r in $results) { + if ($r.Result -ne 'PASS' -and $r.Result -ne 'FAIL') { continue } + $name = ($r.Scenario -creplace '([A-Z])', '_$1').ToLower() + "_$($r.Unit -replace '[^\w.-]', '_').json" + if (-not (Test-Path (Join-Path $reportDir $name))) { + $r.Result = 'ERROR' + $r.Detail = ("no report written ($name). " + $r.Detail).Trim() + } +} +$summary = [pscustomobject]@{ + driver = $sha; rate = $Rate; seconds = $Seconds; crc = $(if ($Crc) { $Crc } else { 'driver default' }) + settleSeconds = $SettleSeconds; results = $results +} +$summary | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $outDir 'summary.json') + +Write-Host '' +$results | Format-Table -AutoSize -Wrap | Out-String -Width 200 | Write-Host +# A skipped scenario did not run, so it is not a pass: the exit code and the gate's status agree. +$failed = @($results | Where-Object { $_.Result -ne 'PASS' }) +$passed = @($results | Where-Object { $_.Result -eq 'PASS' }) + +if ($Report) { + $firmware = @{} + Get-ChildItem $reportDir -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { + $r = Get-Content $_.FullName -Raw | ConvertFrom-Json + if ($r.firmware -and $r.unit) { $firmware[$r.unit] = "$($r.hardware) $($r.firmware)" } + } + $tested = @($units | ForEach-Object { if ($firmware[$_.name]) { $firmware[$_.name] } else { $_.type } }) | Sort-Object -Unique + $state = if ($passed.Count -eq $results.Count) { 'success' } else { 'failure' } + $description = "$($passed.Count)/$($results.Count) passed over Classic: $($tested -join ', ')" + if ($description.Length -gt 140) { $description = $description.Substring(0, 137) + '...' } + gh api -X POST "repos/$githubRepo/statuses/$commit" -f state=$state -f context=java-hil -f description=$description --silent + if ($LASTEXITCODE -ne 0) { + Write-Host 'Could not post the commit status; the release will not see this run.' -ForegroundColor Red + exit 2 + } + Write-Host "Posted 'java-hil' = $state on $githubRepo@$($commit.Substring(0, 8)): $description" +} + +if ($failed.Count -gt 0) { + Write-Host "$($failed.Count) of $($results.Count) did not pass. Reports and console output: $outDir" -ForegroundColor Red + exit 1 +} +Write-Host "All $($results.Count) passed. Reports: $outDir" -ForegroundColor Green +exit 0 From 226d53f66890ffc3445366bc2c7dd3548dfe69ab Mon Sep 17 00:00:00 2001 From: JongChern Date: Wed, 21 Oct 2026 16:53:23 +0800 Subject: [PATCH 27/41] DEV-1269 Java HIL: start SD logging the way Consensys does, retry a link Windows can't open midStreamSet and inStream now start SD logging as Consensys does (DEV-436): stop streaming, start logging on the quiet link, then stream again. SD logging is still stopped mid-stream. Consensys never writes the RTC mid-stream, so the scenarios no longer do either; that is what caused DEV-1245's out-of-order timestamps on the Shimmer3R. The stream is judged per stretch between the deliberate restarts. Loss within 1 s of an SD logging stop is reported but does not fail, because the unit pauses while it closes its file. Loss near a battery read fails, and loss anywhere else is held to the limit. A connect whose Bluetooth link can't be opened (CONNECTION_FAILED; jssc reports "Port busy") is tried once more, and connectRetries in the report counts those retries. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hil/HilSession.java | 65 ++++++- .../hil/JavaHilScenariosTest.java | 162 ++++++++++-------- tools/hil/README.md | 10 +- 3 files changed, 154 insertions(+), 83 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java index 453d829d1..c181419f6 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java @@ -56,10 +56,14 @@ interface Condition { }); private volatile boolean recording; private volatile BT_STATE lastState; + /** State changes reported so far, so a connect can tell its own CONNECTION_FAILED from the last one's. */ + private volatile int stateChanges; /** The call that never returned, if one did not. */ volatile String hung; /** Connects made in this session, for the report: each is a chance for the connect-time race. */ int connects; + /** Connects that failed to open the Bluetooth link and were tried again. */ + int connectRetries; HilSession(String port, String mac, String type) { this.port = port; @@ -93,6 +97,7 @@ protected void processMsgFromCallback(ShimmerMsg msg) { BT_STATE state = ((CallbackObject) msg.mB).mState; if (state != null) { lastState = state; + stateChanges++; event("state " + state); } } else if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_DATA_PACKET && msg.mB instanceof ObjectCluster) { @@ -168,15 +173,29 @@ boolean call(String what, long timeoutMs, Call call) { } } - /** Connects and waits until the driver has read the unit's configuration. */ + /** + * Connects and waits until the driver has read the unit's configuration. If Windows cannot open + * the Bluetooth link at all (CONNECTION_FAILED, which jssc reports as "Port busy" after paging the + * unit for about 5 s), it tries once more, as a user would; connectRetries counts those. + */ boolean connect(long timeoutMs) throws InterruptedException { - connects++; - if (!call("connect on " + port, 10000, () -> device.connect(port, ""))) { - return false; + for (int attempt = 1; ; attempt++) { + connects++; + int mark = stateChanges; + Condition failed = () -> stateChanges > mark && device.getBluetoothRadioState() == BT_STATE.CONNECTION_FAILED; + if (!call("connect on " + port, 10000, () -> device.connect(port, ""))) { + return false; + } + boolean ready = waitFor(() -> (device.isInitialised() && device.getBluetoothRadioState() == BT_STATE.CONNECTED) + || failed.holds(), timeoutMs) && !failed.holds(); + if (ready || attempt > 1 || !failed.holds()) { + event("connected: timers " + device.timers()); + return ready; + } + connectRetries++; + event("the Bluetooth link could not be opened (CONNECTION_FAILED); trying once more in 5 s"); + Thread.sleep(5000); } - boolean ready = waitFor(() -> device.isInitialised() && device.getBluetoothRadioState() == BT_STATE.CONNECTED, timeoutMs); - event("connected: timers " + device.timers()); - return ready; } /** @@ -232,6 +251,38 @@ List stopRecording() { } } + /** + * Ends one stretch of the recording and starts the next, for when the test stops and restarts + * streaming on purpose: each stretch is judged on its own, so the pause is not taken for loss. + */ + List nextSegment() { + synchronized (samples) { + List done = new ArrayList<>(samples); + samples.clear(); + return done; + } + } + + /** + * Starts SD logging on a streaming unit the way Consensys does (PanelLiveData, DEV-436): stop + * streaming, start logging on the quiet link (SET_RWC, then START_LOGGING_ONLY), then stream + * again. The RTC is therefore never written mid-stream. The stretch recorded before the stop is + * returned, and recording carries on into a new one. + */ + List startSdLoggingLikeConsensys(String what, HilReport report) throws InterruptedException { + report.check(stopStreaming(10000), what + ": streaming did not stop before starting SD logging"); + List before = nextSegment(); + boolean started = call("startSDLogging", 10000, device::startSDLogging); + report.check(started, what + ": startSDLogging did not complete: it threw, hung or was skipped (see the events)"); + if (started) { + report.check(waitFor(device::isSDLogging, 5000), what + ": SD logging did not start (state " + state() + "; is there an SD card?)"); + } + report.check(startStreaming(10000), what + ": streaming did not restart after starting SD logging"); + report.check(waitFor(() -> state() == BT_STATE.STREAMING_AND_SDLOGGING, 5000), + what + ": not streaming and logging after the restart (state " + state() + ")"); + return before; + } + boolean startStreaming(long timeoutMs) throws InterruptedException { return call("startStreaming", 10000, device::startStreaming) && waitFor(device::isStreaming, timeoutMs); } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index 7e6d2ef67..e5bd42619 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -104,6 +104,7 @@ private void finish() { /** What the driver did in this session: replies by type, decodes, timer requests, connects. */ private void putDriverCounts() { report.put("connects", session.connects); + report.put("connectRetries", session.connectRetries); report.put("inStreamReplies", device.inStreamReplies()); report.put("statusesDecoded", device.statusesDecoded()); report.put("batteryReadings", device.batteryReadings()); @@ -175,6 +176,60 @@ private void checkTimeline(StreamAnalysis a, List samples } } + /** + * Judges a stream recorded in stretches, split where the test stopped and restarted streaming on + * purpose, so the pause is not taken for loss. Each stretch gets the timestamp checks; loss within + * 1 s of an SD logging stop is reported, not failed (the unit pauses while it closes its file); + * loss near a battery read, or more than maxLossPercent anywhere else, fails. + */ + private void checkStretches(List> stretches, double stoppedAtMs, double deviceRateHz, + double maxLossPercent, List sdStops, List batteryMoments) { + int received = 0; + int missing = 0; + int nearSd = 0; + int nearBattery = 0; + List all = new ArrayList<>(); + List summaries = new ArrayList<>(); + for (int k = 0; k < stretches.size(); k++) { + List stretch = stretches.get(k); + String what = "stretch " + (k + 1); + all.addAll(stretch); + StreamAnalysis a = StreamAnalysis.analyse(stretch, deviceRateHz); + session.event(what + ": " + a); + summaries.add(what + ": " + a); + report.check(a.received > 0, what + ": no packets received"); + int[] attributed = a.attribute(new double[][] {toArray(sdStops), toArray(batteryMoments)}, 1000); + received += a.received; + missing += a.missing; + nearSd += attributed[0]; + nearBattery += attributed[1]; + report.check(a.nonIncreasing == 0, what + ": " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, what); + // The rate needs a second or so of stream; only the last stretch can go silent before a stop + // the test did not make itself. + if (a.received >= deviceRateHz) { + double end = k == stretches.size() - 1 ? stoppedAtMs : stretch.get(stretch.size() - 1).receivedMs; + checkTimeline(a, stretch, end, deviceRateHz, what); + } + } + report.put("stretches", summaries); + report.put("received", received); + report.put("missing", missing); + report.put("missingNearSdLoggingStops", nearSd); + report.put("missingNearBatteryReads", nearBattery); + report.samples(all); + if (nearSd > 0) { + session.event(nearSd + " packets not received within 1 s of an SD logging stop (firmware side; not failed)"); + } + report.check(nearBattery == 0, nearBattery + " packets lost within 1 s of a battery read while streaming"); + int elsewhere = missing - nearSd - nearBattery; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, received + missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLossPercent, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from SD logging stops and battery reads (%d), limit %.2f%%", + elsewherePercent, elsewhere, maxLossPercent)); + } + /** Parser errors and command timeouts the driver logged since index from. */ private void checkDriverLog(int from, String what) { int parserErrors = device.countSince(from, "Unknown parsing error", "Unknown packet error", "Throw Bytes"); @@ -227,8 +282,10 @@ public void streamRest() throws Exception { } /** - * Commands while streaming: start and stop SD logging (SET_RWC and the logging commands, with - * their bare ACKs) and read the battery, each waited on. Needs an SD card in the unit. + * SD logging and battery reads on a streaming unit, in the order Consensys uses (DEV-436): SD + * logging starts with streaming stopped (stop streaming, SET_RWC and START_LOGGING_ONLY on the + * quiet link, stream again), stops mid-stream (STOP_LOGGING_ONLY), and the battery is read + * mid-stream. Each call and reply is waited on. Needs an SD card in the unit. */ @Test public void midStreamSet() throws Exception { @@ -243,62 +300,36 @@ public void midStreamSet() throws Exception { session.startRecording(); report.check(session.startStreaming(10000), "streaming did not start within 10 s"); Thread.sleep(3000); - List sdMoments = new ArrayList<>(); + List> stretches = new ArrayList<>(); + List sdStops = new ArrayList<>(); List batteryMoments = new ArrayList<>(); for (int i = 1; i <= cycles && session.hung == null; i++) { - session.event("cycle " + i); - sdMoments.add(session.nowMs()); - boolean started = session.call("startSDLogging", 10000, device::startSDLogging); - report.check(started, "cycle " + i + ": startSDLogging did not complete: it threw, hung or was skipped (see the events)"); - if (started) { - report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING, 5000), - "cycle " + i + ": SD logging did not start while streaming (state " + session.state() + "; is there an SD card?)"); - } + String what = "cycle " + i; + session.event(what); + stretches.add(session.startSdLoggingLikeConsensys(what, report)); Thread.sleep(2000); - sdMoments.add(session.nowMs()); + sdStops.add(session.nowMs()); boolean stopped = session.call("stopSDLogging", 10000, device::stopSDLogging); - report.check(stopped, "cycle " + i + ": stopSDLogging did not complete: it threw, hung or was skipped (see the events)"); + report.check(stopped, what + ": stopSDLogging did not complete: it threw, hung or was skipped (see the events)"); if (stopped) { report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING, 5000), - "cycle " + i + ": SD logging did not stop (state " + session.state() + ")"); + what + ": SD logging did not stop (state " + session.state() + ")"); } Thread.sleep(1000); batteryMoments.add(session.nowMs()); int readingsBefore = device.batteryReadings(); boolean asked = session.call("readBattery", 10000, device::readBattery); - report.check(asked, "cycle " + i + ": readBattery did not complete: it threw, hung or was skipped (see the events)"); + report.check(asked, what + ": readBattery did not complete: it threw, hung or was skipped (see the events)"); if (asked) { report.check(session.waitFor(() -> device.batteryReadings() > readingsBefore, 5000), - "cycle " + i + ": no battery reply decoded within 5 s of readBattery"); + what + ": no battery reply decoded within 5 s of readBattery"); } Thread.sleep(2000); } double stoppedAtMs = session.nowMs(); - List samples = session.stopRecording(); + stretches.add(session.stopRecording()); report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); - StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); - session.event("stream: " + a); - report.putStream("stream", a); - report.putGaps(a); - report.samples(samples); - // Loss around SD logging start/stop is reported, not failed: the unit itself pauses while it - // opens or closes its file. Loss anywhere else is the driver's. - int[] attributed = a.attribute(new double[][] {toArray(sdMoments), toArray(batteryMoments)}, 1000); - report.put("missingNearSdLoggingStartStop", attributed[0]); - report.put("missingNearBatteryReads", attributed[1]); - if (attributed[0] > 0) { - session.event(attributed[0] + " packets not received within 1 s of SD logging start/stop (firmware side; not failed)"); - } - report.check(attributed[1] == 0, attributed[1] + " packets lost within 1 s of a battery read while streaming"); - int elsewhere = a.missing - attributed[0]; - double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); - report.put("missingElsewhere", elsewhere); - report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, - "stream: %.3f%% packets lost away from SD logging start/stop (%d), limit %.2f%%", elsewherePercent, elsewhere, maxLoss)); - report.check(a.received > 0, "stream: no packets received"); - report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); - checkJumps(a, "stream"); - checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + checkStretches(stretches, stoppedAtMs, deviceRate, maxLoss, sdStops, batteryMoments); checkDriverLog(logMark, "stream"); report.check(session.disconnect(10000), "did not disconnect within 10 s"); } @@ -344,10 +375,11 @@ public void reconnectLoop() throws Exception { /** * In-stream commands while streaming. First each reply on its own: status (0x8A 0x71), directory - * (0x8A 0x88) and battery (0x8A 0x94), each checked to be decoded. Then the overlap: SET_RWC - * followed at once by status and battery requests, so a reply arrives while another command - * waits for its ACK (DEV-1262). Every command must complete, with its own reply, and none may - * time out. Packets the unit drops right after an RTC write are reported; timestamp faults fail. + * (0x8A 0x88) and battery (0x8A 0x94), each checked to be decoded. Then the overlap, as Consensys + * meets it: with the unit streaming and logging (logging started the Consensys way), stop SD + * logging mid-stream (STOP_LOGGING_ONLY) and request status and battery at once, so in-stream + * replies, including any status the unit pushes as logging stops, arrive while that command waits + * for its ACK (DEV-1262). Every command must complete, with its own reply, and none may time out. */ @Test public void inStream() throws Exception { @@ -373,17 +405,20 @@ public void inStream() throws Exception { expectReply(what, "battery", "VBATT", device::readBattery, () -> device.batteryReadings() > batteryBefore); Thread.sleep(500); } - List rtcWrites = new ArrayList<>(); + List> stretches = new ArrayList<>(); + List sdStops = new ArrayList<>(); for (int i = 1; i <= overlaps && session.hung == null; i++) { String what = "overlap " + i; - rtcWrites.add(session.nowMs()); + stretches.add(session.startSdLoggingLikeConsensys(what, report)); + Thread.sleep(1000); int statusReplies = device.inStreamReplies("STATUS"); int batteryReplies = device.inStreamReplies("VBATT"); int statusBefore = device.statusesDecoded(); int batteryBefore = device.batteryReadings(); int timeoutsBefore = device.countSince(logMark, COMMAND_TIMEOUT); - boolean sent = session.call(what + ": SET_RWC, GET_STATUS and GET_VBATT back to back", 10000, () -> { - device.writeRealTimeClock(); + sdStops.add(session.nowMs()); + boolean sent = session.call(what + ": STOP_LOGGING_ONLY, GET_STATUS and GET_VBATT back to back", 10000, () -> { + device.stopSDLogging(); device.readStatusLogAndStream(); device.readBattery(); }); @@ -391,11 +426,11 @@ public void inStream() throws Exception { if (sent) { boolean done = session.waitFor(() -> device.inStreamReplies("STATUS") > statusReplies && device.inStreamReplies("VBATT") > batteryReplies - && session.queuedInstructions() == 0, 5000); + && session.queuedInstructions() == 0 && session.state() == BT_STATE.STREAMING, 5000); report.check(done, String.format(Locale.ROOT, - "%s: not every command completed within 5 s (status replies +%d, battery replies +%d, %d still queued)", + "%s: not every command completed within 5 s (status replies +%d, battery replies +%d, %d still queued, state %s)", what, device.inStreamReplies("STATUS") - statusReplies, device.inStreamReplies("VBATT") - batteryReplies, - session.queuedInstructions())); + session.queuedInstructions(), session.state())); report.check(device.statusesDecoded() > statusBefore, what + ": the status reply was not decoded"); report.check(device.batteryReadings() > batteryBefore, what + ": the battery reply was not decoded"); int timeouts = device.countSince(logMark, COMMAND_TIMEOUT) - timeoutsBefore; @@ -404,32 +439,11 @@ public void inStream() throws Exception { Thread.sleep(1000); } double stoppedAtMs = session.nowMs(); - List samples = session.stopRecording(); + stretches.add(session.stopRecording()); report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); report.check(session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 3000), "after streaming the unit is " + session.state() + ", not CONNECTED"); - StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); - session.event("stream: " + a); - report.putStream("stream", a); - report.putGaps(a); - report.samples(samples); - // Loss right after an RTC write is reported, not failed: the unit itself drops packets when - // its clock is set while streaming (firmware side, as at SD logging start in midStreamSet). - // Timestamps that go backwards or leap ahead still fail: the API hands them to apps. - int nearRtc = a.attribute(new double[][] {toArray(rtcWrites)}, 1000)[0]; - report.put("missingNearRtcWrites", nearRtc); - if (nearRtc > 0) { - session.event(nearRtc + " packets not received within 1 s of an RTC write (firmware side; not failed)"); - } - int elsewhere = a.missing - nearRtc; - double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); - report.put("missingElsewhere", elsewhere); - report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, - "stream: %.3f%% packets lost away from the RTC writes (%d), limit %.2f%%", elsewherePercent, elsewhere, maxLoss)); - report.check(a.received > 0, "stream: no packets received"); - report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); - checkJumps(a, "stream"); - checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + checkStretches(stretches, stoppedAtMs, deviceRate, maxLoss, sdStops, new ArrayList<>()); checkDriverLog(logMark, "stream"); report.check(session.disconnect(10000), "did not disconnect within 10 s"); } diff --git a/tools/hil/README.md b/tools/hil/README.md index d8af1ae6e..8d4cf7c87 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -40,19 +40,25 @@ If anything merges after step 1, the commit changes, so test again. In an emerge | Scenario | What it does | Fails when | |---|---|---| | `streamRest` | Connects, writes an accel-only config at `-Rate` (default 256 Hz) the way Consensys's Bluetooth manager does, streams for `-Seconds` (default 30) with the unit lying still, then stops and disconnects. | It can't connect or configure; more than `-MaxLossPercent` (1 %) of packets are lost; a timestamp doesn't increase, or leaps more than a second ahead of when it arrived (a bad unwrap, not loss); the timestamps give no rate, or one more than 1 % off the configured rate; the stream goes silent for over a second before the test stops it; accel is more than 1 m/s² from 9.81; or the driver logs a parser error or a command timeout. | -| `midStreamSet` | While streaming, starts and stops SD logging and reads the battery, three times. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop; a packet is lost around a battery read, or more than `-MaxLossPercent` away from the SD transitions; the timestamps fail the checks above; or the driver logs a parser error or a command timeout. Packets missing around SD logging start/stop are reported, not failed: the unit itself pauses while it opens its file. Needs an SD card in the unit. | +| `midStreamSet` | On a streaming unit, three times: starts SD logging **the way Consensys does** (stop streaming, start logging on the quiet link, stream again; DEV-436), stops it mid-stream, and reads the battery mid-stream. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop, or streaming doesn't come back; a packet is lost around a battery read, or more than `-MaxLossPercent` elsewhere; a timestamp fault in any stretch between restarts; or the driver logs a parser error or a command timeout. Packets missing around an SD logging stop are reported, not failed: the unit pauses while it closes its file. Needs an SD card in the unit. | | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times on the same `ShimmerPC`. Every connect starts the driver's status timer, which is when the DEV-1266 lock inversion is most likely. | Any cycle fails to connect within 30 s, loses packets, fails the timestamp checks, or the driver logs a parser error or command timeout. | -| `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps: `SET_RWC`, a status request and a battery request back to back, so replies arrive while another command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, the queue empty); the driver logs a command timeout; or the stream fails the checks above. Packets missing within 1 s of an RTC write are reported, not failed (the unit itself drops them; see DEV-1245), but timestamp faults fail. | +| `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps as Consensys meets them: with the unit streaming and logging (logging started the Consensys way), stop SD logging mid-stream and request status and battery at once, so in-stream replies, including any status the unit pushes as logging stops, arrive while that command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, logging stopped, the queue empty); the driver logs a command timeout; or the stream fails the checks above. | Every API call runs under a watchdog. If one doesn't return within 10 s, the scenario stops, and its events hold every thread's stack with the locks each holds and waits on, plus any deadlock the JVM detects (DEV-1266). If the whole run still doesn't finish, the runner stops Gradle after `-ScenarioTimeoutSeconds` (900) and records TIMEOUT. +If Windows can't open the Bluetooth link at all (the driver reports CONNECTION_FAILED, and jssc logs +"Port busy" after paging the unit for about 5 s), a connect is tried once more, as a user would. +That happens on a minority of first connects; `connectRetries` in the report counts them. + Each report counts what the session covered: connects, in-stream replies by type, statuses and battery readings decoded, the status timer's requests, and command timeouts. Its events also say which driver timers were running after connect, after configure and while streaming. +Starting SD logging mid-stream (`startSDLogging()` while streaming) writes the RTC mid-stream, which a Shimmer3R answers with out-of-order timestamps (DEV-1245). Consensys never does that, and neither do these scenarios. + The stream checks are the Android HIL's (`StreamAnalysis`, the same class in ShimmerAndroidAPI's `shimmerHilTest`), so the two judge a stream by the same rules. `StreamAnalysisTest` runs in CI. From 0c8d467226ecabac34fac837982f987a00271402 Mon Sep 17 00:00:00 2001 From: JongChern Date: Wed, 21 Oct 2026 16:54:36 +0800 Subject: [PATCH 28/41] DEV-1269 Java HIL: count command timeouts The pattern expected the command as one word, but the driver logs it as "[0x72] GET_STATUS_COMMAND". So every commandTimeouts count was 0, and the "no command timeouts" checks could never fail. Checked against the bench logs so far: it now finds all 14 timeouts in them. Co-Authored-By: Claude Opus 5.5 --- .../java/com/shimmerresearch/hil/JavaHilScenariosTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index e5bd42619..0489c0114 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -41,8 +41,11 @@ public class JavaHilScenariosTest { private static final double GRAVITY = 9.81; /** At 256 Hz a second without a packet is ~256 packets the gap count cannot see. */ private static final double MAX_SILENCE_AT_END_MS = 1000; - /** The ACK/response timer's own line when a command gets no reply: "Command: timeout". */ - private static final java.util.regex.Pattern COMMAND_TIMEOUT = java.util.regex.Pattern.compile("Command:\t\\S+ timeout"); + /** + * The ACK/response timer's own line when a command gets no reply: "Command: [0x72] GET_STATUS_COMMAND + * timeout", or "[0x..]UNKNOWN" for an opcode the driver doesn't know. + */ + private static final java.util.regex.Pattern COMMAND_TIMEOUT = java.util.regex.Pattern.compile("Command:\t\\[0x[0-9A-F]{2}\\] ?\\S+ timeout"); private HilSession session; private HilReport report; From 8b8c79282b8e134d84f3813e37eb68edb00d2d50 Mon Sep 17 00:00:00 2001 From: JongChern Date: Wed, 21 Oct 2026 20:22:50 +0800 Subject: [PATCH 29/41] DEV-1269 Java HIL: fail streamRest when the accel is NaN The accel check was skipped when the mean was NaN, so a stream with no calibrated accel passed. A Shimmer3 (3E36, LogAndStream v1.1.7) did exactly that: all 7655 samples NaN after the driver read its calibration dump. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hil/JavaHilScenariosTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index 0489c0114..0d24746c2 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -274,10 +274,10 @@ public void streamRest() throws Exception { report.putGaps(a); report.samples(samples); checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); - if (!Double.isNaN(a.accelMagnitudeMean)) { - report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= 1.0, String.format(Locale.ROOT, - "accel at rest %.3f m/s^2, expected %.2f±1.00 (is the unit lying still?)", a.accelMagnitudeMean, GRAVITY)); - } + // NaN fails too: the scenario configured the accel, so a stream without calibrated accel is a fault + report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= 1.0, String.format(Locale.ROOT, + "accel at rest %.3f m/s^2, expected %.2f±1.00 (NaN: no calibrated accel; otherwise is the unit lying still?)", + a.accelMagnitudeMean, GRAVITY)); checkDriverLog(logMark, "stream"); report.check(session.disconnect(10000), "did not disconnect within 10 s"); } From 35d441d737e0fa1abb94f00d301ebc9d2b40bc34 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 11:51:11 +0800 Subject: [PATCH 30/41] DEV-1269 Java HIL: fail a connect whose port is in use, instead of retrying it Every "Port busy" the HIL ever met was another program holding the COM port, never a flaky first connect. Two HILs shared a unit, and that sharing produced the misleading DEV-1276. A retry only lets the run go on while the unit is shared, so its results mean nothing. So a connect is no longer retried. When it fails, the report says why: - the port is open in another program (jssc "Port busy"), which the HIL now sees because HilShimmerPC records the messages the driver prints with consolePrintException; - the link could not be opened for another reason; - the unit did not initialise in time. The README now says that nothing else may use the units during a run, and why a BLE host is the dangerous case. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hil/HilSession.java | 54 ++++++++++++------- .../com/shimmerresearch/hil/HilShimmerPC.java | 7 +++ .../hil/JavaHilScenariosTest.java | 7 +-- tools/hil/README.md | 22 ++++++-- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java index c181419f6..b9a9b896b 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java @@ -11,6 +11,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; import com.shimmerresearch.bluetooth.ShimmerBluetooth; import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; @@ -62,8 +63,10 @@ interface Condition { volatile String hung; /** Connects made in this session, for the report: each is a chance for the connect-time race. */ int connects; - /** Connects that failed to open the Bluetooth link and were tried again. */ - int connectRetries; + /** Why the last connect failed, as a sentence for the report; null after one that succeeded. */ + volatile String connectFailure; + /** jssc's message when Windows refuses to open a COM port that another program already has open. */ + private static final Pattern PORT_BUSY = Pattern.compile("Port busy"); HilSession(String port, String mac, String type) { this.port = port; @@ -174,28 +177,39 @@ boolean call(String what, long timeoutMs, Call call) { } /** - * Connects and waits until the driver has read the unit's configuration. If Windows cannot open - * the Bluetooth link at all (CONNECTION_FAILED, which jssc reports as "Port busy" after paging the - * unit for about 5 s), it tries once more, as a user would; connectRetries counts those. + * Connects and waits until the driver has read the unit's configuration. A connect is never + * retried. When the port can't be opened, connectFailure says why. jssc's "Port busy" means + * another program on this PC has the port open (another HIL, Consensys, a serial terminal), so + * the unit is in use. Results taken while another host shares the unit mean nothing, and that is + * how DEV-1276 was misread. */ boolean connect(long timeoutMs) throws InterruptedException { - for (int attempt = 1; ; attempt++) { - connects++; - int mark = stateChanges; - Condition failed = () -> stateChanges > mark && device.getBluetoothRadioState() == BT_STATE.CONNECTION_FAILED; - if (!call("connect on " + port, 10000, () -> device.connect(port, ""))) { - return false; - } - boolean ready = waitFor(() -> (device.isInitialised() && device.getBluetoothRadioState() == BT_STATE.CONNECTED) - || failed.holds(), timeoutMs) && !failed.holds(); - if (ready || attempt > 1 || !failed.holds()) { - event("connected: timers " + device.timers()); - return ready; + connects++; + connectFailure = null; + int mark = stateChanges; + int logMark = device.logSize(); + Condition failed = () -> stateChanges > mark && device.getBluetoothRadioState() == BT_STATE.CONNECTION_FAILED; + if (!call("connect on " + port, 10000, () -> device.connect(port, ""))) { + connectFailure = "connect on " + port + " did not return (see the events)"; + return false; + } + boolean ready = waitFor(() -> (device.isInitialised() && device.getBluetoothRadioState() == BT_STATE.CONNECTED) + || failed.holds(), timeoutMs) && !failed.holds(); + if (ready) { + event("connected: timers " + device.timers()); + } else { + if (device.countSince(logMark, PORT_BUSY) > 0) { + connectFailure = port + " is open in another program (another HIL, Consensys, a serial terminal?): " + + "jssc reports \"Port busy\". Nothing else may use the unit during a run."; + } else if (failed.holds()) { + connectFailure = "the Bluetooth link on " + port + " could not be opened (CONNECTION_FAILED): " + + "is the unit on, in range, and not connected to another host?"; + } else { + connectFailure = "did not connect and initialise within " + timeoutMs / 1000 + " s (state " + state() + ")"; } - connectRetries++; - event("the Bluetooth link could not be opened (CONNECTION_FAILED); trying once more in 5 s"); - Thread.sleep(5000); + event(connectFailure); } + return ready; } /** diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java index 932a18460..19aeb2042 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java @@ -65,6 +65,13 @@ public void consolePrintErrLn(String message) { super.consolePrintErrLn(message); } + /** The driver prints an exception's message to System.out, past consolePrintLn; keep it in the log too. */ + @Override + protected void consolePrintException(String message, StackTraceElement[] stackTrace) { + record("EXCEPTION " + message); + super.consolePrintException(message, stackTrace); + } + private void record(String message) { String line = String.format(Locale.ROOT, "%9.0f %s", (System.nanoTime() - t0) / 1e6, message); synchronized (log) { diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index 0d24746c2..e85c98e7c 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -107,7 +107,6 @@ private void finish() { /** What the driver did in this session: replies by type, decodes, timer requests, connects. */ private void putDriverCounts() { report.put("connects", session.connects); - report.put("connectRetries", session.connectRetries); report.put("inStreamReplies", device.inStreamReplies()); report.put("statusesDecoded", device.statusesDecoded()); report.put("batteryReadings", device.batteryReadings()); @@ -121,7 +120,7 @@ private boolean connectAndConfigure(double rateHz) throws Exception { long start = System.currentTimeMillis(); boolean connected = session.connect(30000); report.put("connectMs", System.currentTimeMillis() - start); - report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + report.check(connected, String.valueOf(session.connectFailure)); if (!connected) { return false; } @@ -352,7 +351,9 @@ public void reconnectLoop() throws Exception { long start = System.currentTimeMillis(); boolean ready = i == 1 ? connectAndConfigure(rate) : session.connect(30000); connectTimes.add(System.currentTimeMillis() - start); - report.check(ready, what + ": did not connect within 30 s (state " + session.state() + ")"); + // In cycle 1 a failed configure is reported by connectAndConfigure, with connectFailure null + report.check(ready, what + ": " + (session.connectFailure != null ? session.connectFailure + : "not ready to stream (state " + session.state() + ")")); if (!ready) { break; } diff --git a/tools/hil/README.md b/tools/hil/README.md index 8d4cf7c87..83f19b3fa 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -49,9 +49,11 @@ events hold every thread's stack with the locks each holds and waits on, plus an detects (DEV-1266). If the whole run still doesn't finish, the runner stops Gradle after `-ScenarioTimeoutSeconds` (900) and records TIMEOUT. -If Windows can't open the Bluetooth link at all (the driver reports CONNECTION_FAILED, and jssc logs -"Port busy" after paging the unit for about 5 s), a connect is tried once more, as a user would. -That happens on a minority of first connects; `connectRetries` in the report counts them. +A connect is never retried. If the COM port can't be opened, the scenario fails and says why: +- **"Port busy"**: another program on this PC already has the port open: another HIL, Consensys, a + serial terminal. The unit is in use, so a retry would only produce results from a shared unit. +- **CONNECTION_FAILED** for any other reason: the unit is off, out of range, or connected to + another host such as a phone. Each report counts what the session covered: connects, in-stream replies by type, statuses and battery readings decoded, the status timer's requests, and command timeouts. Its events also say @@ -99,8 +101,18 @@ The stream checks are the Android HIL's (`StreamAnalysis`, the same class in Shi 5. **JDK 11 or newer.** The script uses `JAVA_HOME` if it qualifies, otherwise the first one it finds under Amazon Corretto, Eclipse Adoptium or Java in Program Files. -A Shimmer holds one Bluetooth link at a time. Don't run this while Consensys, the Android HIL's phone -or anything else is connected to the same unit. +**Nothing else may use the units during a run.** That includes Consensys, the Android HIL's phone, +and the other HILs that share this bench (C#, shimmer-logandstream). + +Only a second Classic connection is refused outright, as "Port busy". A second host connecting over +BLE is not: +- a Shimmer3R that is streaming over Classic accepts the BLE link; +- it runs the commands sent over BLE and replies down the Classic link, which corrupts this run's + stream; +- the BLE host sees no replies at all. + +On 2026-10-21 that is how two HILs sharing a unit produced a bug report about the unit locking up +after a radio switch (DEV-1276). ## Output From 9bb3d5da80ffa153fe6b47f930179481a25f7e91 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 13:06:04 +0800 Subject: [PATCH 31/41] DEV-1275 Report a Shimmer ready only once its connect sequence has ended The connect sequence reads the unit's status part way through. On that reply, ShimmerPC.eventLogAndStreamStatusChanged() moved the state from CONNECTING to CONNECTED, and ShimmerDeviceCallbackAdapter marks every CONNECTED as initialised. So an app waiting for CONNECTED and initialised was told the unit was ready while the battery read, the calibration dump and the inquiry were still queued. Whatever the app did next went out in among that sequence: - streaming started part way through it, and a reply from the connect sequence landed in the stream, which then went silent; - a configure cut the calibration dump short, which left a Shimmer3's calibrated accel NaN in every sample (7 of 9 runs on the HIL). The sequence's own end, inquiryDone() and then isReadyForStreaming(), already moves CONNECTING to CONNECTED and sends NOTIFICATION_SHIMMER_FULLY_INITIALIZED. A status reply now leaves CONNECTING to it. The Shimmer3R simulator answered INQUIRY with the Shimmer3's 8-byte settings, which never parse as a Shimmer3R's 11. Its tests had only ever passed on the early CONNECTED, before the inquiry timed out. It now answers with settings that match its own InfoMem. API_00031 connects to the Shimmer3 simulator, whose sequence reads the status before the inquiry. It checks that CONNECTED comes only after the sequence's end, with nothing but the RTC write still queued. It fails without the fix. Co-Authored-By: Claude Opus 5.5 --- .../shimmerresearch/pcDriver/ShimmerPC.java | 9 +- ..._ConnectReadyAfterConnectSequenceTest.java | 107 ++++++++++++++++++ .../ByteCommunicationSimulatorS3R.java | 21 +++- 3 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_00031_ConnectReadyAfterConnectSequenceTest.java diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index 7c5ee004d..bd30f78ad 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -900,8 +900,13 @@ else if(isSDLogging()){ } else{ // if(!isStreaming() && !isSDLogging() && isConnected()){ - if(!mIsStreaming && !isSDLogging() && isConnected() && mBluetoothRadioState!=BT_STATE.CONNECTED){ - setBluetoothRadioState(BT_STATE.CONNECTED); + // While connecting, the connect sequence's own end (inquiryDone(), then + // isReadyForStreaming()) moves CONNECTING to CONNECTED. The sequence reads the + // status part way through, and if that reply did it, the app would be told the + // unit is ready while the rest of the sequence is still queued (DEV-1275). + if(!mIsStreaming && !isSDLogging() && isConnected() + && mBluetoothRadioState!=BT_STATE.CONNECTED && mBluetoothRadioState!=BT_STATE.CONNECTING){ + setBluetoothRadioState(BT_STATE.CONNECTED); } // if(getBTState() == BT_STATE.INITIALISED){ // diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_00031_ConnectReadyAfterConnectSequenceTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_00031_ConnectReadyAfterConnectSequenceTest.java new file mode 100644 index 000000000..7bba6aab7 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/API_00031_ConnectReadyAfterConnectSequenceTest.java @@ -0,0 +1,107 @@ +package com.shimmerresearch.shimmer3.communication; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.driver.ShimmerObject; +import com.shimmerresearch.pcDriver.ShimmerPC; + +/** + * DEV-1275: the connect sequence reads the unit's status part way through, and + * ShimmerPC.eventLogAndStreamStatusChanged() moved CONNECTING to CONNECTED on + * that reply. ShimmerDeviceCallbackAdapter marks every CONNECTED initialised, + * so an app waiting for CONNECTED and initialised was told the unit was ready + * while the rest of the sequence (battery, calibration dump, inquiry) was still + * queued. Whatever it did next went out in among that sequence: streaming + * started part way through it, and a configure cut the calibration dump short, + * which left a Shimmer3's accel NaN. The sequence's own end, inquiryDone() and + * then isReadyForStreaming(), is what moves CONNECTING to CONNECTED. + *

+ * No hardware: ByteCommunicationSimulatorS3 answers the whole connect sequence, + * the status read included. + */ +public class API_00031_ConnectReadyAfterConnectSequenceTest { + + private ShimmerPC mDevice; + private volatile boolean mStatusRead; + private volatile boolean mSequenceEnded; + /** What had happened when the state first became CONNECTED; null until then */ + private volatile Boolean mStatusReadAtFirstConnected; + private volatile Boolean mSequenceEndedAtFirstConnected; + private volatile List mQueuedAtFirstConnected; + + @Before + public void setUp() { + mDevice = new ShimmerPC("COM99") { + @Override + public void startTimerReadStatus() { + // As in ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer: only the + // connect sequence's own status read should run + } + + @Override + protected void eventLogAndStreamStatusChanged(byte currentCommand) { + mStatusRead = true; + super.eventLogAndStreamStatusChanged(currentCommand); + } + + @Override + protected void isReadyForStreaming() { + mSequenceEnded = true; + super.isReadyForStreaming(); + } + + @Override + public boolean setBluetoothRadioState(BT_STATE state) { + if(state==BT_STATE.CONNECTED && mSequenceEndedAtFirstConnected==null){ + mStatusReadAtFirstConnected = mStatusRead; + mSequenceEndedAtFirstConnected = mSequenceEnded; + List queued = new ArrayList<>(); + for(byte[] instruction:new ArrayList<>(getListofInstructions())){ + if(instruction!=null){ + queued.add(btCommandToString(instruction[0])); + } + } + mQueuedAtFirstConnected = queued; + } + return super.setBluetoothRadioState(state); + } + }; + mDevice.setTestRadio(new ByteCommunicationSimulatorS3("COM99")); + } + + @After + public void tearDown() { + mDevice.disconnectNoException(); + } + + @Test + public void theUnitIsReportedReadyOnlyOnceTheConnectSequenceHasEnded() { + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + + assertNotNull("the state became CONNECTED", mSequenceEndedAtFirstConnected); + assertTrue("the connect sequence read the status before the end, so the case is covered", mStatusReadAtFirstConnected); + assertTrue("CONNECTED came only once the connect sequence had ended (inquiryDone, isReadyForStreaming); still queued: " + + mQueuedAtFirstConnected, mSequenceEndedAtFirstConnected); + // Every read of the unit's configuration (status, battery, calibration dump, + // inquiry) is answered. What may remain: the inquiry itself, still at the head + // while its reply is handled, and the RTC write that the status reply queues + // when the unit's clock is not set. That write runs before anything the app + // queues, so it never lands in a stream. + List allowed = new ArrayList<>(); + allowed.add(ShimmerBluetooth.btCommandToString(ShimmerObject.INQUIRY_COMMAND)); + allowed.add(ShimmerBluetooth.btCommandToString(ShimmerObject.SET_RWC_COMMAND)); + for(String queued:mQueuedAtFirstConnected){ + assertTrue("no read from the connect sequence is still queued at CONNECTED, but " + queued + " is (queued: " + + mQueuedAtFirstConnected + ")", allowed.contains(queued)); + } + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationSimulatorS3R.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationSimulatorS3R.java index 4d7151238..b9ec057d7 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationSimulatorS3R.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/shimmer3/communication/ByteCommunicationSimulatorS3R.java @@ -24,7 +24,26 @@ protected void txShimmerVersion() { mBuffer.add((byte) 0x0A); } - @Override + /** + * A Shimmer3R's inquiry reply has 11 bytes of settings, with the channel count + * at index 9, so the Shimmer3 reply this class inherited never parsed: the + * inquiry timed out, and the tests only passed because a status reply part way + * through the connect sequence reported the unit ready first (DEV-1275). The + * settings match this simulator's InfoMem: the sampling rate, config setup + * bytes 0-3 from the first block and 4-6 from the second, then no channels + * and a buffer size of 1. + */ + @Override + protected void inquiryResponse() { + mBuffer.add((byte) 0xff); + mBuffer.add((byte) 0x02); + byte[] bytes = UtilShimmer.hexStringToByteArray("80024D9B0E080000000001"); + for (byte b:bytes) { + mBuffer.add(b); + } + } + + @Override protected void txFirmwareVersion() { mBuffer.add((byte) 0xff); mBuffer.add((byte) 0x2f); From c36c38089bf613d28633438217ce4ff44a3423ff Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 13:08:14 +0800 Subject: [PATCH 32/41] DEV-1291 Take an in-stream ACK that comes with no data packet before it After STOP_LOGGING_ONLY_COMMAND the unit stops sending data packets for about 100 ms while it closes its SD file. A command sent in that pause is answered at once, so its ACK comes straight after the previous ACK's CRC, with no data packet before it. processPacket() only looks for an ACK after a data packet, so it threw the reply away. The command then timed out, and while streaming the timeout clears every queued instruction, so whatever the user asked for next (including Stop Streaming) was silently dropped. The Java HIL found it on a Shimmer3 and a Shimmer3R: up to 4 of 5 status and battery requests sent after a logging stop were lost. processWhileStreaming() now looks for an ACK at the very start of the buffer first. It takes one only while a command waits for its ACK, and only when the bytes after it are what that command expects: - for a GET command, its own in-stream reply (FF 8A, then its response opcode); - for a SET command, a bare ACK whose link CRC checks out. So a 0xFF data byte the parser resyncs onto is not taken for one. It looks at no more than 3 bytes, fewer than processPacket() needs, so nothing else changes. Without a link CRC the case can't arise: the previous ACK isn't processed until the stream resumes, so the next command never goes out during the pause. API_00032 feeds the parser the bytes a logging stop produces, then a status reply, a battery reply, or a SET command's bare ACK in the pause, with 1- and 2-byte CRCs. Without the fix, all six of those cases fail. Two more check that an ACK at the start of the buffer is not taken for a different command. Co-Authored-By: Claude Opus 5.5 --- .../bluetooth/ShimmerBluetooth.java | 81 ++++- .../API_00032_StreamingAckAfterPauseTest.java | 302 ++++++++++++++++++ 2 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00032_StreamingAckAfterPauseTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index df8801f60..012d39546 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -1119,10 +1119,89 @@ protected void processWhileStreaming() { printLogDataForDebugging("readbyte null"); } + if(processAckAtStartOfBuffer()){ + return; + } + //If there is a full packet and the subsequent sequence number of following packet if(mByteArrayOutputStream.size()>=getPacketSizeWithCrc()+2){ // +2 because there are two acks processPacket(); - } + } + } + + /** + * Handles an ACK at the very start of the buffer, with no data packet before + * it. The unit sends one when it answers while its data packets are paused, as + * they are for about 100 ms while it closes its SD file after + * STOP_LOGGING_ONLY_COMMAND (DEV-1291). processPacket() only looks for an ACK + * after a data packet, so it threw such a reply away; the command then timed + * out, and the timeout dropped every queued instruction. + *

+ * It is taken only while a command waits for its ACK, and only when the bytes + * after it are what that command expects: its own in-stream reply (FF 8A and + * then its response opcode), or for a SET command a bare ACK whose link CRC + * checks out. A 0xFF data byte the parser resyncs onto is therefore not taken + * for one. Without a link CRC a bare ACK can't be told from a data byte, but + * then the previous ACK isn't processed until the stream resumes either, so the + * next command never goes out during the pause. + * + * @return true while the bytes so far are taken, or may still turn out to be + * such an ACK. The buffer is then at most 3 bytes, too short for + * processPacket() anyway. + */ + private boolean processAckAtStartOfBuffer() { + if(!mWaitForAck || mByteArrayOutputStream.size()==0){ + return false; + } + byte[] bufferTemp = mByteArrayOutputStream.toByteArray(); + if(bufferTemp[0]!=ACK_COMMAND_PROCESSED){ + return false; + } + if(bufferTemp.length==1){ + return true; //wait for what follows the ACK + } + + //The in-stream reply to the GET command in flight + if(isSupportedInStreamCmds() && bufferTemp[1]==INSTREAM_CMD_RESPONSE){ + if(bufferTemp.length==2){ + return true; //wait for the response opcode + } + if(bufferTemp.length>3 || !isReplyToCommandInFlight(bufferTemp[2])){ + return false; + } + printLogDataForDebugging("COMMAND TXed and ACK RECEIVED IN STREAM, with no data packet before it"); + stopTimerCheckForAckOrResp(); //cancel the ack timer + mWaitForResponse=false; + mWaitForAck=false; + + //Reads the rest of the response, and its CRC, from the port + processInstreamResponse(bufferTemp[2], true); + + if(getListofInstructions().size()>0){ + removeInstruction(0); + } + mTransactionCompleted=true; + setInstructionStackLock(false); + clearBuffers(); + return true; + } + + //A bare ACK to the SET command in flight + if(isKnownSetCommand(mCurrentCommand) && mBtCommsCrcModeCurrent!=BT_CRC_MODE.OFF){ + int numCrcBytes = mBtCommsCrcModeCurrent.getNumCrcBytes(); + for(int i=1;i + * Only with a link CRC: without one the previous ACK isn't processed until the + * stream resumes, so the next command never goes out during the pause. + *

+ * No hardware: the parser reads from an in-memory byte queue, one byte at a + * time as the IO thread does, as in API_00027 and API_00030. + */ +public class API_00032_StreamingAckAfterPauseTest { + + /** Bytes after the DATA_PACKET byte: a 3-byte timestamp and three 2-byte channels */ + private static final int PACKET_SIZE = 9; + + /** Docked, sensing, RTC set and streaming; not SD logging */ + private static final byte STATUS_DOCKED_AND_STREAMING = 0x17; + + private static class QueueRadio implements ByteCommunication { + final Deque rx = new ArrayDeque(); + + void queue(byte... bytes) { + for(byte b:bytes){ + rx.add(b); + } + } + + @Override + public byte[] readBytes(int byteCount, int timeout) throws SerialPortTimeoutException { + if(rx.size() expected = new ArrayList(); + for(byte[] payload:expectedPayloads){ + expected.add(UtilShimmer.bytesToHexStringWithSpacesFormatted(payload)); + } + List parsed = new ArrayList(); + // Package-private, so ShimmerPC does not inherit it + ShimmerBluetooth device = mDevice; + for(RawBytePacketWithPCTimeStamp packet:device.mABQPacketByeArray){ + parsed.add(UtilShimmer.bytesToHexStringWithSpacesFormatted(packet.mDataArray)); + } + assertEquals("every data packet is parsed, once and in order", expected, parsed); + } + + private void assertCompleted() { + assertFalse("the ACK is taken", mDevice.mWaitForAck); + assertFalse("the next instruction can be sent", mDevice.isInstructionStackLock()); + assertNull("the ACK timer is cancelled", mDevice.mTimerCheckForAckOrResp); + } + + /** GET_STATUS_COMMAND sent in the pause: its reply must complete it, and nothing queued behind it may be lost */ + private void statusReplyInThePause(BT_CRC_MODE crcMode) { + streamingShimmer3r(crcMode); + mDevice.stopSDLogging(); + mDevice.readStatusLogAndStream(); + mDevice.readBattery(); + stopLoggingAndReceiveItsAck(crcMode); + + sendNextInstruction(); + assertEquals(ShimmerObject.GET_STATUS_COMMAND, mDevice.mCurrentCommand); + receive(statusReply(crcMode), dataPacket(2, crcMode), dataPacket(3, crcMode), new byte[] {ShimmerObject.DATA_PACKET}); + assertCompleted(); + assertTrue("the status reply is decoded", mDevice.isDocked()); + assertEquals("GET_VBATT_COMMAND is still queued", 1, mDevice.getListofInstructions().size()); + assertEquals(ShimmerObject.GET_VBATT_COMMAND, mDevice.getListofInstructions().get(0)[0]); + assertParsed(payload(1), payload(2), payload(3)); + } + + @Test + public void statusReplyInThePauseWithOneByteCrc() { + statusReplyInThePause(BT_CRC_MODE.ONE_BYTE_CRC); + } + + @Test + public void statusReplyInThePauseWithTwoByteCrc() { + statusReplyInThePause(BT_CRC_MODE.TWO_BYTE_CRC); + } + + /** GET_VBATT_COMMAND sent in the pause */ + private void batteryReplyInThePause(BT_CRC_MODE crcMode) { + streamingShimmer3r(crcMode); + mDevice.stopSDLogging(); + mDevice.readBattery(); + stopLoggingAndReceiveItsAck(crcMode); + + sendNextInstruction(); + receive(vbattReply(crcMode), dataPacket(2, crcMode), new byte[] {ShimmerObject.DATA_PACKET}); + assertCompleted(); + assertTrue(mDevice.getListofInstructions().isEmpty()); + assertEquals(0x0B24, mDevice.getBattStatusDetails().getBattAdcValue(), 0); + assertParsed(payload(1), payload(2)); + } + + @Test + public void batteryReplyInThePauseWithOneByteCrc() { + batteryReplyInThePause(BT_CRC_MODE.ONE_BYTE_CRC); + } + + @Test + public void batteryReplyInThePauseWithTwoByteCrc() { + batteryReplyInThePause(BT_CRC_MODE.TWO_BYTE_CRC); + } + + /** A SET command sent in the pause: its bare ACK must complete it */ + private void bareAckInThePause(BT_CRC_MODE crcMode) { + streamingShimmer3r(crcMode); + mDevice.stopSDLogging(); + mDevice.startSDLogging(); // SET_RWC_COMMAND, then START_LOGGING_ONLY_COMMAND + assertEquals(3, mDevice.getListofInstructions().size()); + stopLoggingAndReceiveItsAck(crcMode); + + sendNextInstruction(); + assertEquals(ShimmerObject.SET_RWC_COMMAND, mDevice.mCurrentCommand); + receive(bareAck(crcMode), dataPacket(2, crcMode), new byte[] {ShimmerObject.DATA_PACKET}); + assertCompleted(); + assertEquals("START_LOGGING_ONLY_COMMAND is still queued", 1, mDevice.getListofInstructions().size()); + assertEquals(ShimmerObject.START_LOGGING_ONLY_COMMAND, mDevice.getListofInstructions().get(0)[0]); + assertParsed(payload(1), payload(2)); + } + + @Test + public void bareAckInThePauseWithOneByteCrc() { + bareAckInThePause(BT_CRC_MODE.ONE_BYTE_CRC); + } + + @Test + public void bareAckInThePauseWithTwoByteCrc() { + bareAckInThePause(BT_CRC_MODE.TWO_BYTE_CRC); + } + + /** + * An ACK at the start of the buffer is taken only for the reply the command in + * flight expects: a status push there does not complete GET_VBATT_COMMAND. + */ + private void anotherReplyAtTheStartIsNotTaken(BT_CRC_MODE crcMode) { + streamingShimmer3r(crcMode); + mDevice.readBattery(); + sendNextInstruction(); + + receive(statusReply(crcMode), dataPacket(1, crcMode), dataPacket(2, crcMode)); + assertEquals("GET_VBATT_COMMAND is still in flight", ShimmerObject.GET_VBATT_COMMAND, mDevice.mCurrentCommand); + assertTrue("it is still waiting for its ACK", mDevice.mWaitForAck); + assertTrue("the next instruction is not sent before it completes", mDevice.isInstructionStackLock()); + assertEquals(1, mDevice.getListofInstructions().size()); + } + + @Test + public void anotherReplyAtTheStartIsNotTakenWithOneByteCrc() { + anotherReplyAtTheStartIsNotTaken(BT_CRC_MODE.ONE_BYTE_CRC); + } + + @Test + public void anotherReplyAtTheStartIsNotTakenWithTwoByteCrc() { + anotherReplyAtTheStartIsNotTaken(BT_CRC_MODE.TWO_BYTE_CRC); + } +} From f186c908345276d673743628ef94ba02689dc2d9 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 13:36:08 +0800 Subject: [PATCH 33/41] DEV-1266 Take the instruction list's two locks in one order ShimmerBluetooth guards its instruction list with two locks: the device (its synchronized methods) and the list itself. Every accessor takes them device first, list second. getInstruction(), which the IO thread calls for the next instruction, took the list and then called the synchronized getListofInstructions(), so it took them the other way round. So the IO thread could hold the list while waiting for the device, just as a timer held the device while waiting for the list: - the ACK/response timeout, calling clearAllInstructions() while streaming; - the read-status timer, queuing GET_STATUS_COMMAND. Both then waited for good, and so did every later API call. The Android HIL caught it on a Shimmer3, where stopSDLogging() never returned. It was also the Linux CI hang in DEV-1198, which #325 worked around in the tests only. getInstruction() now reads the head of the list directly, so the IO thread never needs the device while it holds the list. This is Mas's suggestion from her review of #326. API_00033 reproduces the deadlock deterministically. A "timer" thread holds the device the way every synchronized accessor does while the IO thread's real getInstruction() runs; the timer then queues an instruction, which needs the list. Without the fix, the JVM's deadlock detector finds the IO thread blocked in getListofInstructions() and the timer blocked in writeInstruction(), the same stacks as the hardware hang. A second test connects five times with the read-status timer left on, as the product runs. Co-Authored-By: Claude Opus 5.5 --- .../bluetooth/ShimmerBluetooth.java | 9 +- ...PI_00033_InstructionListLockOrderTest.java | 111 ++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00033_InstructionListLockOrderTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index df8801f60..2d4988647 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -6009,11 +6009,18 @@ private synchronized void checkAndRemoveFirstInstructionIfNull(){ } } + /** + * Called by the IO thread for the next instruction. It locks only the list: + * everything else locks the device first and the list second, so taking the + * device here, through the synchronized getListofInstructions(), deadlocked + * the IO thread against the timers that queue and clear instructions + * (DEV-1266). + */ private byte[] getInstruction(){ synchronized(mListofInstructions){ if(!mListofInstructions.isEmpty()){ if(mListofInstructions.get(0)!=null) { - byte[] insBytes = (byte[]) getListofInstructions().get(0); + byte[] insBytes = (byte[]) mListofInstructions.get(0); return insBytes; } else{ return mListofInstructions.remove(0); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00033_InstructionListLockOrderTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00033_InstructionListLockOrderTest.java new file mode 100644 index 000000000..19a734f3c --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00033_InstructionListLockOrderTest.java @@ -0,0 +1,111 @@ +package com.shimmerresearch.bluetooth; + +import static org.junit.Assert.*; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; + +import org.junit.After; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.pcDriver.ShimmerPC; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationSimulatorS3; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; + +/** + * DEV-1266: ShimmerBluetooth guards its instruction list with two locks, the + * device (its synchronized methods) and the list itself. Every accessor took + * them device first, list second, but getInstruction(), which the IO thread + * calls for the next instruction, took the list and then called the + * synchronized getListofInstructions(). So the IO thread could hold the list + * while waiting for the device, as a timer (the ACK/response timeout calling + * clearAllInstructions(), or the read-status timer) held the device while + * waiting for the list. Both then waited for good, and so did every later API + * call. The Android HIL caught it on a Shimmer3 (stopSDLogging() never + * returned), and it was the Linux CI hang in DEV-1198. + */ +public class API_00033_InstructionListLockOrderTest { + + private static final long TIMEOUT_MS = 5000; + + /** Only for the connect test: tearDown must not touch a device whose lock a deadlock may still hold */ + private ShimmerPC mDevice; + + @After + public void tearDown() { + if(mDevice!=null){ + mDevice.disconnectNoException(); + } + } + + /** + * The two locks taken the way they collided: a timer holds the device, as + * every synchronized accessor does, while the IO thread asks for the next + * instruction; the timer then queues an instruction, which needs the list. + * The IO thread's real getInstruction() is called, as it is private. + */ + @Test + public void theIoThreadAndATimerTakeTheTwoLocksInOneOrder() throws Exception { + ShimmerPC device = new ShimmerPC("COM99"); + device.readBattery(); // getInstruction() reads the head of a non-empty list + Method getInstruction = ShimmerBluetooth.class.getDeclaredMethod("getInstruction"); + getInstruction.setAccessible(true); + + CountDownLatch timerHoldsTheDevice = new CountDownLatch(1); + Thread ioThread = daemon("IO thread", () -> { + try { + timerHoldsTheDevice.await(); + getInstruction.invoke(device); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Thread timer = daemon("timer", () -> { + synchronized(device){ + timerHoldsTheDevice.countDown(); + // Wait until the IO thread is either done or stuck waiting for the device + while(ioThread.getState()!=Thread.State.BLOCKED && ioThread.getState()!=Thread.State.TERMINATED){ + Thread.yield(); + } + device.readBattery(); // writeInstruction(): the device, then the list + } + }); + ioThread.start(); + timer.start(); + + ioThread.join(TIMEOUT_MS); + timer.join(TIMEOUT_MS); + long[] deadlocked = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); + assertNull("the IO thread and the timer deadlocked on the device and the instruction list:\n" + + ByteCommunicationTestUtils.dumpAllThreads(), deadlocked); + assertFalse("the IO thread finished", ioThread.isAlive()); + assertFalse("the timer finished", timer.isAlive()); + assertEquals("both instructions are queued", 2, device.getListofInstructions().size()); + } + + /** + * The product as it runs: the read-status timer is left on, so it queues + * GET_STATUS_COMMAND from its own thread while the IO thread works through the + * connect sequence. The simulator tests otherwise build their device without it + * (ByteCommunicationTestUtils.newShimmerPCWithoutStatusTimer), since DEV-1198. + */ + @Test + public void connectsWithTheStatusTimerOn() { + for(int i=1;i<=5;i++){ + mDevice = new ShimmerPC("COM99"); + mDevice.setTestRadio(new ByteCommunicationSimulatorS3("COM99")); + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + assertEquals("connect " + i, BT_STATE.CONNECTED, mDevice.getBluetoothRadioState()); + mDevice.disconnectNoException(); + } + mDevice = null; + } + + private Thread daemon(String name, Runnable runnable) { + Thread thread = new Thread(runnable, name); + thread.setDaemon(true); // a deadlocked pair must not keep the test JVM alive + return thread; + } +} From cb138a9b384b66fd7f78a4c2c58cfa6281ef4478 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 14:40:14 +0800 Subject: [PATCH 34/41] DEV-1269 Java HIL: queueStress, DEV-1266's deadlock on demand While a unit streams, four app threads call readBattery() and readStatusLogAndStream() and check the queue length, in tight loops for 60 s, keeping the queue under 8. Each of those calls locks the device and then the instruction list, so one is nearly always under way while the IO thread, holding the list, fetches the next instruction. That is the window DEV-1266's lock inversion needs. No SD logging is involved. The scenario fails if the calls and replies stop for 10 s, or if the JVM finds a deadlock. Either way the events get every thread's stack, and the unit is left alone. On a Shimmer3R (v1.1.21) without the DEV-1266 fix (#334), it deadlocked within 0.3 s in both runs. With the fix, two 60-second runs were clean, with about 670 calls and 0 timeouts each. It runs by default, so the release gate includes it. Until #334 merges, the gate fails on a deadlock. Co-Authored-By: Claude Opus 5.5 --- .../hil/JavaHilScenariosTest.java | 98 +++++++++++++++++++ tools/hil/README.md | 3 +- tools/hil/run-java-hil.ps1 | 4 +- 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index e85c98e7c..46f370f03 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -31,6 +31,10 @@ * hil.seconds streamRest duration, default 30 * hil.cycles midStreamSet / reconnectLoop / inStream repetitions, default 3 / 5 / 3 * hil.overlaps inStream overlap rounds, default 5 + * hil.threads queueStress app threads, default 4 + * hil.stressSeconds queueStress duration, default 60 + * hil.maxQueued queueStress queue length the threads keep below, default 8 + * hil.stallSeconds queueStress: no progress for this long is a hang, default 10 * hil.crc off, 1 or 2; default: the driver's own default * hil.maxLossPercent default 1.0 * hil.outDir where reports go, default build/hil @@ -468,4 +472,98 @@ private void expectReply(String round, String what, String replyType, HilSession report.check(session.waitFor(decoded, 1000), round + ": the " + what + " reply arrived but was not decoded"); } } + + /** + * DEV-1266 on hardware, with no SD logging. While streaming, app threads queue harmless reads + * (GET_VBATT_COMMAND and GET_STATUS_COMMAND) in tight loops, and check the queue length between + * them. Each of those calls locks the device and then the instruction list, so one is nearly + * always under way while the IO thread, holding the list, fetches the next instruction. That is + * the window the lock inversion needs. The queue is kept short so the unit is not flooded. + *

+ * Fails if the threads and the replies stop for hil.stallSeconds (default 10), or if the JVM + * finds a deadlock. The events then hold every thread's stack. Without the DEV-1266 fix (#334) a + * Shimmer3R deadlocked within 0.3 s in every run; with it, 60 s ran clean. + */ + @Test + public void queueStress() throws Exception { + begin("queue_stress"); + double rate = arg("rate", 256.0); + int threads = arg("threads", 4); + int seconds = arg("stressSeconds", 60); + int maxQueued = arg("maxQueued", 8); + int stallSeconds = arg("stallSeconds", 10); + if (connectAndConfigure(rate)) { + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + java.util.concurrent.atomic.AtomicLong calls = new java.util.concurrent.atomic.AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean stop = new java.util.concurrent.atomic.AtomicBoolean(); + List workers = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + Thread worker = new Thread(() -> { + while (!stop.get()) { + if (device.queuedInstructions() < maxQueued) { + if (calls.incrementAndGet() % 2 == 0) { + device.readBattery(); + } else { + device.readStatusLogAndStream(); + } + } else { + Thread.yield(); + } + } + }, "HilStress-" + t); + worker.setDaemon(true); + workers.add(worker); + } + session.event("queueStress: " + threads + " threads for up to " + seconds + " s, at most " + maxQueued + " queued"); + long start = System.nanoTime(); + workers.forEach(Thread::start); + long lastCalls = -1; + int lastReplies = -1; + long lastProgress = System.nanoTime(); + String hang = null; + while (System.nanoTime() - start < seconds * 1_000_000_000L) { + Thread.sleep(250); + long[] deadlocked = java.lang.management.ManagementFactory.getThreadMXBean().findDeadlockedThreads(); + if (deadlocked != null) { + hang = "the JVM found a deadlock"; + break; + } + long c = calls.get(); + int replies = device.statusesDecoded() + device.batteryReadings(); + if (c != lastCalls || replies != lastReplies) { + lastCalls = c; + lastReplies = replies; + lastProgress = System.nanoTime(); + } else if (System.nanoTime() - lastProgress > stallSeconds * 1_000_000_000L) { + hang = "no call returned and no reply arrived for " + stallSeconds + " s"; + break; + } + } + double ranSeconds = (System.nanoTime() - start) / 1e9; + stop.set(true); + for (Thread worker : workers) { + worker.join(5000); + } + report.put("stressSeconds", Math.round(ranSeconds * 10) / 10.0); + report.put("stressCalls", calls.get()); + report.put("stressReplies", device.statusesDecoded() + device.batteryReadings()); + if (hang == null && workers.stream().anyMatch(Thread::isAlive)) { + hang = "the stress threads did not stop"; + } + if (hang != null) { + session.hung = "queueStress"; + session.event("queueStress hung after " + String.format(Locale.ROOT, "%.1f", ranSeconds) + " s: " + hang + + "; every thread:\n" + com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils.dumpAllThreads()); + report.check(false, String.format(Locale.ROOT, "hung after %.1f s and %d calls: %s (thread dump in the events)", + ranSeconds, calls.get(), hang)); + } else { + session.event(String.format(Locale.ROOT, "queueStress: %d calls, %d replies decoded in %.1f s, no hang", + calls.get(), device.statusesDecoded() + device.batteryReadings(), ranSeconds)); + report.check(session.waitFor(() -> session.queuedInstructions() == 0, 10000), "the queue did not drain after the stress"); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } } diff --git a/tools/hil/README.md b/tools/hil/README.md index 83f19b3fa..4816ca393 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -43,6 +43,7 @@ If anything merges after step 1, the commit changes, so test again. In an emerge | `midStreamSet` | On a streaming unit, three times: starts SD logging **the way Consensys does** (stop streaming, start logging on the quiet link, stream again; DEV-436), stops it mid-stream, and reads the battery mid-stream. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop, or streaming doesn't come back; a packet is lost around a battery read, or more than `-MaxLossPercent` elsewhere; a timestamp fault in any stretch between restarts; or the driver logs a parser error or a command timeout. Packets missing around an SD logging stop are reported, not failed: the unit pauses while it closes its file. Needs an SD card in the unit. | | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times on the same `ShimmerPC`. Every connect starts the driver's status timer, which is when the DEV-1266 lock inversion is most likely. | Any cycle fails to connect within 30 s, loses packets, fails the timestamp checks, or the driver logs a parser error or command timeout. | | `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps as Consensys meets them: with the unit streaming and logging (logging started the Consensys way), stop SD logging mid-stream and request status and battery at once, so in-stream replies, including any status the unit pushes as logging stops, arrive while that command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, logging stopped, the queue empty); the driver logs a command timeout; or the stream fails the checks above. | +| `queueStress` | While streaming, four app threads call `readBattery()` and `readStatusLogAndStream()` and check the queue length, in tight loops for 60 s, keeping the queue under 8. Each of those calls locks the device and then the instruction list, so one is nearly always under way while the IO thread fetches the next instruction. That is the window DEV-1266's lock inversion needs (#334). No SD logging is involved. | The calls and replies stop for 10 s, or the JVM finds a deadlock; the queue doesn't drain afterwards. Without the DEV-1266 fix a Shimmer3R deadlocked within 0.3 s in every run. | Every API call runs under a watchdog. If one doesn't return within 10 s, the scenario stops, and its events hold every thread's stack with the locks each holds and waits on, plus any deadlock the JVM @@ -76,7 +77,7 @@ The stream checks are the Android HIL's (`StreamAnalysis`, the same class in Shi | | Default | Option | |---|---|---| | Units | every unit in the rig file | `-Unit s3r-86f8` | -| Scenarios | all four | `-Scenario inStream` | +| Scenarios | all five | `-Scenario inStream` | | Sampling rate, stream duration | 256 Hz, 30 s | `-Rate 512`, `-Seconds 60` | | Link CRC | the driver's default (1 byte) | `-Crc off`, `-Crc 2` | | Pause between runs | 10 s | `-SettleSeconds 30` | diff --git a/tools/hil/run-java-hil.ps1 b/tools/hil/run-java-hil.ps1 index 1e0278b37..047038bd2 100644 --- a/tools/hil/run-java-hil.ps1 +++ b/tools/hil/run-java-hil.ps1 @@ -22,8 +22,8 @@ Before "Release Java API": the full run on this commit, recorded on GitHub as th #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-java.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress'), [string[]]$Unit, [double]$Rate = 256, [int]$Seconds = 30, From eb2aa97591b1c60de46bb08b85afab1e9d01a75b Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 14:53:00 +0800 Subject: [PATCH 35/41] DEV-1269 Java HIL: report a sample a Shimmer3 skips for a battery reply A Shimmer3 (LogAndStream v1.1.7) sometimes gives up one sample slot to answer GET_VBATT mid-stream: packet n, the battery reply, then packet n+2. That was 4 skips in 33 battery reads. It's the unit, not the host: there were no parser errors, and builds with and without the driver fixes skip at the same rate. Filed for the firmware as DEV-1297. The check charged it as packet loss near a battery read, so midStreamSet failed on the Shimmer3 about one run in three. A gap near a battery read is now excused only when it is exactly one sample and the packet after it arrived within 25 ms of a decoded battery reply. Such a gap is reported, as skippedAtBatteryReplies plus an event naming DEV-1297. Anything else near a battery read still fails. A packet the driver dropped would leave stray bytes, which checkDriverLog() fails on anyway. HilShimmerPC now records when each battery reply is decoded. Checked on 3E36: 4 midStreamSet runs all passed. Two of them reported one skip at a battery reply, and there was no other loss near any read. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hil/HilShimmerPC.java | 11 ++++ .../hil/JavaHilScenariosTest.java | 56 ++++++++++++++++--- tools/hil/README.md | 2 +- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java index 19aeb2042..e18b6cdec 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java @@ -27,6 +27,8 @@ class HilShimmerPC extends ShimmerPC { private final List log = new ArrayList<>(); private volatile int statusesDecoded; private volatile int batteryReadings; + /** When each battery reply was decoded, in ms on the session's clock (HilSession.nowMs()). */ + private final List batteryReplyTimesMs = new ArrayList<>(); HilShimmerPC(String comPort, long t0Nanos) { super(comPort); @@ -50,9 +52,18 @@ protected void eventLogAndStreamStatusChanged(byte currentCommand) { @Override public void setBattStatusDetails(ShimmerBattStatusDetails s) { batteryReadings++; + synchronized (batteryReplyTimesMs) { + batteryReplyTimesMs.add((System.nanoTime() - t0) / 1e6); + } super.setBattStatusDetails(s); } + List batteryReplyTimesMs() { + synchronized (batteryReplyTimesMs) { + return new ArrayList<>(batteryReplyTimesMs); + } + } + @Override public void consolePrintLn(String message) { record(message); diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index 46f370f03..21c08539b 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -45,6 +45,11 @@ public class JavaHilScenariosTest { private static final double GRAVITY = 9.81; /** At 256 Hz a second without a packet is ~256 packets the gap count cannot see. */ private static final double MAX_SILENCE_AT_END_MS = 1000; + /** + * How close the packet after a one-sample gap must arrive to a decoded battery reply for the gap to + * count as the sample a Shimmer3 skips to send that reply (DEV-1297). Measured: 1-7 ms. + */ + private static final double BATTERY_REPLY_WINDOW_MS = 25; /** * The ACK/response timer's own line when a command gets no reply: "Command: [0x72] GET_STATUS_COMMAND * timeout", or "[0x..]UNKNOWN" for an opcode the driver doesn't know. @@ -184,9 +189,17 @@ private void checkTimeline(StreamAnalysis a, List samples /** * Judges a stream recorded in stretches, split where the test stopped and restarted streaming on - * purpose, so the pause is not taken for loss. Each stretch gets the timestamp checks; loss within - * 1 s of an SD logging stop is reported, not failed (the unit pauses while it closes its file); - * loss near a battery read, or more than maxLossPercent anywhere else, fails. + * purpose, so the pause is not taken for loss. Each stretch gets the timestamp checks. Each gap + * is charged, as StreamAnalysis.attribute() does, to the nearest SD logging stop or battery read + * within 1 s: + *

    + *
  • loss near an SD logging stop is reported, not failed (the unit pauses while it closes its + * file);
  • + *
  • a single sample skipped right where a battery reply was decoded is reported, not failed: a + * Shimmer3 gives up a sample slot to answer GET_VBATT mid-stream (DEV-1297). Had the driver lost + * that packet, it would have left bytes behind, which checkDriverLog() fails on;
  • + *
  • any other loss near a battery read, or more than maxLossPercent anywhere else, fails.
  • + *
*/ private void checkStretches(List> stretches, double stoppedAtMs, double deviceRateHz, double maxLossPercent, List sdStops, List batteryMoments) { @@ -194,6 +207,10 @@ private void checkStretches(List> stretches, double int missing = 0; int nearSd = 0; int nearBattery = 0; + int skippedAtBatteryReplies = 0; + double[] sd = toArray(sdStops); + double[] battery = toArray(batteryMoments); + double[] batteryReplies = toArray(device.batteryReplyTimesMs()); List all = new ArrayList<>(); List summaries = new ArrayList<>(); for (int k = 0; k < stretches.size(); k++) { @@ -204,11 +221,22 @@ private void checkStretches(List> stretches, double session.event(what + ": " + a); summaries.add(what + ": " + a); report.check(a.received > 0, what + ": no packets received"); - int[] attributed = a.attribute(new double[][] {toArray(sdStops), toArray(batteryMoments)}, 1000); received += a.received; missing += a.missing; - nearSd += attributed[0]; - nearBattery += attributed[1]; + for (StreamAnalysis.Gap g : a.gaps) { + double toSd = distanceToNearest(sd, g.afterReceivedMs); + double toBattery = distanceToNearest(battery, g.afterReceivedMs); + if (Math.min(toSd, toBattery) > 1000) { + continue; + } + if (toSd <= toBattery) { + nearSd += g.missing; + } else if (g.missing == 1 && distanceToNearest(batteryReplies, g.afterReceivedMs) <= BATTERY_REPLY_WINDOW_MS) { + skippedAtBatteryReplies++; + } else { + nearBattery += g.missing; + } + } report.check(a.nonIncreasing == 0, what + ": " + a.nonIncreasing + " timestamps did not increase"); checkJumps(a, what); // The rate needs a second or so of stream; only the last stretch can go silent before a stop @@ -223,12 +251,17 @@ private void checkStretches(List> stretches, double report.put("missing", missing); report.put("missingNearSdLoggingStops", nearSd); report.put("missingNearBatteryReads", nearBattery); + report.put("skippedAtBatteryReplies", skippedAtBatteryReplies); report.samples(all); if (nearSd > 0) { session.event(nearSd + " packets not received within 1 s of an SD logging stop (firmware side; not failed)"); } + if (skippedAtBatteryReplies > 0) { + session.event(skippedAtBatteryReplies + " single samples skipped where the unit sent a battery reply" + + " (firmware side, DEV-1297; not failed)"); + } report.check(nearBattery == 0, nearBattery + " packets lost within 1 s of a battery read while streaming"); - int elsewhere = missing - nearSd - nearBattery; + int elsewhere = missing - nearSd - nearBattery - skippedAtBatteryReplies; double elsewherePercent = 100.0 * elsewhere / Math.max(1, received + missing); report.put("missingElsewhere", elsewhere); report.check(elsewherePercent <= maxLossPercent, String.format(Locale.ROOT, @@ -252,6 +285,15 @@ private static double[] toArray(List values) { return out; } + /** The distance from t to the nearest of the moments; infinite when there are none. */ + private static double distanceToNearest(double[] momentsMs, double t) { + double nearest = Double.POSITIVE_INFINITY; + for (double m : momentsMs) { + nearest = Math.min(nearest, Math.abs(t - m)); + } + return nearest; + } + // --- scenarios ----------------------------------------------------------------------------- /** Connect, configure, stream at rest, stop, disconnect. */ diff --git a/tools/hil/README.md b/tools/hil/README.md index 4816ca393..5edd52517 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -40,7 +40,7 @@ If anything merges after step 1, the commit changes, so test again. In an emerge | Scenario | What it does | Fails when | |---|---|---| | `streamRest` | Connects, writes an accel-only config at `-Rate` (default 256 Hz) the way Consensys's Bluetooth manager does, streams for `-Seconds` (default 30) with the unit lying still, then stops and disconnects. | It can't connect or configure; more than `-MaxLossPercent` (1 %) of packets are lost; a timestamp doesn't increase, or leaps more than a second ahead of when it arrived (a bad unwrap, not loss); the timestamps give no rate, or one more than 1 % off the configured rate; the stream goes silent for over a second before the test stops it; accel is more than 1 m/s² from 9.81; or the driver logs a parser error or a command timeout. | -| `midStreamSet` | On a streaming unit, three times: starts SD logging **the way Consensys does** (stop streaming, start logging on the quiet link, stream again; DEV-436), stops it mid-stream, and reads the battery mid-stream. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop, or streaming doesn't come back; a packet is lost around a battery read, or more than `-MaxLossPercent` elsewhere; a timestamp fault in any stretch between restarts; or the driver logs a parser error or a command timeout. Packets missing around an SD logging stop are reported, not failed: the unit pauses while it closes its file. Needs an SD card in the unit. | +| `midStreamSet` | On a streaming unit, three times: starts SD logging **the way Consensys does** (stop streaming, start logging on the quiet link, stream again; DEV-436), stops it mid-stream, and reads the battery mid-stream. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop, or streaming doesn't come back; a packet is lost around a battery read, or more than `-MaxLossPercent` elsewhere (a single sample skipped right where the battery reply arrives is reported, not failed: a Shimmer3 gives up a sample slot to answer a battery read, DEV-1297); a timestamp fault in any stretch between restarts; or the driver logs a parser error or a command timeout. Packets missing around an SD logging stop are reported, not failed: the unit pauses while it closes its file. Needs an SD card in the unit. | | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times on the same `ShimmerPC`. Every connect starts the driver's status timer, which is when the DEV-1266 lock inversion is most likely. | Any cycle fails to connect within 30 s, loses packets, fails the timestamp checks, or the driver logs a parser error or command timeout. | | `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps as Consensys meets them: with the unit streaming and logging (logging started the Consensys way), stop SD logging mid-stream and request status and battery at once, so in-stream replies, including any status the unit pushes as logging stops, arrive while that command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, logging stopped, the queue empty); the driver logs a command timeout; or the stream fails the checks above. | | `queueStress` | While streaming, four app threads call `readBattery()` and `readStatusLogAndStream()` and check the queue length, in tight loops for 60 s, keeping the queue under 8. Each of those calls locks the device and then the instruction list, so one is nearly always under way while the IO thread fetches the next instruction. That is the window DEV-1266's lock inversion needs (#334). No SD logging is involved. | The calls and replies stop for 10 s, or the JVM finds a deadlock; the queue doesn't drain afterwards. Without the DEV-1266 fix a Shimmer3R deadlocked within 0.3 s in every run. | From feee0420f5a7a9e656786339af3f333627c1d8a7 Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 15:02:31 +0800 Subject: [PATCH 36/41] DEV-1269 Java HIL: check that every connect reports ready only at the end DEV-1275 was only caught indirectly: a stream lost in reconnectLoop, or NaN accel when a configure cut the calibration dump short. The HIL now checks it directly. For each connect it notes whether the first CONNECTED state came before NOTIFICATION_SHIMMER_FULLY_INITIALIZED, which the driver sends at the end of its connect sequence. The report counts these as connectedBeforeReady, and any one fails the scenario. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hil/HilSession.java | 24 +++++++++++++++++++ .../hil/JavaHilScenariosTest.java | 3 +++ tools/hil/README.md | 2 ++ 3 files changed, 29 insertions(+) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java index b9a9b896b..3fc9c48a9 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java @@ -67,6 +67,17 @@ interface Condition { volatile String connectFailure; /** jssc's message when Windows refuses to open a COM port that another program already has open. */ private static final Pattern PORT_BUSY = Pattern.compile("Port busy"); + /** + * Per connect: whether the driver has sent NOTIFICATION_SHIMMER_FULLY_INITIALIZED (the end of + * its connect sequence), and whether the first CONNECTED state is still to come. + */ + private volatile boolean fullyInitialisedSinceConnect; + private volatile boolean awaitingFirstConnected; + /** + * Connects whose first CONNECTED came before the end of the connect sequence: the app was told + * the unit was ready while the sequence was still queued (DEV-1275). + */ + volatile int connectedBeforeReady; HilSession(String port, String mac, String type) { this.port = port; @@ -102,7 +113,18 @@ protected void processMsgFromCallback(ShimmerMsg msg) { lastState = state; stateChanges++; event("state " + state); + if (state == BT_STATE.CONNECTED && awaitingFirstConnected) { + awaitingFirstConnected = false; + if (!fullyInitialisedSinceConnect) { + connectedBeforeReady++; + event("CONNECTED reported before the connect sequence ended (no FULLY_INITIALIZED yet; DEV-1275)"); + } + } } + } else if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_NOTIFICATION_MESSAGE && msg.mB instanceof CallbackObject + && ((CallbackObject) msg.mB).mIndicator == HilShimmerPC.NOTIFICATION_SHIMMER_FULLY_INITIALIZED) { + fullyInitialisedSinceConnect = true; + event("fully initialised"); } else if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_DATA_PACKET && msg.mB instanceof ObjectCluster) { if (recording) { recordPacket((ObjectCluster) msg.mB); @@ -186,6 +208,8 @@ boolean call(String what, long timeoutMs, Call call) { boolean connect(long timeoutMs) throws InterruptedException { connects++; connectFailure = null; + fullyInitialisedSinceConnect = false; + awaitingFirstConnected = true; int mark = stateChanges; int logMark = device.logSize(); Condition failed = () -> stateChanges > mark && device.getBluetoothRadioState() == BT_STATE.CONNECTION_FAILED; diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index 21c08539b..b075df706 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -104,6 +104,8 @@ public void end() { } private void finish() { + report.check(session.connectedBeforeReady == 0, session.connectedBeforeReady + " of " + session.connects + + " connects reported CONNECTED before the connect sequence ended (DEV-1275)"); if (session.hung != null) { report.put("hungCall", session.hung); report.check(false, session.hung + " never returned: the API call blocked (thread dump in the report's events)"); @@ -116,6 +118,7 @@ private void finish() { /** What the driver did in this session: replies by type, decodes, timer requests, connects. */ private void putDriverCounts() { report.put("connects", session.connects); + report.put("connectedBeforeReady", session.connectedBeforeReady); report.put("inStreamReplies", device.inStreamReplies()); report.put("statusesDecoded", device.statusesDecoded()); report.put("batteryReadings", device.batteryReadings()); diff --git a/tools/hil/README.md b/tools/hil/README.md index 5edd52517..0cacbd5c9 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -56,6 +56,8 @@ A connect is never retried. If the COM port can't be opened, the scenario fails - **CONNECTION_FAILED** for any other reason: the unit is off, out of range, or connected to another host such as a phone. +Every connect, in every scenario, is also checked for DEV-1275: the first CONNECTED must come only after the driver's NOTIFICATION_SHIMMER_FULLY_INITIALIZED, which marks the end of its connect sequence. `connectedBeforeReady` counts the connects that broke this, and any one fails the scenario. + Each report counts what the session covered: connects, in-stream replies by type, statuses and battery readings decoded, the status timer's requests, and command timeouts. Its events also say which driver timers were running after connect, after configure and while streaming. From 3db1ce0c39ec0d3de373476338eef0d86f1a0bfa Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 17:44:04 +0800 Subject: [PATCH 37/41] DEV-1305: Java HIL rollover, idleStart and samplingRates scenarios Three checks the C# and shimmer-logandstream HILs already run on the same bench: - rollover: streams 10 minutes, past the 512 s wrap of the unit's 3-byte timestamp counter, and fails on streamRest's checks or if the raw counter never wrapped. TimestampWraps counts the wraps from the TIMESTAMP channel's raw ticks, which HilSession now records with each sample. - idleStart: leaves the link idle 60 s with the driver's timers on, then fails if the link was lost or a command timed out during the wait, if streaming starts more than 500 ms after startStreaming(), or on the stream checks. - samplingRates: writes 51.2, 256 and 512 Hz in turn the Consensys way, streams 5 s at each and checks the rate, then writes the unit's own rate back. run-java-hil.ps1 runs all eight scenarios by default, so -Report requires the new ones too. TimestampWrapsTest runs in CI. No driver code changes. Co-Authored-By: Claude Opus 5.5 --- .../com/shimmerresearch/hil/HilSession.java | 21 +- .../hil/JavaHilScenariosTest.java | 202 +++++++++++++++++- .../shimmerresearch/hil/TimestampWraps.java | 39 ++++ .../hil/TimestampWrapsTest.java | 48 +++++ tools/hil/README.md | 10 +- tools/hil/run-java-hil.ps1 | 4 +- 6 files changed, 310 insertions(+), 14 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWraps.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWrapsTest.java diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java index 3fc9c48a9..440d4d8e4 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java @@ -49,6 +49,8 @@ interface Condition { private final long t0 = System.nanoTime(); private final List events = new ArrayList<>(); private final List samples = new ArrayList<>(); + /** Each recorded packet's raw device ticks (TIMESTAMP, UNCAL), since startRecording, across stretches. */ + private final List rawTicks = new ArrayList<>(); private final Set otherMessagesSeen = new HashSet<>(); private final ExecutorService calls = Executors.newSingleThreadExecutor(r -> { Thread t = new Thread(r, "HilCall"); @@ -140,14 +142,20 @@ private void recordPacket(ObjectCluster oc) { cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_X), cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Y), cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Z)); + double ticks = value(oc, Configuration.Shimmer3.ObjectClusterSensorName.TIMESTAMP, "UNCAL"); synchronized (samples) { samples.add(s); + rawTicks.add(ticks); } } private static double cal(ObjectCluster oc, String channel) { + return value(oc, channel, "CAL"); + } + + private static double value(ObjectCluster oc, String channel, String format) { Collection formats = oc.getCollectionOfFormatClusters(channel); - FormatCluster c = (FormatCluster) ObjectCluster.returnFormatCluster(formats, "CAL"); + FormatCluster c = (FormatCluster) ObjectCluster.returnFormatCluster(formats, format); return c == null ? Double.NaN : c.mData; } @@ -278,10 +286,21 @@ boolean configureAccelOnly(double rateHz, long timeoutMs) throws InterruptedExce void startRecording() { synchronized (samples) { samples.clear(); + rawTicks.clear(); } recording = true; } + /** + * The raw device ticks of every packet recorded since startRecording, in arrival order: the + * unit's own 3-byte counter, before the driver unwraps it (TimestampWraps counts its wraps). + */ + List rawTicksRecorded() { + synchronized (samples) { + return new ArrayList<>(rawTicks); + } + } + List stopRecording() { recording = false; synchronized (samples) { diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java index b075df706..453c92756 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -35,6 +35,8 @@ * hil.stressSeconds queueStress duration, default 60 * hil.maxQueued queueStress queue length the threads keep below, default 8 * hil.stallSeconds queueStress: no progress for this long is a hang, default 10 + * hil.rolloverSeconds rollover duration, default 600 (past the 512 s wrap from any start) + * hil.idleSeconds idleStart: how long the link is left idle, default 60 * hil.crc off, 1 or 2; default: the driver's own default * hil.maxLossPercent default 1.0 * hil.outDir where reports go, default build/hil @@ -55,6 +57,10 @@ public class JavaHilScenariosTest { * timeout", or "[0x..]UNKNOWN" for an opcode the driver doesn't know. */ private static final java.util.regex.Pattern COMMAND_TIMEOUT = java.util.regex.Pattern.compile("Command:\t\\[0x[0-9A-F]{2}\\] ?\\S+ timeout"); + /** The slowest start idleStart accepts after the wait, as the C# and shimmer-logandstream HILs do. */ + private static final double MAX_START_AFTER_IDLE_MS = 500; + /** The rates samplingRates writes in turn, as the shimmer-logandstream HIL's config.sampling_rate does. */ + private static final double[] SAMPLING_RATES_HZ = {51.2, 256, 512}; private HilSession session; private HilReport report; @@ -129,6 +135,17 @@ private void putDriverCounts() { // --- shared checks ------------------------------------------------------------------------- private boolean connectAndConfigure(double rateHz) throws Exception { + if (!connectAndDescribe()) { + return false; + } + boolean configured = session.configureAccelOnly(rateHz, 20000); + report.put("rateOnDeviceHz", device.getSamplingRateShimmer()); + report.check(configured, "configuration did not complete within 20 s (state " + session.state() + ")"); + return configured; + } + + /** Connects, and records what the unit is: its hardware, firmware and the link CRC in use. */ + private boolean connectAndDescribe() throws Exception { long start = System.currentTimeMillis(); boolean connected = session.connect(30000); report.put("connectMs", System.currentTimeMillis() - start); @@ -139,10 +156,20 @@ private boolean connectAndConfigure(double rateHz) throws Exception { report.put("hardware", device.getHardwareVersionParsed()); report.put("firmware", device.getFirmwareVersionParsed()); report.put("crcMode", String.valueOf(device.getCurrentBtCommsCrcMode())); - boolean configured = session.configureAccelOnly(rateHz, 20000); - report.put("rateOnDeviceHz", device.getSamplingRateShimmer()); - report.check(configured, "configuration did not complete within 20 s (state " + session.state() + ")"); - return configured; + return true; + } + + /** The driver says the link has gone: lost, or closed under the test. */ + private boolean linkLost() { + BT_STATE s = session.state(); + return s == BT_STATE.CONNECTION_LOST || s == BT_STATE.DISCONNECTED; + } + + /** Accel at rest: NaN fails too, since the scenario configured the accel. */ + private void checkAccelAtRest(StreamAnalysis a) { + report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= 1.0, String.format(Locale.ROOT, + "accel at rest %.3f m/s^2, expected %.2f±1.00 (NaN: no calibrated accel; otherwise is the unit lying still?)", + a.accelMagnitudeMean, GRAVITY)); } private void checkRate(double deviceRateHz, String what) { @@ -322,10 +349,7 @@ public void streamRest() throws Exception { report.putGaps(a); report.samples(samples); checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); - // NaN fails too: the scenario configured the accel, so a stream without calibrated accel is a fault - report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= 1.0, String.format(Locale.ROOT, - "accel at rest %.3f m/s^2, expected %.2f±1.00 (NaN: no calibrated accel; otherwise is the unit lying still?)", - a.accelMagnitudeMean, GRAVITY)); + checkAccelAtRest(a); checkDriverLog(logMark, "stream"); report.check(session.disconnect(10000), "did not disconnect within 10 s"); } @@ -611,4 +635,166 @@ public void queueStress() throws Exception { } finish(); } + + /** + * Streams for 10 minutes (hil.rolloverSeconds), past the 512 s wrap of the unit's 3-byte + * timestamp counter from wherever it starts, as the C# HIL's stream.rollover does. The driver's + * unwrap must carry the timeline straight across the wrap: DEV-1023 was that rule, wrong the same + * way in four APIs. Fails on the stream checks, or if the raw counter never wrapped. + */ + @Test + public void rollover() throws Exception { + begin("rollover"); + double rate = arg("rate", 256.0); + int seconds = arg("rolloverSeconds", 600); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + // Watched each second, so a lost link ends the wait instead of sitting out the 10 minutes + long start = System.nanoTime(); + while (System.nanoTime() - start < seconds * 1_000_000_000L && !linkLost()) { + Thread.sleep(1000); + } + double streamedSeconds = (System.nanoTime() - start) / 1e9; + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + int wraps = TimestampWraps.count(session.rawTicksRecorded()); + report.put("streamedSeconds", Math.round(streamedSeconds * 10) / 10.0); + report.put("counterWraps", wraps); + boolean lost = linkLost(); + report.check(!lost, String.format(Locale.ROOT, "the link was lost %.0f s into the stream (state %s)", streamedSeconds, session.state())); + if (!lost) { + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + } + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a + "; the unit's counter wrapped " + wraps + " time(s)"); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); + report.check(wraps >= 1, String.format(Locale.ROOT, + "the unit's counter did not wrap in %.0f s of streaming, so the stream never crossed the 512 s wrap", streamedSeconds)); + checkAccelAtRest(a); + checkDriverLog(logMark, "stream"); + if (!lost) { + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + + /** + * A link left idle, as an app's is between configuring a unit and starting it: 60 s + * (hil.idleSeconds) in which the test sends nothing, with the driver's timers running as they do + * in an app, then a 10 s stream. The link must survive the wait, the status timer's requests + * must be answered, and streaming must start within 500 ms of the call, as the C# and + * shimmer-logandstream HILs require: a Classic link left quiet otherwise answers its next command + * late. + */ + @Test + public void idleStart() throws Exception { + begin("idle_start"); + double rate = arg("rate", 256.0); + int idleSeconds = arg("idleSeconds", 60); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int idleMark = device.logSize(); + session.event("idle for " + idleSeconds + " s: timers " + device.timers()); + long idleStarted = System.nanoTime(); + while (System.nanoTime() - idleStarted < idleSeconds * 1_000_000_000L && !linkLost()) { + Thread.sleep(200); + } + double idled = (System.nanoTime() - idleStarted) / 1e9; + int statusRequests = device.countSince(idleMark, "GET_STATUS_COMMAND Instruction added"); + report.put("idleSeconds", Math.round(idled * 10) / 10.0); + report.put("statusRequestsWhileIdle", statusRequests); + session.event(String.format(Locale.ROOT, "idle %.0f s: %d status requests by the driver, timers %s", idled, statusRequests, device.timers())); + boolean lost = linkLost(); + report.check(!lost, String.format(Locale.ROOT, "the link was lost %.0f s into the idle wait (state %s)", idled, session.state())); + if (!lost) { + checkDriverLog(idleMark, "idle"); + int logMark = device.logSize(); + session.startRecording(); + long asked = System.nanoTime(); + boolean started = session.startStreaming(10000); + double startMs = (System.nanoTime() - asked) / 1e6; + report.put("startMs", Math.round(startMs)); + session.event(String.format(Locale.ROOT, "streaming %s %.0f ms after startStreaming", started ? "started" : "had not started", startMs)); + report.check(started, "streaming did not start within 10 s after the idle wait"); + report.check(!started || startMs <= MAX_START_AFTER_IDLE_MS, String.format(Locale.ROOT, + "streaming started %.0f ms after startStreaming, over %.0f ms", startMs, MAX_START_AFTER_IDLE_MS)); + Thread.sleep(10000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); + checkAccelAtRest(a); + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + + /** + * Writes 51.2, 256 and 512 Hz in turn, each the way Consensys's Bluetooth manager writes a + * configuration (configureAccelOnly), and streams 5 s at each: the unit must stream at the rate + * it was given, as the shimmer-logandstream HIL's config.sampling_rate checks. Then the unit's own + * rate is written back. The unit is left accel only, as every scenario leaves it. + */ + @Test + public void samplingRates() throws Exception { + begin("sampling_rates"); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndDescribe()) { + double ownRate = device.getSamplingRateShimmer(); + report.put("ownRateHz", ownRate); + List streams = new ArrayList<>(); + for (double rate : SAMPLING_RATES_HZ) { + if (session.hung != null) { + break; + } + String what = String.format(Locale.ROOT, "%.1f Hz", rate); + boolean configured = session.configureAccelOnly(rate, 20000); + report.check(configured, String.format(Locale.ROOT, "%s: the configuration did not take within 20 s (rate on device %.2f Hz, state %s)", + what, device.getSamplingRateShimmer(), session.state())); + if (!configured) { + continue; + } + double deviceRate = device.getSamplingRateShimmer(); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + streams.add(what + ": " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + checkDriverLog(logMark, what); + } + report.put("streams", streams); + if (session.hung == null) { + boolean restored = session.configureAccelOnly(ownRate, 20000); + report.check(restored, String.format(Locale.ROOT, "the unit's own rate, %.2f Hz, was not written back (rate on device %.2f Hz)", + ownRate, device.getSamplingRateShimmer())); + report.put("rateOnDeviceHz", device.getSamplingRateShimmer()); + } + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } } diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWraps.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWraps.java new file mode 100644 index 000000000..06493014a --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWraps.java @@ -0,0 +1,39 @@ +package com.shimmerresearch.hil; + +import java.util.List; + +/** + * Counts how often the unit's own timestamp counter wrapped during a stream, from the raw ticks the + * driver passes on over Bluetooth (the TIMESTAMP channel's UNCAL format; CAL is the unwrapped + * time). The counter is 3 bytes at 32768 Hz on Shimmer3 and Shimmer3R, so it wraps every 2^24 + * ticks, 512 s. Pure Java, so it is unit-tested on the JVM (TimestampWrapsTest, in the normal test + * task). Kept apart from StreamAnalysis, which is the same class as the Android HIL's. + */ +final class TimestampWraps { + + /** The raw counter's range: 3 bytes. */ + static final double RANGE_TICKS = 1 << 24; + + private TimestampWraps() { + } + + /** + * Steps where the raw counter fell by more than half its range: a wrap. A smaller step back is + * not a wrap but a fault the stream checks already count (StreamAnalysis.nonIncreasing). A packet + * without ticks (NaN) is skipped, so the step is taken from the last one that had them. + */ + static int count(List rawTicks) { + int wraps = 0; + double previous = Double.NaN; + for (double ticks : rawTicks) { + if (Double.isNaN(ticks)) { + continue; + } + if (previous - ticks > RANGE_TICKS / 2) { + wraps++; + } + previous = ticks; + } + return wraps; + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWrapsTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWrapsTest.java new file mode 100644 index 000000000..dc4c28320 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/TimestampWrapsTest.java @@ -0,0 +1,48 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +/** The rollover scenario's wrap count, checked without a unit. */ +public class TimestampWrapsTest { + + /** Raw 3-byte ticks at 256 Hz (128 ticks a sample) from startTicks, as the counter gives them. */ + private static List rawTicks(double startTicks, int count) { + List ticks = new ArrayList<>(); + for (int i = 0; i < count; i++) { + ticks.add((startTicks + i * 128.0) % TimestampWraps.RANGE_TICKS); + } + return ticks; + } + + @Test + public void countsNoWrapInAShortStream() { + assertEquals(0, TimestampWraps.count(rawTicks(1000, 2560))); + } + + @Test + public void countsTheWrapInTenMinutesWhereverTheStreamStarts() { + // 600 s at 256 Hz is 153,600 samples: one wrap from any start, two from one just before a wrap + assertEquals(1, TimestampWraps.count(rawTicks(0, 153600))); + assertEquals(1, TimestampWraps.count(rawTicks(TimestampWraps.RANGE_TICKS / 2, 153600))); + assertEquals(2, TimestampWraps.count(rawTicks(TimestampWraps.RANGE_TICKS - 1000, 153600))); + } + + @Test + public void aSmallStepBackIsNotAWrap() { + List ticks = rawTicks(5000000, 100); + ticks.set(50, ticks.get(49) - 1000); + assertEquals(0, TimestampWraps.count(ticks)); + } + + @Test + public void ignoresTicksTheDriverDidNotGive() { + List ticks = rawTicks(TimestampWraps.RANGE_TICKS - 640, 10); + ticks.set(4, Double.NaN); + assertEquals(1, TimestampWraps.count(ticks)); + } +} diff --git a/tools/hil/README.md b/tools/hil/README.md index 0cacbd5c9..d73eb49a5 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -44,6 +44,9 @@ If anything merges after step 1, the commit changes, so test again. In an emerge | `reconnectLoop` | Connects, streams for 5 s and disconnects, five times on the same `ShimmerPC`. Every connect starts the driver's status timer, which is when the DEV-1266 lock inversion is most likely. | Any cycle fails to connect within 30 s, loses packets, fails the timestamp checks, or the driver logs a parser error or command timeout. | | `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps as Consensys meets them: with the unit streaming and logging (logging started the Consensys way), stop SD logging mid-stream and request status and battery at once, so in-stream replies, including any status the unit pushes as logging stops, arrive while that command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, logging stopped, the queue empty); the driver logs a command timeout; or the stream fails the checks above. | | `queueStress` | While streaming, four app threads call `readBattery()` and `readStatusLogAndStream()` and check the queue length, in tight loops for 60 s, keeping the queue under 8. Each of those calls locks the device and then the instruction list, so one is nearly always under way while the IO thread fetches the next instruction. That is the window DEV-1266's lock inversion needs (#334). No SD logging is involved. | The calls and replies stop for 10 s, or the JVM finds a deadlock; the queue doesn't drain afterwards. Without the DEV-1266 fix a Shimmer3R deadlocked within 0.3 s in every run. | +| `rollover` | Configures as `streamRest` does and streams for 10 minutes with the unit lying still. That is past the 512 s wrap of the unit's 3-byte timestamp counter from wherever it starts, so the driver's unwrap has to carry the timeline across it (DEV-1023, the same rule wrong in four APIs). The C# HIL's `stream.rollover` does the same. | The stream fails `streamRest`'s checks, or the link is lost; the unit's raw counter never wrapped (counted from the TIMESTAMP channel's raw ticks, so a shortened run doesn't pass). | +| `idleStart` | Connects and configures, then leaves the link idle for 60 s, with the driver's timers running as they do in an app, before streaming 10 s. The C# and shimmer-logandstream HILs check the same: a Bluetooth Classic link left quiet otherwise answers its next command late. The report counts the status timer's requests during the wait. | The link is lost during the wait, or the driver logs a command timeout or parser error in it; streaming starts more than 500 ms after `startStreaming()`; or the stream fails `streamRest`'s checks. | +| `samplingRates` | Writes 51.2, 256 and 512 Hz in turn, each the way Consensys's Bluetooth manager writes a configuration, and streams 5 s at each, as the shimmer-logandstream HIL's `config.sampling_rate` does. Then it writes the unit's own rate back, accel only as every scenario leaves it. | A configuration doesn't take within 20 s, or the unit reads back another rate; a stream loses packets, fails the timestamp checks, or has a rate more than 1 % off the one written; the driver logs a parser error or command timeout; or the unit's own rate isn't written back. | Every API call runs under a watchdog. If one doesn't return within 10 s, the scenario stops, and its events hold every thread's stack with the locks each holds and waits on, plus any deadlock the JVM @@ -65,7 +68,8 @@ which driver timers were running after connect, after configure and while stream Starting SD logging mid-stream (`startSDLogging()` while streaming) writes the RTC mid-stream, which a Shimmer3R answers with out-of-order timestamps (DEV-1245). Consensys never does that, and neither do these scenarios. The stream checks are the Android HIL's (`StreamAnalysis`, the same class in ShimmerAndroidAPI's -`shimmerHilTest`), so the two judge a stream by the same rules. `StreamAnalysisTest` runs in CI. +`shimmerHilTest`), so the two judge a stream by the same rules. `StreamAnalysisTest` runs in CI, as +does `TimestampWrapsTest`, for the wrap count `rollover` relies on. ### What it does not test @@ -79,7 +83,7 @@ The stream checks are the Android HIL's (`StreamAnalysis`, the same class in Shi | | Default | Option | |---|---|---| | Units | every unit in the rig file | `-Unit s3r-86f8` | -| Scenarios | all five | `-Scenario inStream` | +| Scenarios | all eight (`rollover` alone streams 10 minutes a unit) | `-Scenario inStream` | | Sampling rate, stream duration | 256 Hz, 30 s | `-Rate 512`, `-Seconds 60` | | Link CRC | the driver's default (1 byte) | `-Crc off`, `-Crc 2` | | Pause between runs | 10 s | `-SettleSeconds 30` | @@ -90,7 +94,7 @@ The stream checks are the Android HIL's (`StreamAnalysis`, the same class in Shi 1. **Pair each unit with the PC** in Windows Settings → Bluetooth (PIN `1234`). Windows creates an outgoing serial port for it; the script finds that port from the unit's MAC. 2. **Prepare each unit:** SD sync and Bluetooth disable off (either keeps Bluetooth off, DEV-1228), an - SD card in for `midStreamSet`, lying still during `streamRest`. + SD card in for `midStreamSet`, lying still during `streamRest`, `rollover` and `idleStart`. 3. **List the units:** copy `rig-java.example.json` to `rig-java.json` and fill in each name, MAC and `type` (`Shimmer3` or `Shimmer3R`). Add `comPort` only to override the port the script finds. That file is gitignored. diff --git a/tools/hil/run-java-hil.ps1 b/tools/hil/run-java-hil.ps1 index 047038bd2..f2986fb88 100644 --- a/tools/hil/run-java-hil.ps1 +++ b/tools/hil/run-java-hil.ps1 @@ -22,8 +22,8 @@ Before "Release Java API": the full run on this commit, recorded on GitHub as th #> param( [string]$Rig = (Join-Path $PSScriptRoot 'rig-java.json'), - [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress')] - [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress', 'rollover', 'idleStart', 'samplingRates')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress', 'rollover', 'idleStart', 'samplingRates'), [string[]]$Unit, [double]$Rate = 256, [int]$Seconds = 30, From a0a7b5a444d1145e51ed8770e86c24ae5d10649b Mon Sep 17 00:00:00 2001 From: JongChern Date: Thu, 22 Oct 2026 20:28:01 +0800 Subject: [PATCH 38/41] DEV-1306: a stop's reply no longer retires the wrong instruction Every reply path retires the command in flight with removeInstruction(0), taking the head of the instruction list to be the command answered. Two things broke that around a stop: - operationPrepare() cleared the list while the STOP was still in flight. The stop handling, which sleeps 400 ms on the first byte after the STOP, then removed the head in the STOP's place: the first SET_INFOMEM of a configuration started straight after stopStreaming(), the chunk that holds the sampling rate. The unit kept its old rate. operationPrepare() now waits, for up to 3 s, until a command already in flight has been answered and retired, then clears the list. mTransactionCompleted is volatile, so the waiting thread sees it. - The stop handling then let the same byte go on to the ACK processing. When that byte was the unit's ACK, it retired the next instruction as well, a command queued after the stop that was never sent. The stop handling now returns once the stop is handled, and logs when it was answered by its ACK. API_00031_ConfigureStraightAfterStopTest runs the IOThread over an in-memory radio. Without the fix the stop-window configuration sends only chunks 0x0080 and 0x0100, and the command queued behind an ACK-first stop is never sent; with it both pass, as does a configuration after the stop has finished. Co-Authored-By: Claude Opus 5.5 --- .../bluetooth/ShimmerBluetooth.java | 62 ++++- ..._00031_ConfigureStraightAfterStopTest.java | 225 ++++++++++++++++++ 2 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00031_ConfigureStraightAfterStopTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index b02a5339d..00998a855 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -186,7 +186,7 @@ public String toString() { protected byte mCurrentCommand; protected boolean mWaitForAck=false; // This indicates whether the device is waiting for an acknowledge packet from the Shimmer Device protected boolean mWaitForResponse=false; // This indicates whether the device is waiting for a response packet from the Shimmer Device - protected boolean mTransactionCompleted=true; // Variable is used to ensure a command has finished execution prior to executing the next command (see initialize()) + protected volatile boolean mTransactionCompleted=true; // Variable is used to ensure a command has finished execution prior to executing the next command (see initialize()). Volatile: operationPrepare() waits on it from an app thread transient protected IOThread mIOThread; transient protected ProcessingThread mPThread; @Deprecated // mContinousSync doesn't do anything @@ -858,13 +858,21 @@ private void processNotStreamingWaitForAck() { clearSerialBuffer(); - hasStopStreaming(); + hasStopStreaming(); removeInstruction(0); removeAllNulls(); if (mCurrentCommand==STOP_SDBT_COMMAND){ - eventLogAndStreamStatusChanged(mCurrentCommand); + eventLogAndStreamStatusChanged(mCurrentCommand); } setInstructionStackLock(false); + // The stop is handled and the STOP retired. The byte that started the handling + // must not also go on to the ACK processing below: when it was the unit's ACK, + // that retired the next instruction as well, a command queued straight after + // stopStreaming() that was never sent (DEV-1306) + if((byte)byteBuffer[0]==ACK_COMMAND_PROCESSED){ + printLogDataForDebugging("Stop handled on its ACK, the first byte back"); + } + return; } // //TODO: ACK is probably now working for STOP_STREAMING_COMMAND so merge in with others? // if(mCurrentCommand==STOP_SDBT_COMMAND) { //due to not receiving the ack from stop streaming command we will skip looking for it. @@ -2989,6 +2997,14 @@ public void setSendProgressReport(boolean state) { public void operationPrepare(){ stopAllTimers(); + // Let a command already sent finish first. Clearing the list now would remove it while it is + // still in flight, and its reply, which retires the head of the list (removeInstruction(0)), + // would then remove this operation's first instruction in its place. After stopStreaming() the + // STOP is in flight for about 400 ms, as its stop handling sleeps that long, so a configuration + // started straight after a stop lost its first SET_INFOMEM, the chunk that holds the sampling + // rate (DEV-1306). + waitForInstructionInFlight(OPERATION_PREPARE_WAIT_MS); + //make sure no more instructions //shouldnt matter since the configuration are being rewritten by consensys //used in initializeshimmer3 (this class) and BluetoothManager class @@ -6065,6 +6081,46 @@ public void enableCheckifAlive(boolean set){ } } + /** How long operationPrepare() waits for a command in flight: past its 2 s ACK timeout */ + private static final long OPERATION_PREPARE_WAIT_MS = 3000; + + /** + * Waits until the instruction at the head of the list, if it has been sent and is waiting for its + * reply, has been answered and retired, or until the timeout. Returns at once on the IOThread, + * which is the thread that would retire it. + */ + private void waitForInstructionInFlight(long timeoutMs) { + byte[] inFlight; + synchronized(mListofInstructions){ + inFlight = (!mTransactionCompleted && !mListofInstructions.isEmpty()) ? mListofInstructions.get(0) : null; + } + if(inFlight==null || Thread.currentThread()==mIOThread){ + return; + } + long end = System.currentTimeMillis()+timeoutMs; + while(System.currentTimeMillis() + * The STOP stays at the head of the instruction list, in flight, until its stop handling retires it: + * the first byte after it (stream data, while streaming) starts that handling, which sleeps 400 ms, + * then removes the head of the list (removeInstruction(0)). A configuration's operationPrepare() + * clears the list, so inside that window it removed the STOP, the operation queued its chunks, and + * the stop handling then removed the first chunk in the STOP's place. + *

+ * No hardware: the IOThread runs as connect() starts it, over an in-memory radio that acknowledges + * every SET_INFOMEM as the firmware does. It lives in this package because mIOThread is a protected + * member of ShimmerBluetooth. + */ +public class API_00031_ConfigureStraightAfterStopTest { + + /** Three 128-byte chunks, as a Shimmer3 or Shimmer3R configuration is written */ + private static final int CONFIG_BYTES = 384; + + private static class AnsweringRadio implements ByteCommunication { + final ConcurrentLinkedDeque rx = new ConcurrentLinkedDeque(); + final List written = Collections.synchronizedList(new ArrayList()); + final CountDownLatch stopSent = new CountDownLatch(1); + + void queue(byte... bytes) { + for(byte b:bytes){ + rx.add(b); + } + } + + /** The address of each SET_INFOMEM written: [0x8C, length, address LSB, address MSB, data...] */ + List infoMemAddressesWritten() { + List addresses = new ArrayList(); + synchronized(written){ + for(byte[] w:written){ + if(w[0]==ShimmerObject.SET_INFOMEM_COMMAND){ + addresses.add((w[2]&0xFF) | ((w[3]&0xFF)<<8)); + } + } + } + return addresses; + } + + boolean wrote(byte command) { + synchronized(written){ + for(byte[] w:written){ + if(w[0]==command){ + return true; + } + } + } + return false; + } + + @Override + public boolean writeBytes(byte[] buffer) { + written.add(buffer.clone()); + if(buffer[0]==ShimmerObject.STOP_STREAMING_COMMAND){ + // Its reply is the test's to send, so the test controls when the stop handling starts + stopSent.countDown(); + } + else { + // Every other command here is a SET, which the firmware acknowledges + queue(ShimmerObject.ACK_COMMAND_PROCESSED); + } + return true; + } + + @Override + public byte[] readBytes(int byteCount, int timeout) throws SerialPortTimeoutException { + if(rx.size() waitForChunks(AnsweringRadio radio, int count, long timeoutMs) throws InterruptedException { + long end = System.currentTimeMillis()+timeoutMs; + while(radio.infoMemAddressesWritten().size() chunks = waitForChunks(radio, ALL_THREE_CHUNKS.length, 5000); + assertEquals("the SET_INFOMEM chunks sent, by address", java.util.Arrays.asList(ALL_THREE_CHUNKS), chunks); + } + + /** + * When the unit's ACK is the first byte after the STOP, with no data packet in flight, the stop + * handling retired the STOP and the same byte then went on as an ACK and retired the next + * instruction too: here a plain SET queued straight after stopStreaming(), with no configuration + * operation at all. + */ + @Test + public void aCommandQueuedAfterTheStopIsSentWhenTheAckComesFirst() throws Exception { + AnsweringRadio radio = streamingShimmer3r(); + mDevice.stopStreaming(); + mDevice.writeAccelRange(1); + assertTrue("the STOP was sent", radio.stopSent.await(2, TimeUnit.SECONDS)); + startStopHandling(radio, ShimmerObject.ACK_COMMAND_PROCESSED); + + long end = System.currentTimeMillis()+3000; + while(!radio.wrote(ShimmerObject.SET_ACCEL_SENSITIVITY_COMMAND) && System.currentTimeMillis() chunks = waitForChunks(radio, ALL_THREE_CHUNKS.length, 5000); + assertEquals("the SET_INFOMEM chunks sent, by address", java.util.Arrays.asList(ALL_THREE_CHUNKS), chunks); + } +} From 3f010ecfb7b87da3a3c5ce6ba81b52bb0c6f484b Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 10:43:58 +0800 Subject: [PATCH 39/41] DEV-1310: deepClone() no longer copies the connection's queues ShimmerPC.deepClone() copies a device by serialising it, which takes no lock. On a connected device the timers and the IO thread kept changing the instruction list (and, while streaming, the PC timestamp list and packet queue) during the copy, so deepClone() could throw ConcurrentModificationException and the configure built on it failed. A clone is a copy of the configuration, not of the connection: those three queues are now transient and readObject() gives the copy empty ones. The Java HIL's HilShimmerPC had the same problem with its own log, which the driver's threads append to on every message; its counters are now transient too. Tests: API_00034 clones while a thread queues and retires instructions, and a simulator-connected device while an app reads battery and status; HilShimmerPCTest clones while the driver logs. All three fail on master. Co-Authored-By: Claude Opus 5.5 --- .../bluetooth/ShimmerBluetooth.java | 23 +++- .../API_00034_DeepCloneWhileBusyTest.java | 121 ++++++++++++++++++ .../com/shimmerresearch/hil/HilShimmerPC.java | 19 ++- .../shimmerresearch/hil/HilShimmerPCTest.java | 50 ++++++++ 4 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java create mode 100644 ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index 00998a855..0b7b49c01 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -76,6 +76,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.ObjectInputStream; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -200,8 +201,12 @@ public String toString() { private boolean mUseLegacyDelayToDelayForResponse = false; - ArrayBlockingQueue mABQPacketByeArray = new ArrayBlockingQueue(10000); - public List mListofPCTimeStamps = new ArrayList(); + // The connection's queues are transient: the IO thread and the timers change them while a + // connected device is serialised by deepClone(), which threw ConcurrentModificationException + // (DEV-1310). readObject() gives a copy empty ones. + private static final int PACKET_QUEUE_CAPACITY = 10000; + transient ArrayBlockingQueue mABQPacketByeArray = new ArrayBlockingQueue(PACKET_QUEUE_CAPACITY); + public transient List mListofPCTimeStamps = new ArrayList(); protected boolean mIamAlive = false; protected abstract void connect(String address,String bluetoothLibrary); @@ -233,7 +238,7 @@ public String toString() { private SystemTimestampPlot systemTimestampPlot = new SystemTimestampPlot(); - private List mListofInstructions = new ArrayList(); + private transient List mListofInstructions = new ArrayList(); private final int ACK_TIMER_DURATION = 2; // Duration to wait for an ack packet (seconds) protected boolean mDummyRead = false, mDummyReadStarted = false; @Deprecated @@ -6072,6 +6077,18 @@ public void enableCheckifAlive(boolean set){ //endregion + /** + * A copy made by serialisation (deepClone()) is of the configuration, not of the connection, + * so it starts with none of the connection's queues. They are transient (DEV-1310), and a + * deserialised object runs no field initialisers. + */ + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + mABQPacketByeArray = new ArrayBlockingQueue(PACKET_QUEUE_CAPACITY); + mListofPCTimeStamps = new ArrayList(); + mListofInstructions = new ArrayList(); + } + /** * @return the mListofInstructions */ diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java new file mode 100644 index 000000000..5bc4ea94f --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java @@ -0,0 +1,121 @@ +package com.shimmerresearch.bluetooth; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.After; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.pcDriver.ShimmerPC; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationSimulatorS3; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; + +/** + * DEV-1310: ShimmerPC.deepClone() copies a device by serialising it, and Consensys's + * Bluetooth manager configures a connected device through such a copy. Serialisation + * takes no lock, so on a connected device the timers and the IO thread went on changing + * its instruction list while it was copied: the status timer queues GET_STATUS every + * 5 s, the battery timer queues a battery read, and the IO thread removes each + * instruction it sends. A change mid-copy threw ConcurrentModificationException out of + * deepClone(), and the configure failed. The Java HIL saw it about once in 30 configures. + *

+ * A clone is a copy of the configuration, not of the connection, so it now starts with + * none of the connection's queues. + */ +public class API_00034_DeepCloneWhileBusyTest { + + private static final int CLONES = 200; + + private ShimmerPC mDevice; + + @After + public void tearDown() { + if(mDevice!=null){ + mDevice.disconnectNoException(); + } + } + + /** + * Another thread queues an instruction the way a timer does (readBattery() takes the + * device, then the list) and retires one the way the IO thread does, while this one + * copies the device. A long queue keeps the copy of the list busy for long enough that + * nearly every clone overlaps a change. + */ + @Test + public void clonesWhileATimerQueuesInstructions() throws Exception { + ShimmerPC device = new ShimmerPC("COM99"); + for(int i=0;i<2000;i++){ + device.readBattery(); + } + AtomicBoolean stop = new AtomicBoolean(); + Thread timer = daemon("timer", () -> { + while(!stop.get()){ + device.readBattery(); + device.removeInstruction(0); + } + }); + timer.start(); + try { + for(int i=1;i<=CLONES;i++){ + ShimmerPC clone = device.deepClone(); + assertNotNull("clone " + i, clone); + assertTrue("clone " + i + " carries no instructions", clone.getListofInstructions().isEmpty()); + } + } finally { + stop.set(true); + timer.join(5000); + } + assertFalse("the queue was not copied away from the device", device.getListofInstructions().isEmpty()); + } + + /** + * The product as it runs: a device connected to the simulator with its status timer on, + * an app thread reading the battery and status as Consensys does, and clones taken all + * the while. Each clone must be usable to configure from. + */ + @Test + public void clonesAConnectedDeviceWhileItWorksThroughItsQueue() throws Exception { + mDevice = new ShimmerPC("COM99"); + mDevice.setTestRadio(new ByteCommunicationSimulatorS3("COM99")); + ByteCommunicationTestUtils.connectAndWaitUntilInitialised(mDevice); + assertEquals(BT_STATE.CONNECTED, mDevice.getBluetoothRadioState()); + + AtomicBoolean stop = new AtomicBoolean(); + AtomicReference appFailure = new AtomicReference<>(); + Thread app = daemon("app", () -> { + try { + while(!stop.get()){ + if(mDevice.getListofInstructions().size()<8){ + mDevice.readBattery(); + mDevice.readStatusLogAndStream(); + } + Thread.yield(); + } + } catch (Throwable t) { + appFailure.set(t); + } + }); + app.start(); + try { + for(int i=1;i<=CLONES;i++){ + ShimmerPC clone = mDevice.deepClone(); + assertNotNull("clone " + i, clone); + assertTrue("clone " + i + " carries no instructions", clone.getListofInstructions().isEmpty()); + assertEquals("clone " + i + " has the device's rate", mDevice.getSamplingRateShimmer(), clone.getSamplingRateShimmer(), 0); + } + } finally { + stop.set(true); + app.join(5000); + } + assertNull("the app thread failed", appFailure.get()); + } + + private Thread daemon(String name, Runnable runnable) { + Thread thread = new Thread(runnable, name); + thread.setDaemon(true); + return thread; + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java index e18b6cdec..8dfa48413 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java @@ -1,5 +1,7 @@ package com.shimmerresearch.hil; +import java.io.IOException; +import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -23,18 +25,29 @@ class HilShimmerPC extends ShimmerPC { private static final int MAX_LOG_LINES = 50000; private final long t0; - private final Map inStreamReplies = new TreeMap<>(); - private final List log = new ArrayList<>(); + // What the session counted is transient: the driver's threads add to it all the time, and + // configureAccelOnly copies this device by serialising it (deepClone()), which threw + // ConcurrentModificationException when a line was logged mid-copy (DEV-1310). + private transient Map inStreamReplies = new TreeMap<>(); + private transient List log = new ArrayList<>(); private volatile int statusesDecoded; private volatile int batteryReadings; /** When each battery reply was decoded, in ms on the session's clock (HilSession.nowMs()). */ - private final List batteryReplyTimesMs = new ArrayList<>(); + private transient List batteryReplyTimesMs = new ArrayList<>(); HilShimmerPC(String comPort, long t0Nanos) { super(comPort); this.t0 = t0Nanos; } + /** A copy starts counting from nothing; a deserialised object runs no field initialisers. */ + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + inStreamReplies = new TreeMap<>(); + log = new ArrayList<>(); + batteryReplyTimesMs = new ArrayList<>(); + } + @Override protected void processInstreamResponse(byte inStreamResponseCommand, boolean shouldClearCrcFromBuffer) { synchronized (inStreamReplies) { diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java new file mode 100644 index 000000000..171946892 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPCTest.java @@ -0,0 +1,50 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Test; + +import com.shimmerresearch.pcDriver.ShimmerPC; + +/** + * configureAccelOnly copies the device the way Consensys does (deepClone()) while the driver's + * threads keep logging into it. The copy must not trip over the log (DEV-1310). + */ +public class HilShimmerPCTest { + + @Test + public void clonesWhileTheDriverLogs() throws Exception { + HilShimmerPC device = new HilShimmerPC("COM99", System.nanoTime()); + for (int i = 0; i < 5000; i++) { + device.consolePrintLn("line " + i); + } + AtomicBoolean stop = new AtomicBoolean(); + // A line a millisecond or so: often enough to land in most copies, slowly enough to stay + // under the log's cap (MAX_LOG_LINES), past which it no longer changes + Thread driver = new Thread(() -> { + while (!stop.get()) { + device.consolePrintLn("GET_STATUS_COMMAND Instruction added"); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + return; + } + } + }, "driver"); + driver.setDaemon(true); + driver.start(); + try { + for (int i = 1; i <= 100; i++) { + ShimmerPC clone = device.deepClone(); + assertNotNull("clone " + i, clone); + assertEquals("clone " + i + " starts its own log", 0, ((HilShimmerPC) clone).logSize()); + } + } finally { + stop.set(true); + driver.join(5000); + } + } +} From 78ad1e3f93d0529d5d148f281f62f58a618d91ab Mon Sep 17 00:00:00 2001 From: JongChern Date: Fri, 23 Oct 2026 11:45:18 +0800 Subject: [PATCH 40/41] DEV-1310: test the streaming queues a clone no longer copies On a Shimmer3 streaming at 256 Hz, about one copy in a hundred threw ConcurrentModificationException in mListofPCTimeStamps, which the IO thread changes for every packet. And the processing thread's packet queue holds packets that can't be serialised, so deepClone() returned null whenever one was waiting. Both tests fail on master. Co-Authored-By: Claude Opus 5.5 --- .../API_00034_DeepCloneWhileBusyTest.java | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java index 5bc4ea94f..678bce6a1 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00034_DeepCloneWhileBusyTest.java @@ -9,6 +9,7 @@ import org.junit.Test; import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.driver.ShimmerObject; import com.shimmerresearch.pcDriver.ShimmerPC; import com.shimmerresearch.shimmer3.communication.ByteCommunicationSimulatorS3; import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; @@ -20,7 +21,8 @@ * its instruction list while it was copied: the status timer queues GET_STATUS every * 5 s, the battery timer queues a battery read, and the IO thread removes each * instruction it sends. A change mid-copy threw ConcurrentModificationException out of - * deepClone(), and the configure failed. The Java HIL saw it about once in 30 configures. + * deepClone(), and the configure failed. While streaming the same happened with the PC + * timestamp list, and a packet waiting for the processing thread made deepClone() return null. *

* A clone is a copy of the configuration, not of the connection, so it now starts with * none of the connection's queues. @@ -71,6 +73,52 @@ public void clonesWhileATimerQueuesInstructions() throws Exception { assertFalse("the queue was not copied away from the device", device.getListofInstructions().isEmpty()); } + /** + * While streaming, the IO thread notes the PC time as each packet starts and drops it once the + * packet is parsed. On a Shimmer3 streaming at 256 Hz that raced about one clone in a hundred. + */ + @Test + public void clonesWhileTheIoThreadTimestampsPackets() throws Exception { + ShimmerPC device = new ShimmerPC("COM99"); + for(int i=0;i<2000;i++){ + device.mListofPCTimeStamps.add(System.currentTimeMillis()); + } + AtomicBoolean stop = new AtomicBoolean(); + Thread ioThread = daemon("IO thread", () -> { + while(!stop.get()){ + device.mListofPCTimeStamps.add(System.currentTimeMillis()); + device.mListofPCTimeStamps.remove(0); + } + }); + ioThread.start(); + try { + for(int i=1;i<=CLONES;i++){ + ShimmerPC clone = device.deepClone(); + assertNotNull("clone " + i, clone); + assertTrue("clone " + i + " carries no PC timestamps", clone.mListofPCTimeStamps.isEmpty()); + } + } finally { + stop.set(true); + ioThread.join(5000); + } + } + + /** + * With the processing thread on, parsed packets wait in a queue for it. Its packets can't be + * serialised, so deepClone() returned null whenever one was waiting. + */ + @Test + public void clonesWithPacketsWaitingForTheProcessingThread() { + ShimmerPC device = new ShimmerPC("COM99"); + // The queue is package-private to ShimmerBluetooth, so reach it through that type + ShimmerBluetooth original = device; + original.mABQPacketByeArray.add(new RawBytePacketWithPCTimeStamp(new byte[] {ShimmerObject.DATA_PACKET, 0x01, 0x02}, System.currentTimeMillis())); + ShimmerBluetooth clone = device.deepClone(); + assertNotNull("a device with a packet waiting can be copied", clone); + assertTrue("the copy has no packets waiting", clone.mABQPacketByeArray.isEmpty()); + assertEquals("the device keeps its packet", 1, original.mABQPacketByeArray.size()); + } + /** * The product as it runs: a device connected to the simulator with its status timer on, * an app thread reading the battery and status as Consensys does, and clones taken all From 002b674372d252df67d75534365ac4a5f76c95f1 Mon Sep 17 00:00:00 2001 From: Mas Azalya Date: Fri, 23 Oct 2026 14:07:50 +0800 Subject: [PATCH 41/41] DEV-896 Size the live trace margin for post-stall catch-up; volatile pause flag LIVE_VIEWPORT_TRACE_MARGIN_MS was 2 x VIEWPORT_LATENCY_MS (400 ms), but after a data stall shorter than FRAME_CLOCK_RESYNC_MS the window catches up at <=2x instead of snapping, trailing the newest sample by up to FRAME_CLOCK_RESYNC_MS + VIEWPORT_LATENCY_MS. A Bluetooth stall over ~400 ms therefore left a blank strip at the window's left edge while it caught up. The margin is now FRAME_CLOCK_RESYNC_MS + VIEWPORT_LATENCY_MS (1.2 s). New test dataStall_lagStaysWithinTraceMargin_forStallsUpToResync covers stalls of 300-980 ms and fails with the old margin. mIsPlotPaused is now volatile: the frame clock tick reads it on the EDT and the data thread reads it too, while pause can be set from another thread. Co-Authored-By: Claude Opus 5.5 --- .../guiUtilities/plot/BasicPlotManagerPC.java | 15 +++++++++----- .../API_00029_LiveViewportFrameClockTest.java | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java index b36fb7505..eea07eb7f 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/guiUtilities/plot/BasicPlotManagerPC.java @@ -98,7 +98,9 @@ public class BasicPlotManagerPC extends AbstractPlotManager { * sample. Written by the data thread in filterDataAndPlot(); volatile because the live-viewport * frame clock reads it on the EDT (DEV-896). */ protected volatile double mCurrentXValue = 0; - protected boolean mIsPlotPaused = false; + /** volatile: set from GUI handlers but read by the live-viewport frame clock tick on the EDT + * and by the data thread, which may not be the thread that set it (DEV-896). */ + protected volatile boolean mIsPlotPaused = false; public boolean mIsLegendLabelsPainted = true; public boolean mIsScaleLabelsPainted = true; public boolean mIsAxisLabelsPainted = true; @@ -631,13 +633,16 @@ public void setXAxisRangeBasedOnXDurationSubtractSingleSamplingRate(double sampl public static final double VIEWPORT_LATENCY_MS = 200; /** The frame clock stops once no live sample has arrived for this long. */ public static final long FRAME_CLOCK_IDLE_STOP_MS = 2000; - /** Extra history a live trace needs while the frame clock runs. A trace sized to exactly the - * window would leave a blank strip of up to ~VIEWPORT_LATENCY_MS at the window's left edge, - * because the window now trails the newest sample. See getLiveViewportTraceMarginInSeconds(). */ - public static final double LIVE_VIEWPORT_TRACE_MARGIN_MS = 2*VIEWPORT_LATENCY_MS; /** Beyond this distance from its target the playout clock snaps instead of slewing: on the * first tick, after a long stall, or when the X values jump (new session, clock reset). */ private static final double FRAME_CLOCK_RESYNC_MS = 1000; + /** Extra history a live trace needs while the frame clock runs. A trace sized to exactly the + * window would leave a blank strip at the window's left edge, because the window trails the + * newest sample: by ~VIEWPORT_LATENCY_MS in steady running, but by up to + * FRAME_CLOCK_RESYNC_MS + VIEWPORT_LATENCY_MS while it catches up (at <=2x) after a data stall + * just short of the resync threshold - a longer stall snaps instead. The margin covers that + * worst case. See getLiveViewportTraceMarginInSeconds(). */ + public static final double LIVE_VIEWPORT_TRACE_MARGIN_MS = FRAME_CLOCK_RESYNC_MS + VIEWPORT_LATENCY_MS; /** Share of the remaining error the playout clock corrects per tick (~0.3 s time constant). */ private static final double FRAME_CLOCK_CORRECTION_GAIN = 0.05; /** Largest correction per tick, as a share of that tick's elapsed time, for small errors and diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/guiUtilities/plot/API_00029_LiveViewportFrameClockTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/guiUtilities/plot/API_00029_LiveViewportFrameClockTest.java index f689ff5f3..00e9cc395 100644 --- a/ShimmerDriverPC/src/test/java/com/shimmerresearch/guiUtilities/plot/API_00029_LiveViewportFrameClockTest.java +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/guiUtilities/plot/API_00029_LiveViewportFrameClockTest.java @@ -93,6 +93,26 @@ public void dataStall_freezesWithoutBackwardSteps_thenCatchesUpWithinASecond() t assertTrue("caught up within 1 s of the stall ending: "+recoveredAt, recoveredAt>0 && recoveredAt-stallTo <= 1000); } + /** While catching up after a stall the window trails the newest sample by more than in steady + * running; the trace margin must cover that lag at every stall length up to the resync + * threshold, or a blank strip shows at the window's left edge until it has caught up. */ + @Test + public void dataStall_lagStaysWithinTraceMargin_forStallsUpToResync() throws Exception { + for(double stallMs : new double[]{300, 600, 900, 980}){ + setUp(); + double stallFrom = 10000, stallTo = stallFrom+stallMs, worstLag = 0; + for(double t=TICK_MS; t<14000; t+=TICK_MS){ + deliver(t-TICK_MS, t, 60, stallFrom, stallTo); + double end = tick(t); + if(t>2000){ + worstLag = Math.max(worstLag, newestX()-end); + } + } + assertTrue("stall "+stallMs+" ms: lag "+worstLag+" within the trace margin", + worstLag <= BasicPlotManagerPC.LIVE_VIEWPORT_TRACE_MARGIN_MS); + } + } + @Test public void longEdtStall_stepIsElapsedTimeNotDoubled() throws Exception { double t = 0, prevT = 0, prevEnd = Double.NaN;