diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java index 078ceb982..abdbf73ed 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/bluetooth/ShimmerBluetooth.java @@ -584,71 +584,94 @@ public ShimmerBluetooth(String userAssignedName, double samplingRate, int enable } public class ProcessingThread extends Thread { - public boolean stop = false; + public volatile boolean stop = false; int count=0; public void run() { - while (!stop) { - if(!mABQPacketByeArray.isEmpty()){ + while (!stop && !Thread.currentThread().isInterrupted()) { + RawBytePacketWithPCTimeStamp rbp = null; + try { + // Blocks (with a timeout) instead of busy-spinning on isEmpty() while idle. + // The timeout ensures the stop flag is still checked periodically. + rbp = mABQPacketByeArray.poll(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if(rbp!=null){ count++; if(count%1000==0){ consolePrintLn("Queue Size: " + mABQPacketByeArray.size()); printLogDataForDebugging("Queue Size: " + mABQPacketByeArray.size() + "\n"); } - RawBytePacketWithPCTimeStamp rbp = mABQPacketByeArray.remove(); // buildAndSendMsg(rbp.mDataArray, FW_TYPE_BT, false, rbp.mSystemTimeStamp); buildAndSendMsg(rbp.mDataArray, COMMUNICATION_TYPE.BLUETOOTH, false, rbp.mSystemTimeStamp); } - } - } - } + } + } + } //region --------- BLUETOOH STACK --------- public class IOThread extends Thread { protected byte[] byteBuffer = {0}; - public boolean stop = false; + public volatile boolean stop = false; public void run() { - while(!stop) { + while(!stop && !Thread.currentThread().isInterrupted()) { + boolean didWork = false; //In-Shimmer Test here if(InShimmerTest) { if (bytesAvailableToBeRead()) { - byte[] data = readBytes(availableBytes()); - if (data!=null) { - if (data.length>0) { - if (mTestByteListener != null) { - mTestByteListener.eventNewBytesReceived(data); - } + didWork = true; + byte[] data = readBytes(availableBytes()); + if (data!=null) { + if (data.length>0) { + if (mTestByteListener != null) { + mTestByteListener.eventNewBytesReceived(data); + } + } } } - } } else { if (mDummyRead) { performDummyRead(); + didWork = true; } - + // Process Instruction on stack. is an instruction running? if not proceed if(!isInstructionStackLock()){ - processNextInstruction(); + if(processNextInstruction()){ + didWork = true; + } } - + if(mIsStreaming){ processWhileStreaming(); + didWork = true; } else if(bytesAvailableToBeRead()){ + didWork = true; if(mWaitForAck) { processNotStreamingWaitForAck(); - } + } else if(mWaitForResponse) { processNotStreamingWaitForResp(); - } - + } + processBytesAvailableAndInstreamSupported(); - + + } + } + + if(!didWork){ + // Nothing useful happened this iteration (idle, not streaming, no pending + // instructions/bytes) - avoid busy-spinning and pegging a CPU core. + try { + Thread.sleep(2); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } - } } @@ -681,10 +704,16 @@ private void performDummyRead() { mDummyReadStarted = false; } - private void processNextInstruction() { + /** + * @return true if an instruction was found and processed (i.e. useful work was done), + * or if this call already idle-paced the loop with its own threadSleep(50) below - + * in that case reporting true here stops run()'s outer idle-sleep from stacking an + * additional wait on top of this one. + */ + private boolean processNextInstruction() { // check instruction stack, are there any other instructions left to be executed? //checkAndRemoveFirstInstructionIfNull(); - + byte[] insBytes = getInstruction(); if (insBytes!=null){ mCurrentCommand=insBytes[0]; @@ -761,14 +790,32 @@ else if((mCurrentCommand==GET_FW_VERSION_COMMAND) mTransactionCompleted=false; + return true; } else { + // getInstruction() returned null here - either the instruction stack is + // genuinely empty, or its leading entry was a null placeholder that + // getInstruction() just removed (a real instruction may still be next in + // the list and will be picked up on the following iteration). Either way, + // if there's also nothing else to do this iteration, wait here rather than + // busy-spinning. if (!mIsStreaming && !bytesAvailableToBeRead()){ - threadSleep(50); + // Report "did work" (true) so the caller's own idle sleep in run() + // doesn't stack an extra wait on top of this one. + // Not threadSleep(): that helper swallows InterruptedException without + // restoring the interrupt flag, which would defeat run()'s + // !Thread.currentThread().isInterrupted() loop-exit check. + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; } + return false; } } - - /** Process ACK from a GET or SET command while not streaming */ + + /** Process ACK from a GET or SET command while not streaming */ private void processNotStreamingWaitForAck() { //JC TEST:: IMPORTANT TO REMOVE // This is to simulate packet loss /* diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java index dc1634651..8ab6f3a79 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/radioProtocol/LiteProtocol.java @@ -253,49 +253,108 @@ private void startIoThread() { } private void stopIoThread() { - if(mIOThread!=null){ - mIOThread.stop=true; - mIOThread = null; + // Snapshot the field: a concurrent teardown may null mIOThread between the + // null check and the interrupt/join/isAlive calls below. + IOThread ioThread = mIOThread; + if(ioThread!=null){ + ioThread.stop=true; + // Bounded wait for the thread to actually exit its loop before dropping the + // reference. Guard against the self-join case: killConnection() can reach here + // synchronously from within IOThread.run()'s catch block (error-driven teardown), + // and a thread joining itself would just burn the full timeout. + if(Thread.currentThread() != ioThread){ + // Interrupt before joining: run() now exits promptly on + // Thread.currentThread().isInterrupted(), and any idle Thread.sleep()/ + // queue poll() it's waiting in will wake immediately - making shutdown + // faster and more deterministic than waiting out the bounded join first. + ioThread.interrupt(); + try { + ioThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (ioThread.isAlive()) { + // Still not terminated after the interrupt + bounded join - most likely + // blocked in a non-interruptible native serial read. Just warn; a second + // interrupt() here wouldn't unblock it either. + printLogDataForDebugging("Warning: IOThread did not terminate within join timeout; it may be blocked in a non-interruptible read"); + } + } + // Only clear the field if it still refers to the thread we stopped, so a + // newly started IOThread is not clobbered. + if (mIOThread == ioThread) { + mIOThread = null; + } } } public class IOThread extends Thread { byte[] byteBuffer = {0}; - public boolean stop = false; + public volatile boolean stop = false; - public synchronized void run() { - while (!stop) { + /* + * Deliberately NOT synchronized: Thread.join() locks this same Thread + * instance, so a synchronized run() would block external stopIoThread() + * callers on monitor entry (with no timeout) until the loop exits. + */ + public void run() { + while (!stop && !Thread.currentThread().isInterrupted()) { + boolean didWork = false; try { // Process Instruction on stack. is an instruction running? if not proceed if(!isInstructionStackLock()){ - processNextInstruction(); + if(processNextInstruction()){ + didWork = true; + } } - + if(isStreaming()){ processWhileStreaming(); + didWork = true; } else if(bytesAvailableToBeRead()){ + didWork = true; if(mWaitForAck) { processNotStreamingWaitForAck(); - } + } else if(mWaitForResponse) { processNotStreamingWaitForResp(); - } - + } + processBytesAvailableAndInstreamSupported(); } } catch (ShimmerException dE) { // stop=true; - - killConnection(dE); + + // DEV-895: once stop is set, an error here is the teardown interrupt surfacing + // (e.g. as ERR_WRITING_DATA from txBytes), not a real link failure. + if(!stop){ + killConnection(dE); + } // e.printStackTrace(); //TODO send event up the ladder } - } - } - - private void processNextInstruction() throws ShimmerException { + + if(!didWork){ + // Nothing useful happened this iteration (idle, not streaming, no pending + // instructions/bytes) - avoid busy-spinning and pegging a CPU core. + try { + Thread.sleep(2); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + + /** + * @return true if an instruction was found and processed (i.e. useful work was done), + * or if this call already idle-paced the loop with its own threadSleep(50) below - + * in that case reporting true here stops run()'s outer idle-sleep from stacking an + * additional wait on top of this one. + */ + private boolean processNextInstruction() throws ShimmerException { // check instruction stack, are there any other instructions left to be executed? if(!getListofInstructions().isEmpty()) { if(getListofInstructions().get(0)==null) { @@ -387,16 +446,29 @@ else if((mCurrentCommand==InstructionsGet.GET_FW_VERSION_COMMAND_VALUE) } }*/ } - - + + mTransactionCompleted=false; + return true; } - } + } else { if (!isStreaming() && !bytesAvailableToBeRead()){ - threadSleep(50); + // No instruction pending and nothing else to do - wait here rather than + // busy-spinning. Report "did work" (true) so the caller's own idle sleep + // in run() doesn't stack an extra wait on top of this one. + // Not threadSleep(): that helper swallows InterruptedException without + // restoring the interrupt flag, which would defeat run()'s + // !Thread.currentThread().isInterrupted() loop-exit check. + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; } } + return false; } private void processWhileStreaming() throws ShimmerException { diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java index f9dc48001..0f251cc73 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerGRPC.java @@ -290,10 +290,21 @@ protected void writeBytes(byte[] data) { // Remote gRPC service is not available, so disconnect Shimmer device sre.printStackTrace(); System.out.println("ERROR: Lost connection to GRPC Server"); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } } } + /** DEV-895: true when running on an IOThread whose stop flag is set, i.e. the failure is the + * expected consequence of closeConnection() tearing down under an IOThread that outlived the + * bounded join. Reporting CONNECTION_LOST then would turn a user disconnect into an + * auto-reconnect downstream. */ + private boolean isOnStoppedIoThread(){ + Thread t = Thread.currentThread(); + return (t instanceof IOThread) && ((IOThread)t).stop; + } + @Override protected void stop() { // TODO Auto-generated method stub @@ -541,17 +552,62 @@ public void disconnect() throws ShimmerException { private void closeConnection() { try { - if (mIOThread != null) { - mIOThread.stop = true; - mIOThread.interrupt(); - mIOThread.join(); // Wait until the thread terminates - mIOThread = null; - - if (mUseProcessingThread) { - mPThread.stop = true; - mPThread.interrupt(); - mPThread.join(); - mPThread = null; + // closeConnection() is unsynchronized and can be entered concurrently - once from + // connectionLost() running on the IOThread itself, and once from a user-triggered + // disconnect() on another thread. Snapshot the fields into locals before touching + // them so a concurrent caller nulling the field out mid-method can't turn a passed + // null-check into an NPE further down (see ShimmerPC.closeConnection() for the + // same fix). + IOThread ioThread = mIOThread; + if (ioThread != null) { + ioThread.stop = true; + // Skip the join when closeConnection() is reached from the IOThread itself - + // a self-join can never succeed and would just burn the full timeout. The + // interrupt() call moves inside this guard too: calling it unconditionally + // would mark the *current* thread interrupted in the self-close case. + if (Thread.currentThread() != ioThread) { + // Interrupt before joining: run() now exits promptly on + // Thread.currentThread().isInterrupted(), and any idle Thread.sleep() + // it's waiting in will wake immediately - making shutdown faster and + // more deterministic than waiting out the bounded join first. + ioThread.interrupt(); + try { + ioThread.join(2000); // Bounded wait so this can't block indefinitely + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (ioThread.isAlive()) { + // Still not terminated after the interrupt + bounded join. + consolePrintLn("Warning: IOThread did not terminate within join timeout"); + } + } + // Only clear the field if it still refers to the thread we stopped, so a + // late closeConnection() cannot clobber a reconnect's new IOThread. + if (mIOThread == ioThread) { + mIOThread = null; + } + } + // Not nested under the IOThread block: a concurrent caller may already have + // nulled mIOThread while the ProcessingThread is still running. + if (mUseProcessingThread) { + ProcessingThread pThread = mPThread; + if (pThread != null) { + pThread.stop = true; + if (Thread.currentThread() != pThread) { + // Interrupt before joining - see the IOThread teardown above. + pThread.interrupt(); + try { + pThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pThread.isAlive()) { + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); + } + } + if (mPThread == pThread) { + mPThread = null; + } } } mIsStreaming = false; @@ -561,7 +617,7 @@ private void closeConnection() { } catch (Exception ex) { consolePrintException(ex.getMessage(), ex.getStackTrace()); setBluetoothRadioState(BT_STATE.DISCONNECTED); - } + } } //Need to override here because ShimmerDevice class uses a different map diff --git a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java index eda410989..7c5ee004d 100644 --- a/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java +++ b/ShimmerDriverPC/src/main/java/com/shimmerresearch/pcDriver/ShimmerPC.java @@ -434,7 +434,9 @@ public boolean bytesAvailableToBeRead() { } catch (SerialPortException | NullPointerException ex) { consolePrintException(ex.getMessage(), ex.getStackTrace()); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } // e.printStackTrace(); } return false; @@ -450,7 +452,9 @@ public int availableBytes(){ } } catch (SerialPortException | NullPointerException ex) { consolePrintException(ex.getMessage(), ex.getStackTrace()); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } return 0; } } @@ -467,7 +471,9 @@ public void writeBytes(byte[] data) { } catch (SerialPortException | NullPointerException ex) { consolePrintLn("Tried to writeBytes but port is closed"); consolePrintException(ex.getMessage(), ex.getStackTrace()); - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } } } @@ -491,7 +497,9 @@ protected byte[] readBytes(int numberOfBytes) { } } } catch (SerialPortException | NullPointerException e) { - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } consolePrintLn("Tried to readBytes but serial port error"); consolePrintException(e.getMessage(), e.getStackTrace()); // e.printStackTrace(); @@ -505,7 +513,9 @@ protected byte[] readBytes(int numberOfBytes) { // ShimmerBluetooth if(mBluetoothRadioState==BT_STATE.CONNECTING || mBluetoothRadioState==BT_STATE.CONFIGURING){ - connectionLost(); + if(!isOnStoppedIoThread()){ + connectionLost(); + } } } return null; @@ -595,6 +605,15 @@ public void disconnectNoException() { } } + /** DEV-895: true when running on an IOThread whose stop flag is set, i.e. the failure is the + * expected consequence of closeConnection() closing the port under a reader that outlived the + * bounded join. Reporting CONNECTION_LOST then would turn a user disconnect into an + * auto-reconnect downstream. */ + private boolean isOnStoppedIoThread(){ + Thread t = Thread.currentThread(); + return (t instanceof IOThread) && ((IOThread)t).stop; + } + @Override protected void connectionLost() { disconnectNoException(); @@ -605,17 +624,71 @@ protected void connectionLost() { private void closeConnection(){ try { - if (mIOThread != null) { - mIOThread.stop = true; - - // Closing serial port before before thread is finished stopping throws an error so waiting here - while(mIOThread != null && mIOThread.isAlive()); + // closeConnection() is unsynchronized and can be entered concurrently - + // once from connectionLost() running on the IOThread itself, and once from + // a user-triggered disconnect() on another thread. Snapshot the fields into + // locals before touching them so a concurrent caller nulling the field out + // mid-method can't turn a passed null-check into an NPE further down (which + // would otherwise be swallowed by the catch below and skip purgePort/closePort, + // leaking the COM port). + IOThread ioThread = mIOThread; + if (ioThread != null) { + ioThread.stop = true; + + // Closing serial port before thread is finished stopping throws an error so waiting here. + // Bounded join instead of an empty-body busy-wait, so this can't block indefinitely. + // Skip the join when closeConnection() is reached from the IOThread itself + // (connectionLost() fires on it when the serial port errors mid-read/write) - + // a self-join can never succeed and would just burn the full timeout. + if(Thread.currentThread() != ioThread){ + // Interrupt before joining: run() now exits promptly on + // Thread.currentThread().isInterrupted(), and any idle Thread.sleep() + // it's waiting in will wake immediately - making shutdown faster and + // more deterministic than waiting out the bounded join first. + ioThread.interrupt(); + try { + ioThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (ioThread.isAlive()) { + // Still not terminated after the interrupt + bounded join - most + // likely blocked in a non-interruptible native serial read. Just + // warn; a second interrupt() here wouldn't unblock it either. + consolePrintLn("Warning: IOThread did not terminate within join timeout; it may be blocked in a non-interruptible read"); + } + } - mIOThread = null; - - if(mUseProcessingThread){ - mPThread.stop = true; - mPThread = null; + // Only clear the field if it still refers to the thread we stopped, so a + // late closeConnection() cannot clobber a reconnect's new IOThread. + if (mIOThread == ioThread) { + mIOThread = null; + } + } + // Not nested under the IOThread block: a concurrent caller may already have + // nulled mIOThread while the ProcessingThread is still running. + if(mUseProcessingThread){ + ProcessingThread pThread = mPThread; + if (pThread != null) { + pThread.stop = true; + // buildAndSendMsg() runs on the ProcessingThread and dispatches to app + // listeners, so a listener calling disconnect() would self-join here - + // guard the same way as the IOThread above. + if (Thread.currentThread() != pThread) { + // Interrupt before joining - see the IOThread teardown above. + pThread.interrupt(); + try { + pThread.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pThread.isAlive()) { + consolePrintLn("Warning: ProcessingThread did not terminate within join timeout"); + } + } + if (mPThread == pThread) { + mPThread = null; + } } } mIsStreaming = false; diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java new file mode 100644 index 000000000..18f44c653 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/bluetooth/API_00028_IoThreadTeardownTest.java @@ -0,0 +1,193 @@ +package com.shimmerresearch.bluetooth; + +import static org.junit.Assert.*; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Assume; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.pcDriver.ShimmerPC; +import com.shimmerresearch.shimmer3.communication.ByteCommunication; +import com.shimmerresearch.verisense.communication.ByteCommunicationListener; + +import jssc.SerialPortException; +import jssc.SerialPortTimeoutException; + +/** + * DEV-895: the IOThread and ProcessingThread must sleep rather than spin when + * idle, must stop promptly on disconnect(), and a disconnect() must stay a + * disconnect. closeConnection() only waits a bounded 2 s for the IOThread, so + * one blocked in a non-interruptible serial call outlives it, and the port is + * closed under it. When that call then fails, the I/O helpers used to report + * CONNECTION_LOST, which NeuroLynQ answers with an auto-reconnect. + *
+ * No hardware: the threads are started as connect() starts them, over an
+ * in-memory radio. It lives in this package because mIOThread and mPThread
+ * are protected members of ShimmerBluetooth.
+ */
+public class API_00028_IoThreadTeardownTest {
+
+ /** Thread-safe in-memory radio. Its writes can block, ignoring interrupts, as a native jssc call does. */
+ private static class BlockingRadio implements ByteCommunication {
+ final ConcurrentLinkedDeque