From aa692b1dc29372eefc1b524ae7e48d7ab51231bb Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Fri, 25 Sep 2026 12:54:20 +0100 Subject: [PATCH 1/6] DEV-1061: read a NeuroLynQ node's logged files A NeuroLynQ node, a Verisense running the GQ's firmware ID (DEV-1047), now logs sessions to NAND as GQ-format files under its own hardware ID, 61 to 68 (verisense-firmware docs/VERISENSE_NEUROLYNQ_STORAGE.md). The driver keyed everything on the file's hardware ID, so it read those files wrongly, or not at all. ShimmerVerObject - A node takes firmware version code 6. That code is what gives a GQ's records 3-byte timestamps (updateTimestampByteLength). A node had -1, which read them as 2 and misaligned every record. - isSupportedSdDataImport(hwVer, fwId): a node's files are importable. The one-argument form sees only the hardware ID, which a node shares with a stock Verisense, so it is left as it was. - isSupportedNodeStorageDownload(): the host may try to download the device's sessions over its dock link. A node has no SD card, and isSupportedSdCardAccess() stays false for it. ShimmerObject - SDLogHeader.NEUROLYNQ_PPG is bit 16 of the header's sensor bitmap, unused until now. It marks 1 to 3 24-bit PPG counts per record, between GSR and heart rate, one for each channel that header byte 15 enables: bit 0 green, 1 IR, 2 red. No GQ or Shimmer3 sets the bit. - buildMsg's hardware gate admits a node, so its timestamps parse as a GQ's do. - The PPG counts are parsed raw, before ECG_TO_HR_FW, in record order. - A node's GQ sensor map gains SensorPPGNeuroLynQ. A GQ's is unchanged. SensorPPGNeuroLynQ (new) holds the channels, as UINT24 little-endian raw counts, and the mask rules: 0 and 0xFF, an unwritten byte, both mean green only. It exists only on the file path. The node streams no PPG, and ShimmerGQ_802154 must keep a GQ's sensor map (ADV_API_00072), so there the mask is a setting and not a sensor. The rest: - ConfigByteLayoutShimmerGq802154: the mask sits at InfoMem 263, the byte after the radio settings, which no GQ reads. - Configuration: SENSOR_ID.NEUROLYNQ_PPG = 154, and a compatibility list for the node alone. - AbstractSensor: SENSORS.PPG_NEUROLYNQ. GSR needs nothing here. The node re-encodes its GSR into the GQ's ADC domain before writing it, and the SD path's GSR maths is the GQ's whatever the hardware ID. API_00016 covers the driver's side (11 tests). The file-level tests are in Shimmer-Advance-API, where ShimmerSDLog is. They use golden files that the firmware's own writer produced on a PC: no node has logged a file yet. Co-Authored-By: Claude Opus 5.5 --- .../shimmerresearch/driver/Configuration.java | 8 +- .../shimmerresearch/driver/ShimmerObject.java | 32 +- .../ConfigByteLayoutShimmerGq802154.java | 11 + .../driverUtilities/ShimmerVerObject.java | 29 +- .../sensors/AbstractSensor.java | 3 +- .../sensors/SensorPPGNeuroLynQ.java | 290 ++++++++++++++++++ .../API_00016_NeuroLynQNodeSdLogTest.java | 158 ++++++++++ 7 files changed, 526 insertions(+), 5 deletions(-) create mode 100644 ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorPPGNeuroLynQ.java create mode 100644 ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00016_NeuroLynQNodeSdLogTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/Configuration.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/Configuration.java index 12ca2b348..f462ecea8 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/Configuration.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/Configuration.java @@ -791,7 +791,9 @@ public static class SENSOR_ID{ public static final int HOST_REAL_TIME_CLOCK_SYNC = 153; public static final int SHIMMER_ECG_TO_HR_FW = 150; - + /** A NeuroLynQ node's PPG, see SensorPPGNeuroLynQ */ + public static final int NEUROLYNQ_PPG = 154; + // Third party devices @ 1000+ public static final int THIRD_PARTY_NONIN = 1000; public static final int HOST_KEYBOARD_LISTENER = 1001; @@ -1428,6 +1430,10 @@ public static class CompatibilityInfoForMaps{ public static final List listOfCompatibleVersionInfoExgEcgGq = Arrays.asList( svoShimmerGq802154Lr, svoShimmerGq802154Nr, svoShimmer2rGq, svoVerisenseNeuroLynQ); + /** A NeuroLynQ node alone: what it has and a GQ does not, such as PPG */ + public static final List listOfCompatibleVersionInfoNeuroLynQNode = Arrays.asList( + svoVerisenseNeuroLynQ); + public static final List listOfCompatibleVersionInfoExgTest = Arrays.asList( svoExgSdLog, svoExgBtStream, svoExgLogAndStream, svoExgUnifiedSdLog, svoExgUnifiedBtStream, svoExgUnifiedLogAndStream, diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java index 4056fe9fb..07b1fc2c7 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java @@ -68,6 +68,7 @@ import com.shimmerresearch.sensors.SensorEXG; import com.shimmerresearch.sensors.SensorGSR; import com.shimmerresearch.sensors.SensorPPG; +import com.shimmerresearch.sensors.SensorPPGNeuroLynQ; import com.shimmerresearch.sensors.SensorShimmerClock; import com.shimmerresearch.sensors.adxl371.SensorADXL371; import com.shimmerresearch.sensors.AbstractSensor.SENSORS; @@ -251,7 +252,11 @@ public class SDLogHeader { public final static int EXG2_16BIT = 1<<19; public final static int BMPX80 = 1<<18; public final static int MPL_TEMPERATURE = 1<<17; - // 1<<16 + /** A NeuroLynQ node's PPG: 1 to 3 24-bit counts per record, between GSR and heart rate, + * one for each channel header byte 15 enables - bit 0 green, 1 IR, 2 red, in that + * order (verisense-firmware docs/VERISENSE_NEUROLYNQ_STORAGE.md sections 3.1-3.2). No GQ or + * Shimmer3 sets the bit, and on them byte 15 is not a mask. */ + public final static int NEUROLYNQ_PPG = 1<<16; public final static long MPL_QUAT_6DOF = (long)1<<31; // needs to be cast to a long otherwise would overflow public final static int MPL_QUAT_9DOF = 1<<30; public final static int MPL_EULER_6DOF = 1<<29; @@ -931,7 +936,9 @@ public ObjectCluster buildMsg(byte[] newPacket, COMMUNICATION_TYPE fwType, boole Vector3d gyroscope = new Vector3d(); if (getHardwareVersion()==HW_ID.SHIMMER_SR30 || getHardwareVersion()==HW_ID.SHIMMER_3 || getHardwareVersion()==HW_ID.SHIMMER_3R - || getHardwareVersion()==HW_ID.SHIMMER_GQ_802154_LR || getHardwareVersion()==HW_ID.SHIMMER_GQ_802154_NR || getHardwareVersion()==HW_ID.SHIMMER_2R_GQ){ + || getHardwareVersion()==HW_ID.SHIMMER_GQ_802154_LR || getHardwareVersion()==HW_ID.SHIMMER_GQ_802154_NR || getHardwareVersion()==HW_ID.SHIMMER_2R_GQ + // A NeuroLynQ node's files are a GQ's under its own hardware ID + || mShimmerVerObject.isVerisenseNeuroLynQ()){ parseTimestampShimmer3(fwType, objectCluster, uncalibratedData, uncalibratedDataUnits, calibratedData, calibratedDataUnits, sensorNames, newPacketInt); @@ -2332,6 +2339,23 @@ else if (isEXGUsingDefaultThreeUnipolarConfiguration()){ objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.MOTIONANDORIENT,CHANNEL_TYPE.UNCAL.toString(),CHANNEL_UNITS.NO_UNITS,uncalibratedData[iMotOrient]); } + //A NeuroLynQ node's PPG: raw counts, for each channel its header's mask enables + if ((fwType == COMMUNICATION_TYPE.SD) && (mEnabledSensors & SDLogHeader.NEUROLYNQ_PPG) > 0){ + for (String ppgChannel : SensorPPGNeuroLynQ.ObjectClusterSensorName.IN_RECORD_ORDER) { + int sigIndex = getSignalIndex(ppgChannel); + if (sigIndex < 0) { + continue; + } + double counts = (double)newPacketInt[sigIndex]; + objectCluster.addDataToMap(ppgChannel,CHANNEL_TYPE.UNCAL.toString(),CHANNEL_UNITS.NO_UNITS,counts); + objectCluster.addDataToMap(ppgChannel,CHANNEL_TYPE.CAL.toString(),CHANNEL_UNITS.NO_UNITS,counts); + uncalibratedData[sigIndex] = counts; + uncalibratedDataUnits[sigIndex] = CHANNEL_UNITS.NO_UNITS; + calibratedData[sigIndex] = counts; + calibratedDataUnits[sigIndex] = CHANNEL_UNITS.NO_UNITS; + } + } + if ((fwType == COMMUNICATION_TYPE.SD) && (mEnabledSensors & SDLogHeader.ECG_TO_HR_FW) > 0){ int sigIndex = getSignalIndex(Shimmer3.ObjectClusterSensorName.ECG_TO_HR_FW); @@ -5509,6 +5533,10 @@ private void createMapOfSensorClasses() { // addSensorClass(SENSORS.CLOCK, new ShimmerClock(this)); addSensorClass(SENSORS.GSR, new SensorGSR(mShimmerVerObject)); addSensorClass(SENSORS.ECG_TO_HR, new SensorECGToHRFw(mShimmerVerObject)); + //A GQ has no PPG; a NeuroLynQ node may, and its heart rate is the PPG's + if(mShimmerVerObject.isVerisenseNeuroLynQ()){ + addSensorClass(SENSORS.PPG_NEUROLYNQ, new SensorPPGNeuroLynQ(mShimmerVerObject)); + } // addSensorClass(SENSORS.EXG, new SensorEXG(this)); } else if(isShimmerGen3()){ if(isSupportedNoImuSensors()){ diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/shimmerGq/ConfigByteLayoutShimmerGq802154.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/shimmerGq/ConfigByteLayoutShimmerGq802154.java index 6a8212848..0d2de9899 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/shimmerGq/ConfigByteLayoutShimmerGq802154.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/shimmerGq/ConfigByteLayoutShimmerGq802154.java @@ -120,6 +120,16 @@ public class ConfigByteLayoutShimmerGq802154 extends ConfigByteLayout implements public int lengthRadioConfig = 7; + /** + * A NeuroLynQ node's PPG channels: bit 0 green, 1 IR, 2 red (SensorPPGNeuroLynQ.MASK_*). + * The byte after the radio settings, which no GQ reads; a node reads it when a session + * opens and writes it into each file's header byte 15 (verisense-firmware + * gq_file_header.c, IM_PPG_MASK). 0xFF, a byte never written, and 0 both mean green + * only, so an image built from zeros configures a node green only. + */ + public int idxNeuroLynQPpgMask = 128+128+7; // 1byte + public int maskNeuroLynQPpgMask = 0x07; + /** * Hold the Shimmer3's microcontroller information memory layout. This @@ -236,6 +246,7 @@ public HashMap getMapOfByteDescriptions(){ mapOfByteDescriptions.put(idxSrRadioMyAddress+1, "RadioMyAddress_LSB"); mapOfByteDescriptions.put(idxSrRadioResponseWindow, "RadioResponseWindow_MSB"); mapOfByteDescriptions.put(idxSrRadioResponseWindow+1, "RadioResponseWindow_LSB"); + mapOfByteDescriptions.put(idxNeuroLynQPpgMask, "NeuroLynQPpgMask"); return mapOfByteDescriptions; } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/ShimmerVerObject.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/ShimmerVerObject.java index 0ba6e4147..9f0c614f4 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/ShimmerVerObject.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/ShimmerVerObject.java @@ -285,7 +285,9 @@ else if(UtilShimmer.compareVersions(mHardwareVersion,mFirmwareIdentifier,mFirmwa || UtilShimmer.compareVersions(mHardwareVersion,mFirmwareIdentifier,mFirmwareVersionMajor,mFirmwareVersionMinor,mFirmwareVersionInternal,HW_ID.SHIMMER_3,FW_ID.LOGANDSTREAM,0,5,4) || UtilShimmer.compareVersions(mHardwareVersion,mFirmwareIdentifier,mFirmwareVersionMajor,mFirmwareVersionMinor,mFirmwareVersionInternal,HW_ID.SHIMMER_3,FW_ID.SDLOG,0,11,5) || mHardwareVersion==HW_ID.SHIMMER_GQ_802154_NR - || mHardwareVersion==HW_ID.SHIMMER_GQ_802154_LR){ + || mHardwareVersion==HW_ID.SHIMMER_GQ_802154_LR + // A NeuroLynQ node logs a GQ's file, whose records carry 3-byte timestamps + || isVerisenseNeuroLynQ()){ mFirmwareVersionCode = 6; } else if(UtilShimmer.compareVersions(mHardwareVersion,mFirmwareIdentifier,mFirmwareVersionMajor,mFirmwareVersionMinor,mFirmwareVersionInternal,HW_ID.SHIMMER_3,FW_ID.BTSTREAM,0,5,0) @@ -522,6 +524,31 @@ public static boolean isSupportedSdDataImport(int hwVer) { return false; } + /** + * As {@link #isSupportedSdDataImport(int)}, and a NeuroLynQ node too: it logs GQ files + * under its own hardware ID, 61 to 68, which only the firmware ID tells apart from a + * stock Verisense's (DEV-1061). + */ + public static boolean isSupportedSdDataImport(int hwVer, int fwId) { + return isSupportedSdDataImport(hwVer) || isVerisenseNeuroLynQ(hwVer, fwId); + } + + /** + * Whether the host may try to download logged sessions from the device over its dock + * link, as a NeuroLynQ node's are: it has no SD card, and holds its sessions in NAND + * behind the `$` storage component (verisense-firmware + * docs/VERISENSE_NEUROLYNQ_STORAGE.md section 7). A node whose firmware predates storage + * reports the same version, so this says only that it may: the storage INFO request + * answering BAD_CMD is what says it cannot. + */ + public boolean isSupportedNodeStorageDownload() { + return isSupportedNodeStorageDownload(getHardwareVersion(), getFirmwareIdentifier()); + } + + public static boolean isSupportedNodeStorageDownload(int hwVer, int fwId) { + return isVerisenseNeuroLynQ(hwVer, fwId); + } + public boolean isBtMemoryUpdateCommandSupported() { // return isCalibDumpSupported(this); diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/AbstractSensor.java b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/AbstractSensor.java index c1049b229..383fcea55 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/AbstractSensor.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/AbstractSensor.java @@ -71,7 +71,8 @@ public enum SENSORS{ VD6283("VD6283"), MLX90632("MLX90632"), BMP390("BMP390"), - BMP581("BMP581"); + BMP581("BMP581"), + PPG_NEUROLYNQ("NeuroLynQ PPG"); private final String text; diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorPPGNeuroLynQ.java b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorPPGNeuroLynQ.java new file mode 100644 index 000000000..0972f6997 --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorPPGNeuroLynQ.java @@ -0,0 +1,290 @@ +package com.shimmerresearch.sensors; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.shimmerresearch.driver.Configuration; +import com.shimmerresearch.driver.Configuration.CHANNEL_UNITS; +import com.shimmerresearch.driver.Configuration.COMMUNICATION_TYPE; +import com.shimmerresearch.driver.Configuration.Shimmer3.CompatibilityInfoForMaps; +import com.shimmerresearch.driver.ObjectCluster; +import com.shimmerresearch.driver.ShimmerDevice; +import com.shimmerresearch.driver.ShimmerObject.SDLogHeader; +import com.shimmerresearch.driverUtilities.ChannelDetails; +import com.shimmerresearch.driverUtilities.ChannelDetails.CHANNEL_DATA_ENDIAN; +import com.shimmerresearch.driverUtilities.ChannelDetails.CHANNEL_DATA_TYPE; +import com.shimmerresearch.driverUtilities.ChannelDetails.CHANNEL_TYPE; +import com.shimmerresearch.driverUtilities.SensorDetails; +import com.shimmerresearch.driverUtilities.SensorDetailsRef; +import com.shimmerresearch.driverUtilities.ShimmerVerObject; +import com.shimmerresearch.driverUtilities.UtilParseData; + +/** + * A NeuroLynQ node's PPG (DEV-1061): the counts of an SR68's MAX86176, read through its + * MAX32674C sensor hub and logged beside GSR and heart rate in a GQ file + * (verisense-firmware docs/VERISENSE_NEUROLYNQ_STORAGE.md sections 3.2 and 4.2). + *

+ * A record holds one to three 24-bit little-endian counts, one for each channel header + * byte 15 enables - bit 0 green, bit 1 IR, bit 2 red - in that order, between GSR and + * heart rate; header bit {@link SDLogHeader#NEUROLYNQ_PPG} says the block is there. The + * counts are raw: nothing in the file says what LED current or ADC range they were taken + * at, so there is nothing to convert them with yet. Heart rate stays on + * {@link SensorECGToHRFw}, where a GQ's is. + *

+ * Files only: a node streams GSR and heart rate over the air, never PPG. + */ +public class SensorPPGNeuroLynQ extends AbstractSensor implements Serializable { + + private static final long serialVersionUID = -2715183478126653214L; + + /** Header byte 15's bits */ + public static final int MASK_GREEN = 0x01; + public static final int MASK_IR = 0x02; + public static final int MASK_RED = 0x04; + public static final int MASK_ALL = MASK_GREEN | MASK_IR | MASK_RED; + + //--------- Sensor specific variables start -------------- + public static class GuiLabelSensors { + public static final String PPG = "PPG"; + } + + public static class ObjectClusterSensorName { + public static final String PPG_GREEN = "PPG_Green"; + public static final String PPG_IR = "PPG_IR"; + public static final String PPG_RED = "PPG_Red"; + + /** In the mask's bit order, which is the order a record holds them in */ + public static final List IN_RECORD_ORDER = Collections.unmodifiableList( + Arrays.asList(PPG_GREEN, PPG_IR, PPG_RED)); + } + + public static class DatabaseChannelHandles { + public static final String PPG_GREEN = "PPG_Green"; + public static final String PPG_IR = "PPG_IR"; + public static final String PPG_RED = "PPG_Red"; + } + + public static class DatabaseConfigHandle { + public static final String PPG_MASK = "NeuroLynQ_PPG_Mask"; + } + //--------- Sensor specific variables end -------------- + + //--------- Sensor info start -------------- + /** Logged, never streamed: no streaming bitmap */ + public static final SensorDetailsRef sensorPpgNeuroLynQ = new SensorDetailsRef( + 0, + SDLogHeader.NEUROLYNQ_PPG, + GuiLabelSensors.PPG, + CompatibilityInfoForMaps.listOfCompatibleVersionInfoNeuroLynQNode, + null, + ObjectClusterSensorName.IN_RECORD_ORDER); + + public static final Map mSensorMapRef; + static { + Map aMap = new LinkedHashMap(); + aMap.put(Configuration.Shimmer3.SENSOR_ID.NEUROLYNQ_PPG, sensorPpgNeuroLynQ); + mSensorMapRef = Collections.unmodifiableMap(aMap); + } + //--------- Sensor info end -------------- + + //--------- Channel info start -------------- + public static final ChannelDetails channelPpgGreen = createChannel( + ObjectClusterSensorName.PPG_GREEN, DatabaseChannelHandles.PPG_GREEN); + public static final ChannelDetails channelPpgIr = createChannel( + ObjectClusterSensorName.PPG_IR, DatabaseChannelHandles.PPG_IR); + public static final ChannelDetails channelPpgRed = createChannel( + ObjectClusterSensorName.PPG_RED, DatabaseChannelHandles.PPG_RED); + + private static ChannelDetails createChannel(String objectClusterName, String databaseHandle) { + ChannelDetails channel = new ChannelDetails( + objectClusterName, + objectClusterName, + databaseHandle, + CHANNEL_DATA_TYPE.UINT24, 3, CHANNEL_DATA_ENDIAN.LSB, + CHANNEL_UNITS.NO_UNITS, + Arrays.asList(CHANNEL_TYPE.CAL, CHANNEL_TYPE.UNCAL)); + channel.mDefaultUncalUnit = CHANNEL_UNITS.NO_UNITS; + return channel; + } + + public static final Map mChannelMapRef; + static { + Map aMap = new LinkedHashMap(); + aMap.put(ObjectClusterSensorName.PPG_GREEN, channelPpgGreen); + aMap.put(ObjectClusterSensorName.PPG_IR, channelPpgIr); + aMap.put(ObjectClusterSensorName.PPG_RED, channelPpgRed); + mChannelMapRef = Collections.unmodifiableMap(aMap); + } + //--------- Channel info end -------------- + + /** + * The channels a header's byte 15 enables, in record order. Only its low three bits + * count, as the firmware and the web SDK's validator count them. + */ + public static List channelsForMask(int mask) { + List channels = new java.util.ArrayList(); + for (int bit = 0; bit < ObjectClusterSensorName.IN_RECORD_ORDER.size(); bit++) { + if ((mask & (1 << bit)) != 0) { + channels.add(ObjectClusterSensorName.IN_RECORD_ORDER.get(bit)); + } + } + return channels; + } + + /** + * The mask a node reads out of its InfoMem, as its firmware reads it + * (verisense-firmware gq_file_header.c, GqFile_ppgMask()): 0xFF, a byte never + * written, and any value with none of the three bits set are green only. + */ + public static int maskFromInfoMem(int stored) { + stored &= 0xFF; + if (stored == 0xFF || (stored & MASK_ALL) == 0) { + return MASK_GREEN; + } + return stored & MASK_ALL; + } + + /** Green only: what a node logs until configured otherwise, and all a heart rate needs. + * IR and red are for SpO2, and each channel adds a third to a file. */ + private int mChannelMask = MASK_GREEN; + + public SensorPPGNeuroLynQ(ShimmerVerObject svo) { + super(SENSORS.PPG_NEUROLYNQ, svo); + initialise(); + } + + @Override + public void generateSensorMap() { + super.createLocalSensorMapWithCustomParser(mSensorMapRef, mChannelMapRef); + applyChannelMask(); + } + + /** + * The channels a file logs, from its header's byte 15: only those are parsed and + * stored, so a green-only node's database has no IR or red columns. + */ + public void setChannelMask(int mask) { + mChannelMask = mask & MASK_ALL; + applyChannelMask(); + } + + public int getChannelMask() { + return mChannelMask; + } + + private void applyChannelMask() { + SensorDetails sensorDetails = mSensorMap.get(Configuration.Shimmer3.SENSOR_ID.NEUROLYNQ_PPG); + if (sensorDetails == null) { + return; + } + sensorDetails.mListOfChannels.clear(); + for (String channel : channelsForMask(mChannelMask)) { + sensorDetails.mListOfChannels.add(mChannelMapRef.get(channel)); + } + } + + @Override + public void generateSensorGroupMapping() { + //NOT USED IN THIS CLASS + } + + @Override + public void generateConfigOptionsMap() { + //NOT USED IN THIS CLASS + } + + @Override + public ObjectCluster processDataCustom(SensorDetails sensorDetails, byte[] sensorByteArray, COMMUNICATION_TYPE commType, + ObjectCluster objectCluster, boolean isTimeSyncEnabled, double pcTimestampMs) { + int index = 0; + for (ChannelDetails channelDetails : sensorDetails.mListOfChannels) { + byte[] channelByteArray = new byte[channelDetails.mDefaultNumBytes]; + System.arraycopy(sensorByteArray, index, channelByteArray, 0, channelDetails.mDefaultNumBytes); + double counts = (double) UtilParseData.parseData(channelByteArray, + channelDetails.mDefaultChannelDataType, channelDetails.mDefaultChannelDataEndian); + objectCluster.addUncalData(channelDetails, counts); + objectCluster.addCalData(channelDetails, counts); + objectCluster.incrementIndexKeeper(); + index = index + channelDetails.mDefaultNumBytes; + } + return objectCluster; + } + + /** + * Not here: a docked node is configured as a ShimmerGQ_802154, which has a GQ's + * sensors - the node streams what a GQ streams - and so owns the mask's InfoMem byte + * itself. This class is the file's side. + */ + @Override + public void configBytesGenerate(ShimmerDevice shimmerDevice, byte[] configBytes, COMMUNICATION_TYPE commType) { + //NOT USED IN THIS CLASS + } + + @Override + public void configBytesParse(ShimmerDevice shimmerDevice, byte[] configBytes, COMMUNICATION_TYPE commType) { + //NOT USED IN THIS CLASS + } + + @Override + public Object setConfigValueUsingConfigLabel(Integer sensorId, String configLabel, Object valueToSet) { + return null; + } + + @Override + public Object getConfigValueUsingConfigLabel(Integer sensorId, String configLabel) { + return null; + } + + @Override + public void setSensorSamplingRate(double samplingRateHz) { + //The node's rate, from its header + } + + @Override + public boolean setDefaultConfigForSensor(int sensorId, boolean isSensorEnabled) { + return mSensorMap.containsKey(sensorId); + } + + @Override + public boolean checkConfigOptionValues(String stringKey) { + return false; + } + + @Override + public Object getSettings(String componentName, COMMUNICATION_TYPE commType) { + return null; + } + + @Override + public ActionSetting setSettings(String componentName, Object valueToSet, COMMUNICATION_TYPE commType) { + return null; + } + + @Override + public LinkedHashMap generateConfigMap() { + LinkedHashMap mapOfConfig = new LinkedHashMap(); + mapOfConfig.put(DatabaseConfigHandle.PPG_MASK, (double) mChannelMask); + return mapOfConfig; + } + + @Override + public void parseConfigMap(LinkedHashMap mapOfConfigPerShimmer) { + if (mapOfConfigPerShimmer.containsKey(DatabaseConfigHandle.PPG_MASK)) { + setChannelMask(maskFromInfoMem(((Double) mapOfConfigPerShimmer.get(DatabaseConfigHandle.PPG_MASK)).intValue())); + } + } + + @Override + public boolean processResponse(int responseCommand, Object parsedResponse, COMMUNICATION_TYPE commType) { + return false; + } + + @Override + public void checkShimmerConfigBeforeConfiguring() { + //NOT USED IN THIS CLASS + } +} diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00016_NeuroLynQNodeSdLogTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00016_NeuroLynQNodeSdLogTest.java new file mode 100644 index 000000000..2189b2ef7 --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00016_NeuroLynQNodeSdLogTest.java @@ -0,0 +1,158 @@ +package com.shimmerresearch.driverUtilities; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Test; + +import com.shimmerresearch.driver.Configuration; +import com.shimmerresearch.driver.ShimmerObject.SDLogHeader; +import com.shimmerresearch.driver.shimmerGq.ConfigByteLayoutShimmerGq802154; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails.FW_ID; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID; +import com.shimmerresearch.sensors.SensorPPGNeuroLynQ; +import com.shimmerresearch.sensors.SensorPPGNeuroLynQ.ObjectClusterSensorName; + +/** + * What a host needs to read a NeuroLynQ node's logged files (DEV-1061). + *

+ * A node logs a GQ's file under its own hardware ID and the GQ's firmware ID + * (verisense-firmware docs/VERISENSE_NEUROLYNQ_STORAGE.md section 3), with 3-byte timestamps, + * and on an SR68 a PPG block the GQ never had. The file-level tests are in + * Shimmer-Advance-API, where ShimmerSDLog is; these are the driver's parts. + */ +public class API_00016_NeuroLynQNodeSdLogTest { + + private static final ShimmerVerObject SR68_NODE = new ShimmerVerObject(HW_ID.VERISENSE_PULSE_PLUS, FW_ID.GQ_802154, 2, 1, 1); + private static final ShimmerVerObject SR61_NODE = new ShimmerVerObject(HW_ID.VERISENSE_GSR_PLUS, FW_ID.GQ_802154, 2, 1, 1); + private static final ShimmerVerObject STOCK_VERISENSE = new ShimmerVerObject(HW_ID.VERISENSE_PULSE_PLUS, FW_ID.VERISENSE, 2, 1, 1); + private static final ShimmerVerObject GQ = new ShimmerVerObject(HW_ID.SHIMMER_GQ_802154_NR, FW_ID.GQ_802154, 1, 1, 1); + + /** Firmware version code 6 is what gives a GQ's file its 3-byte timestamps + * (ShimmerObject.updateTimestampByteLength). A node had none, which read them as 2. */ + @Test + public void aNodeTakesTheGqsFirmwareVersionCode() { + assertEquals(6, GQ.mFirmwareVersionCode); + assertEquals(GQ.mFirmwareVersionCode, SR68_NODE.mFirmwareVersionCode); + assertEquals(GQ.mFirmwareVersionCode, SR61_NODE.mFirmwareVersionCode); + } + + @Test + public void aStockVerisensesVersionCodeIsUnchanged() { + assertEquals(-1, STOCK_VERISENSE.mFirmwareVersionCode); + } + + @Test + public void aNodesFilesAreImportedAndAStockVerisensesAreNot() { + assertTrue(ShimmerVerObject.isSupportedSdDataImport(SR68_NODE.getHardwareVersion(), SR68_NODE.getFirmwareIdentifier())); + assertTrue(ShimmerVerObject.isSupportedSdDataImport(SR61_NODE.getHardwareVersion(), SR61_NODE.getFirmwareIdentifier())); + assertTrue(ShimmerVerObject.isSupportedSdDataImport(GQ.getHardwareVersion(), GQ.getFirmwareIdentifier())); + assertFalse(ShimmerVerObject.isSupportedSdDataImport(STOCK_VERISENSE.getHardwareVersion(), STOCK_VERISENSE.getFirmwareIdentifier())); + // The one-argument form is unchanged: it cannot tell a node from a stock Verisense + assertFalse(ShimmerVerObject.isSupportedSdDataImport(SR68_NODE.getHardwareVersion())); + } + + /** A node's sessions are downloaded over its dock link; it has no card to scan */ + @Test + public void onlyANodeDownloadsItsStorage() { + assertTrue(SR68_NODE.isSupportedNodeStorageDownload()); + assertTrue(SR61_NODE.isSupportedNodeStorageDownload()); + assertFalse(GQ.isSupportedNodeStorageDownload()); + assertFalse(STOCK_VERISENSE.isSupportedNodeStorageDownload()); + assertFalse(SR68_NODE.isSupportedSdCardAccess()); + } + + /** Header byte 15's low three bits, green, IR and red, in record order */ + @Test + public void thePpgMaskGivesTheChannelsInRecordOrder() { + assertEquals(Collections.emptyList(), SensorPPGNeuroLynQ.channelsForMask(0)); + assertEquals(Arrays.asList(ObjectClusterSensorName.PPG_GREEN), SensorPPGNeuroLynQ.channelsForMask(SensorPPGNeuroLynQ.MASK_GREEN)); + assertEquals(Arrays.asList(ObjectClusterSensorName.PPG_IR, ObjectClusterSensorName.PPG_RED), + SensorPPGNeuroLynQ.channelsForMask(SensorPPGNeuroLynQ.MASK_IR | SensorPPGNeuroLynQ.MASK_RED)); + assertEquals(ObjectClusterSensorName.IN_RECORD_ORDER, SensorPPGNeuroLynQ.channelsForMask(SensorPPGNeuroLynQ.MASK_ALL)); + // Only the low three bits count, as the firmware and the web SDK count them + assertEquals(ObjectClusterSensorName.IN_RECORD_ORDER, SensorPPGNeuroLynQ.channelsForMask(0xFF)); + } + + @Test + public void thePpgSensorIsEnabledByHeaderBit16AndNeverStreamed() { + assertEquals(1 << 16, SDLogHeader.NEUROLYNQ_PPG); + assertEquals(SDLogHeader.NEUROLYNQ_PPG, SensorPPGNeuroLynQ.sensorPpgNeuroLynQ.mSensorBitmapIDSDLogHeader); + assertEquals(0, SensorPPGNeuroLynQ.sensorPpgNeuroLynQ.mSensorBitmapIDStreaming); + } + + /** The sensor stores only the channels its file logs, so a green-only node's database + * gets no IR or red columns */ + @Test + public void theMaskNarrowsTheChannelsStored() { + SensorPPGNeuroLynQ sensor = new SensorPPGNeuroLynQ(SR68_NODE); + SensorDetails details = sensor.mSensorMap.get(Configuration.Shimmer3.SENSOR_ID.NEUROLYNQ_PPG); + assertNotNull(details); + // Green only until told otherwise, as a node logs until configured otherwise + assertEquals(SensorPPGNeuroLynQ.MASK_GREEN, sensor.getChannelMask()); + assertEquals(Arrays.asList(ObjectClusterSensorName.PPG_GREEN), channelNames(details.mListOfChannels)); + assertEquals(SensorPPGNeuroLynQ.DatabaseChannelHandles.PPG_GREEN, details.mListOfChannels.get(0).getDatabaseChannelHandle()); + + sensor.setChannelMask(SensorPPGNeuroLynQ.MASK_ALL); + assertEquals(ObjectClusterSensorName.IN_RECORD_ORDER, channelNames(details.mListOfChannels)); + + sensor.setChannelMask(SensorPPGNeuroLynQ.MASK_GREEN | SensorPPGNeuroLynQ.MASK_RED); + assertEquals(Arrays.asList(ObjectClusterSensorName.PPG_GREEN, ObjectClusterSensorName.PPG_RED), channelNames(details.mListOfChannels)); + } + + /** InfoMem 263 as the firmware reads it (gq_file_header.c GqFile_ppgMask()) */ + @Test + public void anUnwrittenOrEmptyMaskIsGreenOnly() { + assertEquals(SensorPPGNeuroLynQ.MASK_GREEN, SensorPPGNeuroLynQ.maskFromInfoMem(0xFF)); + assertEquals(SensorPPGNeuroLynQ.MASK_GREEN, SensorPPGNeuroLynQ.maskFromInfoMem(0x00)); + assertEquals(SensorPPGNeuroLynQ.MASK_GREEN, SensorPPGNeuroLynQ.maskFromInfoMem(0xF8)); + assertEquals(SensorPPGNeuroLynQ.MASK_IR | SensorPPGNeuroLynQ.MASK_RED, SensorPPGNeuroLynQ.maskFromInfoMem(0x06)); + assertEquals(SensorPPGNeuroLynQ.MASK_ALL, SensorPPGNeuroLynQ.maskFromInfoMem(0x0F)); + assertEquals(SensorPPGNeuroLynQ.MASK_ALL, SensorPPGNeuroLynQ.maskFromInfoMem((byte) 0x87)); + } + + /** The mask is stored with the device's configuration, and comes back from it */ + @Test + public void theMaskRoundTripsThroughTheConfigMap() { + SensorPPGNeuroLynQ sensor = new SensorPPGNeuroLynQ(SR68_NODE); + sensor.setChannelMask(SensorPPGNeuroLynQ.MASK_IR | SensorPPGNeuroLynQ.MASK_RED); + java.util.LinkedHashMap config = sensor.generateConfigMap(); + assertEquals(6.0, (Double) config.get(SensorPPGNeuroLynQ.DatabaseConfigHandle.PPG_MASK), 0.0); + + SensorPPGNeuroLynQ restored = new SensorPPGNeuroLynQ(SR68_NODE); + restored.parseConfigMap(config); + assertEquals(SensorPPGNeuroLynQ.MASK_IR | SensorPPGNeuroLynQ.MASK_RED, restored.getChannelMask()); + } + + @Test + public void eachPpgChannelIsA24BitLittleEndianCount() { + for (ChannelDetails channel : Arrays.asList(SensorPPGNeuroLynQ.channelPpgGreen, SensorPPGNeuroLynQ.channelPpgIr, SensorPPGNeuroLynQ.channelPpgRed)) { + assertEquals(3, channel.mDefaultNumBytes); + assertEquals(ChannelDetails.CHANNEL_DATA_TYPE.UINT24, channel.mDefaultChannelDataType); + assertEquals(ChannelDetails.CHANNEL_DATA_ENDIAN.LSB, channel.mDefaultChannelDataEndian); + } + } + + /** The byte after the radio settings, where the firmware reads it (gq_file_header.c IM_PPG_MASK) */ + @Test + public void theNodesPpgMaskIsInfoMem263() { + ConfigByteLayoutShimmerGq802154 layout = new ConfigByteLayoutShimmerGq802154(FW_ID.GQ_802154, 2, 1, 1); + assertEquals(263, layout.idxNeuroLynQPpgMask); + assertEquals(layout.idxSrRadioConfigStart + layout.lengthRadioConfig, layout.idxNeuroLynQPpgMask); + assertEquals(0x07, layout.maskNeuroLynQPpgMask); + } + + private static List channelNames(List channels) { + String[] names = new String[channels.size()]; + for (int i = 0; i < names.length; i++) { + names[i] = channels.get(i).mObjectClusterName; + } + return Arrays.asList(names); + } +} From 5bea1b1e6c2334a12a96575cdfca2ea260f9e426 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Fri, 25 Sep 2026 12:55:28 +0100 Subject: [PATCH 2/6] DEV-1061: download a NeuroLynQ node's sessions over its dock link A node keeps its sessions in NAND behind a new $ component, STORAGE (0x0C), with properties INFO, SESSION, FILES, READ, ERASE and FORMAT (verisense-firmware docs/VERISENSE_NEUROLYNQ_STORAGE.md section 7). Five of them are one request and one response, as the driver already handles. READ answers with a burst instead: data frames sent a NAND page at a time, then one END that carries the byte count and CRC-32. - UartPacketDetails: the STORAGE component and its properties, for a node only. - ErrorCodesWiredProtocol: an error code for each request, 7040 to 7045. - AbstractCommsProtocolWired: readBurst(). While a burst is read, the parser puts that component's data and END frames on a queue, not on the list other requests poll every 100 ms. A frame whose CRC fails is counted and lost, and the reader asks for it again. A NACK throws, as it does for any request. Requests that are not bursts are unchanged. New in comms/wiredProtocol/neurolynq/: - NeuroLynQStorageCodec: the bytes. It is a port of neurolynq-web-sdk's storage.ts, held to the same vectors as that and the firmware's codec test. - NeuroLynQStorageProtocol: the requests, with the web SDK's read rule. - A window keeps the bytes that arrived in order, and asks again after a gap. - A window whose END contradicts what arrived is thrown away. - A file is whole once every byte has arrived, END or not. - Six empty windows in a row fail the read rather than hang. - isStorageAbsent() means INFO answered BAD_CMD: a GQ, or a node whose firmware predates storage. - SimulatedNeuroLynQNodeSerialPort: a node behind an AbstractSerialPortHal. It sends as the firmware does, a page's frames at a time from its own thread, and on request drops, corrupts, cuts short or NACKs what it sends. It sits in main, not test, because Shimmer-Advance-API's dock tests drive it too. API_00017 tests the codec (8 tests). API_00018 tests the protocol against the simulated node (13 tests). Five faults put into the reader were each caught: no gap tracking, the END's CRC ignored, a CRC error made fatal, the burst not routed, and absence never reported. Tested against the simulated node only; no real node has been downloaded yet. Co-Authored-By: Claude Opus 5.5 --- .../AbstractCommsProtocolWired.java | 98 +++- .../ErrorCodesWiredProtocol.java | 15 + .../wiredProtocol/UartPacketDetails.java | 28 +- .../neurolynq/NeuroLynQStorageCodec.java | 434 +++++++++++++++ .../neurolynq/NeuroLynQStorageProtocol.java | 248 +++++++++ .../SimulatedNeuroLynQNodeSerialPort.java | 520 ++++++++++++++++++ .../API_00017_NeuroLynQStorageCodecTest.java | 182 ++++++ ...PI_00018_NeuroLynQStorageProtocolTest.java | 218 ++++++++ 8 files changed, 1741 insertions(+), 2 deletions(-) create mode 100644 ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageCodec.java create mode 100644 ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java create mode 100644 ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java create mode 100644 ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00017_NeuroLynQStorageCodecTest.java create mode 100644 ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java index 8a0d9ef2e..47d4fa041 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java @@ -6,6 +6,8 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.ArrayUtils; @@ -58,6 +60,12 @@ public abstract class AbstractCommsProtocolWired extends BasicProcessWithCallBac // private List mListOfUartRxPacketObjects = new ArrayList(); private List mListOfUartRxPacketObjects = Collections.synchronizedList(new ArrayList()); public DockException mThrownException = null; + /** A READ burst being collected (readBurst()): its packets, or the DockException the reader + * hit on one, go here as they arrive rather than to mListOfUartRxPacketObjects */ + private volatile LinkedBlockingQueue mBurstQueue = null; + private volatile byte mBurstComponent; + private volatile byte mBurstDataProperty; + private volatile byte mBurstEndProperty; // private UartRxCallback mUartRxCallback = null; private boolean mSendCallbackRxOverride = false; @@ -726,6 +734,10 @@ else if(rxBuf.length burst = mBurstQueue; + if(burst!=null){ + burst.offer(de); + } if(currentPacketCmd!=null && currentMsgArg!=null){ System.out.println(mUniqueId + "\tProblem parsing received packet while waiting for:"); @@ -787,7 +799,11 @@ private void processParsedPacket(UartRxPacketObject uRPO, long timestampMs) thro throw de; } - if(mSendCallbackRxOverride){ + LinkedBlockingQueue burst = mBurstQueue; + if(burst!=null && isBurstPacket(uRPO)){ + burst.offer(uRPO); + } + else if(mSendCallbackRxOverride){ wrapMsgSpanAndSend(MsgDock.MSG_ID_SHIMMERUART_PACKET_RX, uRPO); } else { @@ -805,6 +821,86 @@ private void processParsedPacket(UartRxPacketObject uRPO, long timestampMs) thro } } + private boolean isBurstPacket(UartRxPacketObject uRPO) { + return uRPO.mUartCommandByte == UartPacketDetails.UART_PACKET_CMD.DATA_RESPONSE.toCmdByte() + && uRPO.mUartComponentByte == mBurstComponent + && (uRPO.mUartPropertyByte == mBurstDataProperty || uRPO.mUartPropertyByte == mBurstEndProperty); + } + + /** What a READ burst brought: its packets in order, the end frame last if it came */ + public static final class ReadBurst { + public final List packets; + /** The end frame arrived; otherwise the burst stalled or ran out of time */ + public final boolean ended; + /** Frames lost to a `$` CRC that did not check: their bytes are to be asked for again */ + public final int crcErrors; + + public ReadBurst(List packets, boolean ended, int crcErrors) { + this.packets = Collections.unmodifiableList(packets); + this.ended = ended; + this.crcErrors = crcErrors; + } + } + + /** + * Send a READ and collect the burst that answers it: every data frame with the + * request's component and property, until the one with {@code endProperty}, nothing + * for {@code stallMs}, or {@code totalMs} gone (DEV-1061, a NeuroLynQ node's storage). + * The frames go to a queue as the reader parses them, not to the list other requests + * poll every 100 ms, so a burst is read as fast as it arrives. A frame whose CRC fails + * is counted and lost; a NACK throws, as it does for any request. + */ + public ReadBurst readBurst(UartComponentPropertyDetails request, byte[] payload, byte endProperty, + long stallMs, long totalMs, int errorCode) throws DockException { + if(!mLeavePortOpen) openSafely(); + LinkedBlockingQueue queue = new LinkedBlockingQueue(); + List packets = new ArrayList(); + int crcErrors = 0; + boolean ended = false; + mListOfUartRxPacketObjects.clear(); + mThrownException = null; + mBurstComponent = request.mComponentByte; + mBurstDataProperty = request.mPropertyByte; + mBurstEndProperty = endProperty; + mBurstQueue = queue; + try { + txPacket(UartPacketDetails.UART_PACKET_CMD.READ, request, payload); + long deadline = System.currentTimeMillis() + totalMs; + for(;;) { + long wait = Math.min(stallMs, deadline - System.currentTimeMillis()); + if(wait <= 0) { + break; + } + Object next = queue.poll(wait, TimeUnit.MILLISECONDS); + if(next == null) { + break; + } + if(next instanceof DockException) { + DockException de = (DockException) next; + if(de.mErrorCodeLowLevel == ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_CRC) { + crcErrors++; + continue; + } + de.mErrorCode = errorCode; + throw de; + } + UartRxPacketObject uRPO = (UartRxPacketObject) next; + packets.add(uRPO); + if(uRPO.mUartPropertyByte == endProperty) { + ended = true; + break; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DockException(mComPort, errorCode, ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_TIMEOUT, mUniqueId); + } finally { + mBurstQueue = null; + if(!mLeavePortOpen) closeSafely(); + } + return new ReadBurst(packets, ended, crcErrors); + } + /** remove first and add remaining bytes to start of next serial port read * @return */ private byte[] removeFirstByteFromArray(byte[] rxBuf){ diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/ErrorCodesWiredProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/ErrorCodesWiredProtocol.java index fa14c998c..c23382dc7 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/ErrorCodesWiredProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/ErrorCodesWiredProtocol.java @@ -32,6 +32,14 @@ public class ErrorCodesWiredProtocol extends AbstractErrorCodes { public final static int SHIMMERUART_CMD_ERR_BT_FW_VERSION_INFO_GET = (ERROR_CODES_ID*1000) + 37; public final static int SHIMMERUART_CMD_ERR_ENTER_BOOTLOADER_SET = (ERROR_CODES_ID*1000) + 38; + // A NeuroLynQ node's storage component (DEV-1061) + public final static int SHIMMERUART_CMD_ERR_STORAGE_INFO_GET = (ERROR_CODES_ID*1000) + 40; + public final static int SHIMMERUART_CMD_ERR_STORAGE_SESSION_GET = (ERROR_CODES_ID*1000) + 41; + public final static int SHIMMERUART_CMD_ERR_STORAGE_FILES_GET = (ERROR_CODES_ID*1000) + 42; + public final static int SHIMMERUART_CMD_ERR_STORAGE_READ = (ERROR_CODES_ID*1000) + 43; + public final static int SHIMMERUART_CMD_ERR_STORAGE_ERASE_SET = (ERROR_CODES_ID*1000) + 44; + public final static int SHIMMERUART_CMD_ERR_STORAGE_FORMAT_SET = (ERROR_CODES_ID*1000) + 45; + //SR7 (802.15.4 radio) related errors public final static int SHIMMERUART_CMD_ERR_RADIO_802154_SET_SETTINGS = (ERROR_CODES_ID*1000) + 33; public final static int SHIMMERUART_CMD_ERR_RADIO_802154_GET_SETTINGS = (ERROR_CODES_ID*1000) + 34; @@ -79,6 +87,13 @@ public class ErrorCodesWiredProtocol extends AbstractErrorCodes { aMap.put(SHIMMERUART_CMD_ERR_BT_FW_VERSION_INFO_GET, "Error reading BT FW version info"); aMap.put(SHIMMERUART_CMD_ERR_ENTER_BOOTLOADER_SET, "Error writing enter bootloader command"); + aMap.put(SHIMMERUART_CMD_ERR_STORAGE_INFO_GET, "Error reading the node's storage information"); + aMap.put(SHIMMERUART_CMD_ERR_STORAGE_SESSION_GET, "Error reading a session from the node's storage"); + aMap.put(SHIMMERUART_CMD_ERR_STORAGE_FILES_GET, "Error reading a session's files from the node's storage"); + aMap.put(SHIMMERUART_CMD_ERR_STORAGE_READ, "Error reading a file from the node's storage"); + aMap.put(SHIMMERUART_CMD_ERR_STORAGE_ERASE_SET, "Error erasing sessions from the node's storage"); + aMap.put(SHIMMERUART_CMD_ERR_STORAGE_FORMAT_SET, "Error formatting the node's storage"); + //SR7 (802.15.4 radio) related errors aMap.put(SHIMMERUART_CMD_ERR_RADIO_802154_GET_SETTINGS, "Error reading 802.15.4 radio settings"); aMap.put(SHIMMERUART_CMD_ERR_RADIO_802154_SET_SETTINGS, "Error writing 802.15.4 radio settings"); diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/UartPacketDetails.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/UartPacketDetails.java index 4a8fb0982..975d54c82 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/UartPacketDetails.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/UartPacketDetails.java @@ -65,7 +65,9 @@ public static enum UART_COMPONENT { BEACON ((byte)0x08), RADIO_802154 ((byte)0x09), RADIO_BLUETOOTH ((byte)0x0A), - TEST ((byte)0x0B); + TEST ((byte)0x0B), + /** A NeuroLynQ node's session storage (DEV-1061). A GQ answers it BAD_CMD. */ + STORAGE ((byte)0x0C); private final byte command; @@ -93,6 +95,9 @@ public byte toCmdByte() { public static List listOfCompatibleVersionInfoGq = Arrays.asList(svoGqBle, svoGq802154NR, svoGq802154LR, svoGq802154Shimmer2r); public static List listOfCompatibleVersionInfoTest = Arrays.asList(svoS3Test, svoS3RTest); public static List listOfCompatibleVersionInfoUsbDfu = Arrays.asList(svoS3RUsbComms); + /** A NeuroLynQ node: Verisense hardware running the GQ 802.15.4 firmware (DEV-1061) */ + public static ShimmerVerObject svoNeuroLynQNode = new ShimmerVerObject(ShimmerVerDetails.ANY_VERSION,FW_ID.GQ_802154,ShimmerVerDetails.ANY_VERSION,ShimmerVerDetails.ANY_VERSION,ShimmerVerDetails.ANY_VERSION,ShimmerVerDetails.ANY_VERSION); + public static List listOfCompatibleVersionInfoNeuroLynQNode = Arrays.asList(svoNeuroLynQNode); /** Class listing all of the components and property combinations that can be used with the Shimmer UART commands */ public static class UART_COMPONENT_AND_PROPERTY { @@ -152,6 +157,27 @@ public static class BLUETOOTH { public static final UartComponentPropertyDetails VER = new UartComponentPropertyDetails(UART_COMPONENT.RADIO_BLUETOOTH, 0x03, PERMISSION.READ_ONLY, null, "BT_FW_VER"); } + /** + * A NeuroLynQ node's session storage (DEV-1061): verisense-firmware + * docs/VERISENSE_NEUROLYNQ_STORAGE.md section 7, and neurolynq.NeuroLynQStorageCodec for the bytes. + */ + public static class STORAGE { + /** 36 bytes: state, geometry, capacity, sessions */ + public static final UartComponentPropertyDetails INFO = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x00, PERMISSION.READ_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_INFO"); + /** GET idx u16: one session, oldest first */ + public static final UartComponentPropertyDetails SESSION = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x01, PERMISSION.READ_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_SESSION"); + /** GET handle u16, first u8, max u8: up to 40 {file, size} */ + public static final UartComponentPropertyDetails FILES = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x02, PERMISSION.READ_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_FILES"); + /** GET handle, file, offset u32, len u32: answered by a burst of these, then END */ + public static final UartComponentPropertyDetails READ = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x03, PERMISSION.READ_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_READ"); + /** The frame that ends a READ burst. Never requested. */ + public static final UartComponentPropertyDetails END = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x04, PERMISSION.READ_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_END"); + /** SET scope u8, handle u16, 'ERAS' */ + public static final UartComponentPropertyDetails ERASE = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x05, PERMISSION.WRITE_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_ERASE"); + /** SET mode u8, 'FRMT': ACKed at once, progress in INFO */ + public static final UartComponentPropertyDetails FORMAT = new UartComponentPropertyDetails(UART_COMPONENT.STORAGE, 0x06, PERMISSION.WRITE_ONLY, listOfCompatibleVersionInfoNeuroLynQNode, "STORAGE_FORMAT"); + } + public static class DEVICE_TEST { public static final UartComponentPropertyDetails MAIN_TEST = new UartComponentPropertyDetails(UART_COMPONENT.TEST, 0x00, PERMISSION.WRITE_ONLY, listOfCompatibleVersionInfoTest, "Main Test"); public static final UartComponentPropertyDetails LED_TEST = new UartComponentPropertyDetails(UART_COMPONENT.TEST, 0x01, PERMISSION.WRITE_ONLY, listOfCompatibleVersionInfoTest, "LED Test"); diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageCodec.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageCodec.java new file mode 100644 index 000000000..b1bdb44fa --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageCodec.java @@ -0,0 +1,434 @@ +package com.shimmerresearch.comms.wiredProtocol.neurolynq; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The `$` storage component 0x0C: a NeuroLynQ node's session storage, byte for byte + * (DEV-1061). + *

+ * verisense-firmware docs/VERISENSE_NEUROLYNQ_STORAGE.md section 7 is the definition and + * Includes/ASM_neurolynq_source/GqDockProtocol/gq_dock_storage.c the firmware's codec. + * neurolynq-web-sdk src/neurolynq/storage.ts is the web host's; API_00017 holds this one + * to the same vectors. All fields are little-endian. + *

    + *
  • INFO (36 bytes): what the store is and holds.
  • + *
  • SESSION: one session - its handle, state, counts, and the two folder names + * a GQ would have written it under.
  • + *
  • FILES: up to 40 {file, size} of a session.
  • + *
  • READ: answered by a burst of data frames, each {handle, file, offset} and up + * to 245 bytes, then one END frame with the count and CRC-32 of what was sent.
  • + *
  • ERASE and FORMAT carry a four-letter word, so that no stray frame + * destroys anything.
  • + *
+ * HARDWARE-VERIFY: written from the firmware's codec and its host tests; no node has + * answered one of these frames yet. + */ +public final class NeuroLynQStorageCodec { + + private NeuroLynQStorageCodec() { + } + + /** The component's sizes and limits (gq_dock_storage.h) */ + public static final int INFO_BYTES = 36; + public static final int END_BYTES = 21; + /** Data bytes in one burst frame: LEN is a byte, less COMP, PROP and {handle, file, offset} */ + public static final int READ_DATA_MAX = 245; + /** The most one READ may ask for: more than any file */ + public static final long READ_MAX = 0x01000000L; + public static final int FILES_PER_ANSWER = 40; + /** A session's files: the FILES answer counts them in a byte */ + public static final int FILES_PER_SESSION = 255; + + private static final int SESSION_FIXED = 18; + + /** INFO's state byte */ + public static final class STATE { + public static final int UNFORMATTED = 0; + /** Stock Verisense data no host has fetched: only FORMAT is accepted */ + public static final int STOCK_DATA = 1; + public static final int READY = 2; + public static final int LOGGING = 3; + public static final int FORMATTING = 4; + public static final int NAND_ERROR = 5; + /** Reading what the stock firmware left, just after boot */ + public static final int CHECKING = 6; + } + + /** INFO's flags byte */ + public static final class FLAG { + public static final int SESSION_OPEN = 0x01; + public static final int DIRECTORY_FULL = 0x02; + public static final int STORAGE_FULL = 0x04; + public static final int LOGGING_REFUSED = 0x08; + /** Unformatted over a NAND that holds a store's pages: a directory lost, not a new unit */ + public static final int ORPHANED = 0x10; + } + + /** A session's state, in SESSION */ + public static final class SESSION_STATE { + public static final int OPEN = 1; + public static final int CLOSED = 2; + public static final int RECOVERED = 3; + } + + /** How a READ burst ended, in its END frame */ + public static final class END_STATUS { + public static final int OK = 0; + /** The file ended before the length asked for */ + public static final int EOF = 1; + /** A page failed its check on the node */ + public static final int NAND_ERR = 2; + /** A new request, or a NACK, ended the burst early */ + public static final int ABORTED = 3; + } + + public static final class ERASE_SCOPE { + public static final int ONE = 0; + public static final int UP_TO = 1; + public static final int ALL = 2; + } + + public static final class FORMAT_MODE { + public static final int QUICK = 0; + public static final int FULL = 1; + } + + // ------------------------------------------------------------------ bytes + + private static int le16(byte[] p, int at) { + return (p[at] & 0xFF) | ((p[at + 1] & 0xFF) << 8); + } + + private static long le32(byte[] p, int at) { + return (le16(p, at) | ((long) le16(p, at + 2) << 16)) & 0xFFFFFFFFL; + } + + private static void put16(byte[] p, int at, int v) { + p[at] = (byte) (v & 0xFF); + p[at + 1] = (byte) ((v >>> 8) & 0xFF); + } + + private static void put32(byte[] p, int at, long v) { + put16(p, at, (int) (v & 0xFFFF)); + put16(p, at + 2, (int) ((v >>> 16) & 0xFFFF)); + } + + private static void need(byte[] p, int bytes, String what) { + if (p.length < bytes) { + throw new IllegalArgumentException(what + ": " + p.length + " bytes, need " + bytes); + } + } + + // ------------------------------------------------------------------ INFO + + public static final class Info { + public int layoutVersion; + /** {@link STATE} */ + public int state; + /** {@link FLAG} bits */ + public int flags; + /** 0 unknown, 1 TC58, 2 W25N */ + public int part; + public int pageBytes; + public int pagesPerBlock; + public int blocks; + public int badBlocks; + public long capacityKiB; + public long usedKiB; + public int sessions; + public int maxSessions; + public int generation; + /** A full format's progress, 0-100 */ + public int formatPercent; + /** Data bytes in one READ frame (245) */ + public int readFrameMax; + /** The most one READ may ask for */ + public long readMax; + public int droppedRecords; + public int nandErrors; + } + + public static Info parseInfo(byte[] p) { + need(p, INFO_BYTES, "STORAGE INFO"); + Info i = new Info(); + i.layoutVersion = p[0] & 0xFF; + i.state = p[1] & 0xFF; + i.flags = p[2] & 0xFF; + i.part = p[3] & 0xFF; + i.pageBytes = le16(p, 4); + i.pagesPerBlock = le16(p, 6); + i.blocks = le16(p, 8); + i.badBlocks = le16(p, 10); + i.capacityKiB = le32(p, 12); + i.usedKiB = le32(p, 16); + i.sessions = le16(p, 20); + i.maxSessions = le16(p, 22); + i.generation = le16(p, 24); + i.formatPercent = p[26] & 0xFF; + i.readFrameMax = p[27] & 0xFF; + i.readMax = le32(p, 28); + i.droppedRecords = le16(p, 32); + i.nandErrors = le16(p, 34); + return i; + } + + /** The device's half: INFO as a node answers it */ + public static byte[] buildInfo(Info i) { + byte[] p = new byte[INFO_BYTES]; + p[0] = (byte) i.layoutVersion; + p[1] = (byte) i.state; + p[2] = (byte) i.flags; + p[3] = (byte) i.part; + put16(p, 4, i.pageBytes); + put16(p, 6, i.pagesPerBlock); + put16(p, 8, i.blocks); + put16(p, 10, i.badBlocks); + put32(p, 12, i.capacityKiB); + put32(p, 16, i.usedKiB); + put16(p, 20, i.sessions); + put16(p, 22, i.maxSessions); + put16(p, 24, i.generation); + p[26] = (byte) i.formatPercent; + p[27] = (byte) i.readFrameMax; + put32(p, 28, i.readMax); + put16(p, 32, i.droppedRecords); + put16(p, 34, i.nandErrors); + return p; + } + + // ---------------------------------------------------------------- SESSION + + public static final class Session { + public int total; + public int index; + /** The session's sequence: stable across reboots, and what FILES, READ and ERASE take */ + public int handle; + /** {@link SESSION_STATE} */ + public int state; + /** Bit 0 sync fields, bit 1 SR68 layout, bits 4-6 the PPG mask */ + public int flags; + public int dbSession; + /** The session folder's NNN */ + public int folder; + public int fileCount; + public long totalBytes; + /** {@code _}, as a GQ names it */ + public String trialFolder; + /** {@code -} */ + public String sessionFolder; + } + + public static Session parseSession(byte[] p) { + need(p, SESSION_FIXED + 2, "STORAGE SESSION"); + int trialLen = p[SESSION_FIXED] & 0xFF; + int sessionAt = SESSION_FIXED + 1 + trialLen; + need(p, sessionAt + 1, "STORAGE SESSION"); + int sessionLen = p[sessionAt] & 0xFF; + need(p, sessionAt + 1 + sessionLen, "STORAGE SESSION"); + Session s = new Session(); + s.total = le16(p, 0); + s.index = le16(p, 2); + s.handle = le16(p, 4); + s.state = p[6] & 0xFF; + s.flags = p[7] & 0xFF; + s.dbSession = le16(p, 8); + s.folder = le16(p, 10); + s.fileCount = le16(p, 12); + s.totalBytes = le32(p, 14); + s.trialFolder = new String(p, SESSION_FIXED + 1, trialLen, StandardCharsets.ISO_8859_1); + s.sessionFolder = new String(p, sessionAt + 1, sessionLen, StandardCharsets.ISO_8859_1); + return s; + } + + public static byte[] buildSession(Session s) { + byte[] trial = s.trialFolder.getBytes(StandardCharsets.ISO_8859_1); + byte[] session = s.sessionFolder.getBytes(StandardCharsets.ISO_8859_1); + byte[] p = new byte[SESSION_FIXED + 2 + trial.length + session.length]; + put16(p, 0, s.total); + put16(p, 2, s.index); + put16(p, 4, s.handle); + p[6] = (byte) s.state; + p[7] = (byte) s.flags; + put16(p, 8, s.dbSession); + put16(p, 10, s.folder); + put16(p, 12, s.fileCount); + put32(p, 14, s.totalBytes); + p[SESSION_FIXED] = (byte) trial.length; + System.arraycopy(trial, 0, p, SESSION_FIXED + 1, trial.length); + p[SESSION_FIXED + 1 + trial.length] = (byte) session.length; + System.arraycopy(session, 0, p, SESSION_FIXED + 2 + trial.length, session.length); + return p; + } + + // ------------------------------------------------------------------ FILES + + public static final class FileEntry { + public final int file; + public final long size; + + public FileEntry(int file, long size) { + this.file = file; + this.size = size; + } + } + + public static final class Files { + public int handle; + /** The session's file count */ + public int total; + public int first; + public List files = new ArrayList(); + } + + public static Files parseFiles(byte[] p) { + need(p, 5, "STORAGE FILES"); + int n = p[4] & 0xFF; + need(p, 5 + 6 * n, "STORAGE FILES"); + Files f = new Files(); + f.handle = le16(p, 0); + f.total = p[2] & 0xFF; + f.first = p[3] & 0xFF; + for (int i = 0; i < n; i++) { + f.files.add(new FileEntry(le16(p, 5 + 6 * i), le32(p, 7 + 6 * i))); + } + f.files = Collections.unmodifiableList(f.files); + return f; + } + + public static byte[] buildFiles(Files f) { + byte[] p = new byte[5 + 6 * f.files.size()]; + put16(p, 0, f.handle); + p[2] = (byte) f.total; + p[3] = (byte) f.first; + p[4] = (byte) f.files.size(); + for (int i = 0; i < f.files.size(); i++) { + put16(p, 5 + 6 * i, f.files.get(i).file); + put32(p, 7 + 6 * i, f.files.get(i).size); + } + return p; + } + + // ------------------------------------------------------------- READ burst + + /** One data frame of a READ burst: its bytes and their place in the file */ + public static final class ReadFrame { + public final int handle; + public final int file; + public final long offset; + public final byte[] data; + + public ReadFrame(int handle, int file, long offset, byte[] data) { + this.handle = handle; + this.file = file; + this.offset = offset; + this.data = data; + } + } + + public static ReadFrame parseReadFrame(byte[] p) { + need(p, 8, "STORAGE READ frame"); + byte[] data = new byte[p.length - 8]; + System.arraycopy(p, 8, data, 0, data.length); + return new ReadFrame(le16(p, 0), le16(p, 2), le32(p, 4), data); + } + + public static byte[] buildReadFrame(ReadFrame f) { + if (f.data.length > READ_DATA_MAX) { + throw new IllegalArgumentException("STORAGE READ frame: " + f.data.length + " bytes, at most " + READ_DATA_MAX); + } + byte[] p = new byte[8 + f.data.length]; + put16(p, 0, f.handle); + put16(p, 2, f.file); + put32(p, 4, f.offset); + System.arraycopy(f.data, 0, p, 8, f.data.length); + return p; + } + + /** The frame that ends a READ burst */ + public static final class ReadEnd { + public int handle; + public int file; + public long reqOffset; + /** Data bytes the node sent, from reqOffset */ + public long bytesSent; + public long fileSize; + /** CRC-32/IEEE of the bytes sent, as java.util.zip.CRC32 computes it */ + public long crc32; + /** {@link END_STATUS} */ + public int status; + } + + public static ReadEnd parseReadEnd(byte[] p) { + need(p, END_BYTES, "STORAGE END"); + ReadEnd e = new ReadEnd(); + e.handle = le16(p, 0); + e.file = le16(p, 2); + e.reqOffset = le32(p, 4); + e.bytesSent = le32(p, 8); + e.fileSize = le32(p, 12); + e.crc32 = le32(p, 16); + e.status = p[20] & 0xFF; + return e; + } + + public static byte[] buildReadEnd(ReadEnd e) { + byte[] p = new byte[END_BYTES]; + put16(p, 0, e.handle); + put16(p, 2, e.file); + put32(p, 4, e.reqOffset); + put32(p, 8, e.bytesSent); + put32(p, 12, e.fileSize); + put32(p, 16, e.crc32); + p[20] = (byte) e.status; + return p; + } + + // ------------------------------------------------------ request arguments + + public static byte[] sessionArgs(int index) { + byte[] p = new byte[2]; + put16(p, 0, index); + return p; + } + + public static byte[] filesArgs(int handle, int first, int max) { + byte[] p = new byte[4]; + put16(p, 0, handle); + p[2] = (byte) first; + p[3] = (byte) max; + return p; + } + + public static byte[] readArgs(int handle, int file, long offset, long length) { + byte[] p = new byte[12]; + put16(p, 0, handle); + put16(p, 2, file); + put32(p, 4, offset); + put32(p, 8, length); + return p; + } + + /** The arguments of a READ, as the device's half reads them: {handle, file, offset, length} */ + public static long[] parseReadArgs(byte[] p) { + need(p, 12, "STORAGE READ args"); + return new long[] { le16(p, 0), le16(p, 2), le32(p, 4), le32(p, 8) }; + } + + public static byte[] eraseArgs(int scope, int handle) { + byte[] p = new byte[7]; + p[0] = (byte) scope; + put16(p, 1, handle); + p[3] = 'E'; + p[4] = 'R'; + p[5] = 'A'; + p[6] = 'S'; + return p; + } + + public static byte[] formatArgs(int mode) { + return new byte[] { (byte) mode, 'F', 'R', 'M', 'T' }; + } +} diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java new file mode 100644 index 000000000..d29e2e71f --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java @@ -0,0 +1,248 @@ +package com.shimmerresearch.comms.wiredProtocol.neurolynq; + +import java.util.ArrayList; +import java.util.List; +import java.util.zip.CRC32; + +import com.shimmerresearch.comms.wiredProtocol.AbstractCommsProtocolWired; +import com.shimmerresearch.comms.wiredProtocol.AbstractCommsProtocolWired.ReadBurst; +import com.shimmerresearch.comms.wiredProtocol.DockException; +import com.shimmerresearch.comms.wiredProtocol.ErrorCodesWiredProtocol; +import com.shimmerresearch.comms.wiredProtocol.UartPacketDetails.UART_COMPONENT_AND_PROPERTY; +import com.shimmerresearch.comms.wiredProtocol.UartRxPacketObject; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.END_STATUS; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.FileEntry; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Files; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Info; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ReadEnd; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ReadFrame; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Session; + +/** + * A NeuroLynQ node's session storage, over its dock link: the host's half of the `$` + * storage component 0x0C (DEV-1061), as neurolynq-web-sdk's NeuroLynqNodeClient is the + * web host's. The rules for reading a file are that client's, so both hosts take the + * same thing as a file read whole. + *

+ * A node that answers INFO with BAD_CMD has no storage - a GQ, or a node whose firmware + * predates it - and {@link #isStorageAbsent(DockException)} says so. + *

+ * HARDWARE-VERIFY: tested against SimulatedNeuroLynQNodeSerialPort and the firmware's + * codec vectors; no node has answered it. + */ +public class NeuroLynQStorageProtocol { + + /** The most one READ asks for: between windows, the host takes stock of what arrived */ + public static final long READ_WINDOW_BYTES = 512L * 1024L; + /** Between two frames of a READ burst (storage spec section 7, "Timeouts a host should allow") */ + public static final long READ_STALL_MS = 1000; + /** Windows running that brought nothing before a file read gives up */ + public static final int MAX_STALLS = 5; + + /** Progress through one file */ + public interface ReadProgressListener { + void onProgress(long bytesRead, long fileSize); + } + + /** A file the node could not give whole */ + public static class StorageReadException extends Exception { + private static final long serialVersionUID = 1L; + /** {@link END_STATUS} of the last window, or -1 when none ended */ + public final int endStatus; + + public StorageReadException(String message, int endStatus) { + super(message); + this.endStatus = endStatus; + } + } + + /** One READ's worth: what arrived in order, and how the node ended it */ + public static final class ReadWindow { + public byte[] data = new byte[0]; + public ReadEnd end; + public boolean complete; + public boolean crcMismatch; + public int duplicates; + public int pastGap; + public int badFrames; + } + + private final AbstractCommsProtocolWired mWired; + private long mReadStallMs = READ_STALL_MS; + + public NeuroLynQStorageProtocol(AbstractCommsProtocolWired wired) { + mWired = wired; + } + + public void setReadStallMs(long readStallMs) { + mReadStallMs = readStallMs; + } + + /** INFO answered BAD_CMD: the device has no storage component, and nothing to download */ + public static boolean isStorageAbsent(DockException de) { + return de.mErrorCodeLowLevel == ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_RESPONSE_BAD_CMD; + } + + // ------------------------------------------------------------- listing + + public Info info() throws DockException { + return NeuroLynQStorageCodec.parseInfo(mWired.processShimmerGetCommand( + UART_COMPONENT_AND_PROPERTY.STORAGE.INFO, ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_INFO_GET)); + } + + public Session session(int index) throws DockException { + return NeuroLynQStorageCodec.parseSession(mWired.processShimmerGetCommand( + UART_COMPONENT_AND_PROPERTY.STORAGE.SESSION, ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_SESSION_GET, + NeuroLynQStorageCodec.sessionArgs(index))); + } + + /** Every session on the node, oldest first */ + public List sessions() throws DockException { + List out = new ArrayList(); + int total = info().sessions; + for (int i = 0; i < total; i++) { + Session s = session(i); + out.add(s); + total = s.total; + } + return out; + } + + public Files files(int handle, int first, int max) throws DockException { + return NeuroLynQStorageCodec.parseFiles(mWired.processShimmerGetCommand( + UART_COMPONENT_AND_PROPERTY.STORAGE.FILES, ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_FILES_GET, + NeuroLynQStorageCodec.filesArgs(handle, first, max))); + } + + /** Every file of session {@code handle}, in order */ + public List allFiles(int handle) throws DockException { + List out = new ArrayList(); + int first = 0; + for (;;) { + Files f = files(handle, first, NeuroLynQStorageCodec.FILES_PER_ANSWER); + out.addAll(f.files); + first += f.files.size(); + if (f.files.isEmpty() || first >= f.total) { + return out; + } + } + } + + // ------------------------------------------------------------- reading + + /** + * One READ: what arrived of {@code length} bytes from {@code offset}, in order and + * without a gap, and the END the node closed it with. Frames past a gap are counted + * but not kept: the next window asks for them again. + */ + public ReadWindow readWindow(int handle, int file, long offset, long length) throws DockException { + long limit = offset + length; + ReadWindow w = new ReadWindow(); + java.io.ByteArrayOutputStream kept = new java.io.ByteArrayOutputStream(); + CRC32 crc = new CRC32(); + long next = offset; + boolean gap = false; + + ReadBurst burst = mWired.readBurst(UART_COMPONENT_AND_PROPERTY.STORAGE.READ, + NeuroLynQStorageCodec.readArgs(handle, file, offset, length), + UART_COMPONENT_AND_PROPERTY.STORAGE.END.mPropertyByte, mReadStallMs, burstBudgetMs(length), + ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_READ); + w.badFrames = burst.crcErrors; + for (UartRxPacketObject p : burst.packets) { + if (p.mUartPropertyByte == UART_COMPONENT_AND_PROPERTY.STORAGE.END.mPropertyByte) { + ReadEnd end = NeuroLynQStorageCodec.parseReadEnd(p.getPayload()); + // An earlier READ's, one this host stopped waiting for + if (end.handle != handle || end.file != file || end.reqOffset != offset) { + continue; + } + w.end = end; + break; + } + ReadFrame f = NeuroLynQStorageCodec.parseReadFrame(p.getPayload()); + if (f.handle != handle || f.file != file || f.offset >= limit) { + continue; + } + if (gap || f.offset > next) { + gap = true; + w.pastGap++; + } else if (f.offset + f.data.length <= next) { + w.duplicates++; + } else { + // Whole, or overlapping bytes already held: keep what is new + int from = (int) (next - f.offset); + int to = (int) Math.min(f.data.length, limit - f.offset); + kept.write(f.data, from, to - from); + crc.update(f.data, from, to - from); + next += to - from; + } + } + w.data = kept.toByteArray(); + boolean counted = w.end != null && !gap && w.end.bytesSent == w.data.length; + w.crcMismatch = counted && w.end.crc32 != crc.getValue(); + w.complete = counted && !w.crcMismatch + && (w.end.status == END_STATUS.OK || w.end.status == END_STATUS.EOF); + return w; + } + + /** + * One file, whole: READ from wherever the last window stopped until {@code size} + * bytes have arrived in order. As neurolynq-web-sdk's readFile(): a file is whole once + * every byte has arrived, END or not - each frame carries its place and a `$` CRC of + * its own - and a window whose END contradicts it, by count or CRC-32, is not kept. + */ + public byte[] readFile(int handle, int file, long size, ReadProgressListener progress) + throws DockException, StorageReadException { + if (size > Integer.MAX_VALUE) { + throw new StorageReadException("session " + handle + " file " + file + ": " + size + " bytes", -1); + } + byte[] out = new byte[(int) size]; + long have = 0; + int stalls = 0; + while (have < size) { + ReadWindow w = readWindow(handle, file, have, Math.min(READ_WINDOW_BYTES, size - have)); + if (w.end != null && w.end.fileSize != size) { + throw new StorageReadException("session " + handle + " file " + file + ": the node sizes it " + + w.end.fileSize + ", not " + size, w.end.status); + } + int kept = w.crcMismatch ? 0 : w.data.length; + System.arraycopy(w.data, 0, out, (int) have, kept); + have += kept; + if (kept > 0) { + stalls = 0; + } else if (++stalls > MAX_STALLS) { + int status = (w.end == null) ? -1 : w.end.status; + String why = w.crcMismatch ? "the CRC-32 at " + have + " disagrees " + (MAX_STALLS + 1) + " times" + : status == END_STATUS.NAND_ERR ? "the node cannot read the page at " + have + : "nothing at " + have + " in " + (MAX_STALLS + 1) + " tries"; + throw new StorageReadException("session " + handle + " file " + file + ": " + why, status); + } + if (progress != null) { + progress.onProgress(have, size); + } + } + return out; + } + + // ---------------------------------------------------- erasing, formatting + + /** + * Remove sessions from the node's directory: {@code scope} ONE is session + * {@code handle}, UP_TO every session up to and including it, ALL every one. The node + * refuses (BAD_ARG) a session open for logging. + */ + public void erase(int scope, int handle) throws DockException { + mWired.processShimmerSetCommand(UART_COMPONENT_AND_PROPERTY.STORAGE.ERASE, + NeuroLynQStorageCodec.eraseArgs(scope, handle), ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_ERASE_SET); + } + + /** Start a format: ACKed at once, its progress in INFO. It destroys what the node holds. */ + public void format(int mode) throws DockException { + mWired.processShimmerSetCommand(UART_COMPONENT_AND_PROPERTY.STORAGE.FORMAT, + NeuroLynQStorageCodec.formatArgs(mode), ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_FORMAT_SET); + } + + /** Long enough for {@code bytes} at 100 kB/s, under a third of the link's rate, and 5 s besides */ + static long burstBudgetMs(long bytes) { + return 5000 + (bytes + 99) / 100; + } +} diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java new file mode 100644 index 000000000..5b4d5aa5e --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java @@ -0,0 +1,520 @@ +package com.shimmerresearch.comms.wiredProtocol.neurolynq; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32; + +import com.shimmerresearch.comms.serialPortInterface.AbstractSerialPortHal; +import com.shimmerresearch.comms.serialPortInterface.SerialPortListener; +import com.shimmerresearch.comms.wiredProtocol.ShimmerCrc; +import com.shimmerresearch.comms.wiredProtocol.UartPacketDetails.UART_COMPONENT; +import com.shimmerresearch.comms.wiredProtocol.UartPacketDetails.UART_PACKET_CMD; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.END_STATUS; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ERASE_SCOPE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.FileEntry; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Files; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Info; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ReadEnd; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ReadFrame; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.STATE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.SESSION_STATE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Session; +import com.shimmerresearch.exceptions.ShimmerException; + +/** + * A NeuroLynQ node's storage on a simulated dock link (DEV-1061), to test a host without + * a node: it answers the `$` storage component as the firmware does + * (verisense-firmware GqDockProtocol/gq_dock_storage.c), from sessions held in memory, + * and can drop, corrupt and cut short what it sends. neurolynq-web-sdk's + * SimulatedNeuroLynqNode is the web host's. + *

+ * Like the firmware, a READ goes out a NAND page at a time - {@link #PAGE_PAYLOAD_BYTES} + * of the file, as frames of at most 245 bytes that restart at each page boundary - on a + * thread of its own, as a serial port's bytes arrive on its reader's. Anything but the + * storage component is answered BAD_CMD. + *

+ * It sits with the production code, not the tests, because Shimmer-Advance-API's dock + * tests drive it too, and a project's test classes are not visible to another's. + */ +public class SimulatedNeuroLynQNodeSerialPort extends AbstractSerialPortHal { + + /** A TC58 page's payload (storage spec section 5.2): where the firmware's frames restart */ + public static final int PAGE_PAYLOAD_BYTES = 4080; + + private static final byte COMPONENT = UART_COMPONENT.STORAGE.toCmdByte(); + private static final byte PROP_INFO = 0x00; + private static final byte PROP_SESSION = 0x01; + private static final byte PROP_FILES = 0x02; + private static final byte PROP_READ = 0x03; + private static final byte PROP_END = 0x04; + private static final byte PROP_ERASE = 0x05; + private static final byte PROP_FORMAT = 0x06; + + /** A session the node holds */ + public static final class SimSession { + public int handle; + public int dbSession; + public int folder; + public int flags; + public int state = SESSION_STATE.CLOSED; + public String trialFolder; + public String sessionFolder; + public final List files = new ArrayList(); + } + + private final List mSessions = new ArrayList(); + private final ExecutorService mDevice = Executors.newSingleThreadExecutor(); + private final Object mRxLock = new Object(); + private byte[] mRx = new byte[0]; + private transient SerialPortListener mListener; + private boolean mConnected = false; + + /** False: the component is unknown, as on a GQ or a node without storage */ + public volatile boolean hasStorage = true; + public volatile int state = STATE.READY; + /** Data frames, counted over every burst from 0, that are not sent */ + public final Set dropFrames = new HashSet(); + /** Data frames whose `$` CRC is sent wrong */ + public final Set corruptFrames = new HashSet(); + /** Bursts, counted from 0, whose END is not sent */ + public final Set loseEnds = new HashSet(); + /** Bursts answered with a CRC-32 in END that does not match what was sent */ + public final Set wrongCrcEnds = new HashSet(); + /** Bursts cut off by a BAD_ARG after this many data frames; -1 none */ + public volatile int nackAfterFrames = -1; + /** Each page's bytes arrive in reads of this size, split across frames; 0 a page at a time */ + public volatile int chunkBytes = 0; + /** Requests seen, in order: the first argument byte of each READ's offset is in the log */ + public final List requests = new ArrayList(); + + private int mFramesSent = 0; + private int mBursts = 0; + + public SimSession addSession(int handle, int dbSession, String trialFolder, String sessionFolder, byte[]... files) { + SimSession s = new SimSession(); + s.handle = handle; + s.dbSession = dbSession; + s.trialFolder = trialFolder; + s.sessionFolder = sessionFolder; + for (byte[] f : files) { + s.files.add(f); + } + synchronized (mSessions) { + mSessions.add(s); + } + return s; + } + + public List sessions() { + synchronized (mSessions) { + return new ArrayList(mSessions); + } + } + + // ------------------------------------------------------------ the link + + @Override + public void connect() throws ShimmerException { + mConnected = true; + } + + @Override + public void disconnect() throws ShimmerException { + mConnected = false; + } + + @Override + public void closeSafely() throws ShimmerException { + mConnected = false; + } + + @Override + public void clearSerialPortRxBuffer() throws ShimmerException { + synchronized (mRxLock) { + mRx = new byte[0]; + } + } + + @Override + public boolean isSerialPortReaderStarted() { + return true; + } + + @Override + public void setVerboseMode(boolean verboseMode, boolean isDebugMode) { + } + + @Override + public boolean bytesAvailableToBeRead() throws ShimmerException { + return availableBytes() > 0; + } + + @Override + public int availableBytes() throws ShimmerException { + synchronized (mRxLock) { + return mRx.length; + } + } + + @Override + public boolean isConnected() { + return mConnected; + } + + @Override + public boolean isDisonnected() { + return !mConnected; + } + + @Override + public void registerSerialPortRxEventCallback(SerialPortListener listener) { + mListener = listener; + } + + /** Blocks, as a port does, until {@code numBytes} have arrived or the timeout passes */ + @Override + public byte[] rxBytes(int numBytes) throws ShimmerException { + long deadline = System.currentTimeMillis() + mSerialPortTimeout; + synchronized (mRxLock) { + while (mRx.length < numBytes) { + long wait = deadline - System.currentTimeMillis(); + if (wait <= 0) { + throw new ShimmerException("simulated node: " + mRx.length + " of " + numBytes + " bytes"); + } + try { + mRxLock.wait(wait); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ShimmerException("simulated node: interrupted"); + } + } + byte[] out = new byte[numBytes]; + System.arraycopy(mRx, 0, out, 0, numBytes); + byte[] rest = new byte[mRx.length - numBytes]; + System.arraycopy(mRx, numBytes, rest, 0, rest.length); + mRx = rest; + return out; + } + } + + /** The host's frame: answered on the device's thread, as a node answers its port */ + @Override + public void txBytes(byte[] frame) throws ShimmerException { + final byte[] copy = frame.clone(); + mDevice.execute(new Runnable() { + @Override + public void run() { + answer(copy); + } + }); + } + + /** Stop the device's and the reader's threads */ + public void shutdown() { + mStopped = true; + mReader.interrupt(); + mDevice.shutdownNow(); + try { + mDevice.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** The device's bytes into the port. The reader thread tells the listener, as a port's + * reader does, so a listener waiting in rxBytes() for the rest of a frame never waits + * on the thread that is to send it. */ + private void deliver(byte[] bytes) { + synchronized (mRxLock) { + byte[] joined = new byte[mRx.length + bytes.length]; + System.arraycopy(mRx, 0, joined, 0, mRx.length); + System.arraycopy(bytes, 0, joined, mRx.length, bytes.length); + mRx = joined; + mRxLock.notifyAll(); + } + } + + private volatile boolean mStopped = false; + private final Thread mReader = new Thread(new Runnable() { + @Override + public void run() { + while (!mStopped) { + int available; + synchronized (mRxLock) { + while (mRx.length == 0 && !mStopped) { + try { + mRxLock.wait(100); + } catch (InterruptedException e) { + return; + } + } + available = mRx.length; + } + SerialPortListener listener = mListener; + if (listener != null && available > 0) { + listener.serialPortRxEvent(available); + } + } + } + }, "SimulatedNeuroLynQNode reader"); + + { + mReader.setDaemon(true); + mReader.start(); + } + + private void deliverChunked(byte[] bytes) { + int chunk = chunkBytes; + if (chunk <= 0) { + deliver(bytes); + return; + } + for (int at = 0; at < bytes.length; at += chunk) { + byte[] part = new byte[Math.min(chunk, bytes.length - at)]; + System.arraycopy(bytes, at, part, 0, part.length); + deliver(part); + } + } + + // ----------------------------------------------------------- the frames + + private static byte[] withCrc(byte[] body) { + byte[] crc = ShimmerCrc.shimmerUartCrcCalc(body, body.length); + byte[] out = new byte[body.length + 2]; + System.arraycopy(body, 0, out, 0, body.length); + System.arraycopy(crc, 0, out, body.length, 2); + return out; + } + + private static byte[] response(byte property, byte[] payload) { + byte[] body = new byte[5 + payload.length]; + body[0] = '$'; + body[1] = UART_PACKET_CMD.DATA_RESPONSE.toCmdByte(); + body[2] = (byte) (2 + payload.length); + body[3] = COMPONENT; + body[4] = property; + System.arraycopy(payload, 0, body, 5, payload.length); + return withCrc(body); + } + + private static byte[] shortResponse(UART_PACKET_CMD cmd) { + return withCrc(new byte[] { '$', cmd.toCmdByte() }); + } + + private void answer(byte[] frame) { + if (frame.length < 4 || frame[0] != '$') { + return; + } + if (!ShimmerCrc.shimmerUartCrcCheck(frame)) { + deliver(shortResponse(UART_PACKET_CMD.BAD_CRC_RESPONSE)); + return; + } + byte cmd = frame[1]; + int len = (frame.length >= 7) ? (frame[2] & 0xFF) : 0; + if (len < 2 || frame[3] != COMPONENT || !hasStorage) { + deliver(shortResponse(UART_PACKET_CMD.BAD_CMD_RESPONSE)); + return; + } + byte prop = frame[4]; + byte[] args = new byte[len - 2]; + System.arraycopy(frame, 5, args, 0, args.length); + synchronized (requests) { + requests.add(String.format("%02X/%02X/%02X", cmd, frame[3], prop)); + } + try { + if (cmd == UART_PACKET_CMD.READ.toCmdByte()) { + answerGet(prop, args); + } else if (cmd == UART_PACKET_CMD.WRITE.toCmdByte()) { + answerSet(prop, args); + } else { + deliver(shortResponse(UART_PACKET_CMD.BAD_CMD_RESPONSE)); + } + } catch (IllegalArgumentException e) { + deliver(shortResponse(UART_PACKET_CMD.BAD_ARG_RESPONSE)); + } + } + + private SimSession sessionWithHandle(int handle) { + for (SimSession s : sessions()) { + if (s.handle == handle) { + return s; + } + } + throw new IllegalArgumentException("no session " + handle); + } + + private void answerGet(byte prop, byte[] args) { + List sessions = sessions(); + if (prop == PROP_INFO) { + Info i = new Info(); + i.layoutVersion = 1; + i.state = state; + i.part = 1; + i.pageBytes = 4096; + i.pagesPerBlock = 64; + i.blocks = 2048; + i.capacityKiB = 522240; + i.sessions = sessions.size(); + i.maxSessions = 103; + i.generation = 1; + i.readFrameMax = NeuroLynQStorageCodec.READ_DATA_MAX; + i.readMax = NeuroLynQStorageCodec.READ_MAX; + deliver(response(PROP_INFO, NeuroLynQStorageCodec.buildInfo(i))); + } else if (prop == PROP_SESSION) { + if (args.length != 2) { + throw new IllegalArgumentException("SESSION args"); + } + int index = (args[0] & 0xFF) | ((args[1] & 0xFF) << 8); + if (index >= sessions.size()) { + throw new IllegalArgumentException("SESSION index"); + } + SimSession s = sessions.get(index); + Session out = new Session(); + out.total = sessions.size(); + out.index = index; + out.handle = s.handle; + out.state = s.state; + out.flags = s.flags; + out.dbSession = s.dbSession; + out.folder = s.folder; + out.fileCount = s.files.size(); + long total = 0; + for (byte[] f : s.files) { + total += f.length; + } + out.totalBytes = total; + out.trialFolder = s.trialFolder; + out.sessionFolder = s.sessionFolder; + deliver(response(PROP_SESSION, NeuroLynQStorageCodec.buildSession(out))); + } else if (prop == PROP_FILES) { + if (args.length != 4) { + throw new IllegalArgumentException("FILES args"); + } + SimSession s = sessionWithHandle((args[0] & 0xFF) | ((args[1] & 0xFF) << 8)); + int first = args[2] & 0xFF; + int max = Math.min(args[3] & 0xFF, NeuroLynQStorageCodec.FILES_PER_ANSWER); + Files f = new Files(); + f.handle = s.handle; + f.total = s.files.size(); + f.first = first; + List entries = new ArrayList(); + for (int i = first; i < s.files.size() && entries.size() < max; i++) { + entries.add(new FileEntry(i, s.files.get(i).length)); + } + f.files = entries; + deliver(response(PROP_FILES, NeuroLynQStorageCodec.buildFiles(f))); + } else if (prop == PROP_READ) { + long[] a = NeuroLynQStorageCodec.parseReadArgs(args); + SimSession s = sessionWithHandle((int) a[0]); + if (a[1] >= s.files.size()) { + throw new IllegalArgumentException("READ file"); + } + byte[] file = s.files.get((int) a[1]); + if (a[2] > file.length || a[3] == 0 || a[3] > NeuroLynQStorageCodec.READ_MAX) { + throw new IllegalArgumentException("READ range"); + } + burst(s.handle, (int) a[1], file, a[2], a[3]); + } else { + deliver(shortResponse(UART_PACKET_CMD.BAD_CMD_RESPONSE)); + } + } + + /** A READ's burst: a page's frames at a time, then END, as gq_dock_storage.c sends it */ + private void burst(int handle, int file, byte[] bytes, long offset, long length) { + int burst = mBursts++; + long limit = Math.min(bytes.length, offset + length); + CRC32 crc = new CRC32(); + long sent = 0; + int framesThisBurst = 0; + long at = offset; + while (at < limit) { + long pageEnd = Math.min(limit, (at / PAGE_PAYLOAD_BYTES + 1) * PAGE_PAYLOAD_BYTES); + ByteArrayOutputStream page = new ByteArrayOutputStream(); + while (at < pageEnd) { + if (nackAfterFrames >= 0 && framesThisBurst == nackAfterFrames) { + deliverChunked(page.toByteArray()); + deliver(shortResponse(UART_PACKET_CMD.BAD_ARG_RESPONSE)); + return; + } + int n = (int) Math.min(NeuroLynQStorageCodec.READ_DATA_MAX, pageEnd - at); + byte[] data = new byte[n]; + System.arraycopy(bytes, (int) at, data, 0, n); + byte[] frame = response(PROP_READ, NeuroLynQStorageCodec.buildReadFrame(new ReadFrame(handle, file, at, data))); + int index = mFramesSent++; + framesThisBurst++; + crc.update(data); + sent += n; + at += n; + if (corruptFrames.contains(index)) { + frame[frame.length - 1] ^= 0x5A; + } + if (!dropFrames.contains(index)) { + page.write(frame, 0, frame.length); + } + } + deliverChunked(page.toByteArray()); + } + if (loseEnds.contains(burst)) { + return; + } + ReadEnd end = new ReadEnd(); + end.handle = handle; + end.file = file; + end.reqOffset = offset; + end.bytesSent = sent; + end.fileSize = bytes.length; + end.crc32 = wrongCrcEnds.contains(burst) ? (crc.getValue() ^ 1L) : crc.getValue(); + end.status = (offset + length > bytes.length) ? END_STATUS.EOF : END_STATUS.OK; + deliver(response(PROP_END, NeuroLynQStorageCodec.buildReadEnd(end))); + } + + private void answerSet(byte prop, byte[] args) { + if (prop == PROP_ERASE) { + if (args.length != 7 || args[3] != 'E' || args[4] != 'R' || args[5] != 'A' || args[6] != 'S') { + throw new IllegalArgumentException("ERASE args"); + } + int scope = args[0] & 0xFF; + int handle = (args[1] & 0xFF) | ((args[2] & 0xFF) << 8); + synchronized (mSessions) { + if (scope == ERASE_SCOPE.ALL) { + mSessions.clear(); + } else { + int upTo = -1; + for (int i = 0; i < mSessions.size(); i++) { + if (mSessions.get(i).handle == handle) { + upTo = i; + } + } + if (upTo < 0 || mSessions.get(upTo).state == SESSION_STATE.OPEN) { + throw new IllegalArgumentException("ERASE handle"); + } + if (scope == ERASE_SCOPE.ONE) { + mSessions.remove(upTo); + } else if (scope == ERASE_SCOPE.UP_TO) { + mSessions.subList(0, upTo + 1).clear(); + } else { + throw new IllegalArgumentException("ERASE scope"); + } + } + } + deliver(shortResponse(UART_PACKET_CMD.ACK_RESPONSE)); + } else if (prop == PROP_FORMAT) { + if (args.length != 5 || args[1] != 'F' || args[2] != 'R' || args[3] != 'M' || args[4] != 'T') { + throw new IllegalArgumentException("FORMAT args"); + } + synchronized (mSessions) { + mSessions.clear(); + } + state = STATE.READY; + deliver(shortResponse(UART_PACKET_CMD.ACK_RESPONSE)); + } else { + deliver(shortResponse(UART_PACKET_CMD.BAD_CMD_RESPONSE)); + } + } +} diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00017_NeuroLynQStorageCodecTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00017_NeuroLynQStorageCodecTest.java new file mode 100644 index 000000000..be6b2564b --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00017_NeuroLynQStorageCodecTest.java @@ -0,0 +1,182 @@ +package com.shimmerresearch.comms.wiredProtocol.neurolynq; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.junit.Test; + +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.FileEntry; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Files; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Info; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ReadEnd; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ReadFrame; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.STATE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Session; + +/** + * The storage component's codec, byte for byte (DEV-1061). + *

+ * The vectors are neurolynq-web-sdk's (tests/neurolynq/storage.test.ts), which are + * verisense-firmware's own codec test's (Tools/tests/gq_dock/test_gq_dock_storage.c): + * three implementations held to the same bytes in the same places. + */ +public class API_00017_NeuroLynQStorageCodecTest { + + private static byte[] hex(String s) { + String[] parts = s.trim().split("\\s+"); + byte[] out = new byte[parts.length]; + for (int i = 0; i < parts.length; i++) { + out[i] = (byte) Integer.parseInt(parts[i], 16); + } + return out; + } + + private static void assertThrowsArgument(Runnable r) { + try { + r.run(); + fail("no exception"); + } catch (IllegalArgumentException expected) { + // A short answer is refused rather than read past + } + } + + private static final byte[] INFO = hex("01 02 04 01 00 10 40 00 00 08 34 12 00 f8 07 00 45 23 01 00" + + " 02 00 67 00 cd ab 39 f5 00 00 00 01 02 01 04 03"); + + @Test + public void infoIs36BytesLaidOutAsTheFirmwareLaysItOut() { + assertEquals(NeuroLynQStorageCodec.INFO_BYTES, INFO.length); + Info i = NeuroLynQStorageCodec.parseInfo(INFO); + assertEquals(1, i.layoutVersion); + assertEquals(STATE.READY, i.state); + assertEquals(0x04, i.flags); + assertEquals(1, i.part); + assertEquals(4096, i.pageBytes); + assertEquals(64, i.pagesPerBlock); + assertEquals(2048, i.blocks); + assertEquals(0x1234, i.badBlocks); + assertEquals(0x0007f800L, i.capacityKiB); + assertEquals(0x00012345L, i.usedKiB); + assertEquals(2, i.sessions); + assertEquals(103, i.maxSessions); + assertEquals(0xabcd, i.generation); + assertEquals(57, i.formatPercent); + assertEquals(245, i.readFrameMax); + assertEquals(0x01000000L, i.readMax); + assertEquals(0x0102, i.droppedRecords); + assertEquals(0x0304, i.nandErrors); + assertArrayEquals(INFO, NeuroLynQStorageCodec.buildInfo(i)); + assertThrowsArgument(() -> NeuroLynQStorageCodec.parseInfo(Arrays.copyOf(INFO, 35))); + } + + private static Session session() { + Session s = new Session(); + s.total = 2; + s.index = 1; + s.handle = 8; + s.state = 3; + s.flags = 0x31; + s.dbSession = 0x0506; + s.folder = 12; + s.fileCount = 4; + s.totalBytes = 0x01020304L; + s.trialFolder = "trial1_1727000000"; + s.sessionFolder = "Shimmer_ABCD-012"; + return s; + } + + @Test + public void sessionIs18FixedBytesAndTwoCountedNames() { + byte[] bytes = NeuroLynQStorageCodec.buildSession(session()); + assertEquals(53, bytes.length); + assertArrayEquals(hex("02 00 01 00 08 00 03 31 06 05 0c 00 04 00 04 03 02 01"), Arrays.copyOf(bytes, 18)); + assertEquals(17, bytes[18]); + assertEquals(16, bytes[36]); + assertEquals("Shimmer_ABCD-012", new String(bytes, 37, 16, StandardCharsets.ISO_8859_1)); + Session back = NeuroLynQStorageCodec.parseSession(bytes); + assertEquals(8, back.handle); + assertEquals(0x0506, back.dbSession); + assertEquals(0x01020304L, back.totalBytes); + assertEquals("trial1_1727000000", back.trialFolder); + assertEquals("Shimmer_ABCD-012", back.sessionFolder); + assertThrowsArgument(() -> NeuroLynQStorageCodec.parseSession(Arrays.copyOf(bytes, 52))); + assertThrowsArgument(() -> NeuroLynQStorageCodec.parseSession(Arrays.copyOf(bytes, 30))); + } + + @Test + public void filesAreHandleTotalFirstNAndNFileSizePairs() { + Files f = new Files(); + f.handle = 7; + f.total = 4; + f.first = 1; + f.files = Arrays.asList(new FileEntry(1, 245), new FileEntry(2, 1), new FileEntry(3, 70000)); + byte[] bytes = NeuroLynQStorageCodec.buildFiles(f); + assertArrayEquals(hex("07 00 04 01 03 01 00 f5 00 00 00 02 00 01 00 00 00 03 00 70 11 01 00"), bytes); + Files back = NeuroLynQStorageCodec.parseFiles(bytes); + assertEquals(7, back.handle); + assertEquals(4, back.total); + assertEquals(1, back.first); + assertEquals(3, back.files.size()); + assertEquals(70000L, back.files.get(2).size); + assertThrowsArgument(() -> NeuroLynQStorageCodec.parseFiles(Arrays.copyOf(bytes, 22))); + } + + @Test + public void aReadFrameIsHandleFileOffsetAndItsBytes() { + byte[] bytes = NeuroLynQStorageCodec.buildReadFrame(new ReadFrame(3, 0, 4080, hex("de ad be ef"))); + assertArrayEquals(hex("03 00 00 00 f0 0f 00 00 de ad be ef"), bytes); + ReadFrame back = NeuroLynQStorageCodec.parseReadFrame(bytes); + assertEquals(3, back.handle); + assertEquals(4080L, back.offset); + assertArrayEquals(hex("de ad be ef"), back.data); + } + + /** LEN is a byte: COMP, PROP, the frame's 8 bytes and at most 245 of data */ + @Test + public void aReadFrameCarriesAtMost245Bytes() { + assertEquals(253, NeuroLynQStorageCodec.buildReadFrame(new ReadFrame(1, 0, 0, new byte[245])).length); + assertThrowsArgument(() -> NeuroLynQStorageCodec.buildReadFrame(new ReadFrame(1, 0, 0, new byte[246]))); + } + + @Test + public void endIs21Bytes() { + ReadEnd e = new ReadEnd(); + e.handle = 3; + e.file = 0; + e.reqOffset = 4000; + e.bytesSent = 10000; + e.fileSize = 70000; + e.crc32 = 0xcbf43926L; + e.status = 1; + byte[] bytes = NeuroLynQStorageCodec.buildReadEnd(e); + assertArrayEquals(hex("03 00 00 00 a0 0f 00 00 10 27 00 00 70 11 01 00 26 39 f4 cb 01"), bytes); + assertEquals(NeuroLynQStorageCodec.END_BYTES, bytes.length); + ReadEnd back = NeuroLynQStorageCodec.parseReadEnd(bytes); + assertEquals(0xcbf43926L, back.crc32); + assertEquals(70000L, back.fileSize); + assertThrowsArgument(() -> NeuroLynQStorageCodec.parseReadEnd(Arrays.copyOf(bytes, 20))); + } + + /** 0xCBF43926 is CRC-32/IEEE's check value, over "123456789": the END's CRC is java.util.zip's */ + @Test + public void theEndsCrcIsJavasCrc32() { + java.util.zip.CRC32 crc = new java.util.zip.CRC32(); + crc.update("123456789".getBytes(StandardCharsets.US_ASCII)); + assertEquals(0xcbf43926L, crc.getValue()); + } + + @Test + public void theRequestArgumentsAreTheFirmwaresOrder() { + assertArrayEquals(hex("02 01"), NeuroLynQStorageCodec.sessionArgs(0x0102)); + assertArrayEquals(hex("07 00 28 28"), NeuroLynQStorageCodec.filesArgs(7, 40, 40)); + byte[] read = NeuroLynQStorageCodec.readArgs(0xfffe, 2, 0x00012345L, 0x01000000L); + assertArrayEquals(hex("fe ff 02 00 45 23 01 00 00 00 00 01"), read); + assertArrayEquals(new long[] { 0xfffe, 2, 0x00012345L, 0x01000000L }, NeuroLynQStorageCodec.parseReadArgs(read)); + assertArrayEquals(hex("01 03 02 45 52 41 53"), NeuroLynQStorageCodec.eraseArgs(1, 0x0203)); + assertArrayEquals(hex("01 46 52 4d 54"), NeuroLynQStorageCodec.formatArgs(1)); + } +} diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java new file mode 100644 index 000000000..78fb26cf8 --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java @@ -0,0 +1,218 @@ +package com.shimmerresearch.comms.wiredProtocol.neurolynq; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.shimmerresearch.comms.wiredProtocol.CommsProtocolWiredShimmerViaDock; +import com.shimmerresearch.comms.wiredProtocol.DockException; +import com.shimmerresearch.comms.wiredProtocol.ErrorCodesWiredProtocol; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.ERASE_SCOPE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.FORMAT_MODE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.FileEntry; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Info; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.STATE; +import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.Session; + +/** + * The host's half of a NeuroLynQ node's storage (DEV-1061), over the wired protocol, against + * a simulated node that sends as the firmware does - a NAND page's frames at a time, a READ's + * burst on the reader's thread - and loses, corrupts and cuts short what it sends. + *

+ * The rule under test is neurolynq-web-sdk's: a file is whole once every byte has arrived + * in order, with or without the END that closes the burst, and a window whose END + * contradicts what arrived is not kept. + */ +public class API_00018_NeuroLynQStorageProtocolTest { + + private SimulatedNeuroLynQNodeSerialPort mNode; + private CommsProtocolWiredShimmerViaDock mWired; + private NeuroLynQStorageProtocol mStorage; + + /** Bytes that differ everywhere, so a frame put in the wrong place cannot look right */ + private static byte[] file(int length, long seed) { + byte[] b = new byte[length]; + new Random(seed).nextBytes(b); + return b; + } + + private static final byte[] FILE_A = file(3 * SimulatedNeuroLynQNodeSerialPort.PAGE_PAYLOAD_BYTES + 1234, 1); + private static final byte[] FILE_B = file(700, 2); + private static final byte[] FILE_C = file(100000, 3); + + @Before + public void setUp() { + mNode = new SimulatedNeuroLynQNodeSerialPort(); + mNode.addSession(5, 7, "trial1_1727000000", "NodeA-000", FILE_A, FILE_B); + mNode.addSession(6, 8, "trial1_1727000000", "NodeA-001", FILE_C); + mWired = new CommsProtocolWiredShimmerViaDock("SIM", "node", mNode); + mStorage = new NeuroLynQStorageProtocol(mWired); + mStorage.setReadStallMs(250); + } + + @After + public void tearDown() { + mNode.shutdown(); + } + + @Test + public void itListsTheNodesSessionsAndFiles() throws Exception { + Info info = mStorage.info(); + assertEquals(STATE.READY, info.state); + assertEquals(2, info.sessions); + + List sessions = mStorage.sessions(); + assertEquals(2, sessions.size()); + assertEquals(5, sessions.get(0).handle); + assertEquals("NodeA-001", sessions.get(1).sessionFolder); + assertEquals(FILE_A.length + FILE_B.length, sessions.get(0).totalBytes); + + List files = mStorage.allFiles(5); + assertEquals(2, files.size()); + assertEquals(FILE_A.length, files.get(0).size); + assertEquals(FILE_B.length, files.get(1).size); + } + + @Test + public void aFileIsReadWhole() throws Exception { + final List progress = new ArrayList(); + byte[] got = mStorage.readFile(5, 0, FILE_A.length, (read, size) -> progress.add(read)); + assertArrayEquals(FILE_A, got); + assertEquals(Long.valueOf(FILE_A.length), progress.get(progress.size() - 1)); + assertArrayEquals(FILE_B, mStorage.readFile(5, 1, FILE_B.length, null)); + assertArrayEquals(FILE_C, mStorage.readFile(6, 0, FILE_C.length, null)); + } + + /** Frames cut anywhere across the port's reads parse as whole frames */ + @Test + public void framesSplitAcrossReadsParse() throws Exception { + mNode.chunkBytes = 7; + assertArrayEquals(FILE_A, mStorage.readFile(5, 0, FILE_A.length, null)); + } + + /** A lost frame is a gap: what came after it is asked for again */ + @Test + public void aDroppedFrameIsAskedForAgain() throws Exception { + mNode.dropFrames.add(3); + mNode.dropFrames.add(40); + assertArrayEquals(FILE_A, mStorage.readFile(5, 0, FILE_A.length, null)); + assertTrue("more than one READ", readCount() > 1); + } + + /** A frame whose `$` CRC fails is lost, not believed */ + @Test + public void aCorruptFrameIsAskedForAgain() throws Exception { + mNode.corruptFrames.add(5); + assertArrayEquals(FILE_A, mStorage.readFile(5, 0, FILE_A.length, null)); + assertTrue("more than one READ", readCount() > 1); + } + + /** Every byte arrived: the file is whole without the END that would have closed it */ + @Test + public void aLostEndLosesNothing() throws Exception { + mNode.loseEnds.add(0); + assertArrayEquals(FILE_B, mStorage.readFile(5, 1, FILE_B.length, null)); + } + + /** An END whose CRC-32 disagrees with what arrived: the window is not kept */ + @Test + public void aWindowTheEndContradictsIsReadAgain() throws Exception { + mNode.wrongCrcEnds.add(0); + assertArrayEquals(FILE_B, mStorage.readFile(5, 1, FILE_B.length, null)); + assertEquals(2, readCount()); + } + + /** A NACK in the middle of a burst ends it, as any NACK ends a request */ + @Test + public void aNackInABurstThrows() { + mNode.nackAfterFrames = 4; + try { + mStorage.readFile(5, 0, FILE_A.length, null); + fail("no exception"); + } catch (DockException de) { + assertEquals(ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_RESPONSE_BAD_ARG, de.mErrorCodeLowLevel); + assertEquals(ErrorCodesWiredProtocol.SHIMMERUART_CMD_ERR_STORAGE_READ, de.mErrorCode); + } catch (Exception e) { + fail("threw " + e); + } + } + + /** Nothing arriving at all is a failure, after a few windows, not a hang */ + @Test + public void aNodeThatSendsNothingFailsTheRead() throws Exception { + for (int i = 0; i < 100; i++) { + mNode.dropFrames.add(i); + mNode.loseEnds.add(i); + } + try { + mStorage.readFile(5, 1, FILE_B.length, null); + fail("no exception"); + } catch (NeuroLynQStorageProtocol.StorageReadException e) { + assertEquals(-1, e.endStatus); + assertEquals(NeuroLynQStorageProtocol.MAX_STALLS + 1, readCount()); + } + } + + @Test + public void sessionsAreErasedAndTheNodeFormatted() throws Exception { + mStorage.erase(ERASE_SCOPE.ONE, 5); + assertEquals(1, mStorage.info().sessions); + assertEquals(6, mStorage.session(0).handle); + mStorage.format(FORMAT_MODE.QUICK); + assertEquals(0, mStorage.info().sessions); + } + + @Test + public void erasingASessionTheNodeDoesNotHoldIsRefused() { + try { + mStorage.erase(ERASE_SCOPE.ONE, 99); + fail("no exception"); + } catch (DockException de) { + assertEquals(ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_RESPONSE_BAD_ARG, de.mErrorCodeLowLevel); + } + } + + /** A GQ, or a node whose firmware predates storage: INFO answers BAD_CMD */ + @Test + public void aDeviceWithoutStorageSaysSo() { + mNode.hasStorage = false; + try { + mStorage.info(); + fail("no exception"); + } catch (DockException de) { + assertTrue(NeuroLynQStorageProtocol.isStorageAbsent(de)); + } + } + + @Test + public void aNackIsNotAbsence() { + try { + mStorage.session(9); + fail("no exception"); + } catch (DockException de) { + assertFalse(NeuroLynQStorageProtocol.isStorageAbsent(de)); + } + } + + private int readCount() { + int n = 0; + synchronized (mNode.requests) { + for (String r : mNode.requests) { + if (r.endsWith("/0C/03")) { + n++; + } + } + } + return n; + } +} From 427e3136c45b9397b871e2a7fed09bb1147682bb Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Fri, 25 Sep 2026 13:24:05 +0100 Subject: [PATCH 3/6] DEV-1061: parse a large serial read in a loop, not a call per packet processRxBuf() handled the first packet of a read, then called itself on the rest. Each packet cost one level of stack, and so did each byte of noise it dropped. No response the dock protocol had sent before came close to the limit. A NeuroLynQ node's READ burst does. It sends 4 KB a page, and a read taken after the reader was held up holds everything that arrived in the meantime. At the node's 329 kB/s that is some 1,300 frames a second, and in a test a read of about 1,700 frames ran the reader's thread out of stack. The StackOverflowError is an Error, not an Exception, so serialPortRxEvent()'s catch did not stop it and the thread died. The port then went quiet for good: no later response was parsed. processRxBuf() is now a loop that takes one packet a pass. processFirstPacket() is the old body, less its final self-call, and it parses a packet exactly as before. Nothing else changes, for any dock. The other hosts of the $ protocol were checked. shimmer-web-sdk drains in a loop (drainByteStream), neurolynq-web-sdk walks a cursor, and pyshimmer reads by length. The C# and Swift APIs share the CRC but not the dock protocol. None of them need this. API_00018's aReadHoldingAWholeBurstParses sends a 512 KiB window, about 2,100 frames, as one read (the simulated node's new oneReadPerBurst). With the recursive parser, the read fails and the stack overflow is in the log. With the loop, the file arrives whole and the port answers INFO afterwards. Co-Authored-By: Claude Opus 5.5 --- .../AbstractCommsProtocolWired.java | 27 +++++++++++------ .../SimulatedNeuroLynQNodeSerialPort.java | 30 ++++++++++++++++--- ...PI_00018_NeuroLynQStorageProtocolTest.java | 15 ++++++++++ 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java index 47d4fa041..0fa42df5d 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java @@ -643,8 +643,24 @@ public void serialPortRxEvent(int eventLength){ } } - //TODO not setup to handle streaming via dock connector -> add additional capability? + /** + * Every packet in a read, one a pass. A loop and not a call per packet: a read can + * hold thousands - a NeuroLynQ node's READ burst sends 4 KB a page, and a read taken + * after the reader has been held up holds all that arrived meanwhile - and a call per + * packet, or per byte of noise, ran the reader's thread out of stack (DEV-1061). + */ private void processRxBuf(byte[] rxBuf) throws ShimmerException { + byte[] next = rxBuf; + while(next.length>0){ + processFirstPacket(next); + // What is left of the read after that packet + next = carriedRxBuf; + carriedRxBuf = new byte[]{}; + } + } + + //TODO not setup to handle streaming via dock connector -> add additional capability? + private void processFirstPacket(byte[] rxBuf) throws ShimmerException { byte headerByte = rxBuf[0]; if(headerByte==UartPacketDetails.PACKET_HEADER.toCharArray()[0]){ @@ -758,14 +774,7 @@ else if(rxBuf.length0){ - byte[] tempBuf = carriedRxBuf; - carriedRxBuf = new byte[]{}; - processRxBuf(tempBuf); - } - + // Any bytes left in carriedRxBuf are processRxBuf()'s next pass } /** */ diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java index 5b4d5aa5e..78e203da4 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java @@ -90,6 +90,9 @@ public static final class SimSession { public volatile int nackAfterFrames = -1; /** Each page's bytes arrive in reads of this size, split across frames; 0 a page at a time */ public volatile int chunkBytes = 0; + /** A burst's pages and END arrive in one read, as they do when the host's reader is held + * up while the node sends */ + public volatile boolean oneReadPerBurst = false; /** Requests seen, in order: the first argument byte of each READ's offset is in the log */ public final List requests = new ArrayList(); @@ -433,13 +436,15 @@ private void burst(int handle, int file, byte[] bytes, long offset, long length) long sent = 0; int framesThisBurst = 0; long at = offset; + ByteArrayOutputStream held = oneReadPerBurst ? new ByteArrayOutputStream() : null; while (at < limit) { long pageEnd = Math.min(limit, (at / PAGE_PAYLOAD_BYTES + 1) * PAGE_PAYLOAD_BYTES); ByteArrayOutputStream page = new ByteArrayOutputStream(); while (at < pageEnd) { if (nackAfterFrames >= 0 && framesThisBurst == nackAfterFrames) { - deliverChunked(page.toByteArray()); - deliver(shortResponse(UART_PACKET_CMD.BAD_ARG_RESPONSE)); + emit(held, page.toByteArray()); + emit(held, shortResponse(UART_PACKET_CMD.BAD_ARG_RESPONSE)); + release(held); return; } int n = (int) Math.min(NeuroLynQStorageCodec.READ_DATA_MAX, pageEnd - at); @@ -458,9 +463,10 @@ private void burst(int handle, int file, byte[] bytes, long offset, long length) page.write(frame, 0, frame.length); } } - deliverChunked(page.toByteArray()); + emit(held, page.toByteArray()); } if (loseEnds.contains(burst)) { + release(held); return; } ReadEnd end = new ReadEnd(); @@ -471,7 +477,23 @@ private void burst(int handle, int file, byte[] bytes, long offset, long length) end.fileSize = bytes.length; end.crc32 = wrongCrcEnds.contains(burst) ? (crc.getValue() ^ 1L) : crc.getValue(); end.status = (offset + length > bytes.length) ? END_STATUS.EOF : END_STATUS.OK; - deliver(response(PROP_END, NeuroLynQStorageCodec.buildReadEnd(end))); + emit(held, response(PROP_END, NeuroLynQStorageCodec.buildReadEnd(end))); + release(held); + } + + /** A burst's bytes to the port now, or kept for one read at its end */ + private void emit(ByteArrayOutputStream held, byte[] bytes) { + if (held != null) { + held.write(bytes, 0, bytes.length); + } else { + deliverChunked(bytes); + } + } + + private void release(ByteArrayOutputStream held) { + if (held != null && held.size() > 0) { + deliver(held.toByteArray()); + } } private void answerSet(byte prop, byte[] args) { diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java index 78fb26cf8..f089a03e9 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java @@ -163,6 +163,21 @@ public void aNodeThatSendsNothingFailsTheRead() throws Exception { } } + /** + * A whole window's burst in one read - the reader held up while the node sent it - is + * parsed frame by frame. The parser called itself once a frame, and a read of a couple + * of thousand ran its thread out of stack: the port said nothing more after that. + */ + @Test + public void aReadHoldingAWholeBurstParses() throws Exception { + byte[] big = file((int) NeuroLynQStorageProtocol.READ_WINDOW_BYTES, 5); + mNode.addSession(7, 9, "trial1_1727000000", "NodeA-002", big); + mNode.oneReadPerBurst = true; + assertArrayEquals(big, mStorage.readFile(7, 0, big.length, null)); + assertEquals(1, readCount()); + assertEquals(3, mStorage.info().sessions); + } + @Test public void sessionsAreErasedAndTheNodeFormatted() throws Exception { mStorage.erase(ERASE_SCOPE.ONE, 5); From c53798e36c0ae59a17fe10c4d62589f09fded6e4 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Fri, 25 Sep 2026 13:24:28 +0100 Subject: [PATCH 4/6] DEV-1061: cancel a node's download, and report its progress An eight-hour SR68 session is 68 MB, about three and a half minutes over the dock link. The import's CANCEL has to be able to stop that, and its progress panel needs something to show while it runs. NeuroLynQStorageProtocol - cancel() stops readFile() before its next window, which is under two seconds away at the link's rate. readFile() then throws ReadCancelledException, a StorageReadException. - A window is never cut short, so the burst it asked for is over before anything else goes out. - An instance stays cancelled; a new one reads again. MsgDock: MSG_ID_DOCK_NODE_DOWNLOAD_PROGRESS (60, free) carries a node's mBytesDone of mBytesTotal during an SD scan. A node has no card, so the scan is where its sessions come off it. Shimmer-Advance-API's BasicDock sends it and DockManager passes it on. The simulated node now ends a burst when a request arrives, with END ABORTED, before it answers that request. This is what the firmware does (gq_dock_link.c, storage spec section 7). It can also pause between pages, as a node does while it reads the next page from NAND, and it records each END it sends. API_00018 (3 new tests): - A cancel lands between windows. The link stays usable after it: the next INFO and READ are answered. - A read cancelled before it starts sends nothing. - A request made while a burst is still running is answered within INFO's 500 ms, and the burst ends ABORTED. Unended, that paced burst would outlast the request. Co-Authored-By: Claude Opus 5.5 --- .../neurolynq/NeuroLynQStorageProtocol.java | 29 +++++++++ .../SimulatedNeuroLynQNodeSerialPort.java | 32 +++++++++- .../com/shimmerresearch/driver/MsgDock.java | 8 +++ ...PI_00018_NeuroLynQStorageProtocolTest.java | 60 +++++++++++++++++++ 4 files changed, 127 insertions(+), 2 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java index d29e2e71f..d40f3f268 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/NeuroLynQStorageProtocol.java @@ -56,6 +56,15 @@ public StorageReadException(String message, int endStatus) { } } + /** Thrown by {@link NeuroLynQStorageProtocol#readFile readFile()} once {@link #cancel()} is called */ + public static class ReadCancelledException extends StorageReadException { + private static final long serialVersionUID = 1L; + + public ReadCancelledException(String message) { + super(message, -1); + } + } + /** One READ's worth: what arrived in order, and how the node ended it */ public static final class ReadWindow { public byte[] data = new byte[0]; @@ -69,6 +78,7 @@ public static final class ReadWindow { private final AbstractCommsProtocolWired mWired; private long mReadStallMs = READ_STALL_MS; + private volatile boolean mCancelled = false; public NeuroLynQStorageProtocol(AbstractCommsProtocolWired wired) { mWired = wired; @@ -78,6 +88,21 @@ public void setReadStallMs(long readStallMs) { mReadStallMs = readStallMs; } + /** + * Stop {@link #readFile readFile()} before its next window, which is under two seconds + * away at the link's rate: it throws {@link ReadCancelledException}. A window is never + * cut short, so the burst it asked for is over before anything else is sent, and the + * link is left as a finished read leaves it. An instance stays cancelled; read again + * with a new one. + */ + public void cancel() { + mCancelled = true; + } + + public boolean isCancelled() { + return mCancelled; + } + /** INFO answered BAD_CMD: the device has no storage component, and nothing to download */ public static boolean isStorageAbsent(DockException de) { return de.mErrorCodeLowLevel == ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_RESPONSE_BAD_CMD; @@ -199,6 +224,10 @@ public byte[] readFile(int handle, int file, long size, ReadProgressListener pro long have = 0; int stalls = 0; while (have < size) { + if (mCancelled) { + throw new ReadCancelledException("session " + handle + " file " + file + ": cancelled at " + have + + " of " + size + " bytes"); + } ReadWindow w = readWindow(handle, file, have, Math.min(READ_WINDOW_BYTES, size - have)); if (w.end != null && w.end.fileSize != size) { throw new StorageReadException("session " + handle + " file " + file + ": the node sizes it " diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java index 78e203da4..3639c1ecd 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java @@ -2,12 +2,14 @@ import java.io.ByteArrayOutputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.CRC32; import com.shimmerresearch.comms.serialPortInterface.AbstractSerialPortHal; @@ -93,11 +95,17 @@ public static final class SimSession { /** A burst's pages and END arrive in one read, as they do when the host's reader is held * up while the node sends */ public volatile boolean oneReadPerBurst = false; + /** A pause after each page of a burst, as a node takes to read the next from NAND; 0 none */ + public volatile int pageDelayMs = 0; + /** Each END sent, by its status (END_STATUS), in order */ + public final List endStatuses = Collections.synchronizedList(new ArrayList()); /** Requests seen, in order: the first argument byte of each READ's offset is in the log */ public final List requests = new ArrayList(); private int mFramesSent = 0; private int mBursts = 0; + /** Requests sent and not yet taken up by the device's thread: one ends a burst in progress */ + private final AtomicInteger mQueued = new AtomicInteger(); public SimSession addSession(int handle, int dbSession, String trialFolder, String sessionFolder, byte[]... files) { SimSession s = new SimSession(); @@ -210,9 +218,11 @@ public byte[] rxBytes(int numBytes) throws ShimmerException { @Override public void txBytes(byte[] frame) throws ShimmerException { final byte[] copy = frame.clone(); + mQueued.incrementAndGet(); mDevice.execute(new Runnable() { @Override public void run() { + mQueued.decrementAndGet(); answer(copy); } }); @@ -428,7 +438,11 @@ private void answerGet(byte prop, byte[] args) { } } - /** A READ's burst: a page's frames at a time, then END, as gq_dock_storage.c sends it */ + /** + * A READ's burst: a page's frames at a time, then END, as gq_dock_storage.c sends it. A + * request arriving meanwhile ends it at the next page, with END ABORTED, before that + * request is answered (gq_dock_link.c; storage spec section 7). + */ private void burst(int handle, int file, byte[] bytes, long offset, long length) { int burst = mBursts++; long limit = Math.min(bytes.length, offset + length); @@ -436,8 +450,13 @@ private void burst(int handle, int file, byte[] bytes, long offset, long length) long sent = 0; int framesThisBurst = 0; long at = offset; + boolean aborted = false; ByteArrayOutputStream held = oneReadPerBurst ? new ByteArrayOutputStream() : null; while (at < limit) { + if (mQueued.get() > 0) { + aborted = true; + break; + } long pageEnd = Math.min(limit, (at / PAGE_PAYLOAD_BYTES + 1) * PAGE_PAYLOAD_BYTES); ByteArrayOutputStream page = new ByteArrayOutputStream(); while (at < pageEnd) { @@ -464,6 +483,14 @@ private void burst(int handle, int file, byte[] bytes, long offset, long length) } } emit(held, page.toByteArray()); + if (pageDelayMs > 0) { + try { + Thread.sleep(pageDelayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } } if (loseEnds.contains(burst)) { release(held); @@ -476,7 +503,8 @@ private void burst(int handle, int file, byte[] bytes, long offset, long length) end.bytesSent = sent; end.fileSize = bytes.length; end.crc32 = wrongCrcEnds.contains(burst) ? (crc.getValue() ^ 1L) : crc.getValue(); - end.status = (offset + length > bytes.length) ? END_STATUS.EOF : END_STATUS.OK; + end.status = aborted ? END_STATUS.ABORTED : (offset + length > bytes.length) ? END_STATUS.EOF : END_STATUS.OK; + endStatuses.add(end.status); emit(held, response(PROP_END, NeuroLynQStorageCodec.buildReadEnd(end))); release(held); } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/MsgDock.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/MsgDock.java index 4538e54f1..00f15a8ef 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/MsgDock.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/MsgDock.java @@ -76,6 +76,10 @@ public class MsgDock { public final static int MSG_ID_DOCK_JOB_STARTED_PER_DOCK = 56; public final static int MSG_ID_DOCK_JOB_SUCCESS_PER_DOCK = 57; public final static int MSG_ID_DOCK_JOB_FAIL_PER_DOCK = 58; + + /** A NeuroLynQ node's sessions coming off it during an SD scan, as it has no card: + * {@link #mBytesDone} of {@link #mBytesTotal} (DEV-1061) */ + public final static int MSG_ID_DOCK_NODE_DOWNLOAD_PROGRESS = 60; public final static int MSG_ID_DOCKMANAGER_INITIALIZED_SUCCESS = 91; public final static int MSG_ID_DOCKMANAGER_INITIALIZED_FAIL = 92; @@ -242,6 +246,7 @@ public class MsgDock { aMap.put(MSG_ID_DOCK_JOB_STARTED_PER_DOCK, "MSG_ID_DOCK_JOB_STARTED_PER_DOCK"); aMap.put(MSG_ID_DOCK_JOB_SUCCESS_PER_DOCK, "MSG_ID_DOCK_JOB_SUCCESS_PER_DOCK"); aMap.put(MSG_ID_DOCK_JOB_FAIL_PER_DOCK, "MSG_ID_DOCK_JOB_FAIL_PER_DOCK"); + aMap.put(MSG_ID_DOCK_NODE_DOWNLOAD_PROGRESS, "MSG_ID_DOCK_NODE_DOWNLOAD_PROGRESS"); aMap.put(MSG_ID_DOCKMANAGER_INITIALIZED_SUCCESS, "MSG_ID_DOCKMANAGER_INITIALIZED_SUCCESS"); aMap.put(MSG_ID_DOCKMANAGER_INITIALIZED_FAIL, "MSG_ID_DOCKMANAGER_INITIALIZED_FAIL"); @@ -346,6 +351,9 @@ public class MsgDock { public int mFwImageTotalSize = 0; public int mFwImageWriteProgress = 0; public float mFwImageWriteSpeed = 0; + /** A NeuroLynQ node's download (MSG_ID_DOCK_NODE_DOWNLOAD_PROGRESS): bytes read so far, and in all */ + public long mBytesDone = 0; + public long mBytesTotal = 0; public String mMessage = ""; public double mValue; /** Just used for Shimmer4 BSL programming as each stage takes longer no needs more GUI feedback */ diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java index f089a03e9..7ecdb4ab9 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java @@ -3,10 +3,12 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Random; @@ -163,6 +165,34 @@ public void aNodeThatSendsNothingFailsTheRead() throws Exception { } } + /** + * A cancel lands between windows: the one already asked for is never cut short. The + * next request on the link is answered, whatever of the burst is still arriving: a + * response is matched by its component and property, and a READ frame is neither. + */ + @Test + public void aCancelledReadStopsAtTheNextWindow() throws Exception { + byte[] big = file((int) NeuroLynQStorageProtocol.READ_WINDOW_BYTES + 5000, 4); + mNode.addSession(7, 9, "trial1_1727000000", "NodeA-002", big); + final List progress = new ArrayList(); + try { + mStorage.readFile(7, 0, big.length, (read, size) -> { + progress.add(read); + mStorage.cancel(); + }); + fail("no exception"); + } catch (NeuroLynQStorageProtocol.ReadCancelledException expected) { + assertEquals(1, readCount()); + assertEquals(1, progress.size()); + assertTrue(progress.get(0) < big.length); + } + assertTrue(mStorage.isCancelled()); + + NeuroLynQStorageProtocol again = new NeuroLynQStorageProtocol(mWired); + assertEquals(3, again.info().sessions); + assertArrayEquals(FILE_B, again.readFile(5, 1, FILE_B.length, null)); + } + /** * A whole window's burst in one read - the reader held up while the node sent it - is * parsed frame by frame. The parser called itself once a frame, and a read of a couple @@ -178,6 +208,36 @@ public void aReadHoldingAWholeBurstParses() throws Exception { assertEquals(3, mStorage.info().sessions); } + /** + * The host stopped waiting before the burst ended, and its next request is answered + * all the same: the node ends a burst for a new request, with END ABORTED, and what of + * the burst was already on its way is not taken for the answer. Unended, this burst + * would outlast INFO's 500 ms. + */ + @Test + public void aRequestEndsABurstInProgress() throws Exception { + byte[] big = file((int) NeuroLynQStorageProtocol.READ_WINDOW_BYTES, 6); + mNode.addSession(7, 9, "trial1_1727000000", "NodeA-002", big); + mNode.pageDelayMs = 10; + mStorage.setReadStallMs(1); + NeuroLynQStorageProtocol.ReadWindow w = mStorage.readWindow(7, 0, 0, big.length); + assertNull("the host stopped waiting before the burst ended", w.end); + assertEquals(3, new NeuroLynQStorageProtocol(mWired).info().sessions); + assertEquals(Arrays.asList(NeuroLynQStorageCodec.END_STATUS.ABORTED), mNode.endStatuses); + } + + /** Cancelled before it starts, a read sends nothing */ + @Test + public void aReadCancelledFirstSendsNothing() throws Exception { + mStorage.cancel(); + try { + mStorage.readFile(5, 0, FILE_A.length, null); + fail("no exception"); + } catch (NeuroLynQStorageProtocol.ReadCancelledException expected) { + assertEquals(0, readCount()); + } + } + @Test public void sessionsAreErasedAndTheNodeFormatted() throws Exception { mStorage.erase(ERASE_SCOPE.ONE, 5); From 5bc7950f26cf53074f0ffdba5b01bd44caf5f776 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Fri, 25 Sep 2026 13:58:52 +0100 Subject: [PATCH 5/6] DEV-1061: the simulated node answers the dock's MAC read A BasicDock reads a docked device's details again after a CLEAR, and starts with its MAC: a device whose MAC it cannot read counts as undocked. A real node answers the read (DEV-1047). The simulated node now does too, once it is given a MAC, so Shimmer-Advance-API's dock tests can run a whole CLEAR. It still answers anything else outside the storage component with BAD_CMD. API_00018 checks the answer through readMacId(). Co-Authored-By: Claude Opus 5.5 --- .../SimulatedNeuroLynQNodeSerialPort.java | 18 +++++++++++++++++- ...API_00018_NeuroLynQStorageProtocolTest.java | 7 +++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java index 3639c1ecd..dea181889 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java @@ -15,6 +15,8 @@ import com.shimmerresearch.comms.serialPortInterface.AbstractSerialPortHal; import com.shimmerresearch.comms.serialPortInterface.SerialPortListener; import com.shimmerresearch.comms.wiredProtocol.ShimmerCrc; +import com.shimmerresearch.comms.wiredProtocol.UartComponentPropertyDetails; +import com.shimmerresearch.comms.wiredProtocol.UartPacketDetails.UART_COMPONENT_AND_PROPERTY; import com.shimmerresearch.comms.wiredProtocol.UartPacketDetails.UART_COMPONENT; import com.shimmerresearch.comms.wiredProtocol.UartPacketDetails.UART_PACKET_CMD; import com.shimmerresearch.comms.wiredProtocol.neurolynq.NeuroLynQStorageCodec.END_STATUS; @@ -50,6 +52,7 @@ public class SimulatedNeuroLynQNodeSerialPort extends AbstractSerialPortHal { public static final int PAGE_PAYLOAD_BYTES = 4080; private static final byte COMPONENT = UART_COMPONENT.STORAGE.toCmdByte(); + private static final UartComponentPropertyDetails MAC = UART_COMPONENT_AND_PROPERTY.MAIN_PROCESSOR.MAC; private static final byte PROP_INFO = 0x00; private static final byte PROP_SESSION = 0x01; private static final byte PROP_FILES = 0x02; @@ -79,6 +82,9 @@ public static final class SimSession { /** False: the component is unknown, as on a GQ or a node without storage */ public volatile boolean hasStorage = true; + /** The node's MAC, answered to the dock's MAC read as a node answers it; null, BAD_CMD. + * A dock reads it before and after a CLEAR, and refuses a node it cannot identify. */ + public volatile byte[] macId = null; public volatile int state = STATE.READY; /** Data frames, counted over every burst from 0, that are not sent */ public final Set dropFrames = new HashSet(); @@ -306,11 +312,15 @@ private static byte[] withCrc(byte[] body) { } private static byte[] response(byte property, byte[] payload) { + return response(COMPONENT, property, payload); + } + + private static byte[] response(byte component, byte property, byte[] payload) { byte[] body = new byte[5 + payload.length]; body[0] = '$'; body[1] = UART_PACKET_CMD.DATA_RESPONSE.toCmdByte(); body[2] = (byte) (2 + payload.length); - body[3] = COMPONENT; + body[3] = component; body[4] = property; System.arraycopy(payload, 0, body, 5, payload.length); return withCrc(body); @@ -330,6 +340,12 @@ private void answer(byte[] frame) { } byte cmd = frame[1]; int len = (frame.length >= 7) ? (frame[2] & 0xFF) : 0; + byte[] mac = macId; + if (mac != null && len == 2 && cmd == UART_PACKET_CMD.READ.toCmdByte() + && frame[3] == MAC.mComponentByte && frame[4] == MAC.mPropertyByte) { + deliver(response(MAC.mComponentByte, MAC.mPropertyByte, mac)); + return; + } if (len < 2 || frame[3] != COMPONENT || !hasStorage) { deliver(shortResponse(UART_PACKET_CMD.BAD_CMD_RESPONSE)); return; diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java index 7ecdb4ab9..33ea403ac 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java @@ -269,6 +269,13 @@ public void aDeviceWithoutStorageSaysSo() { } } + /** A dock reads a node's MAC around a CLEAR, so the simulated node answers it as a node does */ + @Test + public void theNodeAnswersItsMacWhenItHasOne() throws Exception { + mNode.macId = new byte[] { 0x00, 0x06, 0x66, (byte) 0xAB, (byte) 0xCD, (byte) 0xEF }; + assertEquals("000666ABCDEF", mWired.readMacId()); + } + @Test public void aNackIsNotAbsence() { try { From dd40ed4c513421179924fa9eb930758296fff63f Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Fri, 25 Sep 2026 14:01:43 +0100 Subject: [PATCH 6/6] DEV-1061: keep a NACK that comes back before the write returns shimmerUartCommandTxRx() cleared mThrownException after txPacket(). A NACK answered and parsed before txPacket() returned was stored there, then wiped. The command waited out SERIAL_PORT_TIMEOUT and failed with TIMEOUT 500 ms later, instead of at once with the NACK the device sent. The three NoWait commands already clear it before sending. This one now does the same. It showed up in the NeuroLynQ node's dock tests, which failed from time to time against a simulated node that answers within microseconds. It matters beyond the tests too: a node whose firmware predates storage answers INFO with BAD_CMD, which is how the host knows it has nothing to import. Reported as a timeout, it would be taken for a failed scan. The simulated node's new answerBeforeTheWriteReturns answers a request, and has the answer parsed, before the write returns. That is the window the NACK was lost in. API_00018's aNackFasterThanTheWriteIsNotLost fails with the old order and passes with the new. API_00018's link also takes a dock-style unique ID now ("Verisense.01.01"): DockException takes the ID apart, and "node" made a timeout throw ArrayIndexOutOfBounds instead. Co-Authored-By: Claude Opus 5.5 --- .../AbstractCommsProtocolWired.java | 4 +++- .../SimulatedNeuroLynQNodeSerialPort.java | 15 ++++++++++++- ...PI_00018_NeuroLynQStorageProtocolTest.java | 21 ++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java index 0fa42df5d..6da1e6fa8 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/AbstractCommsProtocolWired.java @@ -377,9 +377,11 @@ protected byte[] shimmerUartSetMemCommand(UartComponentPropertyDetails msgArg, i */ protected byte[] shimmerUartCommandTxRx(UART_PACKET_CMD packetCmd, UartComponentPropertyDetails msgArg, byte[] payload) throws DockException { mListOfUartRxPacketObjects.clear(); + // Before the request goes, as the NoWait commands do: a NACK can be back and parsed + // before txPacket() returns, and clearing after it lost that NACK to a timeout + mThrownException = null; txPacket(packetCmd, msgArg, payload); - mThrownException = null; return waitForResponse(packetCmd, msgArg); } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java index dea181889..84a314199 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/SimulatedNeuroLynQNodeSerialPort.java @@ -101,6 +101,9 @@ public static final class SimSession { /** A burst's pages and END arrive in one read, as they do when the host's reader is held * up while the node sends */ public volatile boolean oneReadPerBurst = false; + /** A request is answered, and its answer parsed, before the host's write of it returns: + * the most a node can outrun the host, and the window a host must not lose an answer in */ + public volatile boolean answerBeforeTheWriteReturns = false; /** A pause after each page of a burst, as a node takes to read the next from NAND; 0 none */ public volatile int pageDelayMs = 0; /** Each END sent, by its status (END_STATUS), in order */ @@ -224,6 +227,15 @@ public byte[] rxBytes(int numBytes) throws ShimmerException { @Override public void txBytes(byte[] frame) throws ShimmerException { final byte[] copy = frame.clone(); + if (answerBeforeTheWriteReturns) { + answer(copy); + SerialPortListener listener = mListener; + int available = availableBytes(); + if (listener != null && available > 0) { + listener.serialPortRxEvent(available); + } + return; + } mQueued.incrementAndGet(); mDevice.execute(new Runnable() { @Override @@ -266,7 +278,8 @@ public void run() { while (!mStopped) { int available; synchronized (mRxLock) { - while (mRx.length == 0 && !mStopped) { + // Answered on the writer's thread instead, so nothing is parsed twice + while ((mRx.length == 0 || answerBeforeTheWriteReturns) && !mStopped) { try { mRxLock.wait(100); } catch (InterruptedException e) { diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java index 33ea403ac..ede423a63 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/comms/wiredProtocol/neurolynq/API_00018_NeuroLynQStorageProtocolTest.java @@ -57,7 +57,8 @@ public void setUp() { mNode = new SimulatedNeuroLynQNodeSerialPort(); mNode.addSession(5, 7, "trial1_1727000000", "NodeA-000", FILE_A, FILE_B); mNode.addSession(6, 8, "trial1_1727000000", "NodeA-001", FILE_C); - mWired = new CommsProtocolWiredShimmerViaDock("SIM", "node", mNode); + // As a dock names a docked device, which a DockException takes apart + mWired = new CommsProtocolWiredShimmerViaDock("SIM", "Verisense.01.01", mNode); mStorage = new NeuroLynQStorageProtocol(mWired); mStorage.setReadStallMs(250); } @@ -276,6 +277,24 @@ public void theNodeAnswersItsMacWhenItHasOne() throws Exception { assertEquals("000666ABCDEF", mWired.readMacId()); } + /** + * A NACK back before the request's write has returned is not lost. The driver cleared + * its last exception after sending, so a node that answered that fast looked like one + * that never answered: a timeout 500 ms on, and a node without storage not seen as one. + */ + @Test + public void aNackFasterThanTheWriteIsNotLost() { + mNode.hasStorage = false; + mNode.answerBeforeTheWriteReturns = true; + try { + mStorage.info(); + fail("no exception"); + } catch (DockException de) { + assertEquals(ErrorCodesWiredProtocol.SHIMMERUART_COMM_ERR_RESPONSE_BAD_CMD, de.mErrorCodeLowLevel); + assertTrue(NeuroLynQStorageProtocol.isStorageAbsent(de)); + } + } + @Test public void aNackIsNotAbsence() { try {