From e5491a0094fa5448b646a7be38cf5fcc9e44fd52 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Tue, 29 Sep 2026 11:06:42 +0100 Subject: [PATCH 1/4] DEV-1095: anchor each SD file on its first packet's own counter time The SD header's initial timestamp is the RTC when the file was created (sdFileSyncTs), not the first packet's time: the firmware captures that in firstTsForSdFile but never writes it back. Subtracting the first packet's raw timestamp pinned each file's first packet to the header time, erasing a creation-to-first-packet lead that differs per file. File 000 is created after sampling starts, so a Shimmer3R recording stepped back -158.8 ms at the 000 -> 001 split. SdTimestampAnchor rebuilds the first packet's full counter value from the header's high bits and the packet's 24-bit timestamp (signed, wrap- corrected), exact within +-256 s of the header whatever moment the firmware records. Both timestamp paths - ShimmerObject and SensorShimmerClock - use it. 2-byte counters, a zero initial timestamp and a lead over 10 s keep the previous behaviour. Co-Authored-By: Claude Opus 5.5 --- .../shimmerresearch/driver/ShimmerObject.java | 27 ++-- .../driverUtilities/SdTimestampAnchor.java | 106 +++++++++++++ .../sensors/SensorShimmerClock.java | 9 +- .../driver/API_00005_Suite_ShimmerDriver.java | 4 +- .../API_00012_SdTimestampAnchorTest.java | 141 ++++++++++++++++++ 5 files changed, 272 insertions(+), 15 deletions(-) create mode 100644 ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java create mode 100644 ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java index 4056fe9fb..580010904 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java @@ -45,6 +45,7 @@ import com.shimmerresearch.driverUtilities.SensorDetailsRef; import com.shimmerresearch.driverUtilities.SensorGroupingDetails; import com.shimmerresearch.driverUtilities.SensorDetails; +import com.shimmerresearch.driverUtilities.SdTimestampAnchor; import com.shimmerresearch.driverUtilities.ShimmerSDCardDetails; import com.shimmerresearch.driverUtilities.ShimmerVerDetails; import com.shimmerresearch.driverUtilities.ShimmerVerObject; @@ -653,13 +654,13 @@ public String toString() { protected boolean mFirstTime = true; - /** This variable was originally implemented because the - * initial time in the SD header (i.e., the time when the header - * was created) wasn't equal to the first timestamp in the - * subsequent data packets (i.e., the lower 3 bytes of the - * initial timestamp). This problem has since been addressed in - * firmware whereby the header is updated with the timestamp - * from the first packet. Therefore this variable is redundant. */ + /** Subtracted, with the header's initial timestamp added, from each + * unwrapped SD timestamp so that the file's records land on their own + * counter time. It is not redundant: the SD header holds the RTC at the + * time the file was created, not the first packet's timestamp - the + * firmware captures the latter but never writes it back - so it is set by + * {@link SdTimestampAnchor#firstTsOffsetFromInitialTsTicks} from the + * header's low bits and the first packet's raw timestamp (DEV-1095). */ protected double mFirstTsOffsetFromInitialTsTicks = 0; public int OFFSET_LENGTH = 9; //-------- Timestamp related end -------- @@ -2821,17 +2822,19 @@ protected void parseTimestampShimmer3(COMMUNICATION_TYPE fwType, ObjectCluster o objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.UNCAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks); objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.CAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks); if(mFirstTime && fwType==COMMUNICATION_TYPE.SD){ - //this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks. - mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks; - - //This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1 + //This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1 if(getFirmwareIdentifier()==FW_ID.STROKARE && !isThisVerCompatibleWith(FW_ID.STROKARE, 1, 0, 1)){ long initialTsTicksOriginal = getInitialTimeStampTicksSd(); long initialTsTicksNew = (long) ((initialTsTicksOriginal&0xFFFF000000L)+shimmerTimestampTicks); setInitialTimeStampTicksSd(initialTsTicksNew); } - + + //Anchors the file on the first packet's own counter time rather than on + //the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks. + mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks( + getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue); + mFirstTime = false; } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java new file mode 100644 index 000000000..8b1dd2944 --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java @@ -0,0 +1,106 @@ +package com.shimmerresearch.driverUtilities; + +/** + * Places the first record of an SD data file at the time it was actually + * sampled. + *

+ * An SD data file's absolute timeline is its header's initial timestamp plus the + * elapsed ticks of its records. The driver used to anchor that timeline by + * subtracting the first record's raw timestamp, which pins the first record to + * the header time. That is only right if the header holds the first record's + * time, and LogAndStream does not write that: the header carries the RTC at the + * moment the file was created (sdFileSyncTs, set in + * ShimSdDataFile_fileInit and at each hourly split). Records + * already buffered when the file is opened were sampled before that moment, so + * pinning erases a lead time that differs from file to file: + *

+ * The difference surfaces as a permanent step at each file boundary: -158.8 ms + * at the 000 to 001 split of a 66.8 hour Shimmer3R recording (DEV-1095). + *

+ * Nothing is lost to fix it. A record's 3-byte timestamp is the low 24 bits of + * the same 32768 Hz counter the header's initial timestamp reads, so the first + * record's full counter value is the header value moved by the signed, wrap- + * corrected distance between the two low parts. That is exact whenever the + * header lies within half a counter period (256 s) of the first record, and it + * does not depend on which moment the firmware chose for the header - so it + * stays right if the firmware later writes the first record's time instead. + *

+ * The reconstruction is in the device's counter domain. It must be applied to + * the header's initial timestamp before the RTC difference is added: on + * Shimmer3 that difference is the offset from the free-running counter to real + * time, and it is not a multiple of the counter period. + *

+ * Pure and static so that both of the driver's timestamp paths - + * ShimmerObject and SensorShimmerClock - share one rule, + * as with {@link TimestampUnwrap}. + * + * @author Mark Nolan + */ +public final class SdTimestampAnchor { + + /** + * Largest distance between the header's initial timestamp and the first + * record that is read as a lead time: 10 s at 32768 Hz. The real lead is well + * under a second - the buffered records, plus SD start-up for file 000 - so a + * larger distance means the header does not describe this counter, and the + * first record keeps the header time as it always has. + */ + public static final long MAX_LEAD_TICKS = 10L * 32768L; + + private SdTimestampAnchor() { + } + + /** + * The value to subtract from unwrappedTicks + initialTsTicks so + * that each record of an SD data file lands on its own counter time. For the + * first record that is initialTsTicks + d, where d is + * the signed modular distance from the header's low bits to the record's raw + * timestamp. + *

+ * Falls back to the first record's raw timestamp - pinning the first record + * to the header time, the previous behaviour - when there is nothing to + * reconstruct from: + *

+ * + * @param initialTsTicks the header's initial timestamp, in counter ticks, + * without any RTC difference added + * @param firstRawTicks the first record's raw (wrapped) timestamp + * @param ticksMaxValue the counter's modulo: 2^24 for a 3-byte timestamp + * @return the offset to subtract, in ticks + */ + public static double firstTsOffsetFromInitialTsTicks(long initialTsTicks, double firstRawTicks, + int ticksMaxValue) { + if (ticksMaxValue != TimestampUnwrap.TICKS_MAX_3_BYTE || initialTsTicks == 0) { + return firstRawTicks; + } + long lead = signedLeadTicks(initialTsTicks, (long) firstRawTicks, ticksMaxValue); + if (Math.abs(lead) > MAX_LEAD_TICKS) { + return firstRawTicks; + } + return firstRawTicks - lead; + } + + /** + * Signed distance from the header's low bits to the first record's raw + * timestamp, folded into [-ticksMaxValue/2, ticksMaxValue/2): + * positive when the record was sampled after the header was written, negative + * when it was sampled before. + */ + static long signedLeadTicks(long initialTsTicks, long firstRawTicks, int ticksMaxValue) { + long half = ticksMaxValue / 2; + long low = Math.floorMod(initialTsTicks, (long) ticksMaxValue); + return Math.floorMod(firstRawTicks - low + half, (long) ticksMaxValue) - half; + } +} diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java index 8ef35eeb0..638a94f19 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java @@ -17,6 +17,7 @@ import com.shimmerresearch.driverUtilities.SensorDetails; import com.shimmerresearch.driverUtilities.SensorDetailsRef; import com.shimmerresearch.driverUtilities.SensorGroupingDetails; +import com.shimmerresearch.driverUtilities.SdTimestampAnchor; import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID; import com.shimmerresearch.driverUtilities.TimestampUnwrap; import com.shimmerresearch.driverUtilities.UtilParseData; @@ -49,6 +50,8 @@ public class SensorShimmerClock extends AbstractSensor { protected transient boolean mLastTimestampRejected = false; protected boolean mFirstTime = true; + /** Anchors an SD file's records on their own counter time - see + * {@link SdTimestampAnchor} and the matching field in ShimmerObject (DEV-1095). */ double mFirstTsOffsetFromInitialTsTicks = 0; double mSystemTimeStamp = 0; public int OFFSET_LENGTH = 9; @@ -368,8 +371,10 @@ public ObjectCluster processDataCustom(SensorDetails sensorDetails, byte[] senso double shimmerTimestampTicks = UtilParseData.parseData(channelByteArray, channelDetails.mDefaultChannelDataType, channelDetails.mDefaultChannelDataEndian); if(mFirstTime && commType==COMMUNICATION_TYPE.SD){ - //this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks. - mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks; + //Anchors the file on the first packet's own counter time rather than on + //the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks. + mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks( + getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue); mFirstTime = false; } diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java index e6dfddd64..bf74692ee 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java @@ -7,6 +7,7 @@ import com.shimmerresearch.algorithms.API_00002_Filters; import com.shimmerresearch.driverUtilities.API_00009_TimestampUnwrapTest; import com.shimmerresearch.driverUtilities.API_00010_TimestampUnwrapVectorsTest; +import com.shimmerresearch.driverUtilities.API_00012_SdTimestampAnchorTest; import com.shimmerresearch.verisense.API_00004_VerisenseConfigByteParsingAndGeneration; import com.shimmerresearch.verisense.communication.API_00003_VerisenseProtocolByteCommunicationTest; @@ -16,7 +17,8 @@ API_00003_VerisenseProtocolByteCommunicationTest.class, API_00004_VerisenseConfigByteParsingAndGeneration.class, API_00009_TimestampUnwrapTest.class, - API_00010_TimestampUnwrapVectorsTest.class + API_00010_TimestampUnwrapVectorsTest.class, + API_00012_SdTimestampAnchorTest.class }) public class API_00005_Suite_ShimmerDriver { diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java new file mode 100644 index 000000000..abfd9c552 --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java @@ -0,0 +1,141 @@ +package com.shimmerresearch.driverUtilities; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Covers {@link SdTimestampAnchor}: an SD data file's records are placed at their + * own counter time, not pinned to the header's file-creation time. The defect it + * fixes (DEV-1095) was a permanent -158.8 ms step at the 000 to 001 file split of + * a Shimmer3R recording. + * + * @author Mark Nolan + */ +public class API_00012_SdTimestampAnchorTest { + + private static final int MAX_3_BYTE = TimestampUnwrap.TICKS_MAX_3_BYTE; // 2^24 + private static final int MAX_2_BYTE = 65536; + /** A full 40-bit counter value, well away from any 24-bit boundary. */ + private static final long INITIAL_TS = 0x12_3456_7000L; + /** 32 ticks: one sample period at 1024 Hz. */ + private static final long PERIOD_TICKS = 32; + + /** Time the parser gives a record: header + unwrapped - offset. */ + private static double placed(long initialTs, double unwrapped, double offset) { + return initialTs + unwrapped - offset; + } + + private static long low24(long ticks) { + return ticks & (MAX_3_BYTE - 1); + } + + @Test + public void firstRecordSampledAfterHeaderLandsOnItsOwnTime() { + double raw = low24(INITIAL_TS) + 1000; + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE); + assertEquals(INITIAL_TS + 1000, placed(INITIAL_TS, raw, offset), 0); + } + + @Test + public void firstRecordSampledBeforeHeaderLandsBeforeIt() { + // File 000: records buffered during SD start-up predate the header by ~160 ms. + double raw = low24(INITIAL_TS) - 5243; + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE); + assertEquals(INITIAL_TS - 5243, placed(INITIAL_TS, raw, offset), 0); + } + + @Test + public void leadAcrossForwardWrapOfLowBits() { + long initialTs = (0x42L << 24) + MAX_3_BYTE - 100; // low bits 100 below the top + double raw = 200; // the counter wrapped 300 ticks after the header + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE); + assertEquals(initialTs + 300, placed(initialTs, raw, offset), 0); + } + + @Test + public void leadAcrossBackwardWrapOfLowBits() { + long initialTs = (0x42L << 24) + 50; // header just after a wrap + double raw = MAX_3_BYTE - 100; // record sampled 150 ticks before it + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE); + assertEquals(initialTs - 150, placed(initialTs, raw, offset), 0); + } + + @Test + public void laterRecordsKeepTheirSpacing() { + double raw = low24(INITIAL_TS) - 5243; + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE); + double first = placed(INITIAL_TS, raw, offset); + double tenth = placed(INITIAL_TS, raw + 9 * PERIOD_TICKS, offset); + assertEquals(9 * PERIOD_TICKS, tenth - first, 0); + } + + /** + * Two consecutive files of one continuous recording. File 000's header is + * written 5800 ticks (177 ms) after its first record; file 001's 577 ticks + * (17.6 ms) after the split's first buffered record. Pinning each file to its + * header puts a step of the difference at the boundary; anchoring on the + * records' own counter keeps exactly one sample period. + */ + @Test + public void noStepAtFileSplit() { + long c0 = INITIAL_TS; + long lastOf000 = c0 + 1000 * PERIOD_TICKS; + long firstOf001 = lastOf000 + PERIOD_TICKS; + long header000 = c0 + 5800; + long header001 = firstOf001 + 577; + + double off000 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header000, low24(c0), MAX_3_BYTE); + double off001 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header001, low24(firstOf001), MAX_3_BYTE); + // Unwrapped values within a file: raw first + elapsed (no wrap inside this span). + double end000 = placed(header000, low24(c0) + (lastOf000 - c0), off000); + double start001 = placed(header001, low24(firstOf001), off001); + + assertEquals(lastOf000, end000, 0); + assertEquals(firstOf001, start001, 0); + assertEquals(PERIOD_TICKS, start001 - end000, 0); + + // The previous rule - subtract the raw first timestamp - stepped back here. + double oldEnd000 = placed(header000, low24(c0) + (lastOf000 - c0), low24(c0)); + double oldStart001 = placed(header001, low24(firstOf001), low24(firstOf001)); + assertEquals(PERIOD_TICKS - (5800 - 577), oldStart001 - oldEnd000, 0); + } + + @Test + public void strokareCorrectedHeaderStillPinsExactly() { + // The old-StroKare workaround rewrites the header's low bits to the first + // record's, so the lead is zero and the result is unchanged. + double raw = 0x00_ABCDEF; + long initialTs = (INITIAL_TS & 0xFFFF000000L) + (long) raw; + assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE), 0); + } + + @Test + public void twoByteCounterKeepsPreviousBehaviour() { + assertEquals(1234.0, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, 1234, MAX_2_BYTE), 0); + } + + @Test + public void zeroInitialTimestampKeepsPreviousBehaviour() { + double raw = 0x00_F00000; + assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(0, raw, MAX_3_BYTE), 0); + } + + @Test + public void implausibleLeadKeepsPreviousBehaviour() { + double raw = low24(INITIAL_TS) + SdTimestampAnchor.MAX_LEAD_TICKS + 1; + assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE), 0); + raw = low24(INITIAL_TS) + SdTimestampAnchor.MAX_LEAD_TICKS; + assertEquals(raw - SdTimestampAnchor.MAX_LEAD_TICKS, + SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE), 0); + } + + @Test + public void signedLeadIsFoldedIntoHalfRange() { + assertEquals(0, SdTimestampAnchor.signedLeadTicks(INITIAL_TS, low24(INITIAL_TS), MAX_3_BYTE)); + assertEquals(-MAX_3_BYTE / 2, + SdTimestampAnchor.signedLeadTicks(0, MAX_3_BYTE / 2, MAX_3_BYTE)); + assertEquals(MAX_3_BYTE / 2 - 1, + SdTimestampAnchor.signedLeadTicks(0, MAX_3_BYTE / 2 - 1, MAX_3_BYTE)); + } +} From 76ceba1cdd0ce9e4567b7af167cd074550fa6b43 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Tue, 29 Sep 2026 12:12:10 +0100 Subject: [PATCH 2/4] DEV-1095: add the raw Shimmer3R 000/001 split as a test vector Header and first-packet values from the recording's raw files: file 000's first packet predates its header by 5332 ticks (162.72 ms), file 001's by 129 (3.94 ms). The old rule reproduces the export's 158.78 ms step; the anchored split is exactly one sample period. Co-Authored-By: Claude Opus 5.5 --- .../API_00012_SdTimestampAnchorTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java index abfd9c552..4a814cc09 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java @@ -101,6 +101,40 @@ public void noStepAtFileSplit() { assertEquals(PERIOD_TICKS - (5800 - 577), oldStart001 - oldEnd000, 0); } + /** + * Header and packet values read from the raw 000 and 001 files of the + * Shimmer3R recording DEV-1095 was raised on (CE2F, 1024 Hz, 2026-09-25). + * File 000's first packet predates its header by 5332 ticks (162.72 ms), file + * 001's by 129 (3.94 ms); pinned to their headers, the files stepped back + * 5203 ticks (158.78 ms), exactly the step seen in the Consensys export. + * Anchored, the split is one sample period. + */ + @Test + public void realShimmer3rRecordingSplitsByOnePeriod() { + long header000 = 391630116578L; + long firstRaw000 = 16335374L; // 5332 ticks below the header's low bits + long firstCounter000 = 391630111246L; + long lastCounter000 = 391748082222L; // 3,675,280 records later + long header001 = 391748082383L; + long firstRaw001 = 88654L; + long firstCounter001 = 391748082254L; + + double off000 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header000, firstRaw000, MAX_3_BYTE); + double off001 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header001, firstRaw001, MAX_3_BYTE); + assertEquals(firstCounter000, placed(header000, firstRaw000, off000), 0); + assertEquals(firstCounter001, placed(header001, firstRaw001, off001), 0); + + // Unwrapped within file 000: the first raw value plus the ticks elapsed. + double end000 = placed(header000, firstRaw000 + (lastCounter000 - firstCounter000), off000); + assertEquals(lastCounter000, end000, 0); + assertEquals(PERIOD_TICKS, placed(header001, firstRaw001, off001) - end000, 0); + + // The previous rule reproduced the export's step. + double oldEnd000 = placed(header000, firstRaw000 + (lastCounter000 - firstCounter000), firstRaw000); + double oldStart001 = placed(header001, firstRaw001, firstRaw001); + assertEquals(PERIOD_TICKS - 5203, oldStart001 - oldEnd000, 0); + } + @Test public void strokareCorrectedHeaderStillPinsExactly() { // The old-StroKare workaround rewrites the header's low bits to the first From a18fa54a472c9f384bfe8d2649127e53864cd651 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Tue, 29 Sep 2026 13:21:45 +0100 Subject: [PATCH 3/4] DEV-1095: add the raw 041/042 split as a second test vector Leads of 214 and 131 ticks gave the export's other backwards step, -1.556 ms near hour 42. Both real splits now share one assertion helper. Co-Authored-By: Claude Opus 5.5 --- .../API_00012_SdTimestampAnchorTest.java | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java index 4a814cc09..03e84cb55 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java @@ -101,6 +101,32 @@ public void noStepAtFileSplit() { assertEquals(PERIOD_TICKS - (5800 - 577), oldStart001 - oldEnd000, 0); } + /** + * Checks one real file split: each file's first packet lands on its own + * counter value, the split is one sample period, and pinning each file to its + * header - the previous rule - gives the step the Consensys export showed. + * + * @param oldStepTicks the previous rule's step beyond one sample period, i.e. + * the difference between the two files' leads + */ + private static void assertRealSplit(long headerA, long firstRawA, long firstCounterA, long lastCounterA, + long headerB, long firstRawB, long firstCounterB, long oldStepTicks) { + double offA = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(headerA, firstRawA, MAX_3_BYTE); + double offB = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(headerB, firstRawB, MAX_3_BYTE); + assertEquals(firstCounterA, placed(headerA, firstRawA, offA), 0); + assertEquals(firstCounterB, placed(headerB, firstRawB, offB), 0); + + // Unwrapped within file A: the first raw value plus the ticks elapsed. + double unwrappedLastA = firstRawA + (lastCounterA - firstCounterA); + double endA = placed(headerA, unwrappedLastA, offA); + assertEquals(lastCounterA, endA, 0); + assertEquals(PERIOD_TICKS, placed(headerB, firstRawB, offB) - endA, 0); + + // The previous rule reproduced the export's step. + double oldStep = placed(headerB, firstRawB, firstRawB) - placed(headerA, unwrappedLastA, firstRawA); + assertEquals(PERIOD_TICKS + oldStepTicks, oldStep, 0); + } + /** * Header and packet values read from the raw 000 and 001 files of the * Shimmer3R recording DEV-1095 was raised on (CE2F, 1024 Hz, 2026-09-25). @@ -111,28 +137,20 @@ public void noStepAtFileSplit() { */ @Test public void realShimmer3rRecordingSplitsByOnePeriod() { - long header000 = 391630116578L; - long firstRaw000 = 16335374L; // 5332 ticks below the header's low bits - long firstCounter000 = 391630111246L; - long lastCounter000 = 391748082222L; // 3,675,280 records later - long header001 = 391748082383L; - long firstRaw001 = 88654L; - long firstCounter001 = 391748082254L; - - double off000 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header000, firstRaw000, MAX_3_BYTE); - double off001 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header001, firstRaw001, MAX_3_BYTE); - assertEquals(firstCounter000, placed(header000, firstRaw000, off000), 0); - assertEquals(firstCounter001, placed(header001, firstRaw001, off001), 0); - - // Unwrapped within file 000: the first raw value plus the ticks elapsed. - double end000 = placed(header000, firstRaw000 + (lastCounter000 - firstCounter000), off000); - assertEquals(lastCounter000, end000, 0); - assertEquals(PERIOD_TICKS, placed(header001, firstRaw001, off001) - end000, 0); + assertRealSplit(391630116578L, 16335374L, 391630111246L, 391748082222L, // 000: 3,675,280 records + 391748082383L, 88654L, 391748082254L, -5203); + } - // The previous rule reproduced the export's step. - double oldEnd000 = placed(header000, firstRaw000 + (lastCounter000 - firstCounter000), firstRaw000); - double oldStart001 = placed(header001, firstRaw001, firstRaw001); - assertEquals(PERIOD_TICKS - 5203, oldStart001 - oldEnd000, 0); + /** + * The 041 and 042 files of the same recording: leads of 214 ticks (6.53 ms) + * and 131 (4.00 ms). The previous rule's step was only -83 ticks, which the + * export showed as timestamps going back 1.556 ms, about 27 ms before the + * post-split stall. Anchored, this split is one sample period too. + */ + @Test + public void realShimmer3rSplitWithSmallLeadDifference() { + assertRealSplit(396466711492L, 4319982L, 396466711278L, 396584676654L, + 396584676817L, 4844878L, 396584676686L, -83); } @Test From b29098ebb0238091f4c1096482862882e2bee3d0 Mon Sep 17 00:00:00 2001 From: Mark Nolan Date: Sun, 4 Oct 2026 14:37:46 +0100 Subject: [PATCH 4/4] DEV-1095: record the Shimmer3 verification of the SD file anchor Raw files from a 33-file Shimmer3 recording (LogAndStream v1.1.5, 1024 Hz) confirm the rule there. File 000's first record was sampled 1.831 ms after its header and every later file's 0.885 ms before, so the header's low 24 bits share the record counter, and all 32 splits close to exactly one sample period. The old rule's only artefact on that recording is +2.7 ms at the 000 to 001 split. The class doc said file 000's records predate its header by about 160 ms. That is the Shimmer3R; a Shimmer3 takes file 000's header first. Comment only. Co-Authored-By: Claude Opus 5.5 --- .../driverUtilities/SdTimestampAnchor.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java index 8b1dd2944..8454c25ab 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java @@ -14,14 +14,17 @@ * already buffered when the file is opened were sampled before that moment, so * pinning erases a lead time that differs from file to file: *
    - *
  • File 000 is created after sampling has started - SD power-up, - * directory creation, header build - so its first records predate the header by - * roughly 160 ms more than at a mid-stream split.
  • + *
  • File 000 on a Shimmer3R is created after sampling has started - + * SD power-up, directory creation, header build - so its first records predate + * the header by roughly 160 ms more than at a mid-stream split. A Shimmer3 + * (LogAndStream v1.1.5) takes file 000's header 1.8 ms before its first record + * instead.
  • *
  • Later files are opened mid-stream, behind only the records still in * the SD write buffer.
  • *
* The difference surfaces as a permanent step at each file boundary: -158.8 ms - * at the 000 to 001 split of a 66.8 hour Shimmer3R recording (DEV-1095). + * at the 000 to 001 split of a 66.8 hour Shimmer3R recording (DEV-1095), and + * +2.7 ms at the same split of a 32 hour Shimmer3 one. *

* Nothing is lost to fix it. A record's 3-byte timestamp is the low 24 bits of * the same 32768 Hz counter the header's initial timestamp reads, so the first @@ -36,6 +39,14 @@ * Shimmer3 that difference is the offset from the free-running counter to real * time, and it is not a multiple of the counter period. *

+ * Verified against raw files on both platforms. Shimmer3R: the first records of + * 000 and 001 were sampled 162.72 ms and 3.94 ms before their headers (a test + * vector in API_00012), and anchored this way the split is one sample period. + * Shimmer3 (LogAndStream v1.1.5, 1024 Hz, 33 files): file 000's first record was + * sampled 1.831 ms after its header and every later file's 0.885 ms before, so + * the header shares the record counter there too, and all 32 splits are exactly + * one sample period. + *

* Pure and static so that both of the driver's timestamp paths - * ShimmerObject and SensorShimmerClock - share one rule, * as with {@link TimestampUnwrap}.