diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java index 4056fe9fb..580010904 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java @@ -45,6 +45,7 @@ import com.shimmerresearch.driverUtilities.SensorDetailsRef; import com.shimmerresearch.driverUtilities.SensorGroupingDetails; import com.shimmerresearch.driverUtilities.SensorDetails; +import com.shimmerresearch.driverUtilities.SdTimestampAnchor; import com.shimmerresearch.driverUtilities.ShimmerSDCardDetails; import com.shimmerresearch.driverUtilities.ShimmerVerDetails; import com.shimmerresearch.driverUtilities.ShimmerVerObject; @@ -653,13 +654,13 @@ public String toString() { protected boolean mFirstTime = true; - /** This variable was originally implemented because the - * initial time in the SD header (i.e., the time when the header - * was created) wasn't equal to the first timestamp in the - * subsequent data packets (i.e., the lower 3 bytes of the - * initial timestamp). This problem has since been addressed in - * firmware whereby the header is updated with the timestamp - * from the first packet. Therefore this variable is redundant. */ + /** Subtracted, with the header's initial timestamp added, from each + * unwrapped SD timestamp so that the file's records land on their own + * counter time. It is not redundant: the SD header holds the RTC at the + * time the file was created, not the first packet's timestamp - the + * firmware captures the latter but never writes it back - so it is set by + * {@link SdTimestampAnchor#firstTsOffsetFromInitialTsTicks} from the + * header's low bits and the first packet's raw timestamp (DEV-1095). */ protected double mFirstTsOffsetFromInitialTsTicks = 0; public int OFFSET_LENGTH = 9; //-------- Timestamp related end -------- @@ -2821,17 +2822,19 @@ protected void parseTimestampShimmer3(COMMUNICATION_TYPE fwType, ObjectCluster o objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.UNCAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks); objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.CAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks); if(mFirstTime && fwType==COMMUNICATION_TYPE.SD){ - //this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks. - mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks; - - //This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1 + //This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1 if(getFirmwareIdentifier()==FW_ID.STROKARE && !isThisVerCompatibleWith(FW_ID.STROKARE, 1, 0, 1)){ long initialTsTicksOriginal = getInitialTimeStampTicksSd(); long initialTsTicksNew = (long) ((initialTsTicksOriginal&0xFFFF000000L)+shimmerTimestampTicks); setInitialTimeStampTicksSd(initialTsTicksNew); } - + + //Anchors the file on the first packet's own counter time rather than on + //the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks. + mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks( + getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue); + mFirstTime = false; } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java new file mode 100644 index 000000000..8454c25ab --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java @@ -0,0 +1,117 @@ +package com.shimmerresearch.driverUtilities; + +/** + * Places the first record of an SD data file at the time it was actually + * sampled. + *

+ * An SD data file's absolute timeline is its header's initial timestamp plus the + * elapsed ticks of its records. The driver used to anchor that timeline by + * subtracting the first record's raw timestamp, which pins the first record to + * the header time. That is only right if the header holds the first record's + * time, and LogAndStream does not write that: the header carries the RTC at the + * moment the file was created (sdFileSyncTs, set in + * ShimSdDataFile_fileInit and at each hourly split). Records + * already buffered when the file is opened were sampled before that moment, so + * pinning erases a lead time that differs from file to file: + *

+ * The difference surfaces as a permanent step at each file boundary: -158.8 ms + * at the 000 to 001 split of a 66.8 hour Shimmer3R recording (DEV-1095), and + * +2.7 ms at the same split of a 32 hour Shimmer3 one. + *

+ * Nothing is lost to fix it. A record's 3-byte timestamp is the low 24 bits of + * the same 32768 Hz counter the header's initial timestamp reads, so the first + * record's full counter value is the header value moved by the signed, wrap- + * corrected distance between the two low parts. That is exact whenever the + * header lies within half a counter period (256 s) of the first record, and it + * does not depend on which moment the firmware chose for the header - so it + * stays right if the firmware later writes the first record's time instead. + *

+ * The reconstruction is in the device's counter domain. It must be applied to + * the header's initial timestamp before the RTC difference is added: on + * Shimmer3 that difference is the offset from the free-running counter to real + * time, and it is not a multiple of the counter period. + *

+ * Verified against raw files on both platforms. Shimmer3R: the first records of + * 000 and 001 were sampled 162.72 ms and 3.94 ms before their headers (a test + * vector in API_00012), and anchored this way the split is one sample period. + * Shimmer3 (LogAndStream v1.1.5, 1024 Hz, 33 files): file 000's first record was + * sampled 1.831 ms after its header and every later file's 0.885 ms before, so + * the header shares the record counter there too, and all 32 splits are exactly + * one sample period. + *

+ * Pure and static so that both of the driver's timestamp paths - + * ShimmerObject and SensorShimmerClock - share one rule, + * as with {@link TimestampUnwrap}. + * + * @author Mark Nolan + */ +public final class SdTimestampAnchor { + + /** + * Largest distance between the header's initial timestamp and the first + * record that is read as a lead time: 10 s at 32768 Hz. The real lead is well + * under a second - the buffered records, plus SD start-up for file 000 - so a + * larger distance means the header does not describe this counter, and the + * first record keeps the header time as it always has. + */ + public static final long MAX_LEAD_TICKS = 10L * 32768L; + + private SdTimestampAnchor() { + } + + /** + * The value to subtract from unwrappedTicks + initialTsTicks so + * that each record of an SD data file lands on its own counter time. For the + * first record that is initialTsTicks + d, where d is + * the signed modular distance from the header's low bits to the record's raw + * timestamp. + *

+ * Falls back to the first record's raw timestamp - pinning the first record + * to the header time, the previous behaviour - when there is nothing to + * reconstruct from: + *

+ * + * @param initialTsTicks the header's initial timestamp, in counter ticks, + * without any RTC difference added + * @param firstRawTicks the first record's raw (wrapped) timestamp + * @param ticksMaxValue the counter's modulo: 2^24 for a 3-byte timestamp + * @return the offset to subtract, in ticks + */ + public static double firstTsOffsetFromInitialTsTicks(long initialTsTicks, double firstRawTicks, + int ticksMaxValue) { + if (ticksMaxValue != TimestampUnwrap.TICKS_MAX_3_BYTE || initialTsTicks == 0) { + return firstRawTicks; + } + long lead = signedLeadTicks(initialTsTicks, (long) firstRawTicks, ticksMaxValue); + if (Math.abs(lead) > MAX_LEAD_TICKS) { + return firstRawTicks; + } + return firstRawTicks - lead; + } + + /** + * Signed distance from the header's low bits to the first record's raw + * timestamp, folded into [-ticksMaxValue/2, ticksMaxValue/2): + * positive when the record was sampled after the header was written, negative + * when it was sampled before. + */ + static long signedLeadTicks(long initialTsTicks, long firstRawTicks, int ticksMaxValue) { + long half = ticksMaxValue / 2; + long low = Math.floorMod(initialTsTicks, (long) ticksMaxValue); + return Math.floorMod(firstRawTicks - low + half, (long) ticksMaxValue) - half; + } +} diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java index 8ef35eeb0..638a94f19 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java @@ -17,6 +17,7 @@ import com.shimmerresearch.driverUtilities.SensorDetails; import com.shimmerresearch.driverUtilities.SensorDetailsRef; import com.shimmerresearch.driverUtilities.SensorGroupingDetails; +import com.shimmerresearch.driverUtilities.SdTimestampAnchor; import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID; import com.shimmerresearch.driverUtilities.TimestampUnwrap; import com.shimmerresearch.driverUtilities.UtilParseData; @@ -49,6 +50,8 @@ public class SensorShimmerClock extends AbstractSensor { protected transient boolean mLastTimestampRejected = false; protected boolean mFirstTime = true; + /** Anchors an SD file's records on their own counter time - see + * {@link SdTimestampAnchor} and the matching field in ShimmerObject (DEV-1095). */ double mFirstTsOffsetFromInitialTsTicks = 0; double mSystemTimeStamp = 0; public int OFFSET_LENGTH = 9; @@ -368,8 +371,10 @@ public ObjectCluster processDataCustom(SensorDetails sensorDetails, byte[] senso double shimmerTimestampTicks = UtilParseData.parseData(channelByteArray, channelDetails.mDefaultChannelDataType, channelDetails.mDefaultChannelDataEndian); if(mFirstTime && commType==COMMUNICATION_TYPE.SD){ - //this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks. - mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks; + //Anchors the file on the first packet's own counter time rather than on + //the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks. + mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks( + getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue); mFirstTime = false; } diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java index e6dfddd64..bf74692ee 100644 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java @@ -7,6 +7,7 @@ import com.shimmerresearch.algorithms.API_00002_Filters; import com.shimmerresearch.driverUtilities.API_00009_TimestampUnwrapTest; import com.shimmerresearch.driverUtilities.API_00010_TimestampUnwrapVectorsTest; +import com.shimmerresearch.driverUtilities.API_00012_SdTimestampAnchorTest; import com.shimmerresearch.verisense.API_00004_VerisenseConfigByteParsingAndGeneration; import com.shimmerresearch.verisense.communication.API_00003_VerisenseProtocolByteCommunicationTest; @@ -16,7 +17,8 @@ API_00003_VerisenseProtocolByteCommunicationTest.class, API_00004_VerisenseConfigByteParsingAndGeneration.class, API_00009_TimestampUnwrapTest.class, - API_00010_TimestampUnwrapVectorsTest.class + API_00010_TimestampUnwrapVectorsTest.class, + API_00012_SdTimestampAnchorTest.class }) public class API_00005_Suite_ShimmerDriver { diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java new file mode 100644 index 000000000..03e84cb55 --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java @@ -0,0 +1,193 @@ +package com.shimmerresearch.driverUtilities; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Covers {@link SdTimestampAnchor}: an SD data file's records are placed at their + * own counter time, not pinned to the header's file-creation time. The defect it + * fixes (DEV-1095) was a permanent -158.8 ms step at the 000 to 001 file split of + * a Shimmer3R recording. + * + * @author Mark Nolan + */ +public class API_00012_SdTimestampAnchorTest { + + private static final int MAX_3_BYTE = TimestampUnwrap.TICKS_MAX_3_BYTE; // 2^24 + private static final int MAX_2_BYTE = 65536; + /** A full 40-bit counter value, well away from any 24-bit boundary. */ + private static final long INITIAL_TS = 0x12_3456_7000L; + /** 32 ticks: one sample period at 1024 Hz. */ + private static final long PERIOD_TICKS = 32; + + /** Time the parser gives a record: header + unwrapped - offset. */ + private static double placed(long initialTs, double unwrapped, double offset) { + return initialTs + unwrapped - offset; + } + + private static long low24(long ticks) { + return ticks & (MAX_3_BYTE - 1); + } + + @Test + public void firstRecordSampledAfterHeaderLandsOnItsOwnTime() { + double raw = low24(INITIAL_TS) + 1000; + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE); + assertEquals(INITIAL_TS + 1000, placed(INITIAL_TS, raw, offset), 0); + } + + @Test + public void firstRecordSampledBeforeHeaderLandsBeforeIt() { + // File 000: records buffered during SD start-up predate the header by ~160 ms. + double raw = low24(INITIAL_TS) - 5243; + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE); + assertEquals(INITIAL_TS - 5243, placed(INITIAL_TS, raw, offset), 0); + } + + @Test + public void leadAcrossForwardWrapOfLowBits() { + long initialTs = (0x42L << 24) + MAX_3_BYTE - 100; // low bits 100 below the top + double raw = 200; // the counter wrapped 300 ticks after the header + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE); + assertEquals(initialTs + 300, placed(initialTs, raw, offset), 0); + } + + @Test + public void leadAcrossBackwardWrapOfLowBits() { + long initialTs = (0x42L << 24) + 50; // header just after a wrap + double raw = MAX_3_BYTE - 100; // record sampled 150 ticks before it + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE); + assertEquals(initialTs - 150, placed(initialTs, raw, offset), 0); + } + + @Test + public void laterRecordsKeepTheirSpacing() { + double raw = low24(INITIAL_TS) - 5243; + double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE); + double first = placed(INITIAL_TS, raw, offset); + double tenth = placed(INITIAL_TS, raw + 9 * PERIOD_TICKS, offset); + assertEquals(9 * PERIOD_TICKS, tenth - first, 0); + } + + /** + * Two consecutive files of one continuous recording. File 000's header is + * written 5800 ticks (177 ms) after its first record; file 001's 577 ticks + * (17.6 ms) after the split's first buffered record. Pinning each file to its + * header puts a step of the difference at the boundary; anchoring on the + * records' own counter keeps exactly one sample period. + */ + @Test + public void noStepAtFileSplit() { + long c0 = INITIAL_TS; + long lastOf000 = c0 + 1000 * PERIOD_TICKS; + long firstOf001 = lastOf000 + PERIOD_TICKS; + long header000 = c0 + 5800; + long header001 = firstOf001 + 577; + + double off000 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header000, low24(c0), MAX_3_BYTE); + double off001 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header001, low24(firstOf001), MAX_3_BYTE); + // Unwrapped values within a file: raw first + elapsed (no wrap inside this span). + double end000 = placed(header000, low24(c0) + (lastOf000 - c0), off000); + double start001 = placed(header001, low24(firstOf001), off001); + + assertEquals(lastOf000, end000, 0); + assertEquals(firstOf001, start001, 0); + assertEquals(PERIOD_TICKS, start001 - end000, 0); + + // The previous rule - subtract the raw first timestamp - stepped back here. + double oldEnd000 = placed(header000, low24(c0) + (lastOf000 - c0), low24(c0)); + double oldStart001 = placed(header001, low24(firstOf001), low24(firstOf001)); + assertEquals(PERIOD_TICKS - (5800 - 577), oldStart001 - oldEnd000, 0); + } + + /** + * Checks one real file split: each file's first packet lands on its own + * counter value, the split is one sample period, and pinning each file to its + * header - the previous rule - gives the step the Consensys export showed. + * + * @param oldStepTicks the previous rule's step beyond one sample period, i.e. + * the difference between the two files' leads + */ + private static void assertRealSplit(long headerA, long firstRawA, long firstCounterA, long lastCounterA, + long headerB, long firstRawB, long firstCounterB, long oldStepTicks) { + double offA = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(headerA, firstRawA, MAX_3_BYTE); + double offB = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(headerB, firstRawB, MAX_3_BYTE); + assertEquals(firstCounterA, placed(headerA, firstRawA, offA), 0); + assertEquals(firstCounterB, placed(headerB, firstRawB, offB), 0); + + // Unwrapped within file A: the first raw value plus the ticks elapsed. + double unwrappedLastA = firstRawA + (lastCounterA - firstCounterA); + double endA = placed(headerA, unwrappedLastA, offA); + assertEquals(lastCounterA, endA, 0); + assertEquals(PERIOD_TICKS, placed(headerB, firstRawB, offB) - endA, 0); + + // The previous rule reproduced the export's step. + double oldStep = placed(headerB, firstRawB, firstRawB) - placed(headerA, unwrappedLastA, firstRawA); + assertEquals(PERIOD_TICKS + oldStepTicks, oldStep, 0); + } + + /** + * Header and packet values read from the raw 000 and 001 files of the + * Shimmer3R recording DEV-1095 was raised on (CE2F, 1024 Hz, 2026-09-25). + * File 000's first packet predates its header by 5332 ticks (162.72 ms), file + * 001's by 129 (3.94 ms); pinned to their headers, the files stepped back + * 5203 ticks (158.78 ms), exactly the step seen in the Consensys export. + * Anchored, the split is one sample period. + */ + @Test + public void realShimmer3rRecordingSplitsByOnePeriod() { + assertRealSplit(391630116578L, 16335374L, 391630111246L, 391748082222L, // 000: 3,675,280 records + 391748082383L, 88654L, 391748082254L, -5203); + } + + /** + * The 041 and 042 files of the same recording: leads of 214 ticks (6.53 ms) + * and 131 (4.00 ms). The previous rule's step was only -83 ticks, which the + * export showed as timestamps going back 1.556 ms, about 27 ms before the + * post-split stall. Anchored, this split is one sample period too. + */ + @Test + public void realShimmer3rSplitWithSmallLeadDifference() { + assertRealSplit(396466711492L, 4319982L, 396466711278L, 396584676654L, + 396584676817L, 4844878L, 396584676686L, -83); + } + + @Test + public void strokareCorrectedHeaderStillPinsExactly() { + // The old-StroKare workaround rewrites the header's low bits to the first + // record's, so the lead is zero and the result is unchanged. + double raw = 0x00_ABCDEF; + long initialTs = (INITIAL_TS & 0xFFFF000000L) + (long) raw; + assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE), 0); + } + + @Test + public void twoByteCounterKeepsPreviousBehaviour() { + assertEquals(1234.0, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, 1234, MAX_2_BYTE), 0); + } + + @Test + public void zeroInitialTimestampKeepsPreviousBehaviour() { + double raw = 0x00_F00000; + assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(0, raw, MAX_3_BYTE), 0); + } + + @Test + public void implausibleLeadKeepsPreviousBehaviour() { + double raw = low24(INITIAL_TS) + SdTimestampAnchor.MAX_LEAD_TICKS + 1; + assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE), 0); + raw = low24(INITIAL_TS) + SdTimestampAnchor.MAX_LEAD_TICKS; + assertEquals(raw - SdTimestampAnchor.MAX_LEAD_TICKS, + SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE), 0); + } + + @Test + public void signedLeadIsFoldedIntoHalfRange() { + assertEquals(0, SdTimestampAnchor.signedLeadTicks(INITIAL_TS, low24(INITIAL_TS), MAX_3_BYTE)); + assertEquals(-MAX_3_BYTE / 2, + SdTimestampAnchor.signedLeadTicks(0, MAX_3_BYTE / 2, MAX_3_BYTE)); + assertEquals(MAX_3_BYTE / 2 - 1, + SdTimestampAnchor.signedLeadTicks(0, MAX_3_BYTE / 2 - 1, MAX_3_BYTE)); + } +}