diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java index 4056fe9fb..580010904 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driver/ShimmerObject.java @@ -45,6 +45,7 @@ import com.shimmerresearch.driverUtilities.SensorDetailsRef; import com.shimmerresearch.driverUtilities.SensorGroupingDetails; import com.shimmerresearch.driverUtilities.SensorDetails; +import com.shimmerresearch.driverUtilities.SdTimestampAnchor; import com.shimmerresearch.driverUtilities.ShimmerSDCardDetails; import com.shimmerresearch.driverUtilities.ShimmerVerDetails; import com.shimmerresearch.driverUtilities.ShimmerVerObject; @@ -653,13 +654,13 @@ public String toString() { protected boolean mFirstTime = true; - /** This variable was originally implemented because the - * initial time in the SD header (i.e., the time when the header - * was created) wasn't equal to the first timestamp in the - * subsequent data packets (i.e., the lower 3 bytes of the - * initial timestamp). This problem has since been addressed in - * firmware whereby the header is updated with the timestamp - * from the first packet. Therefore this variable is redundant. */ + /** Subtracted, with the header's initial timestamp added, from each + * unwrapped SD timestamp so that the file's records land on their own + * counter time. It is not redundant: the SD header holds the RTC at the + * time the file was created, not the first packet's timestamp - the + * firmware captures the latter but never writes it back - so it is set by + * {@link SdTimestampAnchor#firstTsOffsetFromInitialTsTicks} from the + * header's low bits and the first packet's raw timestamp (DEV-1095). */ protected double mFirstTsOffsetFromInitialTsTicks = 0; public int OFFSET_LENGTH = 9; //-------- Timestamp related end -------- @@ -2821,17 +2822,19 @@ protected void parseTimestampShimmer3(COMMUNICATION_TYPE fwType, ObjectCluster o objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.UNCAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks); objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.CAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks); if(mFirstTime && fwType==COMMUNICATION_TYPE.SD){ - //this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks. - mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks; - - //This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1 + //This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1 if(getFirmwareIdentifier()==FW_ID.STROKARE && !isThisVerCompatibleWith(FW_ID.STROKARE, 1, 0, 1)){ long initialTsTicksOriginal = getInitialTimeStampTicksSd(); long initialTsTicksNew = (long) ((initialTsTicksOriginal&0xFFFF000000L)+shimmerTimestampTicks); setInitialTimeStampTicksSd(initialTsTicksNew); } - + + //Anchors the file on the first packet's own counter time rather than on + //the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks. + mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks( + getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue); + mFirstTime = false; } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java new file mode 100644 index 000000000..8454c25ab --- /dev/null +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/driverUtilities/SdTimestampAnchor.java @@ -0,0 +1,117 @@ +package com.shimmerresearch.driverUtilities; + +/** + * Places the first record of an SD data file at the time it was actually + * sampled. + *
+ * An SD data file's absolute timeline is its header's initial timestamp plus the
+ * elapsed ticks of its records. The driver used to anchor that timeline by
+ * subtracting the first record's raw timestamp, which pins the first record to
+ * the header time. That is only right if the header holds the first record's
+ * time, and LogAndStream does not write that: the header carries the RTC at the
+ * moment the file was created (sdFileSyncTs, set in
+ * ShimSdDataFile_fileInit and at each hourly split). Records
+ * already buffered when the file is opened were sampled before that moment, so
+ * pinning erases a lead time that differs from file to file:
+ *
+ * Nothing is lost to fix it. A record's 3-byte timestamp is the low 24 bits of + * the same 32768 Hz counter the header's initial timestamp reads, so the first + * record's full counter value is the header value moved by the signed, wrap- + * corrected distance between the two low parts. That is exact whenever the + * header lies within half a counter period (256 s) of the first record, and it + * does not depend on which moment the firmware chose for the header - so it + * stays right if the firmware later writes the first record's time instead. + *
+ * The reconstruction is in the device's counter domain. It must be applied to + * the header's initial timestamp before the RTC difference is added: on + * Shimmer3 that difference is the offset from the free-running counter to real + * time, and it is not a multiple of the counter period. + *
+ * Verified against raw files on both platforms. Shimmer3R: the first records of + * 000 and 001 were sampled 162.72 ms and 3.94 ms before their headers (a test + * vector in API_00012), and anchored this way the split is one sample period. + * Shimmer3 (LogAndStream v1.1.5, 1024 Hz, 33 files): file 000's first record was + * sampled 1.831 ms after its header and every later file's 0.885 ms before, so + * the header shares the record counter there too, and all 32 splits are exactly + * one sample period. + *
+ * Pure and static so that both of the driver's timestamp paths -
+ * ShimmerObject and SensorShimmerClock - share one rule,
+ * as with {@link TimestampUnwrap}.
+ *
+ * @author Mark Nolan
+ */
+public final class SdTimestampAnchor {
+
+ /**
+ * Largest distance between the header's initial timestamp and the first
+ * record that is read as a lead time: 10 s at 32768 Hz. The real lead is well
+ * under a second - the buffered records, plus SD start-up for file 000 - so a
+ * larger distance means the header does not describe this counter, and the
+ * first record keeps the header time as it always has.
+ */
+ public static final long MAX_LEAD_TICKS = 10L * 32768L;
+
+ private SdTimestampAnchor() {
+ }
+
+ /**
+ * The value to subtract from unwrappedTicks + initialTsTicks so
+ * that each record of an SD data file lands on its own counter time. For the
+ * first record that is initialTsTicks + d, where d is
+ * the signed modular distance from the header's low bits to the record's raw
+ * timestamp.
+ *
+ * Falls back to the first record's raw timestamp - pinning the first record + * to the header time, the previous behaviour - when there is nothing to + * reconstruct from: + *
[-ticksMaxValue/2, ticksMaxValue/2):
+ * positive when the record was sampled after the header was written, negative
+ * when it was sampled before.
+ */
+ static long signedLeadTicks(long initialTsTicks, long firstRawTicks, int ticksMaxValue) {
+ long half = ticksMaxValue / 2;
+ long low = Math.floorMod(initialTsTicks, (long) ticksMaxValue);
+ return Math.floorMod(firstRawTicks - low + half, (long) ticksMaxValue) - half;
+ }
+}
diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java
index 8ef35eeb0..638a94f19 100644
--- a/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java
+++ b/ShimmerDriver/src/main/java/com/shimmerresearch/sensors/SensorShimmerClock.java
@@ -17,6 +17,7 @@
import com.shimmerresearch.driverUtilities.SensorDetails;
import com.shimmerresearch.driverUtilities.SensorDetailsRef;
import com.shimmerresearch.driverUtilities.SensorGroupingDetails;
+import com.shimmerresearch.driverUtilities.SdTimestampAnchor;
import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID;
import com.shimmerresearch.driverUtilities.TimestampUnwrap;
import com.shimmerresearch.driverUtilities.UtilParseData;
@@ -49,6 +50,8 @@ public class SensorShimmerClock extends AbstractSensor {
protected transient boolean mLastTimestampRejected = false;
protected boolean mFirstTime = true;
+ /** Anchors an SD file's records on their own counter time - see
+ * {@link SdTimestampAnchor} and the matching field in ShimmerObject (DEV-1095). */
double mFirstTsOffsetFromInitialTsTicks = 0;
double mSystemTimeStamp = 0;
public int OFFSET_LENGTH = 9;
@@ -368,8 +371,10 @@ public ObjectCluster processDataCustom(SensorDetails sensorDetails, byte[] senso
double shimmerTimestampTicks = UtilParseData.parseData(channelByteArray, channelDetails.mDefaultChannelDataType, channelDetails.mDefaultChannelDataEndian);
if(mFirstTime && commType==COMMUNICATION_TYPE.SD){
- //this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks.
- mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks;
+ //Anchors the file on the first packet's own counter time rather than on
+ //the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks.
+ mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(
+ getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue);
mFirstTime = false;
}
diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java
index e6dfddd64..bf74692ee 100644
--- a/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java
+++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driver/API_00005_Suite_ShimmerDriver.java
@@ -7,6 +7,7 @@
import com.shimmerresearch.algorithms.API_00002_Filters;
import com.shimmerresearch.driverUtilities.API_00009_TimestampUnwrapTest;
import com.shimmerresearch.driverUtilities.API_00010_TimestampUnwrapVectorsTest;
+import com.shimmerresearch.driverUtilities.API_00012_SdTimestampAnchorTest;
import com.shimmerresearch.verisense.API_00004_VerisenseConfigByteParsingAndGeneration;
import com.shimmerresearch.verisense.communication.API_00003_VerisenseProtocolByteCommunicationTest;
@@ -16,7 +17,8 @@
API_00003_VerisenseProtocolByteCommunicationTest.class,
API_00004_VerisenseConfigByteParsingAndGeneration.class,
API_00009_TimestampUnwrapTest.class,
- API_00010_TimestampUnwrapVectorsTest.class
+ API_00010_TimestampUnwrapVectorsTest.class,
+ API_00012_SdTimestampAnchorTest.class
})
public class API_00005_Suite_ShimmerDriver {
diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java
new file mode 100644
index 000000000..03e84cb55
--- /dev/null
+++ b/ShimmerDriver/src/test/java/com/shimmerresearch/driverUtilities/API_00012_SdTimestampAnchorTest.java
@@ -0,0 +1,193 @@
+package com.shimmerresearch.driverUtilities;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+/**
+ * Covers {@link SdTimestampAnchor}: an SD data file's records are placed at their
+ * own counter time, not pinned to the header's file-creation time. The defect it
+ * fixes (DEV-1095) was a permanent -158.8 ms step at the 000 to 001 file split of
+ * a Shimmer3R recording.
+ *
+ * @author Mark Nolan
+ */
+public class API_00012_SdTimestampAnchorTest {
+
+ private static final int MAX_3_BYTE = TimestampUnwrap.TICKS_MAX_3_BYTE; // 2^24
+ private static final int MAX_2_BYTE = 65536;
+ /** A full 40-bit counter value, well away from any 24-bit boundary. */
+ private static final long INITIAL_TS = 0x12_3456_7000L;
+ /** 32 ticks: one sample period at 1024 Hz. */
+ private static final long PERIOD_TICKS = 32;
+
+ /** Time the parser gives a record: header + unwrapped - offset. */
+ private static double placed(long initialTs, double unwrapped, double offset) {
+ return initialTs + unwrapped - offset;
+ }
+
+ private static long low24(long ticks) {
+ return ticks & (MAX_3_BYTE - 1);
+ }
+
+ @Test
+ public void firstRecordSampledAfterHeaderLandsOnItsOwnTime() {
+ double raw = low24(INITIAL_TS) + 1000;
+ double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE);
+ assertEquals(INITIAL_TS + 1000, placed(INITIAL_TS, raw, offset), 0);
+ }
+
+ @Test
+ public void firstRecordSampledBeforeHeaderLandsBeforeIt() {
+ // File 000: records buffered during SD start-up predate the header by ~160 ms.
+ double raw = low24(INITIAL_TS) - 5243;
+ double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE);
+ assertEquals(INITIAL_TS - 5243, placed(INITIAL_TS, raw, offset), 0);
+ }
+
+ @Test
+ public void leadAcrossForwardWrapOfLowBits() {
+ long initialTs = (0x42L << 24) + MAX_3_BYTE - 100; // low bits 100 below the top
+ double raw = 200; // the counter wrapped 300 ticks after the header
+ double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE);
+ assertEquals(initialTs + 300, placed(initialTs, raw, offset), 0);
+ }
+
+ @Test
+ public void leadAcrossBackwardWrapOfLowBits() {
+ long initialTs = (0x42L << 24) + 50; // header just after a wrap
+ double raw = MAX_3_BYTE - 100; // record sampled 150 ticks before it
+ double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE);
+ assertEquals(initialTs - 150, placed(initialTs, raw, offset), 0);
+ }
+
+ @Test
+ public void laterRecordsKeepTheirSpacing() {
+ double raw = low24(INITIAL_TS) - 5243;
+ double offset = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE);
+ double first = placed(INITIAL_TS, raw, offset);
+ double tenth = placed(INITIAL_TS, raw + 9 * PERIOD_TICKS, offset);
+ assertEquals(9 * PERIOD_TICKS, tenth - first, 0);
+ }
+
+ /**
+ * Two consecutive files of one continuous recording. File 000's header is
+ * written 5800 ticks (177 ms) after its first record; file 001's 577 ticks
+ * (17.6 ms) after the split's first buffered record. Pinning each file to its
+ * header puts a step of the difference at the boundary; anchoring on the
+ * records' own counter keeps exactly one sample period.
+ */
+ @Test
+ public void noStepAtFileSplit() {
+ long c0 = INITIAL_TS;
+ long lastOf000 = c0 + 1000 * PERIOD_TICKS;
+ long firstOf001 = lastOf000 + PERIOD_TICKS;
+ long header000 = c0 + 5800;
+ long header001 = firstOf001 + 577;
+
+ double off000 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header000, low24(c0), MAX_3_BYTE);
+ double off001 = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(header001, low24(firstOf001), MAX_3_BYTE);
+ // Unwrapped values within a file: raw first + elapsed (no wrap inside this span).
+ double end000 = placed(header000, low24(c0) + (lastOf000 - c0), off000);
+ double start001 = placed(header001, low24(firstOf001), off001);
+
+ assertEquals(lastOf000, end000, 0);
+ assertEquals(firstOf001, start001, 0);
+ assertEquals(PERIOD_TICKS, start001 - end000, 0);
+
+ // The previous rule - subtract the raw first timestamp - stepped back here.
+ double oldEnd000 = placed(header000, low24(c0) + (lastOf000 - c0), low24(c0));
+ double oldStart001 = placed(header001, low24(firstOf001), low24(firstOf001));
+ assertEquals(PERIOD_TICKS - (5800 - 577), oldStart001 - oldEnd000, 0);
+ }
+
+ /**
+ * Checks one real file split: each file's first packet lands on its own
+ * counter value, the split is one sample period, and pinning each file to its
+ * header - the previous rule - gives the step the Consensys export showed.
+ *
+ * @param oldStepTicks the previous rule's step beyond one sample period, i.e.
+ * the difference between the two files' leads
+ */
+ private static void assertRealSplit(long headerA, long firstRawA, long firstCounterA, long lastCounterA,
+ long headerB, long firstRawB, long firstCounterB, long oldStepTicks) {
+ double offA = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(headerA, firstRawA, MAX_3_BYTE);
+ double offB = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(headerB, firstRawB, MAX_3_BYTE);
+ assertEquals(firstCounterA, placed(headerA, firstRawA, offA), 0);
+ assertEquals(firstCounterB, placed(headerB, firstRawB, offB), 0);
+
+ // Unwrapped within file A: the first raw value plus the ticks elapsed.
+ double unwrappedLastA = firstRawA + (lastCounterA - firstCounterA);
+ double endA = placed(headerA, unwrappedLastA, offA);
+ assertEquals(lastCounterA, endA, 0);
+ assertEquals(PERIOD_TICKS, placed(headerB, firstRawB, offB) - endA, 0);
+
+ // The previous rule reproduced the export's step.
+ double oldStep = placed(headerB, firstRawB, firstRawB) - placed(headerA, unwrappedLastA, firstRawA);
+ assertEquals(PERIOD_TICKS + oldStepTicks, oldStep, 0);
+ }
+
+ /**
+ * Header and packet values read from the raw 000 and 001 files of the
+ * Shimmer3R recording DEV-1095 was raised on (CE2F, 1024 Hz, 2026-09-25).
+ * File 000's first packet predates its header by 5332 ticks (162.72 ms), file
+ * 001's by 129 (3.94 ms); pinned to their headers, the files stepped back
+ * 5203 ticks (158.78 ms), exactly the step seen in the Consensys export.
+ * Anchored, the split is one sample period.
+ */
+ @Test
+ public void realShimmer3rRecordingSplitsByOnePeriod() {
+ assertRealSplit(391630116578L, 16335374L, 391630111246L, 391748082222L, // 000: 3,675,280 records
+ 391748082383L, 88654L, 391748082254L, -5203);
+ }
+
+ /**
+ * The 041 and 042 files of the same recording: leads of 214 ticks (6.53 ms)
+ * and 131 (4.00 ms). The previous rule's step was only -83 ticks, which the
+ * export showed as timestamps going back 1.556 ms, about 27 ms before the
+ * post-split stall. Anchored, this split is one sample period too.
+ */
+ @Test
+ public void realShimmer3rSplitWithSmallLeadDifference() {
+ assertRealSplit(396466711492L, 4319982L, 396466711278L, 396584676654L,
+ 396584676817L, 4844878L, 396584676686L, -83);
+ }
+
+ @Test
+ public void strokareCorrectedHeaderStillPinsExactly() {
+ // The old-StroKare workaround rewrites the header's low bits to the first
+ // record's, so the lead is zero and the result is unchanged.
+ double raw = 0x00_ABCDEF;
+ long initialTs = (INITIAL_TS & 0xFFFF000000L) + (long) raw;
+ assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(initialTs, raw, MAX_3_BYTE), 0);
+ }
+
+ @Test
+ public void twoByteCounterKeepsPreviousBehaviour() {
+ assertEquals(1234.0, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, 1234, MAX_2_BYTE), 0);
+ }
+
+ @Test
+ public void zeroInitialTimestampKeepsPreviousBehaviour() {
+ double raw = 0x00_F00000;
+ assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(0, raw, MAX_3_BYTE), 0);
+ }
+
+ @Test
+ public void implausibleLeadKeepsPreviousBehaviour() {
+ double raw = low24(INITIAL_TS) + SdTimestampAnchor.MAX_LEAD_TICKS + 1;
+ assertEquals(raw, SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE), 0);
+ raw = low24(INITIAL_TS) + SdTimestampAnchor.MAX_LEAD_TICKS;
+ assertEquals(raw - SdTimestampAnchor.MAX_LEAD_TICKS,
+ SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(INITIAL_TS, raw, MAX_3_BYTE), 0);
+ }
+
+ @Test
+ public void signedLeadIsFoldedIntoHalfRange() {
+ assertEquals(0, SdTimestampAnchor.signedLeadTicks(INITIAL_TS, low24(INITIAL_TS), MAX_3_BYTE));
+ assertEquals(-MAX_3_BYTE / 2,
+ SdTimestampAnchor.signedLeadTicks(0, MAX_3_BYTE / 2, MAX_3_BYTE));
+ assertEquals(MAX_3_BYTE / 2 - 1,
+ SdTimestampAnchor.signedLeadTicks(0, MAX_3_BYTE / 2 - 1, MAX_3_BYTE));
+ }
+}