Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import com.shimmerresearch.driverUtilities.SensorDetailsRef;
import com.shimmerresearch.driverUtilities.SensorGroupingDetails;
import com.shimmerresearch.driverUtilities.SensorDetails;
import com.shimmerresearch.driverUtilities.SdTimestampAnchor;
import com.shimmerresearch.driverUtilities.ShimmerSDCardDetails;
import com.shimmerresearch.driverUtilities.ShimmerVerDetails;
import com.shimmerresearch.driverUtilities.ShimmerVerObject;
Expand Down Expand Up @@ -653,13 +654,13 @@ public String toString() {

protected boolean mFirstTime = true;

/** This variable was originally implemented because the
* initial time in the SD header (i.e., the time when the header
* was created) wasn't equal to the first timestamp in the
* subsequent data packets (i.e., the lower 3 bytes of the
* initial timestamp). This problem has since been addressed in
* firmware whereby the header is updated with the timestamp
* from the first packet. Therefore this variable is redundant. */
/** Subtracted, with the header's initial timestamp added, from each
* unwrapped SD timestamp so that the file's records land on their own
* counter time. It is not redundant: the SD header holds the RTC at the
* time the file was created, not the first packet's timestamp - the
* firmware captures the latter but never writes it back - so it is set by
* {@link SdTimestampAnchor#firstTsOffsetFromInitialTsTicks} from the
* header's low bits and the first packet's raw timestamp (DEV-1095). */
protected double mFirstTsOffsetFromInitialTsTicks = 0;
public int OFFSET_LENGTH = 9;
//-------- Timestamp related end --------
Expand Down Expand Up @@ -2821,17 +2822,19 @@ protected void parseTimestampShimmer3(COMMUNICATION_TYPE fwType, ObjectCluster o
objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.UNCAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks);
objectCluster.addDataToMap(Shimmer3.ObjectClusterSensorName.SHIMMER_CLOCK,CHANNEL_TYPE.CAL.toString(),CHANNEL_UNITS.CLOCK_UNIT,shimmerTimestampTicks);
if(mFirstTime && fwType==COMMUNICATION_TYPE.SD){
//this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks.
mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks;

//This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1
//This is to circumvent a bug with old StroKare firmware. Resolved in firmware v1.0.1
if(getFirmwareIdentifier()==FW_ID.STROKARE
&& !isThisVerCompatibleWith(FW_ID.STROKARE, 1, 0, 1)){
long initialTsTicksOriginal = getInitialTimeStampTicksSd();
long initialTsTicksNew = (long) ((initialTsTicksOriginal&0xFFFF000000L)+shimmerTimestampTicks);
setInitialTimeStampTicksSd(initialTsTicksNew);
}


//Anchors the file on the first packet's own counter time rather than on
//the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks.
mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(
getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue);

mFirstTime = false;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package com.shimmerresearch.driverUtilities;

/**
* Places the first record of an SD data file at the time it was actually
* sampled.
* <p>
* An SD data file's absolute timeline is its header's initial timestamp plus the
* elapsed ticks of its records. The driver used to anchor that timeline by
* subtracting the first record's raw timestamp, which pins the first record to
* the header time. That is only right if the header holds the first record's
* time, and LogAndStream does not write that: the header carries the RTC at the
* moment the file was <em>created</em> (<code>sdFileSyncTs</code>, set in
* <code>ShimSdDataFile_fileInit</code> and at each hourly split). Records
* already buffered when the file is opened were sampled before that moment, so
* pinning erases a lead time that differs from file to file:
* <ul>
* <li><b>File 000</b> on a Shimmer3R is created after sampling has started -
* SD power-up, directory creation, header build - so its first records predate
* the header by roughly 160 ms more than at a mid-stream split. A Shimmer3
* (LogAndStream v1.1.5) takes file 000's header 1.8 ms before its first record
* instead.</li>
* <li><b>Later files</b> are opened mid-stream, behind only the records still in
* the SD write buffer.</li>
* </ul>
* The difference surfaces as a permanent step at each file boundary: -158.8 ms
* at the 000 to 001 split of a 66.8 hour Shimmer3R recording (DEV-1095), and
* +2.7 ms at the same split of a 32 hour Shimmer3 one.
* <p>
* Nothing is lost to fix it. A record's 3-byte timestamp is the low 24 bits of
* the same 32768 Hz counter the header's initial timestamp reads, so the first
* record's full counter value is the header value moved by the signed, wrap-
* corrected distance between the two low parts. That is exact whenever the
* header lies within half a counter period (256 s) of the first record, and it
* does not depend on which moment the firmware chose for the header - so it
* stays right if the firmware later writes the first record's time instead.
* <p>
* The reconstruction is in the device's counter domain. It must be applied to
* the header's initial timestamp <em>before</em> the RTC difference is added: on
* Shimmer3 that difference is the offset from the free-running counter to real
* time, and it is not a multiple of the counter period.
* <p>
* Verified against raw files on both platforms. Shimmer3R: the first records of
* 000 and 001 were sampled 162.72 ms and 3.94 ms before their headers (a test
* vector in API_00012), and anchored this way the split is one sample period.
* Shimmer3 (LogAndStream v1.1.5, 1024 Hz, 33 files): file 000's first record was
* sampled 1.831 ms after its header and every later file's 0.885 ms before, so
* the header shares the record counter there too, and all 32 splits are exactly
* one sample period.
* <p>
* Pure and static so that both of the driver's timestamp paths -
* <code>ShimmerObject</code> and <code>SensorShimmerClock</code> - share one rule,
* as with {@link TimestampUnwrap}.
*
* @author Mark Nolan
*/
public final class SdTimestampAnchor {

/**
* Largest distance between the header's initial timestamp and the first
* record that is read as a lead time: 10 s at 32768 Hz. The real lead is well
* under a second - the buffered records, plus SD start-up for file 000 - so a
* larger distance means the header does not describe this counter, and the
* first record keeps the header time as it always has.
*/
public static final long MAX_LEAD_TICKS = 10L * 32768L;

private SdTimestampAnchor() {
}

/**
* The value to subtract from <code>unwrappedTicks + initialTsTicks</code> so
* that each record of an SD data file lands on its own counter time. For the
* first record that is <code>initialTsTicks + d</code>, where <code>d</code> is
* the signed modular distance from the header's low bits to the record's raw
* timestamp.
* <p>
* Falls back to the first record's raw timestamp - pinning the first record
* to the header time, the previous behaviour - when there is nothing to
* reconstruct from:
* <ul>
* <li>a 2-byte counter, whose firmware predates the current header and whose
* 2 s period leaves no useful margin;</li>
* <li>an initial timestamp of zero, which some log formats write in place of a
* time;</li>
* <li>a distance larger than {@link #MAX_LEAD_TICKS}.</li>
* </ul>
*
* @param initialTsTicks the header's initial timestamp, in counter ticks,
* without any RTC difference added
* @param firstRawTicks the first record's raw (wrapped) timestamp
* @param ticksMaxValue the counter's modulo: 2^24 for a 3-byte timestamp
* @return the offset to subtract, in ticks
*/
public static double firstTsOffsetFromInitialTsTicks(long initialTsTicks, double firstRawTicks,
int ticksMaxValue) {
if (ticksMaxValue != TimestampUnwrap.TICKS_MAX_3_BYTE || initialTsTicks == 0) {
return firstRawTicks;
}
long lead = signedLeadTicks(initialTsTicks, (long) firstRawTicks, ticksMaxValue);
if (Math.abs(lead) > MAX_LEAD_TICKS) {
return firstRawTicks;
}
return firstRawTicks - lead;
}

/**
* Signed distance from the header's low bits to the first record's raw
* timestamp, folded into <code>[-ticksMaxValue/2, ticksMaxValue/2)</code>:
* positive when the record was sampled after the header was written, negative
* when it was sampled before.
*/
static long signedLeadTicks(long initialTsTicks, long firstRawTicks, int ticksMaxValue) {
long half = ticksMaxValue / 2;
long low = Math.floorMod(initialTsTicks, (long) ticksMaxValue);
return Math.floorMod(firstRawTicks - low + half, (long) ticksMaxValue) - half;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.shimmerresearch.driverUtilities.SensorDetails;
import com.shimmerresearch.driverUtilities.SensorDetailsRef;
import com.shimmerresearch.driverUtilities.SensorGroupingDetails;
import com.shimmerresearch.driverUtilities.SdTimestampAnchor;
import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID;
import com.shimmerresearch.driverUtilities.TimestampUnwrap;
import com.shimmerresearch.driverUtilities.UtilParseData;
Expand Down Expand Up @@ -49,6 +50,8 @@ public class SensorShimmerClock extends AbstractSensor {
protected transient boolean mLastTimestampRejected = false;

protected boolean mFirstTime = true;
/** Anchors an SD file's records on their own counter time - see
* {@link SdTimestampAnchor} and the matching field in ShimmerObject (DEV-1095). */
double mFirstTsOffsetFromInitialTsTicks = 0;
double mSystemTimeStamp = 0;
public int OFFSET_LENGTH = 9;
Expand Down Expand Up @@ -368,8 +371,10 @@ public ObjectCluster processDataCustom(SensorDetails sensorDetails, byte[] senso
double shimmerTimestampTicks = UtilParseData.parseData(channelByteArray, channelDetails.mDefaultChannelDataType, channelDetails.mDefaultChannelDataEndian);

if(mFirstTime && commType==COMMUNICATION_TYPE.SD){
//this is to make sure the Raw starts from zero for SD data. See comment for mFirstTsOffsetFromInitialTsTicks.
mFirstTsOffsetFromInitialTsTicks = shimmerTimestampTicks;
//Anchors the file on the first packet's own counter time rather than on
//the header's file-creation time. See comment for mFirstTsOffsetFromInitialTsTicks.
mFirstTsOffsetFromInitialTsTicks = SdTimestampAnchor.firstTsOffsetFromInitialTsTicks(
getInitialTimeStampTicksSd(), shimmerTimestampTicks, mTimeStampTicksMaxValue);
mFirstTime = false;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.shimmerresearch.algorithms.API_00002_Filters;
import com.shimmerresearch.driverUtilities.API_00009_TimestampUnwrapTest;
import com.shimmerresearch.driverUtilities.API_00010_TimestampUnwrapVectorsTest;
import com.shimmerresearch.driverUtilities.API_00012_SdTimestampAnchorTest;
import com.shimmerresearch.verisense.API_00004_VerisenseConfigByteParsingAndGeneration;
import com.shimmerresearch.verisense.communication.API_00003_VerisenseProtocolByteCommunicationTest;

Expand All @@ -16,7 +17,8 @@
API_00003_VerisenseProtocolByteCommunicationTest.class,
API_00004_VerisenseConfigByteParsingAndGeneration.class,
API_00009_TimestampUnwrapTest.class,
API_00010_TimestampUnwrapVectorsTest.class
API_00010_TimestampUnwrapVectorsTest.class,
API_00012_SdTimestampAnchorTest.class
})

public class API_00005_Suite_ShimmerDriver {
Expand Down
Loading
Loading