Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import android.content.Context;
import android.os.Build;
import android.os.Bundle;
import android.os.SystemClock;

import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.platform.app.InstrumentationRegistry;
Expand All @@ -26,6 +27,10 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReference;

/**
* Hardware-in-the-loop scenarios against one real Shimmer3 or Shimmer3R, paired with the phone over
Expand All @@ -40,6 +45,7 @@
* cycles mid_stream_set / reconnect_loop / in_stream repetitions, default 3 / 5 / 3
* crc link CRC to ask for: off, 1 or 2 (default 2; the driver may settle on less)
* maxLossPercent default 1.0
* queueStressSeconds how long queue_stress queues reads, default 60
* accelTolerance m/s^2 either side of 9.81 at rest, default 1.0
* </pre>
*
Expand Down Expand Up @@ -600,6 +606,209 @@ private static double[] toArray(List<Double> values) {
return out;
}

/** The rates samplingRates writes in turn, as the Java and shimmer-logandstream HILs' scenarios of the same name do */
private static final double[] SAMPLING_RATES_HZ = {51.2, 256, 512};

/**
* Writes each rate in SAMPLING_RATES_HZ in turn, the way the examples configure, and streams 5 s
* at each; then writes the unit's own rate back. Every configure but the first comes straight
* after a stopStreaming(), as an app that changes the rate does. Up to driver 0.11.10_beta the
* stop's reply then retired the configuration's first chunk, the one holding the sampling rate,
* and the unit kept its old rate (DEV-1306). No SD card is needed.
*/
@Test
public void samplingRates() throws Exception {
begin("sampling_rates");
double maxLoss = arg("maxLossPercent", 1.0);
long start = System.currentTimeMillis();
boolean connected = session.connect(30000);
report.put("connectMs", System.currentTimeMillis() - start);
report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")");
ShimmerBluetooth d = connected ? session.device() : null;
if (d != null) {
report.put("hardware", d.getHardwareVersionParsed());
report.put("firmware", d.getFirmwareVersionParsed());
report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode()));
double ownRate = d.getSamplingRateShimmer();
report.put("ownRateHz", ownRate);
List<String> streams = new ArrayList<>();
for (double rate : SAMPLING_RATES_HZ) {
if (session.hung != null) {
break;
}
String what = String.format(Locale.ROOT, "%.1f Hz", rate);
if (!configureRate(rate, what)) {
continue;
}
double deviceRate = session.rateOnDevice();
String marker = session.logMarker("sampling_rates " + what);
session.startRecording();
report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s");
Thread.sleep(5000);
double stoppedAtMs = session.nowMs();
List<StreamAnalysis.Sample> samples = session.stopRecording();
report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s");
StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate);
session.event(what + " stream: " + a);
streams.add(what + ": " + a);
checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what);
int parserErrors = session.parserErrorsSince(marker);
report.check(parserErrors == 0, what + ": " + parserErrors + " streaming parser errors logged");
}
report.put("streams", new org.json.JSONArray(streams));
if (session.hung == null) {
configureRate(ownRate, "the unit's own rate");
}
report.put("rateOnDeviceHz", session.rateOnDevice());
report.check(session.disconnect(10000), "did not disconnect within 10 s");
}
finish();
}

/**
* Writes rateHz and checks the unit took it. The configure ends by reading the configuration
* back, so the API's rate is then the unit's. configureAccelOnly can finish on the manager's
* SUCCESS alone, without looking at the rate, so the rate is checked here.
*/
private boolean configureRate(double rateHz, String what) throws InterruptedException {
boolean configured = session.configureAccelOnly(rateHz, 20000);
double onDevice = session.rateOnDevice();
report.check(configured, String.format(Locale.ROOT, "%s: the configuration did not complete within 20 s (rate on device %.2f Hz, state %s)",
what, onDevice, session.state()));
boolean took = Math.abs(onDevice - rateHz) < 0.5;
report.check(!configured || took, String.format(Locale.ROOT,
"%s: the unit kept %.2f Hz after %.2f Hz was written (straight after a stop, that is DEV-1306)", what, onDevice, rateHz));
return configured && took;
}

/** queueStress: the app threads, the most instructions they keep queued, and how long a stuck call or queue takes to count as one */
private static final int QUEUE_STRESS_THREADS = 4;
private static final int QUEUE_STRESS_MAX_QUEUED = 8;
private static final long QUEUE_STRESS_STALL_MS = 10000;

/**
* DEV-1266: the driver guards its instruction queue with two locks, the device and the queue. Its
* IO thread took them queue first when it fetched the next instruction, and every call that queues
* one takes them device first, so the two could deadlock and every later call hung (5 of 12
* inStream runs on driver 0.11.7). With the unit connected and idle, timers on as in an app, four
* threads queue battery and status reads in tight loops for queueStressSeconds (default 60),
* keeping the queue under 8, so a call is nearly always taking the locks while the IO thread
* fetches. Idle, because while streaming Android writes both reads straight out, unqueued. No SD
* card is needed.
*/
@Test
public void queueStress() throws Exception {
begin("queue_stress");
int seconds = arg("queueStressSeconds", 60);
long start = System.currentTimeMillis();
boolean connected = session.connect(30000);
report.put("connectMs", System.currentTimeMillis() - start);
report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")");
final ShimmerBluetooth d = connected ? session.device() : null;
if (d != null) {
report.put("hardware", d.getHardwareVersionParsed());
report.put("firmware", d.getFirmwareVersionParsed());
report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode()));
final AtomicBoolean stop = new AtomicBoolean();
final AtomicLong calls = new AtomicLong();
// When each thread's current call began (elapsedRealtime), or 0 between calls
final AtomicLongArray inCallSince = new AtomicLongArray(QUEUE_STRESS_THREADS);
final AtomicReference<Throwable> thrown = new AtomicReference<>();
List<Thread> threads = new ArrayList<>();
for (int i = 0; i < QUEUE_STRESS_THREADS; i++) {
final int k = i;
Thread t = new Thread(() -> {
try {
while (!stop.get()) {
if (session.queuedInstructions() >= QUEUE_STRESS_MAX_QUEUED) {
Thread.sleep(1);
continue;
}
inCallSince.set(k, SystemClock.elapsedRealtime());
if ((calls.get() + k) % 2 == 0) {
d.readBattery();
} else {
d.readStatusLogAndStream();
}
inCallSince.set(k, 0);
calls.incrementAndGet();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Throwable e) {
thrown.compareAndSet(null, e);
stop.set(true);
}
}, "QueueStress" + i);
t.setDaemon(true); // a deadlocked thread must not keep the test process alive
threads.add(t);
}
session.event("queue stress: " + QUEUE_STRESS_THREADS + " threads for " + seconds + " s");
Object battery = session.batteryDetails();
int batteryReplies = 0;
int maxQueued = 0;
long lastCalls = 0;
long lastProgress = SystemClock.elapsedRealtime();
long end = lastProgress + seconds * 1000L;
String stall = null;
for (Thread t : threads) {
t.start();
}
while (SystemClock.elapsedRealtime() < end && stall == null && !stop.get()) {
Thread.sleep(100);
long now = SystemClock.elapsedRealtime();
Object b = session.batteryDetails();
if (b != null && b != battery) {
battery = b;
batteryReplies++;
}
int queued = session.queuedInstructions();
maxQueued = Math.max(maxQueued, queued);
long c = calls.get();
if (c != lastCalls) {
lastCalls = c;
lastProgress = now;
}
for (int i = 0; i < QUEUE_STRESS_THREADS && stall == null; i++) {
long since = inCallSince.get(i);
if (since != 0 && now - since > QUEUE_STRESS_STALL_MS) {
stall = String.format(Locale.ROOT, "a read queued by QueueStress%d has not returned for %.0f s", i, (now - since) / 1000.0);
}
}
if (stall == null && now - lastProgress > QUEUE_STRESS_STALL_MS) {
stall = String.format(Locale.ROOT, "no read returned for %.0f s, with %d instructions queued", (now - lastProgress) / 1000.0, queued);
}
}
stop.set(true);
report.put("calls", calls.get());
report.put("batteryReplies", batteryReplies);
report.put("maxQueued", maxQueued);
session.event(String.format(Locale.ROOT, "queue stress: %d reads queued, %d battery replies seen, at most %d instructions queued",
calls.get(), batteryReplies, maxQueued));
if (stall != null) {
session.hung = "readBattery/readStatusLogAndStream (queueStress)";
session.event(stall + "; thread dump follows");
session.dumpThreads();
report.check(false, stall + " (the calls and the IO thread are stuck; DEV-1266's deadlock if the dump shows them waiting on each other)");
} else {
for (Thread t : threads) {
t.join(2000);
}
report.check(thrown.get() == null, "a read threw " + thrown.get());
report.check(calls.get() > 0, "no read was queued");
boolean drained = session.waitFor(() -> session.queuedInstructions() == 0, 15000);
report.check(drained, "the queue did not drain within 15 s after the reads stopped ("
+ session.queuedInstructions() + " left; the IO thread is stuck, see the thread dump in the events)");
if (!drained) {
session.event("the queue did not drain; thread dump follows");
session.dumpThreads();
}
report.check(session.disconnect(10000), "did not disconnect within 10 s");
}
}
finish();
}

/** Connect, stream briefly and disconnect, repeatedly: the reconnect path apps hit every day. */
@Test
public void reconnectLoop() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ boolean call(String what, long timeoutMs, Call call) {
}
}

private void dumpThreads() {
void dumpThreads() {
for (java.util.Map.Entry<Thread, StackTraceElement[]> t : Thread.getAllStackTraces().entrySet()) {
StringBuilder b = new StringBuilder(" thread \"" + t.getKey().getName() + "\" " + t.getKey().getState());
for (StackTraceElement frame : t.getValue()) {
Expand Down
2 changes: 2 additions & 0 deletions tools/hil/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ To find regressions before release day, schedule step 2 nightly with Windows Tas
| `reconnectLoop` | Connects, streams for 5 s and disconnects, five times. | Any cycle fails to connect within 30 s, loses packets, fails the rate and silence checks, or logs a parser error. |
| `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. |
| `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. |
| `queueStress` | Connected and idle, with the API's timers on as in an app: four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Each queued read takes the device's lock and then the queue's while the IO thread fetches the next instruction, the window DEV-1266's lock-order deadlock needs. Idle, because while streaming Android writes these reads straight out, unqueued. | A read doesn't return within 10 s, or no read returns for 10 s (the report then holds every thread's stack); a read throws; or the queue doesn't drain within 15 s after the reads stop. No SD card needed. On driver 0.11.7_beta, which has the deadlock, this is the scenario to reproduce it. |
| `samplingRates` | Writes 51.2, 256 and 512 Hz in turn, the way the examples configure, and streams 5 s at each; then writes the unit's own rate back. Every configure but the first comes straight after a `stopStreaming()`, as an app that changes the rate does: up to driver 0.11.10_beta the stop's reply then retired the configuration's first chunk, the one holding the sampling rate (DEV-1306). | A configuration doesn't complete within 20 s, or the rate read back from the unit isn't the one written; or a stream fails `streamRest`'s loss, timestamp, rate and silence checks, or logs a parser error. No SD card needed. |

`-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects.

Expand Down
4 changes: 2 additions & 2 deletions tools/hil/run-android-hil.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ Before "Release Android API": the full run on this commit, recorded on GitHub as
#>
param(
[string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'),
[ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'resetUnit')]
[string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady'),
[ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates', 'resetUnit')]
[string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates'),
[string[]]$Unit,
[ValidateSet('classic', 'ble')]
[string[]]$Transport,
Expand Down