diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83c66e2f8..7f1dcf178 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,9 @@ # One-click release of the Java API: GitHub Packages, a GitHub Release, and the MATLAB jar. # # How to release +# 0. On the bench, at the commit to release: tools/hil/run-java-hil.ps1 -Report (tools/hil/README.md). +# It posts the commit status "java-hil"; the build job stops unless that passed, or hil_waiver +# gives a Jira key (the waiver goes into the release notes). Dry runs only warn. # 1. Actions -> "Release Java API" -> Run workflow (on master). Leave `version` blank for the next # patch after the last API_v release (0.11.8_beta -> 0.11.9_beta), or type one. # 2. Review and merge the ShimmerJavaClass.jar PR it opens in Shimmer-MATLAB-ID. @@ -46,6 +49,10 @@ on: description: 'Build and test only; publish nothing' type: boolean default: false + hil_waiver: + description: 'Jira key to release without a passing hardware test (HIL), e.g. DEV-1234. Blank = require one' + type: string + required: false pull_request: paths: - '.github/workflows/release.yml' @@ -62,12 +69,14 @@ jobs: # job's token. Only the publish job, which runs on real releases alone, can write packages. permissions: contents: read + statuses: read # the hardware test result the bench posts on the commit env: DRY_RUN: ${{ github.event_name != 'workflow_dispatch' || inputs.dry_run }} outputs: version: ${{ steps.version.outputs.version }} tag: ${{ steps.version.outputs.tag }} previous_tag: ${{ steps.version.outputs.previous_tag }} + hil: ${{ steps.hil.outputs.hil }} steps: - name: Release from master only if: env.DRY_RUN != 'true' && github.ref != 'refs/heads/master' @@ -121,6 +130,35 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Releasing $VERSION as $TAG (notes since ${PREVIOUS_TAG:-the first commit})" + - name: Check the hardware test (HIL) result + id: hil + env: + GH_TOKEN: ${{ github.token }} + WAIVER: ${{ inputs.hil_waiver }} + run: | + context=java-hil + if [ -n "$WAIVER" ] && ! [[ "$WAIVER" =~ ^[A-Z][A-Z0-9]+-[0-9]+$ ]]; then + echo "::error::hil_waiver must be a Jira key such as DEV-1234, not '$WAIVER'." + exit 1 + fi + # Newest first: the latest result for this context on this exact commit decides. + line=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/statuses" --paginate \ + --jq '.[] | [.context, .state, .description] | @tsv' | awk -F'\t' -v c="$context" '$1 == c { print; exit }') + state=$(printf '%s' "$line" | cut -f2) + description=$(printf '%s' "$line" | cut -f3) + if [ "$state" = success ]; then + echo "Hardware test passed on $GITHUB_SHA: $description" + echo "hil=Tested on hardware before release: $description." >> "$GITHUB_OUTPUT" + elif [ -n "$WAIVER" ]; then + echo "::warning::No passing '$context' on $GITHUB_SHA (found: ${state:-none}); released under waiver $WAIVER." + echo "hil=Released without a passing hardware test (HIL), waived under $WAIVER." >> "$GITHUB_OUTPUT" + elif [ "$DRY_RUN" = true ]; then + echo "::warning::No passing '$context' on $GITHUB_SHA (found: ${state:-none}). A real release would stop here." + else + echo "::error::No passing '$context' on $GITHUB_SHA (found: ${state:-none}${description:+: $description}). On the bench, at this commit, run: tools/hil/run-java-hil.ps1 -Report. Then run this workflow again, or give a Jira key in hil_waiver." + exit 1 + fi + - name: Set up JDK 11 uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 with: @@ -272,6 +310,7 @@ jobs: VERSION: ${{ needs.build.outputs.version }} TAG: ${{ needs.build.outputs.tag }} PREVIOUS_TAG: ${{ needs.build.outputs.previous_tag }} + HIL: ${{ needs.build.outputs.hil }} steps: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: @@ -286,16 +325,27 @@ jobs: - name: Create the tag and GitHub Release run: | cp matlab-jar/ShimmerJavaClass.jar "release-assets/ShimmerJavaClass-$VERSION.jar" + # The generated notes, with the hardware test result (or its waiver) in front. + args=(-f tag_name="$TAG" -f target_commitish="$GITHUB_SHA") + if [ -n "$PREVIOUS_TAG" ]; then args+=(-f previous_tag_name="$PREVIOUS_TAG"); fi + { + if [ -n "$HIL" ]; then + echo "$HIL" + echo + fi + gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" "${args[@]}" --jq .body + } > notes.md gh release create "$TAG" release-assets/* \ --repo "$GITHUB_REPOSITORY" \ --target "$GITHUB_SHA" \ --title "Shimmer Java API Rev $VERSION" \ - --generate-notes ${PREVIOUS_TAG:+--notes-start-tag "$PREVIOUS_TAG"} \ + --notes-file notes.md \ --latest=false { echo "### Released Java API $VERSION" echo "- Release: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$TAG" echo "- Packages: com.shimmerresearch:shimmerdriver, shimmerbluetoothmanager and shimmerdriverpc at $VERSION" + echo "- ${HIL:-No hardware test result}" echo "- Next: merge the MATLAB PR, then run Release Android API in ShimmerAndroidAPI; it builds against $VERSION unless you pick another driver version." } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 19c5b1390..a480c7ae5 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,7 @@ ShimmerAndroidInstrumentDriver/gen/com/shimmerresearch/androidinstrumentdriver/B ShimmerAndroidInstrumentDriver/gen/com/shimmerresearch/androidinstrumentdriver/R.java gradle.properties github.properties + +# Java HIL (tools/hil): the local rig file and run output +tools/hil/rig-java.json +tools/hil/out/ diff --git a/README.md b/README.md index ab212feab..708d6f538 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ Best examples to start with are For help with using on Ubuntu please check the [wiki](https://github.com/ShimmerEngineering/Shimmer-Java-Android-API/wiki). +## Hardware tests + +[`tools/hil`](tools/hil/README.md) runs the driver on real Shimmer3 and Shimmer3R units over +Bluetooth Classic: streaming, SD logging and in-stream commands while streaming, and reconnects, with +the driver's timers on. **Release Java API** requires a passing run on the commit it releases. + Rev 3.00 - Change to Handler MSGs diff --git a/ShimmerDriverPC/build.gradle b/ShimmerDriverPC/build.gradle index b60226f77..b452317e4 100644 --- a/ShimmerDriverPC/build.gradle +++ b/ShimmerDriverPC/build.gradle @@ -32,6 +32,26 @@ repositories { test { // this test requires a physical Shimmer2r to run exclude '**/API_00001_ShimmerPC_GeneralBluetoothShimmer2R.class' + // the hardware-in-the-loop scenarios need a unit; hilTest runs them (tools/hil/run-java-hil.ps1) + exclude '**/hil/JavaHilScenariosTest.class' +} + +// Hardware-in-the-loop scenarios on a real Shimmer over a paired Bluetooth serial port, run by +// tools/hil/run-java-hil.ps1 (see tools/hil/README.md). Each -Phil.= reaches the tests +// as the system property hil.. +tasks.register('hilTest', Test) { + description = 'Runs the hardware-in-the-loop scenarios on a real unit (tools/hil/run-java-hil.ps1)' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + include '**/hil/JavaHilScenariosTest.class' + dependsOn 'testClasses' + systemProperties project.properties.findAll { k, v -> k.startsWith('hil.') } + outputs.upToDateWhen { false } + testLogging { + events 'passed', 'skipped', 'failed' + showStandardStreams = true + exceptionFormat 'full' + } } publishing { @@ -103,6 +123,8 @@ dependencies { // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add // 'test.useTestNG()' to your build script. testImplementation 'junit:junit:4.12' + // the HIL reports (src/test/java/com/shimmerresearch/hil) + testImplementation 'com.google.code.gson:gson:2.10.1' implementation 'com.parse.bolts:bolts-tasks:1.4.0' // https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java new file mode 100644 index 000000000..683586284 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilReport.java @@ -0,0 +1,161 @@ +package com.shimmerresearch.hil; + +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +/** + * Everything one scenario run found: a JSON report, a CSV of the samples it judged and the + * driver's own log, written to the folder the runner passes (hil.outDir). + */ +final class HilReport { + + private final String scenario; + private final String unit; + private final JsonObject json = new JsonObject(); + private final List failures = new ArrayList<>(); + private List samples; + + HilReport(String scenario, String unit, String port, String mac, String type, Map args) { + this.scenario = scenario; + this.unit = unit; + json.addProperty("scenario", scenario); + json.addProperty("unit", unit); + json.addProperty("port", port); + json.addProperty("mac", mac); + json.addProperty("type", type); + json.addProperty("started", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ROOT).format(new Date())); + JsonObject a = new JsonObject(); + args.forEach(a::addProperty); + json.add("args", a); + } + + /** JSON has no NaN or infinity, and a failing run is the likeliest to produce one: store null. */ + static JsonElement num(double v) { + return Double.isNaN(v) || Double.isInfinite(v) ? JsonNull.INSTANCE : new JsonPrimitive(v); + } + + void put(String key, Object value) { + if (value == null) { + json.add(key, JsonNull.INSTANCE); + } else if (value instanceof Double || value instanceof Float) { + json.add(key, num(((Number) value).doubleValue())); + } else if (value instanceof Number) { + json.addProperty(key, (Number) value); + } else if (value instanceof Boolean) { + json.addProperty(key, (Boolean) value); + } else if (value instanceof Map) { + JsonObject o = new JsonObject(); + ((Map) value).forEach((k, v) -> o.addProperty(String.valueOf(k), v instanceof Number ? (Number) v : null)); + json.add(key, o); + } else if (value instanceof List) { + JsonArray arr = new JsonArray(); + for (Object o : (List) value) { + arr.add(o instanceof Number ? new JsonPrimitive((Number) o) : new JsonPrimitive(String.valueOf(o))); + } + json.add(key, arr); + } else { + json.addProperty(key, String.valueOf(value)); + } + } + + void putStream(String key, StreamAnalysis a) { + JsonObject o = new JsonObject(); + o.addProperty("received", a.received); + o.addProperty("missing", a.missing); + o.add("lossPercent", num(a.lossPercent())); + o.addProperty("nonIncreasing", a.nonIncreasing); + o.addProperty("timestampJumps", a.timestampJumps.size()); + o.add("effectiveRateHz", num(a.effectiveRateHz)); + o.add("accelMagnitudeMean", num(a.accelMagnitudeMean)); + o.add("accelMagnitudeStd", num(a.accelMagnitudeStd)); + o.addProperty("gaps", a.gaps.size()); + json.add(key, o); + } + + /** Every gap and timestamp jump, with when it arrived, to match against what the test was doing. */ + void putGaps(StreamAnalysis a) { + JsonArray gaps = new JsonArray(); + for (StreamAnalysis.Gap g : a.gaps) { + JsonObject o = new JsonObject(); + o.addProperty("afterReceivedMs", Math.round(g.afterReceivedMs)); + o.add("afterTimestampMs", num(g.afterTimestampMs)); + o.addProperty("missing", g.missing); + gaps.add(o); + } + json.add("gapList", gaps); + JsonArray jumps = new JsonArray(); + for (StreamAnalysis.Gap g : a.timestampJumps) { + JsonObject o = new JsonObject(); + o.addProperty("afterReceivedMs", Math.round(g.afterReceivedMs)); + o.add("afterTimestampMs", num(g.afterTimestampMs)); + o.addProperty("periodsAhead", g.missing + 1); + jumps.add(o); + } + json.add("timestampJumpList", jumps); + } + + void samples(List samples) { + this.samples = samples; + } + + /** Records a failed check; the test fails at the end with every one of them. */ + void check(boolean ok, String failure) { + if (!ok) { + failures.add(failure); + System.out.println(scenario + " " + unit + " FAIL: " + failure); + } + } + + List failures() { + return failures; + } + + /** Writes _.json, .csv (samples, if any) and .driver.log into outDir. */ + void write(File outDir, List events, List driverLog) { + try { + json.addProperty("passed", failures.isEmpty()); + JsonArray f = new JsonArray(); + failures.forEach(f::add); + json.add("failures", f); + JsonArray e = new JsonArray(); + events.forEach(e::add); + json.add("events", e); + if (!outDir.exists() && !outDir.mkdirs()) { + throw new IOException("cannot create " + outDir); + } + String base = scenario + "_" + unit.replaceAll("[^\\w.-]", "_"); + try (Writer w = Files.newBufferedWriter(new File(outDir, base + ".json").toPath(), StandardCharsets.UTF_8)) { + new GsonBuilder().setPrettyPrinting().create().toJson(json, w); + } + if (samples != null) { + try (Writer w = Files.newBufferedWriter(new File(outDir, base + ".csv").toPath(), StandardCharsets.UTF_8)) { + w.write("received_ms,timestamp_ms,accel_x,accel_y,accel_z\n"); + for (StreamAnalysis.Sample s : samples) { + w.write(String.format(Locale.ROOT, "%.0f,%.3f,%.4f,%.4f,%.4f\n", + s.receivedMs, s.timestampMs, s.accelX, s.accelY, s.accelZ)); + } + } + } + Files.write(new File(outDir, base + ".driver.log").toPath(), driverLog, StandardCharsets.UTF_8); + System.out.println("report written: " + new File(outDir, base + ".json")); + } catch (IOException ex) { + System.out.println("could not write report: " + ex); + } + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java new file mode 100644 index 000000000..b9a9b896b --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilSession.java @@ -0,0 +1,341 @@ +package com.shimmerresearch.hil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.driver.BasicProcessWithCallBack; +import com.shimmerresearch.driver.CallbackObject; +import com.shimmerresearch.driver.Configuration; +import com.shimmerresearch.driver.FormatCluster; +import com.shimmerresearch.driver.ObjectCluster; +import com.shimmerresearch.driver.ShimmerDevice; +import com.shimmerresearch.driver.ShimmerMsg; +import com.shimmerresearch.driverUtilities.AssembleShimmerConfig; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails; +import com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils; + +/** + * One unit on one serial port, driven through ShimmerPC as an app drives it. Every API call runs + * under a watchdog: if one does not return, the session records which, dumps every thread (with + * the locks each holds and waits on) and refuses further calls, so a deadlock ends the scenario + * with evidence instead of hanging the run. + */ +class HilSession extends BasicProcessWithCallBack implements AutoCloseable { + + interface Call { + void run() throws Exception; + } + + interface Condition { + boolean holds(); + } + + final String port; + final String mac; + final String type; + final HilShimmerPC device; + private final long t0 = System.nanoTime(); + private final List events = new ArrayList<>(); + private final List samples = new ArrayList<>(); + private final Set otherMessagesSeen = new HashSet<>(); + private final ExecutorService calls = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "HilCall"); + t.setDaemon(true); + return t; + }); + private volatile boolean recording; + private volatile BT_STATE lastState; + /** State changes reported so far, so a connect can tell its own CONNECTION_FAILED from the last one's. */ + private volatile int stateChanges; + /** The call that never returned, if one did not. */ + volatile String hung; + /** Connects made in this session, for the report: each is a chance for the connect-time race. */ + int connects; + /** Why the last connect failed, as a sentence for the report; null after one that succeeded. */ + volatile String connectFailure; + /** jssc's message when Windows refuses to open a COM port that another program already has open. */ + private static final Pattern PORT_BUSY = Pattern.compile("Port busy"); + + HilSession(String port, String mac, String type) { + this.port = port; + this.mac = mac; + this.type = type; + this.device = new HilShimmerPC(port, t0); + setWaitForData(device); + } + + double nowMs() { + return (System.nanoTime() - t0) / 1e6; + } + + void event(String text) { + String line = String.format(Locale.ROOT, "%9.0f %s", nowMs(), text); + synchronized (events) { + events.add(line); + } + System.out.println(port + " " + line); + } + + List events() { + synchronized (events) { + return new ArrayList<>(events); + } + } + + @Override + protected void processMsgFromCallback(ShimmerMsg msg) { + if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_STATE_CHANGE && msg.mB instanceof CallbackObject) { + BT_STATE state = ((CallbackObject) msg.mB).mState; + if (state != null) { + lastState = state; + stateChanges++; + event("state " + state); + } + } else if (msg.mIdentifier == HilShimmerPC.MSG_IDENTIFIER_DATA_PACKET && msg.mB instanceof ObjectCluster) { + if (recording) { + recordPacket((ObjectCluster) msg.mB); + } + } else if (otherMessagesSeen.add(msg.mIdentifier)) { + event("message " + msg.mIdentifier + (msg.mB == null ? "" : " " + msg.mB.getClass().getSimpleName())); + } + } + + private void recordPacket(ObjectCluster oc) { + StreamAnalysis.Sample s = new StreamAnalysis.Sample(nowMs(), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.TIMESTAMP), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_X), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Y), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Z)); + synchronized (samples) { + samples.add(s); + } + } + + private static double cal(ObjectCluster oc, String channel) { + Collection formats = oc.getCollectionOfFormatClusters(channel); + FormatCluster c = (FormatCluster) ObjectCluster.returnFormatCluster(formats, "CAL"); + return c == null ? Double.NaN : c.mData; + } + + BT_STATE state() { + BT_STATE s = device.getBluetoothRadioState(); + return s != null ? s : lastState; + } + + boolean waitFor(Condition condition, long timeoutMs) throws InterruptedException { + long deadline = System.nanoTime() + timeoutMs * 1_000_000L; + while (System.nanoTime() < deadline) { + if (condition.holds()) { + return true; + } + Thread.sleep(20); + } + return condition.holds(); + } + + /** + * Runs an API call under the watchdog. After a hung call nothing more is run: the driver is in + * an unknown state, and queuing calls behind the stuck one only adds stuck threads. + */ + boolean call(String what, long timeoutMs, Call call) { + if (hung != null) { + event("skipped " + what + ": " + hung + " never returned"); + return false; + } + event(what); + Future f = calls.submit(() -> { + call.run(); + return null; + }); + try { + f.get(timeoutMs, TimeUnit.MILLISECONDS); + return true; + } catch (TimeoutException e) { + hung = what; + event(what + " did not return within " + timeoutMs + " ms; every thread:\n" + + ByteCommunicationTestUtils.dumpAllThreads()); + return false; + } catch (java.util.concurrent.ExecutionException e) { + event(what + " threw " + e.getCause()); + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** + * Connects and waits until the driver has read the unit's configuration. A connect is never + * retried. When the port can't be opened, connectFailure says why. jssc's "Port busy" means + * another program on this PC has the port open (another HIL, Consensys, a serial terminal), so + * the unit is in use. Results taken while another host shares the unit mean nothing, and that is + * how DEV-1276 was misread. + */ + boolean connect(long timeoutMs) throws InterruptedException { + connects++; + connectFailure = null; + int mark = stateChanges; + int logMark = device.logSize(); + Condition failed = () -> stateChanges > mark && device.getBluetoothRadioState() == BT_STATE.CONNECTION_FAILED; + if (!call("connect on " + port, 10000, () -> device.connect(port, ""))) { + connectFailure = "connect on " + port + " did not return (see the events)"; + return false; + } + boolean ready = waitFor(() -> (device.isInitialised() && device.getBluetoothRadioState() == BT_STATE.CONNECTED) + || failed.holds(), timeoutMs) && !failed.holds(); + if (ready) { + event("connected: timers " + device.timers()); + } else { + if (device.countSince(logMark, PORT_BUSY) > 0) { + connectFailure = port + " is open in another program (another HIL, Consensys, a serial terminal?): " + + "jssc reports \"Port busy\". Nothing else may use the unit during a run."; + } else if (failed.holds()) { + connectFailure = "the Bluetooth link on " + port + " could not be opened (CONNECTION_FAILED): " + + "is the unit on, in range, and not connected to another host?"; + } else { + connectFailure = "did not connect and initialise within " + timeoutMs / 1000 + " s (state " + state() + ")"; + } + event(connectFailure); + } + return ready; + } + + /** + * Accel only (the low-noise accel on Shimmer3, the LSM6DSV's on Shimmer3R) at rateHz, written the + * way Consensys's Bluetooth manager does it: a clone, AssembleShimmerConfig, then + * operationPrepare, configureFromClone and operationStart(CONFIGURING) on the connected device, + * except that the calibration is not written back. + */ + boolean configureAccelOnly(double rateHz, long timeoutMs) throws InterruptedException { + ShimmerDevice clone = device.deepClone(); + clone.disableAllSensors(); + if (clone.getHardwareVersion() == ShimmerVerDetails.HW_ID.SHIMMER_3) { + clone.setSensorEnabledState(Configuration.Shimmer3.SENSOR_ID.SHIMMER_ANALOG_ACCEL, true); + } else { + clone.setSensorEnabledState(Configuration.Shimmer3.SENSOR_ID.SHIMMER_LSM6DSV_ACCEL_LN, true); + } + clone.setShimmerAndSensorsSamplingRate(rateHz); + AssembleShimmerConfig.generateSingleShimmerConfig(clone, Configuration.COMMUNICATION_TYPE.BLUETOOTH); + // Leave the unit's stored calibration alone. configureFromClone otherwise writes the clone's + // calibration dump back (AA-246), which would rewrite the bench units' calibration on every run. + ((ShimmerBluetooth) clone).setWriteCalibrationDumpWhenConfiguringForClone(false); + int logMark = device.logSize(); + boolean sent = call(String.format(Locale.ROOT, "configure accel only at %.1f Hz", rateHz), 10000, () -> { + device.operationPrepare(); + device.configureFromClone(clone); + device.operationStart(BT_STATE.CONFIGURING); + }); + if (!sent) { + return false; + } + // The operation ends by reading the configuration back, finishing with an INQUIRY. Wait for that + // reply (logged after this configure began), an empty queue and the requested rate read back: + // state and rate alone can already hold before the write starts, if the unit kept them. + boolean done = waitFor(() -> device.countSince(logMark, "Inquiry Response Received") > 0 + && device.queuedInstructions() == 0 && state() == BT_STATE.CONNECTED + && Math.abs(device.getSamplingRateShimmer() - rateHz) < 0.5, timeoutMs); + event(String.format(Locale.ROOT, "configure %s: rate on device %.2f Hz, timers %s", done ? "finished" : "timed out", + device.getSamplingRateShimmer(), device.timers())); + return done; + } + + void startRecording() { + synchronized (samples) { + samples.clear(); + } + recording = true; + } + + List stopRecording() { + recording = false; + synchronized (samples) { + return new ArrayList<>(samples); + } + } + + /** + * Ends one stretch of the recording and starts the next, for when the test stops and restarts + * streaming on purpose: each stretch is judged on its own, so the pause is not taken for loss. + */ + List nextSegment() { + synchronized (samples) { + List done = new ArrayList<>(samples); + samples.clear(); + return done; + } + } + + /** + * Starts SD logging on a streaming unit the way Consensys does (PanelLiveData, DEV-436): stop + * streaming, start logging on the quiet link (SET_RWC, then START_LOGGING_ONLY), then stream + * again. The RTC is therefore never written mid-stream. The stretch recorded before the stop is + * returned, and recording carries on into a new one. + */ + List startSdLoggingLikeConsensys(String what, HilReport report) throws InterruptedException { + report.check(stopStreaming(10000), what + ": streaming did not stop before starting SD logging"); + List before = nextSegment(); + boolean started = call("startSDLogging", 10000, device::startSDLogging); + report.check(started, what + ": startSDLogging did not complete: it threw, hung or was skipped (see the events)"); + if (started) { + report.check(waitFor(device::isSDLogging, 5000), what + ": SD logging did not start (state " + state() + "; is there an SD card?)"); + } + report.check(startStreaming(10000), what + ": streaming did not restart after starting SD logging"); + report.check(waitFor(() -> state() == BT_STATE.STREAMING_AND_SDLOGGING, 5000), + what + ": not streaming and logging after the restart (state " + state() + ")"); + return before; + } + + boolean startStreaming(long timeoutMs) throws InterruptedException { + return call("startStreaming", 10000, device::startStreaming) && waitFor(device::isStreaming, timeoutMs); + } + + boolean stopStreaming(long timeoutMs) throws InterruptedException { + event("while streaming: timers " + device.timers()); + return call("stopStreaming", 10000, device::stopStreaming) && waitFor(() -> !device.isStreaming(), timeoutMs); + } + + boolean disconnect(long timeoutMs) throws InterruptedException { + return call("disconnect", 10000, device::disconnect) && waitFor(() -> { + BT_STATE s = state(); + return s == BT_STATE.DISCONNECTED || s == BT_STATE.CONNECTION_LOST; + }, timeoutMs); + } + + /** Instructions waiting to be sent, read as the driver's own accessors read them. */ + int queuedInstructions() { + return device.queuedInstructions(); + } + + @Override + public void close() { + try { + // After a hung call leave the unit alone; the runner's timeout ends the JVM. + if (hung == null) { + BT_STATE s = state(); + if (s != null && s != BT_STATE.DISCONNECTED && s != BT_STATE.CONNECTION_LOST) { + if (device.isStreaming()) { + stopStreaming(5000); + } + disconnect(10000); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + removeSetWaitForData(device); + calls.shutdownNow(); + } + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java new file mode 100644 index 000000000..e18b6cdec --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/HilShimmerPC.java @@ -0,0 +1,188 @@ +package com.shimmerresearch.hil; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; + +import com.shimmerresearch.driverUtilities.ShimmerBattStatusDetails; +import com.shimmerresearch.pcDriver.ShimmerPC; + +/** + * A ShimmerPC that behaves exactly as an app's does (timers on, the real serial port) but counts + * what the tests need to see: each in-stream reply by type, each status the driver decoded, each + * battery reading, and every line the driver logs, whether or not verbose mode is on. Nothing here + * changes what the driver does. + */ +class HilShimmerPC extends ShimmerPC { + + private static final long serialVersionUID = 1L; + + /** Lines kept per session; enough for minutes of commands, bounded if something loops. */ + private static final int MAX_LOG_LINES = 50000; + + private final long t0; + private final Map inStreamReplies = new TreeMap<>(); + private final List log = new ArrayList<>(); + private volatile int statusesDecoded; + private volatile int batteryReadings; + /** When each battery reply was decoded, in ms on the session's clock (HilSession.nowMs()). */ + private final List batteryReplyTimesMs = new ArrayList<>(); + + HilShimmerPC(String comPort, long t0Nanos) { + super(comPort); + this.t0 = t0Nanos; + } + + @Override + protected void processInstreamResponse(byte inStreamResponseCommand, boolean shouldClearCrcFromBuffer) { + synchronized (inStreamReplies) { + inStreamReplies.merge(replyName(inStreamResponseCommand), 1, Integer::sum); + } + super.processInstreamResponse(inStreamResponseCommand, shouldClearCrcFromBuffer); + } + + @Override + protected void eventLogAndStreamStatusChanged(byte currentCommand) { + statusesDecoded++; + super.eventLogAndStreamStatusChanged(currentCommand); + } + + @Override + public void setBattStatusDetails(ShimmerBattStatusDetails s) { + batteryReadings++; + synchronized (batteryReplyTimesMs) { + batteryReplyTimesMs.add((System.nanoTime() - t0) / 1e6); + } + super.setBattStatusDetails(s); + } + + List batteryReplyTimesMs() { + synchronized (batteryReplyTimesMs) { + return new ArrayList<>(batteryReplyTimesMs); + } + } + + @Override + public void consolePrintLn(String message) { + record(message); + super.consolePrintLn(message); + } + + @Override + public void consolePrintErrLn(String message) { + record("ERR " + message); + super.consolePrintErrLn(message); + } + + /** The driver prints an exception's message to System.out, past consolePrintLn; keep it in the log too. */ + @Override + protected void consolePrintException(String message, StackTraceElement[] stackTrace) { + record("EXCEPTION " + message); + super.consolePrintException(message, stackTrace); + } + + private void record(String message) { + String line = String.format(Locale.ROOT, "%9.0f %s", (System.nanoTime() - t0) / 1e6, message); + synchronized (log) { + if (log.size() < MAX_LOG_LINES) { + log.add(line); + } + } + } + + static String replyName(byte b) { + switch (b) { + case STATUS_RESPONSE: + return "STATUS"; + case DIR_RESPONSE: + return "DIR"; + case VBATT_RESPONSE: + return "VBATT"; + default: + return String.format(Locale.ROOT, "0x%02X", b & 0xFF); + } + } + + int inStreamReplies(String type) { + synchronized (inStreamReplies) { + return inStreamReplies.getOrDefault(type, 0); + } + } + + Map inStreamReplies() { + synchronized (inStreamReplies) { + return new TreeMap<>(inStreamReplies); + } + } + + /** Which of the driver's periodic timers exist right now: an app runs with all three. */ + String timers() { + List t = new ArrayList<>(); + if (mTimerReadStatus != null) { + t.add("status"); + } + if (mTimerCheckAlive != null) { + t.add("alive"); + } + if (mTimerReadBattStatus != null) { + t.add("battery"); + } + return t.isEmpty() ? "none" : String.join("+", t); + } + + /** Instructions waiting to be sent, read through the driver's own (synchronized) accessor. */ + int queuedInstructions() { + return getListofInstructions().size(); + } + + int statusesDecoded() { + return statusesDecoded; + } + + int batteryReadings() { + return batteryReadings; + } + + int logSize() { + synchronized (log) { + return log.size(); + } + } + + List log() { + synchronized (log) { + return new ArrayList<>(log); + } + } + + /** Lines logged since index from that match the pattern anywhere. */ + int countSince(int from, java.util.regex.Pattern pattern) { + int n = 0; + synchronized (log) { + for (int i = Math.max(0, from); i < log.size(); i++) { + if (pattern.matcher(log.get(i)).find()) { + n++; + } + } + } + return n; + } + + /** Lines logged since index from that contain any of the given texts. */ + int countSince(int from, String... texts) { + int n = 0; + synchronized (log) { + for (int i = Math.max(0, from); i < log.size(); i++) { + for (String t : texts) { + if (log.get(i).contains(t)) { + n++; + break; + } + } + } + } + return n; + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java new file mode 100644 index 000000000..21c08539b --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/JavaHilScenariosTest.java @@ -0,0 +1,611 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_CRC_MODE; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; + +/** + * Hardware-in-the-loop scenarios for the Java driver: a real Shimmer3 or Shimmer3R on a paired + * Bluetooth serial port, driven through ShimmerPC with its timers on, as Consensys and MATLAB run + * it. Run by tools/hil/run-java-hil.ps1 through the hilTest task, one scenario per JVM. Without + * hil.port every test is skipped, so the class never runs (or fails) without hardware. + * + *
+ *   hil.port            COM port of the unit (required)
+ *   hil.mac, hil.unit   for the report
+ *   hil.type            Shimmer3 (default) or Shimmer3R
+ *   hil.rate            sampling rate in Hz, default 256
+ *   hil.seconds         streamRest duration, default 30
+ *   hil.cycles          midStreamSet / reconnectLoop / inStream repetitions, default 3 / 5 / 3
+ *   hil.overlaps        inStream overlap rounds, default 5
+ *   hil.threads         queueStress app threads, default 4
+ *   hil.stressSeconds   queueStress duration, default 60
+ *   hil.maxQueued       queueStress queue length the threads keep below, default 8
+ *   hil.stallSeconds    queueStress: no progress for this long is a hang, default 10
+ *   hil.crc             off, 1 or 2; default: the driver's own default
+ *   hil.maxLossPercent  default 1.0
+ *   hil.outDir          where reports go, default build/hil
+ * 
+ */ +public class JavaHilScenariosTest { + + private static final double GRAVITY = 9.81; + /** At 256 Hz a second without a packet is ~256 packets the gap count cannot see. */ + private static final double MAX_SILENCE_AT_END_MS = 1000; + /** + * How close the packet after a one-sample gap must arrive to a decoded battery reply for the gap to + * count as the sample a Shimmer3 skips to send that reply (DEV-1297). Measured: 1-7 ms. + */ + private static final double BATTERY_REPLY_WINDOW_MS = 25; + /** + * The ACK/response timer's own line when a command gets no reply: "Command: [0x72] GET_STATUS_COMMAND + * timeout", or "[0x..]UNKNOWN" for an opcode the driver doesn't know. + */ + private static final java.util.regex.Pattern COMMAND_TIMEOUT = java.util.regex.Pattern.compile("Command:\t\\[0x[0-9A-F]{2}\\] ?\\S+ timeout"); + + private HilSession session; + private HilReport report; + private HilShimmerPC device; + private int logStart; + + private static String arg(String name, String fallback) { + String v = System.getProperty("hil." + name); + return v == null || v.isEmpty() ? fallback : v; + } + + private static double arg(String name, double fallback) { + return Double.parseDouble(arg(name, Double.toString(fallback))); + } + + private static int arg(String name, int fallback) { + return Integer.parseInt(arg(name, Integer.toString(fallback))); + } + + private void begin(String scenario) { + String port = arg("port", null); + assumeTrue("No unit given: pass -Phil.port=COMx (tools/hil/run-java-hil.ps1 does)", port != null); + String crc = arg("crc", null); + if (crc != null) { + ShimmerBluetooth.setDefaultBtCrcModeToUseIfFwSupported( + "off".equals(crc) ? BT_CRC_MODE.OFF : "1".equals(crc) ? BT_CRC_MODE.ONE_BYTE_CRC : BT_CRC_MODE.TWO_BYTE_CRC); + } + Map args = new LinkedHashMap<>(); + System.getProperties().stringPropertyNames().stream().filter(k -> k.startsWith("hil.")).sorted() + .forEach(k -> args.put(k.substring(4), System.getProperty(k))); + String mac = arg("mac", ""); + String type = arg("type", "Shimmer3"); + report = new HilReport(scenario, arg("unit", port), port, mac, type, args); + session = new HilSession(port, mac, type); + device = session.device; + logStart = device.logSize(); + } + + @After + public void end() { + if (session == null) { + return; + } + putDriverCounts(); + session.close(); + report.write(new File(arg("outDir", "build/hil")), session.events(), device.log()); + } + + private void finish() { + if (session.hung != null) { + report.put("hungCall", session.hung); + report.check(false, session.hung + " never returned: the API call blocked (thread dump in the report's events)"); + } + if (!report.failures().isEmpty()) { + Assert.fail(String.join("; ", report.failures())); + } + } + + /** What the driver did in this session: replies by type, decodes, timer requests, connects. */ + private void putDriverCounts() { + report.put("connects", session.connects); + report.put("inStreamReplies", device.inStreamReplies()); + report.put("statusesDecoded", device.statusesDecoded()); + report.put("batteryReadings", device.batteryReadings()); + report.put("statusRequests", device.countSince(logStart, "GET_STATUS_COMMAND Instruction added")); + report.put("commandTimeouts", device.countSince(logStart, COMMAND_TIMEOUT)); + } + + // --- shared checks ------------------------------------------------------------------------- + + private boolean connectAndConfigure(double rateHz) throws Exception { + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, String.valueOf(session.connectFailure)); + if (!connected) { + return false; + } + report.put("hardware", device.getHardwareVersionParsed()); + report.put("firmware", device.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(device.getCurrentBtCommsCrcMode())); + boolean configured = session.configureAccelOnly(rateHz, 20000); + report.put("rateOnDeviceHz", device.getSamplingRateShimmer()); + report.check(configured, "configuration did not complete within 20 s (state " + session.state() + ")"); + return configured; + } + + private void checkRate(double deviceRateHz, String what) { + report.check(!Double.isNaN(deviceRateHz) && deviceRateHz > 0, + what + ": the API has no sampling rate for the unit (" + deviceRateHz + " Hz)"); + } + + private void checkStream(StreamAnalysis a, List samples, double stoppedAtMs, + double maxLossPercent, double deviceRateHz, String what) { + report.check(a.received > 0, what + ": no packets received"); + report.check(a.lossPercent() <= maxLossPercent, String.format(Locale.ROOT, + "%s: %.3f%% packets lost (%d of %d), limit %.2f%%", what, a.lossPercent(), a.missing, + a.received + a.missing, maxLossPercent)); + report.check(a.nonIncreasing == 0, what + ": " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, what); + checkTimeline(a, samples, stoppedAtMs, deviceRateHz, what); + } + + /** A device timestamp that leapt ahead of the arrival time is wrong data (DEV-1023 class), never loss. */ + private void checkJumps(StreamAnalysis a, String what) { + for (StreamAnalysis.Gap j : a.timestampJumps) { + report.check(false, String.format(Locale.ROOT, + "%s: timestamp jumped %.1f s ahead at %.0f ms (device time %.0f ms) while packets kept arriving", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs, j.afterTimestampMs)); + } + } + + /** + * Loss is counted between packets received, so two things it cannot see are checked here: the + * timestamps must give a rate within 1 % of the configured one, and packets must have kept + * arriving until recording stopped. + */ + private void checkTimeline(StreamAnalysis a, List samples, double stoppedAtMs, + double deviceRateHz, String what) { + if (Double.isNaN(a.effectiveRateHz) || Double.isInfinite(a.effectiveRateHz)) { + report.check(false, what + ": no usable device timestamps to measure the rate from (" + a.received + " packets)"); + } else { + report.check(Math.abs(a.effectiveRateHz - deviceRateHz) <= 0.01 * deviceRateHz, String.format(Locale.ROOT, + "%s: effective rate %.2f Hz, device set to %.2f Hz", what, a.effectiveRateHz, deviceRateHz)); + } + if (!samples.isEmpty()) { + double silentMs = stoppedAtMs - samples.get(samples.size() - 1).receivedMs; + report.check(silentMs <= MAX_SILENCE_AT_END_MS, String.format(Locale.ROOT, + "%s: no packet in the last %.1f s before recording stopped (the stream went silent)", what, silentMs / 1000.0)); + } + } + + /** + * Judges a stream recorded in stretches, split where the test stopped and restarted streaming on + * purpose, so the pause is not taken for loss. Each stretch gets the timestamp checks. Each gap + * is charged, as StreamAnalysis.attribute() does, to the nearest SD logging stop or battery read + * within 1 s: + *
    + *
  • loss near an SD logging stop is reported, not failed (the unit pauses while it closes its + * file);
  • + *
  • a single sample skipped right where a battery reply was decoded is reported, not failed: a + * Shimmer3 gives up a sample slot to answer GET_VBATT mid-stream (DEV-1297). Had the driver lost + * that packet, it would have left bytes behind, which checkDriverLog() fails on;
  • + *
  • any other loss near a battery read, or more than maxLossPercent anywhere else, fails.
  • + *
+ */ + private void checkStretches(List> stretches, double stoppedAtMs, double deviceRateHz, + double maxLossPercent, List sdStops, List batteryMoments) { + int received = 0; + int missing = 0; + int nearSd = 0; + int nearBattery = 0; + int skippedAtBatteryReplies = 0; + double[] sd = toArray(sdStops); + double[] battery = toArray(batteryMoments); + double[] batteryReplies = toArray(device.batteryReplyTimesMs()); + List all = new ArrayList<>(); + List summaries = new ArrayList<>(); + for (int k = 0; k < stretches.size(); k++) { + List stretch = stretches.get(k); + String what = "stretch " + (k + 1); + all.addAll(stretch); + StreamAnalysis a = StreamAnalysis.analyse(stretch, deviceRateHz); + session.event(what + ": " + a); + summaries.add(what + ": " + a); + report.check(a.received > 0, what + ": no packets received"); + received += a.received; + missing += a.missing; + for (StreamAnalysis.Gap g : a.gaps) { + double toSd = distanceToNearest(sd, g.afterReceivedMs); + double toBattery = distanceToNearest(battery, g.afterReceivedMs); + if (Math.min(toSd, toBattery) > 1000) { + continue; + } + if (toSd <= toBattery) { + nearSd += g.missing; + } else if (g.missing == 1 && distanceToNearest(batteryReplies, g.afterReceivedMs) <= BATTERY_REPLY_WINDOW_MS) { + skippedAtBatteryReplies++; + } else { + nearBattery += g.missing; + } + } + report.check(a.nonIncreasing == 0, what + ": " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, what); + // The rate needs a second or so of stream; only the last stretch can go silent before a stop + // the test did not make itself. + if (a.received >= deviceRateHz) { + double end = k == stretches.size() - 1 ? stoppedAtMs : stretch.get(stretch.size() - 1).receivedMs; + checkTimeline(a, stretch, end, deviceRateHz, what); + } + } + report.put("stretches", summaries); + report.put("received", received); + report.put("missing", missing); + report.put("missingNearSdLoggingStops", nearSd); + report.put("missingNearBatteryReads", nearBattery); + report.put("skippedAtBatteryReplies", skippedAtBatteryReplies); + report.samples(all); + if (nearSd > 0) { + session.event(nearSd + " packets not received within 1 s of an SD logging stop (firmware side; not failed)"); + } + if (skippedAtBatteryReplies > 0) { + session.event(skippedAtBatteryReplies + " single samples skipped where the unit sent a battery reply" + + " (firmware side, DEV-1297; not failed)"); + } + report.check(nearBattery == 0, nearBattery + " packets lost within 1 s of a battery read while streaming"); + int elsewhere = missing - nearSd - nearBattery - skippedAtBatteryReplies; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, received + missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLossPercent, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from SD logging stops and battery reads (%d), limit %.2f%%", + elsewherePercent, elsewhere, maxLossPercent)); + } + + /** Parser errors and command timeouts the driver logged since index from. */ + private void checkDriverLog(int from, String what) { + int parserErrors = device.countSince(from, "Unknown parsing error", "Unknown packet error", "Throw Bytes"); + report.check(parserErrors == 0, what + ": " + parserErrors + " parser errors logged by the driver (see the .driver.log)"); + int timeouts = device.countSince(from, COMMAND_TIMEOUT); + report.check(timeouts == 0, what + ": " + timeouts + " command timeouts logged by the driver (see the .driver.log)"); + } + + private static double[] toArray(List values) { + double[] out = new double[values.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = values.get(i); + } + return out; + } + + /** The distance from t to the nearest of the moments; infinite when there are none. */ + private static double distanceToNearest(double[] momentsMs, double t) { + double nearest = Double.POSITIVE_INFINITY; + for (double m : momentsMs) { + nearest = Math.min(nearest, Math.abs(t - m)); + } + return nearest; + } + + // --- scenarios ----------------------------------------------------------------------------- + + /** Connect, configure, stream at rest, stop, disconnect. */ + @Test + public void streamRest() throws Exception { + begin("stream_rest"); + double rate = arg("rate", 256.0); + int seconds = arg("seconds", 30); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(seconds * 1000L); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); + // NaN fails too: the scenario configured the accel, so a stream without calibrated accel is a fault + report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= 1.0, String.format(Locale.ROOT, + "accel at rest %.3f m/s^2, expected %.2f±1.00 (NaN: no calibrated accel; otherwise is the unit lying still?)", + a.accelMagnitudeMean, GRAVITY)); + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** + * SD logging and battery reads on a streaming unit, in the order Consensys uses (DEV-436): SD + * logging starts with streaming stopped (stop streaming, SET_RWC and START_LOGGING_ONLY on the + * quiet link, stream again), stops mid-stream (STOP_LOGGING_ONLY), and the battery is read + * mid-stream. Each call and reply is waited on. Needs an SD card in the unit. + */ + @Test + public void midStreamSet() throws Exception { + begin("mid_stream_set"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 3); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(3000); + List> stretches = new ArrayList<>(); + List sdStops = new ArrayList<>(); + List batteryMoments = new ArrayList<>(); + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "cycle " + i; + session.event(what); + stretches.add(session.startSdLoggingLikeConsensys(what, report)); + Thread.sleep(2000); + sdStops.add(session.nowMs()); + boolean stopped = session.call("stopSDLogging", 10000, device::stopSDLogging); + report.check(stopped, what + ": stopSDLogging did not complete: it threw, hung or was skipped (see the events)"); + if (stopped) { + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING, 5000), + what + ": SD logging did not stop (state " + session.state() + ")"); + } + Thread.sleep(1000); + batteryMoments.add(session.nowMs()); + int readingsBefore = device.batteryReadings(); + boolean asked = session.call("readBattery", 10000, device::readBattery); + report.check(asked, what + ": readBattery did not complete: it threw, hung or was skipped (see the events)"); + if (asked) { + report.check(session.waitFor(() -> device.batteryReadings() > readingsBefore, 5000), + what + ": no battery reply decoded within 5 s of readBattery"); + } + Thread.sleep(2000); + } + double stoppedAtMs = session.nowMs(); + stretches.add(session.stopRecording()); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + checkStretches(stretches, stoppedAtMs, deviceRate, maxLoss, sdStops, batteryMoments); + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Connect, stream for 5 s and disconnect, repeatedly. Every connect starts the status timer. */ + @Test + public void reconnectLoop() throws Exception { + begin("reconnect_loop"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 5); + double maxLoss = arg("maxLossPercent", 1.0); + List connectTimes = new ArrayList<>(); + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "cycle " + i; + long start = System.currentTimeMillis(); + boolean ready = i == 1 ? connectAndConfigure(rate) : session.connect(30000); + connectTimes.add(System.currentTimeMillis() - start); + // In cycle 1 a failed configure is reported by connectAndConfigure, with connectFailure null + report.check(ready, what + ": " + (session.connectFailure != null ? session.connectFailure + : "not ready to stream (state " + session.state() + ")")); + if (!ready) { + break; + } + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, what); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + checkDriverLog(logMark, what); + report.check(session.disconnect(10000), what + ": did not disconnect within 10 s"); + Thread.sleep(2000); + } + report.put("connectMs", connectTimes); + finish(); + } + + /** + * In-stream commands while streaming. First each reply on its own: status (0x8A 0x71), directory + * (0x8A 0x88) and battery (0x8A 0x94), each checked to be decoded. Then the overlap, as Consensys + * meets it: with the unit streaming and logging (logging started the Consensys way), stop SD + * logging mid-stream (STOP_LOGGING_ONLY) and request status and battery at once, so in-stream + * replies, including any status the unit pushes as logging stops, arrive while that command waits + * for its ACK (DEV-1262). Every command must complete, with its own reply, and none may time out. + */ + @Test + public void inStream() throws Exception { + begin("in_stream"); + double rate = arg("rate", 256.0); + int rounds = arg("cycles", 3); + int overlaps = arg("overlaps", 5); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = device.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + int logMark = device.logSize(); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(2000); + for (int i = 1; i <= rounds && session.hung == null; i++) { + String what = "round " + i; + int statusBefore = device.statusesDecoded(); + expectReply(what, "status", "STATUS", device::readStatusLogAndStream, () -> device.statusesDecoded() > statusBefore); + expectReply(what, "directory", "DIR", device::readDirectoryName, + () -> device.getDirectoryName() != null && !device.getDirectoryName().isEmpty()); + int batteryBefore = device.batteryReadings(); + expectReply(what, "battery", "VBATT", device::readBattery, () -> device.batteryReadings() > batteryBefore); + Thread.sleep(500); + } + List> stretches = new ArrayList<>(); + List sdStops = new ArrayList<>(); + for (int i = 1; i <= overlaps && session.hung == null; i++) { + String what = "overlap " + i; + stretches.add(session.startSdLoggingLikeConsensys(what, report)); + Thread.sleep(1000); + int statusReplies = device.inStreamReplies("STATUS"); + int batteryReplies = device.inStreamReplies("VBATT"); + int statusBefore = device.statusesDecoded(); + int batteryBefore = device.batteryReadings(); + int timeoutsBefore = device.countSince(logMark, COMMAND_TIMEOUT); + sdStops.add(session.nowMs()); + boolean sent = session.call(what + ": STOP_LOGGING_ONLY, GET_STATUS and GET_VBATT back to back", 10000, () -> { + device.stopSDLogging(); + device.readStatusLogAndStream(); + device.readBattery(); + }); + report.check(sent, what + ": the three calls did not complete (see the events)"); + if (sent) { + boolean done = session.waitFor(() -> device.inStreamReplies("STATUS") > statusReplies + && device.inStreamReplies("VBATT") > batteryReplies + && session.queuedInstructions() == 0 && session.state() == BT_STATE.STREAMING, 5000); + report.check(done, String.format(Locale.ROOT, + "%s: not every command completed within 5 s (status replies +%d, battery replies +%d, %d still queued, state %s)", + what, device.inStreamReplies("STATUS") - statusReplies, device.inStreamReplies("VBATT") - batteryReplies, + session.queuedInstructions(), session.state())); + report.check(device.statusesDecoded() > statusBefore, what + ": the status reply was not decoded"); + report.check(device.batteryReadings() > batteryBefore, what + ": the battery reply was not decoded"); + int timeouts = device.countSince(logMark, COMMAND_TIMEOUT) - timeoutsBefore; + report.check(timeouts == 0, what + ": " + timeouts + " command timeouts (a reply matched to the wrong command?)"); + } + Thread.sleep(1000); + } + double stoppedAtMs = session.nowMs(); + stretches.add(session.stopRecording()); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + report.check(session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 3000), + "after streaming the unit is " + session.state() + ", not CONNECTED"); + checkStretches(stretches, stoppedAtMs, deviceRate, maxLoss, sdStops, new ArrayList<>()); + checkDriverLog(logMark, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Sends one in-stream request and checks its reply both arrived and was decoded within 3 s. */ + private void expectReply(String round, String what, String replyType, HilSession.Call request, + HilSession.Condition decoded) throws InterruptedException { + int before = device.inStreamReplies(replyType); + if (!session.call(round + ": request " + what, 10000, request)) { + report.check(false, round + ": the " + what + " request did not complete (see the events)"); + return; + } + boolean arrived = session.waitFor(() -> device.inStreamReplies(replyType) > before, 3000); + report.check(arrived, round + ": no in-stream " + what + " reply within 3 s"); + if (arrived) { + report.check(session.waitFor(decoded, 1000), round + ": the " + what + " reply arrived but was not decoded"); + } + } + + /** + * DEV-1266 on hardware, with no SD logging. While streaming, app threads queue harmless reads + * (GET_VBATT_COMMAND and GET_STATUS_COMMAND) in tight loops, and check the queue length between + * them. Each of those calls locks the device and then the instruction list, so one is nearly + * always under way while the IO thread, holding the list, fetches the next instruction. That is + * the window the lock inversion needs. The queue is kept short so the unit is not flooded. + *

+ * Fails if the threads and the replies stop for hil.stallSeconds (default 10), or if the JVM + * finds a deadlock. The events then hold every thread's stack. Without the DEV-1266 fix (#334) a + * Shimmer3R deadlocked within 0.3 s in every run; with it, 60 s ran clean. + */ + @Test + public void queueStress() throws Exception { + begin("queue_stress"); + double rate = arg("rate", 256.0); + int threads = arg("threads", 4); + int seconds = arg("stressSeconds", 60); + int maxQueued = arg("maxQueued", 8); + int stallSeconds = arg("stallSeconds", 10); + if (connectAndConfigure(rate)) { + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + java.util.concurrent.atomic.AtomicLong calls = new java.util.concurrent.atomic.AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean stop = new java.util.concurrent.atomic.AtomicBoolean(); + List workers = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + Thread worker = new Thread(() -> { + while (!stop.get()) { + if (device.queuedInstructions() < maxQueued) { + if (calls.incrementAndGet() % 2 == 0) { + device.readBattery(); + } else { + device.readStatusLogAndStream(); + } + } else { + Thread.yield(); + } + } + }, "HilStress-" + t); + worker.setDaemon(true); + workers.add(worker); + } + session.event("queueStress: " + threads + " threads for up to " + seconds + " s, at most " + maxQueued + " queued"); + long start = System.nanoTime(); + workers.forEach(Thread::start); + long lastCalls = -1; + int lastReplies = -1; + long lastProgress = System.nanoTime(); + String hang = null; + while (System.nanoTime() - start < seconds * 1_000_000_000L) { + Thread.sleep(250); + long[] deadlocked = java.lang.management.ManagementFactory.getThreadMXBean().findDeadlockedThreads(); + if (deadlocked != null) { + hang = "the JVM found a deadlock"; + break; + } + long c = calls.get(); + int replies = device.statusesDecoded() + device.batteryReadings(); + if (c != lastCalls || replies != lastReplies) { + lastCalls = c; + lastReplies = replies; + lastProgress = System.nanoTime(); + } else if (System.nanoTime() - lastProgress > stallSeconds * 1_000_000_000L) { + hang = "no call returned and no reply arrived for " + stallSeconds + " s"; + break; + } + } + double ranSeconds = (System.nanoTime() - start) / 1e9; + stop.set(true); + for (Thread worker : workers) { + worker.join(5000); + } + report.put("stressSeconds", Math.round(ranSeconds * 10) / 10.0); + report.put("stressCalls", calls.get()); + report.put("stressReplies", device.statusesDecoded() + device.batteryReadings()); + if (hang == null && workers.stream().anyMatch(Thread::isAlive)) { + hang = "the stress threads did not stop"; + } + if (hang != null) { + session.hung = "queueStress"; + session.event("queueStress hung after " + String.format(Locale.ROOT, "%.1f", ranSeconds) + " s: " + hang + + "; every thread:\n" + com.shimmerresearch.shimmer3.communication.ByteCommunicationTestUtils.dumpAllThreads()); + report.check(false, String.format(Locale.ROOT, "hung after %.1f s and %d calls: %s (thread dump in the events)", + ranSeconds, calls.get(), hang)); + } else { + session.event(String.format(Locale.ROOT, "queueStress: %d calls, %d replies decoded in %.1f s, no hang", + calls.get(), device.statusesDecoded() + device.batteryReadings(), ranSeconds)); + report.check(session.waitFor(() -> session.queuedInstructions() == 0, 10000), "the queue did not drain after the stress"); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java new file mode 100644 index 000000000..85154a618 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysis.java @@ -0,0 +1,196 @@ +package com.shimmerresearch.hil; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** + * Judges a recorded stream from the device timestamps alone. The checks mirror the firmware HIL + * harness in log-and-stream-common Test/hil (DEV-1165): packet loss and non-increasing timestamps + * from the driver's already-unwrapped timestamps, the effective sample rate, and accel at rest. + * Pure Java, so it is unit-tested on the JVM (StreamAnalysisTest, in the normal test task). + *

+ * The same class as the Android HIL's (ShimmerAndroidAPI, shimmerHilTest), so both judge a stream + * by the same rules; a change here belongs there too. + */ +public final class StreamAnalysis { + + /** One received packet. Times in ms; accel in m/s^2, NaN when the channel is absent. */ + public static final class Sample { + public final double receivedMs; + public final double timestampMs; + public final double accelX, accelY, accelZ; + + public Sample(double receivedMs, double timestampMs, double accelX, double accelY, double accelZ) { + this.receivedMs = receivedMs; + this.timestampMs = timestampMs; + this.accelX = accelX; + this.accelY = accelY; + this.accelZ = accelZ; + } + + double accelMagnitude() { + return Math.sqrt(accelX * accelX + accelY * accelY + accelZ * accelZ); + } + } + + /** A run of packets missing between two received ones. */ + public static final class Gap { + public final double afterTimestampMs; + public final double afterReceivedMs; + public final int missing; + + Gap(double afterTimestampMs, double afterReceivedMs, int missing) { + this.afterTimestampMs = afterTimestampMs; + this.afterReceivedMs = afterReceivedMs; + this.missing = missing; + } + } + + public final int received; + public final int missing; + public final int nonIncreasing; + /** + * Steps where the device timestamp leapt far ahead of the arrival time: not lost packets but a + * bad timestamp, such as a 24-bit wrap unwrapped wrongly (512 s at a 32768 Hz clock, DEV-1023). + */ + public final List timestampJumps; + public final double expectedPeriodMs; + public final double effectiveRateHz; + public final double accelMagnitudeMean; + public final double accelMagnitudeStd; + public final List gaps; + + private StreamAnalysis(int received, int missing, int nonIncreasing, List timestampJumps, double expectedPeriodMs, + double effectiveRateHz, double accelMagnitudeMean, double accelMagnitudeStd, List gaps) { + this.received = received; + this.missing = missing; + this.nonIncreasing = nonIncreasing; + this.timestampJumps = timestampJumps; + this.expectedPeriodMs = expectedPeriodMs; + this.effectiveRateHz = effectiveRateHz; + this.accelMagnitudeMean = accelMagnitudeMean; + this.accelMagnitudeStd = accelMagnitudeStd; + this.gaps = gaps; + } + + /** + * @param samples in arrival order + * @param rateHz the sampling rate the device was configured with + */ + public static StreamAnalysis analyse(List samples, double rateHz) { + double period = 1000.0 / rateHz; + int missing = 0; + int nonIncreasing = 0; + // The effective rate counts sample periods over device time, from ordinary steps only: a + // jump or a step backwards says nothing about the rate. + double periods = 0, spanMs = 0; + List gaps = new ArrayList<>(); + List jumps = new ArrayList<>(); + for (int i = 1; i < samples.size(); i++) { + double step = samples.get(i).timestampMs - samples.get(i - 1).timestampMs; + double arrivalStep = samples.get(i).receivedMs - samples.get(i - 1).receivedMs; + if (step <= 0) { + nonIncreasing++; + } else if (step > 1.5 * period) { + int lost = (int) Math.round(step / period) - 1; + Gap g = new Gap(samples.get(i - 1).timestampMs, samples.get(i - 1).receivedMs, lost); + // Packets really lost leave a pause in arrival about as long as the gap. A timestamp that + // leaps a second or more beyond the arrival time is a bad timestamp, not lost data. + if (step - arrivalStep > JUMP_MS) { + jumps.add(g); + } else { + missing += lost; + gaps.add(g); + periods += lost + 1; + spanMs += step; + } + } else { + periods += 1; + spanMs += step; + } + } + double effectiveRate = spanMs > 0 ? periods * 1000.0 / spanMs : Double.NaN; + double sum = 0, sumSq = 0; + int n = 0; + for (Sample s : samples) { + double m = s.accelMagnitude(); + if (!Double.isNaN(m)) { + sum += m; + sumSq += m * m; + n++; + } + } + double mean = n > 0 ? sum / n : Double.NaN; + double std = n > 1 ? Math.sqrt(Math.max(0, (sumSq - n * mean * mean) / (n - 1))) : Double.NaN; + return new StreamAnalysis(samples.size(), missing, nonIncreasing, jumps, period, effectiveRate, mean, std, gaps); + } + + /** How far ahead of the arrival time a timestamp must leap to count as a jump, not loss. */ + static final double JUMP_MS = 1000; + + /** + * Charges each gap to the kind of command nearest to it (by arrival time), if one is within + * windowMs. Commands that follow each other closely therefore never share a loss. + * + * @param momentsByKind moments (ms) of each kind of command + * @return packets lost per kind, in the same order + */ + public int[] attribute(double[][] momentsByKind, double windowMs) { + int[] lost = new int[momentsByKind.length]; + for (Gap g : gaps) { + int nearestKind = -1; + double nearest = Double.MAX_VALUE; + for (int k = 0; k < momentsByKind.length; k++) { + for (double t : momentsByKind[k]) { + double d = Math.abs(g.afterReceivedMs - t); + if (d < nearest) { + nearest = d; + nearestKind = k; + } + } + } + if (nearestKind >= 0 && nearest <= windowMs) { + lost[nearestKind] += g.missing; + } + } + return lost; + } + + /** Packets lost within windowMs (by arrival time) of any of the given moments. */ + public int missingNear(double[] momentsMs, double windowMs) { + int total = 0; + for (Gap g : gaps) { + for (double t : momentsMs) { + if (Math.abs(g.afterReceivedMs - t) <= windowMs) { + total += g.missing; + break; + } + } + } + return total; + } + + public double lossPercent() { + int expected = received + missing; + return expected == 0 ? 0 : 100.0 * missing / expected; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, + "received=%d missing=%d (%.3f%%) nonIncreasing=%d timestampJumps=%d effectiveRate=%.2fHz accel=%.3f±%.3f m/s^2 gaps=%s", + received, missing, lossPercent(), nonIncreasing, timestampJumps.size(), effectiveRateHz, + accelMagnitudeMean, accelMagnitudeStd, gapsSummary()); + } + + private String gapsSummary() { + int shown = Math.min(gaps.size(), 5); + String[] parts = new String[shown]; + for (int i = 0; i < shown; i++) { + parts[i] = String.format(Locale.ROOT, "%d@%.0fms", gaps.get(i).missing, gaps.get(i).afterReceivedMs); + } + return Arrays.toString(parts) + (gaps.size() > shown ? "+" + (gaps.size() - shown) : ""); + } +} diff --git a/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java new file mode 100644 index 000000000..472366319 --- /dev/null +++ b/ShimmerDriverPC/src/test/java/com/shimmerresearch/hil/StreamAnalysisTest.java @@ -0,0 +1,102 @@ +package com.shimmerresearch.hil; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +/** The HIL verdicts are only as good as this arithmetic, so it is checked without a phone. */ +public class StreamAnalysisTest { + + private static List stream(double rateHz, int count, int... droppedIndices) { + List samples = new ArrayList<>(); + double period = 1000.0 / rateHz; + outer: + for (int i = 0; i < count; i++) { + for (int d : droppedIndices) { + if (d == i) { + continue outer; + } + } + samples.add(new StreamAnalysis.Sample(i * period, 1000 + i * period, 0, 0, 9.81)); + } + return samples; + } + + @Test + public void cleanStreamHasNoLoss() { + StreamAnalysis a = StreamAnalysis.analyse(stream(256, 2560), 256); + assertEquals(2560, a.received); + assertEquals(0, a.missing); + assertEquals(0, a.nonIncreasing); + assertEquals(256, a.effectiveRateHz, 0.01); + assertEquals(9.81, a.accelMagnitudeMean, 1e-9); + } + + @Test + public void countsEachDroppedPacket() { + StreamAnalysis a = StreamAnalysis.analyse(stream(256, 1000, 100, 500, 501, 502), 256); + assertEquals(996, a.received); + assertEquals(4, a.missing); + assertEquals(2, a.gaps.size()); + assertEquals(256, a.effectiveRateHz, 0.01); + } + + @Test + public void attributesLossToTheMomentItHappened() { + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + // Packet 300 is missing; packet 299 arrived at 2990 ms. + assertEquals(1, a.missingNear(new double[] {3000}, 500)); + assertEquals(0, a.missingNear(new double[] {9000}, 500)); + } + + @Test + public void chargesEachGapToTheNearestCommandOnly() { + // 100 Hz; packet 300 (arrives ~3000 ms) is missing. An SD stop at 2900 ms and a battery read + // at 3920 ms are both within 1 s of it: the SD stop is nearer, so it alone is charged. + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + int[] lost = a.attribute(new double[][] {{2900}, {3920}}, 1000); + assertEquals(1, lost[0]); + assertEquals(0, lost[1]); + } + + @Test + public void chargesNothingOutsideTheWindow() { + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + int[] lost = a.attribute(new double[][] {{5000}, {8000}}, 1000); + assertEquals(0, lost[0] + lost[1]); + } + + @Test + public void tellsABadTimestampFromLostPackets() { + // What a Shimmer3R did over BLE at an SD logging start: one packet stamped a 24-bit wrap + // (512 s) ahead while packets kept arriving on time, then the next one back on track. + List s = stream(256, 1000); + StreamAnalysis.Sample bad = s.get(500); + s.set(500, new StreamAnalysis.Sample(bad.receivedMs, bad.timestampMs + 512000, 0, 0, 9.81)); + StreamAnalysis a = StreamAnalysis.analyse(s, 256); + assertEquals(1, a.timestampJumps.size()); + assertEquals(0, a.missing); + assertEquals(1, a.nonIncreasing); + assertEquals(256, a.effectiveRateHz, 0.5); + } + + @Test + public void flagsTimestampsThatGoBackwards() { + List s = stream(100, 10); + s.add(5, new StreamAnalysis.Sample(45, 1000, 0, 0, 9.81)); + StreamAnalysis a = StreamAnalysis.analyse(s, 100); + assertEquals(1, a.nonIncreasing); + } + + @Test + public void toleratesJitterBelowHalfAPeriod() { + List s = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + s.add(new StreamAnalysis.Sample(i * 10, 1000 + i * 10 + (i % 2 == 0 ? 0 : 4), 0, 0, 9.81)); + } + assertEquals(0, StreamAnalysis.analyse(s, 100).missing); + } +} diff --git a/tools/hil/README.md b/tools/hil/README.md new file mode 100644 index 000000000..5edd52517 --- /dev/null +++ b/tools/hil/README.md @@ -0,0 +1,140 @@ +# Java HIL: the Java driver on real Shimmers + +Hardware-in-the-loop tests of the Java driver (`ShimmerDriver` with `ShimmerDriverPC`). A PC connects +to real Shimmer3 and Shimmer3R units over their paired Bluetooth Classic serial ports, through the +same API Consensys and MATLAB use (`ShimmerPC`), **with the driver's timers on** as an app runs it. +One command builds the tests, runs every scenario on every unit, and prints pass or fail. + +```powershell +.\tools\hil\run-java-hil.ps1 +``` + +It exits 0 when everything passed, 1 when anything failed, errored, timed out or was skipped, and 2 +when the bench isn't set up. + +The CI tests run against byte simulators, and since DEV-1198 (#325, #326) with the status timer off. +This is where the driver meets real firmware, a real radio and its own timers. + +## Before every Java API release (the gate) + +**Release Java API** won't publish a commit that hasn't passed this on hardware. Each round: + +1. On the bench, on `master` with a clean working tree and every unit in the rig, run: + ```powershell + git checkout master; git pull + .\tools\hil\run-java-hil.ps1 -Report + ``` + With `-Report`, the result is posted to GitHub as the commit status **`java-hil`** on that commit: + passed or failed, with each unit's firmware. It refuses to report anything but the full gate run: + - every scenario, on at least one Shimmer3 and one Shimmer3R; + - the default rate, duration, link CRC and loss limit, and a fresh build (no `-SkipBuild`); + - a clean working tree, pushed. +2. Run **Release Java API**. It looks up that status for the commit it's releasing and stops before + building anything unless it passed. The release notes say what was tested. + +If anything merges after step 1, the commit changes, so test again. In an emergency, put a Jira key in +**hil_waiver** to release without a pass; the release notes record the waiver. Dry runs only warn. + +## What it tests + +| Scenario | What it does | Fails when | +|---|---|---| +| `streamRest` | Connects, writes an accel-only config at `-Rate` (default 256 Hz) the way Consensys's Bluetooth manager does, streams for `-Seconds` (default 30) with the unit lying still, then stops and disconnects. | It can't connect or configure; more than `-MaxLossPercent` (1 %) of packets are lost; a timestamp doesn't increase, or leaps more than a second ahead of when it arrived (a bad unwrap, not loss); the timestamps give no rate, or one more than 1 % off the configured rate; the stream goes silent for over a second before the test stops it; accel is more than 1 m/s² from 9.81; or the driver logs a parser error or a command timeout. | +| `midStreamSet` | On a streaming unit, three times: starts SD logging **the way Consensys does** (stop streaming, start logging on the quiet link, stream again; DEV-436), stops it mid-stream, and reads the battery mid-stream. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop, or streaming doesn't come back; a packet is lost around a battery read, or more than `-MaxLossPercent` elsewhere (a single sample skipped right where the battery reply arrives is reported, not failed: a Shimmer3 gives up a sample slot to answer a battery read, DEV-1297); a timestamp fault in any stretch between restarts; or the driver logs a parser error or a command timeout. Packets missing around an SD logging stop are reported, not failed: the unit pauses while it closes its file. Needs an SD card in the unit. | +| `reconnectLoop` | Connects, streams for 5 s and disconnects, five times on the same `ShimmerPC`. Every connect starts the driver's status timer, which is when the DEV-1266 lock inversion is most likely. | Any cycle fails to connect within 30 s, loses packets, fails the timestamp checks, or the driver logs a parser error or command timeout. | +| `inStream` | While streaming: first each in-stream reply on its own, three rounds: status (`0x8A 0x71`), directory (`0x8A 0x88`) and battery (`0x8A 0x94`). Then five overlaps as Consensys meets them: with the unit streaming and logging (logging started the Consensys way), stop SD logging mid-stream and request status and battery at once, so in-stream replies, including any status the unit pushes as logging stops, arrive while that command waits for its ACK (DEV-1262). | A reply doesn't arrive within 3 s or isn't decoded; an overlap doesn't complete within 5 s (both replies decoded, logging stopped, the queue empty); the driver logs a command timeout; or the stream fails the checks above. | +| `queueStress` | While streaming, four app threads call `readBattery()` and `readStatusLogAndStream()` and check the queue length, in tight loops for 60 s, keeping the queue under 8. Each of those calls locks the device and then the instruction list, so one is nearly always under way while the IO thread fetches the next instruction. That is the window DEV-1266's lock inversion needs (#334). No SD logging is involved. | The calls and replies stop for 10 s, or the JVM finds a deadlock; the queue doesn't drain afterwards. Without the DEV-1266 fix a Shimmer3R deadlocked within 0.3 s in every run. | + +Every API call runs under a watchdog. If one doesn't return within 10 s, the scenario stops, and its +events hold every thread's stack with the locks each holds and waits on, plus any deadlock the JVM +detects (DEV-1266). If the whole run still doesn't finish, the runner stops Gradle after +`-ScenarioTimeoutSeconds` (900) and records TIMEOUT. + +A connect is never retried. If the COM port can't be opened, the scenario fails and says why: +- **"Port busy"**: another program on this PC already has the port open: another HIL, Consensys, a + serial terminal. The unit is in use, so a retry would only produce results from a shared unit. +- **CONNECTION_FAILED** for any other reason: the unit is off, out of range, or connected to + another host such as a phone. + +Each report counts what the session covered: connects, in-stream replies by type, statuses and +battery readings decoded, the status timer's requests, and command timeouts. Its events also say +which driver timers were running after connect, after configure and while streaming. + +Starting SD logging mid-stream (`startSDLogging()` while streaming) writes the RTC mid-stream, which a Shimmer3R answers with out-of-order timestamps (DEV-1245). Consensys never does that, and neither do these scenarios. + +The stream checks are the Android HIL's (`StreamAnalysis`, the same class in ShimmerAndroidAPI's +`shimmerHilTest`), so the two judge a stream by the same rules. `StreamAnalysisTest` runs in CI. + +### What it does not test + +- BLE: on the PC it goes through the gRPC BLE server (`ShimmerGRPC`); not in v1. +- Status the firmware pushes on its own (dock, undock, button press), which needs someone at the + bench; the managed USB hub in `bench-tools/hub` could do that later. +- The SD and file parsers, and the Android API's own Classic parser (the Android HIL covers that). + +## Choosing what to test + +| | Default | Option | +|---|---|---| +| Units | every unit in the rig file | `-Unit s3r-86f8` | +| Scenarios | all five | `-Scenario inStream` | +| Sampling rate, stream duration | 256 Hz, 30 s | `-Rate 512`, `-Seconds 60` | +| Link CRC | the driver's default (1 byte) | `-Crc off`, `-Crc 2` | +| Pause between runs | 10 s | `-SettleSeconds 30` | +| Record the result for the release gate | no | `-Report` (full run on a clean, pushed commit only) | + +## One-time setup + +1. **Pair each unit with the PC** in Windows Settings → Bluetooth (PIN `1234`). Windows creates an + outgoing serial port for it; the script finds that port from the unit's MAC. +2. **Prepare each unit:** SD sync and Bluetooth disable off (either keeps Bluetooth off, DEV-1228), an + SD card in for `midStreamSet`, lying still during `streamRest`. +3. **List the units:** copy `rig-java.example.json` to `rig-java.json` and fill in each name, MAC + and `type` (`Shimmer3` or `Shimmer3R`). Add `comPort` only to override the port the script finds. + That file is gitignored. +4. **The checkout's folder must be named `Shimmer-Java-Android-API`**, because + `ShimmerDriverPC/settings.gradle` finds its sibling projects by that name. In a worktree, make a + junction with that name and run the script through it: + ```powershell + New-Item -ItemType Junction -Path C:\wt\Shimmer-Java-Android-API -Target C:\wt\ + C:\wt\Shimmer-Java-Android-API\tools\hil\run-java-hil.ps1 + ``` +5. **JDK 11 or newer.** The script uses `JAVA_HOME` if it qualifies, otherwise the first one it finds + under Amazon Corretto, Eclipse Adoptium or Java in Program Files. + +**Nothing else may use the units during a run.** That includes Consensys, the Android HIL's phone, +and the other HILs that share this bench (C#, shimmer-logandstream). + +Only a second Classic connection is refused outright, as "Port busy". A second host connecting over +BLE is not: +- a Shimmer3R that is streaming over Classic accepts the BLE link; +- it runs the commands sent over BLE and replies down the Classic link, which corrupts this run's + stream; +- the BLE host sees no replies at all. + +On 2026-10-21 that is how two HILs sharing a unit produced a bug report about the unit locking up +after a radio switch (DEV-1276). + +## Output + +The run's folder, `tools/hil/out//`, holds: +- `summary.json` and the table the script prints; +- `reports/`, per scenario and unit: + - `_.json`: versions, the CRC mode in use, loss, rate, accel, the counts above, a + timeline of events and the failures; + - `.csv`: every sample; + - `.driver.log`: every line the driver logged, verbose or not; +- `*.console.txt` and `*.junit.xml`: Gradle's output and the JUnit result for each run; +- `build.log`. + +## Running one scenario by hand + +```powershell +cd C:\wt\Shimmer-Java-Android-API\ShimmerDriverPC +.\gradlew :hilTest --no-watch-fs --tests com.shimmerresearch.hil.JavaHilScenariosTest.inStream "-Phil.port=COM36" "-Phil.type=Shimmer3R" "-Phil.unit=s3r-86f8" +``` + +Every `-Phil.` reaches the tests as `hil.`; the names are listed in +`JavaHilScenariosTest`. Without `hil.port` every scenario is skipped, and the normal `test` task +leaves them out. `--no-watch-fs` matters when you edit through a worktree's real path but build +through the junction: Gradle's file watcher misses those edits and would run stale classes. diff --git a/tools/hil/rig-java.example.json b/tools/hil/rig-java.example.json new file mode 100644 index 000000000..8d62516d9 --- /dev/null +++ b/tools/hil/rig-java.example.json @@ -0,0 +1,6 @@ +{ + "units": [ + { "name": "s3-xxxx", "mac": "00:06:66:XX:XX:XX", "type": "Shimmer3" }, + { "name": "s3r-xxxx", "mac": "E8:EB:1B:XX:XX:XX", "type": "Shimmer3R" } + ] +} diff --git a/tools/hil/run-java-hil.ps1 b/tools/hil/run-java-hil.ps1 new file mode 100644 index 000000000..047038bd2 --- /dev/null +++ b/tools/hil/run-java-hil.ps1 @@ -0,0 +1,234 @@ +<# +.SYNOPSIS +Runs the Java driver's hardware-in-the-loop (HIL) scenarios on every Shimmer in a rig file, over +the PC's paired Bluetooth serial ports, and prints a pass/fail table. Exits 1 if anything did not pass. + +.DESCRIPTION +One-time setup is in tools/hil/README.md: pair each unit with this PC, list it in +tools/hil/rig-java.json (copy rig-java.example.json), and check out the repository in a folder named +Shimmer-Java-Android-API (ShimmerDriverPC's settings.gradle finds its siblings by that name). + +.EXAMPLE +.\tools\hil\run-java-hil.ps1 +Every scenario on every unit. + +.EXAMPLE +.\tools\hil\run-java-hil.ps1 -Unit s3r-86f8 -Scenario inStream + +.EXAMPLE +.\tools\hil\run-java-hil.ps1 -Report +Before "Release Java API": the full run on this commit, recorded on GitHub as the commit status +"java-hil". The release refuses to publish a commit without a passing one. +#> +param( + [string]$Rig = (Join-Path $PSScriptRoot 'rig-java.json'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'queueStress'), + [string[]]$Unit, + [double]$Rate = 256, + [int]$Seconds = 30, + # Link CRC to ask for; by default the driver's own default, as an app gets it. + [ValidateSet('off', '1', '2')] + [string]$Crc, + [double]$MaxLossPercent = 1.0, + [switch]$SkipBuild, + [switch]$Report, + [int]$ScenarioTimeoutSeconds = 900, + # Pause between runs, so the unit has dropped the last link before the next connect. + [int]$SettleSeconds = 10 +) + +$ErrorActionPreference = 'Continue' +$repo = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path +$gradleRoot = Join-Path $repo 'ShimmerDriverPC' +$testClass = 'com.shimmerresearch.hil.JavaHilScenariosTest' + +function Fail([string]$message) { + Write-Host "ERROR: $message" -ForegroundColor Red + exit 2 +} + +# --- Rig ----------------------------------------------------------------------------------------- +if (-not (Test-Path $Rig)) { Fail "No rig file at $Rig. Copy rig-java.example.json to rig-java.json and list your units (README)." } +$rigData = Get-Content $Rig -Raw | ConvertFrom-Json +$units = @($rigData.units) +if ($Unit) { $units = @($units | Where-Object { $Unit -contains $_.name }) } +if ($units.Count -eq 0) { Fail "No units to test (rig: $Rig, -Unit: $Unit)." } + +# Each unit's outgoing Bluetooth serial port, from its MAC, unless the rig names one. +$ports = @{} +Get-CimInstance Win32_PnPEntity -Filter "PNPClass='Ports'" -ErrorAction SilentlyContinue | + Where-Object { $_.DeviceID -match 'BTHENUM' } | ForEach-Object { + $m = [regex]::Match($_.DeviceID, '&([0-9A-F]{12})_') + $p = [regex]::Match($_.Name, 'COM\d+') + if ($m.Success -and $p.Success -and $m.Groups[1].Value -ne '000000000000') { $ports[$m.Groups[1].Value] = $p.Value } + } +foreach ($u in $units) { + if (-not $u.type) { Fail "Unit $($u.name): give its type (Shimmer3 or Shimmer3R)." } + if (-not $u.comPort) { + $key = ($u.mac -replace '[:-]', '').ToUpper() + if (-not $ports.ContainsKey($key)) { Fail "Unit $($u.name) ($($u.mac)) is not paired with this PC: no Bluetooth serial port has its MAC. Pair it (README) or give comPort." } + $u | Add-Member -NotePropertyName comPort -NotePropertyValue $ports[$key] -Force + } +} + +# The release gate trusts this status, so only report a run that tested exactly this commit, with +# the gate's own settings: every scenario, both device types, defaults, a fresh build, clean tree. +$commit = $null +$githubRepo = $null +if ($Report) { + if ($PSBoundParameters.ContainsKey('Scenario') -or $Unit) { Fail '-Report needs every scenario on every unit in the rig; drop -Scenario and -Unit.' } + $shaping = @('Rate', 'Seconds', 'Crc', 'MaxLossPercent', 'SkipBuild') | Where-Object { $PSBoundParameters.ContainsKey($_) } + if ($shaping) { Fail "-Report runs with the default settings and a fresh build; drop $((@($shaping) | ForEach-Object { "-$_" }) -join ', ')." } + $types = @($units | ForEach-Object { $_.type }) + if (-not ($types -contains 'Shimmer3') -or -not ($types -contains 'Shimmer3R')) { + Fail "-Report needs at least one Shimmer3 and one Shimmer3R in the rig (types found: $($types -join ', '))." + } + $dirty = (git -C $repo status --porcelain) -join "`n" + if ($dirty) { Fail "-Report needs a clean working tree, so the tested code is exactly the commit:`n$dirty" } + $commit = ((git -C $repo rev-parse HEAD) -join '').Trim() + $remote = ((git -C $repo remote get-url origin) -join '').Trim() + $githubRepo = ([regex]::Match($remote, 'github\.com[:/](.+?)(\.git)?$')).Groups[1].Value + if (-not $githubRepo) { Fail "Cannot tell the GitHub repository from origin ($remote)." } + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { Fail '-Report posts the result with the GitHub CLI (gh); install it and log in.' } + gh api "repos/$githubRepo/commits/$commit" --silent 2>$null + if ($LASTEXITCODE -ne 0) { Fail "Commit $commit is not on GitHub ($githubRepo); push it first." } +} + +# --- Build --------------------------------------------------------------------------------------- +# ShimmerDriverPC's settings.gradle finds ShimmerDriver at ../../Shimmer-Java-Android-API/ShimmerDriver. +if ((Split-Path $repo -Leaf) -ne 'Shimmer-Java-Android-API') { + Fail ("ShimmerDriverPC only builds from a folder named Shimmer-Java-Android-API (its settings.gradle finds its siblings that way), " + + "and this checkout is $repo. Make a junction and run from it:`n" + + " New-Item -ItemType Junction -Path '$(Join-Path (Split-Path $repo -Parent) 'Shimmer-Java-Android-API')' -Target '$repo'") +} +function JavaMajor([string]$javaHome) { + $release = Join-Path $javaHome 'release' + if (-not (Test-Path $release)) { return 0 } + $m = [regex]::Match((Get-Content $release -Raw), 'JAVA_VERSION="(?:1\.)?(\d+)') + if ($m.Success) { return [int]$m.Groups[1].Value } else { return 0 } +} +if (-not $env:JAVA_HOME -or (JavaMajor $env:JAVA_HOME) -lt 11) { + $jdk = @(Get-ChildItem 'C:\Program Files\Amazon Corretto', 'C:\Program Files\Eclipse Adoptium', 'C:\Program Files\Java' -Directory -ErrorAction SilentlyContinue | + Where-Object { (JavaMajor $_.FullName) -ge 11 } | Sort-Object { JavaMajor $_.FullName }) | Select-Object -First 1 + if (-not $jdk) { Fail 'No JDK 11 or newer found; set JAVA_HOME.' } + $env:JAVA_HOME = $jdk.FullName +} + +$outDir = Join-Path $PSScriptRoot ('out\' + (Get-Date -Format 'yyyyMMdd-HHmmss')) +$reportDir = Join-Path $outDir 'reports' +New-Item -ItemType Directory -Force $reportDir | Out-Null +$sha = ((git -C $repo rev-parse --short HEAD) -join '').Trim() +Write-Host "Java HIL: driver source $sha, JDK $(JavaMajor $env:JAVA_HOME) ($env:JAVA_HOME)" +Write-Host "Results: $outDir" + +# --no-watch-fs: Gradle's file watcher can miss edits made through the real path of a junction, +# and would then run stale test classes. +$gradle = Join-Path $gradleRoot 'gradlew.bat' +if (-not $SkipBuild) { + Write-Host 'Building...' + Push-Location $gradleRoot + & $gradle ':testClasses' --no-watch-fs --console=plain *> (Join-Path $outDir 'build.log') + $buildExit = $LASTEXITCODE + Pop-Location + if ($buildExit -ne 0) { Fail "Build failed; see $(Join-Path $outDir 'build.log')" } +} + +# --- Run ----------------------------------------------------------------------------------------- +function Invoke-Scenario([string[]]$gradleArgs, [int]$timeoutSeconds, [string]$logFile) { + $p = Start-Process -FilePath $gradle -ArgumentList $gradleArgs -WorkingDirectory $gradleRoot -NoNewWindow -PassThru ` + -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err" + if (-not $p.WaitForExit($timeoutSeconds * 1000)) { + # A test JVM stuck in a deadlock outlives the Gradle client; stopping the daemons ends it. + & $gradle --stop *> $null + $p.Kill() + return $true + } + return $false +} + +$results = @() +$runs = 0 +foreach ($u in $units) { + foreach ($s in $Scenario) { + if ($runs++ -gt 0) { Start-Sleep -Seconds $SettleSeconds } + $label = "$($u.name) $s" + Write-Host "Running $label on $($u.comPort)..." -NoNewline + $gradleArgs = @(':hilTest', '--no-watch-fs', '--console=plain', '--tests', "$testClass.$s", + "-Phil.port=$($u.comPort)", "-Phil.mac=$($u.mac)", "-Phil.unit=$($u.name)", "-Phil.type=$($u.type)", + "-Phil.rate=$Rate", "-Phil.seconds=$Seconds", "-Phil.maxLossPercent=$MaxLossPercent", "-Phil.outDir=$reportDir") + if ($Crc) { $gradleArgs += "-Phil.crc=$Crc" } + $safe = "$($u.name)_$s" -replace '[^\w.-]', '_' + $log = Join-Path $outDir "$safe.console.txt" + $timedOut = Invoke-Scenario $gradleArgs $ScenarioTimeoutSeconds $log + + $verdict = 'ERROR' + $detail = '' + $xml = Join-Path $gradleRoot "build\test-results\hilTest\TEST-$testClass.xml" + if ($timedOut) { + $verdict = 'TIMEOUT' + $detail = "did not finish within $ScenarioTimeoutSeconds s; stopped (see $safe.console.txt)" + } + elseif (Test-Path $xml) { + Copy-Item $xml (Join-Path $outDir "$safe.junit.xml") + $case = ([xml](Get-Content $xml -Raw -Encoding UTF8)).testsuite.testcase | Where-Object { $_.name -eq $s } | Select-Object -First 1 + if (-not $case) { $detail = 'the scenario did not run (see the console)' } + elseif ($case.failure) { $verdict = 'FAIL'; $detail = (($case.failure.message -split "`n")[0]) -replace '^java\.lang\.AssertionError: ', '' } + elseif ($case.skipped) { $verdict = 'SKIP'; $detail = 'skipped' } + else { $verdict = 'PASS' } + Remove-Item $xml + } + else { + $detail = ((Get-Content $log, "$log.err" -ErrorAction SilentlyContinue | Select-String 'error|FAILURE|wrong' | Select-Object -First 2) -join ' ').Trim() + } + Write-Host " $verdict" -ForegroundColor $(if ($verdict -eq 'PASS') { 'Green' } elseif ($verdict -eq 'SKIP') { 'Yellow' } else { 'Red' }) + $results += [pscustomobject]@{ Unit = $u.name; Port = $u.comPort; Scenario = $s; Result = $verdict; Detail = $detail } + } +} + +# --- Collect ------------------------------------------------------------------------------------- +# Every run that reached a verdict wrote a report; the gate's status rests on them. +foreach ($r in $results) { + if ($r.Result -ne 'PASS' -and $r.Result -ne 'FAIL') { continue } + $name = ($r.Scenario -creplace '([A-Z])', '_$1').ToLower() + "_$($r.Unit -replace '[^\w.-]', '_').json" + if (-not (Test-Path (Join-Path $reportDir $name))) { + $r.Result = 'ERROR' + $r.Detail = ("no report written ($name). " + $r.Detail).Trim() + } +} +$summary = [pscustomobject]@{ + driver = $sha; rate = $Rate; seconds = $Seconds; crc = $(if ($Crc) { $Crc } else { 'driver default' }) + settleSeconds = $SettleSeconds; results = $results +} +$summary | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $outDir 'summary.json') + +Write-Host '' +$results | Format-Table -AutoSize -Wrap | Out-String -Width 200 | Write-Host +# A skipped scenario did not run, so it is not a pass: the exit code and the gate's status agree. +$failed = @($results | Where-Object { $_.Result -ne 'PASS' }) +$passed = @($results | Where-Object { $_.Result -eq 'PASS' }) + +if ($Report) { + $firmware = @{} + Get-ChildItem $reportDir -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { + $r = Get-Content $_.FullName -Raw | ConvertFrom-Json + if ($r.firmware -and $r.unit) { $firmware[$r.unit] = "$($r.hardware) $($r.firmware)" } + } + $tested = @($units | ForEach-Object { if ($firmware[$_.name]) { $firmware[$_.name] } else { $_.type } }) | Sort-Object -Unique + $state = if ($passed.Count -eq $results.Count) { 'success' } else { 'failure' } + $description = "$($passed.Count)/$($results.Count) passed over Classic: $($tested -join ', ')" + if ($description.Length -gt 140) { $description = $description.Substring(0, 137) + '...' } + gh api -X POST "repos/$githubRepo/statuses/$commit" -f state=$state -f context=java-hil -f description=$description --silent + if ($LASTEXITCODE -ne 0) { + Write-Host 'Could not post the commit status; the release will not see this run.' -ForegroundColor Red + exit 2 + } + Write-Host "Posted 'java-hil' = $state on $githubRepo@$($commit.Substring(0, 8)): $description" +} + +if ($failed.Count -gt 0) { + Write-Host "$($failed.Count) of $($results.Count) did not pass. Reports and console output: $outDir" -ForegroundColor Red + exit 1 +} +Write-Host "All $($results.Count) passed. Reports: $outDir" -ForegroundColor Green +exit 0