diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d0d42b7..cbd35d23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,12 @@ # # If only the release job fails, "Re-run failed jobs" re-runs just that job. # +# Hardware test gate: before releasing, run tools/hil/run-android-hil.ps1 -Report on the bench at the +# commit being released (see tools/hil/README.md). It posts the commit status +# "android-hil (driver X)". A real release refuses to build unless that status passed for this commit +# and this driver, or hil_waiver gives a Jira key (the waiver goes into the release notes). Dry runs +# only warn. The release notes say what the hardware test found. +# # The driver is read from Shimmer-Java-Android-API's GitHub Packages with this repo's GITHUB_TOKEN. # If that is ever refused (401/403 resolving com.shimmerresearch:shimmerdriver), add a classic token # with read:packages as the PACKAGES_READ_TOKEN secret and it is used instead. @@ -45,6 +51,10 @@ on: description: 'Java API (driver) version to build against, e.g. 0.11.9_beta. Blank = newest release' type: string required: false + hil_waiver: + description: 'Jira key to release without a passing hardware test (HIL), e.g. DEV-1234. Blank = require one' + type: string + required: false dry_run: description: 'Build only; publish nothing' type: boolean @@ -71,6 +81,7 @@ jobs: permissions: contents: read packages: read + statuses: read # the hardware test result the bench posts on the commit env: DRY_RUN: ${{ github.event_name != 'workflow_dispatch' || inputs.dry_run }} # Read by the repositories in the root build.gradle to resolve the driver. @@ -81,6 +92,7 @@ jobs: tag: ${{ steps.version.outputs.tag }} previous_tag: ${{ steps.version.outputs.previous_tag }} driver: ${{ steps.version.outputs.driver }} + hil: ${{ steps.hil.outputs.hil }} steps: - name: Release from master only if: env.DRY_RUN != 'true' && github.ref != 'refs/heads/master' @@ -167,6 +179,38 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Releasing $VERSION as $TAG against driver $DRIVER (master's default: ${default_driver:-none}; notes since ${PREVIOUS_TAG:-the first commit})" + # Before any building, so a missing hardware test fails the release in seconds. + - name: Check the hardware test (HIL) result + id: hil + working-directory: . + env: + GH_TOKEN: ${{ github.token }} + DRIVER: ${{ steps.version.outputs.driver }} + WAIVER: ${{ inputs.hil_waiver }} + run: | + context="android-hil (driver $DRIVER)" + if [ -n "$WAIVER" ] && ! [[ "$WAIVER" =~ ^[A-Z][A-Z0-9]+-[0-9]+$ ]]; then + echo "::error::hil_waiver must be a Jira key such as DEV-1234, not '$WAIVER'." + exit 1 + fi + # Newest first: the latest result for this context on this exact commit decides. + line=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/statuses" --paginate \ + --jq '.[] | [.context, .state, .description] | @tsv' | awk -F'\t' -v c="$context" '$1 == c { print; exit }') + state=$(printf '%s' "$line" | cut -f2) + description=$(printf '%s' "$line" | cut -f3) + if [ "$state" = success ]; then + echo "Hardware test passed on $GITHUB_SHA with driver $DRIVER: $description" + echo "hil=Tested on hardware before release (driver $DRIVER): $description." >> "$GITHUB_OUTPUT" + elif [ -n "$WAIVER" ]; then + echo "::warning::No passing '$context' on $GITHUB_SHA (found: ${state:-none}); released under waiver $WAIVER." + echo "hil=Released without a passing hardware test (HIL), waived under $WAIVER." >> "$GITHUB_OUTPUT" + elif [ "$DRY_RUN" = true ]; then + echo "::warning::No passing '$context' on $GITHUB_SHA (found: ${state:-none}). A real release would stop here." + else + echo "::error::No passing '$context' on $GITHUB_SHA (found: ${state:-none}${description:+: $description}). On the bench, at this commit, run: tools/hil/run-android-hil.ps1 -Report -DriverVersion $DRIVER. Then run this workflow again, or give a Jira key in hil_waiver." + exit 1 + fi + # AGP 8 needs JDK 17. - name: Set up JDK 17 uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 @@ -271,6 +315,7 @@ jobs: TAG: ${{ needs.build.outputs.tag }} PREVIOUS_TAG: ${{ needs.build.outputs.previous_tag }} DRIVER: ${{ needs.build.outputs.driver }} + HIL: ${{ needs.build.outputs.hil }} steps: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: @@ -290,6 +335,10 @@ jobs: # The driver's JNA is desktop-only; its two jars clash on META-INF/AL2.0 in an Android app. echo "Exclude \`net.java.dev.jna\` from both, or the app fails packaging with \"2 files found with path 'META-INF/AL2.0'\". The exclusions this library builds with are in [ShimmerAndroidInstrumentDriver/build.gradle]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/$TAG/ShimmerAndroidInstrumentDriver/build.gradle)." echo + if [ -n "$HIL" ]; then + echo "$HIL" + echo + fi gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" "${args[@]}" --jq .body } > notes.md gh release create "$TAG" release-assets/* \ @@ -302,4 +351,5 @@ jobs: echo "- Release: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$TAG" echo "- Package: com.shimmerresearch:shimmerandroidinstrumentdriver:$VERSION" echo "- Built against driver $DRIVER" + echo "- ${HIL:-No hardware test result}" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 475319fc..30472aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,8 @@ lint/tmp/ gradle.properties ShimmerAndroidInstrumentDriver/.idea/jarRepositories.xml ShimmerAndroidInstrumentDriver/.idea/gradle.xml + +# Android HIL (tools/hil): the local rig file and run output +tools/hil/rig-android.json +tools/hil/out/ +tools/hil/__pycache__/ diff --git a/README.md b/README.md index a6a0e203..8d7f519b 100644 --- a/README.md +++ b/README.md @@ -91,3 +91,6 @@ compile 'ShimmerAndroidInstrumentDriver:ShimmerAndroidInstrumentDriver:3.0.69Bet + +# Hardware tests +The Android API can be tested on a phone against real Shimmer3 and Shimmer3R units with one command: see [tools/hil/README.md](tools/hil/README.md). diff --git a/ShimmerAndroidInstrumentDriver/settings.gradle b/ShimmerAndroidInstrumentDriver/settings.gradle index e523ef6f..386fd352 100644 --- a/ShimmerAndroidInstrumentDriver/settings.gradle +++ b/ShimmerAndroidInstrumentDriver/settings.gradle @@ -5,3 +5,4 @@ include ':shimmer3DOrientationExample', ':shimmerConnectionTest' //project (":ShimmerAdvanceLibrary").projectDir = new File("../../ShimmerAdvanceAPI/ShimmerAdvanceLibrary") include ':multiverisenseblebasicexample' , ':shimmer3BLEBasicExample' include ':shimmerspeedtest' +include ':shimmerHilTest' diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/build.gradle b/ShimmerAndroidInstrumentDriver/shimmerHilTest/build.gradle new file mode 100644 index 00000000..c223e99d --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/build.gradle @@ -0,0 +1,52 @@ +// Hardware-in-the-loop (HIL) tests of the Android API on a USB-attached phone against real +// Shimmer3 / Shimmer3R units. The tests are in src/androidTest; this app only hosts them. Run them +// with tools/hil/run-android-hil.ps1 (see tools/hil/README.md). +// +// Which Android API is tested: +// default the ShimmerAndroidInstrumentDriver source in this checkout +// -PandroidApiVersion=3.2.10_beta the published .aar, as a customer app gets it +// Which driver: -PdriverVersion, read by the root build.gradle like every other module. +apply plugin: 'com.android.application' + +def androidApiVersion = findProperty('androidApiVersion') + +android { + namespace 'com.shimmerresearch.hiltest' + compileSdkVersion 34 + + defaultConfig { + applicationId "com.shimmerresearch.hiltest" + minSdkVersion 21 + targetSdkVersion 34 + versionCode 1 + versionName "1.0" + multiDexEnabled true + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + packagingOptions { + resources { + excludes += ['META-INF/INDEX.LIST', 'META-INF/DEPENDENCIES', 'META-INF/LICENSE', + 'META-INF/LICENSE.txt', 'META-INF/license.txt', 'META-INF/NOTICE', + 'META-INF/NOTICE.txt', 'META-INF/notice.txt', 'META-INF/ASL2.0'] + } + } + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } +} + +dependencies { + if (androidApiVersion) { + implementation(group: 'com.shimmerresearch', name: 'shimmerandroidinstrumentdriver', version: androidApiVersion, ext: 'aar') + } else { + implementation project(':ShimmerAndroidInstrumentDriver') + } + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test:runner:1.5.2' + androidTestImplementation 'androidx.test:rules:1.5.0' + androidTestImplementation 'androidx.test.ext:junit:1.1.5' + androidTestImplementation 'junit:junit:4.13.2' +} diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilReport.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilReport.java new file mode 100644 index 00000000..c501e98c --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilReport.java @@ -0,0 +1,152 @@ +package com.shimmerresearch.hiltest; + +import android.content.Context; +import android.os.Bundle; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +/** + * Everything one scenario run found, written as JSON (plus a CSV of samples) to the app's + * external files dir, hil/, for tools/hil/run-android-hil.ps1 to pull. + */ +final class HilReport { + + private final Context context; + private final String scenario; + private final String transport; + private final String mac; + private final JSONObject json = new JSONObject(); + private final List failures = new ArrayList<>(); + private List samples; + + HilReport(Context context, String scenario, String transport, String mac, Bundle args) throws Exception { + this.context = context; + this.scenario = scenario; + this.transport = transport; + this.mac = mac; + json.put("scenario", scenario); + json.put("transport", transport); + json.put("mac", mac); + json.put("started", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ROOT).format(new Date())); + JSONObject a = new JSONObject(); + for (String key : args.keySet()) { + a.put(key, String.valueOf(args.get(key))); + } + json.put("args", a); + } + + /** org.json refuses NaN and infinities, which a failing run is the likeliest to produce; store null. */ + private static Object num(double v) { + return Double.isNaN(v) || Double.isInfinite(v) ? JSONObject.NULL : v; + } + + void put(String key, Object value) { + try { + json.put(key, value instanceof Double || value instanceof Float ? num(((Number) value).doubleValue()) : value); + } catch (Exception e) { + Log.w(ShimmerSession.TAG, "report: " + key + ": " + e); + } + } + + void putStream(String key, StreamAnalysis a) { + try { + JSONObject o = new JSONObject(); + o.put("received", a.received); + o.put("missing", a.missing); + o.put("lossPercent", num(a.lossPercent())); + o.put("nonIncreasing", a.nonIncreasing); + o.put("timestampJumps", a.timestampJumps.size()); + o.put("effectiveRateHz", num(a.effectiveRateHz)); + o.put("accelMagnitudeMean", num(a.accelMagnitudeMean)); + o.put("accelMagnitudeStd", num(a.accelMagnitudeStd)); + o.put("gaps", a.gaps.size()); + json.put(key, o); + } catch (Exception e) { + Log.w(ShimmerSession.TAG, "report: " + key + ": " + e); + } + } + + /** Every gap, with when it arrived, so a loss can be matched to what the test was doing. */ + void putGaps(StreamAnalysis a) { + try { + JSONArray gaps = new JSONArray(); + for (StreamAnalysis.Gap g : a.gaps) { + JSONObject o = new JSONObject(); + o.put("afterReceivedMs", Math.round(g.afterReceivedMs)); + o.put("afterTimestampMs", num(g.afterTimestampMs)); + o.put("missing", g.missing); + gaps.put(o); + } + json.put("gapList", gaps); + JSONArray jumps = new JSONArray(); + for (StreamAnalysis.Gap g : a.timestampJumps) { + JSONObject o = new JSONObject(); + o.put("afterReceivedMs", Math.round(g.afterReceivedMs)); + o.put("afterTimestampMs", num(g.afterTimestampMs)); + o.put("periodsAhead", g.missing + 1); + jumps.put(o); + } + json.put("timestampJumpList", jumps); + } catch (Exception e) { + Log.w(ShimmerSession.TAG, "report: gapList: " + e); + } + } + + void samples(List samples) { + this.samples = samples; + } + + /** Records a failed check; the test fails at the end with every one of them. */ + void check(boolean ok, String failure) { + if (!ok) { + failures.add(failure); + Log.e(ShimmerSession.TAG, scenario + " " + transport + " " + mac + " FAIL: " + failure); + } + } + + List failures() { + return failures; + } + + void write(List events) { + try { + json.put("passed", failures.isEmpty()); + json.put("failures", new JSONArray(failures)); + json.put("events", new JSONArray(events)); + File dir = new File(context.getExternalFilesDir(null), "hil"); + if (!dir.exists() && !dir.mkdirs()) { + throw new IllegalStateException("cannot create " + dir); + } + String base = scenario + "_" + transport + "_" + mac.replace(":", "") + "_" + + new SimpleDateFormat("yyyyMMdd-HHmmss", Locale.ROOT).format(new Date()); + try (Writer w = new OutputStreamWriter(new FileOutputStream(new File(dir, base + ".json")), StandardCharsets.UTF_8)) { + w.write(json.toString(2)); + } + if (samples != null) { + try (Writer w = new OutputStreamWriter(new FileOutputStream(new File(dir, base + ".csv")), StandardCharsets.UTF_8)) { + w.write("received_ms,timestamp_ms,accel_x,accel_y,accel_z\n"); + for (StreamAnalysis.Sample s : samples) { + w.write(String.format(Locale.ROOT, "%.0f,%.3f,%.4f,%.4f,%.4f\n", + s.receivedMs, s.timestampMs, s.accelX, s.accelY, s.accelZ)); + } + } + } + Log.i(ShimmerSession.TAG, "report written: " + new File(dir, base + ".json")); + } catch (Exception e) { + Log.e(ShimmerSession.TAG, "could not write report: " + e); + } + } +} diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java new file mode 100644 index 00000000..24bf7030 --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/HilScenariosTest.java @@ -0,0 +1,849 @@ +package com.shimmerresearch.hiltest; + +import static org.junit.Assume.assumeTrue; + +import android.Manifest; +import android.content.Context; +import android.os.Build; +import android.os.Bundle; +import android.os.SystemClock; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.rule.GrantPermissionRule; + +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_CRC_MODE; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Hardware-in-the-loop scenarios against one real Shimmer3 or Shimmer3R, paired with the phone over + * Bluetooth Classic. Arguments come from `am instrument -e key value`: + * + *
+ *   mac             XX:XX:XX:XX:XX:XX (required; without it every test is skipped)
+ *   transport       classic (default; the unit must be paired) or ble (no pairing)
+ *   type            Shimmer3 (default) or Shimmer3R; over BLE it picks the device class
+ *   rate            sampling rate in Hz, default 256
+ *   seconds         stream_rest duration, default 30
+ *   cycles          mid_stream_set / reconnect_loop / in_stream repetitions, default 3 / 5 / 3
+ *   crc             link CRC to ask for: off, 1 or 2 (default 2; the driver may settle on less)
+ *   maxLossPercent  default 1.0
+ *   queueStressSeconds  how long queue_stress queues reads, default 60
+ *   accelTolerance  m/s^2 either side of 9.81 at rest, default 1.0
+ * 
+ * + * tools/hil/run-android-hil.ps1 runs each scenario for each unit in a rig file and pulls the reports. + */ +@RunWith(AndroidJUnit4.class) +public class HilScenariosTest { + + private static final double GRAVITY = 9.81; + + /** + * Connecting to an already paired unit needs no runtime permission before Android 12 (BLUETOOTH is + * granted at install; location is only for scanning, which these tests never do). From Android 12, + * BLUETOOTH_CONNECT is a runtime permission. Some phones (Xiaomi) refuse runtime grants over USB + * without "USB debugging (Security settings)", so grant nothing where nothing is needed. + */ + @Rule + public TestRule permissions = Build.VERSION.SDK_INT >= Build.VERSION_CODES.S + ? GrantPermissionRule.grant(Manifest.permission.BLUETOOTH_CONNECT, Manifest.permission.BLUETOOTH_SCAN) + : RuleChain.emptyRuleChain(); + + private Bundle args; + private Context context; + private ShimmerSession session; + private HilReport report; + + private String arg(String key, String fallback) { + String v = args.getString(key); + return v == null || v.isEmpty() ? fallback : v; + } + + private double arg(String key, double fallback) { + return Double.parseDouble(arg(key, String.valueOf(fallback))); + } + + private int arg(String key, int fallback) { + return Integer.parseInt(arg(key, String.valueOf(fallback))); + } + + private void begin(String scenario) throws Exception { + args = InstrumentationRegistry.getArguments(); + String mac = args.getString("mac"); + assumeTrue("No unit given: pass -e mac XX:XX:XX:XX:XX:XX", mac != null && !mac.isEmpty()); + mac = mac.toUpperCase(Locale.ROOT); + context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + String crc = arg("crc", "2"); + ShimmerBluetooth.setDefaultBtCrcModeToUseIfFwSupported( + "off".equals(crc) ? BT_CRC_MODE.OFF : "1".equals(crc) ? BT_CRC_MODE.ONE_BYTE_CRC : BT_CRC_MODE.TWO_BYTE_CRC); + String transport = arg("transport", "classic"); + assumeTrue("transport must be classic or ble, not " + transport, "classic".equals(transport) || "ble".equals(transport)); + report = new HilReport(context, scenario, transport, mac, args); + session = new ShimmerSession(context, mac, transport, arg("type", "Shimmer3")); + } + + @After + public void end() throws Exception { + try { + if (session != null) { + session.close(); + } + } finally { + if (report != null) { + report.write(session != null ? session.events() : new ArrayList()); + } + } + } + + private void finish() { + if (session != null && session.hung != null) { + report.put("hungCall", session.hung); + report.check(false, session.hung + " never returned: the API call blocked (thread dump in the report's events)"); + } + if (!report.failures().isEmpty()) { + Assert.fail(android.text.TextUtils.join("; ", report.failures())); + } + } + + /** Connects, records what the unit is, and writes the accel-only config at the test's rate. */ + private boolean connectAndConfigure(double rateHz) throws Exception { + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + if (!connected) { + return false; + } + ShimmerBluetooth d = session.device(); + if (d == null) { + report.check(false, "the connection was lost right after connecting (state " + session.state() + ")"); + return false; + } + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + boolean configured = session.configureAccelOnly(rateHz, 20000); + report.put("rateOnDeviceHz", d.getSamplingRateShimmer()); + report.check(configured, "configuration did not complete within 20 s (state " + session.state() + ")"); + if (configured && session.device() == null) { + report.check(false, "the connection was lost right after configuring (state " + session.state() + ")"); + return false; + } + return configured; + } + + private void checkStream(StreamAnalysis a, List samples, double stoppedAtMs, + double maxLossPercent, double deviceRateHz, String what) { + report.check(a.received > 0, what + ": no packets received"); + report.check(a.lossPercent() <= maxLossPercent, String.format(Locale.ROOT, + "%s: %.3f%% packets lost (%d of %d), limit %.2f%%", what, a.lossPercent(), a.missing, + a.received + a.missing, maxLossPercent)); + report.check(a.nonIncreasing == 0, what + ": " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, what); + checkTimeline(a, samples, stoppedAtMs, deviceRateHz, what); + } + + /** + * Loss is counted between packets received, so two things it cannot see are checked here: the + * device timestamps must give a rate (at least two usable ones) within 1 % of the configured + * rate, and packets must have kept arriving until the test stopped recording, since a stream + * that fell silent leaves no gap between the packets it did deliver. + */ + private void checkTimeline(StreamAnalysis a, List samples, double stoppedAtMs, + double deviceRateHz, String what) { + if (Double.isNaN(a.effectiveRateHz) || Double.isInfinite(a.effectiveRateHz)) { + report.check(false, what + ": no usable device timestamps to measure the rate from (" + a.received + " packets)"); + } else { + report.check(Math.abs(a.effectiveRateHz - deviceRateHz) <= 0.01 * deviceRateHz, String.format(Locale.ROOT, + "%s: effective rate %.2f Hz, device set to %.2f Hz", what, a.effectiveRateHz, deviceRateHz)); + } + if (!samples.isEmpty()) { + double silentMs = stoppedAtMs - samples.get(samples.size() - 1).receivedMs; + report.check(silentMs <= MAX_SILENCE_AT_END_MS, String.format(Locale.ROOT, + "%s: no packet in the last %.1f s before recording stopped (the stream went silent)", what, silentMs / 1000.0)); + } + } + + /** At 256 Hz a second without a packet is ~256 packets the gap count cannot see. */ + private static final double MAX_SILENCE_AT_END_MS = 1000; + + /** Every stream check is relative to the unit's rate, so a run without one cannot pass. */ + private void checkRate(double deviceRateHz, String what) { + report.check(!Double.isNaN(deviceRateHz) && deviceRateHz > 0, + what + ": the API has no sampling rate for the unit (" + deviceRateHz + " Hz; connection lost?)"); + } + + /** A device timestamp that leapt ahead of the arrival time is wrong data (DEV-1023 class), never loss. */ + private void checkJumps(StreamAnalysis a, String what) { + for (StreamAnalysis.Gap j : a.timestampJumps) { + report.check(false, String.format(Locale.ROOT, + "%s: timestamp jumped %.1f s ahead at %.0f ms (device time %.0f ms) while packets kept arriving", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs, j.afterTimestampMs)); + } + } + + /** Connect, configure, stream at rest, stop, disconnect. */ + @Test + public void streamRest() throws Exception { + begin("stream_rest"); + double rate = arg("rate", 256.0); + int seconds = arg("seconds", 30); + double maxLoss = arg("maxLossPercent", 1.0); + double accelTolerance = arg("accelTolerance", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = session.rateOnDevice(); + checkRate(deviceRate, "stream"); + String marker = session.logMarker("stream_rest"); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(seconds * 1000L); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, "stream"); + if (!Double.isNaN(a.accelMagnitudeMean)) { + report.check(Math.abs(a.accelMagnitudeMean - GRAVITY) <= accelTolerance, String.format(Locale.ROOT, + "accel at rest %.3f m/s^2, expected %.2f±%.2f (is the unit lying still?)", + a.accelMagnitudeMean, GRAVITY, accelTolerance)); + } else { + report.check(false, "no calibrated accel in the packets"); + } + int parserErrors = session.parserErrorsSince(marker); + report.put("parserErrors", parserErrors); + report.check(parserErrors == 0, parserErrors + " streaming parser errors logged (\"Packet syncing problem\")"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** + * Commands while streaming: start and stop SD logging (SET_RWC_COMMAND, whose bare ACK carries the + * link CRC: DEV-1147 in the driver, DEV-1148 for Android's own parser) and read the battery. No + * packet may be lost around any of them, and logging must actually start and stop. Needs an SD + * card in the unit and a link CRC on to exercise DEV-1148. + */ + @Test + public void midStreamSet() throws Exception { + begin("mid_stream_set"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 3); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + ShimmerBluetooth d = session.device(); + double deviceRate = d.getSamplingRateShimmer(); + checkRate(deviceRate, "stream"); + BT_CRC_MODE crc = d.getCurrentBtCommsCrcMode(); + if (crc == BT_CRC_MODE.OFF) { + session.event("link CRC is off: the DEV-1148 path is not exercised"); + } + String marker = session.logMarker("mid_stream_set"); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(3000); + List sdMoments = new ArrayList<>(); + List batteryMoments = new ArrayList<>(); + for (int i = 1; i <= cycles && session.hung == null; i++) { + sdMoments.add(session.nowMs()); + session.event("cycle " + i); + // A call that threw, hung or was skipped is a failure in itself, not a reason to skip the + // state check; the events say which. + boolean started = session.startSdLogging(); + report.check(started, "cycle " + i + ": startSDLogging did not complete: it threw, hung or was skipped after a hung call (see the events)"); + if (started) { + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING, 5000), + "cycle " + i + ": SD logging did not start while streaming (state " + session.state() + + "; is there an SD card? With a link CRC on, DEV-1148)"); + } + Thread.sleep(2000); + sdMoments.add(session.nowMs()); + boolean stopped = session.stopSdLogging(); + report.check(stopped, "cycle " + i + ": stopSDLogging did not complete: it threw, hung or was skipped after a hung call (see the events)"); + if (stopped) { + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING, 5000), + "cycle " + i + ": SD logging did not stop (state " + session.state() + ")"); + } + Thread.sleep(1000); + batteryMoments.add(session.nowMs()); + // readBattery() only sends the command, so wait for the reply to be decoded as well. + final Object batteryBefore = session.batteryDetails(); + boolean batteryAsked = session.readBattery(); + report.check(batteryAsked, "cycle " + i + ": readBattery did not complete: it threw, hung or was skipped after a hung call (see the events)"); + if (batteryAsked) { + report.check(session.waitFor(() -> { + Object now = session.batteryDetails(); + return now != null && now != batteryBefore; + }, 5000), "cycle " + i + ": no battery reply decoded within 5 s of readBattery"); + } + Thread.sleep(2000); + } + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + report.samples(samples); + // Loss around SD logging start/stop is reported, not failed: with no parser error and + // logging starting, the unit itself did not send those samples while it opened or closed + // its file, which is firmware behaviour for the firmware HIL bench to judge. Loss around a + // battery read, or anywhere else, is the Android API's. + int[] attributed = a.attribute(new double[][] {toArray(sdMoments), toArray(batteryMoments)}, 1000); + int nearSd = attributed[0]; + int nearBattery = attributed[1]; + report.put("missingNearSdLoggingStartStop", nearSd); + report.put("missingNearBatteryReads", nearBattery); + if (nearSd > 0) { + session.event(nearSd + " packets not received within 1 s of SD logging start/stop (firmware side; not failed)"); + } + report.check(nearBattery == 0, nearBattery + " packets lost within 1 s of a battery read while streaming"); + int elsewhere = a.missing - nearSd; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from SD logging start/stop (%d), limit %.2f%%", + elsewherePercent, elsewhere, maxLoss)); + report.check(a.received > 0, "stream: no packets received"); + report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); + checkJumps(a, "stream"); + checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + int parserErrors = session.parserErrorsSince(marker); + report.put("parserErrors", parserErrors); + report.check(parserErrors == 0, parserErrors + " streaming parser errors logged (\"Packet syncing problem\")"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Reads sent after each SD logging stop in inStream, alternately battery and status */ + private static final int READS_PER_BURST = 10; + + /** Between those reads: the unit pauses for about 100 ms, so two or three land in the pause */ + private static final int READ_INTERVAL_MS = 40; + + /** What the driver logs when a command gets no reply in time: "Command: [0x95] GET_VBATT_COMMAND timeout" */ + private static final String[] COMMAND_TIMEOUTS = {"_COMMAND timeout", "UNKNOWN timeout"}; + + /** + * In-stream replies while the unit pauses its data packets (DEV-1291). After STOP_LOGGING_ONLY the + * unit sends no data packets for about 100 ms while it closes its SD file, and answers a command + * sent then at once, with no data packet before the reply. Android's Classic parser threw such a + * reply away. Each round starts SD logging, then stops it and reads the battery and the status + * every 40 ms for 400 ms: Android writes those reads straight out, so some land in the pause. Then + * each restart stops logging and starts it again back to back, so the queued START_LOGGING_ONLY goes + * out in the pause and its bare ACK comes first. Every battery read must be answered, logging must + * restart, and nothing may log a parser error or a command timeout. Needs an SD card in the unit. + */ + @Test + public void inStream() throws Exception { + begin("in_stream"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 3); + double maxLoss = arg("maxLossPercent", 1.0); + if (connectAndConfigure(rate)) { + double deviceRate = session.rateOnDevice(); + checkRate(deviceRate, "stream"); + LogTally tally = new LogTally(); + String marker = session.logMarker("in_stream start"); + session.startRecording(); + report.check(session.startStreaming(10000), "streaming did not start within 10 s"); + Thread.sleep(2000); + tally.add(marker); + List sdMoments = new ArrayList<>(); + List readMoments = new ArrayList<>(); + int batteryReads = 0; + int statusReads = 0; + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "round " + i; + session.event(what); + marker = session.logMarker("in_stream " + what); + if (!startSdLoggingWhileStreaming(what, sdMoments)) { + break; + } + Thread.sleep(1000); + sdMoments.add(session.nowMs()); + report.check(session.stopSdLogging(), what + ": stopSDLogging did not complete (see the events)"); + for (int k = 0; k < READS_PER_BURST && session.hung == null; k++) { + readMoments.add(session.nowMs()); + if (k % 2 == 0) { + batteryReads += session.readBattery() ? 1 : 0; + } else { + statusReads += session.readStatus() ? 1 : 0; + } + Thread.sleep(READ_INTERVAL_MS); + } + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING && session.queuedInstructions() == 0, 5000), + what + ": SD logging did not stop (state " + session.state() + ", " + session.queuedInstructions() + " instructions queued)"); + Thread.sleep(1000); + tally.add(marker); + } + marker = session.logMarker("in_stream restarts"); + if (session.hung == null && startSdLoggingWhileStreaming("restarts", sdMoments)) { + for (int i = 1; i <= cycles && session.hung == null; i++) { + String what = "restart " + i; + session.event(what); + Thread.sleep(1000); + tally.add(marker); + marker = session.logMarker("in_stream " + what); + sdMoments.add(session.nowMs()); + report.check(session.stopSdLogging() && session.startSdLogging(), + what + ": stopSDLogging or startSDLogging did not complete (see the events)"); + report.check(session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING && session.queuedInstructions() == 0, 5000), + what + ": logging did not restart after a stop and a start back to back (state " + session.state() + + ", " + session.queuedInstructions() + " instructions queued)"); + } + Thread.sleep(1000); + sdMoments.add(session.nowMs()); + report.check(session.stopSdLogging() && session.waitFor(() -> session.state() == BT_STATE.STREAMING, 5000), + "SD logging did not stop at the end (state " + session.state() + ")"); + } + Thread.sleep(1000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), "streaming did not stop within 10 s"); + tally.add(marker); + + report.put("batteryReads", batteryReads); + report.put("batteryReplies", tally.batteryReplies); + report.put("statusReads", statusReads); + report.put("statusReplies", tally.statusReplies); + report.put("repliesInAPause", tally.repliesInAPause); + report.put("commandTimeouts", tally.timeouts); + report.put("parserErrors", tally.parserErrors); + session.event(String.format(Locale.ROOT, "battery %d reads, %d replies; status %d reads, %d replies (and pushes); %d with no data packet before them", + batteryReads, tally.batteryReplies, statusReads, tally.statusReplies, tally.repliesInAPause)); + report.check(tally.complete, "the phone's log rolled past a round's start, so its replies and errors could not all be counted (see the events)"); + report.check(tally.batteryReplies == batteryReads, batteryReads + " battery reads but " + tally.batteryReplies + " replies decoded"); + report.check(tally.statusReplies >= statusReads, statusReads + " status reads but " + tally.statusReplies + " status replies decoded"); + report.check(tally.timeouts == 0, tally.timeouts + " command timeouts logged"); + report.check(tally.parserErrors == 0, tally.parserErrors + " streaming parser errors logged (\"Packet syncing problem\")"); + + // Loss near SD logging start/stop is the unit's own (see midStreamSet). A Shimmer3 also skips a + // sample now and then around a battery read, with no parser error, so loss near the reads is + // reported, not failed. + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event("stream: " + a); + report.putStream("stream", a); + report.putGaps(a); + int[] attributed = a.attribute(new double[][] {toArray(sdMoments), toArray(readMoments)}, 1000); + report.put("missingNearSdLoggingStartStop", attributed[0]); + report.put("missingNearReads", attributed[1]); + int elsewhere = a.missing - attributed[0] - attributed[1]; + double elsewherePercent = 100.0 * elsewhere / Math.max(1, a.received + a.missing); + report.put("missingElsewhere", elsewhere); + report.check(elsewherePercent <= maxLoss, String.format(Locale.ROOT, + "stream: %.3f%% packets lost away from SD logging start/stop and the reads (%d), limit %.2f%%", + elsewherePercent, elsewhere, maxLoss)); + report.check(a.received > 0, "stream: no packets received"); + report.check(a.nonIncreasing == 0, "stream: " + a.nonIncreasing + " timestamps did not increase"); + checkJumpsAwayFrom(a, sdMoments, "stream"); + checkTimeline(a, samples, stoppedAtMs, deviceRate, "stream"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** inStream's in-stream replies, pause replies, timeouts and parser errors, counted a round at a time */ + private final class LogTally { + int batteryReplies; + int statusReplies; + int repliesInAPause; + int timeouts; + int parserErrors; + /** False once a round's lines could not be counted */ + boolean complete = true; + + /** Adds the lines logged since the marker. Called soon after it, before the phone's small log buffer rolls past it. */ + void add(String marker) { + int[] c = session.logCountsSince(marker, new String[] {"IN STREAM: VBATT_RESPONSE"}, new String[] {"IN STREAM: STATUS_RESPONSE"}, + new String[] {"with no data packet before it"}, COMMAND_TIMEOUTS, ShimmerSession.PARSER_ERRORS); + if (c == null) { + complete = false; + return; + } + batteryReplies += c[0]; + statusReplies += c[1]; + repliesInAPause += c[2]; + timeouts += c[3]; + parserErrors += c[4]; + } + } + + /** + * As checkJumps(), except that a jump within 3 s after an SD logging start or stop is reported, not + * failed. The unit can pause for over a second while it opens its file, then send a packet or two it + * held from before the pause and, a millisecond later, the next one it sampled: a timestamp leap + * with no pause in arrival, but the unit's own (the bytes captured on a Shimmer3 show it). + */ + private void checkJumpsAwayFrom(StreamAnalysis a, List sdMoments, String what) { + int nearSd = 0; + for (StreamAnalysis.Gap j : a.timestampJumps) { + boolean near = false; + for (double moment : sdMoments) { + near |= j.afterReceivedMs >= moment && j.afterReceivedMs - moment <= 3000; + } + if (near) { + nearSd++; + session.event(String.format(Locale.ROOT, "%s: timestamp jumped %.1f s ahead at %.0f ms, within 3 s of an SD logging start or stop (the unit's own; not failed)", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs)); + } else { + report.check(false, String.format(Locale.ROOT, + "%s: timestamp jumped %.1f s ahead at %.0f ms (device time %.0f ms) while packets kept arriving", + what, (j.missing + 1) * a.expectedPeriodMs / 1000.0, j.afterReceivedMs, j.afterTimestampMs)); + } + } + report.put("timestampJumpsNearSdLoggingStartStop", nearSd); + } + + /** Starts SD logging while streaming and waits until it has started */ + private boolean startSdLoggingWhileStreaming(String what, List sdMoments) throws InterruptedException { + sdMoments.add(session.nowMs()); + boolean started = session.startSdLogging() + && session.waitFor(() -> session.state() == BT_STATE.STREAMING_AND_SDLOGGING && session.queuedInstructions() == 0, 5000); + report.check(started, what + ": SD logging did not start while streaming (state " + session.state() + "; is there an SD card?)"); + return started; + } + + /** + * The unit is reported ready only once its connect sequence has ended (DEV-1275), which + * NOTIFICATION_SHIMMER_FULLY_INITIALIZED marks: CONNECTED, or SDLOGGING for a unit that is logging + * to its SD card. The sequence reads the status part way through, before the calibration dump and + * the inquiry, and that reply used to report the unit ready early, so an app's next command went + * out in among the rest of the sequence. Connects to the unit idle, then again while it logs. + * Needs an SD card in the unit. + */ + @Test + public void connectReady() throws Exception { + begin("connect_ready"); + if (connectAndCheckReady("idle", BT_STATE.CONNECTED)) { + boolean logging = session.startSdLogging() + && session.waitFor(() -> session.state() == BT_STATE.SDLOGGING && session.queuedInstructions() == 0, 5000); + report.check(logging, "SD logging did not start (state " + session.state() + "; is there an SD card?)"); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + if (logging && session.hung == null) { + Thread.sleep(2000); + if (!connectAndCheckReady("logging", BT_STATE.SDLOGGING) && session.hung == null) { + // Leave the unit as it was found, not logging: connect again to stop it + session.disconnect(10000); + Thread.sleep(2000); + session.connect(30000, BT_STATE.CONNECTED, BT_STATE.SDLOGGING); + } + stopSdLoggingIfLogging(); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + + private void stopSdLoggingIfLogging() throws InterruptedException { + ShimmerBluetooth d = session.device(); + if (session.hung == null && d != null && d.isSDLogging()) { + report.check(session.stopSdLogging() && session.waitFor(() -> session.state() == BT_STATE.CONNECTED, 5000), + "SD logging did not stop, so the unit was left logging (state " + session.state() + ")"); + } + } + + /** + * Not a check: brings the unit back to idle after a run that left it logging to its SD card. It + * connects, stops SD logging if the unit logs, and disconnects. Run it on its own, with + * -Scenario resetUnit; it isn't part of a default run. + */ + @Test + public void resetUnit() throws Exception { + begin("reset_unit"); + boolean connected = session.connect(30000, BT_STATE.CONNECTED, BT_STATE.SDLOGGING); + report.check(connected, "did not connect within 30 s (state " + session.state() + ")"); + if (connected) { + session.event("connected: " + session.state()); + stopSdLoggingIfLogging(); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** Connects and checks the unit was first reported ready, in readyState, at the end of the connect sequence */ + private boolean connectAndCheckReady(String what, BT_STATE readyState) throws InterruptedException { + boolean ready = session.connect(30000, readyState) + && session.waitFor(() -> !Double.isNaN(session.fullyInitialisedAtMs()), 2000); + report.check(ready, what + ": did not connect, end in " + readyState + " and send FULLY_INITIALIZED within 30 s (state " + session.state() + ")"); + session.event(what + ": " + session.readyTimeline()); + report.put(what + "ReadyEarlyMs", session.readyEarlyMs()); + if (ready) { + report.check(session.readyEarlyMs() <= 250, String.format(Locale.ROOT, + "%s: reported ready %.0f ms before the connect sequence ended (%s)", what, session.readyEarlyMs(), session.readyTimeline())); + } + return ready; + } + + private static double[] toArray(List values) { + double[] out = new double[values.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = values.get(i); + } + return out; + } + + /** The rates samplingRates writes in turn, as the Java and shimmer-logandstream HILs' scenarios of the same name do */ + private static final double[] SAMPLING_RATES_HZ = {51.2, 256, 512}; + + /** + * Writes each rate in SAMPLING_RATES_HZ in turn, the way the examples configure, and streams 5 s + * at each; then writes the unit's own rate back. Every configure but the first comes straight + * after a stopStreaming(), as an app that changes the rate does. Up to driver 0.11.10_beta the + * stop's reply then retired the configuration's first chunk, the one holding the sampling rate, + * and the unit kept its old rate (DEV-1306). No SD card is needed. + */ + @Test + public void samplingRates() throws Exception { + begin("sampling_rates"); + double maxLoss = arg("maxLossPercent", 1.0); + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + ShimmerBluetooth d = connected ? session.device() : null; + if (d != null) { + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + double ownRate = d.getSamplingRateShimmer(); + report.put("ownRateHz", ownRate); + List streams = new ArrayList<>(); + for (double rate : SAMPLING_RATES_HZ) { + if (session.hung != null) { + break; + } + String what = String.format(Locale.ROOT, "%.1f Hz", rate); + if (!configureRate(rate, what)) { + continue; + } + double deviceRate = session.rateOnDevice(); + String marker = session.logMarker("sampling_rates " + what); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + streams.add(what + ": " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + int parserErrors = session.parserErrorsSince(marker); + report.check(parserErrors == 0, what + ": " + parserErrors + " streaming parser errors logged"); + } + report.put("streams", new org.json.JSONArray(streams)); + if (session.hung == null) { + configureRate(ownRate, "the unit's own rate"); + } + report.put("rateOnDeviceHz", session.rateOnDevice()); + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + finish(); + } + + /** + * Writes rateHz and checks the unit took it. The configure ends by reading the configuration + * back, so the API's rate is then the unit's. configureAccelOnly can finish on the manager's + * SUCCESS alone, without looking at the rate, so the rate is checked here. + */ + private boolean configureRate(double rateHz, String what) throws InterruptedException { + boolean configured = session.configureAccelOnly(rateHz, 20000); + double onDevice = session.rateOnDevice(); + report.check(configured, String.format(Locale.ROOT, "%s: the configuration did not complete within 20 s (rate on device %.2f Hz, state %s)", + what, onDevice, session.state())); + boolean took = Math.abs(onDevice - rateHz) < 0.5; + report.check(!configured || took, String.format(Locale.ROOT, + "%s: the unit kept %.2f Hz after %.2f Hz was written (straight after a stop, that is DEV-1306)", what, onDevice, rateHz)); + return configured && took; + } + + /** queueStress: the app threads, the most instructions they keep queued, and how long a stuck call or queue takes to count as one */ + private static final int QUEUE_STRESS_THREADS = 4; + private static final int QUEUE_STRESS_MAX_QUEUED = 8; + private static final long QUEUE_STRESS_STALL_MS = 10000; + + /** + * DEV-1266: the driver guards its instruction queue with two locks, the device and the queue. Its + * IO thread took them queue first when it fetched the next instruction, and every call that queues + * one takes them device first, so the two could deadlock and every later call hung (5 of 12 + * inStream runs on driver 0.11.7). With the unit connected and idle, timers on as in an app, four + * threads queue battery and status reads in tight loops for queueStressSeconds (default 60), + * keeping the queue under 8, so a call is nearly always taking the locks while the IO thread + * fetches. Idle, because while streaming Android writes both reads straight out, unqueued. No SD + * card is needed. + */ + @Test + public void queueStress() throws Exception { + begin("queue_stress"); + int seconds = arg("queueStressSeconds", 60); + long start = System.currentTimeMillis(); + boolean connected = session.connect(30000); + report.put("connectMs", System.currentTimeMillis() - start); + report.check(connected, "did not connect and initialise within 30 s (state " + session.state() + ")"); + final ShimmerBluetooth d = connected ? session.device() : null; + if (d != null) { + report.put("hardware", d.getHardwareVersionParsed()); + report.put("firmware", d.getFirmwareVersionParsed()); + report.put("crcMode", String.valueOf(d.getCurrentBtCommsCrcMode())); + final AtomicBoolean stop = new AtomicBoolean(); + final AtomicLong calls = new AtomicLong(); + // When each thread's current call began (elapsedRealtime), or 0 between calls + final AtomicLongArray inCallSince = new AtomicLongArray(QUEUE_STRESS_THREADS); + final AtomicReference thrown = new AtomicReference<>(); + List threads = new ArrayList<>(); + for (int i = 0; i < QUEUE_STRESS_THREADS; i++) { + final int k = i; + Thread t = new Thread(() -> { + try { + while (!stop.get()) { + if (session.queuedInstructions() >= QUEUE_STRESS_MAX_QUEUED) { + Thread.sleep(1); + continue; + } + inCallSince.set(k, SystemClock.elapsedRealtime()); + if ((calls.get() + k) % 2 == 0) { + d.readBattery(); + } else { + d.readStatusLogAndStream(); + } + inCallSince.set(k, 0); + calls.incrementAndGet(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable e) { + thrown.compareAndSet(null, e); + stop.set(true); + } + }, "QueueStress" + i); + t.setDaemon(true); // a deadlocked thread must not keep the test process alive + threads.add(t); + } + session.event("queue stress: " + QUEUE_STRESS_THREADS + " threads for " + seconds + " s"); + Object battery = session.batteryDetails(); + int batteryReplies = 0; + int maxQueued = 0; + long lastCalls = 0; + long lastProgress = SystemClock.elapsedRealtime(); + long end = lastProgress + seconds * 1000L; + String stall = null; + for (Thread t : threads) { + t.start(); + } + while (SystemClock.elapsedRealtime() < end && stall == null && !stop.get()) { + Thread.sleep(100); + long now = SystemClock.elapsedRealtime(); + Object b = session.batteryDetails(); + if (b != null && b != battery) { + battery = b; + batteryReplies++; + } + int queued = session.queuedInstructions(); + maxQueued = Math.max(maxQueued, queued); + long c = calls.get(); + if (c != lastCalls) { + lastCalls = c; + lastProgress = now; + } + for (int i = 0; i < QUEUE_STRESS_THREADS && stall == null; i++) { + long since = inCallSince.get(i); + if (since != 0 && now - since > QUEUE_STRESS_STALL_MS) { + stall = String.format(Locale.ROOT, "a read queued by QueueStress%d has not returned for %.0f s", i, (now - since) / 1000.0); + } + } + if (stall == null && now - lastProgress > QUEUE_STRESS_STALL_MS) { + stall = String.format(Locale.ROOT, "no read returned for %.0f s, with %d instructions queued", (now - lastProgress) / 1000.0, queued); + } + } + stop.set(true); + report.put("calls", calls.get()); + report.put("batteryReplies", batteryReplies); + report.put("maxQueued", maxQueued); + session.event(String.format(Locale.ROOT, "queue stress: %d reads queued, %d battery replies seen, at most %d instructions queued", + calls.get(), batteryReplies, maxQueued)); + if (stall != null) { + session.hung = "readBattery/readStatusLogAndStream (queueStress)"; + session.event(stall + "; thread dump follows"); + session.dumpThreads(); + report.check(false, stall + " (the calls and the IO thread are stuck; DEV-1266's deadlock if the dump shows them waiting on each other)"); + } else { + for (Thread t : threads) { + t.join(2000); + } + report.check(thrown.get() == null, "a read threw " + thrown.get()); + report.check(calls.get() > 0, "no read was queued"); + boolean drained = session.waitFor(() -> session.queuedInstructions() == 0, 15000); + report.check(drained, "the queue did not drain within 15 s after the reads stopped (" + + session.queuedInstructions() + " left; the IO thread is stuck, see the thread dump in the events)"); + if (!drained) { + session.event("the queue did not drain; thread dump follows"); + session.dumpThreads(); + } + report.check(session.disconnect(10000), "did not disconnect within 10 s"); + } + } + finish(); + } + + /** Connect, stream briefly and disconnect, repeatedly: the reconnect path apps hit every day. */ + @Test + public void reconnectLoop() throws Exception { + begin("reconnect_loop"); + double rate = arg("rate", 256.0); + int cycles = arg("cycles", 5); + double maxLoss = arg("maxLossPercent", 1.0); + List connectTimes = new ArrayList<>(); + for (int i = 1; i <= cycles; i++) { + String what = "cycle " + i; + long start = System.currentTimeMillis(); + boolean ready = i == 1 ? connectAndConfigure(rate) : session.connect(30000); + connectTimes.add(System.currentTimeMillis() - start); + report.check(ready, what + ": did not connect within 30 s (state " + session.state() + ")"); + if (!ready) { + break; + } + double deviceRate = session.rateOnDevice(); + checkRate(deviceRate, what); + String marker = session.logMarker(what); + session.startRecording(); + report.check(session.startStreaming(10000), what + ": streaming did not start within 10 s"); + Thread.sleep(5000); + double stoppedAtMs = session.nowMs(); + List samples = session.stopRecording(); + report.check(session.stopStreaming(10000), what + ": streaming did not stop within 10 s"); + StreamAnalysis a = StreamAnalysis.analyse(samples, deviceRate); + session.event(what + " stream: " + a); + checkStream(a, samples, stoppedAtMs, maxLoss, deviceRate, what); + int parserErrors = session.parserErrorsSince(marker); + report.check(parserErrors == 0, what + ": " + parserErrors + " streaming parser errors logged"); + report.check(session.disconnect(10000), what + ": did not disconnect within 10 s"); + Thread.sleep(2000); + } + report.put("connectMs", new org.json.JSONArray(connectTimes)); + finish(); + } +} diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java new file mode 100644 index 00000000..c6b66d9b --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/androidTest/java/com/shimmerresearch/hiltest/ShimmerSession.java @@ -0,0 +1,575 @@ +package com.shimmerresearch.hiltest; + +import android.app.Application; +import android.content.Context; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Message; +import android.os.Process; +import android.os.SystemClock; +import android.util.Log; + +import androidx.test.platform.app.InstrumentationRegistry; + +import com.clj.fastble.BleManager; +import com.shimmerresearch.android.manager.ShimmerBluetoothManagerAndroid; +import com.shimmerresearch.bluetooth.BluetoothProgressReportPerDevice; +import com.shimmerresearch.bluetooth.ShimmerBluetooth; +import com.shimmerresearch.bluetooth.ShimmerBluetooth.BT_STATE; +import com.shimmerresearch.driver.CallbackObject; +import com.shimmerresearch.driver.Configuration; +import com.shimmerresearch.driver.FormatCluster; +import com.shimmerresearch.driver.ObjectCluster; +import com.shimmerresearch.driver.ShimmerDevice; +import com.shimmerresearch.driverUtilities.AssembleShimmerConfig; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; + +/** + * One Shimmer, driven through the public Android API the way an app would: the Bluetooth manager, + * a Handler for callbacks, and the ShimmerBluetooth calls. Records state changes and, while + * recording, every data packet. + */ +final class ShimmerSession implements AutoCloseable { + + static final String TAG = "ShimmerHil"; + + /** What the API logs (Log.d, tag "Shimmer") when its streaming parser loses its place. */ + static final String[] PARSER_ERRORS = {"Packet syncing problem", "Unknown parsing error while streaming"}; + + interface Condition { + boolean holds(); + } + + final String mac; + /** "classic" (Android's own Shimmer class and parser) or "ble" (Shimmer3BLEAndroid, the driver's parser). */ + final String transport; + /** "Shimmer3" or "Shimmer3R": over BLE the manager picks the device class from it. */ + final String deviceType; + private final HandlerThread callbackThread = new HandlerThread("ShimmerHilCallbacks"); + private final ShimmerBluetoothManagerAndroid manager; + private final long t0 = SystemClock.elapsedRealtime(); + private final List events = new ArrayList<>(); + private final List samples = new ArrayList<>(); + private volatile boolean recording; + private volatile BT_STATE lastState; + private volatile BluetoothProgressReportPerDevice.OperationState configureResult; + private volatile int stateMessages; + private final java.util.Set otherMessagesSeen = new java.util.HashSet<>(); + + /** + * Since the last connect: when the app was first told the unit is ready (a CONNECTED or + * SDLOGGING state message), when the API's state first showed it (polled), and when the API sent + * NOTIFICATION_SHIMMER_FULLY_INITIALIZED, which marks the end of the connect sequence. NaN until + * then. + */ + private volatile double readyMessageAtMs = Double.NaN; + private volatile double readyStateAtMs = Double.NaN; + private volatile double fullyInitialisedAtMs = Double.NaN; + private volatile BT_STATE readyMessageState; + private volatile int queuedAtReadyState = -1; + + ShimmerSession(Context context, String mac, String transport, String deviceType) throws Exception { + this.mac = mac; + this.transport = transport; + this.deviceType = deviceType; + callbackThread.start(); + Handler handler = new Handler(callbackThread.getLooper()) { + @Override + public void handleMessage(Message msg) { + onMessage(msg); + } + }; + // On the main thread, as an app does: the manager creates a Handler on whatever thread + // constructs it, which fails on the instrumentation thread (no Looper). + final ShimmerBluetoothManagerAndroid[] created = new ShimmerBluetoothManagerAndroid[1]; + final Exception[] failure = new Exception[1]; + InstrumentationRegistry.getInstrumentation().runOnMainSync(() -> { + try { + BleManager.getInstance().init((Application) context.getApplicationContext()); + created[0] = new ShimmerBluetoothManagerAndroid(context, handler); + } catch (Exception e) { + failure[0] = e; + } + }); + if (failure[0] != null) { + callbackThread.quitSafely(); + throw failure[0]; + } + manager = created[0]; + } + + double nowMs() { + return SystemClock.elapsedRealtime() - t0; + } + + void event(String text) { + String line = String.format(Locale.ROOT, "%9.0f %s", nowMs(), text); + synchronized (events) { + events.add(line); + } + Log.i(TAG, mac + " " + line); + } + + List events() { + synchronized (events) { + return new ArrayList<>(events); + } + } + + private void onMessage(Message msg) { + if (msg.what == ShimmerBluetooth.MSG_IDENTIFIER_STATE_CHANGE) { + BT_STATE state = null; + if (msg.obj instanceof ObjectCluster) { + state = ((ObjectCluster) msg.obj).mState; + } else if (msg.obj instanceof CallbackObject) { + state = ((CallbackObject) msg.obj).mState; + } + if (state != null) { + lastState = state; + stateMessages++; + event("state " + state); + if (isReady(state) && Double.isNaN(readyMessageAtMs)) { + readyMessageAtMs = nowMs(); + readyMessageState = state; + } + } + } else if (msg.what == ShimmerBluetooth.MSG_IDENTIFIER_NOTIFICATION_MESSAGE && msg.obj instanceof CallbackObject + && ((CallbackObject) msg.obj).mIndicator == ShimmerBluetooth.NOTIFICATION_SHIMMER_FULLY_INITIALIZED) { + if (Double.isNaN(fullyInitialisedAtMs)) { + fullyInitialisedAtMs = nowMs(); + } + event("fully initialised"); + } else if (msg.what == ShimmerBluetooth.MSG_IDENTIFIER_PROGRESS_REPORT_PER_DEVICE + && msg.obj instanceof BluetoothProgressReportPerDevice) { + // The manager's own verdict on an operation such as configureShimmer. + BluetoothProgressReportPerDevice p = (BluetoothProgressReportPerDevice) msg.obj; + if (p.mCurrentOperationBtState == BT_STATE.CONFIGURING + && p.mOperationState != BluetoothProgressReportPerDevice.OperationState.PENDING + && p.mOperationState != BluetoothProgressReportPerDevice.OperationState.INPROGRESS) { + configureResult = p.mOperationState; + event("configure progress: " + p.mOperationState); + } + } else if (msg.what == ShimmerBluetooth.MSG_IDENTIFIER_DATA_PACKET) { + if (recording && msg.obj instanceof ObjectCluster) { + recordPacket((ObjectCluster) msg.obj); + } + } else if (otherMessagesSeen.add(msg.what)) { + event("message " + msg.what + (msg.obj == null ? "" : " " + msg.obj.getClass().getSimpleName())); + } + } + + private void recordPacket(ObjectCluster oc) { + StreamAnalysis.Sample s = new StreamAnalysis.Sample(nowMs(), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.TIMESTAMP), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_X), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Y), + cal(oc, Configuration.Shimmer3.ObjectClusterSensorName.ACCEL_LN_Z)); + synchronized (samples) { + samples.add(s); + } + } + + private static double cal(ObjectCluster oc, String channel) { + Collection formats = oc.getCollectionOfFormatClusters(channel); + FormatCluster c = (FormatCluster) ObjectCluster.returnFormatCluster(formats, "CAL"); + return c == null ? Double.NaN : c.mData; + } + + ShimmerBluetooth device() { + return (ShimmerBluetooth) manager.getShimmer(mac); + } + + /** + * The device, or an exception that the call reports. The API discards its device object when the + * link drops, so a lost connection shows up here rather than as a NullPointerException. + */ + ShimmerBluetooth requireDevice() { + ShimmerBluetooth d = device(); + if (d == null) { + throw new IllegalStateException("the API no longer has the unit (connection lost; last state " + lastState + ")"); + } + return d; + } + + /** The rate the API holds for the unit, or NaN once the connection is gone. */ + double rateOnDevice() { + ShimmerBluetooth d = device(); + return d == null ? Double.NaN : d.getSamplingRateShimmer(); + } + + boolean streaming() { + ShimmerBluetooth d = device(); + return d != null && d.isStreaming(); + } + + BT_STATE state() { + ShimmerBluetooth d = device(); + return d == null ? lastState : d.getBluetoothRadioState(); + } + + boolean waitFor(Condition condition, long timeoutMs) throws InterruptedException { + long deadline = SystemClock.elapsedRealtime() + timeoutMs; + while (SystemClock.elapsedRealtime() < deadline) { + if (condition.holds()) { + return true; + } + Thread.sleep(50); + } + return condition.holds(); + } + + interface Call { + void run() throws Exception; + } + + /** Set to the first library call that never returned; nothing more is attempted after it. */ + volatile String hung; + + private final java.util.concurrent.ExecutorService calls = java.util.concurrent.Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r, "ShimmerHilCall"); + t.setDaemon(true); + return t; + }); + + /** + * Runs one API call with a time limit, so a call that blocks cannot hang the run. A call that + * never returns is recorded with every thread's stack (the evidence a phone without root cannot + * otherwise give) and returns false; so does one that throws. + */ + boolean call(String what, long timeoutMs, Call call) { + if (hung != null) { + event("skipped " + what + ": " + hung + " never returned"); + return false; + } + event(what); + java.util.concurrent.Future f = calls.submit(() -> { + call.run(); + return null; + }); + try { + f.get(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + return true; + } catch (java.util.concurrent.TimeoutException e) { + hung = what; + event(what + " did not return within " + timeoutMs + " ms; thread dump follows"); + dumpThreads(); + return false; + } catch (java.util.concurrent.ExecutionException e) { + event(what + " threw " + e.getCause()); + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + void dumpThreads() { + for (java.util.Map.Entry t : Thread.getAllStackTraces().entrySet()) { + StringBuilder b = new StringBuilder(" thread \"" + t.getKey().getName() + "\" " + t.getKey().getState()); + for (StackTraceElement frame : t.getValue()) { + b.append("\n at ").append(frame); + } + event(b.toString()); + } + } + + /** + * Connects over the session's transport and waits until the API has read the device's + * configuration. Classic needs the unit paired with the phone; BLE does not. + */ + boolean connect(long timeoutMs) throws InterruptedException { + return connect(timeoutMs, BT_STATE.CONNECTED); + } + + /** + * As connect(long), for a unit that ends its connect in one of readyStates: SDLOGGING for one that + * is logging to its SD card. It records when the unit was first reported ready (readyEarlyMs()). + */ + boolean connect(long timeoutMs, final BT_STATE... readyStates) throws InterruptedException { + final java.util.List accepted = java.util.Arrays.asList(readyStates); + readyMessageAtMs = Double.NaN; + readyStateAtMs = Double.NaN; + fullyInitialisedAtMs = Double.NaN; + readyMessageState = null; + queuedAtReadyState = -1; + final boolean ble = "ble".equals(transport); + if (!call("connect over " + transport, 10000, () -> manager.connectShimmerThroughBTAddress(mac, deviceType, + ble ? ShimmerBluetoothManagerAndroid.BT_TYPE.BLE : ShimmerBluetoothManagerAndroid.BT_TYPE.BT_CLASSIC))) { + return false; + } + boolean ready = waitFor(() -> { + ShimmerBluetooth d = device(); + if (d != null && isReady(d.getBluetoothRadioState()) && Double.isNaN(readyStateAtMs)) { + readyStateAtMs = nowMs(); + queuedAtReadyState = queuedInstructions(); + event("state first shows " + d.getBluetoothRadioState() + " with " + queuedAtReadyState + " instructions queued"); + } + return d != null && d.isInitialised() && accepted.contains(d.getBluetoothRadioState()); + }, timeoutMs); + if (!ready) { + // The connect call returned, so nothing else dumps the threads: a stalled connect sequence + // (such as the DEV-1198 lock-order deadlock) shows here + event("connect did not finish within " + timeoutMs + " ms (state " + state() + "); thread dump follows"); + dumpThreads(); + } + ShimmerBluetooth d = device(); + if (ready && ble && d != null) { + // Over BLE the driver logs parser errors with consolePrintLn, which only prints in verbose + // mode; Android's Classic class logs them always. Same evidence either way. + d.setVerboseMode(true); + } + return ready && d != null; + } + + private static boolean isReady(BT_STATE state) { + return state == BT_STATE.CONNECTED || state == BT_STATE.SDLOGGING; + } + + /** When the last connect sent NOTIFICATION_SHIMMER_FULLY_INITIALIZED, or NaN if it has not */ + double fullyInitialisedAtMs() { + return fullyInitialisedAtMs; + } + + /** + * How long before NOTIFICATION_SHIMMER_FULLY_INITIALIZED the last connect first reported the unit + * ready, by state message or in the API's state, whichever came first. NaN if either is missing. + */ + double readyEarlyMs() { + double first = Double.isNaN(readyMessageAtMs) ? readyStateAtMs + : Double.isNaN(readyStateAtMs) ? readyMessageAtMs : Math.min(readyMessageAtMs, readyStateAtMs); + return fullyInitialisedAtMs - first; + } + + /** What readyEarlyMs() is made of, for the report */ + String readyTimeline() { + return String.format(Locale.ROOT, "first ready message %s at %.0f ms, state first ready at %.0f ms with %d instructions queued, fully initialised at %.0f ms", + readyMessageState, readyMessageAtMs, readyStateAtMs, queuedAtReadyState, fullyInitialisedAtMs); + } + + /** + * The API's instruction queue length, or -1 without a device. The queue is protected, so an app + * can't see it; the test reads the field by reflection. Not through getListofInstructions(), which + * takes the device's lock: the IO thread takes the queue's lock and then the device's + * (DEV-1198), so holding the device's lock from here would widen that deadlock's window. + */ + int queuedInstructions() { + ShimmerBluetooth d = device(); + if (d == null) { + return -1; + } + try { + java.lang.reflect.Field f = ShimmerBluetooth.class.getDeclaredField("mListofInstructions"); + f.setAccessible(true); + return ((List) f.get(d)).size(); + } catch (Exception e) { + event("could not read the instruction queue: " + e); + return -1; + } + } + + /** + * Accel only (the low-noise accel on Shimmer3, the LSM6DSV's on Shimmer3R) at rateHz, written + * the way the examples do: a clone, AssembleShimmerConfig, then configureShimmer. + */ + boolean configureAccelOnly(double rateHz, long timeoutMs) throws InterruptedException { + ShimmerDevice original = manager.getShimmer(mac); + if (original == null) { + event("configure: the API no longer has the unit (connection lost; last state " + lastState + ")"); + return false; + } + ShimmerDevice clone = original.deepClone(); + clone.disableAllSensors(); + if (clone.getHardwareVersion() == ShimmerVerDetails.HW_ID.SHIMMER_3) { + clone.setSensorEnabledState(Configuration.Shimmer3.SENSOR_ID.SHIMMER_ANALOG_ACCEL, true); + } else { + clone.setSensorEnabledState(Configuration.Shimmer3.SENSOR_ID.SHIMMER_LSM6DSV_ACCEL_LN, true); + } + clone.setShimmerAndSensorsSamplingRate(rateHz); + AssembleShimmerConfig.generateSingleShimmerConfig(clone, Configuration.COMMUNICATION_TYPE.BLUETOOTH); + configureResult = null; + final int stateMessagesBefore = stateMessages; + if (!call(String.format(Locale.ROOT, "configure accel only at %.1f Hz", rateHz), 10000, + () -> manager.configureShimmer(clone))) { + return false; + } + // Done when the operation has finished: either the manager's progress report (not every + // version forwards it to the Handler) or the fresh "Ready" the unit posts when the operation + // ends, with the requested rate read back. The status timer is stopped during an operation, + // so that "Ready" comes from the configuration. + boolean done = waitFor(() -> state() == BT_STATE.CONNECTED + && (configureResult != null + || (stateMessages > stateMessagesBefore && Math.abs(rateOnDevice() - rateHz) < 0.5)), + timeoutMs); + event(String.format(Locale.ROOT, "configure %s (%s): rate on device %.2f Hz", + done ? "finished" : "timed out", configureResult != null ? configureResult : "no progress report", + rateOnDevice())); + return done && (configureResult == null || configureResult == BluetoothProgressReportPerDevice.OperationState.SUCCESS); + } + + void startRecording() { + synchronized (samples) { + samples.clear(); + } + recording = true; + } + + List stopRecording() { + recording = false; + synchronized (samples) { + return new ArrayList<>(samples); + } + } + + int sampleCount() { + synchronized (samples) { + return samples.size(); + } + } + + boolean startStreaming(long timeoutMs) throws InterruptedException { + return call("startStreaming", 10000, () -> requireDevice().startStreaming()) + && waitFor(this::streaming, timeoutMs); + } + + boolean stopStreaming(long timeoutMs) throws InterruptedException { + return call("stopStreaming", 10000, () -> requireDevice().stopStreaming()) + && waitFor(() -> !streaming(), timeoutMs); + } + + /** Returns once the call has returned; the caller waits for the state it expects. */ + boolean startSdLogging() { + return call("startSDLogging", 10000, () -> requireDevice().startSDLogging()); + } + + boolean stopSdLogging() { + return call("stopSDLogging", 10000, () -> requireDevice().stopSDLogging()); + } + + boolean readBattery() { + return call("readBattery", 10000, () -> requireDevice().readBattery()); + } + + boolean readStatus() { + return call("readStatusLogAndStream", 10000, () -> requireDevice().readStatusLogAndStream()); + } + + /** + * The API's current battery details. Both the Classic and the BLE parser replace the object with + * a new one for each battery reply they decode, so a different object means a reply arrived. + */ + Object batteryDetails() { + ShimmerBluetooth d = device(); + return d == null ? null : d.getBattStatusDetails(); + } + + boolean disconnect(long timeoutMs) throws InterruptedException { + final ShimmerBluetooth d = device(); + if (d != null && !call("disconnect", 10000, d::disconnect)) { + return false; + } + return waitFor(() -> { + BT_STATE s = state(); + return s == null || s == BT_STATE.DISCONNECTED || s == BT_STATE.CONNECTION_LOST; + }, timeoutMs); + } + + /** Writes a marker to this process's log; parserErrorsSince(marker) counts lines after it. */ + String logMarker(String label) { + String marker = "HIL-MARK " + label + " " + SystemClock.elapsedRealtimeNanos(); + Log.i(TAG, marker); + return marker; + } + + /** Parser errors this process logged after the marker. Apps may read their own process's log. */ + int parserErrorsSince(String marker) { + return logLinesSince(marker, PARSER_ERRORS); + } + + /** Lines this process logged after the marker that contain any of the texts, or -1 if the log can't be read */ + int logLinesSince(String marker, String... texts) { + int count = 0; + try { + java.lang.Process p = Runtime.getRuntime().exec(new String[] {"logcat", "-d", "--pid=" + Process.myPid()}); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + boolean afterMarker = false; + for (String line; (line = r.readLine()) != null; ) { + if (line.contains(marker)) { + afterMarker = true; + } else if (afterMarker) { + for (String e : texts) { + if (line.contains(e)) { + count++; + } + } + } + } + p.waitFor(); + } catch (Exception e) { + event("could not read logcat: " + e); + return -1; + } + return count; + } + + /** + * For each set of texts, the lines this process logged after the marker that contain any of them, + * or null if they can't be counted. Unlike logLinesSince(), that includes a log that no longer + * holds the marker: the phone's log buffer is small, and with the Bluetooth snoop log on it can + * roll past a marker within seconds, which would read as no lines at all. + */ + int[] logCountsSince(String marker, String[]... textSets) { + int[] counts = new int[textSets.length]; + boolean afterMarker = false; + try { + java.lang.Process p = Runtime.getRuntime().exec(new String[] {"logcat", "-d", "--pid=" + Process.myPid()}); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + for (String line; (line = r.readLine()) != null; ) { + if (line.contains(marker)) { + afterMarker = true; + } else if (afterMarker) { + for (int k = 0; k < textSets.length; k++) { + for (String e : textSets[k]) { + if (line.contains(e)) { + counts[k]++; + } + } + } + } + } + p.waitFor(); + } catch (Exception e) { + event("could not read logcat: " + e); + return null; + } + if (!afterMarker) { + event("the phone's log no longer holds \"" + marker + "\", so the lines after it can't be counted"); + return null; + } + return counts; + } + + @Override + public void close() throws Exception { + try { + // After a hung call the API is in an unknown state; leave the unit for the runner's + // force-stop rather than queue more calls behind the stuck one. + BT_STATE s = state(); + if (hung == null && s != null && s != BT_STATE.DISCONNECTED && s != BT_STATE.CONNECTION_LOST) { + if (streaming()) { + stopStreaming(5000); + } + disconnect(10000); + } + } finally { + callbackThread.quitSafely(); + calls.shutdownNow(); + } + } +} diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/main/AndroidManifest.xml b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/main/AndroidManifest.xml new file mode 100644 index 00000000..1ebfe4a9 --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/main/AndroidManifest.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/main/java/com/shimmerresearch/hiltest/StreamAnalysis.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/main/java/com/shimmerresearch/hiltest/StreamAnalysis.java new file mode 100644 index 00000000..d38c1dd2 --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/main/java/com/shimmerresearch/hiltest/StreamAnalysis.java @@ -0,0 +1,193 @@ +package com.shimmerresearch.hiltest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** + * Judges a recorded stream from the device timestamps alone. The checks mirror the firmware HIL + * harness in log-and-stream-common Test/hil (DEV-1165): packet loss and non-increasing timestamps + * from the driver's already-unwrapped timestamps, the effective sample rate, and accel at rest. + * Pure Java, so it is unit-tested on the JVM (src/test) as well as used on the phone. + */ +public final class StreamAnalysis { + + /** One received packet. Times in ms; accel in m/s^2, NaN when the channel is absent. */ + public static final class Sample { + public final double receivedMs; + public final double timestampMs; + public final double accelX, accelY, accelZ; + + public Sample(double receivedMs, double timestampMs, double accelX, double accelY, double accelZ) { + this.receivedMs = receivedMs; + this.timestampMs = timestampMs; + this.accelX = accelX; + this.accelY = accelY; + this.accelZ = accelZ; + } + + double accelMagnitude() { + return Math.sqrt(accelX * accelX + accelY * accelY + accelZ * accelZ); + } + } + + /** A run of packets missing between two received ones. */ + public static final class Gap { + public final double afterTimestampMs; + public final double afterReceivedMs; + public final int missing; + + Gap(double afterTimestampMs, double afterReceivedMs, int missing) { + this.afterTimestampMs = afterTimestampMs; + this.afterReceivedMs = afterReceivedMs; + this.missing = missing; + } + } + + public final int received; + public final int missing; + public final int nonIncreasing; + /** + * Steps where the device timestamp leapt far ahead of the arrival time: not lost packets but a + * bad timestamp, such as a 24-bit wrap unwrapped wrongly (512 s at a 32768 Hz clock, DEV-1023). + */ + public final List timestampJumps; + public final double expectedPeriodMs; + public final double effectiveRateHz; + public final double accelMagnitudeMean; + public final double accelMagnitudeStd; + public final List gaps; + + private StreamAnalysis(int received, int missing, int nonIncreasing, List timestampJumps, double expectedPeriodMs, + double effectiveRateHz, double accelMagnitudeMean, double accelMagnitudeStd, List gaps) { + this.received = received; + this.missing = missing; + this.nonIncreasing = nonIncreasing; + this.timestampJumps = timestampJumps; + this.expectedPeriodMs = expectedPeriodMs; + this.effectiveRateHz = effectiveRateHz; + this.accelMagnitudeMean = accelMagnitudeMean; + this.accelMagnitudeStd = accelMagnitudeStd; + this.gaps = gaps; + } + + /** + * @param samples in arrival order + * @param rateHz the sampling rate the device was configured with + */ + public static StreamAnalysis analyse(List samples, double rateHz) { + double period = 1000.0 / rateHz; + int missing = 0; + int nonIncreasing = 0; + // The effective rate counts sample periods over device time, from ordinary steps only: a + // jump or a step backwards says nothing about the rate. + double periods = 0, spanMs = 0; + List gaps = new ArrayList<>(); + List jumps = new ArrayList<>(); + for (int i = 1; i < samples.size(); i++) { + double step = samples.get(i).timestampMs - samples.get(i - 1).timestampMs; + double arrivalStep = samples.get(i).receivedMs - samples.get(i - 1).receivedMs; + if (step <= 0) { + nonIncreasing++; + } else if (step > 1.5 * period) { + int lost = (int) Math.round(step / period) - 1; + Gap g = new Gap(samples.get(i - 1).timestampMs, samples.get(i - 1).receivedMs, lost); + // Packets really lost leave a pause in arrival about as long as the gap. A timestamp that + // leaps a second or more beyond the arrival time is a bad timestamp, not lost data. + if (step - arrivalStep > JUMP_MS) { + jumps.add(g); + } else { + missing += lost; + gaps.add(g); + periods += lost + 1; + spanMs += step; + } + } else { + periods += 1; + spanMs += step; + } + } + double effectiveRate = spanMs > 0 ? periods * 1000.0 / spanMs : Double.NaN; + double sum = 0, sumSq = 0; + int n = 0; + for (Sample s : samples) { + double m = s.accelMagnitude(); + if (!Double.isNaN(m)) { + sum += m; + sumSq += m * m; + n++; + } + } + double mean = n > 0 ? sum / n : Double.NaN; + double std = n > 1 ? Math.sqrt(Math.max(0, (sumSq - n * mean * mean) / (n - 1))) : Double.NaN; + return new StreamAnalysis(samples.size(), missing, nonIncreasing, jumps, period, effectiveRate, mean, std, gaps); + } + + /** How far ahead of the arrival time a timestamp must leap to count as a jump, not loss. */ + static final double JUMP_MS = 1000; + + /** + * Charges each gap to the kind of command nearest to it (by arrival time), if one is within + * windowMs. Commands that follow each other closely therefore never share a loss. + * + * @param momentsByKind moments (ms) of each kind of command + * @return packets lost per kind, in the same order + */ + public int[] attribute(double[][] momentsByKind, double windowMs) { + int[] lost = new int[momentsByKind.length]; + for (Gap g : gaps) { + int nearestKind = -1; + double nearest = Double.MAX_VALUE; + for (int k = 0; k < momentsByKind.length; k++) { + for (double t : momentsByKind[k]) { + double d = Math.abs(g.afterReceivedMs - t); + if (d < nearest) { + nearest = d; + nearestKind = k; + } + } + } + if (nearestKind >= 0 && nearest <= windowMs) { + lost[nearestKind] += g.missing; + } + } + return lost; + } + + /** Packets lost within windowMs (by arrival time) of any of the given moments. */ + public int missingNear(double[] momentsMs, double windowMs) { + int total = 0; + for (Gap g : gaps) { + for (double t : momentsMs) { + if (Math.abs(g.afterReceivedMs - t) <= windowMs) { + total += g.missing; + break; + } + } + } + return total; + } + + public double lossPercent() { + int expected = received + missing; + return expected == 0 ? 0 : 100.0 * missing / expected; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, + "received=%d missing=%d (%.3f%%) nonIncreasing=%d timestampJumps=%d effectiveRate=%.2fHz accel=%.3f±%.3f m/s^2 gaps=%s", + received, missing, lossPercent(), nonIncreasing, timestampJumps.size(), effectiveRateHz, + accelMagnitudeMean, accelMagnitudeStd, gapsSummary()); + } + + private String gapsSummary() { + int shown = Math.min(gaps.size(), 5); + String[] parts = new String[shown]; + for (int i = 0; i < shown; i++) { + parts[i] = String.format(Locale.ROOT, "%d@%.0fms", gaps.get(i).missing, gaps.get(i).afterReceivedMs); + } + return Arrays.toString(parts) + (gaps.size() > shown ? "+" + (gaps.size() - shown) : ""); + } +} diff --git a/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/test/java/com/shimmerresearch/hiltest/StreamAnalysisTest.java b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/test/java/com/shimmerresearch/hiltest/StreamAnalysisTest.java new file mode 100644 index 00000000..02126b7f --- /dev/null +++ b/ShimmerAndroidInstrumentDriver/shimmerHilTest/src/test/java/com/shimmerresearch/hiltest/StreamAnalysisTest.java @@ -0,0 +1,102 @@ +package com.shimmerresearch.hiltest; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +/** The HIL verdicts are only as good as this arithmetic, so it is checked without a phone. */ +public class StreamAnalysisTest { + + private static List stream(double rateHz, int count, int... droppedIndices) { + List samples = new ArrayList<>(); + double period = 1000.0 / rateHz; + outer: + for (int i = 0; i < count; i++) { + for (int d : droppedIndices) { + if (d == i) { + continue outer; + } + } + samples.add(new StreamAnalysis.Sample(i * period, 1000 + i * period, 0, 0, 9.81)); + } + return samples; + } + + @Test + public void cleanStreamHasNoLoss() { + StreamAnalysis a = StreamAnalysis.analyse(stream(256, 2560), 256); + assertEquals(2560, a.received); + assertEquals(0, a.missing); + assertEquals(0, a.nonIncreasing); + assertEquals(256, a.effectiveRateHz, 0.01); + assertEquals(9.81, a.accelMagnitudeMean, 1e-9); + } + + @Test + public void countsEachDroppedPacket() { + StreamAnalysis a = StreamAnalysis.analyse(stream(256, 1000, 100, 500, 501, 502), 256); + assertEquals(996, a.received); + assertEquals(4, a.missing); + assertEquals(2, a.gaps.size()); + assertEquals(256, a.effectiveRateHz, 0.01); + } + + @Test + public void attributesLossToTheMomentItHappened() { + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + // Packet 300 is missing; packet 299 arrived at 2990 ms. + assertEquals(1, a.missingNear(new double[] {3000}, 500)); + assertEquals(0, a.missingNear(new double[] {9000}, 500)); + } + + @Test + public void chargesEachGapToTheNearestCommandOnly() { + // 100 Hz; packet 300 (arrives ~3000 ms) is missing. An SD stop at 2900 ms and a battery read + // at 3920 ms are both within 1 s of it: the SD stop is nearer, so it alone is charged. + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + int[] lost = a.attribute(new double[][] {{2900}, {3920}}, 1000); + assertEquals(1, lost[0]); + assertEquals(0, lost[1]); + } + + @Test + public void chargesNothingOutsideTheWindow() { + StreamAnalysis a = StreamAnalysis.analyse(stream(100, 1000, 300), 100); + int[] lost = a.attribute(new double[][] {{5000}, {8000}}, 1000); + assertEquals(0, lost[0] + lost[1]); + } + + @Test + public void tellsABadTimestampFromLostPackets() { + // What a Shimmer3R did over BLE at an SD logging start: one packet stamped a 24-bit wrap + // (512 s) ahead while packets kept arriving on time, then the next one back on track. + List s = stream(256, 1000); + StreamAnalysis.Sample bad = s.get(500); + s.set(500, new StreamAnalysis.Sample(bad.receivedMs, bad.timestampMs + 512000, 0, 0, 9.81)); + StreamAnalysis a = StreamAnalysis.analyse(s, 256); + assertEquals(1, a.timestampJumps.size()); + assertEquals(0, a.missing); + assertEquals(1, a.nonIncreasing); + assertEquals(256, a.effectiveRateHz, 0.5); + } + + @Test + public void flagsTimestampsThatGoBackwards() { + List s = stream(100, 10); + s.add(5, new StreamAnalysis.Sample(45, 1000, 0, 0, 9.81)); + StreamAnalysis a = StreamAnalysis.analyse(s, 100); + assertEquals(1, a.nonIncreasing); + } + + @Test + public void toleratesJitterBelowHalfAPeriod() { + List s = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + s.add(new StreamAnalysis.Sample(i * 10, 1000 + i * 10 + (i % 2 == 0 ? 0 : 4), 0, 0, 9.81)); + } + assertEquals(0, StreamAnalysis.analyse(s, 100).missing); + } +} diff --git a/tools/hil/README.md b/tools/hil/README.md new file mode 100644 index 00000000..13356075 --- /dev/null +++ b/tools/hil/README.md @@ -0,0 +1,211 @@ +# Android HIL: the Android API on real Shimmers + +Hardware-in-the-loop tests of the Android API. A phone on USB connects to real Shimmer3 and Shimmer3R +units, over Bluetooth Classic, BLE or both, through the same public API an app uses +(`ShimmerBluetoothManagerAndroid`, `ShimmerBluetooth`). One command builds the tests, installs them, +runs every scenario on every unit and transport, pulls the reports and prints pass or fail. + +**Both transports matter, because on Android they are different code.** + +| Transport | Device class | Parser | Pairing | +|---|---|---|---| +| Classic | the Android API's own `Shimmer` | the API's own copy, overriding the driver's | needed once | +| BLE | `Shimmer3BLEAndroid` | the Java driver's | none | + +A defect in one transport says nothing about the other. DEV-1148 is Classic only, for example: the +driver has had its fix, DEV-1147, since 0.11.9_beta. + +```powershell +.\tools\hil\run-android-hil.ps1 +``` + +It exits 0 when everything passed, 1 when anything failed, errored, timed out or was skipped, and 2 when the bench isn't set up. + +## Before every Android release (the gate) + +**Release Android API** won't publish a commit that hasn't passed this on hardware. Each round: + +1. Release the Java API, if the driver changed. +2. On the bench, on `master` with a clean working tree and every unit in the rig, run: + ```powershell + git checkout master; git pull + .\tools\hil\run-android-hil.ps1 -Report + ``` + With `-Report`, the result is posted to GitHub as the commit status + **`android-hil (driver X)`** on that commit: passed or failed, with the phone and each unit's + firmware. It refuses to report a partial run or a run of uncommitted code: + - every scenario, at least one Shimmer3 and one Shimmer3R, and both transports, Classic and BLE; + - the default rate, duration, link CRC and loss limit, and a fresh build (no `-SkipBuild`); + - this checkout's library source; + - a clean working tree, pushed. +3. Run **Release Android API** with the same driver (blank picks the newest, as the script does). + It looks up that status for the commit it's releasing and the driver it builds against, and + stops before building anything unless it passed. The release notes then say what was tested. + +If anything merges after step 2, the commit changes, so test again. In an emergency, put a Jira key +in **hil_waiver** to release without a pass; the release notes record the waiver. Dry runs only warn. + +To find regressions before release day, schedule step 2 nightly with Windows Task Scheduler, without +`-Report`. + +## What it tests + +| Scenario | What it does | Fails when | +|---|---|---| +| `streamRest` | Connects, writes an accel-only config at `-Rate` (default 256 Hz), streams for `-Seconds` (default 30) with the unit lying still, then stops and disconnects. | It can't connect or configure; more than `-MaxLossPercent` (1 %) of packets are lost; a timestamp doesn't increase, or leaps more than a second ahead of when it arrived (a bad unwrap, not loss); the timestamps give no rate, or one more than 1 % off the configured rate; the stream goes silent for over a second before the test stops it; accel is more than 1 m/s² from 9.81; or the API logs a "Packet syncing problem". | +| `midStreamSet` | While streaming, starts and stops SD logging and reads the battery, three times. | One of those calls fails, or a battery reply isn't decoded within 5 s; logging doesn't start or stop (the DEV-1148 symptom with a link CRC on); a packet is lost around a battery read, or more than `-MaxLossPercent` away from the SD transitions; the timestamps fail the same rate and silence checks as `streamRest`; or there's a parser error. Packets missing around SD logging start/stop are reported, not failed: the unit itself pauses while it opens its file, which is for the firmware HIL bench to judge. Needs an SD card in the unit. | +| `reconnectLoop` | Connects, streams for 5 s and disconnects, five times. | Any cycle fails to connect within 30 s, loses packets, fails the rate and silence checks, or logs a parser error. | +| `inStream` | While streaming, three times: starts SD logging, stops it, and reads the battery and the status every 40 ms for 400 ms, so some reads land in the ~100 ms the unit pauses its data packets while it closes its file (DEV-1291). Then, three times, stops and starts logging back to back, so the queued start goes out in that pause. | A battery read isn't answered, or fewer status replies than reads are decoded; logging doesn't stop or restart, or a command stays queued; a command times out; there's a parser error; more than `-MaxLossPercent` of packets are lost away from the SD transitions and the reads; or the timestamps fail the same rate and silence checks as `streamRest`. Loss near the reads is reported, not failed: a Shimmer3 now and then skips a sample around a battery read itself. The report's `repliesInAPause` counts the replies that came with no data packet before them. Needs an SD card in the unit. | +| `connectReady` | Connects to the unit idle, starts SD logging, disconnects and connects again while it logs, then stops logging. | The unit is reported ready (a CONNECTED or SDLOGGING state message, or that state in the API) more than 250 ms before `NOTIFICATION_SHIMMER_FULLY_INITIALIZED`, which marks the end of the connect sequence (DEV-1275); or the connect doesn't end in CONNECTED, or SDLOGGING while logging. Needs an SD card in the unit. | +| `queueStress` | Connected and idle, with the API's timers on as in an app: four threads queue battery and status reads in tight loops for 60 s, keeping the queue under 8. Each queued read takes the device's lock and then the queue's while the IO thread fetches the next instruction, the window DEV-1266's lock-order deadlock needs. Idle, because while streaming Android writes these reads straight out, unqueued. | A read doesn't return within 10 s, or no read returns for 10 s (the report then holds every thread's stack); a read throws; or the queue doesn't drain within 15 s after the reads stop. No SD card needed. On driver 0.11.7_beta, which has the deadlock, this is the scenario to reproduce it. | +| `samplingRates` | Writes 51.2, 256 and 512 Hz in turn, the way the examples configure, and streams 5 s at each; then writes the unit's own rate back. Every configure but the first comes straight after a `stopStreaming()`, as an app that changes the rate does: up to driver 0.11.10_beta the stop's reply then retired the configuration's first chunk, the one holding the sampling rate (DEV-1306). | A configuration doesn't complete within 20 s, or the rate read back from the unit isn't the one written; or a stream fails `streamRest`'s loss, timestamp, rate and silence checks, or logs a parser error. No SD card needed. | + +`-Scenario resetUnit` is not a check and is not part of a default run: it connects, stops SD logging if a run left the unit logging, and disconnects. + +Every scenario also fails if an API call doesn't return within 10 s. The report then holds every thread's stack, which a phone without root can't otherwise give, and the scenario stops. If the whole test process still never finishes, the runner force-stops it after `-ScenarioTimeoutSeconds` (600) and records TIMEOUT. + +The packet checks use the device's own timestamps, so a loss is a gap in the device's sequence, not +phone scheduling. They follow the firmware HIL harness in log-and-stream-common `Test/hil` (DEV-1165), +so this can later run there as a tier. + +### What it does not test +Over BLE the tests run the driver's own streaming parser, the one the PC and MATLAB use. Over Classic +they run the Android API's copy of it (`Shimmer.java` overrides `processPacket`, +`processWhileStreaming`, `processInstreamResponse`, `clearSingleDataPacketFromBuffers`, `readBattery` +and `readStatusLogAndStream`). Neither covers: + +- `shimmerdriverpc`: the serial port, BLE gRPC, and Shimmer3R USB; +- the SD and file parsers. + +Both cover the driver's connect sequence, configuration, calibration, decoding and timestamp unwrap, +since Android uses those unchanged. + +## Choosing what to test + +| | Default | Option | +|---|---|---| +| Android API | this checkout's library source | `-AndroidApiVersion 3.2.10_beta`: the published `.aar`, as a customer app gets it | +| Java driver | the newest `API_v` release | `-DriverVersion 0.11.8_beta` | +| Units | every unit in the rig file | `-Unit s3r-15bc` | +| Transports | every transport each unit lists | `-Transport ble` | +| Scenarios | all five | `-Scenario midStreamSet` | +| Link CRC | ask for 2 bytes (the driver may settle on 1 or off) | `-Crc off`, `-Crc 1` | +| Pause between runs | 10 s | `-SettleSeconds 60` | +| Record the result for the release gate | no | `-Report` (full run on a clean, pushed commit only) | +| Save the bytes exchanged with each unit | no | `-CaptureBytes` (see [Capturing the bytes](#capturing-the-bytes)) | + +## One-time setup + +1. **Phone.** Turn on Developer options, then USB debugging, and "Stay awake while charging". + Connect it to the PC and accept the debugging prompt. On Xiaomi phones also turn on "Install via + USB". On Android 12 and later also turn on "USB debugging (Security settings)", which lets the + tests be granted `BLUETOOTH_CONNECT`. Up to Android 11 nothing needs granting: connecting to a + paired unit uses install-time permissions only. +2. **Pair each unit you test over Classic** with the phone, in Settings → Bluetooth, PIN `1234`. BLE + needs no pairing. A Shimmer3R usually bonds over BLE instead (DEV-970). The phone then reaches + the unit, but the serial channel is refused, and the Classic test reports that it "did not + connect" within 2 s. Unpair it, then either pair again with the unit's BLE turned off, or connect + once from a classic serial-terminal app so that Android makes a Bluetooth Classic bond. Until + then, test that Shimmer3R over BLE only. +3. **Prepare each unit.** + - **SD sync and Bluetooth disable off.** Either one keeps Bluetooth off (DEV-1228). Check in + Consensys. + - **SD card in,** for `midStreamSet`, `inStream` and `connectReady`. + - **Lying still** during `streamRest`. +4. **List the units:** copy `rig-android.example.json` to `rig-android.json` and fill in each name, + MAC, `type` (`Shimmer3` or `Shimmer3R`) and `transports` (`["classic"]`, `["ble"]` or both; + Classic if left out). That file is gitignored. You can read the MACs of paired units off the + phone: + ```powershell + adb shell dumpsys bluetooth_manager | Select-String -Context 0,6 'Bonded devices' + ``` +5. **Credentials for GitHub Packages,** where the driver comes from. Either set `gpr.usr` and + `gpr.key` in `%USERPROFILE%\.gradle\gradle.properties`, or let the script use your `gh` login + after granting it the packages scope once: + ```powershell + gh auth refresh -h github.com -s read:packages + ``` +6. **JDK 17 or newer.** The script uses `JAVA_HOME` if it qualifies, otherwise the JDK Android Studio + ships in its `jbr` folder. It also finds the Android SDK (`ANDROID_HOME`, or Android Studio's default + location) when the checkout has no `local.properties`. + +A Shimmer holds one Bluetooth link at a time. Don't run this while Consensys, the PC HIL harness or +another phone is connected to the same unit. + +## Output + +The run's folder, `tools/hil/out//`, holds: +- `summary.json` and the table the script prints; +- `reports/`: per scenario and unit, a JSON report (versions, CRC mode actually in use, loss, + rate, accel, parser errors, a timeline of events and the failures) and, for `streamRest` and + `midStreamSet`, a CSV of every sample; +- `*.logcat.txt` and `*.instrument.txt`: the phone's log and the test runner's output for each run; +- with `-CaptureBytes`, `*.bytes.log` for each run and `btsnoop/`, the phone's snoop log they came from; +- `build.log`. + +## Capturing the bytes + +When a run finds something, the bytes that caused it are what a fix needs: replayed, they turn the +finding into a byte-level test that runs anywhere, with no phone or unit. With `-CaptureBytes`, the +script saves each run's bytes as `__.bytes.log`, in the trace format +shimmer-logandstream's replay tests read (`tests/data/*.bytes.log`, loaded by `load_session` in +`tests/common/mod.rs`): + +``` +14:33:05.120 TX h17 [0x07] +14:33:05.163 RX h14 [0xFF 0x00 0x12 0x34 ...] +``` + +TX is phone to unit, RX unit to phone. Over BLE, each line is one ATT write or notification, and +`h` is its attribute handle. Over Classic, each line is one RFCOMM frame's payload and `r` its +channel. Those are kept as the radio delivered them, not split into packets, so a bug that depends +on how bytes arrive replays the same way. The write that turns notifications on is left out, as in +the recorded traces. + +**One-time setup:** +- On the phone: Developer options → **Enable Bluetooth HCI snoop log**. Choose "Enabled", not + "Filtered", if it asks. +- **Then turn Bluetooth off and on.** The log only starts when Bluetooth does. adb can do neither step. + The script checks with the Bluetooth service (`mSnoopLogSettingAtEnable` in + `adb shell dumpsys bluetooth_manager`), and refuses to start until it reports `full`. +- Python 3 on the PATH. + +**How it works:** the script notes the phone's clock before and after each run. After the last run +it takes one bug report (about 3 minutes), because that is the only way adb can read the snoop log +on a phone that isn't rooted. It keeps `btsnoop_hci.log` and deletes the bug report, which holds the +phone's whole state, from the PC and the phone. Then `btsnoop_to_trace.py` cuts out each run's bytes, +by the unit's MAC and the run's time window. Android stamps the snoop log in the phone's local time, +so the script passes the phone's UTC offset along. Each run's window and that offset are saved in +`summary.json` (`captureWindows`, `phoneUtcOffsetMin`), so the bytes can be cut out again later. + +None of this changes a verdict: if it fails, the script says so and carries on. + +**Limits:** +- Android keeps only the current snoop log and the one before it, so a long session can lose its + first runs. Capture a few runs at a time, with `-Unit` and `-Scenario`. +- `btsnoop/` holds all the phone's Bluetooth traffic, not only the unit's. It stays in the run's + folder, which is gitignored. + +To extract by hand, or from a snoop log taken some other way: + +```powershell +python tools\hil\btsnoop_to_trace.py btsnoop_hci.log --mac EC:06:D8:0D:86:F8 --utc-offset-min 480 --out run.bytes.log +``` + +Leave out `--mac` to keep every device's traffic; add `--from-ms` and `--to-ms` (Unix time, from +`captureWindows`) to keep one run. Its tests run without a phone: +`cd tools\hil; python -m unittest test_btsnoop_to_trace`. + +## Running a single test by hand + +```powershell +cd ShimmerAndroidInstrumentDriver +.\gradlew :shimmerHilTest:assembleDebug :shimmerHilTest:assembleDebugAndroidTest -PdriverVersion=0.11.9_beta -Pandroid.useAndroidX=true +adb install -r -g shimmerHilTest\build\outputs\apk\debug\shimmerHilTest-debug.apk +adb install -r -g shimmerHilTest\build\outputs\apk\androidTest\debug\shimmerHilTest-debug-androidTest.apk +adb shell am instrument -w -e class com.shimmerresearch.hiltest.HilScenariosTest#streamRest -e mac 00:06:66:XX:XX:XX com.shimmerresearch.hiltest.test/androidx.test.runner.AndroidJUnitRunner +``` + +Add `-e transport ble -e type Shimmer3R` to run the same test over BLE against a Shimmer3R. + +Without `-e mac`, every test is skipped. That's how CI can compile the tests without hardware. diff --git a/tools/hil/btsnoop_to_trace.py b/tools/hil/btsnoop_to_trace.py new file mode 100644 index 00000000..b112ff77 --- /dev/null +++ b/tools/hil/btsnoop_to_trace.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +""" +Turns an Android Bluetooth HCI snoop log (btsnoop_hci.log) into the byte-trace format that +shimmer-logandstream's replay tests read (tests/data/*.bytes.log): one line per write to the unit +(TX) or notification from it (RX), with its time of day. + + HH:MM:SS.mmm TX h [0x.. 0x..] BLE: ATT write command or request + HH:MM:SS.mmm RX h [0x.. 0x..] BLE: ATT notification + HH:MM:SS.mmm TX r [0x.. 0x..] Classic: RFCOMM payload + HH:MM:SS.mmm RX r [0x.. 0x..] + +Classic payloads are kept as the radio delivered them, chunk by chunk, so a parser bug that +depends on how bytes arrive (a bulk read splitting a packet) replays the same way. + +Standard library only. Usage: + + python btsnoop_to_trace.py btsnoop_hci.log --mac EC:06:D8:0D:86:F8 --out run.bytes.log + [--from-ms EPOCH_MS] [--to-ms EPOCH_MS] [--utc-offset-min 480] [--snoop-clock local|utc] + [--title TEXT] + +--utc-offset-min is the phone's offset from UTC. Android writes the snoop log in local time, so +it is needed both to place the --from-ms/--to-ms window and to print times of day. +""" +import argparse +import datetime +import struct +import sys + +# btsnoop counts microseconds from 0000-01-01; this is 1970-01-01 on that scale. +BTSNOOP_EPOCH_DELTA_US = 0x00DCDDB30F2F8000 + +H4_COMMAND, H4_ACL, H4_SCO, H4_EVENT = 1, 2, 3, 4 +L2CAP_SIGNALLING, L2CAP_ATT = 0x0001, 0x0004 +PSM_RFCOMM = 0x0003 +ATT_WRITE_REQ, ATT_NOTIFY, ATT_WRITE_CMD = 0x12, 0x1B, 0x52 +ATT_MTU_REQ, ATT_MTU_RSP = 0x02, 0x03 +RFCOMM_UIH = 0xEF + + +class Record: + __slots__ = ("time_us", "received", "data") + + def __init__(self, time_us, received, data): + self.time_us = time_us # Unix time, microseconds + self.received = received # True: controller to host (from the unit) + self.data = data + + +def read_btsnoop(path, clock_offset_us=0): + """clock_offset_us: how far the log's clock is ahead of UTC. Android stamps its snoop log in + the phone's local time, so on a phone at UTC+8 it is 8 hours.""" + with open(path, "rb") as f: + header = f.read(16) + if len(header) < 16 or header[:8] != b"btsnoop\0": + raise ValueError(f"{path}: not a btsnoop file") + _version, datalink = struct.unpack(">II", header[8:16]) + if datalink != 1002: + raise ValueError(f"{path}: datalink {datalink}, expected 1002 (HCI UART, H4)") + while True: + head = f.read(24) + if len(head) < 24: + return + _orig_len, incl_len, flags, _drops, ts = struct.unpack(">IIIIq", head) + data = f.read(incl_len) + if len(data) < incl_len: + return + yield Record(ts - BTSNOOP_EPOCH_DELTA_US - clock_offset_us, bool(flags & 1), data) + + +def mac_from_le(b): + return ":".join(f"{x:02X}" for x in reversed(b)) + + +class Link: + """One ACL connection: its address, and how to read the data on it.""" + + def __init__(self, handle, mac, le): + self.handle = handle + self.mac = mac + self.le = le + self.rfcomm_cids = set() + self.pending_rfcomm = {} # signalling id -> source CID of a PSM 3 connection request + self.reassembly = {} # direction -> bytearray of an L2CAP PDU in progress + self.mtu = None + + +class Extractor: + def __init__(self, mac=None): + self.mac = mac.upper() if mac else None + self.links = {} # ACL handle -> Link (current connection on that handle) + self.entries = [] # (time_us, tx, token, payload) + self.connections = [] # (time_us, mac, le, handle) + + def feed(self, rec): + if not rec.data: + return + kind = rec.data[0] + if kind == H4_EVENT: + self._event(rec) + elif kind == H4_ACL: + self._acl(rec) + + def _event(self, rec): + d = rec.data + if len(d) < 3: + return + code, params = d[1], d[3:] + if code == 0x03 and len(params) >= 9 and params[0] == 0: # Connection Complete (BR/EDR) + handle = struct.unpack_from("= 12: + if params[1] == 0: # LE Connection Complete / Enhanced Connection Complete + handle = struct.unpack_from("= 3: # Disconnection Complete + handle = struct.unpack_from("> 12) & 0x3 + link = self.links.get(handle) + if link is None: + # A connection made before the log started: its address is unknown. + link = self.links[handle] = Link(handle, None, None) + payload = d[5:5 + length] + key = rec.received + if pb == 0x1: # continuing fragment + buf = link.reassembly.get(key) + if buf is None: + return + buf.extend(payload) + else: # start of an L2CAP PDU + buf = link.reassembly[key] = bytearray(payload) + if len(buf) < 4: + return + l2len, cid = struct.unpack_from("= 3: + mtu = struct.unpack_from("= 4: # Connection Request + psm, scid = struct.unpack_from("= 8: # Connection Response + dcid, scid, result = struct.unpack_from("> 2 + control = pdu[1] + if dlci == 0 or (control & ~0x10) != RFCOMM_UIH: + return # the multiplexer's control channel, or not a data frame + # TS 07.10 length: one octet (EA = 1) up to 127, otherwise two octets holding len << 1 as a + # little-endian 16-bit value. Only the first octet carries the EA bit; the second is len >> 7 + # whole (Linux: __len16 in net/bluetooth/rfcomm/core.c; Android: RFCOMM_PARSE_LEN_FIELD). + if pdu[2] & 1: + length, i = pdu[2] >> 1, 3 + else: + length, i = (pdu[2] >> 1) | (pdu[3] << 7), 4 + if control & 0x10: + i += 1 # a credit byte (credit-based flow control) precedes the data + payload = pdu[i:i + length] + if payload: + self.entries.append((rec.time_us, not rec.received, f"r{dlci}", payload, link)) + + +def drop_cccd_writes(entries): + """A write that turns notifications on (to the descriptor after a notifying handle) is GATT + housekeeping, not a command to the unit; the recorded traces leave it out. Handles belong to one + connection, so a write is only matched against notifications on the same one.""" + notify_handles = {(id(e[4]), int(e[2][1:])) for e in entries if not e[1] and e[2].startswith("h")} + return [e for e in entries + if not (e[1] and e[2].startswith("h") and (id(e[4]), int(e[2][1:]) - 1) in notify_handles + and len(e[3]) == 2 and e[3][1] == 0 and e[3][0] in (0, 1, 2))] + + +def time_of_day(time_us, utc_offset_min): + t = datetime.datetime.fromtimestamp(time_us / 1e6, datetime.timezone.utc) + t += datetime.timedelta(minutes=utc_offset_min) + return t.strftime("%H:%M:%S.") + f"{t.microsecond // 1000:03d}" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("log", nargs="+", help="btsnoop_hci.log (older file first, if it rotated)") + ap.add_argument("--mac", help="only this unit's connection") + ap.add_argument("--from-ms", type=int, help="Unix time, ms: drop what came before") + ap.add_argument("--to-ms", type=int, help="Unix time, ms: drop what came after") + ap.add_argument("--utc-offset-min", type=int, default=0, help="the phone's offset from UTC, in minutes") + ap.add_argument("--snoop-clock", choices=("local", "utc"), default="local", + help="the clock the log is stamped in (Android: local)") + ap.add_argument("--title", default="", help="first comment line") + ap.add_argument("--out", required=True) + args = ap.parse_args(argv) + + clock_offset_us = args.utc_offset_min * 60_000_000 if args.snoop_clock == "local" else 0 + ex = Extractor(args.mac) + first = last = None + for path in args.log: + for rec in read_btsnoop(path, clock_offset_us): + first = rec.time_us if first is None else first + last = rec.time_us + ex.feed(rec) + lo = args.from_ms * 1000 if args.from_ms is not None else None + hi = args.to_ms * 1000 if args.to_ms is not None else None + entries = [e for e in drop_cccd_writes(ex.entries) + if (lo is None or e[0] >= lo) and (hi is None or e[0] <= hi)] + + with open(args.out, "w", encoding="utf-8", newline="\n") as out: + if args.title: + out.write(f"# {args.title}\n") + out.write("# Extracted from the phone's Bluetooth HCI snoop log by tools/hil/btsnoop_to_trace.py.\n") + out.write("# TX = phone to unit, RX = unit to phone; h is a BLE ATT handle, r an RFCOMM channel (DLCI).\n") + out.write("# Classic payloads are as the radio delivered them, not split into packets.\n") + seen = set() + for time_us, mac, le, handle in ex.connections: + if (args.mac is None or mac == args.mac.upper()) and (lo is None or time_us >= lo - 30_000_000) \ + and (hi is None or time_us <= hi): + link_mtu = next((e[4].mtu for e in entries if e[4].handle == handle and e[4].mac == mac), None) + key = (time_us, handle) + if key not in seen: + seen.add(key) + mtu = f" mtu={link_mtu}" if link_mtu else "" + out.write(f"{time_of_day(time_us, args.utc_offset_min)} connected {mac} " + f"over {'BLE' if le else 'Classic'} handle={handle}{mtu}\n") + for time_us, tx, token, payload, _link in entries: + data = " ".join(f"0x{b:02X}" for b in payload) + out.write(f"{time_of_day(time_us, args.utc_offset_min)} {'TX' if tx else 'RX'} {token} [{data}]\n") + + tx = [e for e in entries if e[1]] + rx = [e for e in entries if not e[1]] + span = "" + if first is not None: + span = (f"; the log covers {time_of_day(first, args.utc_offset_min)}" + f" to {time_of_day(last, args.utc_offset_min)}") + print(f"{args.out}: {len(tx)} TX ({sum(len(e[3]) for e in tx)} bytes), " + f"{len(rx)} RX ({sum(len(e[3]) for e in rx)} bytes){span}") + return 0 if entries else 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hil/rig-android.example.json b/tools/hil/rig-android.example.json new file mode 100644 index 00000000..49baa299 --- /dev/null +++ b/tools/hil/rig-android.example.json @@ -0,0 +1,7 @@ +{ + "phone": "", + "units": [ + { "name": "s3-xxxx", "mac": "00:06:66:XX:XX:XX", "type": "Shimmer3", "transports": ["classic"] }, + { "name": "s3r-xxxx", "mac": "E8:EB:1B:XX:XX:XX", "type": "Shimmer3R", "transports": ["ble", "classic"] } + ] +} diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 new file mode 100644 index 00000000..b3822c19 --- /dev/null +++ b/tools/hil/run-android-hil.ps1 @@ -0,0 +1,454 @@ +<# +.SYNOPSIS +Runs the Android API hardware-in-the-loop (HIL) scenarios on a USB-attached phone against every +Shimmer in a rig file, pulls the reports and prints a pass/fail table. Exits 1 if anything failed. + +.DESCRIPTION +One-time setup is in tools/hil/README.md: list each unit, with the transports to test it over, in +tools/hil/rig-android.json (copy rig-android.example.json), pair it with the phone if it is tested over +Bluetooth Classic, and give Gradle a token that can read GitHub Packages. + +.EXAMPLE +.\tools\hil\run-android-hil.ps1 +Every scenario on every unit, against the newest Java API (driver) release and this checkout's library. + +.EXAMPLE +.\tools\hil\run-android-hil.ps1 -AndroidApiVersion 3.2.10_beta -DriverVersion 0.11.9_beta +The published .aar, as a customer app gets it. + +.EXAMPLE +.\tools\hil\run-android-hil.ps1 -Scenario midStreamSet -Unit s3r-15bc -Crc 1 + +.EXAMPLE +.\tools\hil\run-android-hil.ps1 -Unit s3r-15bc -Scenario midStreamSet -CaptureBytes +Also saves every byte exchanged with the unit, per run, as a trace shimmer-logandstream's tests can +replay. Needs the phone's Bluetooth HCI snoop log on (see the README) and Python 3. + +.EXAMPLE +.\tools\hil\run-android-hil.ps1 -Report +Before "Release Android API": the full run on this commit, recorded on GitHub as the commit status +"android-hil (driver X)". The release refuses to publish that commit with driver X without a passing one. +#> +param( + [string]$Rig = (Join-Path $PSScriptRoot 'rig-android.json'), + [ValidateSet('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates', 'resetUnit')] + [string[]]$Scenario = @('streamRest', 'midStreamSet', 'reconnectLoop', 'inStream', 'connectReady', 'queueStress', 'samplingRates'), + [string[]]$Unit, + [ValidateSet('classic', 'ble')] + [string[]]$Transport, + [string]$DriverVersion, + [string]$AndroidApiVersion, + [string]$Serial, + [double]$Rate = 256, + [int]$Seconds = 30, + [ValidateSet('off', '1', '2')] + [string]$Crc = '2', + [double]$MaxLossPercent = 1.0, + [switch]$SkipBuild, + [switch]$Report, + [int]$ScenarioTimeoutSeconds = 600, + # Pause between runs, so the unit has advertised again and the phone's Bluetooth stack has let go + # of the last connection before the next one starts. + [int]$SettleSeconds = 10, + # Save the bytes exchanged with each unit, from the phone's Bluetooth HCI snoop log, as + # __.bytes.log beside each run's logcat. Adds a bug report (~3 min). + [switch]$CaptureBytes +) + +$ErrorActionPreference = 'Continue' +$repo = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path +$gradleRoot = Join-Path $repo 'ShimmerAndroidInstrumentDriver' +$package = 'com.shimmerresearch.hiltest' +$runner = "$package.test/androidx.test.runner.AndroidJUnitRunner" +$deviceReports = "/sdcard/Android/data/$package/files/hil" + +function Fail([string]$message) { + Write-Host "ERROR: $message" -ForegroundColor Red + exit 2 +} + +# --- Rig --------------------------------------------------------------------------------------- +if (-not (Test-Path $Rig)) { + Fail "No rig file at $Rig. Copy tools\hil\rig-android.example.json to rig-android.json and list your units." +} +$rigData = Get-Content $Rig -Raw | ConvertFrom-Json +$units = @($rigData.units) +if ($Unit) { $units = @($units | Where-Object { $Unit -contains $_.name }) } +if ($units.Count -eq 0) { Fail "No units to test (rig: $Rig, -Unit: $Unit)." } +if (-not $Serial -and $rigData.phone) { $Serial = $rigData.phone } + +# Each unit runs over the transports it lists ("classic" needs it paired with the phone; "ble" +# doesn't). They are different code: Classic goes through the Android API's own Shimmer class and +# parser, BLE through Shimmer3BLEAndroid and the driver's parser. +$targets = @(foreach ($u in $units) { + $unitTransports = @($u.transports) + if (-not $u.transports) { $unitTransports = @('classic') } + foreach ($t in $unitTransports) { + if ($t -ne 'classic' -and $t -ne 'ble') { Fail "Unit $($u.name): transport '$t' is not classic or ble." } + if (-not $Transport -or $Transport -contains $t) { [pscustomobject]@{ Unit = $u; Transport = $t } } + } +}) +if ($targets.Count -eq 0) { Fail "Nothing to run: no unit lists the transport(s) $($Transport -join ', ')." } + +# --- adb --------------------------------------------------------------------------------------- +$sdk = $env:ANDROID_HOME +if (-not $sdk) { $sdk = $env:ANDROID_SDK_ROOT } +if (-not $sdk) { $sdk = Join-Path $env:LOCALAPPDATA 'Android\Sdk' } +$adbExe = Join-Path $sdk 'platform-tools\adb.exe' +if (-not (Test-Path $adbExe)) { Fail "adb not found at $adbExe. Set ANDROID_HOME." } +$adbTarget = @() +if ($Serial) { $adbTarget = @('-s', $Serial) } +function Adb { & $adbExe @adbTarget @args } + +$phones = @(& $adbExe devices | Select-String '\tdevice$' | ForEach-Object { ($_ -split "`t")[0] }) +if (-not $Serial -and $phones.Count -ne 1) { + Fail "Expected one phone on adb, found $($phones.Count): $($phones -join ', '). Pass -Serial." +} +$phoneModel = (Adb shell getprop ro.product.model) -join '' +$phoneAndroid = (Adb shell getprop ro.build.version.release) -join '' +function PhoneMs { [long](((Adb shell date +%s%3N) -join '').Trim()) } + +if ($CaptureBytes) { + # The Bluetooth service reports the snoop mode it started with, which is the one in force: the + # setting only takes effect when Bluetooth next starts. (adb can't read the setting itself on + # Android 11.) "filtered" keeps only the start of each data packet, which is no use for replay. + $snoopLine = @(Adb shell dumpsys bluetooth_manager | Select-String 'mSnoopLogSettingAtEnable = (\S+)') | Select-Object -First 1 + $snoopMode = if ($snoopLine) { $snoopLine.Matches[0].Groups[1].Value } else { ((Adb shell getprop persist.bluetooth.btsnooplogmode) -join '').Trim() } + if ($snoopMode -ne 'full') { + Fail ("-CaptureBytes needs the phone's full Bluetooth HCI snoop log, and Bluetooth started with it in mode " + + "'$snoopMode'. In Developer options turn on 'Enable Bluetooth HCI snoop log' (choose 'Enabled', not " + + "'Filtered', where it asks), then turn Bluetooth off and on.") + } + $python = @(Get-Command python, python3 -CommandType Application -ErrorAction SilentlyContinue) | Select-Object -First 1 + # Windows' "python" can be a Store shortcut that runs nothing, so check that it runs Python 3. + if ($python) { & $python.Source -c "import sys; sys.exit(sys.version_info[0] != 3)" 2>$null | Out-Null } + if (-not $python -or $LASTEXITCODE -ne 0) { Fail '-CaptureBytes needs Python 3 on the PATH, to extract the bytes from the snoop log.' } + $z = ((Adb shell date +%z) -join '').Trim() # e.g. +0800 + $phoneUtcOffsetMin = [int]($z.Substring(0, 1) + '1') * ([int]$z.Substring(1, 2) * 60 + [int]$z.Substring(3, 2)) +} +$windows = @{} + +# --- Versions ---------------------------------------------------------------------------------- +if (-not $DriverVersion) { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + $releases = Invoke-RestMethod 'https://api.github.com/repos/ShimmerResearch/Shimmer-Java-Android-API/releases?per_page=100' + $newest = $releases | Where-Object { $_.tag_name -like 'API_v*' } | Sort-Object created_at -Descending | Select-Object -First 1 + if (-not $newest) { Fail 'Could not find an API_v release; pass -DriverVersion.' } + $DriverVersion = $newest.tag_name.Substring(5) +} +$apiLabel = $AndroidApiVersion +if (-not $apiLabel) { $apiLabel = 'source ' + ((git -C $repo rev-parse --short HEAD) -join '') } + +# --- What -Report vouches for ----------------------------------------------------------------- +# The release gate trusts this status, so only report a run that tested exactly what the release +# would build: this commit's library source, unmodified and pushed, every scenario, both device types. +$commit = $null +$githubRepo = $null +if ($Report) { + if ($AndroidApiVersion) { Fail '-Report tests the library source at a commit; drop -AndroidApiVersion.' } + if ($PSBoundParameters.ContainsKey('Scenario') -or $Unit -or $Transport) { + Fail '-Report needs every scenario on every unit and transport in the rig; drop -Scenario, -Unit and -Transport.' + } + # The gate's pass criteria are the defaults; a run that changed them, or reused an old build, is + # not evidence about this commit. + $shaping = @('Rate', 'Seconds', 'Crc', 'MaxLossPercent', 'SkipBuild') | Where-Object { $PSBoundParameters.ContainsKey($_) } + if ($shaping) { + Fail "-Report runs with the default settings and a fresh build; drop $((@($shaping) | ForEach-Object { "-$_" }) -join ', ')." + } + $types = @($units | ForEach-Object { $_.type }) + if (-not ($types -contains 'Shimmer3') -or -not ($types -contains 'Shimmer3R')) { + Fail "-Report needs at least one Shimmer3 and one Shimmer3R in the rig (types found: $($types -join ', '))." + } + $transportsCovered = @($targets | ForEach-Object { $_.Transport }) | Sort-Object -Unique + if (-not ($transportsCovered -contains 'classic') -or -not ($transportsCovered -contains 'ble')) { + Fail "-Report needs both transports in the rig, Classic and BLE: they run different parsers (found: $($transportsCovered -join ', '))." + } + $dirty = (git -C $repo status --porcelain) -join "`n" + if ($dirty) { Fail "-Report needs a clean working tree, so the tested code is exactly the commit:`n$dirty" } + $commit = ((git -C $repo rev-parse HEAD) -join '').Trim() + $origin = ((git -C $repo remote get-url origin) -join '').Trim() + if ($origin -notmatch 'github\.com[:/](.+?)(\.git)?$') { Fail "origin is not a GitHub remote: $origin" } + $githubRepo = $Matches[1] + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { Fail '-Report posts the result with gh; install it and log in.' } + gh api "repos/$githubRepo/commits/$commit" --silent 2>$null + if ($LASTEXITCODE -ne 0) { Fail "Commit $commit is not on GitHub ($githubRepo). Push it first: the release checks the commit it builds." } +} + +$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' +$outDir = Join-Path $PSScriptRoot "out\$stamp" +New-Item -ItemType Directory -Force $outDir | Out-Null + +Write-Host "Android HIL: $phoneModel (Android $phoneAndroid), Android API $apiLabel, driver $DriverVersion" +Write-Host "Results: $outDir" + +# --- Build ------------------------------------------------------------------------------------- +$apk = Join-Path $gradleRoot 'shimmerHilTest\build\outputs\apk\debug\shimmerHilTest-debug.apk' +$testApk = Join-Path $gradleRoot 'shimmerHilTest\build\outputs\apk\androidTest\debug\shimmerHilTest-debug-androidTest.apk' +if (-not $SkipBuild) { + # Gradle needs JDK 17 or newer (AGP 8). Use JAVA_HOME if it qualifies, else a JDK Android Studio ships. + $saved = @{ JAVA_HOME = $env:JAVA_HOME; ANDROID_HOME = $env:ANDROID_HOME + GPR_USR = ${env:ORG_GRADLE_PROJECT_gpr.usr}; GPR_KEY = ${env:ORG_GRADLE_PROJECT_gpr.key} } + try { + # A fresh checkout has no local.properties (Android Studio writes it), so tell Gradle where + # the SDK is: the one adb came from. + if (-not $env:ANDROID_HOME -and -not $env:ANDROID_SDK_ROOT -and -not (Test-Path (Join-Path $gradleRoot 'local.properties'))) { + $env:ANDROID_HOME = $sdk + } + $javaOk = $false + if ($env:JAVA_HOME -and (Test-Path (Join-Path $env:JAVA_HOME 'release'))) { + $javaOk = (Get-Content (Join-Path $env:JAVA_HOME 'release') -Raw) -match 'JAVA_VERSION="(1[7-9]|[2-9]\d)' + } + if (-not $javaOk) { + $candidates = @(Get-ChildItem "$env:ProgramFiles\Android" -Directory -ErrorAction SilentlyContinue | + ForEach-Object { Join-Path $_.FullName 'jbr' } | Where-Object { Test-Path (Join-Path $_ 'bin\java.exe') }) + if ($candidates.Count -eq 0) { Fail 'Need JDK 17 or newer: set JAVA_HOME (Android Studio ships one in its jbr folder).' } + $env:JAVA_HOME = $candidates[-1] + Write-Host "Using JAVA_HOME=$env:JAVA_HOME" + } + + # GitHub Packages: gpr.usr/gpr.key from ~/.gradle/gradle.properties if set, else the gh login's + # token, passed to this Gradle run only (it needs the read:packages scope). + $userProps = Join-Path $env:USERPROFILE '.gradle\gradle.properties' + $hasGpr = (Test-Path $userProps) -and ((Get-Content $userProps -Raw) -match '(?m)^\s*gpr\.key\s*=') + if (-not $hasGpr -and -not $env:TOKEN) { + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + Fail 'No GitHub Packages credentials: set gpr.usr/gpr.key in ~/.gradle/gradle.properties, or install and log in to gh.' + } + if (-not ((gh auth status 2>&1 | Out-String) -match 'read:packages')) { + Fail "The gh login cannot read GitHub Packages. Run: gh auth refresh -h github.com -s read:packages" + } + ${env:ORG_GRADLE_PROJECT_gpr.usr} = (gh api user --jq .login) + ${env:ORG_GRADLE_PROJECT_gpr.key} = (gh auth token) + } + + $gradleArgs = @(':shimmerHilTest:assembleDebug', ':shimmerHilTest:assembleDebugAndroidTest', + "-PdriverVersion=$DriverVersion", '-Pandroid.useAndroidX=true', + '-Dorg.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1g', '--console=plain') + if ($AndroidApiVersion) { $gradleArgs += "-PandroidApiVersion=$AndroidApiVersion" } + Write-Host 'Building the test APKs...' + Push-Location $gradleRoot + & .\gradlew.bat @gradleArgs *> (Join-Path $outDir 'build.log') + $buildExit = $LASTEXITCODE + Pop-Location + if ($buildExit -ne 0) { Fail "Build failed; see $(Join-Path $outDir 'build.log')" } + } finally { + $env:JAVA_HOME = $saved.JAVA_HOME + $env:ANDROID_HOME = $saved.ANDROID_HOME + ${env:ORG_GRADLE_PROJECT_gpr.usr} = $saved.GPR_USR + ${env:ORG_GRADLE_PROJECT_gpr.key} = $saved.GPR_KEY + } +} + +# --- Install ----------------------------------------------------------------------------------- +# -g pre-grants runtime permissions. Some phones (Xiaomi) refuse it without "USB debugging (Security +# settings)"; the tests need no runtime permission before Android 12, so install without it there. +function Install-Apk([string]$file, [string[]]$flags) { + ((& $adbExe @adbTarget install @flags $file 2>&1) | ForEach-Object { "$_" }) -join "`n" +} +foreach ($file in @($apk, $testApk)) { + $result = Install-Apk $file @('-r', '-g') + if ($result -match 'INSTALL_GRANT_RUNTIME_PERMISSIONS') { + Write-Host "The phone refused to pre-grant permissions (-g); installing $(Split-Path $file -Leaf) without." + $result = Install-Apk $file @('-r') + } + if ($result -notmatch 'Success') { + Fail "Installing $(Split-Path $file -Leaf) failed:`n$($result.Trim())`nOn Xiaomi, turn on 'Install via USB' (and on Android 12+ 'USB debugging (Security settings)') in Developer options." + } +} +# Clear the last run's reports, and make sure they went: the collect step takes any report it finds +# as this run's evidence. +Adb shell rm -rf $deviceReports | Out-Null +$left = (Adb shell "ls $deviceReports 2>/dev/null") -join '' +if ($left.Trim()) { Fail "Could not clear the old reports on the phone ($deviceReports); remove them and run again." } + +# --- Run --------------------------------------------------------------------------------------- +# One scenario, with a time limit: the tests guard every API call, but if the whole process still +# never finishes, force-stop it so an unattended run cannot hang. +function Invoke-Instrument([string[]]$arguments, [int]$timeoutSeconds) { + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $adbExe + $psi.Arguments = (@($adbTarget) + $arguments | ForEach-Object { if ("$_" -match '\s') { '"' + $_ + '"' } else { "$_" } }) -join ' ' + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $p = [System.Diagnostics.Process]::Start($psi) + $stdout = $p.StandardOutput.ReadToEndAsync() + $stderr = $p.StandardError.ReadToEndAsync() + $finished = $p.WaitForExit($timeoutSeconds * 1000) + if (-not $finished) { + & $adbExe @adbTarget shell am force-stop $package 2>&1 | Out-Null + if (-not $p.WaitForExit(15000)) { $p.Kill() } + } + [pscustomobject]@{ Output = $stdout.Result + $stderr.Result; TimedOut = (-not $finished) } +} + +$results = @() +$runs = 0 +foreach ($target in $targets) { + $u = $target.Unit + $t = $target.Transport + foreach ($s in $Scenario) { + if ($runs++ -gt 0) { Start-Sleep -Seconds $SettleSeconds } + $label = "$($u.name) $t $s" + Write-Host "Running $label..." -NoNewline + Adb logcat -c | Out-Null + $instrumentArgs = @('shell', 'am', 'instrument', '-w', '-r', + '-e', 'class', "$package.HilScenariosTest#$s", + '-e', 'mac', $u.mac, '-e', 'unit', $u.name, '-e', 'type', $u.type, '-e', 'transport', $t, + '-e', 'rate', $Rate, '-e', 'seconds', $Seconds, + '-e', 'crc', $Crc, '-e', 'maxLossPercent', $MaxLossPercent, + '-e', 'driverVersion', $DriverVersion, '-e', 'androidApi', ($apiLabel -replace ' ', '_'), + $runner) + $startMs = if ($CaptureBytes) { PhoneMs } else { 0 } + $run = Invoke-Instrument $instrumentArgs $ScenarioTimeoutSeconds + $output = $run.Output + $safe = "$($u.name)_$($t)_$s" -replace '[^\w.-]', '_' + if ($CaptureBytes) { $windows[$safe] = @{ From = $startMs; To = (PhoneMs); Mac = $u.mac; Label = $label } } + $output | Set-Content (Join-Path $outDir "$safe.instrument.txt") + Adb logcat -d -v threadtime | Set-Content (Join-Path $outDir "$safe.logcat.txt") + + $codes = [regex]::Matches($output, 'INSTRUMENTATION_STATUS_CODE: (-?\d+)') + $code = if ($codes.Count -gt 0) { [int]$codes[$codes.Count - 1].Groups[1].Value } else { $null } + $detail = '' + switch ($code) { + 0 { $verdict = 'PASS' } + -2 { $verdict = 'FAIL' } + -4 { $verdict = 'SKIP' } + default { $verdict = 'ERROR' } + } + $m = [regex]::Match($output, '(?:AssertionError|AssumptionViolatedException|Exception): ([^\r\n]+)') + if ($run.TimedOut) { + $verdict = 'TIMEOUT' + $detail = "did not finish within $ScenarioTimeoutSeconds s; force-stopped (see the logcat)" + } + elseif ($m.Success) { $detail = $m.Groups[1].Value } + elseif ($verdict -eq 'ERROR') { $detail = ($output -split "`n" | Select-Object -Last 3) -join ' ' } + Write-Host " $verdict" -ForegroundColor $(if ($verdict -eq 'PASS') { 'Green' } elseif ($verdict -eq 'SKIP') { 'Yellow' } else { 'Red' }) + $results += [pscustomobject]@{ Unit = $u.name; Transport = $t; Scenario = $s; Result = $verdict; Detail = $detail } + } +} + +# --- Collect ----------------------------------------------------------------------------------- +$reportDir = Join-Path $outDir 'reports' +$pullOutput = & $adbExe @adbTarget pull $deviceReports $reportDir 2>&1 | Out-String +$pullFailed = ($LASTEXITCODE -ne 0) +# Every run that reached its verdict wrote a report, and the gate's status rests on them: a run whose +# report did not come back is an error, not a pass. +$macByUnit = @{} +foreach ($u in $units) { $macByUnit[$u.name] = $u.mac } +foreach ($r in $results) { + if ($r.Result -ne 'PASS' -and $r.Result -ne 'FAIL') { continue } + $prefix = ($r.Scenario -creplace '([A-Z])', '_$1').ToLower() + "_$($r.Transport)_$($macByUnit[$r.Unit] -replace ':', '')_" + $found = @(Get-ChildItem $reportDir -Filter "$prefix*.json" -ErrorAction SilentlyContinue) + if ($found.Count -eq 0) { + $why = if ($pullFailed) { "adb pull failed: $($pullOutput.Trim())" } else { 'the phone had no report for it' } + $r.Result = 'ERROR' + $r.Detail = (("no report collected ($why). " + $r.Detail).Trim()) + } +} +# The bytes, from the phone's HCI snoop log. Diagnostic only: a problem here never changes a verdict. +if ($CaptureBytes) { + Write-Host 'Collecting the Bluetooth snoop log through a bug report (about 3 minutes)...' + $zip = Join-Path $outDir 'bugreport.zip' + $brOutput = & $adbExe @adbTarget bugreport $zip 2>&1 | Out-String + $snoopDir = Join-Path $outDir 'btsnoop' + $logs = @() + try { + if (Test-Path $zip) { + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [IO.Compression.ZipFile]::OpenRead($zip) + try { + New-Item -ItemType Directory -Force $snoopDir | Out-Null + foreach ($entry in @($archive.Entries | Where-Object { $_.Name -match '^btsnoop_hci\.log(\.last)?$' })) { + $dest = Join-Path $snoopDir $entry.Name + [IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $dest, $true) + $logs += $dest + } + } + finally { $archive.Dispose() } + } + else { + Write-Host "The bug report did not arrive: $($brOutput.Trim())" -ForegroundColor Yellow + } + } + catch { + Write-Host "Could not read the bug report: $($_.Exception.Message)" -ForegroundColor Yellow + $logs = @() + } + finally { + # A bug report holds the phone's whole state; keep only the snoop log, on the PC and the + # phone, whether or not reading it worked. + Remove-Item $zip -ErrorAction SilentlyContinue + foreach ($onPhone in [regex]::Matches($brOutput, '(/data/user_de/\S+/bugreports/)(bugreport-[^\s/]+?)\.zip')) { + $dir = $onPhone.Groups[1].Value + $name = $onPhone.Groups[2].Value + Adb shell "rm -f $dir$name.zip $dir$name-dumpstate_log-*.txt" | Out-Null + } + } + # The rotated-out file (.last) is older, so it goes first. + $logs = @($logs | Sort-Object { if ($_ -like '*.last') { 0 } else { 1 } }) + if ($logs.Count -eq 0) { + Write-Host 'No btsnoop_hci.log in the bug report, so no bytes were saved. Is the snoop log on (Developer options)?' -ForegroundColor Yellow + } + else { + foreach ($safe in $windows.Keys) { + $w = $windows[$safe] + $title = "HIL run $($w.Label) on $phoneModel (Android $phoneAndroid), Android API $apiLabel, driver $DriverVersion" + & $python.Source (Join-Path $PSScriptRoot 'btsnoop_to_trace.py') @logs --mac $w.Mac --from-ms $w.From --to-ms $w.To ` + --utc-offset-min $phoneUtcOffsetMin --title $title --out (Join-Path $outDir "$safe.bytes.log") | Write-Host + if ($LASTEXITCODE -ne 0) { + Write-Host " no bytes for $($w.Label) in the snoop log (it may have rotated past this run)" -ForegroundColor Yellow + } + } + } +} + +$summary = [pscustomobject]@{ + phone = "$phoneModel (Android $phoneAndroid)"; androidApi = $apiLabel; driverVersion = $DriverVersion + rate = $Rate; seconds = $Seconds; crc = $Crc; settleSeconds = $SettleSeconds; captureBytes = [bool]$CaptureBytes + results = $results +} +if ($CaptureBytes) { + # Each run's window on the phone's clock, to cut its bytes out of btsnoop/ again later. + $summary | Add-Member -MemberType NoteProperty -Name captureWindows -Value ([pscustomobject]$windows) + $summary | Add-Member -MemberType NoteProperty -Name phoneUtcOffsetMin -Value $phoneUtcOffsetMin +} +$summary | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $outDir 'summary.json') + +Write-Host '' +$results | Format-Table -AutoSize -Wrap | Out-String -Width 200 | Write-Host +# A skipped scenario did not run, so it is not a pass: the exit code and the gate's status agree on that. +$failed = @($results | Where-Object { $_.Result -ne 'PASS' }) +$passed = @($results | Where-Object { $_.Result -eq 'PASS' }) + +if ($Report) { + # Firmware per unit, from the reports the tests wrote, with the transports each ran over. + $firmware = @{} + Get-ChildItem $reportDir -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { + $r = Get-Content $_.FullName -Raw | ConvertFrom-Json + if ($r.firmware -and $r.args.unit) { $firmware[$r.args.unit] = "$($r.hardware) $($r.firmware)" } + } + $tested = @($units | ForEach-Object { + $name = $_.name + $over = (@($targets | Where-Object { $_.Unit.name -eq $name } | ForEach-Object { $_.Transport }) -join '+') + $what = if ($firmware[$name]) { $firmware[$name] } else { $_.type } + "$what ($over)" + }) | Sort-Object -Unique + $state = if ($passed.Count -eq $results.Count) { 'success' } else { 'failure' } + $description = "$($passed.Count)/$($results.Count) passed on $phoneModel`: $($tested -join ', ')" + if ($description.Length -gt 140) { $description = $description.Substring(0, 137) + '...' } + $context = "android-hil (driver $DriverVersion)" + gh api -X POST "repos/$githubRepo/statuses/$commit" -f state=$state -f context=$context -f description=$description --silent + if ($LASTEXITCODE -ne 0) { + Write-Host "Could not post the commit status; the release will not see this run." -ForegroundColor Red + exit 2 + } + Write-Host "Posted '$context' = $state on $githubRepo@$($commit.Substring(0, 8)): $description" +} + +if ($failed.Count -gt 0) { + Write-Host "$($failed.Count) of $($results.Count) did not pass. Reports, logcat and instrument output: $outDir" -ForegroundColor Red + exit 1 +} +Write-Host "All $($results.Count) passed. Reports: $outDir" -ForegroundColor Green +exit 0 diff --git a/tools/hil/test_btsnoop_to_trace.py b/tools/hil/test_btsnoop_to_trace.py new file mode 100644 index 00000000..27a054c2 --- /dev/null +++ b/tools/hil/test_btsnoop_to_trace.py @@ -0,0 +1,161 @@ +"""Checks btsnoop_to_trace.py on a synthetic snoop log. Run: python -m unittest test_btsnoop_to_trace""" +import os +import shutil +import struct +import tempfile +import unittest + +import btsnoop_to_trace as bt + +S3R, S3, OTHER = "EC:06:D8:0D:86:F8", "E8:EB:1B:71:3E:36", "11:22:33:44:55:66" +T0 = 1_792_476_000_000_000 # Unix time, us (2026-10-20 06:00 UTC) +PHONE_UTC_OFFSET_US = 8 * 3600 * 1_000_000 # a phone at UTC+8 + + +def mac_le(mac): + return bytes(reversed(bytes.fromhex(mac.replace(":", "")))) + + +class Snoop: + def __init__(self): + self.records = [] + self.t = T0 + + def add(self, received, data, dt_ms=1): + self.t += dt_ms * 1000 + self.records.append((self.t, received, bytes(data))) + + def event(self, code, params): + self.add(True, bytes([bt.H4_EVENT, code, len(params)]) + params) + + def le_connected(self, handle, mac): + self.event(0x3E, bytes([0x01, 0x00]) + struct.pack("II", 1, 1002)) + for t, received, data in self.records: + f.write(struct.pack(">IIIIq", len(data), len(data), 1 if received else 0, 0, + t + bt.BTSNOOP_EPOCH_DELTA_US + clock_offset_us)) + f.write(data) + + +def rfcomm(dlci, payload, credit=None): + control = bt.RFCOMM_UIH | (0x10 if credit is not None else 0) + n = len(payload) + # As Linux sends it (rfcomm_make_uih): len << 1 | EA in one octet, or len << 1 as a 16-bit + # little-endian value in two. 130 bytes is [0x04, 0x01]. + length = bytes([(n << 1) | 1]) if n < 128 else (n << 1).to_bytes(2, "little") + return bytes([(dlci << 2) | 0x3, control]) + length + (bytes([credit]) if credit is not None else b"") \ + + payload + b"\x9A" + + +class ExtractTest(unittest.TestCase): + + def setUp(self): + s = Snoop() + # BLE: the Shimmer3R, and another device that must be left out. + s.le_connected(0x40, S3R) + s.le_connected(0x41, OTHER) + s.acl(False, 0x40, bt.L2CAP_ATT, b"\x02" + struct.pack("