From ef77412f0c9acba76dcb7c0dd8741a39c4fbf032 Mon Sep 17 00:00:00 2001 From: JongChern Date: Tue, 20 Oct 2026 14:06:21 +0800 Subject: [PATCH 1/4] HIL: save each run's Bluetooth bytes as a replayable trace (-CaptureBytes) A HIL finding is only fixed for good once its bytes are a byte-level test. With -CaptureBytes the runner notes the phone's clock around each run, takes one bug report at the end (the only way adb can read the HCI snoop log on an unrooted phone), keeps btsnoop_hci.log, deletes the bug report from the PC and the phone, and cuts out each run's bytes by the unit's MAC and time window. btsnoop_to_trace.py writes shimmer-logandstream's trace format (tests/data/*.bytes.log): BLE ATT writes and notifications, and Classic RFCOMM payloads as the radio delivered them, with L2CAP fragments reassembled. It is standard-library Python with unit tests on a synthetic snoop log. Capturing is diagnostic only and never changes a verdict. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + tools/hil/README.md | 51 +++++- tools/hil/btsnoop_to_trace.py | 282 +++++++++++++++++++++++++++++ tools/hil/run-android-hil.ps1 | 79 +++++++- tools/hil/test_btsnoop_to_trace.py | 129 +++++++++++++ 5 files changed, 538 insertions(+), 4 deletions(-) create mode 100644 tools/hil/btsnoop_to_trace.py create mode 100644 tools/hil/test_btsnoop_to_trace.py diff --git a/.gitignore b/.gitignore index 7edd5ce6..30472aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,4 @@ ShimmerAndroidInstrumentDriver/.idea/gradle.xml # Android HIL (tools/hil): the local rig file and run output tools/hil/rig-android.json tools/hil/out/ +tools/hil/__pycache__/ diff --git a/tools/hil/README.md b/tools/hil/README.md index 9aa46d80..a95539f7 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -86,6 +86,7 @@ since Android uses those unchanged. | Link CRC | ask for 2 bytes (the driver may settle on 1 or off) | `-Crc off`, `-Crc 1` | | Pause between runs | 10 s | `-SettleSeconds 60` | | Record the result for the release gate | no | `-Report` (full run on a clean, pushed commit only) | +| Save the bytes exchanged with each unit | no | `-CaptureBytes` (see [Capturing the bytes](#capturing-the-bytes)) | ## One-time setup @@ -130,11 +131,57 @@ another phone is connected to the same unit. The run's folder, `tools/hil/out//`, holds: - `summary.json` and the table the script prints; - `reports/`: per scenario and unit, a JSON report (versions, CRC mode actually in use, loss, - rate, accel, parser errors, a timeline of events and the failures) and, for `streamRest`, a CSV of - every sample; + rate, accel, parser errors, a timeline of events and the failures) and, for `streamRest` and + `midStreamSet`, a CSV of every sample; - `*.logcat.txt` and `*.instrument.txt`: the phone's log and the test runner's output for each run; +- with `-CaptureBytes`, `*.bytes.log` for each run and `btsnoop/`, the phone's snoop log they came from; - `build.log`. +## Capturing the bytes + +When a run finds something, the bytes that caused it are what a fix needs: replayed, they turn the +finding into a byte-level test that runs anywhere, with no phone or unit. With `-CaptureBytes`, the +script saves each run's bytes as `__.bytes.log`, in the trace format +shimmer-logandstream's replay tests read (`tests/data/*.bytes.log`, loaded by `load_session` in +`tests/common/mod.rs`): + +``` +14:33:05.120 TX h17 [0x07] +14:33:05.163 RX h14 [0xFF 0x00 0x12 0x34 ...] +``` + +TX is phone to unit, RX unit to phone. Over BLE, each line is one ATT write or notification, and +`h` is its attribute handle. Over Classic, each line is one RFCOMM frame's payload and `r` its +channel. Those are kept as the radio delivered them, not split into packets, so a bug that depends +on how bytes arrive replays the same way. The write that turns notifications on is left out, as in +the recorded traces. + +**One-time setup:** +- On the phone: Developer options → **Enable Bluetooth HCI snoop log**. Choose "Enabled", not + "Filtered", if it asks. Then turn Bluetooth off and on. adb cannot switch this on itself. +- Python 3 on the PATH. + +**How it works:** the script notes the phone's clock before and after each run. After the last run +it takes one bug report (about 3 minutes), because that is the only way adb can read the snoop log +on a phone that isn't rooted. It keeps `btsnoop_hci.log` and deletes the bug report, which holds the +phone's whole state, from the PC and the phone. Then `btsnoop_to_trace.py` cuts out each run's bytes, +by the unit's MAC and the run's time window. None of this changes a verdict: if it fails, the script +says so and carries on. + +**Limits:** +- Android keeps only the current snoop log and the one before it, so a long session can lose its + first runs. Capture a few runs at a time, with `-Unit` and `-Scenario`. +- `btsnoop/` holds all the phone's Bluetooth traffic, not only the unit's. It stays in the run's + folder, which is gitignored. + +To extract by hand, or from a snoop log taken some other way: + +```powershell +python tools\hil\btsnoop_to_trace.py btsnoop_hci.log --mac EC:06:D8:0D:86:F8 --utc-offset-min 480 --out run.bytes.log +``` + +Its tests run without a phone: `cd tools\hil; python -m unittest test_btsnoop_to_trace`. + ## Running a single test by hand ```powershell diff --git a/tools/hil/btsnoop_to_trace.py b/tools/hil/btsnoop_to_trace.py new file mode 100644 index 00000000..edc3f9d6 --- /dev/null +++ b/tools/hil/btsnoop_to_trace.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +Turns an Android Bluetooth HCI snoop log (btsnoop_hci.log) into the byte-trace format that +shimmer-logandstream's replay tests read (tests/data/*.bytes.log): one line per write to the unit +(TX) or notification from it (RX), with its time of day. + + HH:MM:SS.mmm TX h [0x.. 0x..] BLE: ATT write command or request + HH:MM:SS.mmm RX h [0x.. 0x..] BLE: ATT notification + HH:MM:SS.mmm TX r [0x.. 0x..] Classic: RFCOMM payload + HH:MM:SS.mmm RX r [0x.. 0x..] + +Classic payloads are kept as the radio delivered them, chunk by chunk, so a parser bug that +depends on how bytes arrive (a bulk read splitting a packet) replays the same way. + +Standard library only. Usage: + + python btsnoop_to_trace.py btsnoop_hci.log --mac EC:06:D8:0D:86:F8 --out run.bytes.log + [--from-ms EPOCH_MS] [--to-ms EPOCH_MS] [--utc-offset-min 480] [--title TEXT] +""" +import argparse +import datetime +import struct +import sys + +# btsnoop counts microseconds from 0000-01-01; this is 1970-01-01 on that scale. +BTSNOOP_EPOCH_DELTA_US = 0x00DCDDB30F2F8000 + +H4_COMMAND, H4_ACL, H4_SCO, H4_EVENT = 1, 2, 3, 4 +L2CAP_SIGNALLING, L2CAP_ATT = 0x0001, 0x0004 +PSM_RFCOMM = 0x0003 +ATT_WRITE_REQ, ATT_NOTIFY, ATT_WRITE_CMD = 0x12, 0x1B, 0x52 +ATT_MTU_REQ, ATT_MTU_RSP = 0x02, 0x03 +RFCOMM_UIH = 0xEF + + +class Record: + __slots__ = ("time_us", "received", "data") + + def __init__(self, time_us, received, data): + self.time_us = time_us # Unix time, microseconds + self.received = received # True: controller to host (from the unit) + self.data = data + + +def read_btsnoop(path): + with open(path, "rb") as f: + header = f.read(16) + if len(header) < 16 or header[:8] != b"btsnoop\0": + raise ValueError(f"{path}: not a btsnoop file") + _version, datalink = struct.unpack(">II", header[8:16]) + if datalink != 1002: + raise ValueError(f"{path}: datalink {datalink}, expected 1002 (HCI UART, H4)") + while True: + head = f.read(24) + if len(head) < 24: + return + _orig_len, incl_len, flags, _drops, ts = struct.unpack(">IIIIq", head) + data = f.read(incl_len) + if len(data) < incl_len: + return + yield Record(ts - BTSNOOP_EPOCH_DELTA_US, bool(flags & 1), data) + + +def mac_from_le(b): + return ":".join(f"{x:02X}" for x in reversed(b)) + + +class Link: + """One ACL connection: its address, and how to read the data on it.""" + + def __init__(self, handle, mac, le): + self.handle = handle + self.mac = mac + self.le = le + self.rfcomm_cids = set() + self.pending_rfcomm = {} # signalling id -> source CID of a PSM 3 connection request + self.reassembly = {} # direction -> bytearray of an L2CAP PDU in progress + self.mtu = None + + +class Extractor: + def __init__(self, mac=None): + self.mac = mac.upper() if mac else None + self.links = {} # ACL handle -> Link (current connection on that handle) + self.entries = [] # (time_us, tx, token, payload) + self.connections = [] # (time_us, mac, le, handle) + + def feed(self, rec): + if not rec.data: + return + kind = rec.data[0] + if kind == H4_EVENT: + self._event(rec) + elif kind == H4_ACL: + self._acl(rec) + + def _event(self, rec): + d = rec.data + if len(d) < 3: + return + code, params = d[1], d[3:] + if code == 0x03 and len(params) >= 9 and params[0] == 0: # Connection Complete (BR/EDR) + handle = struct.unpack_from("= 12: + if params[1] == 0: # LE Connection Complete / Enhanced Connection Complete + handle = struct.unpack_from("= 3: # Disconnection Complete + handle = struct.unpack_from("> 12) & 0x3 + link = self.links.get(handle) + if link is None: + # A connection made before the log started: its address is unknown. + link = self.links[handle] = Link(handle, None, None) + payload = d[5:5 + length] + key = rec.received + if pb == 0x1: # continuing fragment + buf = link.reassembly.get(key) + if buf is None: + return + buf.extend(payload) + else: # start of an L2CAP PDU + buf = link.reassembly[key] = bytearray(payload) + if len(buf) < 4: + return + l2len, cid = struct.unpack_from("= 3: + mtu = struct.unpack_from("= 4: # Connection Request + psm, scid = struct.unpack_from("= 8: # Connection Response + dcid, scid, result = struct.unpack_from("> 2 + control = pdu[1] + if dlci == 0 or (control & ~0x10) != RFCOMM_UIH: + return # the multiplexer's control channel, or not a data frame + if pdu[2] & 1: + length, i = pdu[2] >> 1, 3 + else: + length, i = (pdu[2] >> 1) | (pdu[3] << 7), 4 + if control & 0x10: + i += 1 # a credit byte (credit-based flow control) precedes the data + payload = pdu[i:i + length] + if payload: + self.entries.append((rec.time_us, not rec.received, f"r{dlci}", payload, link)) + + +def drop_cccd_writes(entries): + """A write that turns notifications on (to the descriptor after a notifying handle) is GATT + housekeeping, not a command to the unit; the recorded traces leave it out.""" + notify_handles = {int(e[2][1:]) for e in entries if not e[1] and e[2].startswith("h")} + return [e for e in entries + if not (e[1] and e[2].startswith("h") and int(e[2][1:]) - 1 in notify_handles + and len(e[3]) == 2 and e[3][1] == 0 and e[3][0] in (0, 1, 2))] + + +def time_of_day(time_us, utc_offset_min): + t = datetime.datetime.fromtimestamp(time_us / 1e6, datetime.timezone.utc) + t += datetime.timedelta(minutes=utc_offset_min) + return t.strftime("%H:%M:%S.") + f"{t.microsecond // 1000:03d}" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("log", nargs="+", help="btsnoop_hci.log (older file first, if it rotated)") + ap.add_argument("--mac", help="only this unit's connection") + ap.add_argument("--from-ms", type=int, help="Unix time, ms: drop what came before") + ap.add_argument("--to-ms", type=int, help="Unix time, ms: drop what came after") + ap.add_argument("--utc-offset-min", type=int, default=0, help="the phone's offset, for times of day") + ap.add_argument("--title", default="", help="first comment line") + ap.add_argument("--out", required=True) + args = ap.parse_args(argv) + + ex = Extractor(args.mac) + first = last = None + for path in args.log: + for rec in read_btsnoop(path): + first = rec.time_us if first is None else first + last = rec.time_us + ex.feed(rec) + lo = args.from_ms * 1000 if args.from_ms is not None else None + hi = args.to_ms * 1000 if args.to_ms is not None else None + entries = [e for e in drop_cccd_writes(ex.entries) + if (lo is None or e[0] >= lo) and (hi is None or e[0] <= hi)] + + with open(args.out, "w", encoding="utf-8", newline="\n") as out: + if args.title: + out.write(f"# {args.title}\n") + out.write("# Extracted from the phone's Bluetooth HCI snoop log by tools/hil/btsnoop_to_trace.py.\n") + out.write("# TX = phone to unit, RX = unit to phone; h is a BLE ATT handle, r an RFCOMM channel (DLCI).\n") + out.write("# Classic payloads are as the radio delivered them, not split into packets.\n") + seen = set() + for time_us, mac, le, handle in ex.connections: + if (args.mac is None or mac == args.mac.upper()) and (lo is None or time_us >= lo - 30_000_000) \ + and (hi is None or time_us <= hi): + link_mtu = next((e[4].mtu for e in entries if e[4].handle == handle and e[4].mac == mac), None) + key = (time_us, handle) + if key not in seen: + seen.add(key) + mtu = f" mtu={link_mtu}" if link_mtu else "" + out.write(f"{time_of_day(time_us, args.utc_offset_min)} connected {mac} " + f"over {'BLE' if le else 'Classic'} handle={handle}{mtu}\n") + for time_us, tx, token, payload, _link in entries: + data = " ".join(f"0x{b:02X}" for b in payload) + out.write(f"{time_of_day(time_us, args.utc_offset_min)} {'TX' if tx else 'RX'} {token} [{data}]\n") + + tx = [e for e in entries if e[1]] + rx = [e for e in entries if not e[1]] + span = "" + if first is not None: + span = (f"; the log covers {time_of_day(first, args.utc_offset_min)}" + f" to {time_of_day(last, args.utc_offset_min)}") + print(f"{args.out}: {len(tx)} TX ({sum(len(e[3]) for e in tx)} bytes), " + f"{len(rx)} RX ({sum(len(e[3]) for e in rx)} bytes){span}") + return 0 if entries else 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index d060245c..54764bab 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -19,6 +19,11 @@ The published .aar, as a customer app gets it. .EXAMPLE .\tools\hil\run-android-hil.ps1 -Scenario midStreamSet -Unit s3r-15bc -Crc 1 +.EXAMPLE +.\tools\hil\run-android-hil.ps1 -Unit s3r-15bc -Scenario midStreamSet -CaptureBytes +Also saves every byte exchanged with the unit, per run, as a trace shimmer-logandstream's tests can +replay. Needs the phone's Bluetooth HCI snoop log on (see the README) and Python 3. + .EXAMPLE .\tools\hil\run-android-hil.ps1 -Report Before "Release Android API": the full run on this commit, recorded on GitHub as the commit status @@ -44,7 +49,10 @@ param( [int]$ScenarioTimeoutSeconds = 600, # Pause between runs, so the unit has advertised again and the phone's Bluetooth stack has let go # of the last connection before the next one starts. - [int]$SettleSeconds = 10 + [int]$SettleSeconds = 10, + # Save the bytes exchanged with each unit, from the phone's Bluetooth HCI snoop log, as + # __.bytes.log beside each run's logcat. Adds a bug report (~3 min). + [switch]$CaptureBytes ) $ErrorActionPreference = 'Continue' @@ -98,6 +106,25 @@ if (-not $Serial -and $phones.Count -ne 1) { } $phoneModel = (Adb shell getprop ro.product.model) -join '' $phoneAndroid = (Adb shell getprop ro.build.version.release) -join '' +function PhoneMs { [long](((Adb shell date +%s%3N) -join '').Trim()) } + +if ($CaptureBytes) { + # "filtered" keeps only the start of each data packet, which is no use for replay. + $snoopMode = ((Adb shell getprop persist.bluetooth.btsnooplogmode) -join '').Trim() + $snoopOld = ((Adb shell getprop persist.bluetooth.btsnoopenable) -join '').Trim() + if ($snoopMode -ne 'full' -and -not (-not $snoopMode -and $snoopOld -eq 'true')) { + Fail ("-CaptureBytes needs the phone's full Bluetooth HCI snoop log (persist.bluetooth.btsnooplogmode is " + + "'$snoopMode'). In Developer options turn on 'Enable Bluetooth HCI snoop log' (choose 'Enabled', not " + + "'Filtered', where it asks), then turn Bluetooth off and on.") + } + $python = @(Get-Command python, python3 -CommandType Application -ErrorAction SilentlyContinue) | Select-Object -First 1 + # Windows' "python" can be a Store shortcut that runs nothing, so check that it runs Python 3. + if ($python) { & $python.Source -c "import sys; sys.exit(sys.version_info[0] != 3)" 2>$null | Out-Null } + if (-not $python -or $LASTEXITCODE -ne 0) { Fail '-CaptureBytes needs Python 3 on the PATH, to extract the bytes from the snoop log.' } + $z = ((Adb shell date +%z) -join '').Trim() # e.g. +0800 + $phoneUtcOffsetMin = [int]($z.Substring(0, 1) + '1') * ([int]$z.Substring(1, 2) * 60 + [int]$z.Substring(3, 2)) +} +$windows = @{} # --- Versions ---------------------------------------------------------------------------------- if (-not $DriverVersion) { @@ -270,9 +297,11 @@ foreach ($target in $targets) { '-e', 'crc', $Crc, '-e', 'maxLossPercent', $MaxLossPercent, '-e', 'driverVersion', $DriverVersion, '-e', 'androidApi', ($apiLabel -replace ' ', '_'), $runner) + $startMs = if ($CaptureBytes) { PhoneMs } else { 0 } $run = Invoke-Instrument $instrumentArgs $ScenarioTimeoutSeconds $output = $run.Output $safe = "$($u.name)_$($t)_$s" -replace '[^\w.-]', '_' + if ($CaptureBytes) { $windows[$safe] = @{ From = $startMs; To = (PhoneMs); Mac = $u.mac; Label = $label } } $output | Set-Content (Join-Path $outDir "$safe.instrument.txt") Adb logcat -d -v threadtime | Set-Content (Join-Path $outDir "$safe.logcat.txt") @@ -315,9 +344,55 @@ foreach ($r in $results) { $r.Detail = (("no report collected ($why). " + $r.Detail).Trim()) } } +# The bytes, from the phone's HCI snoop log. Diagnostic only: a problem here never changes a verdict. +if ($CaptureBytes) { + Write-Host 'Collecting the Bluetooth snoop log through a bug report (about 3 minutes)...' + $zip = Join-Path $outDir 'bugreport.zip' + $brOutput = & $adbExe @adbTarget bugreport $zip 2>&1 | Out-String + $snoopDir = Join-Path $outDir 'btsnoop' + $logs = @() + if (Test-Path $zip) { + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [IO.Compression.ZipFile]::OpenRead($zip) + try { + New-Item -ItemType Directory -Force $snoopDir | Out-Null + foreach ($entry in @($archive.Entries | Where-Object { $_.Name -match '^btsnoop_hci\.log(\.last)?$' })) { + $dest = Join-Path $snoopDir $entry.Name + [IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $dest, $true) + $logs += $dest + } + } + finally { $archive.Dispose() } + # A bug report holds the phone's whole state; keep only the snoop log, on the PC and the phone. + Remove-Item $zip + foreach ($onPhone in [regex]::Matches($brOutput, '(/data/user_de/\S+/bugreports/)(bugreport-[^\s/]+?)\.zip')) { + $dir = $onPhone.Groups[1].Value + $name = $onPhone.Groups[2].Value + Adb shell "rm -f $dir$name.zip $dir$name-dumpstate_log-*.txt" | Out-Null + } + } + # The rotated-out file (.last) is older, so it goes first. + $logs = @($logs | Sort-Object { if ($_ -like '*.last') { 0 } else { 1 } }) + if ($logs.Count -eq 0) { + Write-Host 'No btsnoop_hci.log in the bug report, so no bytes were saved. Is the snoop log on (Developer options)?' -ForegroundColor Yellow + } + else { + foreach ($safe in $windows.Keys) { + $w = $windows[$safe] + $title = "HIL run $($w.Label) on $phoneModel (Android $phoneAndroid), Android API $apiLabel, driver $DriverVersion" + & $python.Source (Join-Path $PSScriptRoot 'btsnoop_to_trace.py') @logs --mac $w.Mac --from-ms $w.From --to-ms $w.To ` + --utc-offset-min $phoneUtcOffsetMin --title $title --out (Join-Path $outDir "$safe.bytes.log") | Write-Host + if ($LASTEXITCODE -ne 0) { + Write-Host " no bytes for $($w.Label) in the snoop log (it may have rotated past this run)" -ForegroundColor Yellow + } + } + } +} + $summary = [pscustomobject]@{ phone = "$phoneModel (Android $phoneAndroid)"; androidApi = $apiLabel; driverVersion = $DriverVersion - rate = $Rate; seconds = $Seconds; crc = $Crc; settleSeconds = $SettleSeconds; results = $results + rate = $Rate; seconds = $Seconds; crc = $Crc; settleSeconds = $SettleSeconds; captureBytes = [bool]$CaptureBytes + results = $results } $summary | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $outDir 'summary.json') diff --git a/tools/hil/test_btsnoop_to_trace.py b/tools/hil/test_btsnoop_to_trace.py new file mode 100644 index 00000000..79e1faab --- /dev/null +++ b/tools/hil/test_btsnoop_to_trace.py @@ -0,0 +1,129 @@ +"""Checks btsnoop_to_trace.py on a synthetic snoop log. Run: python -m unittest test_btsnoop_to_trace""" +import os +import shutil +import struct +import tempfile +import unittest + +import btsnoop_to_trace as bt + +S3R, S3, OTHER = "EC:06:D8:0D:86:F8", "E8:EB:1B:71:3E:36", "11:22:33:44:55:66" +T0 = 1_792_476_000_000_000 # Unix time, us (2026-10-20 06:00 UTC) + + +def mac_le(mac): + return bytes(reversed(bytes.fromhex(mac.replace(":", "")))) + + +class Snoop: + def __init__(self): + self.records = [] + self.t = T0 + + def add(self, received, data, dt_ms=1): + self.t += dt_ms * 1000 + self.records.append((self.t, received, bytes(data))) + + def event(self, code, params): + self.add(True, bytes([bt.H4_EVENT, code, len(params)]) + params) + + def le_connected(self, handle, mac): + self.event(0x3E, bytes([0x01, 0x00]) + struct.pack("II", 1, 1002)) + for t, received, data in self.records: + f.write(struct.pack(">IIIIq", len(data), len(data), 1 if received else 0, 0, + t + bt.BTSNOOP_EPOCH_DELTA_US)) + f.write(data) + + +def rfcomm(dlci, payload, credit=None): + control = bt.RFCOMM_UIH | (0x10 if credit is not None else 0) + n = len(payload) + length = bytes([(n << 1) | 1]) if n < 128 else bytes([(n << 1) & 0xFE, n >> 7]) + return bytes([(dlci << 2) | 0x3, control]) + length + (bytes([credit]) if credit is not None else b"") \ + + payload + b"\x9A" + + +class ExtractTest(unittest.TestCase): + + def setUp(self): + s = Snoop() + # BLE: the Shimmer3R, and another device that must be left out. + s.le_connected(0x40, S3R) + s.le_connected(0x41, OTHER) + s.acl(False, 0x40, bt.L2CAP_ATT, b"\x02" + struct.pack(" Date: Tue, 20 Oct 2026 15:10:56 +0800 Subject: [PATCH 2/4] HIL capture: the snoop log is in local time; check the mode that's in force Found on the first real capture (Xiaomi, Android 11): - Android stamps btsnoop_hci.log in the phone's local time, so every run's window missed its bytes by the UTC offset. The extractor now takes the log as local time by default (--snoop-clock utc for one that isn't). - adb cannot read persist.bluetooth.btsnooplogmode on Android 11, so the runner asks the Bluetooth service which mode it started with (mSnoopLogSettingAtEnable), which is also the one actually in force. - summary.json keeps each run's capture window and the phone's UTC offset, so the bytes can be cut out of btsnoop/ again later. Verified on a Shimmer3R over BLE: every sample the API delivered in streamRest (7607) and midStreamSet (5273) is a CRC-valid packet in the captured bytes, in the same order. Co-Authored-By: Claude Opus 5.5 --- tools/hil/README.md | 16 ++++++++++++---- tools/hil/btsnoop_to_trace.py | 19 ++++++++++++++----- tools/hil/run-android-hil.ps1 | 19 +++++++++++++------ tools/hil/test_btsnoop_to_trace.py | 15 +++++++++++++-- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/tools/hil/README.md b/tools/hil/README.md index a95539f7..a3f1ed09 100644 --- a/tools/hil/README.md +++ b/tools/hil/README.md @@ -158,15 +158,21 @@ the recorded traces. **One-time setup:** - On the phone: Developer options → **Enable Bluetooth HCI snoop log**. Choose "Enabled", not - "Filtered", if it asks. Then turn Bluetooth off and on. adb cannot switch this on itself. + "Filtered", if it asks. +- **Then turn Bluetooth off and on.** The log only starts when Bluetooth does. adb can do neither step. + The script checks with the Bluetooth service (`mSnoopLogSettingAtEnable` in + `adb shell dumpsys bluetooth_manager`), and refuses to start until it reports `full`. - Python 3 on the PATH. **How it works:** the script notes the phone's clock before and after each run. After the last run it takes one bug report (about 3 minutes), because that is the only way adb can read the snoop log on a phone that isn't rooted. It keeps `btsnoop_hci.log` and deletes the bug report, which holds the phone's whole state, from the PC and the phone. Then `btsnoop_to_trace.py` cuts out each run's bytes, -by the unit's MAC and the run's time window. None of this changes a verdict: if it fails, the script -says so and carries on. +by the unit's MAC and the run's time window. Android stamps the snoop log in the phone's local time, +so the script passes the phone's UTC offset along. Each run's window and that offset are saved in +`summary.json` (`captureWindows`, `phoneUtcOffsetMin`), so the bytes can be cut out again later. + +None of this changes a verdict: if it fails, the script says so and carries on. **Limits:** - Android keeps only the current snoop log and the one before it, so a long session can lose its @@ -180,7 +186,9 @@ To extract by hand, or from a snoop log taken some other way: python tools\hil\btsnoop_to_trace.py btsnoop_hci.log --mac EC:06:D8:0D:86:F8 --utc-offset-min 480 --out run.bytes.log ``` -Its tests run without a phone: `cd tools\hil; python -m unittest test_btsnoop_to_trace`. +Leave out `--mac` to keep every device's traffic; add `--from-ms` and `--to-ms` (Unix time, from +`captureWindows`) to keep one run. Its tests run without a phone: +`cd tools\hil; python -m unittest test_btsnoop_to_trace`. ## Running a single test by hand diff --git a/tools/hil/btsnoop_to_trace.py b/tools/hil/btsnoop_to_trace.py index edc3f9d6..5df89b3d 100644 --- a/tools/hil/btsnoop_to_trace.py +++ b/tools/hil/btsnoop_to_trace.py @@ -15,7 +15,11 @@ Standard library only. Usage: python btsnoop_to_trace.py btsnoop_hci.log --mac EC:06:D8:0D:86:F8 --out run.bytes.log - [--from-ms EPOCH_MS] [--to-ms EPOCH_MS] [--utc-offset-min 480] [--title TEXT] + [--from-ms EPOCH_MS] [--to-ms EPOCH_MS] [--utc-offset-min 480] [--snoop-clock local|utc] + [--title TEXT] + +--utc-offset-min is the phone's offset from UTC. Android writes the snoop log in local time, so +it is needed both to place the --from-ms/--to-ms window and to print times of day. """ import argparse import datetime @@ -42,7 +46,9 @@ def __init__(self, time_us, received, data): self.data = data -def read_btsnoop(path): +def read_btsnoop(path, clock_offset_us=0): + """clock_offset_us: how far the log's clock is ahead of UTC. Android stamps its snoop log in + the phone's local time, so on a phone at UTC+8 it is 8 hours.""" with open(path, "rb") as f: header = f.read(16) if len(header) < 16 or header[:8] != b"btsnoop\0": @@ -58,7 +64,7 @@ def read_btsnoop(path): data = f.read(incl_len) if len(data) < incl_len: return - yield Record(ts - BTSNOOP_EPOCH_DELTA_US, bool(flags & 1), data) + yield Record(ts - BTSNOOP_EPOCH_DELTA_US - clock_offset_us, bool(flags & 1), data) def mac_from_le(b): @@ -229,15 +235,18 @@ def main(argv=None): ap.add_argument("--mac", help="only this unit's connection") ap.add_argument("--from-ms", type=int, help="Unix time, ms: drop what came before") ap.add_argument("--to-ms", type=int, help="Unix time, ms: drop what came after") - ap.add_argument("--utc-offset-min", type=int, default=0, help="the phone's offset, for times of day") + ap.add_argument("--utc-offset-min", type=int, default=0, help="the phone's offset from UTC, in minutes") + ap.add_argument("--snoop-clock", choices=("local", "utc"), default="local", + help="the clock the log is stamped in (Android: local)") ap.add_argument("--title", default="", help="first comment line") ap.add_argument("--out", required=True) args = ap.parse_args(argv) + clock_offset_us = args.utc_offset_min * 60_000_000 if args.snoop_clock == "local" else 0 ex = Extractor(args.mac) first = last = None for path in args.log: - for rec in read_btsnoop(path): + for rec in read_btsnoop(path, clock_offset_us): first = rec.time_us if first is None else first last = rec.time_us ex.feed(rec) diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index 54764bab..0e4299cb 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -109,12 +109,14 @@ $phoneAndroid = (Adb shell getprop ro.build.version.release) -join '' function PhoneMs { [long](((Adb shell date +%s%3N) -join '').Trim()) } if ($CaptureBytes) { - # "filtered" keeps only the start of each data packet, which is no use for replay. - $snoopMode = ((Adb shell getprop persist.bluetooth.btsnooplogmode) -join '').Trim() - $snoopOld = ((Adb shell getprop persist.bluetooth.btsnoopenable) -join '').Trim() - if ($snoopMode -ne 'full' -and -not (-not $snoopMode -and $snoopOld -eq 'true')) { - Fail ("-CaptureBytes needs the phone's full Bluetooth HCI snoop log (persist.bluetooth.btsnooplogmode is " + - "'$snoopMode'). In Developer options turn on 'Enable Bluetooth HCI snoop log' (choose 'Enabled', not " + + # The Bluetooth service reports the snoop mode it started with, which is the one in force: the + # setting only takes effect when Bluetooth next starts. (adb can't read the setting itself on + # Android 11.) "filtered" keeps only the start of each data packet, which is no use for replay. + $snoopLine = @(Adb shell dumpsys bluetooth_manager | Select-String 'mSnoopLogSettingAtEnable = (\S+)') | Select-Object -First 1 + $snoopMode = if ($snoopLine) { $snoopLine.Matches[0].Groups[1].Value } else { ((Adb shell getprop persist.bluetooth.btsnooplogmode) -join '').Trim() } + if ($snoopMode -ne 'full') { + Fail ("-CaptureBytes needs the phone's full Bluetooth HCI snoop log, and Bluetooth started with it in mode " + + "'$snoopMode'. In Developer options turn on 'Enable Bluetooth HCI snoop log' (choose 'Enabled', not " + "'Filtered', where it asks), then turn Bluetooth off and on.") } $python = @(Get-Command python, python3 -CommandType Application -ErrorAction SilentlyContinue) | Select-Object -First 1 @@ -394,6 +396,11 @@ $summary = [pscustomobject]@{ rate = $Rate; seconds = $Seconds; crc = $Crc; settleSeconds = $SettleSeconds; captureBytes = [bool]$CaptureBytes results = $results } +if ($CaptureBytes) { + # Each run's window on the phone's clock, to cut its bytes out of btsnoop/ again later. + $summary | Add-Member captureWindows ([pscustomobject]$windows) + $summary | Add-Member phoneUtcOffsetMin $phoneUtcOffsetMin +} $summary | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $outDir 'summary.json') Write-Host '' diff --git a/tools/hil/test_btsnoop_to_trace.py b/tools/hil/test_btsnoop_to_trace.py index 79e1faab..44682ad6 100644 --- a/tools/hil/test_btsnoop_to_trace.py +++ b/tools/hil/test_btsnoop_to_trace.py @@ -9,6 +9,7 @@ S3R, S3, OTHER = "EC:06:D8:0D:86:F8", "E8:EB:1B:71:3E:36", "11:22:33:44:55:66" T0 = 1_792_476_000_000_000 # Unix time, us (2026-10-20 06:00 UTC) +PHONE_UTC_OFFSET_US = 8 * 3600 * 1_000_000 # a phone at UTC+8 def mac_le(mac): @@ -40,12 +41,13 @@ def acl(self, received, handle, cid, payload, split=None): pb = 0x2 if n == 0 else 0x1 self.add(received, bytes([bt.H4_ACL]) + struct.pack("II", 1, 1002)) for t, received, data in self.records: f.write(struct.pack(">IIIIq", len(data), len(data), 1 if received else 0, 0, - t + bt.BTSNOOP_EPOCH_DELTA_US)) + t + bt.BTSNOOP_EPOCH_DELTA_US + clock_offset_us)) f.write(data) @@ -82,6 +84,7 @@ def setUp(self): self.log = os.path.join(self.tmp, "btsnoop_hci.log") s.write(self.log) self.t = s.records + self.snoop = s def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) @@ -103,6 +106,14 @@ def test_ble_session_of_one_unit(self): # Times of day are the phone's: 06:00 UTC + 8 h. self.assertTrue(lines[-1].startswith("14:00:00.")) + def test_log_stamped_in_utc(self): + self.snoop.write(self.log, clock_offset_us=0) + rx_time_ms = self.t[6][0] // 1000 + _, lines = self.run_extract("--mac", S3R, "--snoop-clock", "utc", "--from-ms", str(rx_time_ms)) + rx = [l for l in lines if " RX " in l] + self.assertEqual(1, len(rx)) + self.assertTrue(rx[0].startswith("14:00:00.")) + def test_classic_session_keeps_payloads_and_drops_control(self): _, lines = self.run_extract("--mac", S3) data = [l[14:] for l in lines if " TX " in l or " RX " in l] From 532cc8b43d3e266682883885fcd092aece496a62 Mon Sep 17 00:00:00 2001 From: JongChern Date: Tue, 20 Oct 2026 15:25:55 +0800 Subject: [PATCH 3/4] DEV-1251 HIL capture: per-connection CCCD filter; delete an unreadable bug report - The notification-on write is matched only against notifications on the same connection: ATT handles belong to one connection, so after a reconnect a real 2-byte command could be dropped (new test; it fails on the old filter). - If the bug report can't be read, say so and still delete it from the PC and the phone; it holds the phone's whole state. - The time-window test now checks both ends at the record that completes the notification. - summary.json's extra fields use Add-Member's named parameters (the positional form worked; this is for readability). Co-Authored-By: Claude Opus 5.5 --- tools/hil/btsnoop_to_trace.py | 7 +++--- tools/hil/run-android-hil.ps1 | 40 +++++++++++++++++++----------- tools/hil/test_btsnoop_to_trace.py | 23 +++++++++++++++-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/tools/hil/btsnoop_to_trace.py b/tools/hil/btsnoop_to_trace.py index 5df89b3d..7d735836 100644 --- a/tools/hil/btsnoop_to_trace.py +++ b/tools/hil/btsnoop_to_trace.py @@ -216,10 +216,11 @@ def _rfcomm(self, rec, link, pdu): def drop_cccd_writes(entries): """A write that turns notifications on (to the descriptor after a notifying handle) is GATT - housekeeping, not a command to the unit; the recorded traces leave it out.""" - notify_handles = {int(e[2][1:]) for e in entries if not e[1] and e[2].startswith("h")} + housekeeping, not a command to the unit; the recorded traces leave it out. Handles belong to one + connection, so a write is only matched against notifications on the same one.""" + notify_handles = {(id(e[4]), int(e[2][1:])) for e in entries if not e[1] and e[2].startswith("h")} return [e for e in entries - if not (e[1] and e[2].startswith("h") and int(e[2][1:]) - 1 in notify_handles + if not (e[1] and e[2].startswith("h") and (id(e[4]), int(e[2][1:]) - 1) in notify_handles and len(e[3]) == 2 and e[3][1] == 0 and e[3][0] in (0, 1, 2))] diff --git a/tools/hil/run-android-hil.ps1 b/tools/hil/run-android-hil.ps1 index 0e4299cb..e80321c6 100644 --- a/tools/hil/run-android-hil.ps1 +++ b/tools/hil/run-android-hil.ps1 @@ -353,20 +353,32 @@ if ($CaptureBytes) { $brOutput = & $adbExe @adbTarget bugreport $zip 2>&1 | Out-String $snoopDir = Join-Path $outDir 'btsnoop' $logs = @() - if (Test-Path $zip) { - Add-Type -AssemblyName System.IO.Compression.FileSystem - $archive = [IO.Compression.ZipFile]::OpenRead($zip) - try { - New-Item -ItemType Directory -Force $snoopDir | Out-Null - foreach ($entry in @($archive.Entries | Where-Object { $_.Name -match '^btsnoop_hci\.log(\.last)?$' })) { - $dest = Join-Path $snoopDir $entry.Name - [IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $dest, $true) - $logs += $dest + try { + if (Test-Path $zip) { + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [IO.Compression.ZipFile]::OpenRead($zip) + try { + New-Item -ItemType Directory -Force $snoopDir | Out-Null + foreach ($entry in @($archive.Entries | Where-Object { $_.Name -match '^btsnoop_hci\.log(\.last)?$' })) { + $dest = Join-Path $snoopDir $entry.Name + [IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $dest, $true) + $logs += $dest + } } + finally { $archive.Dispose() } + } + else { + Write-Host "The bug report did not arrive: $($brOutput.Trim())" -ForegroundColor Yellow } - finally { $archive.Dispose() } - # A bug report holds the phone's whole state; keep only the snoop log, on the PC and the phone. - Remove-Item $zip + } + catch { + Write-Host "Could not read the bug report: $($_.Exception.Message)" -ForegroundColor Yellow + $logs = @() + } + finally { + # A bug report holds the phone's whole state; keep only the snoop log, on the PC and the + # phone, whether or not reading it worked. + Remove-Item $zip -ErrorAction SilentlyContinue foreach ($onPhone in [regex]::Matches($brOutput, '(/data/user_de/\S+/bugreports/)(bugreport-[^\s/]+?)\.zip')) { $dir = $onPhone.Groups[1].Value $name = $onPhone.Groups[2].Value @@ -398,8 +410,8 @@ $summary = [pscustomobject]@{ } if ($CaptureBytes) { # Each run's window on the phone's clock, to cut its bytes out of btsnoop/ again later. - $summary | Add-Member captureWindows ([pscustomobject]$windows) - $summary | Add-Member phoneUtcOffsetMin $phoneUtcOffsetMin + $summary | Add-Member -MemberType NoteProperty -Name captureWindows -Value ([pscustomobject]$windows) + $summary | Add-Member -MemberType NoteProperty -Name phoneUtcOffsetMin -Value $phoneUtcOffsetMin } $summary | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $outDir 'summary.json') diff --git a/tools/hil/test_btsnoop_to_trace.py b/tools/hil/test_btsnoop_to_trace.py index 44682ad6..7cc22847 100644 --- a/tools/hil/test_btsnoop_to_trace.py +++ b/tools/hil/test_btsnoop_to_trace.py @@ -108,7 +108,7 @@ def test_ble_session_of_one_unit(self): def test_log_stamped_in_utc(self): self.snoop.write(self.log, clock_offset_us=0) - rx_time_ms = self.t[6][0] // 1000 + rx_time_ms = self.t[7][0] // 1000 _, lines = self.run_extract("--mac", S3R, "--snoop-clock", "utc", "--from-ms", str(rx_time_ms)) rx = [l for l in lines if " RX " in l] self.assertEqual(1, len(rx)) @@ -121,10 +121,29 @@ def test_classic_session_keeps_payloads_and_drops_control(self): "RX r2 [" + " ".join(f"0x{b:02X}" for b in range(130)) + "]"], data) def test_time_window(self): - rx_time_ms = self.t[6][0] // 1000 # the second fragment of the BLE notification + # A notification counts at the record that completes it: its second fragment, record 7. The + # command before it is record 5, 2 ms earlier. Both ends of the window are inclusive. + rx_time_ms = self.t[7][0] // 1000 _, lines = self.run_extract("--mac", S3R, "--from-ms", str(rx_time_ms)) self.assertEqual(1, sum(1 for l in lines if " RX " in l)) self.assertEqual(0, sum(1 for l in lines if " TX " in l)) + _, lines = self.run_extract("--mac", S3R, "--to-ms", str(rx_time_ms - 1)) + self.assertEqual(0, sum(1 for l in lines if " RX " in l)) + self.assertEqual(1, sum(1 for l in lines if " TX " in l)) + + def test_notification_write_filter_is_per_connection(self): + # The unit reconnects on the same ACL handle; on the new connection a 2-byte command to + # h15 must not be taken for the notification-on write of the old connection's h14. + s = Snoop() + s.le_connected(0x40, S3R) + s.acl(True, 0x40, bt.L2CAP_ATT, b"\x1B" + struct.pack(" Date: Tue, 20 Oct 2026 15:32:50 +0800 Subject: [PATCH 4/4] DEV-1251 HIL capture: cite the RFCOMM length encoding A review read the second length octet as carrying an EA bit. It doesn't: Linux sends a long length as len << 1, little-endian over two octets (__len16), so 130 is [0x04, 0x01] and the high octet is len >> 7 whole, as the extractor decodes it. Comments now say so, and the test builds the length the way Linux does. No change in behaviour. Co-Authored-By: Claude Opus 5.5 --- tools/hil/btsnoop_to_trace.py | 3 +++ tools/hil/test_btsnoop_to_trace.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/hil/btsnoop_to_trace.py b/tools/hil/btsnoop_to_trace.py index 7d735836..b112ff77 100644 --- a/tools/hil/btsnoop_to_trace.py +++ b/tools/hil/btsnoop_to_trace.py @@ -203,6 +203,9 @@ def _rfcomm(self, rec, link, pdu): control = pdu[1] if dlci == 0 or (control & ~0x10) != RFCOMM_UIH: return # the multiplexer's control channel, or not a data frame + # TS 07.10 length: one octet (EA = 1) up to 127, otherwise two octets holding len << 1 as a + # little-endian 16-bit value. Only the first octet carries the EA bit; the second is len >> 7 + # whole (Linux: __len16 in net/bluetooth/rfcomm/core.c; Android: RFCOMM_PARSE_LEN_FIELD). if pdu[2] & 1: length, i = pdu[2] >> 1, 3 else: diff --git a/tools/hil/test_btsnoop_to_trace.py b/tools/hil/test_btsnoop_to_trace.py index 7cc22847..27a054c2 100644 --- a/tools/hil/test_btsnoop_to_trace.py +++ b/tools/hil/test_btsnoop_to_trace.py @@ -54,7 +54,9 @@ def write(self, path, clock_offset_us=PHONE_UTC_OFFSET_US): def rfcomm(dlci, payload, credit=None): control = bt.RFCOMM_UIH | (0x10 if credit is not None else 0) n = len(payload) - length = bytes([(n << 1) | 1]) if n < 128 else bytes([(n << 1) & 0xFE, n >> 7]) + # As Linux sends it (rfcomm_make_uih): len << 1 | EA in one octet, or len << 1 as a 16-bit + # little-endian value in two. 130 bytes is [0x04, 0x01]. + length = bytes([(n << 1) | 1]) if n < 128 else (n << 1).to_bytes(2, "little") return bytes([(dlci << 2) | 0x3, control]) + length + (bytes([credit]) if credit is not None else b"") \ + payload + b"\x9A"