From 3fa458a4a3fdad61d9aa8ab6c9faf85627b839e4 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 11:30:39 +0200 Subject: [PATCH 01/47] build: create script for PGO builds --- scripts/pgo/README.md | 40 ++++++++++++++ scripts/pgo/android-local-proof.sh | 83 ++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 scripts/pgo/README.md create mode 100755 scripts/pgo/android-local-proof.sh diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md new file mode 100644 index 000000000000..96bf6281f7b9 --- /dev/null +++ b/scripts/pgo/README.md @@ -0,0 +1,40 @@ +# Local Android PGO proof + +This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the project's `release` variant: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. + +## Preconditions + +- A physical arm64 Android device connected through `adb`. +- `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. +- A signed-in, seeded test account and the agreed manual NewDot journey. +- Dependencies have had the repository's patches applied. + +## Collect one profile + +1. Build and install the instrumented application: + + ```bash + scripts/pgo/android-local-proof.sh build-instrumented + adb install -r Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release.apk + ``` + +2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. +3. Persist the counters before force-stopping the application: + + ```bash + scripts/pgo/android-local-proof.sh dump + scripts/pgo/android-local-proof.sh pull + scripts/pgo/android-local-proof.sh merge + ``` + +4. Build the matched optimized application: + + ```bash + scripts/pgo/android-local-proof.sh build-optimized + ``` + +The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. + +## Compare + +Install the non-PGO and PGO APKs separately. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh new file mode 100755 index 000000000000..cf28b86b0138 --- /dev/null +++ b/scripts/pgo/android-local-proof.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euo pipefail + +readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" +readonly PACKAGE_NAME="org.me.mobiexpensifyg" +readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly BUILD_VARIANT="Release" + +function usage() { + echo "Usage: $0 {build-instrumented|dump|pull|merge|build-optimized}" +} + +function ndk_tool() { + local tool_name="$1" + local ndk_root="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" + if [[ -z "$ndk_root" ]]; then + echo "Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build." >&2 + exit 1 + fi + + local host_tag="darwin-x86_64" + if [[ "$(uname -s)" == "Linux" ]]; then + host_tag="linux-x86_64" + fi + echo "$ndk_root/toolchains/llvm/prebuilt/$host_tag/bin/$tool_name" +} + +function gradle() { + ( + cd "$ANDROID_DIR" + ./gradlew "$@" + ) +} + +case "${1:-}" in + build-instrumented) + gradle ":assemble$BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=generate + ;; + dump) + adb shell am broadcast \ + -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$PACKAGE_NAME/.PgoProfileReceiver" + ;; + pull) + rm -rf "$PROFILE_DIR/raw" + mkdir -p "$PROFILE_DIR/raw" + adb exec-out run-as "$PACKAGE_NAME" tar -C cache -cf - . | tar -C "$PROFILE_DIR/raw" -xf - + find "$PROFILE_DIR/raw" -name '*.profraw' -print + ;; + merge) + mkdir -p "$PROFILE_DIR" + profiles=() + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found. Run dump and pull first." >&2 + exit 1 + fi + "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" + "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" + ;; + build-optimized) + if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then + echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 + exit 1 + fi + gradle ":app:assemble$BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=use \ + -PpgoProfile="$PROFILE_DIR/newdot.profdata" + ;; + *) + usage + exit 1 + ;; +esac From 3295d3865472f07a4650f028202682e87f2ac8d6 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 11:30:55 +0200 Subject: [PATCH 02/47] build: configure PGO build --- android/app/build.gradle | 11 ++ android/app/src/main/AndroidManifest.xml | 8 ++ .../expensify/chat/PgoProfileReceiver.java | 27 +++++ .../com/expensify/chat/PgoProfileWriter.java | 19 +++ .../android/CMakeLists.txt | 25 +++- .../ExpensifyNitroUtils/android/build.gradle | 19 ++- .../android/src/main/cpp/PgoProfileWriter.cpp | 53 +++++++++ patches/react-native/details.md | 7 ++ ....85.3+040+add-local-pgo-build-switch.patch | 112 ++++++++++++++++++ 9 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 android/app/src/main/java/com/expensify/chat/PgoProfileReceiver.java create mode 100644 android/app/src/main/java/com/expensify/chat/PgoProfileWriter.java create mode 100644 modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp create mode 100644 patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch diff --git a/android/app/build.gradle b/android/app/build.gradle index 319f661b7eab..2eb448716d3b 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -81,6 +81,16 @@ project.ext.envConfigFiles = [ * Set this to true to Run Proguard on Release builds to minify the Java bytecode. */ def enableProguardInReleaseBuilds = true +def pgoMode = providers.gradleProperty('pgoMode').getOrElse('off') +def pgoProfile = providers.gradleProperty('pgoProfile').orNull + +if (!(pgoMode in ['off', 'generate', 'use'])) { + throw new GradleException("Unsupported pgoMode '${pgoMode}'. Use off, generate, or use.") +} + +if (pgoMode == 'use' && !pgoProfile) { + throw new GradleException('pgoProfile is required when pgoMode=use.') +} /** * The preferred build flavor of JavaScriptCore (JSC) @@ -116,6 +126,7 @@ android { // Supported language variants must be declared here to avoid from being removed during the compilation. // This also helps us to not include unnecessary language variants in the APK. resConfigs "en", "es" + buildConfigField 'boolean', 'PGO_PROFILE_GENERATION', (pgoMode == 'generate').toString() } flavorDimensions "default" diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 2cf668b50b25..d2bbf6ad3914 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -27,6 +27,14 @@ android:theme="@style/AppTheme" tools:replace="android:supportsRtl"> + + + + + + +#include + +#include +#include + +namespace { +using SetFilename = int (*)(const char *); +using WriteFile = int (*)(); + +constexpr std::array kInstrumentedLibraries = { + "libhermesvm.so", + "libreactnative.so", + "libjsi.so", + "libExpensifyNitroUtils.so", +}; + +bool writeProfile(void *handle, const std::string &profilePath) { + const auto setFilename = reinterpret_cast(dlsym(handle, "__llvm_profile_set_filename")); + const auto writeFile = reinterpret_cast(dlsym(handle, "__llvm_profile_write_file")); + if (setFilename == nullptr || writeFile == nullptr) { + return false; + } + + return setFilename(profilePath.c_str()) == 0 && writeFile() == 0; +} +} // namespace + +extern "C" JNIEXPORT jint JNICALL +Java_com_expensify_chat_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { + const char *directoryChars = env->GetStringUTFChars(directory, nullptr); + if (directoryChars == nullptr) { + return 0; + } + + const std::string profilePath = std::string(directoryChars) + "/newdot-%m.profraw"; + env->ReleaseStringUTFChars(directory, directoryChars); + + jint writtenProfiles = 0; + for (const char *library : kInstrumentedLibraries) { + void *handle = dlopen(library, RTLD_NOW | RTLD_NOLOAD); + if (handle == nullptr) { + continue; + } + + if (writeProfile(handle, profilePath)) { + ++writtenProfiles; + } + dlclose(handle); + } + + return writtenProfiles; +} diff --git a/patches/react-native/details.md b/patches/react-native/details.md index f59d0a5ec3d0..0fcb8da72a13 100644 --- a/patches/react-native/details.md +++ b/patches/react-native/details.md @@ -1,5 +1,12 @@ # `react-native` patches +### [react-native+0.85.3+040+add-local-pgo-build-switch.patch](react-native+0.85.3+040+add-local-pgo-build-switch.patch) + +- Reason: Adds an opt-in local LLVM PGO mode to the React Native Android runtime and Hermes source builds. The mode is disabled by default and is used only by the local PGO proof tooling. +- Upstream PR/issue: 🛑 +- E/App issue: 🛑 +- PR Introducing Patch: 🛑 + ### [react-native+0.85.3+001+initial.patch](react-native+0.85.3+001+initial.patch) - Reason: Fixes keyboard flickering issue when opening/closing modals. When an input is blurred and a modal is opened, the `rootView` becomes the `firstResponder`, causing the system to retain an incorrect keyboard state. This leads to keyboard flickering when the modal is closed. The patch adds code to resign the `rootView`'s `firstResponder` status before presenting the modal to prevent this issue. diff --git a/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch new file mode 100644 index 000000000000..57b7b6fe96df --- /dev/null +++ b/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch @@ -0,0 +1,112 @@ +diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modules/react-native/ReactAndroid/build.gradle.kts +--- a/node_modules/react-native/ReactAndroid/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/build.gradle.kts +@@ -40,6 +40,15 @@ val reactNativeRootDir = projectDir.parent + val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} + + // We put the publishing version from gradle.properties inside ext. so other + // subprojects can access it as well. +@@ -579,6 +588,10 @@ android { + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW", ++ "-DPGO_MODE=$pgoMode", + ) ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } + + if (hermesV1Enabled) { +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +--- a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +@@ -52,6 +52,15 @@ val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} + val reactNativeRootDir = project(":packages:react-native:ReactAndroid").projectDir.parent + val customDownloadDir = System.getenv("REACT_NATIVE_DOWNLOADS_DIR") + val downloadsDir = +@@ -383,6 +392,11 @@ android { + // We intentionally build Hermes with Intl support only. This is to simplify + // the build setup and to avoid overcomplicating the build-type matrix. + "-DHERMES_ENABLE_INTL=True", ++ "-DPGO_MODE=$pgoMode", + ) ++ ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } + + if (hermesV1Enabled) { +diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +--- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt ++++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +@@ -8,6 +8,14 @@ set(CMAKE_VERBOSE_MAKEFILE on) + + project(ReactAndroid) ++ ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() + + include(CheckIPOSupported) + check_ipo_supported(RESULT IPO_SUPPORT) +diff --git a/node_modules/react-native/sdks/hermes/CMakeLists.txt b/node_modules/react-native/sdks/hermes/CMakeLists.txt +--- a/node_modules/react-native/sdks/hermes/CMakeLists.txt ++++ b/node_modules/react-native/sdks/hermes/CMakeLists.txt +@@ -476,6 +476,12 @@ if (HERMES_ENABLE_CODE_COVERAGE) + endif() + endif() ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ append("-fprofile-generate" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) ++ append("-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -Wl,--export-dynamic-symbol=__llvm_profile_write_file" CMAKE_SHARED_LINKER_FLAGS) ++elseif(PGO_MODE STREQUAL "use") ++ append("-fprofile-use=${PGO_PROFILE}" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) ++endif() + if(HERMES_FACEBOOK_BUILD) + add_definitions(-DHERMES_FACEBOOK_BUILD) + endif() From e1747b547df47c7f5e2bfd90d3fdb4b6b6900894 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 11:47:28 +0200 Subject: [PATCH 03/47] build: add `verify-instrumented` tool --- scripts/pgo/android-local-proof.sh | 56 ++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index cf28b86b0138..e84932dcbabd 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -2,14 +2,19 @@ set -euo pipefail +readonly NDK_VERSION="27.1.12297006" +readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" + readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly PACKAGE_NAME="org.me.mobiexpensifyg" -readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly BUILD_VARIANT="Release" +readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" +readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" + function usage() { - echo "Usage: $0 {build-instrumented|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -34,6 +39,48 @@ function gradle() { ) } +function verify_pgo_instrumentation() ( + local apk_path="${1:-$APK_PATH}" + if [[ ! -f "$apk_path" ]]; then + echo "Missing APK at $apk_path. Build the instrumented release first." >&2 + exit 1 + fi + + local extracted_dir + extracted_dir="$(mktemp -d "${TMPDIR:-/tmp}/expensify-pgo-apk.XXXXXX")" + trap 'rm -rf "$extracted_dir"' EXIT + + local llvm_readelf + if ! llvm_readelf="$(ndk_tool llvm-readelf)"; then + exit 1 + fi + + local libraries=( + libreactnative.so + libhermesvm.so + libjsi.so + libExpensifyNitroUtils.so + ) + local library + for library in "${libraries[@]}"; do + local apk_entry="lib/arm64-v8a/$library" + if ! zipinfo -1 "$apk_path" | rg -Fx "$apk_entry" >/dev/null; then + echo "Missing expected arm64 library in APK: $apk_entry" >&2 + exit 1 + fi + + local extracted_library="$extracted_dir/$library" + unzip -p "$apk_path" "$apk_entry" > "$extracted_library" + local section_headers + section_headers="$("$llvm_readelf" -SW "$extracted_library")" + if [[ "$section_headers" != *"__llvm_prf_data"* || "$section_headers" != *"__llvm_prf_cnts"* || "$section_headers" != *"__llvm_prf_names"* ]]; then + echo "LLVM PGO instrumentation is missing from $apk_entry." >&2 + exit 1 + fi + echo "Verified LLVM PGO instrumentation: $apk_entry" + done +) + case "${1:-}" in build-instrumented) gradle ":assemble$BUILD_VARIANT" \ @@ -41,6 +88,9 @@ case "${1:-}" in -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate ;; + verify-instrumented) + verify_pgo_instrumentation + ;; dump) adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ From 96518ea7e01c7f12537005ab5a113bfd4926b771 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 12:30:04 +0200 Subject: [PATCH 04/47] chore: update RN patch --- ...atch => react-native+0.85.3+040+pgo.patch} | 91 ++++++++++++------- 1 file changed, 56 insertions(+), 35 deletions(-) rename patches/react-native/{react-native+0.85.3+040+add-local-pgo-build-switch.patch => react-native+0.85.3+040+pgo.patch} (68%) diff --git a/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.85.3+040+pgo.patch similarity index 68% rename from patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch rename to patches/react-native/react-native+0.85.3+040+pgo.patch index 57b7b6fe96df..7132cc63a984 100644 --- a/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch +++ b/patches/react-native/react-native+0.85.3+040+pgo.patch @@ -1,9 +1,11 @@ diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modules/react-native/ReactAndroid/build.gradle.kts +index bae034d..94d11e1 100644 --- a/node_modules/react-native/ReactAndroid/build.gradle.kts +++ b/node_modules/react-native/ReactAndroid/build.gradle.kts -@@ -40,6 +40,15 @@ val reactNativeRootDir = projectDir.parent +@@ -42,6 +42,16 @@ val reactNativeRootDir = projectDir.parent val hermesV1Enabled = rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() + +val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" +val pgoProfile = project.findProperty("pgoProfile")?.toString() + @@ -13,26 +15,36 @@ diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modu +if (pgoMode == "use") { + check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } +} - ++ // We put the publishing version from gradle.properties inside ext. so other // subprojects can access it as well. -@@ -579,6 +588,10 @@ android { + extra["publishing_version"] = project.findProperty("VERSION_NAME")?.toString()!! +@@ -579,8 +589,13 @@ android { + "-DANDROID_TOOLCHAIN=clang", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW", + "-DPGO_MODE=$pgoMode", ) + + if (pgoMode == "use") { + arguments("-DPGO_PROFILE=$pgoProfile") + } - ++ if (hermesV1Enabled) { + arguments("-DHERMES_V1_ENABLED=1") + } diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +index 6610782..853b937 100644 --- a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +++ b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts -@@ -52,6 +52,15 @@ val hermesV1Enabled = +@@ -52,6 +52,18 @@ fun getSDKManagerPath(): String { + + val hermesV1Enabled = rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++ +val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" +val pgoProfile = project.findProperty("pgoProfile")?.toString() ++val pgoCmakePath = File(projectDir, "pgo.cmake") + +check(pgoMode in setOf("off", "generate", "use")) { + "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." @@ -40,28 +52,60 @@ diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.k +if (pgoMode == "use") { + check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } +} ++ val reactNativeRootDir = project(":packages:react-native:ReactAndroid").projectDir.parent val customDownloadDir = System.getenv("REACT_NATIVE_DOWNLOADS_DIR") val downloadsDir = -@@ -383,6 +392,11 @@ android { +@@ -415,8 +427,14 @@ android { // We intentionally build Hermes with Intl support only. This is to simplify // the build setup and to avoid overcomplicating the build-type matrix. "-DHERMES_ENABLE_INTL=True", + "-DPGO_MODE=$pgoMode", ++ "-DCMAKE_PROJECT_Hermes_INCLUDE=${pgoCmakePath.absolutePath}", ) -+ + + if (pgoMode == "use") { + arguments("-DPGO_PROFILE=$pgoProfile") + } - ++ if (hermesV1Enabled) { + arguments("-DHERMESVM_HEAP_HV_MODE=HEAP_HV_PREFER32") + } +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +new file mode 100644 +index 0000000..bcc335d +--- /dev/null ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +@@ -0,0 +1,21 @@ ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +index 8125e02..d466798 100644 --- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt -@@ -8,6 +8,14 @@ set(CMAKE_VERBOSE_MAKEFILE on) +@@ -8,6 +8,28 @@ set(CMAKE_VERBOSE_MAKEFILE on) project(ReactAndroid) -+ + +if(PGO_MODE STREQUAL "generate") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir @@ -83,30 +127,7 @@ diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt + add_compile_options("-fprofile-use=${PGO_PROFILE}") + add_link_options("-fprofile-use=${PGO_PROFILE}") +endif() - ++ include(CheckIPOSupported) check_ipo_supported(RESULT IPO_SUPPORT) -diff --git a/node_modules/react-native/sdks/hermes/CMakeLists.txt b/node_modules/react-native/sdks/hermes/CMakeLists.txt ---- a/node_modules/react-native/sdks/hermes/CMakeLists.txt -+++ b/node_modules/react-native/sdks/hermes/CMakeLists.txt -@@ -476,6 +476,12 @@ if (HERMES_ENABLE_CODE_COVERAGE) - endif() - endif() -+if(PGO_MODE STREQUAL "generate") -+ execute_process( -+ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir -+ OUTPUT_VARIABLE CLANG_RESOURCE_DIR -+ OUTPUT_STRIP_TRAILING_WHITESPACE -+ ) -+ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") -+ if(NOT EXISTS "${PGO_RUNTIME}") -+ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") -+ endif() -+ append("-fprofile-generate" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) -+ append("-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -Wl,--export-dynamic-symbol=__llvm_profile_write_file" CMAKE_SHARED_LINKER_FLAGS) -+elseif(PGO_MODE STREQUAL "use") -+ append("-fprofile-use=${PGO_PROFILE}" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) -+endif() - if(HERMES_FACEBOOK_BUILD) - add_definitions(-DHERMES_FACEBOOK_BUILD) - endif() + if (IPO_SUPPORT) From 27c7035fae767217ccba087983192dbf63060817 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:36:36 +0200 Subject: [PATCH 05/47] chore: add `.pgo` dir to `.gitignore` --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e547a16b7ca4..b483094cf1da 100644 --- a/.gitignore +++ b/.gitignore @@ -134,6 +134,9 @@ config/rsbuild/*.pem # Rspack build profiling output (RSPACK_PROFILE=OVERVIEW|ALL) .rspack-profile-*/ +# Local native PGO profiles +.pgo/ + # Expo .expo dist/ From e8d11043bab2f9bbb9da76593c52562a5a489c9d Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:37:34 +0200 Subject: [PATCH 06/47] fix: PGO merging --- .../android/CMakeLists.txt | 8 ++++-- .../android/src/main/cpp/PgoProfileWriter.cpp | 11 ++++---- .../react-native+0.85.3+040+pgo.patch | 20 +++++++++---- scripts/pgo/android-local-proof.sh | 28 +++++++++++++++++-- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index 4fa4657d5931..0936f76325d7 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -19,8 +19,12 @@ if(EXPENSIFY_PGO_MODE STREQUAL "generate") add_link_options( -fprofile-generate "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" - -Wl,--export-dynamic-symbol=__llvm_profile_set_filename - -Wl,--export-dynamic-symbol=__llvm_profile_write_file + -Wl,--undefined=__llvm_profile_set_filename + -Wl,--undefined=__llvm_profile_write_file + -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename + -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file + -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename + -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file ) elseif(EXPENSIFY_PGO_MODE STREQUAL "use") add_compile_options("-fprofile-use=${EXPENSIFY_PGO_PROFILE}") diff --git a/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp index 32e7477301b4..d4a0357f7f2f 100644 --- a/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp +++ b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp @@ -5,7 +5,7 @@ #include namespace { -using SetFilename = int (*)(const char *); +using SetFilename = void (*)(const char *); using WriteFile = int (*)(); constexpr std::array kInstrumentedLibraries = { @@ -16,18 +16,19 @@ constexpr std::array kInstrumentedLibraries = { }; bool writeProfile(void *handle, const std::string &profilePath) { - const auto setFilename = reinterpret_cast(dlsym(handle, "__llvm_profile_set_filename")); - const auto writeFile = reinterpret_cast(dlsym(handle, "__llvm_profile_write_file")); + const auto setFilename = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_set_filename")); + const auto writeFile = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_write_file")); if (setFilename == nullptr || writeFile == nullptr) { return false; } - return setFilename(profilePath.c_str()) == 0 && writeFile() == 0; + setFilename(profilePath.c_str()); + return writeFile() == 0; } } // namespace extern "C" JNIEXPORT jint JNICALL -Java_com_expensify_chat_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { +Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { const char *directoryChars = env->GetStringUTFChars(directory, nullptr); if (directoryChars == nullptr) { return 0; diff --git a/patches/react-native/react-native+0.85.3+040+pgo.patch b/patches/react-native/react-native+0.85.3+040+pgo.patch index 7132cc63a984..14ca0c45eb91 100644 --- a/patches/react-native/react-native+0.85.3+040+pgo.patch +++ b/patches/react-native/react-native+0.85.3+040+pgo.patch @@ -76,7 +76,7 @@ new file mode 100644 index 0000000..bcc335d --- /dev/null +++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake -@@ -0,0 +1,21 @@ +@@ -0,0 +1,25 @@ +if(PGO_MODE STREQUAL "generate") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir @@ -91,8 +91,12 @@ index 0000000..bcc335d + add_link_options( + -fprofile-generate + "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" -+ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -+ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file + ) +elseif(PGO_MODE STREQUAL "use") + add_compile_options("-fprofile-use=${PGO_PROFILE}") @@ -102,7 +106,7 @@ diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt index 8125e02..d466798 100644 --- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt -@@ -8,6 +8,28 @@ set(CMAKE_VERBOSE_MAKEFILE on) +@@ -8,6 +8,32 @@ set(CMAKE_VERBOSE_MAKEFILE on) project(ReactAndroid) @@ -120,8 +124,12 @@ index 8125e02..d466798 100644 + add_link_options( + -fprofile-generate + "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" -+ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -+ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file + ) +elseif(PGO_MODE STREQUAL "use") + add_compile_options("-fprofile-use=${PGO_PROFILE}") diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index e84932dcbabd..56dc140520b4 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -12,6 +12,7 @@ readonly BUILD_VARIANT="Release" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" function usage() { echo "Usage: $0 {build-instrumented|verify-instrumented|dump|pull|merge|build-optimized}" @@ -77,6 +78,18 @@ function verify_pgo_instrumentation() ( echo "LLVM PGO instrumentation is missing from $apk_entry." >&2 exit 1 fi + + local dynamic_symbols + dynamic_symbols="$("$llvm_nm" -D --defined-only "$extracted_library")" + if [[ "$dynamic_symbols" != *"expensify_llvm_profile_set_filename"* || "$dynamic_symbols" != *"expensify_llvm_profile_write_file"* ]]; then + echo "LLVM PGO profile-writing APIs are not exported from $apk_entry." >&2 + exit 1 + fi + + if [[ "$library" == "libExpensifyNitroUtils.so" && "$dynamic_symbols" != *"Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles"* ]]; then + echo "The PGO JNI writer is missing from $apk_entry. Rebuild the native module before installing." >&2 + exit 1 + fi echo "Verified LLVM PGO instrumentation: $apk_entry" done ) @@ -99,8 +112,16 @@ case "${1:-}" in pull) rm -rf "$PROFILE_DIR/raw" mkdir -p "$PROFILE_DIR/raw" - adb exec-out run-as "$PACKAGE_NAME" tar -C cache -cf - . | tar -C "$PROFILE_DIR/raw" -xf - - find "$PROFILE_DIR/raw" -name '*.profraw' -print + adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" + profiles=() + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 + exit 1 + fi + printf '%s\n' "${profiles[@]}" ;; merge) mkdir -p "$PROFILE_DIR" @@ -114,13 +135,14 @@ case "${1:-}" in fi "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" + echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" ;; build-optimized) if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 exit 1 fi - gradle ":app:assemble$BUILD_VARIANT" \ + gradle ":assemble$BUILD_VARIANT" \ -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ From 2b114fdb3aa2ad5ac485132746a9b665b43e8140 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:38:02 +0200 Subject: [PATCH 07/47] build: add `install-instrumented` command --- scripts/pgo/android-local-proof.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 56dc140520b4..1f5bb99b59a5 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -15,7 +15,7 @@ readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|install|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -40,6 +40,15 @@ function gradle() { ) } +function install_apk() { + if [[ ! -f "$APK_PATH" ]]; then + echo "Missing APK at $APK_PATH. Build the APK first." >&2 + exit 1 + fi + + adb install -r "$APK_PATH" +} + function verify_pgo_instrumentation() ( local apk_path="${1:-$APK_PATH}" if [[ ! -f "$apk_path" ]]; then @@ -104,6 +113,9 @@ case "${1:-}" in verify-instrumented) verify_pgo_instrumentation ;; + install) + install_apk + ;; dump) adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ From 9322dba115d31c188998ab60e1c3b25de4b462a7 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:38:11 +0200 Subject: [PATCH 08/47] Update README.md --- scripts/pgo/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 96bf6281f7b9..2c53d2e6fac9 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -15,7 +15,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android ```bash scripts/pgo/android-local-proof.sh build-instrumented - adb install -r Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release.apk + scripts/pgo/android-local-proof.sh install ``` 2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. @@ -27,6 +27,8 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh merge ``` + Profiles are written to the app's external cache so they can be retrieved from a non-debuggable release APK with `adb pull`. + 4. Build the matched optimized application: ```bash From 47f9d5bcd9a3e3ffdd6ed175b58a3d9ed7fec4ee Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:38:14 +0200 Subject: [PATCH 09/47] Update android-local-proof.sh --- scripts/pgo/android-local-proof.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 1f5bb99b59a5..f11ea9b9eeb5 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -64,6 +64,10 @@ function verify_pgo_instrumentation() ( if ! llvm_readelf="$(ndk_tool llvm-readelf)"; then exit 1 fi + local llvm_nm + if ! llvm_nm="$(ndk_tool llvm-nm)"; then + exit 1 + fi local libraries=( libreactnative.so From 6ba45d606aef7076dfe485e859d03b2735d5144c Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:08:12 +0200 Subject: [PATCH 10/47] build: create record-startups command --- scripts/pgo/README.md | 18 +++++++++++++++ scripts/pgo/android-local-proof.sh | 36 +++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 2c53d2e6fac9..6c36f0375d85 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,6 +35,24 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` +### Collect a startup-focused profile + +To replace the manual journey with ten cold-process startup runs, then pull and merge the accumulated profiles automatically, run: + +```bash +scripts/pgo/android-local-proof.sh record-startups +``` + +The optional arguments set the number of runs and the fixed settling time after `am start -W` respectively: + +```bash +scripts/pgo/android-local-proof.sh record-startups 10 10 +``` + +The command clears only existing `newdot-*.profraw` files on the device before the first run. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. + +The settling time is currently a fallback because the release app does not expose a device-visible NewDot-ready signal. A benchmark-only build should emit a native Logcat marker and call Android's `reportFullyDrawn()` from `Expensify.tsx`'s `onSplashHide`, after the splash exit animation and startup gates have completed. Keep that measurement marker on the separate metrics branch. + The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. ## Compare diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index f11ea9b9eeb5..842e64ae9892 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -13,9 +13,10 @@ readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" +readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|install|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|install|record-startups [runs] [settle-seconds]|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -49,6 +50,36 @@ function install_apk() { adb install -r "$APK_PATH" } +function record_startups() { + local runs="${1:-10}" + local settle_seconds="${2:-10}" + + if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then + echo "Startup run count must be a positive integer, received: $runs" >&2 + exit 1 + fi + if [[ ! "$settle_seconds" =~ ^[0-9]+$ ]]; then + echo "Startup settle time must be a non-negative integer, received: $settle_seconds" >&2 + exit 1 + fi + + echo "Clearing previous device PGO profiles from $DEVICE_PROFILE_DIR." + adb shell "rm -f '$DEVICE_PROFILE_DIR'/newdot-*.profraw" + + local run + for ((run = 1; run <= runs; run++)); do + echo "Recording cold-process startup $run/$runs." + adb shell am force-stop "$PACKAGE_NAME" + adb shell am start -W -n "$START_ACTIVITY" + echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." + sleep "$settle_seconds" + "$0" dump + done + + "$0" pull + "$0" merge +} + function verify_pgo_instrumentation() ( local apk_path="${1:-$APK_PATH}" if [[ ! -f "$apk_path" ]]; then @@ -120,6 +151,9 @@ case "${1:-}" in install) install_apk ;; + record-startups) + record_startups "${2:-10}" "${3:-10}" + ;; dump) adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ From 62808e8d919a90a4a73308927cfda4f47d07c6c1 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:25:01 +0200 Subject: [PATCH 11/47] fix: prevent startup profile merge scope error --- scripts/pgo/android-local-proof.sh | 79 ++++++++++++++++++------------ 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 842e64ae9892..41c0ab08d3d8 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -50,6 +50,49 @@ function install_apk() { adb install -r "$APK_PATH" } +function dump_profiles() { + adb shell am broadcast \ + -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$PACKAGE_NAME/.PgoProfileReceiver" +} + +function pull_profiles() { + rm -rf "$PROFILE_DIR/raw" + mkdir -p "$PROFILE_DIR/raw" + adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" + + local -a profiles=() + local profile + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 + exit 1 + fi + printf '%s\n' "${profiles[@]}" +} + +function merge_profiles() { + mkdir -p "$PROFILE_DIR" + + local -a profiles=() + local profile + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found. Run dump and pull first." >&2 + exit 1 + fi + + "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" + "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" + echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" +} + function record_startups() { local runs="${1:-10}" local settle_seconds="${2:-10}" @@ -73,11 +116,11 @@ function record_startups() { adb shell am start -W -n "$START_ACTIVITY" echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." sleep "$settle_seconds" - "$0" dump + dump_profiles done - "$0" pull - "$0" merge + pull_profiles + merge_profiles } function verify_pgo_instrumentation() ( @@ -155,37 +198,13 @@ case "${1:-}" in record_startups "${2:-10}" "${3:-10}" ;; dump) - adb shell am broadcast \ - -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$PACKAGE_NAME/.PgoProfileReceiver" + dump_profiles ;; pull) - rm -rf "$PROFILE_DIR/raw" - mkdir -p "$PROFILE_DIR/raw" - adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" - profiles=() - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 - exit 1 - fi - printf '%s\n' "${profiles[@]}" + pull_profiles ;; merge) - mkdir -p "$PROFILE_DIR" - profiles=() - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found. Run dump and pull first." >&2 - exit 1 - fi - "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" - "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" - echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" + merge_profiles ;; build-optimized) if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then From 0da014af9d201d2b61bf984e3123a89299c9e4c9 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 16:12:37 +0200 Subject: [PATCH 12/47] build: add dedicated PGO build variants --- scripts/pgo/README.md | 10 +++- scripts/pgo/android-local-proof.sh | 83 ++++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 6c36f0375d85..2230e0576ce7 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,6 +1,6 @@ # Local Android PGO proof -This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the project's `release` variant: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the release-derived `pgoInstrumented` and `pgoOptimized` variants: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. ## Preconditions @@ -15,7 +15,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android ```bash scripts/pgo/android-local-proof.sh build-instrumented - scripts/pgo/android-local-proof.sh install + scripts/pgo/android-local-proof.sh install-instrumented ``` 2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. @@ -35,6 +35,12 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` + The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. Install the optimized APK with: + + ```bash + scripts/pgo/android-local-proof.sh install-optimized + ``` + ### Collect a startup-focused profile To replace the manual journey with ten cold-process startup runs, then pull and merge the accumulated profiles automatically, run: diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 41c0ab08d3d8..18bc7dc48da7 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -7,16 +7,24 @@ readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" readonly PACKAGE_NAME="org.me.mobiexpensifyg" -readonly BUILD_VARIANT="Release" +readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" +readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" -readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" +readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" +readonly APP_READY_LOG_TAG="NewDotStartup" +readonly APP_READY_LOG_MESSAGE="APP_READY" +readonly DEFAULT_STARTUP_RUNS=10 +readonly DEFAULT_APP_READY_TIMEOUT_SECONDS=30 +readonly STARTUP_RELAUNCH_DELAY_SECONDS=0.5 +readonly PROFILE_DUMP_TIMEOUT_SECONDS=5 function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|install|record-startups [runs] [settle-seconds]|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -41,19 +49,71 @@ function gradle() { ) } +function build_instrumented() { + gradle ":assemble$INSTRUMENTED_BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=generate +} + +function build_optimized() { + if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then + echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 + exit 1 + fi + + gradle ":assemble$OPTIMIZED_BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=use \ + -PpgoProfile="$PROFILE_DIR/newdot.profdata" +} + function install_apk() { - if [[ ! -f "$APK_PATH" ]]; then - echo "Missing APK at $APK_PATH. Build the APK first." >&2 + local apk_path="$1" + if [[ ! -f "$apk_path" ]]; then + echo "Missing APK at $apk_path. Build the APK first." >&2 exit 1 fi - adb install -r "$APK_PATH" + adb install -r "$apk_path" } function dump_profiles() { + adb logcat -c adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ -n "$PACKAGE_NAME/.PgoProfileReceiver" + + wait_for_profile_dump +} + +function wait_for_profile_dump() { + local started_at="$SECONDS" + + while ((SECONDS - started_at < PROFILE_DUMP_TIMEOUT_SECONDS)); do + local profile_logs + profile_logs="$(adb logcat -d -s 'PgoProfileReceiver:I' '*:S')" + + if [[ "$profile_logs" =~ Wrote\ ([1-9][0-9]*)\ LLVM\ PGO\ profile ]]; then + echo "${BASH_REMATCH[0]}(s)." + return 0 + fi + if [[ "$profile_logs" == *"Ignoring PGO profile request in a non-instrumented build."* ]]; then + echo "The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files." >&2 + return 1 + fi + if [[ "$profile_logs" == *"Wrote 0 LLVM PGO profile(s)."* ]]; then + echo "The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles." >&2 + return 1 + fi + + sleep 0.1 + done + + echo "The PGO receiver did not confirm a profile write within ${PROFILE_DUMP_TIMEOUT_SECONDS}s." >&2 + adb logcat -d -s 'PgoProfileReceiver:I' '*:S' >&2 + return 1 } function pull_profiles() { @@ -113,6 +173,8 @@ function record_startups() { for ((run = 1; run <= runs; run++)); do echo "Recording cold-process startup $run/$runs." adb shell am force-stop "$PACKAGE_NAME" + sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" + adb logcat -c adb shell am start -W -n "$START_ACTIVITY" echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." sleep "$settle_seconds" @@ -124,7 +186,7 @@ function record_startups() { } function verify_pgo_instrumentation() ( - local apk_path="${1:-$APK_PATH}" + local apk_path="${1:-$INSTRUMENTED_APK_PATH}" if [[ ! -f "$apk_path" ]]; then echo "Missing APK at $apk_path. Build the instrumented release first." >&2 exit 1 @@ -191,8 +253,11 @@ case "${1:-}" in verify-instrumented) verify_pgo_instrumentation ;; - install) - install_apk + install | install-instrumented) + install_apk "$INSTRUMENTED_APK_PATH" + ;; + install-optimized) + install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) record_startups "${2:-10}" "${3:-10}" From ea4b7fb6631933b05198f71b42d0f7b254641be2 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:28:19 +0200 Subject: [PATCH 13/47] Wait for native app-ready signal during PGO startup recording --- Mobile-Expensify | 2 +- scripts/pgo/README.md | 8 +++---- scripts/pgo/android-local-proof.sh | 31 ++++++++++++++++++++----- src/Expensify.tsx | 2 ++ src/libs/BootSplash/types.ts | 1 + src/libs/StartupMetrics/index.native.ts | 7 ++++++ src/libs/StartupMetrics/index.ts | 3 +++ 7 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 src/libs/StartupMetrics/index.native.ts create mode 100644 src/libs/StartupMetrics/index.ts diff --git a/Mobile-Expensify b/Mobile-Expensify index 3d7ad2f66ba5..9c1320c6911b 160000 --- a/Mobile-Expensify +++ b/Mobile-Expensify @@ -1 +1 @@ -Subproject commit 3d7ad2f66ba5fe2ccb45c4d1b173fd41d922775d +Subproject commit 9c1320c6911b7106c7fdaf8648d3910609d99087 diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 2230e0576ce7..0739cd7c0cb8 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -49,15 +49,15 @@ To replace the manual journey with ten cold-process startup runs, then pull and scripts/pgo/android-local-proof.sh record-startups ``` -The optional arguments set the number of runs and the fixed settling time after `am start -W` respectively: +The optional arguments set the number of runs and the timeout for NewDot's native app-ready signal respectively: ```bash -scripts/pgo/android-local-proof.sh record-startups 10 10 +scripts/pgo/android-local-proof.sh record-startups 10 30 ``` -The command clears only existing `newdot-*.profraw` files on the device before the first run. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. +The command clears only existing `newdot-*.profraw` files on the device before the first run. It clears Logcat before each launch, waits for `NewDotStartup: APP_READY`, and fails rather than recording an incomplete startup if the marker does not appear before the timeout. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. -The settling time is currently a fallback because the release app does not expose a device-visible NewDot-ready signal. A benchmark-only build should emit a native Logcat marker and call Android's `reportFullyDrawn()` from `Expensify.tsx`'s `onSplashHide`, after the splash exit animation and startup gates have completed. Keep that measurement marker on the separate metrics branch. +The metrics build calls Android's `reportFullyDrawn()` and emits the native marker from `Expensify.tsx`'s `onSplashHide`, immediately after the `ManualAppStartup` Sentry span ends and after the splash exit animation and startup gates have completed. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 18bc7dc48da7..99e47e78bad8 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -155,14 +155,14 @@ function merge_profiles() { function record_startups() { local runs="${1:-10}" - local settle_seconds="${2:-10}" + local ready_timeout_seconds="${2:-30}" if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then echo "Startup run count must be a positive integer, received: $runs" >&2 exit 1 fi - if [[ ! "$settle_seconds" =~ ^[0-9]+$ ]]; then - echo "Startup settle time must be a non-negative integer, received: $settle_seconds" >&2 + if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 exit 1 fi @@ -176,8 +176,7 @@ function record_startups() { sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" adb logcat -c adb shell am start -W -n "$START_ACTIVITY" - echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." - sleep "$settle_seconds" + wait_for_app_ready "$ready_timeout_seconds" dump_profiles done @@ -185,6 +184,26 @@ function record_startups() { merge_profiles } +function wait_for_app_ready() { + local timeout_seconds="$1" + local started_at="$SECONDS" + + echo "Waiting up to ${timeout_seconds}s for $APP_READY_LOG_TAG: $APP_READY_LOG_MESSAGE." + while ((SECONDS - started_at < timeout_seconds)); do + local startup_logs + startup_logs="$(adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S')" + if [[ "$startup_logs" == *"$APP_READY_LOG_MESSAGE"* ]]; then + echo "NewDot reported APP_READY." + return 0 + fi + sleep 0.25 + done + + echo "NewDot did not report APP_READY within ${timeout_seconds}s." >&2 + adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S' >&2 + return 1 +} + function verify_pgo_instrumentation() ( local apk_path="${1:-$INSTRUMENTED_APK_PATH}" if [[ ! -f "$apk_path" ]]; then @@ -260,7 +279,7 @@ case "${1:-}" in install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) - record_startups "${2:-10}" "${3:-10}" + record_startups "${2:-10}" "${3:-30}" ;; dump) dump_profiles diff --git a/src/Expensify.tsx b/src/Expensify.tsx index 245c3a6a2c26..93ac0c72074c 100644 --- a/src/Expensify.tsx +++ b/src/Expensify.tsx @@ -34,6 +34,7 @@ import './libs/NetworkState'; import Navigation from './libs/Navigation/Navigation'; import NavigationRoot from './libs/Navigation/NavigationRoot'; import PushNotification from './libs/Notification/PushNotification'; +import StartupMetrics from './libs/StartupMetrics'; import {endSpan, getSpan, startSpan} from './libs/telemetry/activeSpans'; import {startBootsplashMonitor} from './libs/telemetry/bootsplashTelemetry'; import {cleanupTelemetryTrackers, initializeTelemetryTrackers} from './libs/telemetry/TelemetrySynchronizer'; @@ -208,6 +209,7 @@ function Expensify() { endSpan(CONST.TELEMETRY.SPAN_APP_STARTUP); endSpan(CONST.TELEMETRY.SPAN_BOOTSPLASH.ROOT); endSpan(CONST.TELEMETRY.SPAN_BOOTSPLASH.SPLASH_HIDER); + StartupMetrics.reportFullyDrawn(); }, [setSplashScreenState]); useLayoutEffect(() => { diff --git a/src/libs/BootSplash/types.ts b/src/libs/BootSplash/types.ts index 8fa41b2d1af2..3f20032d575b 100644 --- a/src/libs/BootSplash/types.ts +++ b/src/libs/BootSplash/types.ts @@ -2,6 +2,7 @@ type BootSplashModule = { logoSizeRatio: number; navigationBarHeight: number; hide: () => Promise; + reportFullyDrawn?: () => void; }; // eslint-disable-next-line import/prefer-default-export diff --git a/src/libs/StartupMetrics/index.native.ts b/src/libs/StartupMetrics/index.native.ts new file mode 100644 index 000000000000..b9c684246afa --- /dev/null +++ b/src/libs/StartupMetrics/index.native.ts @@ -0,0 +1,7 @@ +import {NativeModules} from 'react-native'; + +function reportFullyDrawn(): void { + NativeModules.BootSplash.reportFullyDrawn?.(); +} + +export default {reportFullyDrawn}; diff --git a/src/libs/StartupMetrics/index.ts b/src/libs/StartupMetrics/index.ts new file mode 100644 index 000000000000..f879a0dc9d99 --- /dev/null +++ b/src/libs/StartupMetrics/index.ts @@ -0,0 +1,3 @@ +const reportFullyDrawn = (): void => undefined; + +export default {reportFullyDrawn}; From 151f4f1f7f001782c2fe27fa81ec1b6333fc6e14 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:39:10 +0200 Subject: [PATCH 14/47] chore: improve script --- scripts/pgo/android-local-proof.sh | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 99e47e78bad8..47a437a6d2c7 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -154,8 +154,8 @@ function merge_profiles() { } function record_startups() { - local runs="${1:-10}" - local ready_timeout_seconds="${2:-30}" + local runs="${1:-$DEFAULT_STARTUP_RUNS}" + local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then echo "Startup run count must be a positive integer, received: $runs" >&2 @@ -264,10 +264,7 @@ function verify_pgo_instrumentation() ( case "${1:-}" in build-instrumented) - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=generate + build_instrumented ;; verify-instrumented) verify_pgo_instrumentation @@ -279,7 +276,7 @@ case "${1:-}" in install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) - record_startups "${2:-10}" "${3:-30}" + record_startups "${2:-}" "${3:-}" ;; dump) dump_profiles @@ -291,15 +288,7 @@ case "${1:-}" in merge_profiles ;; build-optimized) - if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then - echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 - exit 1 - fi - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=use \ - -PpgoProfile="$PROFILE_DIR/newdot.profdata" + build_optimized ;; *) usage From 192125505df87d12fa69105975c14ec6a512dc0c Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 16:27:40 +0200 Subject: [PATCH 15/47] chore: add PGO benchmarking suite --- scripts/pgo/README.md | 39 ++++++- scripts/pgo/android-local-proof.sh | 176 +++++++++++++++++++++++++++-- 2 files changed, 204 insertions(+), 11 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 0739cd7c0cb8..2fd8629fcf4d 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -57,7 +57,44 @@ scripts/pgo/android-local-proof.sh record-startups 10 30 The command clears only existing `newdot-*.profraw` files on the device before the first run. It clears Logcat before each launch, waits for `NewDotStartup: APP_READY`, and fails rather than recording an incomplete startup if the marker does not appear before the timeout. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. -The metrics build calls Android's `reportFullyDrawn()` and emits the native marker from `Expensify.tsx`'s `onSplashHide`, immediately after the `ManualAppStartup` Sentry span ends and after the splash exit animation and startup gates have completed. +The metrics build calls Android's `reportFullyDrawn()` and emits the machine-readable `NewDotStartup: APP_READY durationMs=` native marker from `Expensify.tsx`'s `onSplashHide`, immediately after the `ManualAppStartup` Sentry span ends and after the splash exit animation and startup gates have completed. The duration starts at the same native timestamp used by the Sentry span. + +## Benchmark startup + +Build the release and PGO-optimized APK from the same source revision, compiler, NDK, and ABI: + +```bash +scripts/pgo/android-local-proof.sh build-release +scripts/pgo/android-local-proof.sh build-optimized +``` + +Collect each benchmark independently. Each command installs its APK without clearing application data, performs one unmeasured warm-up, then records ten cold-process startup samples by default: + +```bash +scripts/pgo/android-local-proof.sh benchmark-release +scripts/pgo/android-local-proof.sh benchmark-optimized +``` + +The optional arguments select the measured run count and app-ready timeout: + +```bash +scripts/pgo/android-local-proof.sh benchmark-release 20 30 +scripts/pgo/android-local-proof.sh benchmark-optimized 20 30 +``` + +Compare previously collected samples: + +```bash +scripts/pgo/android-local-proof.sh compare-benchmarks +``` + +Or run the release benchmark first, then the optimized benchmark, and compare them in one command: + +```bash +scripts/pgo/android-local-proof.sh benchmark 10 30 +``` + +Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the mean, median, minimum, maximum, and percentage improvement. Positive improvement percentages mean the optimized build was faster. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 47a437a6d2c7..db00a15d658c 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -7,13 +7,18 @@ readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" readonly PACKAGE_NAME="org.me.mobiexpensifyg" +readonly RELEASE_BUILD_VARIANT="Release" readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" +readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" +readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" +readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" readonly APP_READY_LOG_TAG="NewDotStartup" @@ -23,8 +28,10 @@ readonly DEFAULT_APP_READY_TIMEOUT_SECONDS=30 readonly STARTUP_RELAUNCH_DELAY_SECONDS=0.5 readonly PROFILE_DUMP_TIMEOUT_SECONDS=5 +APP_READY_DURATION_MS="" + function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}" } function ndk_tool() { @@ -56,6 +63,13 @@ function build_instrumented() { -PpgoMode=generate } +function build_release() { + gradle ":assemble$RELEASE_BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=off +} + function build_optimized() { if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 @@ -157,14 +171,7 @@ function record_startups() { local runs="${1:-$DEFAULT_STARTUP_RUNS}" local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then - echo "Startup run count must be a positive integer, received: $runs" >&2 - exit 1 - fi - if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then - echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 - exit 1 - fi + validate_startup_arguments "$runs" "$ready_timeout_seconds" echo "Clearing previous device PGO profiles from $DEVICE_PROFILE_DIR." adb shell "rm -f '$DEVICE_PROFILE_DIR'/newdot-*.profraw" @@ -184,16 +191,147 @@ function record_startups() { merge_profiles } +function validate_startup_arguments() { + local runs="$1" + local ready_timeout_seconds="$2" + + if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then + echo "Startup run count must be a positive integer, received: $runs" >&2 + exit 1 + fi + if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 + exit 1 + fi +} + +function measure_startup() { + local ready_timeout_seconds="$1" + + adb shell am force-stop "$PACKAGE_NAME" + sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" + adb logcat -c + adb shell am start -W -n "$START_ACTIVITY" + wait_for_app_ready "$ready_timeout_seconds" + + if [[ -z "$APP_READY_DURATION_MS" ]]; then + echo "APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes." >&2 + return 1 + fi +} + +function benchmark_startups() { + local label="$1" + local apk_path="$2" + local benchmark_path="$3" + local runs="$4" + local ready_timeout_seconds="$5" + + validate_startup_arguments "$runs" "$ready_timeout_seconds" + install_apk "$apk_path" + mkdir -p "$BENCHMARK_DIR" + + echo "Running one unmeasured warm-up startup for $label." + measure_startup "$ready_timeout_seconds" + + printf 'run,duration_ms\n' > "$benchmark_path" + local run + for ((run = 1; run <= runs; run++)); do + echo "Benchmarking $label cold-process startup $run/$runs." + measure_startup "$ready_timeout_seconds" + printf '%d,%s\n' "$run" "$APP_READY_DURATION_MS" >> "$benchmark_path" + done + + echo "Recorded $runs $label startup samples: $benchmark_path" +} + +function benchmark_stats() { + local benchmark_path="$1" + + awk -F, ' + NR > 1 && $2 ~ /^[0-9]+$/ { + values[++count] = $2 + sum += $2 + } + END { + if (count == 0) { + exit 1 + } + for (i = 1; i <= count; i++) { + for (j = i + 1; j <= count; j++) { + if (values[i] > values[j]) { + temporary = values[i] + values[i] = values[j] + values[j] = temporary + } + } + } + if (count % 2 == 1) { + median = values[(count + 1) / 2] + } else { + median = (values[count / 2] + values[count / 2 + 1]) / 2 + } + printf "%d %.2f %.2f %d %d\n", count, sum / count, median, values[1], values[count] + } + ' "$benchmark_path" +} + +function compare_benchmarks() { + if [[ ! -f "$RELEASE_BENCHMARK_PATH" || ! -f "$OPTIMIZED_BENCHMARK_PATH" ]]; then + echo "Missing benchmark data. Run benchmark-release and benchmark-optimized first." >&2 + exit 1 + fi + + local release_count release_mean release_median release_min release_max + read -r release_count release_mean release_median release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") + local optimized_count optimized_mean optimized_median optimized_min optimized_max + read -r optimized_count optimized_mean optimized_median optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") + + local mean_improvement median_improvement + mean_improvement="$(awk -v release="$release_mean" -v optimized="$optimized_mean" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" + median_improvement="$(awk -v release="$release_median" -v optimized="$optimized_median" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" + + printf '%-18s %6s %10s %10s %8s %8s\n' 'Build' 'Runs' 'Mean ms' 'Median ms' 'Min ms' 'Max ms' + printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'Release' "$release_count" "$release_mean" "$release_median" "$release_min" "$release_max" + printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_mean" "$optimized_median" "$optimized_min" "$optimized_max" + echo "PGO mean startup improvement: ${mean_improvement}%" + echo "PGO median startup improvement: ${median_improvement}%" + echo "Positive percentages are faster; negative percentages are regressions." +} + +function benchmark_release() { + benchmark_startups "release" "$RELEASE_APK_PATH" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" +} + +function benchmark_optimized() { + benchmark_startups "PGO optimized" "$OPTIMIZED_APK_PATH" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" +} + +function benchmark_all() { + local runs="${1:-$DEFAULT_STARTUP_RUNS}" + local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" + + benchmark_release "$runs" "$ready_timeout_seconds" + benchmark_optimized "$runs" "$ready_timeout_seconds" + compare_benchmarks +} + function wait_for_app_ready() { local timeout_seconds="$1" local started_at="$SECONDS" + APP_READY_DURATION_MS="" echo "Waiting up to ${timeout_seconds}s for $APP_READY_LOG_TAG: $APP_READY_LOG_MESSAGE." while ((SECONDS - started_at < timeout_seconds)); do local startup_logs startup_logs="$(adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S')" if [[ "$startup_logs" == *"$APP_READY_LOG_MESSAGE"* ]]; then - echo "NewDot reported APP_READY." + if [[ "$startup_logs" =~ APP_READY\ durationMs=([0-9]+) ]]; then + APP_READY_DURATION_MS="${BASH_REMATCH[1]}" + echo "NewDot reported APP_READY after ${APP_READY_DURATION_MS}ms." + else + echo "NewDot reported APP_READY without a numeric duration marker." + fi return 0 fi sleep 0.25 @@ -263,6 +401,9 @@ function verify_pgo_instrumentation() ( ) case "${1:-}" in + build-release) + build_release + ;; build-instrumented) build_instrumented ;; @@ -272,6 +413,9 @@ case "${1:-}" in install | install-instrumented) install_apk "$INSTRUMENTED_APK_PATH" ;; + install-release) + install_apk "$RELEASE_APK_PATH" + ;; install-optimized) install_apk "$OPTIMIZED_APK_PATH" ;; @@ -290,6 +434,18 @@ case "${1:-}" in build-optimized) build_optimized ;; + benchmark-release) + benchmark_release "${2:-}" "${3:-}" + ;; + benchmark-optimized) + benchmark_optimized "${2:-}" "${3:-}" + ;; + benchmark) + benchmark_all "${2:-}" "${3:-}" + ;; + compare-benchmarks) + compare_benchmarks + ;; *) usage exit 1 From 13df28ae16f7f88ca78ea1929d3d6dcda457c78c Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 16:36:43 +0200 Subject: [PATCH 16/47] build: add dedicated PGO build variants --- scripts/pgo/README.md | 4 ++-- scripts/pgo/android-local-proof.sh | 35 ++++++++++++------------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 2230e0576ce7..b22567559abc 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. Install the optimized APK with: + The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. They use the application IDs `org.me.mobiexpensifyg.pgo.instrumented` and `org.me.mobiexpensifyg.pgo.optimized`, with the launcher names “Expensify PGO Instrumented” and “Expensify PGO Optimized”, so both can remain installed simultaneously. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -63,4 +63,4 @@ The `.pgo/` directory is intentionally local-only. Never apply this profile to a ## Compare -Install the non-PGO and PGO APKs separately. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. +Install the instrumented and optimized APKs side by side. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 18bc7dc48da7..1a7923df9cad 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -6,7 +6,9 @@ readonly NDK_VERSION="27.1.12297006" readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly PACKAGE_NAME="org.me.mobiexpensifyg" +readonly BASE_PACKAGE_NAME="org.me.mobiexpensifyg" +readonly INSTRUMENTED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.instrumented" +readonly OPTIMIZED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.optimized" readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" @@ -14,8 +16,8 @@ readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" -readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" -readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" +readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$INSTRUMENTED_PACKAGE_NAME/cache" +readonly START_ACTIVITY="$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.ExpensifyActivityBase" readonly APP_READY_LOG_TAG="NewDotStartup" readonly APP_READY_LOG_MESSAGE="APP_READY" readonly DEFAULT_STARTUP_RUNS=10 @@ -71,19 +73,21 @@ function build_optimized() { function install_apk() { local apk_path="$1" + local package_name="$2" if [[ ! -f "$apk_path" ]]; then echo "Missing APK at $apk_path. Build the APK first." >&2 exit 1 fi adb install -r "$apk_path" + echo "Installed $package_name." } function dump_profiles() { adb logcat -c adb shell am broadcast \ - -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$PACKAGE_NAME/.PgoProfileReceiver" + -a "$INSTRUMENTED_PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.PgoProfileReceiver" wait_for_profile_dump } @@ -172,7 +176,7 @@ function record_startups() { local run for ((run = 1; run <= runs; run++)); do echo "Recording cold-process startup $run/$runs." - adb shell am force-stop "$PACKAGE_NAME" + adb shell am force-stop "$INSTRUMENTED_PACKAGE_NAME" sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" adb logcat -c adb shell am start -W -n "$START_ACTIVITY" @@ -245,19 +249,16 @@ function verify_pgo_instrumentation() ( case "${1:-}" in build-instrumented) - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=generate + build_instrumented ;; verify-instrumented) verify_pgo_instrumentation ;; install | install-instrumented) - install_apk "$INSTRUMENTED_APK_PATH" + install_apk "$INSTRUMENTED_APK_PATH" "$INSTRUMENTED_PACKAGE_NAME" ;; install-optimized) - install_apk "$OPTIMIZED_APK_PATH" + install_apk "$OPTIMIZED_APK_PATH" "$OPTIMIZED_PACKAGE_NAME" ;; record-startups) record_startups "${2:-10}" "${3:-10}" @@ -272,15 +273,7 @@ case "${1:-}" in merge_profiles ;; build-optimized) - if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then - echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 - exit 1 - fi - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=use \ - -PpgoProfile="$PROFILE_DIR/newdot.profdata" + build_optimized ;; *) usage From 7bf73a043a20e2ea4be59f3aa5e474a5a176b728 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 17:45:35 +0200 Subject: [PATCH 17/47] revert: separate build variants for PGO --- scripts/pgo/README.md | 12 +++--- scripts/pgo/android-local-proof.sh | 64 +++++++++++++++++------------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 8049f1f44b64..46ec0f9d5546 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,6 +1,6 @@ # Local Android PGO proof -This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the release-derived `pgoInstrumented` and `pgoOptimized` variants: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 release build: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. ## Preconditions @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. They use the application IDs `org.me.mobiexpensifyg.pgo.instrumented` and `org.me.mobiexpensifyg.pgo.optimized`, with the launcher names “Expensify PGO Instrumented” and “Expensify PGO Optimized”, so both can remain installed simultaneously. Install the optimized APK with: + The script renames the instrumented and optimized release outputs to `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-instrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-optimized.apk`. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -61,11 +61,11 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the release and PGO-optimized APK from the same source revision, compiler, NDK, and ABI: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is renamed before the release APK is built: ```bash -scripts/pgo/android-local-proof.sh build-release scripts/pgo/android-local-proof.sh build-optimized +scripts/pgo/android-local-proof.sh build-release ``` Collect each benchmark independently. Each command installs its APK without clearing application data, performs one unmeasured warm-up, then records ten cold-process startup samples by default: @@ -88,7 +88,7 @@ Compare previously collected samples: scripts/pgo/android-local-proof.sh compare-benchmarks ``` -Or run the release benchmark first, then the optimized benchmark, and compare them in one command: +Or install and benchmark the release APK first, then install and benchmark the renamed optimized APK, and compare them in one command: ```bash scripts/pgo/android-local-proof.sh benchmark 10 30 @@ -100,4 +100,4 @@ The `.pgo/` directory is intentionally local-only. Never apply this profile to a ## Compare -Install the instrumented and optimized APKs side by side. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. +Install the release and optimized APKs consecutively. For each build, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 867137080be8..5f5e9922ae7f 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -6,23 +6,19 @@ readonly NDK_VERSION="27.1.12297006" readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly BASE_PACKAGE_NAME="org.me.mobiexpensifyg" -readonly INSTRUMENTED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.instrumented" -readonly OPTIMIZED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.optimized" +readonly PACKAGE_NAME="org.me.mobiexpensifyg" readonly RELEASE_BUILD_VARIANT="Release" -readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" -readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" -readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" -readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" +readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-instrumented.apk" +readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-optimized.apk" readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" -readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$INSTRUMENTED_PACKAGE_NAME/cache" -readonly START_ACTIVITY="$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.ExpensifyActivityBase" +readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" +readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" readonly APP_READY_LOG_TAG="NewDotStartup" readonly APP_READY_LOG_MESSAGE="APP_READY" readonly DEFAULT_STARTUP_RUNS=10 @@ -59,10 +55,11 @@ function gradle() { } function build_instrumented() { - gradle ":assemble$INSTRUMENTED_BUILD_VARIANT" \ + gradle ":assemble$RELEASE_BUILD_VARIANT" \ -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate + rename_release_apk "$INSTRUMENTED_APK_PATH" } function build_release() { @@ -78,30 +75,41 @@ function build_optimized() { exit 1 fi - gradle ":assemble$OPTIMIZED_BUILD_VARIANT" \ + gradle ":assemble$RELEASE_BUILD_VARIANT" \ -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ -PpgoProfile="$PROFILE_DIR/newdot.profdata" + rename_release_apk "$OPTIMIZED_APK_PATH" +} + +function rename_release_apk() { + local renamed_apk_path="$1" + if [[ ! -f "$RELEASE_APK_PATH" ]]; then + echo "Missing built release APK at $RELEASE_APK_PATH." >&2 + exit 1 + fi + + mv -f "$RELEASE_APK_PATH" "$renamed_apk_path" + echo "Renamed release APK: $renamed_apk_path" } function install_apk() { local apk_path="$1" - local package_name="$2" if [[ ! -f "$apk_path" ]]; then echo "Missing APK at $apk_path. Build the APK first." >&2 exit 1 fi adb install -r "$apk_path" - echo "Installed $package_name." + echo "Installed APK: $apk_path" } function dump_profiles() { adb logcat -c adb shell am broadcast \ - -a "$INSTRUMENTED_PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.PgoProfileReceiver" + -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$PACKAGE_NAME/.PgoProfileReceiver" wait_for_profile_dump } @@ -183,7 +191,7 @@ function record_startups() { local run for ((run = 1; run <= runs; run++)); do echo "Recording cold-process startup $run/$runs." - adb shell am force-stop "$INSTRUMENTED_PACKAGE_NAME" + adb shell am force-stop "$PACKAGE_NAME" sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" adb logcat -c adb shell am start -W -n "$START_ACTIVITY" @@ -226,13 +234,11 @@ function measure_startup() { function benchmark_startups() { local label="$1" - local apk_path="$2" - local benchmark_path="$3" - local runs="$4" - local ready_timeout_seconds="$5" + local benchmark_path="$2" + local runs="$3" + local ready_timeout_seconds="$4" validate_startup_arguments "$runs" "$ready_timeout_seconds" - install_apk "$apk_path" mkdir -p "$BENCHMARK_DIR" echo "Running one unmeasured warm-up startup for $label." @@ -304,19 +310,23 @@ function compare_benchmarks() { } function benchmark_release() { - benchmark_startups "release" "$RELEASE_APK_PATH" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" + install_apk "$RELEASE_APK_PATH" + benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" } function benchmark_optimized() { - benchmark_startups "PGO optimized" "$OPTIMIZED_APK_PATH" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" + install_apk "$OPTIMIZED_APK_PATH" + benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" } function benchmark_all() { local runs="${1:-$DEFAULT_STARTUP_RUNS}" local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - benchmark_release "$runs" "$ready_timeout_seconds" - benchmark_optimized "$runs" "$ready_timeout_seconds" + install_apk "$RELEASE_APK_PATH" + benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" + install_apk "$OPTIMIZED_APK_PATH" + benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" compare_benchmarks } @@ -415,13 +425,13 @@ case "${1:-}" in verify_pgo_instrumentation ;; install | install-instrumented) - install_apk "$INSTRUMENTED_APK_PATH" "$INSTRUMENTED_PACKAGE_NAME" + install_apk "$INSTRUMENTED_APK_PATH" ;; install-release) install_apk "$RELEASE_APK_PATH" ;; install-optimized) - install_apk "$OPTIMIZED_APK_PATH" "$OPTIMIZED_PACKAGE_NAME" + install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) record_startups "${2:-}" "${3:-}" From 4ce4248b88cd0aef8314dd3e35f870fefa1af689 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:04:07 +0200 Subject: [PATCH 18/47] chore: move apk outputs to `.pgo` folder --- scripts/pgo/README.md | 4 ++-- scripts/pgo/android-local-proof.sh | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 46ec0f9d5546..18e7f64286c6 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The script renames the instrumented and optimized release outputs to `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-instrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-optimized.apk`. All builds use the release application ID and are installed consecutively. Install the optimized APK with: + The script moves the instrumented and optimized release outputs to `.pgo/android/arm64-v8a/apk/Expensify-release-instrumented.apk` and `.pgo/android/arm64-v8a/apk/Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -61,7 +61,7 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is renamed before the release APK is built: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is moved into `.pgo/android/arm64-v8a/apk/` before the release APK is built: ```bash scripts/pgo/android-local-proof.sh build-optimized diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 5f5e9922ae7f..7c7da32ed25b 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -10,11 +10,12 @@ readonly PACKAGE_NAME="org.me.mobiexpensifyg" readonly RELEASE_BUILD_VARIANT="Release" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly APK_DIR="$PROFILE_DIR/apk" readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" -readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-instrumented.apk" -readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-optimized.apk" +readonly INSTRUMENTED_APK_PATH="$APK_DIR/Expensify-release-instrumented.apk" +readonly OPTIMIZED_APK_PATH="$APK_DIR/Expensify-release-optimized.apk" readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" @@ -59,7 +60,7 @@ function build_instrumented() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate - rename_release_apk "$INSTRUMENTED_APK_PATH" + archive_release_apk "$INSTRUMENTED_APK_PATH" } function build_release() { @@ -80,18 +81,19 @@ function build_optimized() { -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ -PpgoProfile="$PROFILE_DIR/newdot.profdata" - rename_release_apk "$OPTIMIZED_APK_PATH" + archive_release_apk "$OPTIMIZED_APK_PATH" } -function rename_release_apk() { - local renamed_apk_path="$1" +function archive_release_apk() { + local archived_apk_path="$1" if [[ ! -f "$RELEASE_APK_PATH" ]]; then echo "Missing built release APK at $RELEASE_APK_PATH." >&2 exit 1 fi - mv -f "$RELEASE_APK_PATH" "$renamed_apk_path" - echo "Renamed release APK: $renamed_apk_path" + mkdir -p "$APK_DIR" + mv -f "$RELEASE_APK_PATH" "$archived_apk_path" + echo "Archived release APK: $archived_apk_path" } function install_apk() { From c267adb2cfb4059328466266a78fcb0b48662f50 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:12:33 +0200 Subject: [PATCH 19/47] chore: archive release build --- scripts/pgo/README.md | 6 +++--- scripts/pgo/android-local-proof.sh | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 18e7f64286c6..c22b3a422872 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The script moves the instrumented and optimized release outputs to `.pgo/android/arm64-v8a/apk/Expensify-release-instrumented.apk` and `.pgo/android/arm64-v8a/apk/Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: + The script archives all release outputs under `.pgo/android/arm64-v8a/apk/`: `Expensify-release.apk`, `Expensify-release-instrumented.apk`, and `Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -61,7 +61,7 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is moved into `.pgo/android/arm64-v8a/apk/` before the release APK is built: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode initially uses Gradle's normal release output path, then the tool moves it into `.pgo/android/arm64-v8a/apk/`: ```bash scripts/pgo/android-local-proof.sh build-optimized @@ -88,7 +88,7 @@ Compare previously collected samples: scripts/pgo/android-local-proof.sh compare-benchmarks ``` -Or install and benchmark the release APK first, then install and benchmark the renamed optimized APK, and compare them in one command: +Or install and benchmark the archived release APK first, then install and benchmark the archived optimized APK, and compare them in one command: ```bash scripts/pgo/android-local-proof.sh benchmark 10 30 diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 7c7da32ed25b..8bdcf25afbe0 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -13,7 +13,8 @@ readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly APK_DIR="$PROFILE_DIR/apk" readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" -readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly GRADLE_RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly RELEASE_APK_PATH="$APK_DIR/Expensify-release.apk" readonly INSTRUMENTED_APK_PATH="$APK_DIR/Expensify-release-instrumented.apk" readonly OPTIMIZED_APK_PATH="$APK_DIR/Expensify-release-optimized.apk" readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" @@ -68,6 +69,7 @@ function build_release() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=off + archive_release_apk "$RELEASE_APK_PATH" } function build_optimized() { @@ -86,13 +88,13 @@ function build_optimized() { function archive_release_apk() { local archived_apk_path="$1" - if [[ ! -f "$RELEASE_APK_PATH" ]]; then - echo "Missing built release APK at $RELEASE_APK_PATH." >&2 + if [[ ! -f "$GRADLE_RELEASE_APK_PATH" ]]; then + echo "Missing built release APK at $GRADLE_RELEASE_APK_PATH." >&2 exit 1 fi mkdir -p "$APK_DIR" - mv -f "$RELEASE_APK_PATH" "$archived_apk_path" + mv -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" echo "Archived release APK: $archived_apk_path" } From 43425ff725d3a2c296c1660c8a2bef85016e3cef Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:24:32 +0200 Subject: [PATCH 20/47] chore: copy generated release file instead of moving it --- scripts/pgo/README.md | 2 +- scripts/pgo/android-local-proof.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index c22b3a422872..644bc125d79b 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -61,7 +61,7 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode initially uses Gradle's normal release output path, then the tool moves it into `.pgo/android/arm64-v8a/apk/`: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode uses Gradle's normal release output path, then the tool copies it into `.pgo/android/arm64-v8a/apk/`. The currently built APK remains in Gradle's release directory: ```bash scripts/pgo/android-local-proof.sh build-optimized diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 8bdcf25afbe0..406fe4fd61c1 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -61,7 +61,7 @@ function build_instrumented() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate - archive_release_apk "$INSTRUMENTED_APK_PATH" + copy_release_apk "$INSTRUMENTED_APK_PATH" } function build_release() { @@ -69,7 +69,7 @@ function build_release() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=off - archive_release_apk "$RELEASE_APK_PATH" + copy_release_apk "$RELEASE_APK_PATH" } function build_optimized() { @@ -83,10 +83,10 @@ function build_optimized() { -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ -PpgoProfile="$PROFILE_DIR/newdot.profdata" - archive_release_apk "$OPTIMIZED_APK_PATH" + copy_release_apk "$OPTIMIZED_APK_PATH" } -function archive_release_apk() { +function copy_release_apk() { local archived_apk_path="$1" if [[ ! -f "$GRADLE_RELEASE_APK_PATH" ]]; then echo "Missing built release APK at $GRADLE_RELEASE_APK_PATH." >&2 @@ -94,8 +94,8 @@ function archive_release_apk() { fi mkdir -p "$APK_DIR" - mv -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" - echo "Archived release APK: $archived_apk_path" + cp -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" + echo "Copied release APK: $archived_apk_path" } function install_apk() { From aa96634e2366c371be6de4bea25f85af37ef5ff6 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:29:25 +0200 Subject: [PATCH 21/47] chore: extend PGO benchmark suite --- scripts/pgo/README.md | 2 +- scripts/pgo/android-local-proof.sh | 59 ++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 644bc125d79b..7ad4f9c868a3 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -94,7 +94,7 @@ Or install and benchmark the archived release APK first, then install and benchm scripts/pgo/android-local-proof.sh benchmark 10 30 ``` -Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the mean, median, minimum, maximum, and percentage improvement. Positive improvement percentages mean the optimized build was faster. +Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the average, P50, P75, P90, P95, P99, minimum, maximum, and percentage improvements. Percentiles use linear interpolation, and positive improvement percentages mean the optimized build was faster. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 406fe4fd61c1..994b9ac5d180 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -263,6 +263,15 @@ function benchmark_stats() { local benchmark_path="$1" awk -F, ' + function percentile(fraction, position, lower_index, remainder) { + position = 1 + (count - 1) * fraction + lower_index = int(position) + remainder = position - lower_index + if (lower_index >= count) { + return values[count] + } + return values[lower_index] + remainder * (values[lower_index + 1] - values[lower_index]) + } NR > 1 && $2 ~ /^[0-9]+$/ { values[++count] = $2 sum += $2 @@ -280,36 +289,46 @@ function benchmark_stats() { } } } - if (count % 2 == 1) { - median = values[(count + 1) / 2] - } else { - median = (values[count / 2] + values[count / 2 + 1]) / 2 - } - printf "%d %.2f %.2f %d %d\n", count, sum / count, median, values[1], values[count] + printf "%d %.2f %.2f %.2f %.2f %.2f %.2f %d %d\n", count, sum / count, percentile(0.50), percentile(0.75), percentile(0.90), percentile(0.95), percentile(0.99), values[1], values[count] } ' "$benchmark_path" } +function percentage_improvement() { + local release_value="$1" + local optimized_value="$2" + + awk -v release="$release_value" -v optimized="$optimized_value" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }' +} + function compare_benchmarks() { if [[ ! -f "$RELEASE_BENCHMARK_PATH" || ! -f "$OPTIMIZED_BENCHMARK_PATH" ]]; then echo "Missing benchmark data. Run benchmark-release and benchmark-optimized first." >&2 exit 1 fi - local release_count release_mean release_median release_min release_max - read -r release_count release_mean release_median release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") - local optimized_count optimized_mean optimized_median optimized_min optimized_max - read -r optimized_count optimized_mean optimized_median optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") - - local mean_improvement median_improvement - mean_improvement="$(awk -v release="$release_mean" -v optimized="$optimized_mean" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" - median_improvement="$(awk -v release="$release_median" -v optimized="$optimized_median" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" - - printf '%-18s %6s %10s %10s %8s %8s\n' 'Build' 'Runs' 'Mean ms' 'Median ms' 'Min ms' 'Max ms' - printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'Release' "$release_count" "$release_mean" "$release_median" "$release_min" "$release_max" - printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_mean" "$optimized_median" "$optimized_min" "$optimized_max" - echo "PGO mean startup improvement: ${mean_improvement}%" - echo "PGO median startup improvement: ${median_improvement}%" + local release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max + read -r release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") + local optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max + read -r optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") + + local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement + average_improvement="$(percentage_improvement "$release_average" "$optimized_average")" + p50_improvement="$(percentage_improvement "$release_p50" "$optimized_p50")" + p75_improvement="$(percentage_improvement "$release_p75" "$optimized_p75")" + p90_improvement="$(percentage_improvement "$release_p90" "$optimized_p90")" + p95_improvement="$(percentage_improvement "$release_p95" "$optimized_p95")" + p99_improvement="$(percentage_improvement "$release_p99" "$optimized_p99")" + + printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'Build' 'Runs' 'Average' 'P50' 'P75' 'P90' 'P95' 'P99' 'Min' 'Max' + printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'Release' "$release_count" "$release_average" "$release_p50" "$release_p75" "$release_p90" "$release_p95" "$release_p99" "$release_min" "$release_max" + printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_average" "$optimized_p50" "$optimized_p75" "$optimized_p90" "$optimized_p95" "$optimized_p99" "$optimized_min" "$optimized_max" + echo "PGO average startup improvement: ${average_improvement}%" + echo "PGO P50 startup improvement: ${p50_improvement}%" + echo "PGO P75 startup improvement: ${p75_improvement}%" + echo "PGO P90 startup improvement: ${p90_improvement}%" + echo "PGO P95 startup improvement: ${p95_improvement}%" + echo "PGO P99 startup improvement: ${p99_improvement}%" echo "Positive percentages are faster; negative percentages are regressions." } From dfceb966d93a24a8e327e6031f78a800b4c9ba2a Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:49:00 +0200 Subject: [PATCH 22/47] chore: reformat benchmark output --- scripts/pgo/android-local-proof.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 994b9ac5d180..95e89d1f9b25 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -312,24 +312,21 @@ function compare_benchmarks() { local optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max read -r optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") - local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement + local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement min_improvement max_improvement average_improvement="$(percentage_improvement "$release_average" "$optimized_average")" p50_improvement="$(percentage_improvement "$release_p50" "$optimized_p50")" p75_improvement="$(percentage_improvement "$release_p75" "$optimized_p75")" p90_improvement="$(percentage_improvement "$release_p90" "$optimized_p90")" p95_improvement="$(percentage_improvement "$release_p95" "$optimized_p95")" p99_improvement="$(percentage_improvement "$release_p99" "$optimized_p99")" + min_improvement="$(percentage_improvement "$release_min" "$optimized_min")" + max_improvement="$(percentage_improvement "$release_max" "$optimized_max")" + echo "Positive percentages are faster; negative percentages are regressions." printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'Build' 'Runs' 'Average' 'P50' 'P75' 'P90' 'P95' 'P99' 'Min' 'Max' printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'Release' "$release_count" "$release_average" "$release_p50" "$release_p75" "$release_p90" "$release_p95" "$release_p99" "$release_min" "$release_max" printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_average" "$optimized_p50" "$optimized_p75" "$optimized_p90" "$optimized_p95" "$optimized_p99" "$optimized_min" "$optimized_max" - echo "PGO average startup improvement: ${average_improvement}%" - echo "PGO P50 startup improvement: ${p50_improvement}%" - echo "PGO P75 startup improvement: ${p75_improvement}%" - echo "PGO P90 startup improvement: ${p90_improvement}%" - echo "PGO P95 startup improvement: ${p95_improvement}%" - echo "PGO P99 startup improvement: ${p99_improvement}%" - echo "Positive percentages are faster; negative percentages are regressions." + printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'PGO improvement' '-' "${average_improvement}%" "${p50_improvement}%" "${p75_improvement}%" "${p90_improvement}%" "${p95_improvement}%" "${p99_improvement}%" "${min_improvement}%" "${max_improvement}%" } function benchmark_release() { From e25759f4c53b104296d17c85df870b8c17c2ac5b Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 10:48:08 +0200 Subject: [PATCH 23/47] refactor: migrate script to TypeScript --- scripts/pgo/README.md | 38 +- scripts/pgo/android-local-proof.sh | 487 ------------------------- scripts/pgo/local-proof.ts | 561 +++++++++++++++++++++++++++++ 3 files changed, 580 insertions(+), 506 deletions(-) delete mode 100755 scripts/pgo/android-local-proof.sh create mode 100755 scripts/pgo/local-proof.ts diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 7ad4f9c868a3..ab4e91407ebb 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,6 +1,6 @@ -# Local Android PGO proof +# Local PGO proof -This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 release build: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow. The TypeScript tool separates platform-specific operations from the shared build, profile, and benchmark workflows so iOS support can be added without duplicating the orchestration. Currently only the standalone Android arm64 release build is implemented: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. ## Preconditions @@ -14,17 +14,17 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android 1. Build and install the instrumented application: ```bash - scripts/pgo/android-local-proof.sh build-instrumented - scripts/pgo/android-local-proof.sh install-instrumented + scripts/pgo/local-proof.ts android build-instrumented + scripts/pgo/local-proof.ts android install-instrumented ``` 2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. 3. Persist the counters before force-stopping the application: ```bash - scripts/pgo/android-local-proof.sh dump - scripts/pgo/android-local-proof.sh pull - scripts/pgo/android-local-proof.sh merge + scripts/pgo/local-proof.ts android dump + scripts/pgo/local-proof.ts android pull + scripts/pgo/local-proof.ts android merge ``` Profiles are written to the app's external cache so they can be retrieved from a non-debuggable release APK with `adb pull`. @@ -32,13 +32,13 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android 4. Build the matched optimized application: ```bash - scripts/pgo/android-local-proof.sh build-optimized + scripts/pgo/local-proof.ts android build-optimized ``` The script archives all release outputs under `.pgo/android/arm64-v8a/apk/`: `Expensify-release.apk`, `Expensify-release-instrumented.apk`, and `Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash - scripts/pgo/android-local-proof.sh install-optimized + scripts/pgo/local-proof.ts android install-optimized ``` ### Collect a startup-focused profile @@ -46,13 +46,13 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android To replace the manual journey with ten cold-process startup runs, then pull and merge the accumulated profiles automatically, run: ```bash -scripts/pgo/android-local-proof.sh record-startups +scripts/pgo/local-proof.ts android record-startups ``` The optional arguments set the number of runs and the timeout for NewDot's native app-ready signal respectively: ```bash -scripts/pgo/android-local-proof.sh record-startups 10 30 +scripts/pgo/local-proof.ts android record-startups 10 30 ``` The command clears only existing `newdot-*.profraw` files on the device before the first run. It clears Logcat before each launch, waits for `NewDotStartup: APP_READY`, and fails rather than recording an incomplete startup if the marker does not appear before the timeout. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. @@ -64,34 +64,34 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode uses Gradle's normal release output path, then the tool copies it into `.pgo/android/arm64-v8a/apk/`. The currently built APK remains in Gradle's release directory: ```bash -scripts/pgo/android-local-proof.sh build-optimized -scripts/pgo/android-local-proof.sh build-release +scripts/pgo/local-proof.ts android build-optimized +scripts/pgo/local-proof.ts android build-release ``` Collect each benchmark independently. Each command installs its APK without clearing application data, performs one unmeasured warm-up, then records ten cold-process startup samples by default: ```bash -scripts/pgo/android-local-proof.sh benchmark-release -scripts/pgo/android-local-proof.sh benchmark-optimized +scripts/pgo/local-proof.ts android benchmark-release +scripts/pgo/local-proof.ts android benchmark-optimized ``` The optional arguments select the measured run count and app-ready timeout: ```bash -scripts/pgo/android-local-proof.sh benchmark-release 20 30 -scripts/pgo/android-local-proof.sh benchmark-optimized 20 30 +scripts/pgo/local-proof.ts android benchmark-release 20 30 +scripts/pgo/local-proof.ts android benchmark-optimized 20 30 ``` Compare previously collected samples: ```bash -scripts/pgo/android-local-proof.sh compare-benchmarks +scripts/pgo/local-proof.ts android compare-benchmarks ``` Or install and benchmark the archived release APK first, then install and benchmark the archived optimized APK, and compare them in one command: ```bash -scripts/pgo/android-local-proof.sh benchmark 10 30 +scripts/pgo/local-proof.ts android benchmark 10 30 ``` Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the average, P50, P75, P90, P95, P99, minimum, maximum, and percentage improvements. Percentiles use linear interpolation, and positive improvement percentages mean the optimized build was faster. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh deleted file mode 100755 index 95e89d1f9b25..000000000000 --- a/scripts/pgo/android-local-proof.sh +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -readonly NDK_VERSION="27.1.12297006" -readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" - -readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly PACKAGE_NAME="org.me.mobiexpensifyg" -readonly RELEASE_BUILD_VARIANT="Release" - -readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" -readonly APK_DIR="$PROFILE_DIR/apk" -readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" -readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" -readonly GRADLE_RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" -readonly RELEASE_APK_PATH="$APK_DIR/Expensify-release.apk" -readonly INSTRUMENTED_APK_PATH="$APK_DIR/Expensify-release-instrumented.apk" -readonly OPTIMIZED_APK_PATH="$APK_DIR/Expensify-release-optimized.apk" -readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" -readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" -readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" -readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" -readonly APP_READY_LOG_TAG="NewDotStartup" -readonly APP_READY_LOG_MESSAGE="APP_READY" -readonly DEFAULT_STARTUP_RUNS=10 -readonly DEFAULT_APP_READY_TIMEOUT_SECONDS=30 -readonly STARTUP_RELAUNCH_DELAY_SECONDS=0.5 -readonly PROFILE_DUMP_TIMEOUT_SECONDS=5 - -APP_READY_DURATION_MS="" - -function usage() { - echo "Usage: $0 {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}" -} - -function ndk_tool() { - local tool_name="$1" - local ndk_root="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" - if [[ -z "$ndk_root" ]]; then - echo "Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build." >&2 - exit 1 - fi - - local host_tag="darwin-x86_64" - if [[ "$(uname -s)" == "Linux" ]]; then - host_tag="linux-x86_64" - fi - echo "$ndk_root/toolchains/llvm/prebuilt/$host_tag/bin/$tool_name" -} - -function gradle() { - ( - cd "$ANDROID_DIR" - ./gradlew "$@" - ) -} - -function build_instrumented() { - gradle ":assemble$RELEASE_BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=generate - copy_release_apk "$INSTRUMENTED_APK_PATH" -} - -function build_release() { - gradle ":assemble$RELEASE_BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=off - copy_release_apk "$RELEASE_APK_PATH" -} - -function build_optimized() { - if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then - echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 - exit 1 - fi - - gradle ":assemble$RELEASE_BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=use \ - -PpgoProfile="$PROFILE_DIR/newdot.profdata" - copy_release_apk "$OPTIMIZED_APK_PATH" -} - -function copy_release_apk() { - local archived_apk_path="$1" - if [[ ! -f "$GRADLE_RELEASE_APK_PATH" ]]; then - echo "Missing built release APK at $GRADLE_RELEASE_APK_PATH." >&2 - exit 1 - fi - - mkdir -p "$APK_DIR" - cp -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" - echo "Copied release APK: $archived_apk_path" -} - -function install_apk() { - local apk_path="$1" - if [[ ! -f "$apk_path" ]]; then - echo "Missing APK at $apk_path. Build the APK first." >&2 - exit 1 - fi - - adb install -r "$apk_path" - echo "Installed APK: $apk_path" -} - -function dump_profiles() { - adb logcat -c - adb shell am broadcast \ - -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$PACKAGE_NAME/.PgoProfileReceiver" - - wait_for_profile_dump -} - -function wait_for_profile_dump() { - local started_at="$SECONDS" - - while ((SECONDS - started_at < PROFILE_DUMP_TIMEOUT_SECONDS)); do - local profile_logs - profile_logs="$(adb logcat -d -s 'PgoProfileReceiver:I' '*:S')" - - if [[ "$profile_logs" =~ Wrote\ ([1-9][0-9]*)\ LLVM\ PGO\ profile ]]; then - echo "${BASH_REMATCH[0]}(s)." - return 0 - fi - if [[ "$profile_logs" == *"Ignoring PGO profile request in a non-instrumented build."* ]]; then - echo "The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files." >&2 - return 1 - fi - if [[ "$profile_logs" == *"Wrote 0 LLVM PGO profile(s)."* ]]; then - echo "The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles." >&2 - return 1 - fi - - sleep 0.1 - done - - echo "The PGO receiver did not confirm a profile write within ${PROFILE_DUMP_TIMEOUT_SECONDS}s." >&2 - adb logcat -d -s 'PgoProfileReceiver:I' '*:S' >&2 - return 1 -} - -function pull_profiles() { - rm -rf "$PROFILE_DIR/raw" - mkdir -p "$PROFILE_DIR/raw" - adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" - - local -a profiles=() - local profile - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 - exit 1 - fi - printf '%s\n' "${profiles[@]}" -} - -function merge_profiles() { - mkdir -p "$PROFILE_DIR" - - local -a profiles=() - local profile - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found. Run dump and pull first." >&2 - exit 1 - fi - - "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" - "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" - echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" -} - -function record_startups() { - local runs="${1:-$DEFAULT_STARTUP_RUNS}" - local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - - validate_startup_arguments "$runs" "$ready_timeout_seconds" - - echo "Clearing previous device PGO profiles from $DEVICE_PROFILE_DIR." - adb shell "rm -f '$DEVICE_PROFILE_DIR'/newdot-*.profraw" - - local run - for ((run = 1; run <= runs; run++)); do - echo "Recording cold-process startup $run/$runs." - adb shell am force-stop "$PACKAGE_NAME" - sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" - adb logcat -c - adb shell am start -W -n "$START_ACTIVITY" - wait_for_app_ready "$ready_timeout_seconds" - dump_profiles - done - - pull_profiles - merge_profiles -} - -function validate_startup_arguments() { - local runs="$1" - local ready_timeout_seconds="$2" - - if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then - echo "Startup run count must be a positive integer, received: $runs" >&2 - exit 1 - fi - if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then - echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 - exit 1 - fi -} - -function measure_startup() { - local ready_timeout_seconds="$1" - - adb shell am force-stop "$PACKAGE_NAME" - sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" - adb logcat -c - adb shell am start -W -n "$START_ACTIVITY" - wait_for_app_ready "$ready_timeout_seconds" - - if [[ -z "$APP_READY_DURATION_MS" ]]; then - echo "APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes." >&2 - return 1 - fi -} - -function benchmark_startups() { - local label="$1" - local benchmark_path="$2" - local runs="$3" - local ready_timeout_seconds="$4" - - validate_startup_arguments "$runs" "$ready_timeout_seconds" - mkdir -p "$BENCHMARK_DIR" - - echo "Running one unmeasured warm-up startup for $label." - measure_startup "$ready_timeout_seconds" - - printf 'run,duration_ms\n' > "$benchmark_path" - local run - for ((run = 1; run <= runs; run++)); do - echo "Benchmarking $label cold-process startup $run/$runs." - measure_startup "$ready_timeout_seconds" - printf '%d,%s\n' "$run" "$APP_READY_DURATION_MS" >> "$benchmark_path" - done - - echo "Recorded $runs $label startup samples: $benchmark_path" -} - -function benchmark_stats() { - local benchmark_path="$1" - - awk -F, ' - function percentile(fraction, position, lower_index, remainder) { - position = 1 + (count - 1) * fraction - lower_index = int(position) - remainder = position - lower_index - if (lower_index >= count) { - return values[count] - } - return values[lower_index] + remainder * (values[lower_index + 1] - values[lower_index]) - } - NR > 1 && $2 ~ /^[0-9]+$/ { - values[++count] = $2 - sum += $2 - } - END { - if (count == 0) { - exit 1 - } - for (i = 1; i <= count; i++) { - for (j = i + 1; j <= count; j++) { - if (values[i] > values[j]) { - temporary = values[i] - values[i] = values[j] - values[j] = temporary - } - } - } - printf "%d %.2f %.2f %.2f %.2f %.2f %.2f %d %d\n", count, sum / count, percentile(0.50), percentile(0.75), percentile(0.90), percentile(0.95), percentile(0.99), values[1], values[count] - } - ' "$benchmark_path" -} - -function percentage_improvement() { - local release_value="$1" - local optimized_value="$2" - - awk -v release="$release_value" -v optimized="$optimized_value" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }' -} - -function compare_benchmarks() { - if [[ ! -f "$RELEASE_BENCHMARK_PATH" || ! -f "$OPTIMIZED_BENCHMARK_PATH" ]]; then - echo "Missing benchmark data. Run benchmark-release and benchmark-optimized first." >&2 - exit 1 - fi - - local release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max - read -r release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") - local optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max - read -r optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") - - local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement min_improvement max_improvement - average_improvement="$(percentage_improvement "$release_average" "$optimized_average")" - p50_improvement="$(percentage_improvement "$release_p50" "$optimized_p50")" - p75_improvement="$(percentage_improvement "$release_p75" "$optimized_p75")" - p90_improvement="$(percentage_improvement "$release_p90" "$optimized_p90")" - p95_improvement="$(percentage_improvement "$release_p95" "$optimized_p95")" - p99_improvement="$(percentage_improvement "$release_p99" "$optimized_p99")" - min_improvement="$(percentage_improvement "$release_min" "$optimized_min")" - max_improvement="$(percentage_improvement "$release_max" "$optimized_max")" - - echo "Positive percentages are faster; negative percentages are regressions." - printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'Build' 'Runs' 'Average' 'P50' 'P75' 'P90' 'P95' 'P99' 'Min' 'Max' - printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'Release' "$release_count" "$release_average" "$release_p50" "$release_p75" "$release_p90" "$release_p95" "$release_p99" "$release_min" "$release_max" - printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_average" "$optimized_p50" "$optimized_p75" "$optimized_p90" "$optimized_p95" "$optimized_p99" "$optimized_min" "$optimized_max" - printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'PGO improvement' '-' "${average_improvement}%" "${p50_improvement}%" "${p75_improvement}%" "${p90_improvement}%" "${p95_improvement}%" "${p99_improvement}%" "${min_improvement}%" "${max_improvement}%" -} - -function benchmark_release() { - install_apk "$RELEASE_APK_PATH" - benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" -} - -function benchmark_optimized() { - install_apk "$OPTIMIZED_APK_PATH" - benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" -} - -function benchmark_all() { - local runs="${1:-$DEFAULT_STARTUP_RUNS}" - local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - - install_apk "$RELEASE_APK_PATH" - benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" - install_apk "$OPTIMIZED_APK_PATH" - benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" - compare_benchmarks -} - -function wait_for_app_ready() { - local timeout_seconds="$1" - local started_at="$SECONDS" - - APP_READY_DURATION_MS="" - echo "Waiting up to ${timeout_seconds}s for $APP_READY_LOG_TAG: $APP_READY_LOG_MESSAGE." - while ((SECONDS - started_at < timeout_seconds)); do - local startup_logs - startup_logs="$(adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S')" - if [[ "$startup_logs" == *"$APP_READY_LOG_MESSAGE"* ]]; then - if [[ "$startup_logs" =~ APP_READY\ durationMs=([0-9]+) ]]; then - APP_READY_DURATION_MS="${BASH_REMATCH[1]}" - echo "NewDot reported APP_READY after ${APP_READY_DURATION_MS}ms." - else - echo "NewDot reported APP_READY without a numeric duration marker." - fi - return 0 - fi - sleep 0.25 - done - - echo "NewDot did not report APP_READY within ${timeout_seconds}s." >&2 - adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S' >&2 - return 1 -} - -function verify_pgo_instrumentation() ( - local apk_path="${1:-$INSTRUMENTED_APK_PATH}" - if [[ ! -f "$apk_path" ]]; then - echo "Missing APK at $apk_path. Build the instrumented release first." >&2 - exit 1 - fi - - local extracted_dir - extracted_dir="$(mktemp -d "${TMPDIR:-/tmp}/expensify-pgo-apk.XXXXXX")" - trap 'rm -rf "$extracted_dir"' EXIT - - local llvm_readelf - if ! llvm_readelf="$(ndk_tool llvm-readelf)"; then - exit 1 - fi - local llvm_nm - if ! llvm_nm="$(ndk_tool llvm-nm)"; then - exit 1 - fi - - local libraries=( - libreactnative.so - libhermesvm.so - libjsi.so - libExpensifyNitroUtils.so - ) - local library - for library in "${libraries[@]}"; do - local apk_entry="lib/arm64-v8a/$library" - if ! zipinfo -1 "$apk_path" | rg -Fx "$apk_entry" >/dev/null; then - echo "Missing expected arm64 library in APK: $apk_entry" >&2 - exit 1 - fi - - local extracted_library="$extracted_dir/$library" - unzip -p "$apk_path" "$apk_entry" > "$extracted_library" - local section_headers - section_headers="$("$llvm_readelf" -SW "$extracted_library")" - if [[ "$section_headers" != *"__llvm_prf_data"* || "$section_headers" != *"__llvm_prf_cnts"* || "$section_headers" != *"__llvm_prf_names"* ]]; then - echo "LLVM PGO instrumentation is missing from $apk_entry." >&2 - exit 1 - fi - - local dynamic_symbols - dynamic_symbols="$("$llvm_nm" -D --defined-only "$extracted_library")" - if [[ "$dynamic_symbols" != *"expensify_llvm_profile_set_filename"* || "$dynamic_symbols" != *"expensify_llvm_profile_write_file"* ]]; then - echo "LLVM PGO profile-writing APIs are not exported from $apk_entry." >&2 - exit 1 - fi - - if [[ "$library" == "libExpensifyNitroUtils.so" && "$dynamic_symbols" != *"Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles"* ]]; then - echo "The PGO JNI writer is missing from $apk_entry. Rebuild the native module before installing." >&2 - exit 1 - fi - echo "Verified LLVM PGO instrumentation: $apk_entry" - done -) - -case "${1:-}" in - build-release) - build_release - ;; - build-instrumented) - build_instrumented - ;; - verify-instrumented) - verify_pgo_instrumentation - ;; - install | install-instrumented) - install_apk "$INSTRUMENTED_APK_PATH" - ;; - install-release) - install_apk "$RELEASE_APK_PATH" - ;; - install-optimized) - install_apk "$OPTIMIZED_APK_PATH" - ;; - record-startups) - record_startups "${2:-}" "${3:-}" - ;; - dump) - dump_profiles - ;; - pull) - pull_profiles - ;; - merge) - merge_profiles - ;; - build-optimized) - build_optimized - ;; - benchmark-release) - benchmark_release "${2:-}" "${3:-}" - ;; - benchmark-optimized) - benchmark_optimized "${2:-}" "${3:-}" - ;; - benchmark) - benchmark_all "${2:-}" "${3:-}" - ;; - compare-benchmarks) - compare_benchmarks - ;; - *) - usage - exit 1 - ;; -esac diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts new file mode 100755 index 000000000000..07ab17346b11 --- /dev/null +++ b/scripts/pgo/local-proof.ts @@ -0,0 +1,561 @@ +#!/usr/bin/env -S node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON + +import {spawnSync} from 'node:child_process'; +import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; +import {homedir, platform, tmpdir} from 'node:os'; +import {basename, dirname, join, resolve} from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +const DEFAULT_STARTUP_RUNS = 10; +const DEFAULT_APP_READY_TIMEOUT_SECONDS = 30; +const STARTUP_RELAUNCH_DELAY_MS = 500; + +type BuildKind = 'release' | 'instrumented' | 'optimized'; +type BenchmarkKind = Extract; + +type BuildArtifactPaths = Record; + +type BenchmarkStats = { + count: number; + average: number; + p50: number; + p75: number; + p90: number; + p95: number; + p99: number; + min: number; + max: number; +}; + +type PlatformAdapter = { + readonly name: string; + readonly profileDirectory: string; + readonly rawProfileDirectory: string; + readonly mergedProfilePath: string; + readonly benchmarkDirectory: string; + readonly benchmarkPaths: Record; + readonly artifactPaths: BuildArtifactPaths; + build: (kind: BuildKind) => void; + install: (kind: BuildKind) => void; + verifyInstrumentation: () => void; + clearDeviceProfiles: () => void; + dumpProfiles: () => Promise; + pullProfiles: () => void; + forceStop: () => void; + clearSystemLogs: () => void; + launch: () => void; + waitForAppReady: (timeoutSeconds: number) => Promise; + llvmTool: (name: string) => string; +}; + +const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const rootDirectory = resolve(scriptDirectory, '../..'); + +function fail(message: string): never { + throw new Error(message); +} + +function run(command: string, args: string[], cwd = rootDirectory): void { + const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } +} + +function capture(command: string, args: string[], cwd = rootDirectory): string { + const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const stderr = result.stderr.trim(); + fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { + const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolvePromise) => { + setTimeout(resolvePromise, milliseconds); + }); +} + +function findFiles(directory: string, extension: string): string[] { + if (!existsSync(directory)) { + return []; + } + + return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return findFiles(path, extension); + } + return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; + }); +} + +function requirePositiveInteger(rawValue: string | undefined, defaultValue: number, label: string): number { + if (rawValue === undefined || rawValue.length === 0) { + return defaultValue; + } + + const value = Number(rawValue); + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive integer, received: ${rawValue}`); + } + return value; +} + +function valueAt(values: T[], index: number): T { + const value = values.at(index); + if (value === undefined) { + fail(`Expected a value at index ${index}.`); + } + return value; +} + +function createAndroidAdapter(): PlatformAdapter { + const ndkVersion = '27.1.12297006'; + const packageName = 'org.me.mobiexpensifyg'; + const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const apkDirectory = join(profileDirectory, 'apk'); + const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); + const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); + const gradleReleaseApkPath = join(androidDirectory, 'build/outputs/apk/release/Expensify-release.apk'); + const deviceProfileDirectory = `/sdcard/Android/data/${packageName}/cache`; + const startActivity = `${packageName}/.ExpensifyActivityBase`; + const appReadyLogTag = 'NewDotStartup'; + const appReadyLogMessage = 'APP_READY'; + const profileDumpTimeoutSeconds = 5; + + const artifactPaths: BuildArtifactPaths = { + release: join(apkDirectory, 'Expensify-release.apk'), + instrumented: join(apkDirectory, 'Expensify-release-instrumented.apk'), + optimized: join(apkDirectory, 'Expensify-release-optimized.apk'), + }; + + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + function adb(args: string[]): void { + run('adb', args); + } + + function adbCapture(args: string[]): string { + return capture('adb', args); + } + + function llvmTool(name: string): string { + const androidNdkHome: unknown = process.env.ANDROID_NDK_HOME; + const androidNdkRoot: unknown = process.env.ANDROID_NDK_ROOT; + const configuredNdkRoot = [androidNdkHome, androidNdkRoot].find((value): value is string => typeof value === 'string'); + const ndkRoot = configuredNdkRoot ?? join(homedir(), `Library/Android/sdk/ndk/${ndkVersion}`); + const hostTag = platform() === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; + const toolPath = join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); + if (!existsSync(toolPath)) { + fail(`Missing ${name} at ${toolPath}. Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build.`); + } + return toolPath; + } + + function archiveReleaseApk(destination: string): void { + if (!existsSync(gradleReleaseApkPath)) { + fail(`Missing built release APK at ${gradleReleaseApkPath}.`); + } + + mkdirSync(apkDirectory, {recursive: true}); + copyFileSync(gradleReleaseApkPath, destination); + console.log(`Copied release APK: ${destination}`); + } + + function build(kind: BuildKind): void { + if (kind === 'optimized' && !existsSync(join(profileDirectory, 'newdot.profdata'))) { + fail(`Missing ${join(profileDirectory, 'newdot.profdata')}. Run merge first.`); + } + + const pgoModes: Record = { + release: 'off', + instrumented: 'generate', + optimized: 'use', + }; + const pgoMode = pgoModes[kind]; + const gradleArguments = [':assembleRelease', '-PpatchedArtifacts.forceBuildFromSource=true', '-PreactNativeArchitectures=arm64-v8a', `-PpgoMode=${pgoMode}`]; + if (kind === 'optimized') { + gradleArguments.push(`-PpgoProfile=${join(profileDirectory, 'newdot.profdata')}`); + } + + run('./gradlew', gradleArguments, androidDirectory); + archiveReleaseApk(artifactPaths[kind]); + } + + function install(kind: BuildKind): void { + const apkPath = artifactPaths[kind]; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the APK first.`); + } + + adb(['install', '-r', apkPath]); + console.log(`Installed APK: ${apkPath}`); + } + + async function waitForProfileDump(): Promise { + const deadline = Date.now() + profileDumpTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + const profileCount = /Wrote ([1-9][0-9]*) LLVM PGO profile/.exec(profileLogs)?.[1]; + if (profileCount) { + console.log(`Wrote ${profileCount} LLVM PGO profile(s).`); + return; + } + if (profileLogs.includes('Ignoring PGO profile request in a non-instrumented build.')) { + fail('The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files.'); + } + if (profileLogs.includes('Wrote 0 LLVM PGO profile(s).')) { + fail('The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles.'); + } + await sleep(100); + } + + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + fail(`The PGO receiver did not confirm a profile write within ${profileDumpTimeoutSeconds}s.\n${profileLogs}`); + } + + async function dumpProfiles(): Promise { + adb(['logcat', '-c']); + adb(['shell', 'am', 'broadcast', '-a', `${packageName}.action.WRITE_PGO_PROFILES`, '-n', `${packageName}/.PgoProfileReceiver`]); + await waitForProfileDump(); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(rawProfileDirectory, {recursive: true}); + adb(['pull', `${deviceProfileDirectory}/.`, rawProfileDirectory]); + + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in ${deviceProfileDirectory}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + async function waitForAppReady(timeoutSeconds: number): Promise { + const deadline = Date.now() + timeoutSeconds * 1000; + console.log(`Waiting up to ${timeoutSeconds}s for ${appReadyLogTag}: ${appReadyLogMessage}.`); + while (Date.now() < deadline) { + const startupLogs = adbCapture(['logcat', '-d', '-s', `${appReadyLogTag}:I`, '*:S']); + if (startupLogs.includes(appReadyLogMessage)) { + const duration = /APP_READY durationMs=([0-9]+)/.exec(startupLogs)?.[1]; + if (duration) { + console.log(`NewDot reported APP_READY after ${duration}ms.`); + return Number(duration); + } + console.log('NewDot reported APP_READY without a numeric duration marker.'); + return null; + } + await sleep(250); + } + + const startupLogs = adbCapture(['logcat', '-d', '-s', `${appReadyLogTag}:I`, '*:S']); + fail(`NewDot did not report APP_READY within ${timeoutSeconds}s.\n${startupLogs}`); + } + + function verifyInstrumentation(): void { + const apkPath = artifactPaths.instrumented; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the instrumented release first.`); + } + + const extractedDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-apk-')); + const llvmReadelf = llvmTool('llvm-readelf'); + const llvmNm = llvmTool('llvm-nm'); + const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); + const libraries = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; + + try { + for (const library of libraries) { + const apkEntry = `lib/arm64-v8a/${library}`; + if (!apkEntries.has(apkEntry)) { + fail(`Missing expected arm64 library in APK: ${apkEntry}`); + } + + const extractedLibrary = join(extractedDirectory, library); + writeFileSync(extractedLibrary, captureBinary('unzip', ['-p', apkPath, apkEntry])); + const sectionHeaders = capture(llvmReadelf, ['-SW', extractedLibrary]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => sectionHeaders.includes(section))) { + fail(`LLVM PGO instrumentation is missing from ${apkEntry}.`); + } + + const dynamicSymbols = capture(llvmNm, ['-D', '--defined-only', extractedLibrary]); + if (!['expensify_llvm_profile_set_filename', 'expensify_llvm_profile_write_file'].every((symbol) => dynamicSymbols.includes(symbol))) { + fail(`LLVM PGO profile-writing APIs are not exported from ${apkEntry}.`); + } + if (library === 'libExpensifyNitroUtils.so' && !dynamicSymbols.includes('Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles')) { + fail(`The PGO JNI writer is missing from ${apkEntry}. Rebuild the native module before installing.`); + } + console.log(`Verified LLVM PGO instrumentation: ${apkEntry}`); + } + } finally { + rmSync(extractedDirectory, {recursive: true, force: true}); + } + } + + return { + name: 'android', + profileDirectory, + rawProfileDirectory, + mergedProfilePath: join(profileDirectory, 'newdot.profdata'), + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + build, + install, + verifyInstrumentation, + clearDeviceProfiles: () => { + console.log(`Clearing previous device PGO profiles from ${deviceProfileDirectory}.`); + adb(['shell', `rm -f '${deviceProfileDirectory}'/newdot-*.profraw`]); + }, + dumpProfiles, + pullProfiles, + forceStop: () => adb(['shell', 'am', 'force-stop', packageName]), + clearSystemLogs: () => adb(['logcat', '-c']), + launch: () => adb(['shell', 'am', 'start', '-W', '-n', startActivity]), + waitForAppReady, + llvmTool, + }; +} + +function mergeProfiles(adapter: PlatformAdapter): void { + mkdirSync(adapter.profileDirectory, {recursive: true}); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail('No .profraw files found. Run dump and pull first.'); + } + + const llvmProfdata = adapter.llvmTool('llvm-profdata'); + run(llvmProfdata, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + const profileReport = capture(llvmProfdata, ['show', '--all-functions', adapter.mergedProfilePath]); + writeFileSync(`${adapter.mergedProfilePath}.txt`, profileReport); + console.log(`Merged PGO profile: ${adapter.mergedProfilePath}`); +} + +async function measureStartup(adapter: PlatformAdapter, readyTimeoutSeconds: number): Promise { + adapter.forceStop(); + await sleep(STARTUP_RELAUNCH_DELAY_MS); + adapter.clearSystemLogs(); + adapter.launch(); + const duration = await adapter.waitForAppReady(readyTimeoutSeconds); + if (duration === null) { + fail('APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes.'); + } + return duration; +} + +async function recordStartups(adapter: PlatformAdapter, runs: number, readyTimeoutSeconds: number): Promise { + adapter.clearDeviceProfiles(); + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + console.log(`Recording cold-process startup ${runNumber}/${runs}.`); + await measureStartup(adapter, readyTimeoutSeconds); + await adapter.dumpProfiles(); + } + + adapter.pullProfiles(); + mergeProfiles(adapter); +} + +async function benchmarkStartups(adapter: PlatformAdapter, kind: BenchmarkKind, runs: number, readyTimeoutSeconds: number): Promise { + const label = kind === 'release' ? 'release' : 'PGO optimized'; + const benchmarkPath = adapter.benchmarkPaths[kind]; + mkdirSync(adapter.benchmarkDirectory, {recursive: true}); + + console.log(`Running one unmeasured warm-up startup for ${label}.`); + await measureStartup(adapter, readyTimeoutSeconds); + + const samples: number[] = []; + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + console.log(`Benchmarking ${label} cold-process startup ${runNumber}/${runs}.`); + samples.push(await measureStartup(adapter, readyTimeoutSeconds)); + } + + const csv = ['run,duration_ms', ...samples.map((duration, index) => `${index + 1},${duration}`), ''].join('\n'); + writeFileSync(benchmarkPath, csv); + console.log(`Recorded ${runs} ${label} startup samples: ${benchmarkPath}`); +} + +function percentile(sortedValues: number[], fraction: number): number { + const position = (sortedValues.length - 1) * fraction; + const lowerIndex = Math.floor(position); + const upperIndex = Math.ceil(position); + const remainder = position - lowerIndex; + const lowerValue = valueAt(sortedValues, lowerIndex); + return lowerValue + remainder * (valueAt(sortedValues, upperIndex) - lowerValue); +} + +function benchmarkStats(path: string): BenchmarkStats { + const values = readFileSync(path, 'utf8') + .trim() + .split('\n') + .slice(1) + .map((line) => Number(line.split(',').at(1))) + .filter(Number.isFinite) + .sort((left, right) => left - right); + if (values.length === 0) { + fail(`No benchmark samples found in ${path}.`); + } + + return { + count: values.length, + average: values.reduce((sum, value) => sum + value, 0) / values.length, + p50: percentile(values, 0.5), + p75: percentile(values, 0.75), + p90: percentile(values, 0.9), + p95: percentile(values, 0.95), + p99: percentile(values, 0.99), + min: valueAt(values, 0), + max: valueAt(values, -1), + }; +} + +function percentageImprovement(releaseValue: number, optimizedValue: number): number { + return ((releaseValue - optimizedValue) / releaseValue) * 100; +} + +function compareBenchmarks(adapter: PlatformAdapter): void { + const releasePath = adapter.benchmarkPaths.release; + const optimizedPath = adapter.benchmarkPaths.optimized; + if (!existsSync(releasePath) || !existsSync(optimizedPath)) { + fail('Missing benchmark data. Run benchmark-release and benchmark-optimized first.'); + } + + const release = benchmarkStats(releasePath); + const optimized = benchmarkStats(optimizedPath); + const numericColumns: Array> = ['average', 'p50', 'p75', 'p90', 'p95', 'p99', 'min', 'max']; + const improvements = numericColumns.map((key) => percentageImprovement(release[key], optimized[key])); + + const formatLabel = (value: string) => value.padEnd(18); + const formatCount = (value: string) => value.padStart(5); + const formatMetric = (value: string) => value.padStart(10); + const row = (label: string, count: string, values: string[]) => [formatLabel(label), formatCount(count), ...values.map(formatMetric)].join(' '); + const statsRow = (label: string, stats: BenchmarkStats) => + row( + label, + String(stats.count), + [stats.average, stats.p50, stats.p75, stats.p90, stats.p95, stats.p99, stats.min, stats.max].map((value) => value.toFixed(2)), + ); + + console.log('Positive percentages are faster; negative percentages are regressions.'); + console.log(row('Build', 'Runs', ['Average', 'P50', 'P75', 'P90', 'P95', 'P99', 'Min', 'Max'])); + console.log(statsRow('Release', release)); + console.log(statsRow('PGO optimized', optimized)); + console.log( + row( + 'PGO improvement', + '-', + improvements.map((improvement) => `${improvement.toFixed(2)}%`), + ), + ); +} + +function printUsage(): void { + console.log( + `Usage: ${basename(process.argv.at(1) ?? 'local-proof.ts')} android {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}`, + ); +} + +function getAdapter(platformName: string | undefined): PlatformAdapter { + if (platformName === 'android') { + return createAndroidAdapter(); + } + if (platformName === 'ios') { + fail('The iOS PGO workflow is not implemented yet.'); + } + + printUsage(); + fail(`Unsupported or missing platform: ${platformName ?? '(none)'}`); +} + +async function main(): Promise { + const [, , platformName, command, runsArgument, timeoutArgument] = process.argv; + const adapter = getAdapter(platformName); + const runs = requirePositiveInteger(runsArgument, DEFAULT_STARTUP_RUNS, 'Startup run count'); + const timeoutSeconds = requirePositiveInteger(timeoutArgument, DEFAULT_APP_READY_TIMEOUT_SECONDS, 'App-ready timeout'); + + switch (command) { + case 'build-release': + adapter.build('release'); + return; + case 'build-instrumented': + adapter.build('instrumented'); + return; + case 'build-optimized': + adapter.build('optimized'); + return; + case 'verify-instrumented': + adapter.verifyInstrumentation(); + return; + case 'install-release': + adapter.install('release'); + return; + case 'install-instrumented': + adapter.install('instrumented'); + return; + case 'install-optimized': + adapter.install('optimized'); + return; + case 'record-startups': + await recordStartups(adapter, runs, timeoutSeconds); + return; + case 'benchmark-release': + adapter.install('release'); + await benchmarkStartups(adapter, 'release', runs, timeoutSeconds); + return; + case 'benchmark-optimized': + adapter.install('optimized'); + await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); + return; + case 'benchmark': + adapter.install('release'); + await benchmarkStartups(adapter, 'release', runs, timeoutSeconds); + adapter.install('optimized'); + await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); + compareBenchmarks(adapter); + return; + case 'compare-benchmarks': + compareBenchmarks(adapter); + return; + case 'dump': + await adapter.dumpProfiles(); + return; + case 'pull': + adapter.pullProfiles(); + return; + case 'merge': + mergeProfiles(adapter); + return; + default: + printUsage(); + fail(`Unsupported or missing command: ${command ?? '(none)'}`); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); From 04f20ae06dea0c930fb97f5070ae66faa1917d8c Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 11:10:33 +0200 Subject: [PATCH 24/47] refactor: use CLI tool in the PGO tool --- scripts/pgo/local-proof.ts | 118 +++++++++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 07ab17346b11..55f18fd02c46 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -1,11 +1,13 @@ #!/usr/bin/env -S node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON +import type {TupleToUnion} from 'type-fest'; + +import CLI from 'expensify-common/CLI'; import {spawnSync} from 'node:child_process'; import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; import {homedir, platform, tmpdir} from 'node:os'; -import {basename, dirname, join, resolve} from 'node:path'; +import {dirname, join, resolve} from 'node:path'; import process from 'node:process'; -import {fileURLToPath} from 'node:url'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_APP_READY_TIMEOUT_SECONDS = 30; @@ -13,6 +15,8 @@ const STARTUP_RELAUNCH_DELAY_MS = 500; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; +type PlatformName = TupleToUnion; +type WorkflowCommand = TupleToUnion; type BuildArtifactPaths = Record; @@ -49,7 +53,30 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const PLATFORM_NAMES = ['android', 'ios'] as const; +const WORKFLOW_COMMANDS = [ + 'build-release', + 'build-instrumented', + 'build-optimized', + 'verify-instrumented', + 'install-release', + 'install-instrumented', + 'install-optimized', + 'record-startups', + 'benchmark-release', + 'benchmark-optimized', + 'benchmark', + 'compare-benchmarks', + 'dump', + 'pull', + 'merge', +] as const; + +const scriptPath = process.argv.at(1); +if (!scriptPath) { + throw new Error('Unable to resolve the PGO script path.'); +} +const scriptDirectory = dirname(resolve(scriptPath)); const rootDirectory = resolve(scriptDirectory, '../..'); function fail(message: string): never { @@ -109,18 +136,25 @@ function findFiles(directory: string, extension: string): string[] { }); } -function requirePositiveInteger(rawValue: string | undefined, defaultValue: number, label: string): number { - if (rawValue === undefined || rawValue.length === 0) { - return defaultValue; - } - - const value = Number(rawValue); +function requirePositiveInteger(value: number, label: string): number { if (!Number.isSafeInteger(value) || value <= 0) { - fail(`${label} must be a positive integer, received: ${rawValue}`); + fail(`${label} must be a positive integer, received: ${value}`); } return value; } +function parsePositiveInteger(rawValue: string, label: string): number { + return requirePositiveInteger(Number(rawValue), label); +} + +function parseChoice(rawValue: string, choices: readonly T[], label: string): T { + const choice = choices.find((candidate) => candidate === rawValue); + if (!choice) { + fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); + } + return choice; +} + function valueAt(values: T[], index: number): T { const value = values.at(index); if (value === undefined) { @@ -473,31 +507,17 @@ function compareBenchmarks(adapter: PlatformAdapter): void { ); } -function printUsage(): void { - console.log( - `Usage: ${basename(process.argv.at(1) ?? 'local-proof.ts')} android {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}`, - ); -} - -function getAdapter(platformName: string | undefined): PlatformAdapter { +function getAdapter(platformName: PlatformName): PlatformAdapter { if (platformName === 'android') { return createAndroidAdapter(); } - if (platformName === 'ios') { - fail('The iOS PGO workflow is not implemented yet.'); - } - - printUsage(); - fail(`Unsupported or missing platform: ${platformName ?? '(none)'}`); + fail('The iOS PGO workflow is not implemented yet.'); } -async function main(): Promise { - const [, , platformName, command, runsArgument, timeoutArgument] = process.argv; +async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number): Promise { const adapter = getAdapter(platformName); - const runs = requirePositiveInteger(runsArgument, DEFAULT_STARTUP_RUNS, 'Startup run count'); - const timeoutSeconds = requirePositiveInteger(timeoutArgument, DEFAULT_APP_READY_TIMEOUT_SECONDS, 'App-ready timeout'); - switch (command) { + switch (workflow) { case 'build-release': adapter.build('release'); return; @@ -548,13 +568,49 @@ async function main(): Promise { return; case 'merge': mergeProfiles(adapter); - return; + break; default: - printUsage(); - fail(`Unsupported or missing command: ${command ?? '(none)'}`); + fail('Unsupported workflow.'); } } +async function main(): Promise { + const cli = new CLI({ + positionalArgs: [ + { + name: 'platform', + description: `Native platform to target (${PLATFORM_NAMES.join(', ')})`, + parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform'), + }, + { + name: 'workflow', + description: `PGO workflow to run (${WORKFLOW_COMMANDS.join(', ')})`, + parse: (value): WorkflowCommand => parseChoice(value, WORKFLOW_COMMANDS, 'Workflow'), + }, + { + name: 'runs', + description: 'Number of measured startup runs', + default: DEFAULT_STARTUP_RUNS, + parse: (value) => parsePositiveInteger(value, 'Startup run count'), + }, + { + name: 'timeout', + description: 'Seconds to wait for the app-ready marker', + default: DEFAULT_APP_READY_TIMEOUT_SECONDS, + parse: (value) => parsePositiveInteger(value, 'App-ready timeout'), + }, + ], + }); + + const {platform: platformName, workflow, runs, timeout} = cli.positionalArgs; + await runWorkflow( + parseChoice(String(platformName), PLATFORM_NAMES, 'Platform'), + parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), + requirePositiveInteger(Number(runs), 'Startup run count'), + requirePositiveInteger(Number(timeout), 'App-ready timeout'), + ); +} + main().catch((error: unknown) => { console.error(error instanceof Error ? error.message : error); process.exitCode = 1; From c68accc8a96721d442ab6c60b96157225fdfe3a1 Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 15:31:36 +0200 Subject: [PATCH 25/47] feat: implement iOS PGO --- ios/NewExpensify/RCTBootSplash.h | 1 + ios/NewExpensify/RCTBootSplash.mm | 112 +++++ .../react-native+0.85.3+040+pgo.patch | 43 ++ scripts/pgo/README.md | 60 ++- scripts/pgo/local-proof.ts | 440 +++++++++++++++++- scripts/pod-install.sh | 5 +- 6 files changed, 651 insertions(+), 10 deletions(-) diff --git a/ios/NewExpensify/RCTBootSplash.h b/ios/NewExpensify/RCTBootSplash.h index f499fe59f916..29b22858a561 100644 --- a/ios/NewExpensify/RCTBootSplash.h +++ b/ios/NewExpensify/RCTBootSplash.h @@ -4,6 +4,7 @@ + (void)initWithStoryboard:(NSString * _Nonnull)storyboardName rootView:(UIView * _Nullable)rootView; ++ (void)initializePGOProfileCollection; + (void)hide:(BOOL)fade; + (void)bringSubviewToFrontIfInitialized; diff --git a/ios/NewExpensify/RCTBootSplash.mm b/ios/NewExpensify/RCTBootSplash.mm index c2cff4c5b833..0ba896813d56 100644 --- a/ios/NewExpensify/RCTBootSplash.mm +++ b/ios/NewExpensify/RCTBootSplash.mm @@ -13,6 +13,22 @@ #import #import +#ifndef EXPENSIFY_PGO_GENERATE +#define EXPENSIFY_PGO_GENERATE 0 +#endif + +#if EXPENSIFY_PGO_GENERATE +extern "C" void __llvm_profile_set_filename(const char *FilenamePat); +extern "C" int __llvm_profile_write_file(void); +extern "C" void __llvm_profile_reset_counters(void); +#endif + +static NSString *const PGO_DIRECTORY_NAME = @"ExpensifyPGO"; +static NSString *const PGO_APP_READY_MARKER = @"app-ready.txt"; +static NSString *const PGO_STATUS_MARKER = @"profile-status.txt"; +static CFStringRef const PGO_WRITE_NOTIFICATION = CFSTR("com.expensify.pgo.write-profiles"); +static CFStringRef const PGO_CLEAR_NOTIFICATION = CFSTR("com.expensify.pgo.clear-profiles"); + static RCTSurfaceHostingProxyRootView *_rootView = nil; static UIView *_loadingView = nil; @@ -20,6 +36,29 @@ static bool _fade = false; static bool _nativeHidden = false; +static long long CurrentTimeMilliseconds(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000); +} + +@interface RCTBootSplash () ++ (NSURL *)pgoDirectoryURL; ++ (void)writePGOStatus:(NSString *)status result:(int)result; ++ (void)writePGOProfiles; ++ (void)clearPGOProfiles; +@end + +static void HandlePGONotification(__unused CFNotificationCenterRef center, + __unused void *observer, + CFStringRef name, + __unused const void *object, + __unused CFDictionaryRef userInfo) { + if (CFEqual(name, PGO_WRITE_NOTIFICATION)) { + [RCTBootSplash writePGOProfiles]; + } else if (CFEqual(name, PGO_CLEAR_NOTIFICATION)) { + [RCTBootSplash clearPGOProfiles]; + } +} + @implementation RCTBootSplash RCT_EXPORT_MODULE(); @@ -32,6 +71,69 @@ + (BOOL)requiresMainQueueSetup { return NO; } ++ (NSURL *)pgoDirectoryURL { + NSURL *cachesURL = [[[NSFileManager defaultManager] URLsForDirectory:NSCachesDirectory + inDomains:NSUserDomainMask] firstObject]; + NSURL *directoryURL = [cachesURL URLByAppendingPathComponent:PGO_DIRECTORY_NAME isDirectory:YES]; + [[NSFileManager defaultManager] createDirectoryAtURL:directoryURL + withIntermediateDirectories:YES + attributes:nil + error:nil]; + return directoryURL; +} + ++ (void)writePGOStatus:(NSString *)status result:(int)result { + NSString *contents = [NSString stringWithFormat:@"%lld,%@,%d", CurrentTimeMilliseconds(), status, result]; + NSURL *statusURL = [[self pgoDirectoryURL] URLByAppendingPathComponent:PGO_STATUS_MARKER]; + [contents writeToURL:statusURL atomically:YES encoding:NSUTF8StringEncoding error:nil]; +} + ++ (void)initializePGOProfileCollection { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + CFNotificationCenterRef notificationCenter = CFNotificationCenterGetDarwinNotifyCenter(); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_WRITE_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_CLEAR_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + +#if EXPENSIFY_PGO_GENERATE + NSString *profilePattern = [[[self pgoDirectoryURL] URLByAppendingPathComponent:@"newdot-%m.profraw"] path]; + __llvm_profile_set_filename(profilePattern.fileSystemRepresentation); +#endif + }); +} + ++ (void)writePGOProfiles { +#if EXPENSIFY_PGO_GENERATE + int result = __llvm_profile_write_file(); + [self writePGOStatus:@"written" result:result]; + NSLog(@"ExpensifyPGO: wrote LLVM profile with result=%d", result); +#else + [self writePGOStatus:@"not-instrumented" result:1]; + NSLog(@"ExpensifyPGO: ignored profile write in a non-instrumented build"); +#endif +} + ++ (void)clearPGOProfiles { + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSURL *directoryURL = [self pgoDirectoryURL]; + NSArray *files = [fileManager contentsOfDirectoryAtURL:directoryURL + includingPropertiesForKeys:nil + options:0 + error:nil]; + for (NSURL *fileURL in files) { + if ([fileURL.lastPathComponent hasSuffix:@".profraw"] || [fileURL.lastPathComponent isEqualToString:PGO_STATUS_MARKER]) { + [fileManager removeItemAtURL:fileURL error:nil]; + } + } + +#if EXPENSIFY_PGO_GENERATE + __llvm_profile_reset_counters(); + [self writePGOStatus:@"cleared" result:0]; +#else + [self writePGOStatus:@"not-instrumented" result:1]; +#endif +} + + (bool)isLoadingViewVisible { return _loadingView != nil && ![_loadingView isHidden]; } @@ -202,4 +304,14 @@ - (void)hideImpl:(BOOL)fade [self hideImpl:0 resolve:resolve]; } +RCT_EXPORT_METHOD(reportFullyDrawn) { + double startedAt = [[NSUserDefaults standardUserDefaults] doubleForKey:@"AppStartTime"]; + long long readyAt = CurrentTimeMilliseconds(); + long long duration = startedAt > 0 ? MAX(0, readyAt - (long long)startedAt) : 0; + NSString *contents = [NSString stringWithFormat:@"%lld,%lld", readyAt, duration]; + NSURL *markerURL = [[RCTBootSplash pgoDirectoryURL] URLByAppendingPathComponent:PGO_APP_READY_MARKER]; + [contents writeToURL:markerURL atomically:YES encoding:NSUTF8StringEncoding error:nil]; + NSLog(@"NewDotStartup: APP_READY durationMs=%lld", duration); +} + @end diff --git a/patches/react-native/react-native+0.85.3+040+pgo.patch b/patches/react-native/react-native+0.85.3+040+pgo.patch index 14ca0c45eb91..ae4cd4becbf6 100644 --- a/patches/react-native/react-native+0.85.3+040+pgo.patch +++ b/patches/react-native/react-native+0.85.3+040+pgo.patch @@ -139,3 +139,46 @@ index 8125e02..d466798 100644 include(CheckIPOSupported) check_ipo_supported(RESULT IPO_SUPPORT) if (IPO_SUPPORT) +diff --git a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +--- a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh ++++ b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +@@ -59,6 +59,19 @@ if [[ $xcode_major_version -ge 15 ]]; then + xcode_15_flags="LINKER:-ld_classic" + fi + ++pgo_compile_flags="" ++pgo_linker_flags="" ++if [[ "${EXPENSIFY_PGO_MODE:-off}" == "generate" ]]; then ++ pgo_compile_flags="-fprofile-instr-generate" ++ pgo_linker_flags="-fprofile-instr-generate" ++elif [[ "${EXPENSIFY_PGO_MODE:-off}" == "use" ]]; then ++ if [[ -z "${EXPENSIFY_PGO_PROFILE:-}" ]]; then ++ echo "EXPENSIFY_PGO_PROFILE is required when EXPENSIFY_PGO_MODE=use" >&2 ++ exit 1 ++ fi ++ pgo_compile_flags="-fprofile-instr-use=${EXPENSIFY_PGO_PROFILE} -Wno-profile-instr-unprofiled -Wno-profile-instr-out-of-date" ++fi ++ + boost_context_flag="" + if [[ $PLATFORM_NAME == "catalyst" ]]; then + boost_context_flag="-DHERMES_ALLOW_BOOST_CONTEXT=0" +@@ -70,7 +84,7 @@ echo "Configure Apple framework" + "$CMAKE_BINARY" \ + -S "${PODS_ROOT}/hermes-engine" \ + -B "${PODS_ROOT}/hermes-engine/build/${PLATFORM_NAME}" \ +- -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags" \ ++ -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags;$pgo_linker_flags" \ + -DHERMES_APPLE_TARGET_PLATFORM:STRING="$PLATFORM_NAME" \ + -DCMAKE_OSX_ARCHITECTURES:STRING="$architectures" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING="$deployment_target" \ +@@ -82,8 +96,8 @@ echo "Configure Apple framework" + -DHERMES_ENABLE_BITCODE:BOOLEAN=false \ + -DHERMES_BUILD_APPLE_FRAMEWORK:BOOLEAN=true \ + -DHERMES_BUILD_SHARED_JSI:BOOLEAN=false \ +- -DCMAKE_CXX_FLAGS:STRING="-gdwarf" \ +- -DCMAKE_C_FLAGS:STRING="-gdwarf" \ ++ -DCMAKE_CXX_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ ++ -DCMAKE_C_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ + -DIMPORT_HOST_COMPILERS:PATH="${hermesc_path}" \ + -DJSI_DIR="$jsi_path" \ + -DHERMES_RELEASE_VERSION="for RN $release_version" \ diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index ab4e91407ebb..d9bf17e07284 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,14 +1,24 @@ # Local PGO proof -This is an experimental, local-only LLVM PGO workflow. The TypeScript tool separates platform-specific operations from the shared build, profile, and benchmark workflows so iOS support can be added without duplicating the orchestration. Currently only the standalone Android arm64 release build is implemented: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow for Android and iOS. The TypeScript tool shares the build, profile, merge, and benchmark orchestration while each platform adapter handles its native build system and physical-device tooling. It builds React Native, Hermes, Nitro modules, and other source-based native dependencies from source so they can participate in PGO. Precompiled vendored frameworks cannot be instrumented retroactively. + +The workflow does not publish artifacts. It uses the existing NewDot `APP_READY` point immediately after the `ManualAppStartup` Sentry span ends as both the profile boundary and startup benchmark marker. ## Preconditions -- A physical arm64 Android device connected through `adb`. -- `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. +- A physical arm64 Android or iOS device. Simulators are not suitable for collecting a production-device PGO profile. +- For Android, `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. +- For iOS, Xcode command-line tools, a connected and unlocked device, and locally valid Apple Development signing settings for the app and its remaining extensions. The tool respects the project development team by default; override it with `IOS_DEVELOPMENT_TEAM` and select a device with `IOS_DEVICE_ID` when necessary. It reads the locally signed bundle identifier from the archived `.app`; `IOS_BUNDLE_IDENTIFIER` is available as an explicit override. - A signed-in, seeded test account and the agreed manual NewDot journey. - Dependencies have had the repository's patches applied. +Every command starts with a platform: + +```bash +scripts/pgo/local-proof.ts android --help +scripts/pgo/local-proof.ts ios --help +``` + ## Collect one profile 1. Build and install the instrumented application: @@ -101,3 +111,47 @@ The `.pgo/` directory is intentionally local-only. Never apply this profile to a ## Compare Install the release and optimized APKs consecutively. For each build, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. + +## iOS workflow + +The iOS commands mirror Android: + +```bash +scripts/pgo/local-proof.ts ios build-instrumented +scripts/pgo/local-proof.ts ios verify-instrumented +scripts/pgo/local-proof.ts ios install-instrumented +scripts/pgo/local-proof.ts ios record-startups 10 30 +scripts/pgo/local-proof.ts ios build-optimized +scripts/pgo/local-proof.ts ios build-release +scripts/pgo/local-proof.ts ios benchmark 10 30 +``` + +`record-startups` launches the installed instrumented app, clears old counters, performs the requested cold-process startups, explicitly flushes LLVM counters after every `APP_READY`, copies the raw profiles from the app data container, and merges them into `.pgo/ios/arm64/newdot.profdata`. The app must remain in the foreground until each dump completes; force-terminating an iOS app is not a reliable profile-flush mechanism. + +For a broader manual profile, use the same instrumented build, perform the agreed authenticated NewDot journey, and then run: + +```bash +scripts/pgo/local-proof.ts ios dump +scripts/pgo/local-proof.ts ios pull +scripts/pgo/local-proof.ts ios merge +scripts/pgo/local-proof.ts ios build-optimized +``` + +The iOS adapter uses an arm64 Release device build with local development signing. It forces React Native core, React Native dependencies, and Hermes to build from source. Xcode-level Clang and Swift frontend instrumentation covers app and source-based CocoaPods targets; a React Native patch forwards the matching flags into Hermes' nested CMake build. Release, instrumented, and optimized `.app` bundles are archived under `.pgo/ios/arm64/app/` and benchmarks under `.pgo/ios/benchmarks/`. + +The source-build preparation also validates the source roots CocoaPods needs for Hermes, `libdav1d`, and `libwebp`. If an existing generated checkout is incomplete, the tool removes only that Pod directory and lets the locked `pod install` restore it before compiling. This prevents missing private-header failures caused by a partially materialized CocoaPods sandbox. + +If signing is temporarily unavailable, `IOS_CODE_SIGNING_ALLOWED=NO scripts/pgo/local-proof.ts ios build-instrumented` builds an unsigned device artifact for compilation and instrumentation verification only. Unsigned apps cannot be installed, profiled, or benchmarked; the connected device and all app-extension bundle identifiers must be covered by valid development provisioning profiles for the complete workflow. + +The three-stage procedure is the same on both platforms, but the mechanics differ: + +| Stage | Android | iOS | +| --- | --- | --- | +| Build | Gradle/NDK | CocoaPods + `xcodebuild` | +| Instrument | Clang `-fprofile-generate` | Clang `-fprofile-instr-generate`, Swift `-profile-generate` | +| Flush | Android broadcast/JNI | Darwin notification/native LLVM runtime | +| Retrieve | `adb pull` from external cache | CoreDevice copy from the app data container | +| Merge | NDK `llvm-profdata` | Xcode `llvm-profdata` | +| Consume | Clang `-fprofile-use` | Clang `-fprofile-instr-use`, Swift `-profile-use` | + +Profiles are tied to the exact compiler, architecture, source, dependencies, and build configuration that produced them. Rebuild all three app variants from the same revision and Xcode version, do not reuse an Android profile on iOS, and regenerate the iOS profile after meaningful native-source or toolchain changes. The release and optimized benchmarks preserve app data and use the same bundle identifier so the authenticated state and data set remain comparable. diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 55f18fd02c46..2385734cc56c 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -4,7 +4,7 @@ import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; import {spawnSync} from 'node:child_process'; -import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; +import {copyFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; import {homedir, platform, tmpdir} from 'node:os'; import {dirname, join, resolve} from 'node:path'; import process from 'node:process'; @@ -15,6 +15,7 @@ const STARTUP_RELAUNCH_DELAY_MS = 500; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; +type PgoMode = 'off' | 'generate' | 'use'; type PlatformName = TupleToUnion; type WorkflowCommand = TupleToUnion; @@ -43,7 +44,8 @@ type PlatformAdapter = { build: (kind: BuildKind) => void; install: (kind: BuildKind) => void; verifyInstrumentation: () => void; - clearDeviceProfiles: () => void; + prepareProfileCollection?: (timeoutSeconds: number) => Promise; + clearDeviceProfiles: () => Promise; dumpProfiles: () => Promise; pullProfiles: () => void; forceStop: () => void; @@ -116,6 +118,11 @@ function captureBinary(command: string, args: string[], cwd = rootDirectory): Bu return result.stdout; } +function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { + const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); + return !result.error && result.status === 0; +} + function sleep(milliseconds: number): Promise { return new Promise((resolvePromise) => { setTimeout(resolvePromise, milliseconds); @@ -155,6 +162,15 @@ function parseChoice(rawValue: string, choices: readonly T[], return choice; } +function environmentString(name: string): string | undefined { + const value: unknown = process.env[name]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + function valueAt(values: T[], index: number): T { const value = values.at(index); if (value === undefined) { @@ -362,7 +378,7 @@ function createAndroidAdapter(): PlatformAdapter { build, install, verifyInstrumentation, - clearDeviceProfiles: () => { + clearDeviceProfiles: async () => { console.log(`Clearing previous device PGO profiles from ${deviceProfileDirectory}.`); adb(['shell', `rm -f '${deviceProfileDirectory}'/newdot-*.profraw`]); }, @@ -376,6 +392,415 @@ function createAndroidAdapter(): PlatformAdapter { }; } +function createIosAdapter(): PlatformAdapter { + const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const appDirectory = join(profileDirectory, 'app'); + const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); + const markerDirectory = join(profileDirectory, 'markers'); + const iosDirectory = join(rootDirectory, 'Mobile-Expensify/iOS'); + const hermesVersionPath = join(rootDirectory, 'node_modules/react-native/sdks/.hermesv1version'); + const remotePgoDirectory = 'Library/Caches/ExpensifyPGO'; + const profileStatusPath = `${remotePgoDirectory}/profile-status.txt`; + const appReadyMarkerPath = `${remotePgoDirectory}/app-ready.txt`; + const profileWriteNotification = 'com.expensify.pgo.write-profiles'; + const profileClearNotification = 'com.expensify.pgo.clear-profiles'; + const profileOperationTimeoutSeconds = 10; + const developmentTeam = environmentString('IOS_DEVELOPMENT_TEAM'); + const codeSigningAllowed = !['0', 'false', 'no'].includes((environmentString('IOS_CODE_SIGNING_ALLOWED') ?? 'yes').toLowerCase()); + const configuredBundleIdentifier = environmentString('IOS_BUNDLE_IDENTIFIER'); + const mergedProfilePath = join(profileDirectory, 'newdot.profdata'); + + const artifactPaths: BuildArtifactPaths = { + release: join(appDirectory, 'Expensify-release.app'), + instrumented: join(appDirectory, 'Expensify-release-instrumented.app'), + optimized: join(appDirectory, 'Expensify-release-optimized.app'), + }; + + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + let cachedDeviceIdentifier: string | undefined; + let cachedBundleIdentifier: string | undefined; + let previousAppReadyMarker: string | undefined; + + function bundleIdentifier(): string { + if (cachedBundleIdentifier) { + return cachedBundleIdentifier; + } + if (configuredBundleIdentifier) { + cachedBundleIdentifier = configuredBundleIdentifier; + return configuredBundleIdentifier; + } + + const archivedApp = [artifactPaths.instrumented, artifactPaths.optimized, artifactPaths.release].find((path) => existsSync(join(path, 'Info.plist'))); + if (!archivedApp) { + fail('No archived iOS app is available to determine its bundle identifier. Build an app first, or set IOS_BUNDLE_IDENTIFIER.'); + } + cachedBundleIdentifier = capture('/usr/libexec/PlistBuddy', ['-c', 'Print :CFBundleIdentifier', join(archivedApp, 'Info.plist')]).trim(); + if (!cachedBundleIdentifier) { + fail(`The archived app at ${archivedApp} has no CFBundleIdentifier.`); + } + console.log(`Using iOS bundle identifier ${cachedBundleIdentifier}.`); + return cachedBundleIdentifier; + } + + function deviceIdentifier(): string { + if (cachedDeviceIdentifier) { + return cachedDeviceIdentifier; + } + + const configuredIdentifier = environmentString('IOS_DEVICE_ID'); + if (configuredIdentifier) { + cachedDeviceIdentifier = configuredIdentifier; + return configuredIdentifier; + } + + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-ios-devices-')); + const jsonPath = join(temporaryDirectory, 'devices.json'); + try { + run('xcrun', ['devicectl', 'list', 'devices', '--json-output', jsonPath]); + const parsedDevices: unknown = JSON.parse(readFileSync(jsonPath, 'utf8')); + if (!isRecord(parsedDevices) || !isRecord(parsedDevices.result) || !Array.isArray(parsedDevices.result.devices)) { + fail('CoreDevice returned an unexpected device-list response.'); + } + + const physicalDevices = parsedDevices.result.devices.flatMap((device) => { + if (!isRecord(device) || !isRecord(device.hardwareProperties) || !isRecord(device.deviceProperties)) { + return []; + } + const {hardwareProperties, deviceProperties} = device; + if (hardwareProperties.platform !== 'iOS' || hardwareProperties.reality !== 'physical' || deviceProperties.bootState !== 'booted') { + return []; + } + return typeof hardwareProperties.udid === 'string' ? [{name: String(deviceProperties.name), udid: hardwareProperties.udid}] : []; + }); + + if (physicalDevices.length === 0) { + fail('No booted physical iOS device is connected. Connect and unlock a device, or set IOS_DEVICE_ID.'); + } + if (physicalDevices.length > 1) { + fail(`Multiple physical iOS devices are connected (${physicalDevices.map(({name}) => name).join(', ')}). Set IOS_DEVICE_ID to the desired UDID.`); + } + + const selectedDevice = valueAt(physicalDevices, 0); + cachedDeviceIdentifier = selectedDevice.udid; + console.log(`Using iOS device ${selectedDevice.name} (${selectedDevice.udid}).`); + return selectedDevice.udid; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } + } + + function llvmTool(name: string): string { + return capture('xcrun', ['--find', name]).trim(); + } + + function sourceBuildEnvironment(mode: PgoMode): string[] { + const environment = ['RCT_USE_RN_DEP=0', 'RCT_USE_PREBUILT_RNCORE=0', 'RCT_BUILD_HERMES_FROM_SOURCE=true', `EXPENSIFY_PGO_MODE=${mode}`]; + if (mode === 'use') { + environment.push(`EXPENSIFY_PGO_PROFILE=${mergedProfilePath}`); + } + return environment; + } + + function prepareSourcePods(mode: PgoMode): void { + if (!existsSync(hermesVersionPath)) { + fail(`Missing React Native's pinned Hermes version file at ${hermesVersionPath}.`); + } + + const requiredPodSources = [ + {directory: join(iosDirectory, 'Pods/hermes-engine'), requiredFile: 'CMakeLists.txt'}, + {directory: join(iosDirectory, 'Pods/libdav1d'), requiredFile: 'dav1d/include/common/intops.h'}, + {directory: join(iosDirectory, 'Pods/libwebp'), requiredFile: 'src/webp/types.h'}, + ]; + for (const {directory, requiredFile} of requiredPodSources) { + if (existsSync(directory) && !existsSync(join(directory, requiredFile))) { + console.log(`Removing incomplete CocoaPods checkout: ${directory}`); + rmSync(directory, {recursive: true, force: true}); + } + } + + run('/usr/bin/env', [...sourceBuildEnvironment(mode), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); + + for (const {directory, requiredFile} of requiredPodSources) { + if (!existsSync(join(directory, requiredFile))) { + fail(`CocoaPods did not install the required source file ${join(directory, requiredFile)}.`); + } + } + + const hermesBuildDirectory = join(iosDirectory, 'Pods/hermes-engine/build'); + rmSync(hermesBuildDirectory, {recursive: true, force: true}); + } + + function pgoBuildSettings(kind: BuildKind): string[] { + if (kind === 'instrumented') { + return [ + 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', + 'OTHER_CFLAGS=$(inherited) -fprofile-instr-generate', + 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -profile-generate', + 'OTHER_LDFLAGS=$(inherited) -fprofile-instr-generate', + ]; + } + if (kind === 'optimized') { + const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-profile-instr-unprofiled -Wno-profile-instr-out-of-date`; + return [ + 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_SWIFT_FLAGS=$(inherited) -profile-use=${mergedProfilePath}`, + ]; + } + return ['CLANG_USE_OPTIMIZATION_PROFILE=NO']; + } + + function build(kind: BuildKind): void { + if (kind === 'optimized' && !existsSync(mergedProfilePath)) { + fail(`Missing ${mergedProfilePath}. Run merge first.`); + } + + let mode: PgoMode = 'off'; + if (kind === 'instrumented') { + mode = 'generate'; + } else if (kind === 'optimized') { + mode = 'use'; + } + prepareSourcePods(mode); + + const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); + const builtAppPath = join(derivedDataDirectory, 'Build/Products/Release-iphoneos/Expensify.app'); + const signingSettings = codeSigningAllowed + ? [ + 'CODE_SIGN_STYLE=Automatic', + 'CODE_SIGN_IDENTITY=Apple Development', + 'PROVISIONING_PROFILE=', + 'PROVISIONING_PROFILE_SPECIFIER=', + ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), + ] + : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; + const xcodeArguments = [ + 'xcodebuild', + '-workspace', + 'Expensify.xcworkspace', + '-scheme', + 'Expensify', + '-configuration', + 'Release', + '-destination', + `id=${deviceIdentifier()}`, + '-derivedDataPath', + derivedDataDirectory, + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), + '-quiet', + 'build', + 'ARCHS=arm64', + 'ONLY_ACTIVE_ARCH=YES', + 'COMPILER_INDEX_STORE_ENABLE=NO', + ...signingSettings, + ...pgoBuildSettings(kind), + ]; + run('/usr/bin/env', [...sourceBuildEnvironment(mode), ...xcodeArguments], iosDirectory); + + if (!existsSync(builtAppPath)) { + fail(`Missing built iOS app at ${builtAppPath}.`); + } + mkdirSync(appDirectory, {recursive: true}); + rmSync(artifactPaths[kind], {recursive: true, force: true}); + cpSync(builtAppPath, artifactPaths[kind], {recursive: true, preserveTimestamps: true}); + console.log(`Copied release app: ${artifactPaths[kind]}`); + } + + function install(kind: BuildKind): void { + const appPath = artifactPaths[kind]; + if (!existsSync(appPath)) { + fail(`Missing app at ${appPath}. Build it first.`); + } + if (!existsSync(join(appPath, '_CodeSignature/CodeResources'))) { + fail(`The app at ${appPath} is unsigned. Rebuild without IOS_CODE_SIGNING_ALLOWED=NO before installing it on a device.`); + } + run('xcrun', ['devicectl', 'device', 'install', 'app', '--device', deviceIdentifier(), appPath]); + console.log(`Installed app: ${appPath}`); + } + + function tryReadDeviceFile(remotePath: string, localName: string): string | undefined { + mkdirSync(markerDirectory, {recursive: true}); + const localPath = join(markerDirectory, localName); + rmSync(localPath, {recursive: true, force: true}); + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePath, + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8').trim() : undefined; + } + + function postNotification(name: string): void { + run('xcrun', ['devicectl', 'device', 'notification', 'post', '--device', deviceIdentifier(), '--name', name]); + } + + async function waitForProfileStatus(expectedStatus: string, previousStatus: string | undefined): Promise { + const deadline = Date.now() + profileOperationTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const statusContents = tryReadDeviceFile(profileStatusPath, 'profile-status.txt'); + if (statusContents && statusContents !== previousStatus) { + const [, status, rawResult] = statusContents.split(','); + if (status === 'not-instrumented') { + fail('The installed iOS app is not instrumented. Build and install build-instrumented before recording profiles.'); + } + if (status === expectedStatus && Number(rawResult) === 0) { + console.log(`iOS PGO profiles ${expectedStatus}.`); + return; + } + fail(`The iOS PGO profile operation failed: ${statusContents}`); + } + await sleep(250); + } + fail(`The iOS app did not confirm that PGO profiles were ${expectedStatus} within ${profileOperationTimeoutSeconds}s.`); + } + + async function clearDeviceProfiles(): Promise { + console.log(`Clearing previous iOS PGO profiles from ${remotePgoDirectory}.`); + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-clear.txt'); + postNotification(profileClearNotification); + await waitForProfileStatus('cleared', previousStatus); + } + + async function dumpProfiles(): Promise { + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-write.txt'); + postNotification(profileWriteNotification); + await waitForProfileStatus('written', previousStatus); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(dirname(rawProfileDirectory), {recursive: true}); + run('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePgoDirectory, + '--destination', + rawProfileDirectory, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + ]); + + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in the iOS app container at ${remotePgoDirectory}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + function forceStop(): void { + runAllowFailure('xcrun', ['devicectl', 'device', 'process', 'terminate', '--device', deviceIdentifier(), bundleIdentifier()]); + } + + function launch(): void { + previousAppReadyMarker = tryReadDeviceFile(appReadyMarkerPath, 'app-ready-before-launch.txt'); + run('xcrun', ['devicectl', 'device', 'process', 'launch', '--device', deviceIdentifier(), '--terminate-existing', bundleIdentifier()]); + } + + async function waitForAppReady(timeoutSeconds: number): Promise { + const deadline = Date.now() + timeoutSeconds * 1000; + console.log(`Waiting up to ${timeoutSeconds}s for the iOS NewDot APP_READY marker.`); + while (Date.now() < deadline) { + const marker = tryReadDeviceFile(appReadyMarkerPath, 'app-ready.txt'); + if (marker && marker !== previousAppReadyMarker) { + const [, rawDuration] = marker.split(','); + const duration = Number(rawDuration); + if (Number.isFinite(duration) && duration >= 0) { + console.log(`NewDot reported APP_READY after ${duration}ms.`); + return duration; + } + return null; + } + await sleep(250); + } + fail(`NewDot did not report APP_READY on iOS within ${timeoutSeconds}s.`); + } + + function verifyInstrumentation(): void { + const appPath = artifactPaths.instrumented; + const executablePath = join(appPath, 'Expensify'); + if (!existsSync(executablePath)) { + fail(`Missing instrumented iOS executable at ${executablePath}.`); + } + + const sectionHeaders = capture('xcrun', ['size', '-m', executablePath]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => sectionHeaders.includes(section))) { + fail('LLVM PGO instrumentation is missing from the iOS app executable.'); + } + const executableStrings = capture('strings', ['-a', executablePath]); + if (!['ExpensifyPGO: wrote LLVM profile', 'NewDotStartup: APP_READY'].every((marker) => executableStrings.includes(marker))) { + fail('The iOS app executable is missing the PGO flush or APP_READY marker.'); + } + console.log(`Verified LLVM PGO instrumentation: ${executablePath}`); + + const hermesExecutable = join(appPath, 'Frameworks/hermesvm.framework/hermesvm'); + if (!existsSync(hermesExecutable)) { + fail(`Missing embedded source-built Hermes framework at ${hermesExecutable}.`); + } + const hermesSections = capture('xcrun', ['size', '-m', hermesExecutable]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => hermesSections.includes(section))) { + fail('LLVM PGO instrumentation is missing from the source-built iOS Hermes framework.'); + } + console.log(`Verified LLVM PGO instrumentation: ${hermesExecutable}`); + } + + async function prepareProfileCollection(timeoutSeconds: number): Promise { + console.log('Launching the instrumented iOS app once so its PGO notification handlers are active.'); + forceStop(); + await sleep(STARTUP_RELAUNCH_DELAY_MS); + launch(); + await waitForAppReady(timeoutSeconds); + } + + return { + name: 'ios', + profileDirectory, + rawProfileDirectory, + mergedProfilePath, + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + build, + install, + verifyInstrumentation, + prepareProfileCollection, + clearDeviceProfiles, + dumpProfiles, + pullProfiles, + forceStop, + clearSystemLogs: () => undefined, + launch, + waitForAppReady, + llvmTool, + }; +} + function mergeProfiles(adapter: PlatformAdapter): void { mkdirSync(adapter.profileDirectory, {recursive: true}); const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); @@ -397,13 +822,16 @@ async function measureStartup(adapter: PlatformAdapter, readyTimeoutSeconds: num adapter.launch(); const duration = await adapter.waitForAppReady(readyTimeoutSeconds); if (duration === null) { - fail('APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes.'); + fail(`APP_READY did not contain a numeric durationMs marker. Rebuild the ${adapter.name} app with the startup metric changes.`); } return duration; } async function recordStartups(adapter: PlatformAdapter, runs: number, readyTimeoutSeconds: number): Promise { - adapter.clearDeviceProfiles(); + if (adapter.prepareProfileCollection) { + await adapter.prepareProfileCollection(readyTimeoutSeconds); + } + await adapter.clearDeviceProfiles(); for (let runNumber = 1; runNumber <= runs; runNumber += 1) { console.log(`Recording cold-process startup ${runNumber}/${runs}.`); await measureStartup(adapter, readyTimeoutSeconds); @@ -511,7 +939,7 @@ function getAdapter(platformName: PlatformName): PlatformAdapter { if (platformName === 'android') { return createAndroidAdapter(); } - fail('The iOS PGO workflow is not implemented yet.'); + return createIosAdapter(); } async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number): Promise { diff --git a/scripts/pod-install.sh b/scripts/pod-install.sh index b0739b28d4a4..fbbb1ce890cb 100755 --- a/scripts/pod-install.sh +++ b/scripts/pod-install.sh @@ -54,7 +54,10 @@ if [[ "$IS_HYBRID_APP_REPO" == "true" && "$NEW_DOT_FLAG" == "false" ]]; then # Navigate to the OldDot repository, and run bundle install and pod install cd Mobile-Expensify/ios bundle install - RCT_USE_RN_DEP=0 RCT_USE_PREBUILT_RNCORE=0 bundle exec pod install + RCT_USE_RN_DEP=0\ + RCT_USE_PREBUILT_RNCORE=0\ + RCT_BUILD_HERMES_FROM_SOURCE=true\ + bundle exec pod install exit 0 fi From fa69456727635b15cf1ff87bd16845895a628d20 Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 18:20:23 +0200 Subject: [PATCH 26/47] fix: ios optimized build --- scripts/pgo/README.md | 6 +++--- scripts/pgo/local-proof.ts | 26 ++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index d9bf17e07284..c7fc898dfcff 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -137,7 +137,7 @@ scripts/pgo/local-proof.ts ios merge scripts/pgo/local-proof.ts ios build-optimized ``` -The iOS adapter uses an arm64 Release device build with local development signing. It forces React Native core, React Native dependencies, and Hermes to build from source. Xcode-level Clang and Swift frontend instrumentation covers app and source-based CocoaPods targets; a React Native patch forwards the matching flags into Hermes' nested CMake build. Release, instrumented, and optimized `.app` bundles are archived under `.pgo/ios/arm64/app/` and benchmarks under `.pgo/ios/benchmarks/`. +The iOS adapter uses an arm64 Release device build with local development signing. It forces React Native core, React Native dependencies, and Hermes to build from source. Xcode-level Clang frontend instrumentation and Swift IR instrumentation cover app and source-based CocoaPods targets; a React Native patch forwards the matching flags into Hermes' nested CMake build. Release, instrumented, and optimized `.app` bundles are archived under `.pgo/ios/arm64/app/` and benchmarks under `.pgo/ios/benchmarks/`. The source-build preparation also validates the source roots CocoaPods needs for Hermes, `libdav1d`, and `libwebp`. If an existing generated checkout is incomplete, the tool removes only that Pod directory and lets the locked `pod install` restore it before compiling. This prevents missing private-header failures caused by a partially materialized CocoaPods sandbox. @@ -148,10 +148,10 @@ The three-stage procedure is the same on both platforms, but the mechanics diffe | Stage | Android | iOS | | --- | --- | --- | | Build | Gradle/NDK | CocoaPods + `xcodebuild` | -| Instrument | Clang `-fprofile-generate` | Clang `-fprofile-instr-generate`, Swift `-profile-generate` | +| Instrument | Clang `-fprofile-generate` | Clang `-fprofile-instr-generate`, Swift `-ir-profile-generate` | | Flush | Android broadcast/JNI | Darwin notification/native LLVM runtime | | Retrieve | `adb pull` from external cache | CoreDevice copy from the app data container | | Merge | NDK `llvm-profdata` | Xcode `llvm-profdata` | -| Consume | Clang `-fprofile-use` | Clang `-fprofile-instr-use`, Swift `-profile-use` | +| Consume | Clang `-fprofile-use` | Clang `-fprofile-instr-use`, Swift `-ir-profile-use` | Profiles are tied to the exact compiler, architecture, source, dependencies, and build configuration that produced them. Rebuild all three app variants from the same revision and Xcode version, do not reuse an Android profile on iOS, and regenerate the iOS profile after meaningful native-source or toolchain changes. The release and optimized benchmarks preserve app data and use the same bundle identifier so the authenticated state and data set remain comparable. diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 2385734cc56c..32b8c7a44c14 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -41,6 +41,7 @@ type PlatformAdapter = { readonly benchmarkDirectory: string; readonly benchmarkPaths: Record; readonly artifactPaths: BuildArtifactPaths; + readonly profileFormat?: string; build: (kind: BuildKind) => void; install: (kind: BuildKind) => void; verifyInstrumentation: () => void; @@ -393,6 +394,7 @@ function createAndroidAdapter(): PlatformAdapter { } function createIosAdapter(): PlatformAdapter { + const profileFormat = 'ios-clang-frontend-swift-ir-v1'; const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); const rawProfileDirectory = join(profileDirectory, 'raw'); const appDirectory = join(profileDirectory, 'app'); @@ -536,32 +538,40 @@ function createIosAdapter(): PlatformAdapter { } function pgoBuildSettings(kind: BuildKind): string[] { + const baseSettings = ['ENABLE_CODE_COVERAGE=NO', 'CLANG_COVERAGE_MAPPING=NO', 'CLANG_USE_OPTIMIZATION_PROFILE=NO']; if (kind === 'instrumented') { return [ - 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + ...baseSettings, 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', 'OTHER_CFLAGS=$(inherited) -fprofile-instr-generate', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-generate', - 'OTHER_SWIFT_FLAGS=$(inherited) -profile-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-generate', 'OTHER_LDFLAGS=$(inherited) -fprofile-instr-generate', ]; } if (kind === 'optimized') { const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-profile-instr-unprofiled -Wno-profile-instr-out-of-date`; return [ - 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, - `OTHER_SWIFT_FLAGS=$(inherited) -profile-use=${mergedProfilePath}`, + `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${mergedProfilePath}`, ]; } - return ['CLANG_USE_OPTIMIZATION_PROFILE=NO']; + return baseSettings; } function build(kind: BuildKind): void { if (kind === 'optimized' && !existsSync(mergedProfilePath)) { fail(`Missing ${mergedProfilePath}. Run merge first.`); } + if (kind === 'optimized') { + const profileFormatPath = `${mergedProfilePath}.format`; + const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; + if (recordedFormat !== profileFormat) { + fail('The merged iOS profile predates the current Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); + } + } let mode: PgoMode = 'off'; if (kind === 'instrumented') { @@ -591,7 +601,7 @@ function createIosAdapter(): PlatformAdapter { '-configuration', 'Release', '-destination', - `id=${deviceIdentifier()}`, + 'generic/platform=iOS', '-derivedDataPath', derivedDataDirectory, ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), @@ -786,6 +796,7 @@ function createIosAdapter(): PlatformAdapter { benchmarkDirectory, benchmarkPaths, artifactPaths, + profileFormat, build, install, verifyInstrumentation, @@ -810,6 +821,9 @@ function mergeProfiles(adapter: PlatformAdapter): void { const llvmProfdata = adapter.llvmTool('llvm-profdata'); run(llvmProfdata, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + if (adapter.profileFormat) { + writeFileSync(`${adapter.mergedProfilePath}.format`, `${adapter.profileFormat}\n`); + } const profileReport = capture(llvmProfdata, ['show', '--all-functions', adapter.mergedProfilePath]); writeFileSync(`${adapter.mergedProfilePath}.txt`, profileReport); console.log(`Merged PGO profile: ${adapter.mergedProfilePath}`); From 5ddd179edffa7463a9e4f31d2840b4c26d735eff Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 18:28:06 +0200 Subject: [PATCH 27/47] fix: benchmark release first --- scripts/pgo/local-proof.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 32b8c7a44c14..7e263a05cd3e 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -875,6 +875,21 @@ async function benchmarkStartups(adapter: PlatformAdapter, kind: BenchmarkKind, console.log(`Recorded ${runs} ${label} startup samples: ${benchmarkPath}`); } +async function benchmarkAll(adapter: PlatformAdapter, runs: number, readyTimeoutSeconds: number): Promise { + console.log('=== Benchmark phase 1/2: Release ==='); + console.log(`Installing release artifact: ${adapter.artifactPaths.release}`); + adapter.install('release'); + await benchmarkStartups(adapter, 'release', runs, readyTimeoutSeconds); + + console.log('=== Benchmark phase 2/2: PGO optimized ==='); + console.log(`Installing PGO optimized artifact: ${adapter.artifactPaths.optimized}`); + adapter.install('optimized'); + await benchmarkStartups(adapter, 'optimized', runs, readyTimeoutSeconds); + + console.log('=== Benchmark comparison ==='); + compareBenchmarks(adapter); +} + function percentile(sortedValues: number[], fraction: number): number { const position = (sortedValues.length - 1) * fraction; const lowerIndex = Math.floor(position); @@ -993,11 +1008,7 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); return; case 'benchmark': - adapter.install('release'); - await benchmarkStartups(adapter, 'release', runs, timeoutSeconds); - adapter.install('optimized'); - await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); - compareBenchmarks(adapter); + await benchmarkAll(adapter, runs, timeoutSeconds); return; case 'compare-benchmarks': compareBenchmarks(adapter); From 22a19cd57bf91882be7468a80a825ae886c2a633 Mon Sep 17 00:00:00 2001 From: chrispader Date: Thu, 6 Aug 2026 15:07:47 +0200 Subject: [PATCH 28/47] feat: allow defining device --- scripts/pgo/local-proof.ts | 61 +++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 7e263a05cd3e..6823d63bffec 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -180,7 +180,7 @@ function valueAt(values: T[], index: number): T { return value; } -function createAndroidAdapter(): PlatformAdapter { +function createAndroidAdapter(configuredDeviceIdentifier?: string): PlatformAdapter { const ndkVersion = '27.1.12297006'; const packageName = 'org.me.mobiexpensifyg'; const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); @@ -207,11 +207,11 @@ function createAndroidAdapter(): PlatformAdapter { }; function adb(args: string[]): void { - run('adb', args); + run('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); } function adbCapture(args: string[]): string { - return capture('adb', args); + return capture('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); } function llvmTool(name: string): string { @@ -393,7 +393,7 @@ function createAndroidAdapter(): PlatformAdapter { }; } -function createIosAdapter(): PlatformAdapter { +function createIosAdapter(cliDeviceIdentifier?: string): PlatformAdapter { const profileFormat = 'ios-clang-frontend-swift-ir-v1'; const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); const rawProfileDirectory = join(profileDirectory, 'raw'); @@ -454,12 +454,7 @@ function createIosAdapter(): PlatformAdapter { return cachedDeviceIdentifier; } - const configuredIdentifier = environmentString('IOS_DEVICE_ID'); - if (configuredIdentifier) { - cachedDeviceIdentifier = configuredIdentifier; - return configuredIdentifier; - } - + const configuredIdentifier = cliDeviceIdentifier ?? environmentString('IOS_DEVICE_ID'); const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-ios-devices-')); const jsonPath = join(temporaryDirectory, 'devices.json'); try { @@ -477,17 +472,40 @@ function createIosAdapter(): PlatformAdapter { if (hardwareProperties.platform !== 'iOS' || hardwareProperties.reality !== 'physical' || deviceProperties.bootState !== 'booted') { return []; } - return typeof hardwareProperties.udid === 'string' ? [{name: String(deviceProperties.name), udid: hardwareProperties.udid}] : []; + if (typeof hardwareProperties.udid !== 'string' || typeof deviceProperties.name !== 'string') { + return []; + } + const name = deviceProperties.name.trim(); + const osVersion = typeof deviceProperties.osVersionNumber === 'string' ? deviceProperties.osVersionNumber : undefined; + const aliases = [name, hardwareProperties.udid]; + if (typeof device.identifier === 'string') { + aliases.push(device.identifier); + } + if (typeof hardwareProperties.serialNumber === 'string') { + aliases.push(hardwareProperties.serialNumber); + } + if (osVersion) { + aliases.push(`${name} (${osVersion})`, `${name} (${osVersion}) (${hardwareProperties.udid})`); + } + return [{aliases, name, udid: hardwareProperties.udid}]; }); if (physicalDevices.length === 0) { fail('No booted physical iOS device is connected. Connect and unlock a device, or set IOS_DEVICE_ID.'); } - if (physicalDevices.length > 1) { + + const normalizeDeviceSelector = (value: string) => value.trim().replaceAll(/\s+/g, ' ').toLowerCase(); + const matchingDevices = configuredIdentifier + ? physicalDevices.filter(({aliases}) => aliases.some((alias) => normalizeDeviceSelector(alias) === normalizeDeviceSelector(configuredIdentifier))) + : physicalDevices; + if (matchingDevices.length === 0) { + fail(`No connected physical iOS device matches "${configuredIdentifier}". Available devices: ${physicalDevices.map(({name, udid}) => `${name} (${udid})`).join(', ')}.`); + } + if (matchingDevices.length > 1) { fail(`Multiple physical iOS devices are connected (${physicalDevices.map(({name}) => name).join(', ')}). Set IOS_DEVICE_ID to the desired UDID.`); } - const selectedDevice = valueAt(physicalDevices, 0); + const selectedDevice = valueAt(matchingDevices, 0); cachedDeviceIdentifier = selectedDevice.udid; console.log(`Using iOS device ${selectedDevice.name} (${selectedDevice.udid}).`); return selectedDevice.udid; @@ -964,15 +982,15 @@ function compareBenchmarks(adapter: PlatformAdapter): void { ); } -function getAdapter(platformName: PlatformName): PlatformAdapter { +function getAdapter(platformName: PlatformName, deviceIdentifier?: string): PlatformAdapter { if (platformName === 'android') { - return createAndroidAdapter(); + return createAndroidAdapter(deviceIdentifier); } - return createIosAdapter(); + return createIosAdapter(deviceIdentifier); } -async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number): Promise { - const adapter = getAdapter(platformName); +async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number, deviceIdentifier?: string): Promise { + const adapter = getAdapter(platformName, deviceIdentifier); switch (workflow) { case 'build-release': @@ -1053,6 +1071,12 @@ async function main(): Promise { parse: (value) => parsePositiveInteger(value, 'App-ready timeout'), }, ], + namedArgs: { + device: { + description: 'Device identifier to use (adb serial on Android; CoreDevice identifier, UDID, serial number, or device name on iOS)', + required: false, + }, + }, }); const {platform: platformName, workflow, runs, timeout} = cli.positionalArgs; @@ -1061,6 +1085,7 @@ async function main(): Promise { parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), requirePositiveInteger(Number(runs), 'Startup run count'), requirePositiveInteger(Number(timeout), 'App-ready timeout'), + cli.namedArgs.device, ); } From d35b7f5ef5c44f582c210ab0fe251c1f8ed402c0 Mon Sep 17 00:00:00 2001 From: chrispader Date: Thu, 6 Aug 2026 15:09:26 +0200 Subject: [PATCH 29/47] Update README.md --- scripts/pgo/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index c7fc898dfcff..981b5b803c6a 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -8,7 +8,7 @@ The workflow does not publish artifacts. It uses the existing NewDot `APP_READY` - A physical arm64 Android or iOS device. Simulators are not suitable for collecting a production-device PGO profile. - For Android, `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. -- For iOS, Xcode command-line tools, a connected and unlocked device, and locally valid Apple Development signing settings for the app and its remaining extensions. The tool respects the project development team by default; override it with `IOS_DEVELOPMENT_TEAM` and select a device with `IOS_DEVICE_ID` when necessary. It reads the locally signed bundle identifier from the archived `.app`; `IOS_BUNDLE_IDENTIFIER` is available as an explicit override. +- For iOS, Xcode command-line tools, a connected and unlocked device, and locally valid Apple Development signing settings for the app and its remaining extensions. The tool respects the project development team by default; override it with `IOS_DEVELOPMENT_TEAM`. Select a device with `--device`; `IOS_DEVICE_ID` remains available as an environment fallback. It reads the locally signed bundle identifier from the archived `.app`; `IOS_BUNDLE_IDENTIFIER` is available as an explicit override. - A signed-in, seeded test account and the agreed manual NewDot journey. - Dependencies have had the repository's patches applied. @@ -104,6 +104,18 @@ Or install and benchmark the archived release APK first, then install and benchm scripts/pgo/local-proof.ts android benchmark 10 30 ``` +Select a specific connected device for install or benchmark commands with `--device`. Use an adb serial for Android and any identifier accepted by CoreDevice for iOS, such as a CoreDevice identifier, UDID, serial number, or device name: + +```bash +scripts/pgo/local-proof.ts --device emulator-5554 android install-optimized +scripts/pgo/local-proof.ts --device emulator-5554 android benchmark 10 30 +scripts/pgo/local-proof.ts --device Chris14Pro ios install-optimized +scripts/pgo/local-proof.ts --device Chris14Pro ios benchmark 10 30 +scripts/pgo/local-proof.ts --device "Chris14Pro (26.6) (00008120-00065D541E3B401E)" ios benchmark 10 30 +``` + +Quote selectors containing spaces or parentheses. The iOS resolver accepts either the plain device name or the complete physical-device entry printed under `== Devices ==` by `xcrun xctrace list devices`; it does not select entries from the simulator section. + Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the average, P50, P75, P90, P95, P99, minimum, maximum, and percentage improvements. Percentiles use linear interpolation, and positive improvement percentages mean the optimized build was faster. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. From 9befe34fdb832f100984705b5e69c5f6c4b8a389 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 12:59:42 +0200 Subject: [PATCH 30/47] chore: allow native PGO terminology in spelling checks --- modules/ExpensifyNitroUtils/android/CMakeLists.txt | 2 ++ scripts/pgo/README.md | 2 ++ scripts/pgo/android.ts | 2 ++ scripts/pgo/ios.ts | 2 ++ scripts/pgo/pgo.ts | 2 ++ tests/tooling/pgo.test.ts | 1 + 6 files changed, 11 insertions(+) diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index 0936f76325d7..ab05b09f3c0c 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -1,4 +1,6 @@ project(ExpensifyNitroUtils) +# cspell:ignore STREQUAL libclang fprofile defsym + cmake_minimum_required(VERSION 3.9.0) set(PACKAGE_NAME ExpensifyNitroUtils) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index ad7122bb583c..54729c66c1ee 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,5 +1,7 @@ # Native profile-guided optimization + + This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. The command enables the existing `ManualAppStartup` benchmark span in every app it builds. Profile collection waits for that span to finish, then explicitly writes the LLVM counters before the app process is stopped. Benchmarking uses the same native span tooling as the repository's general startup benchmark. diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 55b0d532c76d..0a35e42a7770 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -1,3 +1,5 @@ +// cspell:ignore sdcard profdata Ppatched Ppgo profraw Readelf readelf zipinfo libreactnative libhermesvm libjsi cnts + /** Android-specific release builds, native profile persistence, retrieval, and instrumentation checks. */ import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index cd76d60c9dce..c262db94059d 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -1,3 +1,5 @@ +// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm + /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index e64a854d2d1d..1b2767c7bb6e 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -1,5 +1,7 @@ #!/usr/bin/env bun +// cspell:ignore profraw Profdata profdata + import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; diff --git a/tests/tooling/pgo.test.ts b/tests/tooling/pgo.test.ts index 5b18494f2ed0..8839e05fc7c7 100644 --- a/tests/tooling/pgo.test.ts +++ b/tests/tooling/pgo.test.ts @@ -1,4 +1,5 @@ import {describe, expect, it} from 'bun:test'; +// cspell:ignore chrispader import {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; import {percentageImprovement} from '@scripts/pgo/pgo'; From 169f09f7a45876333679dd42e20628344d697d77 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 13:12:31 +0200 Subject: [PATCH 31/47] docs: clarify PGO scenario isolation and artifact matching --- scripts/pgo/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 54729c66c1ee..ecf487661453 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -85,6 +85,8 @@ scripts/pgo/pgo.ts android merge Use the equivalent `ios` commands on iOS. A representative journey should exercise the common signed-in path: open and scroll chats, send a message, visit a workspace setting, attach and view a file, and run a search. Keep account state, data size, and network conditions stable between collections. +Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The current command automates this lifecycle for startup collection; a mixed-scenario CI runner still needs that orchestration. + ## Build and benchmark the optimized app Build the optimized app only after recording a fresh profile. Build the baseline release from the same source revision and toolchain: @@ -129,6 +131,8 @@ The current `record-startups` command accumulates and merges the startup runs au Run profile generation as a scheduled or release-candidate job, not on every pull request. Use pinned self-hosted runners or a device farm, preferably with physical arm64 devices, the production compiler versions, stable thermal and power conditions, a seeded account, and deterministic local fixtures where possible. Build one instrumented artifact per platform, run the fixed startup and interaction suites against that exact artifact, flush after every successful scenario, merge the separate outputs with documented weights, then build the optimized artifact without changing the checkout or toolchain. +Store both repository revisions, dependency locks, compiler and SDK versions, ABI, build settings, scenario version, and test-data fixture with each profile. Reject mismatched artifacts before the optimized build. Keep the instrumented build and raw profiles so a failed collection can be investigated without rerunning it. + Benchmark both variants on the same device after a cooldown, discard post-install warm-ups, and retain raw samples. The `benchmark` workflow runs all baseline samples before all optimized samples, so thermal or time-dependent drift can bias the comparison. The two archived builds intentionally share an identifier to preserve app data, which prevents side-by-side alternation. CI can reduce drift with ABBA install blocks that preserve the shared identifier and discard each post-install warm-up. Another option is to give the builds distinct identifiers, seed matched state, and use the repository's alternating startup benchmark. Evaluate both startup and held-out interactive journeys that were excluded from training. Gate on several releases of data rather than a single noisy run. Hosted CI is usually insufficient for the complete workflow because hosted runners rarely expose stable physical devices. Android emulators can validate automation and may contribute useful same-ABI counts when the workload is representative, but final validation should use target devices. iOS device collection also needs signing and physical-device access. A small self-hosted device pool or managed device farm is the realistic route to automation. From 66a65c1b7e593f6d377554b9b5d28ff89f09e7e8 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 13:38:35 +0200 Subject: [PATCH 32/47] fix: support Xcode 27 benchmark process discovery --- scripts/lib/nativeAppBenchmark.ts | 2 ++ scripts/lib/nativeAppBenchmark/ios.ts | 32 +++++++++++------------ tests/tooling/benchmarkAppStartup.test.ts | 32 ++++++++++++++++++++++- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/scripts/lib/nativeAppBenchmark.ts b/scripts/lib/nativeAppBenchmark.ts index bce54b015663..434fc2c038fe 100644 --- a/scripts/lib/nativeAppBenchmark.ts +++ b/scripts/lib/nativeAppBenchmark.ts @@ -6,6 +6,7 @@ import {assertAndroidAppInstalled, createAndroidAdapter, parseAndroidProcessIden import { createIOSAdapter, iOSBenchmarkMarkerPath, + iOSProcessListArguments, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, @@ -27,6 +28,7 @@ export { createNativeAppBenchmarkAdapter, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseBenchmarkLogEvents, diff --git a/scripts/lib/nativeAppBenchmark/ios.ts b/scripts/lib/nativeAppBenchmark/ios.ts index fa82833d1719..d5a82d60b543 100644 --- a/scripts/lib/nativeAppBenchmark/ios.ts +++ b/scripts/lib/nativeAppBenchmark/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore BEGINSWITH devicectl +// cspell:ignore devicectl /** Implements iOS benchmark setup, launch, process control, and marker collection through CoreDevice. */ @@ -67,20 +67,7 @@ async function createIOSAdapter({rootDirectory, deviceIdentifier, appID}: Omit 0 ? processIdentifier : undefined; } +/** Lists every process so client-side matching works across CoreDevice versions with different filter field names. */ +function iOSProcessListArguments(deviceIdentifier: string, outputPath: string): string[] { + return ['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']; +} + /** Maps a span name to its encoded marker path inside the iOS app data container. */ function iOSBenchmarkMarkerPath(spanName: string): string { return `${IOS_BENCHMARK_DIRECTORY}/${encodeURIComponent(spanName)}.log`; @@ -283,5 +275,13 @@ function isCoreDeviceInstalledApp(value: JsonValue): value is CoreDeviceInstalle return isJSONObject(value) && typeof value.bundleIdentifier === 'string' && typeof value.url === 'string'; } -export {createIOSAdapter, iOSBenchmarkMarkerPath, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, parseIOSRunningAppProcessIdentifier}; +export { + createIOSAdapter, + iOSBenchmarkMarkerPath, + iOSProcessListArguments, + parseIOSInstalledAppURL, + parseIOSInstalledAppsResponse, + parseIOSLaunchProcessIdentifier, + parseIOSRunningAppProcessIdentifier, +}; export type {CoreDeviceInstalledAppsResponse}; diff --git a/tests/tooling/benchmarkAppStartup.test.ts b/tests/tooling/benchmarkAppStartup.test.ts index 0385db56a12a..4c213eb7ef12 100644 --- a/tests/tooling/benchmarkAppStartup.test.ts +++ b/tests/tooling/benchmarkAppStartup.test.ts @@ -1,4 +1,4 @@ -// cspell:ignore appex +// cspell:ignore appex devicectl import {describe, expect, it, jest} from 'bun:test'; @@ -7,6 +7,7 @@ import { assertAndroidAppInstalled, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseIOSInstalledAppURL, @@ -78,6 +79,35 @@ describe('benchmarkAppStartup', () => { expect(processIdentifier).toBe(456); }); + it('lists iOS processes without a version-specific CoreDevice filter', () => { + // Given + const deviceIdentifier = '00008120-EXAMPLE'; + const outputPath = '/tmp/processes.json'; + + // When + const args = iOSProcessListArguments(deviceIdentifier, outputPath); + + // Then + expect(args).toEqual(['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']); + expect(args).not.toContain('--filter'); + }); + + it('ignores an extension process when the main iOS app is not running', () => { + // Given + const appURL = 'file:///private/var/containers/Bundle/Application/EXAMPLE/Expensify.app/'; + const response = { + result: { + runningProcesses: [{executable: `${appURL}PlugIns/LiveActivityExtension.appex/LiveActivityExtension`, processIdentifier: 20702}], + }, + }; + + // When + const processIdentifier = parseIOSRunningAppProcessIdentifier(response, appURL); + + // Then + expect(processIdentifier).toBeUndefined(); + }); + it('rejects malformed CoreDevice installed-app responses before using them', () => { expect(() => parseIOSInstalledAppsResponse({result: {apps: [{bundleIdentifier: 'com.example.app'}]}})).toThrow('unexpected installed-app response'); }); From d94d768870b7bb22352e68c22c64a26dc20fe41d Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 13:55:06 +0200 Subject: [PATCH 33/47] docs: clarify iOS PGO profile coverage --- scripts/pgo/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index ecf487661453..b9f78fd5a8c2 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -141,6 +141,7 @@ Hosted CI is usually insufficient for the complete workflow because hosted runne - LLVM PGO improves compiled native code. It can optimize a source-built Hermes engine, but it does not directly reorder JavaScript or replace JavaScript startup analysis. - Precompiled vendored frameworks cannot be instrumented after the fact. +- The iOS flush currently exports the app executable's profile runtime and Hermes's separate runtime. Other dynamically linked frameworks need their own export path before their instrumentation can contribute to the merged profile. - Profiles become stale after native source, dependency, compiler, SDK, ABI, or important build-setting changes. Regenerate them instead of accepting out-of-date warnings. - Instrumented builds have overhead. Never use them as the performance baseline. - Local bootstrap settings disable Android minification so synthetic identifiers and debug signing work. A production decision still needs a production-like, minified CI build with the repository's signing and dependency issues resolved. From c4bd27d2eb7fd2533da1229abe40be83be52ea0d Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 22 Sep 2026 10:17:11 +0200 Subject: [PATCH 34/47] refactor: share PGO script utilities --- scripts/lib/scriptUtils.ts | 104 ++++++++++++++++++++++++++++++ scripts/pgo/android.ts | 6 +- scripts/pgo/ios.ts | 5 +- scripts/pgo/pgo.ts | 7 ++- scripts/pgo/shared.ts | 125 +------------------------------------ tests/tooling/pgo.test.ts | 8 ++- 6 files changed, 124 insertions(+), 131 deletions(-) create mode 100644 scripts/lib/scriptUtils.ts diff --git a/scripts/lib/scriptUtils.ts b/scripts/lib/scriptUtils.ts new file mode 100644 index 000000000000..9a665b430b86 --- /dev/null +++ b/scripts/lib/scriptUtils.ts @@ -0,0 +1,104 @@ +/** Reusable helpers for repository tooling scripts. */ + +import {spawnSync} from 'node:child_process'; +import {existsSync, readdirSync} from 'node:fs'; +import {join, resolve} from 'node:path'; +import process from 'node:process'; + +const rootDirectory = resolve(import.meta.dirname, '../..'); + +function fail(message: string): never { + throw new Error(message); +} + +function run(command: string, args: string[], cwd = rootDirectory): void { + const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } +} + +function capture(command: string, args: string[], cwd = rootDirectory): string { + const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const stderr = result.stderr.trim(); + fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { + const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { + const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); + return !result.error && result.status === 0; +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolvePromise) => { + setTimeout(resolvePromise, milliseconds); + }); +} + +function findFiles(directory: string, extension: string): string[] { + if (!existsSync(directory)) { + return []; + } + + return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return findFiles(path, extension); + } + return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; + }); +} + +function requirePositiveInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive integer, received: ${value}`); + } + return value; +} + +function parsePositiveInteger(rawValue: string, label: string): number { + return requirePositiveInteger(Number(rawValue), label); +} + +function parseChoice(rawValue: string, choices: readonly T[], label: string): T { + const choice = choices.find((candidate) => candidate === rawValue); + if (!choice) { + fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); + } + return choice; +} + +function environmentString(name: string): string | undefined { + const value: unknown = process.env[name]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function valueAt(values: readonly T[], index: number): T { + const value = values.at(index); + if (value === undefined) { + fail(`Expected a value at index ${index}.`); + } + return value; +} + +export {capture, captureBinary, environmentString, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run, runAllowFailure, sleep, valueAt}; diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 0a35e42a7770..0463bf8073d4 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -9,7 +9,8 @@ import process from 'node:process'; import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PlatformAdapter} from './shared'; -import {BENCHMARK_SPANS_ENVIRONMENT, capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from './shared'; +import {capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from '../lib/scriptUtils'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; @@ -212,4 +213,5 @@ function androidProfileReceiverComponent(appID: string): string { return `${appID}/${ANDROID_JAVA_NAMESPACE}.PgoProfileReceiver`; } -export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent, createAndroidPgoAdapter}; +export default createAndroidPgoAdapter; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent}; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index c262db94059d..765684d6d349 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -2,13 +2,16 @@ /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ +import {isRecord} from '@libs/ObjectUtils'; + import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {dirname, join} from 'node:path'; import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter} from './shared'; -import {BENCHMARK_SPANS_ENVIRONMENT, capture, environmentString, fail, findFiles, isRecord, rootDirectory, run, runAllowFailure, sleep, valueAt} from './shared'; +import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllowFailure, sleep, valueAt} from '../lib/scriptUtils'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { const profileFormat = 'ios-clang-frontend-swift-ir-v1'; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 1b2767c7bb6e..486c3660bce9 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -14,9 +14,10 @@ import type {BenchmarkKind, PlatformAdapter} from './shared'; import {benchmarkStartups as runBenchmarkStartups} from '../lib/benchmarkAppStartup'; import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppBenchmark'; -import {createAndroidPgoAdapter} from './android'; +import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; -import {STARTUP_SPAN_NAME, capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from './shared'; +import {STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_STARTUP_WAIT_SECONDS = 30; @@ -281,4 +282,4 @@ if (import.meta.main) { }); } -export {main, percentageImprovement}; +export default percentageImprovement; diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index c7522ce26d74..c8c9daccf786 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -1,15 +1,9 @@ -/** Shared PGO workflow types, paths, subprocess helpers, and argument validation. */ - -import {spawnSync} from 'node:child_process'; -import {existsSync, readdirSync} from 'node:fs'; -import {join, resolve} from 'node:path'; -import process from 'node:process'; +/** Shared PGO workflow types and constants. */ import type {PlatformName} from '../lib/nativeAppBenchmark'; const STARTUP_SPAN_NAME = 'ManualAppStartup'; const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${STARTUP_SPAN_NAME}`; -const rootDirectory = resolve(import.meta.dirname, '../..'); type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; @@ -35,120 +29,5 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -function fail(message: string): never { - throw new Error(message); -} - -function run(command: string, args: string[], cwd = rootDirectory): void { - const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); - if (result.error) { - fail(`Failed to run ${command}: ${result.error.message}`); - } - if (result.status !== 0) { - fail(`${command} exited with status ${result.status ?? 'unknown'}.`); - } -} - -function capture(command: string, args: string[], cwd = rootDirectory): string { - const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); - if (result.error) { - fail(`Failed to run ${command}: ${result.error.message}`); - } - if (result.status !== 0) { - const stderr = result.stderr.trim(); - fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); - } - return result.stdout; -} - -function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { - const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); - if (result.error) { - fail(`Failed to run ${command}: ${result.error.message}`); - } - if (result.status !== 0) { - fail(`${command} exited with status ${result.status ?? 'unknown'}.`); - } - return result.stdout; -} - -function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { - const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); - return !result.error && result.status === 0; -} - -function sleep(milliseconds: number): Promise { - return new Promise((resolvePromise) => { - setTimeout(resolvePromise, milliseconds); - }); -} - -function findFiles(directory: string, extension: string): string[] { - if (!existsSync(directory)) { - return []; - } - - return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { - const path = join(directory, entry.name); - if (entry.isDirectory()) { - return findFiles(path, extension); - } - return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; - }); -} - -function requirePositiveInteger(value: number, label: string): number { - if (!Number.isSafeInteger(value) || value <= 0) { - fail(`${label} must be a positive integer, received: ${value}`); - } - return value; -} - -function parsePositiveInteger(rawValue: string, label: string): number { - return requirePositiveInteger(Number(rawValue), label); -} - -function parseChoice(rawValue: string, choices: readonly T[], label: string): T { - const choice = choices.find((candidate) => candidate === rawValue); - if (!choice) { - fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); - } - return choice; -} - -function environmentString(name: string): string | undefined { - const value: unknown = process.env[name]; - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function valueAt(values: T[], index: number): T { - const value = values.at(index); - if (value === undefined) { - fail(`Expected a value at index ${index}.`); - } - return value; -} - -export { - BENCHMARK_SPANS_ENVIRONMENT, - STARTUP_SPAN_NAME, - capture, - captureBinary, - environmentString, - fail, - findFiles, - isRecord, - parseChoice, - parsePositiveInteger, - requirePositiveInteger, - rootDirectory, - run, - runAllowFailure, - sleep, - valueAt, -}; +export {BENCHMARK_SPANS_ENVIRONMENT, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgo.test.ts b/tests/tooling/pgo.test.ts index 8839e05fc7c7..ac3ff4ba66e0 100644 --- a/tests/tooling/pgo.test.ts +++ b/tests/tooling/pgo.test.ts @@ -1,10 +1,14 @@ import {describe, expect, it} from 'bun:test'; // cspell:ignore chrispader -import {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; -import {percentageImprovement} from '@scripts/pgo/pgo'; +import createAndroidPgoAdapter, {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; +import percentageImprovement from '@scripts/pgo/pgo'; describe('PGO tooling', () => { + it('exports the Android adapter as the default export', () => { + expect(createAndroidPgoAdapter).toBeFunction(); + }); + it('reads the bootstrapped release application ID', () => { // Given const buildGradle = ` From 6aa10315586deab1d2e09bccc0ac5a7f51552ddd Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 11:45:36 +0200 Subject: [PATCH 35/47] fix: allow partial iOS PGO profiles --- scripts/pgo/ios.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 765684d6d349..8fd607438ebc 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm +// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ @@ -186,7 +186,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ]; } if (kind === 'optimized') { - const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath}`; + const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-error=profile-instr-unprofiled`; return [ ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, From f6978f17f1ef3953008b839294d59d8f45ef397c Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 14:00:36 +0200 Subject: [PATCH 36/47] perf: automate native PGO journeys --- package.json | 1 + scripts/pgo/JOURNEY.md | 77 +++++++++++ scripts/pgo/README.md | 4 +- scripts/pgo/journey.ts | 154 ++++++++++++++++++++++ scripts/pgo/journeyConfig.ts | 71 ++++++++++ scripts/pgo/journeyDevice.ts | 143 ++++++++++++++++++++ scripts/pgo/journeyWorkload.ts | 215 +++++++++++++++++++++++++++++++ tests/tooling/pgoJourney.test.ts | 135 +++++++++++++++++++ 8 files changed, 799 insertions(+), 1 deletion(-) create mode 100644 scripts/pgo/JOURNEY.md create mode 100755 scripts/pgo/journey.ts create mode 100644 scripts/pgo/journeyConfig.ts create mode 100644 scripts/pgo/journeyDevice.ts create mode 100644 scripts/pgo/journeyWorkload.ts create mode 100644 tests/tooling/pgoJourney.test.ts diff --git a/package.json b/package.json index 0e4259356074..bbc1b65e7393 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build-staging": "ENV=staging bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "build-adhoc": "ENV=adhoc bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "benchmark-app-startup": "bun scripts/benchmarkAppStartup.ts", + "pgo:journey": "bun scripts/pgo/journey.ts", "bootstrap-device": "bun scripts/bootstrapForDevice.ts", "createDocsRoutes": "bun .github/scripts/createDocsRoutes.ts", "generateAllowedUrls": "bun .github/scripts/generateAllowedUrls.ts", diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md new file mode 100644 index 000000000000..34757f14e5e6 --- /dev/null +++ b/scripts/pgo/JOURNEY.md @@ -0,0 +1,77 @@ +# Automated native PGO journey + + + +`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full Android journey, including message sending and native profile export, has been validated on a physical Samsung phone. iOS execution still needs live validation. + +## Account and device setup + +Use a dedicated, owner-approved heavy account with a populated Inbox, a long chat history, and enough expenses and expense reports to scroll. Keep the account and its data between runs. Installing over the existing app preserves authentication; uninstalling or clearing app data does not. + +The runner never enters credentials, retrieves a magic code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. + +Before every run, it opens Inbox to allow the app's automatic large-account preference update, then opens Account → Preferences and ensures Priority mode is **Most recent**. If it is **#focus**, the runner changes it and verifies the result. It checks the preference again at the end and rejects the run if #focus became enabled. + +Install `agent-device` on the runner and pin its version. Android needs USB debugging and a connected, authorized device. The runner temporarily enables the tool's test keyboard for reliable text replacement and restores the previous keyboard when its session closes. iOS requires macOS, Xcode, a trusted device with Developer Mode enabled, and signing for the tool's XCTest runner. The existing iOS profile collector supports physical devices; simulator profiles are not inputs to a device build. + +Create an ignored fixture, for example `.pgo/journey-fixture.json`: + +```json +{ + "accountEmail": "heavy@example.com", + "accountClass": "heavy", + "description": "Owner-approved high-traffic test account with populated chats, expenses, and reports.", + "report": { + "query": "busy", + "resultLabelPrefix": "#busy, Test workspace", + "title": "#busy" + }, + "personalChat": { + "query": "heavy@example.com", + "resultLabelPrefix": "Test User (you), heavy@example.com", + "title": "Test User (you)" + }, + "allowMessages": false, + "scrolls": 8, + "tabCycles": 3 +} +``` + +Choose the report name and workspace from the actual account. Search matching permits a changing message preview after the configured identity, and rejects ambiguous results. The personal-chat query must be the exact approved account email. That result can show either the email or the latest message beneath its `(you)` title, so both forms are supported for repeated runs. Confirm the personal chat with Chris before setting `allowMessages` to `true`. Messages are restricted to that account's `(you)` chat; each run sends two uniquely marked test messages and leaves them in place. An existing composer draft causes a failure. + +## Validate and record + +First check the account and #focus setting without sending messages: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --preflight +``` + +Use `--navigation-only` instead of `--preflight` to exercise all navigation and scrolling without sending messages. This mode cannot collect a training profile. Omit both flags for the complete journey once message permission is recorded in the fixture. + +The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes and edits two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. + +After installing and verifying an instrumented build with the existing PGO commands, collect three independent repetitions: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +scripts/pgo/journey.ts ios --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +``` + +Start with three repetitions per platform. Use the recorded duration and resulting native counter coverage to adjust the workload; extra repetitions of the same data do not add new paths. Most activity here exercises Hermes, Fabric mounting, layout, text, native input, and scrolling. This trains the compiled native implementation used by JavaScript; it does not compile the application's JavaScript with LLVM. Avoid spending most of the run waiting on network responses or looping over trivial settings screens. + +On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. + +The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. + +Each batch lives under `.pgo//journeys//` and includes `result.json`, per-run raw profiles, `journey.profdata`, and a function summary. The result records the fixture, app and device identifiers, both repository revisions, automation version, completed runs, and durations. These artifacts contain account identifiers and must stay in restricted CI storage. + +The journey profile is kept separate from the startup profile. The existing optimized-build command reads `newdot.profdata` from its platform directory: CI must explicitly select or weight the successful scenario profiles, write that final input, and preserve the iOS `.format` marker. Do not silently replace the startup input with an incomplete navigation run. + +## Release pipeline integration + +Use an exclusive device lease on a persistent runner: Linux or macOS with an arm64 Android device, and macOS with an arm64 iPhone. Build the instrumented artifact from the release candidate, install it over the signed-in app, run preflight and the full collection, then build the optimized artifact from the same source and toolchain. Gate publication on a separate performance and correctness check. The instrumented artifact is a CI input, not the store upload. + +Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. + +This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows, validating iOS, and measuring held-out interactive performance remain separate integration steps. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index b9f78fd5a8c2..6c01fac3d088 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -75,6 +75,8 @@ The app must remain in the foreground until each profile write completes. Force- ## Collect an interactive profile +For the automated heavy-account workload, see [Automated native PGO journey](./JOURNEY.md). It checks the signed-in account, disables #focus, drives the critical user flows, and can archive and merge profiles from each successful run. + For a broader profile, install the instrumented app, perform the agreed journey, and write the counters while the app is still running: ```bash @@ -85,7 +87,7 @@ scripts/pgo/pgo.ts android merge Use the equivalent `ios` commands on iOS. A representative journey should exercise the common signed-in path: open and scroll chats, send a message, visit a workspace setting, attach and view a file, and run a search. Keep account state, data size, and network conditions stable between collections. -Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The current command automates this lifecycle for startup collection; a mixed-scenario CI runner still needs that orchestration. +Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The startup recorder and journey runner each handle that lifecycle; CI must still choose and combine the scenario profiles. ## Build and benchmark the optimized app diff --git a/scripts/pgo/journey.ts b/scripts/pgo/journey.ts new file mode 100755 index 000000000000..624f67f2c56e --- /dev/null +++ b/scripts/pgo/journey.ts @@ -0,0 +1,154 @@ +#!/usr/bin/env bun + +// cspell:ignore profraw profdata + +import CLI from 'expensify-common/CLI'; +import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {dirname, join, relative, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {PLATFORM_NAMES} from '../lib/nativeAppBenchmark'; +import {capture, findFiles, parseChoice, parsePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {parseJourneyFixture} from './journeyConfig'; +import {createJourneyDevice} from './journeyDevice'; +import {prepareJourney, runJourneyWorkload, verifyJourneyAccount} from './journeyWorkload'; + +/** Run the same UI workload on a selected Android or iOS device; never automate authentication. */ +async function main(): Promise { + /* eslint-disable @typescript-eslint/naming-convention */ + const cli = new CLI({ + positionalArgs: [{name: 'platform', description: 'android or ios', parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform')}], + namedArgs: { + fixture: {description: 'JSON fixture identifying the approved heavy account and reports'}, + device: {description: 'Android serial or iOS UDID'}, + 'app-id': {description: 'Installed application or bundle identifier'}, + runs: {description: 'Complete journeys to run', default: 1, parse: (value) => parsePositiveInteger(value, 'Runs')}, + }, + flags: { + collect: {description: 'Flush, archive, and merge LLVM profiles from an already-installed instrumented app'}, + preflight: {description: 'Verify account and disable #focus, then stop without running the workload'}, + 'navigation-only': {description: 'Validate navigation without sending messages; cannot collect training profiles'}, + }, + }); + /* eslint-enable @typescript-eslint/naming-convention */ + const platform = parseChoice(String(cli.positionalArgs.platform), PLATFORM_NAMES, 'Platform'); + const fixtureInput: unknown = JSON.parse(readFileSync(resolve(cli.namedArgs.fixture), 'utf8')); + const fixture = parseJourneyFixture(fixtureInput); + if (cli.flags['navigation-only'] && cli.flags.collect) { + throw new Error('Navigation-only validation cannot produce a complete training profile.'); + } + if (!cli.flags.preflight && !cli.flags['navigation-only'] && !fixture.allowMessages) { + throw new Error('Confirm the personal chat with Chris before setting allowMessages=true in the fixture. Use --navigation-only to validate without sending.'); + } + const runs = parsePositiveInteger(String(cli.namedArgs.runs), 'Runs'); + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const batchDirectory = join(rootDirectory, '.pgo', platform, 'journeys', batchID); + mkdirSync(batchDirectory, {recursive: true}); + const device = createJourneyDevice({platform, device: cli.namedArgs.device, appID: cli.namedArgs['app-id'], session: `pgo-${platform}-${batchID}`}); + const adapter = platform === 'android' ? createAndroidPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device) : createIOSPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device); + const completedProfiles: string[] = []; + const evidence = { + platform, + device: cli.namedArgs.device, + appID: cli.namedArgs['app-id'], + accountEmail: fixture.accountEmail, + accountClass: fixture.accountClass, + fixture, + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + automationVersion: capture('agent-device', ['--version']).trim(), + requestedRuns: runs, + collectProfiles: cli.flags.collect, + navigationOnly: cli.flags['navigation-only'], + completedRuns: 0, + runDurationsSeconds: [] as number[], + status: 'running', + failure: '', + }; + const saveEvidence = () => writeFileSync(join(batchDirectory, 'result.json'), `${JSON.stringify(evidence, null, 2)}\n`); + saveEvidence(); + try { + device.open(false); + await prepareJourney(device, fixture); + if (cli.flags.preflight) { + evidence.status = 'preflight-passed'; + return; + } + if (cli.flags.collect) { + // Verify the installed build's writer before spending time on the workload; discard these setup counters. + await adapter.dumpProfiles(); + await adapter.clearDeviceProfiles(); + } + for (let index = 1; index <= runs; index += 1) { + const runID = `${batchID}-${index}`; + const runDirectory = join(batchDirectory, `run-${index}`); + mkdirSync(runDirectory); + if (index > 1) { + await prepareJourney(device, fixture); + if (cli.flags.collect) { + await adapter.clearDeviceProfiles(); + } + } + console.log(`Starting journey ${index}/${runs}.`); + const startedAt = Date.now(); + device.open(true); + await runJourneyWorkload(device, fixture, runID, cli.flags['navigation-only']); + await verifyJourneyAccount(device, fixture); + if (cli.flags.collect) { + // Flush once per fresh process. Never merge partial or failed journeys. + await adapter.dumpProfiles(); + adapter.pullProfiles(); + const rawDirectory = join(runDirectory, 'raw'); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + throw new Error('The completed journey produced no LLVM profiles.'); + } + for (const profile of profiles) { + // Archive counters only, not unrelated files pulled from the app's cache directory. + const archivedProfile = join(rawDirectory, relative(adapter.rawProfileDirectory, profile)); + mkdirSync(dirname(archivedProfile), {recursive: true}); + copyFileSync(profile, archivedProfile); + completedProfiles.push(archivedProfile); + } + } + evidence.completedRuns = index; + evidence.runDurationsSeconds.push(Math.round((Date.now() - startedAt) / 1000)); + saveEvidence(); + } + if (cli.flags.collect) { + const mergedProfile = join(batchDirectory, 'journey.profdata'); + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${mergedProfile}`, ...completedProfiles]); + if (adapter.profileFormat) { + writeFileSync(`${mergedProfile}.format`, `${adapter.profileFormat}\n`); + } + writeFileSync(`${mergedProfile}.txt`, capture(adapter.llvmTool('llvm-profdata'), ['show', '--all-functions', mergedProfile])); + console.log(`Journey profile: ${mergedProfile}`); + } + evidence.status = cli.flags['navigation-only'] ? 'navigation-passed' : 'passed'; + } catch (error) { + evidence.status = 'failed'; + evidence.failure = error instanceof Error ? error.message : String(error); + throw error; + } finally { + saveEvidence(); + // Release the device and restore Android's original keyboard, including on failed attempts. + // Closing a session leaves persisted authentication intact. + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + console.log(`Journey result: ${join(batchDirectory, 'result.json')}`); + } +} + +if (import.meta.main) { + main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = message.startsWith('SIGN_IN_REQUIRED:') ? 2 : 1; + }); +} diff --git a/scripts/pgo/journeyConfig.ts b/scripts/pgo/journeyConfig.ts new file mode 100644 index 000000000000..8a6f0cb0fe6a --- /dev/null +++ b/scripts/pgo/journeyConfig.ts @@ -0,0 +1,71 @@ +import {isRecord} from '@libs/ObjectUtils'; + +type JourneyReport = { + query: string; + resultLabelPrefix: string; + title: string; +}; + +type JourneyFixture = { + accountEmail: string; + accountClass: 'heavy'; + description: string; + report: JourneyReport; + personalChat: JourneyReport; + allowMessages: boolean; + scrolls: number; + tabCycles: number; +}; + +/** Require an explicitly selected heavy account instead of silently training on whoever is signed in. */ +function parseJourneyFixture(input: unknown): JourneyFixture { + if (!isRecord(input) || input.accountClass !== 'heavy') { + throw new Error('The journey requires a fixture with accountClass="heavy", approved by the account owner.'); + } + const accountEmail = requiredString(input.accountEmail, 'accountEmail'); + if (!accountEmail.includes('@')) { + throw new Error('accountEmail must identify the approved heavy account.'); + } + const personalChat = parseReport(input.personalChat, 'personalChat'); + if (personalChat.query !== accountEmail || !personalChat.title.endsWith('(you)') || personalChat.resultLabelPrefix !== `${personalChat.title}, ${accountEmail}`) { + throw new Error('personalChat must be the approved account\'s own "(you)" chat, with its email as query and its title and email in resultLabelPrefix.'); + } + return { + accountEmail, + accountClass: 'heavy', + description: requiredString(input.description, 'description'), + report: parseReport(input.report, 'report'), + personalChat, + allowMessages: input.allowMessages === true, + scrolls: boundedInteger(input.scrolls ?? 8, 'scrolls', 4, 30), + tabCycles: boundedInteger(input.tabCycles ?? 3, 'tabCycles', 2, 10), + }; +} + +function parseReport(input: unknown, name: string): JourneyReport { + if (!isRecord(input)) { + throw new Error(`${name} must specify query, resultLabelPrefix, and title.`); + } + return { + query: requiredString(input.query, `${name}.query`), + resultLabelPrefix: requiredString(input.resultLabelPrefix, `${name}.resultLabelPrefix`), + title: requiredString(input.title, `${name}.title`), + }; +} + +function requiredString(input: unknown, name: string): string { + if (typeof input !== 'string' || input.trim().length === 0) { + throw new Error(`${name} must be a nonempty string.`); + } + return input.trim(); +} + +function boundedInteger(input: unknown, name: string, minimum: number, maximum: number): number { + if (typeof input !== 'number' || !Number.isSafeInteger(input) || input < minimum || input > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`); + } + return input; +} + +export {parseJourneyFixture}; +export type {JourneyFixture, JourneyReport}; diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts new file mode 100644 index 000000000000..d5e888639f3d --- /dev/null +++ b/scripts/pgo/journeyDevice.ts @@ -0,0 +1,143 @@ +import {isRecord} from '@libs/ObjectUtils'; + +import {spawnSync} from 'node:child_process'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +type JourneyNode = { + label: string; + value: string; + identifier: string; + type: string; + enabled: boolean; + width: number; + height: number; +}; + +type JourneyDevice = ReturnType; + +/** Execute deterministic device commands. Authentication is always handled by the user. */ +function createJourneyDevice(options: {platform: PlatformName; device: string; appID: string; session: string}) { + const targetArguments = ['--platform', options.platform, options.platform === 'android' ? '--serial' : '--udid', options.device, '--session', options.session, '--json']; + + function command(...args: string[]): unknown { + const result = spawnSync('agent-device', [...args, ...targetArguments], {encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024}); + if (result.error) { + throw new Error(`Device command ${args.at(0)} failed: ${result.error.message}`); + } + let response: unknown; + try { + response = JSON.parse(result.stdout); + } catch { + throw new Error(`Device command ${args.at(0)} did not return JSON: ${result.stderr.trim()}`); + } + if (!isRecord(response) || response.success !== true || result.status !== 0) { + const error = isRecord(response) && isRecord(response.error) ? response.error : undefined; + throw new Error(`Device command ${args.at(0)} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); + } + return response.data; + } + + function snapshot(): JourneyNode[] { + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const nodes = parseJourneySnapshot(command('snapshot', '-i', '--force-full'), options.appID); + assertSignedIn(nodes); + return nodes; + } catch (error) { + // The Android accessibility service can briefly return only system UI while a heavy list mounts. + // Retry this read alone; repeating a tap or send could perform the action twice. + if (attempt === 2 || !(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + command('wait', '500'); + } + } + throw new Error('Unable to read the app accessibility tree.'); + } + + function wait(selector: string): void { + try { + command('wait', selector, '30000'); + } catch (error) { + // Give sign-in a specific, actionable failure instead of continuing through an expired session. + snapshot(); + throw error; + } + } + + function press(selector: string): void { + wait(selector); + command('press', selector, '--settle'); + } + + function labels(): string[] { + return snapshot().map((node) => normalizeLabel(node.label)); + } + + function hasLabel(label: string): boolean { + return labels().includes(normalizeLabel(label)); + } + + return { + command, + snapshot, + wait, + press, + hasLabel, + pressLabel: (label: string) => press(labelSelector(label)), + waitLabel: (label: string) => wait(labelSelector(label)), + fill: (identifier: string, text: string) => { + wait(idSelector(identifier)); + command('fill', idSelector(identifier), text, '--settle'); + }, + back: () => command('back', '--settle'), + open: (relaunch: boolean) => command('open', options.appID, ...(relaunch ? ['--relaunch'] : []), ...(options.platform === 'android' ? ['--test-ime'] : [])), + }; +} + +function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { + if (!isRecord(data) || !Array.isArray(data.nodes)) { + throw new Error('The device did not return an accessibility tree.'); + } + if (data.appBundleId !== appID) { + throw new Error('The automation is no longer attached to the expected app.'); + } + return data.nodes + .filter(isRecord) + .filter((node) => !node.bundleId || node.bundleId === appID) + .map((node) => ({ + label: typeof node.label === 'string' ? node.label : '', + value: typeof node.value === 'string' ? node.value : '', + identifier: typeof node.identifier === 'string' ? node.identifier : '', + type: typeof node.type === 'string' ? node.type : '', + enabled: node.enabled !== false, + width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, + height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + })); +} + +function assertSignedIn(nodes: JourneyNode[]): void { + const signInLabels = new Set(['Sign in', 'Email or phone number', 'Enter your email or phone number', 'Enter your magic code']); + if (nodes.some((node) => ['username', 'validateCode'].includes(node.identifier) || signInLabels.has(normalizeLabel(node.label)))) { + throw new Error('SIGN_IN_REQUIRED: Ask Chris to sign into the approved heavy account, then restart the journey. Automated sign-in is disabled.'); + } +} + +function normalizeLabel(label: string): string { + return label + .replaceAll(/[\u2066-\u2069]/g, '') + .replaceAll(/\s+/g, ' ') + .trim(); +} + +function labelSelector(label: string): string { + return `label=${JSON.stringify(label)}`; +} + +function idSelector(identifier: string): string { + return `id=${JSON.stringify(identifier)}`; +} + +export {assertSignedIn, createJourneyDevice, idSelector, labelSelector, normalizeLabel, parseJourneySnapshot}; +export type {JourneyDevice, JourneyNode}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts new file mode 100644 index 000000000000..7592cc6fcbb2 --- /dev/null +++ b/scripts/pgo/journeyWorkload.ts @@ -0,0 +1,215 @@ +import type {JourneyFixture, JourneyReport} from './journeyConfig'; +import type {JourneyDevice, JourneyNode} from './journeyDevice'; + +import {sleep} from '../lib/scriptUtils'; +import {idSelector, normalizeLabel} from './journeyDevice'; + +/** Run only after the user has signed into and approved the fixture's heavy account. */ +async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture, runID: string, navigationOnly = false): Promise { + console.log('Scrolling the full inbox.'); + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + device.pressLabel('All'); + await scrollBothWays(device, fixture.scrolls, true); + + console.log('Opening and scrolling a populated report.'); + await openReport(device, fixture.report); + await scrollBothWays(device, fixture.scrolls, true, 'up'); + + console.log(navigationOnly ? 'Opening the personal chat without sending.' : 'Composing and sending messages in the personal chat.'); + await openReport(device, fixture.personalChat, true); + for (let messageNumber = 1; messageNumber <= (navigationOnly ? 0 : 2); messageNumber += 1) { + // Check the destination immediately before every write. Search-result matching alone is insufficient. + device.waitLabel(fixture.personalChat.title); + const composer = device.snapshot().find((node) => node.identifier === 'composer'); + if (!composer) { + throw new Error('The personal chat composer is missing.'); + } + if (composer.value && composer.value !== 'Write something...') { + throw new Error('The personal chat contains an existing draft. Clear or send it manually before running the journey.'); + } + const token = `PGO ${runID} message ${messageNumber}`; + device.fill('composer', `${token} draft`); + device.fill('composer', `${token}. Answered by Codex, instructed by Chris.`); + device.pressLabel('Send'); + await waitForMessage(device, token); + } + + console.log('Scrolling Spend expenses and reports.'); + await showTab(device, 'Spend'); + device.pressLabel('Expenses'); + device.waitLabel('Filters'); + await scrollBothWays(device, fixture.scrolls, true); + device.pressLabel('Reports'); + device.waitLabel('Filters'); + await scrollBothWays(device, Math.max(4, Math.floor(fixture.scrolls / 2)), true); + device.pressLabel('Expenses'); + + console.log('Switching tabs and reopening the report.'); + for (let cycle = 0; cycle < fixture.tabCycles; cycle += 1) { + for (const tab of ['Inbox', 'Spend', 'Workspaces', 'Inbox']) { + await showTab(device, tab); + } + } + await openReport(device, fixture.report); + await showTab(device, 'Inbox'); +} + +/** Check identity before changing preferences; never sign in or switch accounts automatically. */ +async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): Promise { + // Entering Inbox first allows the app's automatic large-account focus prompt to appear before we check preferences. + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + try { + device.waitLabel(fixture.accountEmail); + } catch { + throw new Error(`SIGN_IN_REQUIRED: Ask Chris to sign into ${fixture.accountEmail}. The journey cannot run on another account.`); + } + device.pressLabel('Preferences'); + await ensureFocusDisabled(device, true); + device.back(); + await showTab(device, 'Inbox'); + console.log('Verified the approved heavy account and disabled #focus.'); +} + +/** End-of-run verification rejects a profile if focus was enabled during the workload. */ +async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixture): Promise { + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + device.waitLabel(fixture.accountEmail); + device.pressLabel('Preferences'); + await ensureFocusDisabled(device, false); + device.back(); + await showTab(device, 'Inbox'); +} + +async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): Promise { + device.waitLabel('Priority mode'); + if (device.hasLabel('Priority mode, Most recent')) { + return; + } + if (!device.hasLabel('Priority mode, #focus')) { + throw new Error('Cannot verify Priority mode. The journey requires English UI labels and #focus disabled.'); + } + if (!allowChange) { + throw new Error('#focus became enabled during the journey. Discard this run and restart after disabling it.'); + } + device.pressLabel('Priority mode, #focus'); + device.pressLabel('Most recent'); + device.pressLabel('Save'); + device.waitLabel('Priority mode, Most recent'); +} + +async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false): Promise { + await showTab(device, 'Inbox'); + device.press(idSelector('searchButton')); + device.fill('search-autocomplete-text-input', report.query); + const deadline = Date.now() + 30_000; + let resultLabel: string | undefined; + while (!resultLabel && Date.now() < deadline) { + resultLabel = findReportResult(device.snapshot(), report.resultLabelPrefix, isPersonalChat ? report.title : undefined); + if (!resultLabel) { + await sleep(500); + } + } + if (!resultLabel) { + throw new Error(`No search result matched ${report.resultLabelPrefix}.`); + } + device.pressLabel(resultLabel); + device.wait(idSelector('composer')); + device.waitLabel(report.title); +} + +async function showTab(device: JourneyDevice, tab: string): Promise { + // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. + for (let attempt = 0; attempt < 5; attempt += 1) { + if (device.hasLabel(tab)) { + device.pressLabel(tab); + return; + } + device.back(); + await sleep(300); + } + throw new Error(`Cannot return to the ${tab} tab.`); +} + +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { + const initialNodes = device.snapshot(); + const pixels = scrollDistance(initialNodes); + let previous = contentSignature(initialNodes); + let changed = 0; + for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { + let stationary = 0; + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', String(pixels), '--settle'); + const current = contentSignature(device.snapshot()); + if (current !== previous) { + changed += 1; + stationary = 0; + } else { + stationary += 1; + } + previous = current; + if (stationary === 2) { + break; + } + } + } + if (requireMovement && changed < 2) { + throw new Error('The list did not expose enough changing content. Use a populated heavy-account fixture; do not collect this run.'); + } +} + +async function waitForMessage(device: JourneyDevice, token: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const composer = nodes.find((node) => node.identifier === 'composer'); + const visibleMessage = nodes.some((node) => node.identifier !== 'composer' && !/EditText|TextField|TextViewInput/i.test(node.type) && node.label.includes(token)); + if (composer && !composer.value.includes(token) && visibleMessage) { + return; + } + await sleep(500); + } + throw new Error('The sent message did not appear with an empty composer. Do not retry the send automatically.'); +} + +function contentSignature(nodes: JourneyNode[]): string { + return nodes + .map((node) => normalizeLabel(node.label)) + .filter(Boolean) + .join('\n'); +} + +/** Keep both ends of the gesture inside the center of portrait lists, below Spend's fixed header. */ +function scrollDistance(nodes: JourneyNode[]): number { + const scrollAreas = nodes.filter((node) => /ScrollView|RecyclerView|ScrollArea/i.test(node.type) && node.width > 0 && node.height > 0); + const viewport = scrollAreas.toSorted((left, right) => right.width * right.height - left.width * left.height).at(0); + if (!viewport) { + throw new Error('Cannot determine scroll bounds on this device.'); + } + return Math.round(Math.min(viewport.height * 0.25, viewport.width * 0.6)); +} + +/** Match stable report identity while allowing the last-message preview to change. */ +function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitle?: string): string | undefined { + const identity = normalizeLabel(prefix); + // The fixture binds this title to the signed-in account and an exact email search. A populated self-chat replaces the email subtitle with its last message. + const personalPrefix = personalChatTitle ? `${normalizeLabel(personalChatTitle)}, ` : undefined; + const matches = nodes.filter((node) => { + const label = normalizeLabel(node.label); + return label === identity || label.startsWith(`${identity} •`) || (personalPrefix !== undefined && label.startsWith(personalPrefix)); + }); + if (matches.length > 1) { + throw new Error(`Ambiguous report search: ${prefix}. Use a more specific fixture.`); + } + return matches.at(0)?.label; +} + +export {findReportResult, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts new file mode 100644 index 000000000000..49c9c1476c84 --- /dev/null +++ b/tests/tooling/pgoJourney.test.ts @@ -0,0 +1,135 @@ +import {describe, expect, it} from 'bun:test'; + +import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; +import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; +import {findReportResult, scrollDistance} from '@scripts/pgo/journeyWorkload'; + +const fixture = { + accountEmail: 'heavy@example.com', + accountClass: 'heavy', + description: 'Owner-approved account with a populated inbox, message history, expenses, and reports.', + report: {query: 'busy', resultLabelPrefix: '#busy, Test workspace', title: '#busy'}, + personalChat: {query: 'heavy@example.com', resultLabelPrefix: 'Test User (you), heavy@example.com', title: 'Test User (you)'}, +}; + +describe('PGO journey safeguards', () => { + it('requires explicit message approval in the fixture', () => { + // Given account approval alone does not authorize a particular message destination. + for (const allowMessages of [undefined, false, 'true']) { + // When / Then full journeys remain unavailable until approval is recorded explicitly. + expect(parseJourneyFixture({...fixture, allowMessages}).allowMessages).toBe(false); + } + expect(parseJourneyFixture({...fixture, allowMessages: true}).allowMessages).toBe(true); + }); + + it('requires the account owner to select a heavy account explicitly', () => { + // Given an unclassified or ordinary account, a populated-looking screen is insufficient evidence. + for (const accountClass of [undefined, 'ordinary', '']) { + // When / Then the fixture cannot start a training run. + expect(() => parseJourneyFixture({...fixture, accountClass})).toThrow('accountClass="heavy"'); + } + }); + + it("rejects message destinations that are not the approved account's personal chat", () => { + // Given a fixture that would send messages to another person or to a different account's personal chat. + for (const personalChat of [ + {...fixture.personalChat, query: 'another@example.com'}, + {...fixture.personalChat, title: 'Another person'}, + {...fixture.personalChat, resultLabelPrefix: 'Other User (you), other@example.com'}, + {...fixture.personalChat, resultLabelPrefix: 'Test User (you), heavy@example.com.invalid'}, + ]) { + // When / Then no device action is permitted for that fixture. + expect(() => parseJourneyFixture({...fixture, personalChat})).toThrow('own "(you)" chat'); + } + }); + + it('rejects an empty workload or an unbounded scrolling loop', () => { + // Given invalid workload sizes that would create an unrepresentative or runaway recording. + for (const scrolls of [0, 1, 31, NaN, '8']) { + // When / Then parsing fails before the app is touched. + expect(() => parseJourneyFixture({...fixture, scrolls})).toThrow('scrolls'); + } + }); + + it('filters system keyboard content out of app assertions', () => { + // Given Android's keyboard can display labels unrelated to the foreground app. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {bundleId: 'keyboard.app', label: 'Sign in'}, + {bundleId: 'test.app', label: 'Inbox'}, + ], + }, + 'test.app', + ); + // When / Then keyboard content cannot falsely trigger a sign-in failure. + expect(nodes).toHaveLength(1); + expect(() => assertSignedIn(nodes)).not.toThrow(); + }); + + it('stops for manual sign-in on both platforms', () => { + // Given either platform has reached an authentication or code-entry screen. + for (const identifier of ['username', 'validateCode']) { + for (const type of ['android.widget.EditText', 'TextField']) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{identifier, type}]}, 'test.app'); + // When / Then the runner requests user input instead of filling credentials. + expect(() => assertSignedIn(nodes)).toThrow('SIGN_IN_REQUIRED'); + } + } + }); + + it('rejects a snapshot from the wrong app', () => { + // Given device focus has moved to an unexpected application. + const snapshot = {appBundleId: 'other.app', nodes: []}; + // When / Then its labels cannot satisfy the journey's assertions. + expect(() => parseJourneySnapshot(snapshot, 'test.app')).toThrow('expected app'); + }); + + it('normalizes accessibility text direction markers without changing its words', () => { + // Given translated UI labels include bidirectional isolation and non-breaking spaces. + const label = '\u2066Most\u00a0recent\u2069'; + // When / Then the same label can be compared across native platforms. + expect(normalizeLabel(label)).toBe('Most recent'); + }); + + it('matches report identity independently of the changing last-message preview', () => { + // Given the workspace has similarly named reports and a new message changes the preview. + const label = '#busy, Test workspace • New message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: '#busy, Test workspace copy • Other message'}]}, 'test.app'); + // When / Then only the specified workspace can be selected. + expect(findReportResult(nodes, '#busy, Test workspace')).toBe(label); + }); + + it('refuses ambiguous report matches', () => { + // Given two results share the fixture identity. + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label: '#busy, Test workspace'}, {label: '#busy, Test workspace • Message'}]}, 'test.app'); + // When / Then choosing the first result would risk opening an unintended destination. + expect(() => findReportResult(nodes, '#busy, Test workspace')).toThrow('Ambiguous'); + }); + + it('finds the approved personal chat after its email subtitle becomes a message preview', () => { + // Given a previous run sent a message, while another account has a similarly named chat. + const label = 'Test User (you), PGO previous run message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: 'Test User (you)'}, {label: 'Test User, Message from another account'}]}, 'test.app'); + // When / Then the validated personal-chat title matches only the full result row. + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix, fixture.personalChat.title)).toBe(label); + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix)).toBeUndefined(); + }); + + it('sizes scrolls for different screens without starting inside the fixed header', () => { + // Given Android uses physical pixels while iOS accessibility exposes logical screen dimensions. + for (const [width, height] of [ + [1080, 2280], + [390, 844], + ]) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width, height}}]}, 'test.app'); + // When the gesture is centered, both ends stay within the middle half of the portrait viewport. + const distance = scrollDistance(nodes); + expect(distance).toBeGreaterThan(0); + expect(distance).toBeLessThanOrEqual((height ?? 0) / 2); + } + // Then missing viewport information must fail instead of falling back to a screen-edge gesture. + expect(() => scrollDistance([])).toThrow('scroll bounds'); + }); +}); From a9cae54bf29e740d49971d2d6b62b96ec6d46daf Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 14:25:19 +0200 Subject: [PATCH 37/47] fix: match iOS PGO tab status labels --- scripts/pgo/journeyDevice.ts | 2 ++ scripts/pgo/journeyWorkload.ts | 21 ++++++++++++++++++--- tests/tooling/pgoJourney.test.ts | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts index d5e888639f3d..bee0f7d4f9c0 100644 --- a/scripts/pgo/journeyDevice.ts +++ b/scripts/pgo/journeyDevice.ts @@ -12,6 +12,7 @@ type JourneyNode = { enabled: boolean; width: number; height: number; + y: number; }; type JourneyDevice = ReturnType; @@ -114,6 +115,7 @@ function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { enabled: node.enabled !== false, width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + y: isRecord(node.rect) && typeof node.rect.y === 'number' ? node.rect.y : 0, })); } diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 7592cc6fcbb2..19274cad56dc 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -129,8 +129,9 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson async function showTab(device: JourneyDevice, tab: string): Promise { // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. for (let attempt = 0; attempt < 5; attempt += 1) { - if (device.hasLabel(tab)) { - device.pressLabel(tab); + const tabLabel = findTabLabel(device.snapshot(), tab); + if (tabLabel) { + device.pressLabel(tabLabel); return; } device.back(); @@ -139,6 +140,20 @@ async function showTab(device: JourneyDevice, tab: string): Promise { throw new Error(`Cannot return to the ${tab} tab.`); } +/** iOS appends unread/review status to tab accessibility labels. */ +function findTabLabel(nodes: JourneyNode[], tab: string): string | undefined { + const screenHeight = nodes.find((node) => node.type === 'Application')?.height; + const candidates = nodes.filter((node) => { + const label = normalizeLabel(node.label); + const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); + return matches && (!screenHeight || node.y >= screenHeight * 0.7); + }); + if (candidates.length > 1) { + throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); + } + return candidates.at(0)?.label; +} + async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { const initialNodes = device.snapshot(); const pixels = scrollDistance(initialNodes); @@ -212,4 +227,4 @@ function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitl return matches.at(0)?.label; } -export {findReportResult, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; +export {findReportResult, findTabLabel, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 49c9c1476c84..eb6e90f68c27 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -2,7 +2,7 @@ import {describe, expect, it} from 'bun:test'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {findReportResult, scrollDistance} from '@scripts/pgo/journeyWorkload'; +import {findReportResult, findTabLabel, scrollDistance} from '@scripts/pgo/journeyWorkload'; const fixture = { accountEmail: 'heavy@example.com', @@ -132,4 +132,21 @@ describe('PGO journey safeguards', () => { // Then missing viewport information must fail instead of falling back to a screen-edge gesture. expect(() => scrollDistance([])).toThrow('scroll bounds'); }); + + it('selects iOS status-bearing bottom tabs without confusing them with page headings', () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Inbox', rect: {width: 390, height: 50, y: 80}}, + {label: 'Inbox. Your review is required', rect: {width: 78, height: 71, y: 749}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + expect(findTabLabel(nodes, 'Inbox')).toBe('Inbox. Your review is required'); + expect(findTabLabel(nodes, 'Account')).toBe('Account, My settings. Your review is required.'); + }); }); From 394ac25641bd04cb47bfdcc85baa51da1bec7ddf Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 15:22:41 +0200 Subject: [PATCH 38/47] fix: complete iOS PGO journey on physical device --- scripts/pgo/JOURNEY.md | 4 +- scripts/pgo/journeyDevice.ts | 5 +- scripts/pgo/journeyWorkload.ts | 114 ++++++++++++++++++++++++------- tests/tooling/pgoJourney.test.ts | 68 +++++++++++++++++- 4 files changed, 162 insertions(+), 29 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index 34757f14e5e6..6e410d5002e2 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -49,7 +49,7 @@ scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture Use `--navigation-only` instead of `--preflight` to exercise all navigation and scrolling without sending messages. This mode cannot collect a training profile. Omit both flags for the complete journey once message permission is recorded in the fixture. -The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes and edits two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. +The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. Android also edits each draft before sending; iOS enters the final text once because replacing non-empty text in its composer was unreliable during live validation. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. After installing and verifying an instrumented build with the existing PGO commands, collect three independent repetitions: @@ -62,6 +62,8 @@ Start with three repetitions per platform. Use the recorded duration and resulti On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. +On the same day, a complete instrumented iOS recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. It sent two approved messages to the account's personal chat, exported two raw profiles, and produced a 15 MB merged profile with 23,822 functions showing nonzero execution counts. The final check confirmed the heavy account and #focus off. This is one validation recording; the three-repetition training batch remains to be collected. + The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. Each batch lives under `.pgo//journeys//` and includes `result.json`, per-run raw profiles, `journey.profdata`, and a function summary. The result records the fixture, app and device identifiers, both repository revisions, automation version, completed runs, and durations. These artifacts contain account identifiers and must stay in restricted CI storage. diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts index bee0f7d4f9c0..89dc251fa1e2 100644 --- a/scripts/pgo/journeyDevice.ts +++ b/scripts/pgo/journeyDevice.ts @@ -12,6 +12,7 @@ type JourneyNode = { enabled: boolean; width: number; height: number; + x: number; y: number; }; @@ -34,7 +35,7 @@ function createJourneyDevice(options: {platform: PlatformName; device: string; a } if (!isRecord(response) || response.success !== true || result.status !== 0) { const error = isRecord(response) && isRecord(response.error) ? response.error : undefined; - throw new Error(`Device command ${args.at(0)} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); + throw new Error(`Device command ${args.at(0)} ${args.at(1) ?? ''} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); } return response.data; } @@ -81,6 +82,7 @@ function createJourneyDevice(options: {platform: PlatformName; device: string; a } return { + platform: options.platform, command, snapshot, wait, @@ -115,6 +117,7 @@ function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { enabled: node.enabled !== false, width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + x: isRecord(node.rect) && typeof node.rect.x === 'number' ? node.rect.x : 0, y: isRecord(node.rect) && typeof node.rect.y === 'number' ? node.rect.y : 0, })); } diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 19274cad56dc..919c2d3f7be4 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -11,7 +11,7 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture if (device.hasLabel('No thanks')) { device.pressLabel('No thanks'); } - device.pressLabel('All'); + selectAllInboxFilter(device); await scrollBothWays(device, fixture.scrolls, true); console.log('Opening and scrolling a populated report.'); @@ -22,7 +22,9 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture await openReport(device, fixture.personalChat, true); for (let messageNumber = 1; messageNumber <= (navigationOnly ? 0 : 2); messageNumber += 1) { // Check the destination immediately before every write. Search-result matching alone is insufficient. - device.waitLabel(fixture.personalChat.title); + if (!device.hasLabel(fixture.personalChat.title)) { + throw new Error('The personal chat title is not visible before sending.'); + } const composer = device.snapshot().find((node) => node.identifier === 'composer'); if (!composer) { throw new Error('The personal chat composer is missing.'); @@ -31,7 +33,9 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture throw new Error('The personal chat contains an existing draft. Clear or send it manually before running the journey.'); } const token = `PGO ${runID} message ${messageNumber}`; - device.fill('composer', `${token} draft`); + if (device.platform === 'android') { + device.fill('composer', `${token} draft`); + } device.fill('composer', `${token}. Answered by Codex, instructed by Chris.`); device.pressLabel('Send'); await waitForMessage(device, token); @@ -39,13 +43,13 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture console.log('Scrolling Spend expenses and reports.'); await showTab(device, 'Spend'); - device.pressLabel('Expenses'); + selectSpendSection(device, 'Expenses'); device.waitLabel('Filters'); await scrollBothWays(device, fixture.scrolls, true); - device.pressLabel('Reports'); + selectSpendSection(device, 'Reports'); device.waitLabel('Filters'); await scrollBothWays(device, Math.max(4, Math.floor(fixture.scrolls / 2)), true); - device.pressLabel('Expenses'); + selectSpendSection(device, 'Expenses'); console.log('Switching tabs and reopening the report.'); for (let cycle = 0; cycle < fixture.tabCycles; cycle += 1) { @@ -66,12 +70,10 @@ async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): P } await showTab(device, 'Account'); device.command('scroll', 'top', '--settle'); - try { - device.waitLabel(fixture.accountEmail); - } catch { + if (!device.hasLabel(fixture.accountEmail)) { throw new Error(`SIGN_IN_REQUIRED: Ask Chris to sign into ${fixture.accountEmail}. The journey cannot run on another account.`); } - device.pressLabel('Preferences'); + pressPreferences(device); await ensureFocusDisabled(device, true); device.back(); await showTab(device, 'Inbox'); @@ -82,15 +84,16 @@ async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): P async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixture): Promise { await showTab(device, 'Account'); device.command('scroll', 'top', '--settle'); - device.waitLabel(fixture.accountEmail); - device.pressLabel('Preferences'); + if (!device.hasLabel(fixture.accountEmail)) { + throw new Error(`The account changed during the journey. Expected ${fixture.accountEmail}; discard this run.`); + } + pressPreferences(device); await ensureFocusDisabled(device, false); device.back(); await showTab(device, 'Inbox'); } async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): Promise { - device.waitLabel('Priority mode'); if (device.hasLabel('Priority mode, Most recent')) { return; } @@ -123,15 +126,17 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson } device.pressLabel(resultLabel); device.wait(idSelector('composer')); - device.waitLabel(report.title); + if (!device.hasLabel(report.title)) { + throw new Error(`The opened report does not show the expected title ${report.title}.`); + } } async function showTab(device: JourneyDevice, tab: string): Promise { // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. for (let attempt = 0; attempt < 5; attempt += 1) { - const tabLabel = findTabLabel(device.snapshot(), tab); - if (tabLabel) { - device.pressLabel(tabLabel); + const tabNode = findTabNode(device.snapshot(), tab); + if (tabNode) { + device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } device.back(); @@ -140,18 +145,77 @@ async function showTab(device: JourneyDevice, tab: string): Promise { throw new Error(`Cannot return to the ${tab} tab.`); } +function selectAllInboxFilter(device: JourneyDevice): void { + const tapPoint = allFilterTapPoint(device.snapshot()); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel('All'); +} + +function pressPreferences(device: JourneyDevice): void { + const cell = device.snapshot().filter((node) => normalizeLabel(node.label) === 'Preferences' && node.type === 'Cell' && node.width > 0 && node.height > 0); + const preferenceCell = cell.length === 1 ? cell.at(0) : undefined; + if (preferenceCell) { + device.command('press', String(Math.round(preferenceCell.x + preferenceCell.width / 2)), String(Math.round(preferenceCell.y + preferenceCell.height / 2)), '--settle'); + return; + } + device.pressLabel('Preferences'); +} + +function selectSpendSection(device: JourneyDevice, section: 'Expenses' | 'Reports'): void { + const tapPoint = spendSectionTapPoint(device.snapshot(), section); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel(section); +} + +/** On iOS, Expenses labels the whole horizontal strip while Reports has its own bounds. */ +function spendSectionTapPoint(nodes: JourneyNode[], section: 'Expenses' | 'Reports'): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'Expenses' && node.type === 'Cell' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const reports = nodes.find((node) => normalizeLabel(node.label) === 'Reports' && node.x > 0 && node.height > 0); + if (!reports) { + throw new Error('Cannot locate the Expenses and Reports segments in Spend.'); + } + return {x: Math.round(section === 'Reports' ? reports.x + reports.width / 2 : reports.x / 2), y: Math.round(reports.y + reports.height / 2)}; +} + +/** iOS exposes the whole filter strip as the All cell; its center lands on Unread. */ +function allFilterTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'All' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const unread = nodes.find((node) => normalizeLabel(node.label) === 'Unread' && node.x > 0 && node.height > 0); + if (!unread) { + throw new Error('Cannot locate the All filter inside the iOS Inbox strip.'); + } + return {x: Math.round(unread.x / 2), y: Math.round(unread.y + unread.height / 2)}; +} + /** iOS appends unread/review status to tab accessibility labels. */ -function findTabLabel(nodes: JourneyNode[], tab: string): string | undefined { - const screenHeight = nodes.find((node) => node.type === 'Application')?.height; +function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined { + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); const candidates = nodes.filter((node) => { const label = normalizeLabel(node.label); const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); - return matches && (!screenHeight || node.y >= screenHeight * 0.7); + return matches && (!screen || (node.y >= screen.height * 0.7 && node.width >= screen.width * 0.1 && node.height >= screen.height * 0.05)); }); if (candidates.length > 1) { throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); } - return candidates.at(0)?.label; + return candidates.at(0); } async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { @@ -196,9 +260,11 @@ async function waitForMessage(device: JourneyDevice, token: string): Promise node.type === 'Application')?.height; return nodes - .map((node) => normalizeLabel(node.label)) - .filter(Boolean) + .filter((node) => !screenHeight || (node.y + node.height > screenHeight * 0.13 && node.y < screenHeight * 0.88)) + .filter((node) => !!normalizeLabel(node.label)) + .map((node) => `${normalizeLabel(node.label)}:${Math.round(node.y / 8)}`) .join('\n'); } @@ -227,4 +293,4 @@ function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitl return matches.at(0)?.label; } -export {findReportResult, findTabLabel, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; +export {allFilterTapPoint, contentSignature, findReportResult, findTabNode, prepareJourney, runJourneyWorkload, scrollDistance, spendSectionTapPoint, verifyJourneyAccount}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index eb6e90f68c27..06da11572f7b 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -2,7 +2,7 @@ import {describe, expect, it} from 'bun:test'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {findReportResult, findTabLabel, scrollDistance} from '@scripts/pgo/journeyWorkload'; +import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; const fixture = { accountEmail: 'heavy@example.com', @@ -141,12 +141,74 @@ describe('PGO journey safeguards', () => { {type: 'Application', rect: {width: 390, height: 844, y: 0}}, {label: 'Inbox', rect: {width: 390, height: 50, y: 80}}, {label: 'Inbox. Your review is required', rect: {width: 78, height: 71, y: 749}}, + {label: 'Inbox', type: 'StaticText', rect: {width: 26, height: 14, y: 793}}, {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, ], }, 'test.app', ); - expect(findTabLabel(nodes, 'Inbox')).toBe('Inbox. Your review is required'); - expect(findTabLabel(nodes, 'Account')).toBe('Account, My settings. Your review is required.'); + expect(findTabNode(nodes, 'Inbox')?.label).toBe('Inbox. Your review is required'); + expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); + }); + + it('targets the visible All filter rather than the iOS strip cell center over Unread', () => { + // Given an iOS accessibility cell whose bounds cover the whole Inbox filter strip. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'All', type: 'Cell', rect: {width: 390, height: 52, x: 0, y: 115}}, + {label: 'Unread', type: 'Other', rect: {width: 136, height: 40, x: 102, y: 115}}, + ], + }, + 'test.app', + ); + // When the journey chooses the All filter's tap point. + const tapPoint = allFilterTapPoint(nodes); + // Then it stays left of Unread. + expect(tapPoint).toEqual({x: 51, y: 135}); + }); + + it('detects iOS report scrolling when offscreen accessibility labels stay loaded', () => { + // Given the same report labels at different vertical positions after a scroll. + const reportNodes = (firstMessageY: number) => + parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Old message', rect: {width: 390, height: 40, y: firstMessageY - 80}}, + {label: 'Recent message', rect: {width: 390, height: 40, y: firstMessageY}}, + ], + }, + 'test.app', + ); + // When content signatures are compared. + const before = contentSignature(reportNodes(500)); + const after = contentSignature(reportNodes(266)); + // Then movement is visible despite identical labels. + expect(before).not.toBe(after); + }); + + it('targets the visible iOS Spend segments despite a wide Expenses strip cell', () => { + // Given an Expenses cell that covers the full segment strip and a bounded Reports control. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Expenses', type: 'Cell', rect: {width: 1426, height: 52, x: 0, y: 115}}, + {label: 'Reports', type: 'Other', rect: {width: 94, height: 40, x: 125, y: 115}}, + ], + }, + 'test.app', + ); + // When each section is selected. + const expenses = spendSectionTapPoint(nodes, 'Expenses'); + const reports = spendSectionTapPoint(nodes, 'Reports'); + // Then both tap points land inside their visible segments. + expect(expenses).toEqual({x: 63, y: 135}); + expect(reports).toEqual({x: 172, y: 135}); }); }); From b7b4105a2c53327f82d2f635406ff1843eed9524 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 17:20:21 +0200 Subject: [PATCH 39/47] fix(pgo): handle mobile navigation tree edge cases --- scripts/pgo/JOURNEY.md | 6 +-- scripts/pgo/journeyWorkload.ts | 71 +++++++++++++++++++++++++++++--- tests/tooling/pgoJourney.test.ts | 30 +++++++++++++- 3 files changed, 97 insertions(+), 10 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index 6e410d5002e2..aa32aa722d49 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -2,7 +2,7 @@ -`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full Android journey, including message sending and native profile export, has been validated on a physical Samsung phone. iOS execution still needs live validation. +`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full journey, including message sending and native profile export, has been validated on a physical Samsung phone and iPhone 13 Pro. ## Account and device setup @@ -62,7 +62,7 @@ Start with three repetitions per platform. Use the recorded duration and resulti On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. -On the same day, a complete instrumented iOS recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. It sent two approved messages to the account's personal chat, exported two raw profiles, and produced a 15 MB merged profile with 23,822 functions showing nonzero execution counts. The final check confirmed the heavy account and #focus off. This is one validation recording; the three-repetition training batch remains to be collected. +On the same day, a complete instrumented iOS validation recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. A subsequent three-repetition training batch passed in 6 minutes 41 seconds, 6 minutes 35 seconds, and 6 minutes 37 seconds per journey. The batch exported six raw profiles and produced a 15 MB merged profile containing 77,590 functions. Every repetition sent two approved messages to the account's personal chat, then confirmed the heavy account and #focus off. These timings include automation overhead; they are not app performance measurements. The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. @@ -76,4 +76,4 @@ Use an exclusive device lease on a persistent runner: Linux or macOS with an arm Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. -This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows, validating iOS, and measuring held-out interactive performance remain separate integration steps. +This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows and measuring held-out interactive performance remain separate integration steps. diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 919c2d3f7be4..d807dac61693 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -75,7 +75,7 @@ async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): P } pressPreferences(device); await ensureFocusDisabled(device, true); - device.back(); + pressInAppBack(device); await showTab(device, 'Inbox'); console.log('Verified the approved heavy account and disabled #focus.'); } @@ -89,7 +89,7 @@ async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixtu } pressPreferences(device); await ensureFocusDisabled(device, false); - device.back(); + pressInAppBack(device); await showTab(device, 'Inbox'); } @@ -139,7 +139,7 @@ async function showTab(device: JourneyDevice, tab: string): Promise { device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } - device.back(); + pressInAppBack(device); await sleep(300); } throw new Error(`Cannot return to the ${tab} tab.`); @@ -164,6 +164,42 @@ function pressPreferences(device: JourneyDevice): void { device.pressLabel('Preferences'); } +function pressInAppBack(device: JourneyDevice): void { + if (device.platform === 'ios') { + const tapPoint = inAppBackTapPoint(device.snapshot()); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + } + device.back(); +} + +/** The iOS search Back button remains visible but is reported non-hittable while the keyboard is open. */ +function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screen = nodes.find((node) => node.type === 'Application'); + if (!screen) { + return undefined; + } + const candidates = nodes.filter( + (node) => + node.type === 'Button' && + normalizeLabel(node.label) === 'Back' && + node.enabled && + node.width > 0 && + node.height > 0 && + node.x >= 0 && + node.y >= 0 && + node.x < screen.width * 0.25 && + node.y < screen.height * 0.2, + ); + if (candidates.length > 1) { + throw new Error('Ambiguous in-app Back control on iOS.'); + } + const back = candidates.at(0); + return back ? {x: Math.round(back.x + back.width / 2), y: Math.round(back.y + back.height / 2)} : undefined; +} + function selectSpendSection(device: JourneyDevice, section: 'Expenses' | 'Reports'): void { const tapPoint = spendSectionTapPoint(device.snapshot(), section); if (tapPoint) { @@ -212,10 +248,22 @@ function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); return matches && (!screen || (node.y >= screen.height * 0.7 && node.width >= screen.width * 0.1 && node.height >= screen.height * 0.05)); }); - if (candidates.length > 1) { + const first = candidates.at(0); + const duplicatedNode = + first && + candidates.every( + (node) => + node.type === first.type && + normalizeLabel(node.label) === normalizeLabel(first.label) && + node.x === first.x && + node.y === first.y && + node.width === first.width && + node.height === first.height, + ); + if (candidates.length > 1 && !duplicatedNode) { throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); } - return candidates.at(0); + return first; } async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { @@ -293,4 +341,15 @@ function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitl return matches.at(0)?.label; } -export {allFilterTapPoint, contentSignature, findReportResult, findTabNode, prepareJourney, runJourneyWorkload, scrollDistance, spendSectionTapPoint, verifyJourneyAccount}; +export { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + prepareJourney, + runJourneyWorkload, + scrollDistance, + spendSectionTapPoint, + verifyJourneyAccount, +}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 06da11572f7b..de4d031da0d5 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -2,7 +2,7 @@ import {describe, expect, it} from 'bun:test'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; +import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; const fixture = { accountEmail: 'heavy@example.com', @@ -151,6 +151,15 @@ describe('PGO journey safeguards', () => { expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { + const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; + const snapshot = (tabs: Array) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'Application', rect: {x: 0, y: 0, width: 1080, height: 2280}}, ...tabs]}, 'test.app'); + + expect(findTabNode(snapshot([tab, tab]), 'Inbox')?.x).toBe(216); + expect(() => findTabNode(snapshot([tab, {...tab, rect: {...tab.rect, x: 432}}]), 'Inbox')).toThrow('Ambiguous Inbox tab'); + }); + it('targets the visible All filter rather than the iOS strip cell center over Unread', () => { // Given an iOS accessibility cell whose bounds cover the whole Inbox filter strip. const nodes = parseJourneySnapshot( @@ -170,6 +179,25 @@ describe('PGO journey safeguards', () => { expect(tapPoint).toEqual({x: 51, y: 135}); }); + it('taps the iOS search Back button even when the keyboard marks it non-hittable', () => { + // Given the search screen's bounded Back button beneath an open keyboard. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Back', type: 'Button', hittable: false, rect: {width: 40, height: 40, x: 8, y: 59}}, + {label: 'Back', type: 'Button', rect: {width: 40, height: 40, x: 300, y: 700}}, + ], + }, + 'test.app', + ); + // When the journey looks for the in-app Back control. + const tapPoint = inAppBackTapPoint(nodes); + // Then it chooses the top-left button rather than the keyboard area. + expect(tapPoint).toEqual({x: 28, y: 79}); + }); + it('detects iOS report scrolling when offscreen accessibility labels stay loaded', () => { // Given the same report labels at different vertical positions after a scroll. const reportNodes = (firstMessageY: number) => From 7673df96f954a4b7eae33c6e282ffd0b0d26b693 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 24 Sep 2026 15:08:13 +0200 Subject: [PATCH 40/47] perf(pgo): benchmark short signed-in journeys --- scripts/pgo/JOURNEY.md | 11 ++ scripts/pgo/README.md | 13 +- scripts/pgo/android.ts | 22 ++- scripts/pgo/ios.ts | 94 ++++++++++-- scripts/pgo/journeyBenchmark.ts | 196 +++++++++++++++++++++++++ scripts/pgo/journeyBenchmarkAndroid.ts | 52 +++++++ scripts/pgo/journeyBenchmarkIOS.ts | 102 +++++++++++++ scripts/pgo/journeyWorkload.ts | 21 +++ scripts/pgo/pgo.ts | 54 +++++-- scripts/pgo/shared.ts | 7 +- tests/tooling/pgoJourney.test.ts | 48 +++++- 11 files changed, 584 insertions(+), 36 deletions(-) create mode 100644 scripts/pgo/journeyBenchmark.ts create mode 100644 scripts/pgo/journeyBenchmarkAndroid.ts create mode 100644 scripts/pgo/journeyBenchmarkIOS.ts diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index aa32aa722d49..bd6ff35593bd 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -77,3 +77,14 @@ Use an exclusive device lease on a persistent runner: Linux or macOS with an arm Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows and measuring held-out interactive performance remain separate integration steps. + +## Held-out interaction benchmark + +The PGO workflow has a shorter, read-only benchmark that is separate from the six-minute profile-collection journey. Each sample relaunches the signed-in app, selects Spend → Expenses, returns to Inbox, then times a warm Inbox → Spend → Inbox switch. The app's existing first-paint, content-load, and Inbox navigation spans measure the UI work. One complete warm-up is discarded for each build, followed by the requested number of fresh-process samples. The benchmark checks the same approved account and #focus preference, but sends no messages. + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Build both the release and optimized artifacts with the current span allowlist first. `build-optimized --profile /absolute/path/to/merged.profdata` selects an immutable archived profile; without `--profile`, it uses `newdot.profdata`. The output uses the existing startup benchmark's raw-sample and summary CSV format in a unique `.pgo//benchmarks/journey//` directory. The two phases run together, so a failed or repeated phase cannot silently pair with stale data. Compare each interactive span separately; adding their durations together does not give a full journey time because there is automation time between taps. Keep this benchmark out of the profile-training batch, and avoid using its result as a release gate until build-order drift and device temperature have been characterized. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 6c01fac3d088..f9ec34b76b04 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -4,7 +4,7 @@ This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. -The command enables the existing `ManualAppStartup` benchmark span in every app it builds. Profile collection waits for that span to finish, then explicitly writes the LLVM counters before the app process is stopped. Benchmarking uses the same native span tooling as the repository's general startup benchmark. +The command enables `ManualAppStartup` and the existing Spend/Inbox navigation spans in every app it builds. Startup profile collection waits for `ManualAppStartup` to finish, then explicitly writes the LLVM counters before the app process is stopped. Both benchmark modes use app-defined span durations rather than device-automation wall time. ## Prepare local release identifiers @@ -99,8 +99,19 @@ scripts/pgo/pgo.ts android build-release scripts/pgo/pgo.ts android benchmark 20 45 ``` +Pass `--profile /absolute/path/to/merged.profdata` to `build-optimized` when comparing an archived profile. The path becomes part of the compiler input; keep the file immutable for the build. Without it, the command uses the platform's `newdot.profdata`. Local PGO builds disable Sentry uploads. iOS temporarily skips the FullStory and Sentry upload phases and restores the original Xcode project afterward. Android excludes `fullstoryRelease` and archives the pre-FullStory APK, so its local benchmarks are internally paired but do not represent the fully post-processed production APK. Do not compare their absolute timings directly with earlier or store builds that used FullStory processing. + The benchmark installs the archived baseline, runs one warm-up and the requested `ManualAppStartup` samples, then repeats the process with the optimized app. Both builds use the same application identifier, so installing the second artifact preserves the seeded account and data. Results are stored under `.pgo//benchmarks/` in the repository benchmark CSV format. +For a separate, read-only interaction benchmark, use the approved signed-in account fixture described in [JOURNEY.md](./JOURNEY.md): + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id com.chrispader.expensify.pgo --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id com.chrispader.expensify.expensifylite.pgo --fixture .pgo/journey-fixture.json +``` + +Each variant gets one discarded warm-up and 20 measured fresh-process journeys. Before each sample, the runner selects Spend → Expenses and returns to Inbox; it then times a warm Inbox → Spend → Inbox tab switch. Expenses can restore the account's saved query and filters, so keep that state fixed across both phases. The CSVs under a unique `.pgo//benchmarks/journey//` directory contain `ManualNavigateToReportsFirstPaint`, `ManualNavigateToReportsContentLoad`, and `ManualNavigateToInboxTab` samples, alongside a run manifest with artifact hashes. These are app-defined tap-to-paint timings, so automation overhead is outside the measurement. The journey sends no messages and leaves account data unchanged. Both phases run in one command and are compared only within that batch. Rebuild both archived variants with this version of the tooling before benchmarking; older PGO builds only logged startup. + The stages can also run independently: ```bash diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 0463bf8073d4..e0ef640aa0a8 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -23,7 +23,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); const buildGradlePath = join(androidDirectory, 'build.gradle'); - const gradleReleaseApkPath = join(androidDirectory, 'build/outputs/apk/release/Expensify-release.apk'); + const gradleReleaseApkPath = join(androidDirectory, 'build/intermediates/apk/release/packageRelease/Expensify-release.apk'); const profileDumpTimeoutSeconds = 5; const artifactPaths: BuildArtifactPaths = { @@ -77,18 +77,26 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent console.log(`Copied release APK: ${destination}`); } - function build(kind: BuildKind): void { - if (kind === 'optimized' && !existsSync(join(profileDirectory, 'newdot.profdata'))) { - fail(`Missing ${join(profileDirectory, 'newdot.profdata')}. Run merge first.`); + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? join(profileDirectory, 'newdot.profdata'); + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); } const pgoModes: Record = {release: 'off', instrumented: 'generate', optimized: 'use'}; - const gradleArguments = [':assembleRelease', '-PpatchedArtifacts.forceBuildFromSource=true', '-PreactNativeArchitectures=arm64-v8a', `-PpgoMode=${pgoModes[kind]}`]; + const gradleArguments = [ + ':packageRelease', + '-PpatchedArtifacts.forceBuildFromSource=true', + '-PreactNativeArchitectures=arm64-v8a', + `-PpgoMode=${pgoModes[kind]}`, + '-x', + 'fullstoryRelease', + ]; if (kind === 'optimized') { - gradleArguments.push(`-PpgoProfile=${join(profileDirectory, 'newdot.profdata')}`); + gradleArguments.push(`-PpgoProfile=${profilePath}`); } - run('/usr/bin/env', [BENCHMARK_SPANS_ENVIRONMENT, './gradlew', ...gradleArguments], androidDirectory); + run('/usr/bin/env', [BENCHMARK_SPANS_ENVIRONMENT, 'SENTRY_DISABLE_AUTO_UPLOAD=true', './gradlew', ...gradleArguments], androidDirectory); archiveReleaseApk(artifactPaths[kind]); } diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 8fd607438ebc..bc2ba74c7a1c 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -4,7 +4,7 @@ import {isRecord} from '@libs/ObjectUtils'; -import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {dirname, join} from 'node:path'; @@ -21,6 +21,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); const markerDirectory = join(profileDirectory, 'markers'); const iosDirectory = join(rootDirectory, 'Mobile-Expensify/iOS'); + const projectPath = join(iosDirectory, 'Expensify.xcodeproj/project.pbxproj'); const hermesVersionPath = join(rootDirectory, 'node_modules/react-native/sdks/.hermesv1version'); const remotePgoDirectory = 'Library/Caches/ExpensifyPGO'; const profileStatusPath = `${remotePgoDirectory}/profile-status.txt`; @@ -134,22 +135,23 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str return capture('xcrun', ['--find', name]).trim(); } - function sourceBuildEnvironment(mode: PgoMode): string[] { + function sourceBuildEnvironment(mode: PgoMode, profilePath = mergedProfilePath): string[] { const environment = [ 'BUILD_RN_FROM_SOURCE=1', 'RCT_USE_RN_DEP=0', 'RCT_USE_PREBUILT_RNCORE=0', 'RCT_BUILD_HERMES_FROM_SOURCE=true', BENCHMARK_SPANS_ENVIRONMENT, + 'SENTRY_DISABLE_AUTO_UPLOAD=true', `EXPENSIFY_PGO_MODE=${mode}`, ]; if (mode === 'use') { - environment.push(`EXPENSIFY_PGO_PROFILE=${mergedProfilePath}`); + environment.push(`EXPENSIFY_PGO_PROFILE=${profilePath}`); } return environment; } - function prepareSourcePods(mode: PgoMode): void { + function prepareSourcePods(mode: PgoMode, profilePath: string): void { if (!existsSync(hermesVersionPath)) { fail(`Missing React Native's pinned Hermes version file at ${hermesVersionPath}.`); } @@ -164,7 +166,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str rmSync(directory, {recursive: true, force: true}); } } - run('/usr/bin/env', [...sourceBuildEnvironment(mode), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); + run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); for (const {directory, requiredFile} of requiredPodSources) { if (!existsSync(join(directory, requiredFile))) { fail(`CocoaPods did not install the required source file ${join(directory, requiredFile)}.`); @@ -173,7 +175,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str rmSync(join(iosDirectory, 'Pods/hermes-engine/build'), {recursive: true, force: true}); } - function pgoBuildSettings(kind: BuildKind): string[] { + function pgoBuildSettings(kind: BuildKind, profilePath: string): string[] { const baseSettings = ['ENABLE_CODE_COVERAGE=NO', 'CLANG_COVERAGE_MAPPING=NO', 'CLANG_USE_OPTIMIZATION_PROFILE=NO']; if (kind === 'instrumented') { return [ @@ -186,23 +188,24 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ]; } if (kind === 'optimized') { - const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-error=profile-instr-unprofiled`; + const clangProfileFlags = `-fprofile-instr-use=${profilePath} -Wno-error=profile-instr-unprofiled`; return [ ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, - `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${mergedProfilePath}`, + `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${profilePath}`, ]; } return baseSettings; } - function build(kind: BuildKind): void { - if (kind === 'optimized' && !existsSync(mergedProfilePath)) { - fail(`Missing ${mergedProfilePath}. Run merge first.`); + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? mergedProfilePath; + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); } if (kind === 'optimized') { - const profileFormatPath = `${mergedProfilePath}.format`; + const profileFormatPath = `${profilePath}.format`; const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; if (recordedFormat !== profileFormat) { fail('The merged iOS profile predates the current Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); @@ -215,7 +218,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str } else if (kind === 'optimized') { mode = 'use'; } - prepareSourcePods(mode); + prepareSourcePods(mode, profilePath); const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); const builtAppPath = join(derivedDataDirectory, 'Build/Products/Release-iphoneos/Expensify.app'); const signingSettings = codeSigningAllowed @@ -246,9 +249,9 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str 'ONLY_ACTIVE_ARCH=YES', 'COMPILER_INDEX_STORE_ENABLE=NO', ...signingSettings, - ...pgoBuildSettings(kind), + ...pgoBuildSettings(kind, profilePath), ]; - run('/usr/bin/env', [...sourceBuildEnvironment(mode), ...xcodeArguments], iosDirectory); + withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), ...xcodeArguments], iosDirectory)); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -258,6 +261,59 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str console.log(`Copied release app: ${artifactPaths[kind]}`); } + /** Prevent local release builds from publishing source maps or FullStory assets; restore the existing project text even if Xcode fails. */ + function withLocalUploadPhasesDisabled(buildApp: () => void): void { + const replacements = [ + // Xcode expands these literal variables inside the project build phase. + { + // eslint-disable-next-line no-template-curly-in-string + command: '\\"${PODS_ROOT}/FullStory/tools/FullStoryCommandLine\\" \\"${CONFIGURATION_BUILD_DIR}/${WRAPPER_NAME}\\"', + disabled: 'echo Skipping FullStory upload for local PGO build', + }, + {command: 'source \\"$PROJECT_ROOT/tools/upload-sentry-source-maps.sh\\"', disabled: 'echo Skipping Sentry source-map upload for local PGO build'}, + ]; + const originalProject = readFileSync(projectPath, 'utf8'); + const guardedProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, command, disabled), originalProject); + let restored = false; + let guarded = false; + const restoreProject = () => { + if (restored) { + return; + } + if (!guarded) { + writeFileSync(projectPath, originalProject); + restored = true; + return; + } + const currentProject = readFileSync(projectPath, 'utf8'); + const restoredProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, disabled, command), currentProject); + writeFileSync(projectPath, restoredProject); + restored = true; + }; + const onInterrupt = () => { + restoreProject(); + process.exit(130); + }; + const onTermination = () => { + restoreProject(); + process.exit(143); + }; + process.once('SIGINT', onInterrupt); + process.once('SIGTERM', onTermination); + try { + writeFileSync(projectPath, guardedProject); + guarded = true; + buildApp(); + } finally { + try { + restoreProject(); + } finally { + process.off('SIGINT', onInterrupt); + process.off('SIGTERM', onTermination); + } + } + } + function install(kind: BuildKind): void { const appPath = artifactPaths[kind]; if (!existsSync(appPath)) { @@ -408,4 +464,12 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str }; } +function replaceOnce(content: string, previous: string, next: string): string { + const offset = content.indexOf(previous); + if (offset < 0 || content.indexOf(previous, offset + previous.length) >= 0) { + fail('The iOS project upload phases changed. Refusing to build until the local PGO upload guard is updated.'); + } + return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; +} + export default createIOSPgoAdapter; diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts new file mode 100644 index 000000000000..2783f0221c01 --- /dev/null +++ b/scripts/pgo/journeyBenchmark.ts @@ -0,0 +1,196 @@ +/** Repeats a small, read-only signed-in journey and records app-defined navigation spans. */ + +import {createHash} from 'node:crypto'; +import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readlinkSync, writeFileSync} from 'node:fs'; +import {join, relative} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; +import type {JourneyFixture} from './journeyConfig'; +import type {PlatformAdapter, BenchmarkKind} from './shared'; + +import {createBenchmarkRecorder} from '../lib/benchmarkAppStartup'; +import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; +import {capture, fail} from '../lib/scriptUtils'; +import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; +import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; +import {createJourneyDevice} from './journeyDevice'; +import {selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {JOURNEY_SPAN_NAMES} from './shared'; + +type JourneyBenchmarkReader = { + begin: (spanNames: readonly string[]) => void; + collect: (spanNames: readonly string[], timeoutSeconds: number) => Promise; +}; + +function journeyBenchmarkPaths(adapter: Pick, batchID: string): Record { + const directory = join(adapter.benchmarkDirectory, 'journey', batchID); + return {release: join(directory, 'release.csv'), optimized: join(directory, 'pgo-optimized.csv')}; +} + +/** Installs the selected build, discards one complete warm-up, then records every requested run. */ +async function benchmarkJourneyBuild( + adapter: PlatformAdapter, + kind: BenchmarkKind, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + outputPath: string, +): Promise { + console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); + adapter.install(kind); + + const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-journey-benchmark-${adapter.name}-${kind}-${Date.now()}`}); + const reader: JourneyBenchmarkReader = + adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); + try { + device.open(false); + await waitForTab(device, 'Account'); + await verifyJourneyAccount(device, fixture); + console.log(`Running one unmeasured ${kind} journey warm-up.`); + await measureJourney(device, reader, timeoutSeconds); + + const recorder = createBenchmarkRecorder({spanNames: [...JOURNEY_SPAN_NAMES], runs, outputPath}); + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + const events = await measureJourney(device, reader, timeoutSeconds); + console.log(`${kind} journey ${runNumber}/${runs}: ${recorder.record(events, runNumber).join(', ')}`); + } + await verifyJourneyAccount(device, fixture); + await recorder.complete(`${kind} journey metrics`); + } finally { + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +/** Uses a new app process for each sample; automation latency is outside the timed spans. */ +async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, timeoutSeconds: number): Promise { + device.open(true); + await waitForTab(device, 'Account'); + // Normalize the saved Spend view before timing either tab transition. This makes the measured taps warm navigations from a known Expenses state. + await showTab(device, 'Spend'); + selectSpendSection(device, 'Expenses'); + device.waitLabel('Filters'); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + + const spendSpans = JOURNEY_SPAN_NAMES.slice(0, 2); + const inboxSpans = [JOURNEY_SPAN_NAMES[2]]; + reader.begin(spendSpans); + await showTab(device, 'Spend'); + device.waitLabel('Filters'); + const spendEvents = await reader.collect(spendSpans, timeoutSeconds); + + reader.begin(inboxSpans); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + const inboxEvents = await reader.collect(inboxSpans, timeoutSeconds); + return [...spendEvents, ...inboxEvents]; +} + +async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, fixture: JourneyFixture, runs: number, timeoutSeconds: number): Promise { + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const paths = journeyBenchmarkPaths(adapter, batchID); + const directory = join(adapter.benchmarkDirectory, 'journey', batchID); + if (existsSync(directory)) { + fail(`Journey benchmark batch already exists: ${directory}`); + } + mkdirSync(directory, {recursive: true}); + writeFileSync( + join(directory, 'run.json'), + `${JSON.stringify( + { + platform: adapter.name, + deviceID, + appID: adapter.appID(), + accountEmail: fixture.accountEmail, + runs, + timeoutSeconds, + spans: JOURNEY_SPAN_NAMES, + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + artifacts: adapter.artifactPaths, + artifactSHA256: { + release: await sha256Artifact(adapter.artifactPaths.release), + optimized: await sha256Artifact(adapter.artifactPaths.optimized), + }, + }, + null, + 2, + )}\n`, + ); + console.log(`Journey benchmark batch: ${directory}`); + console.log('=== Journey benchmark phase 1/2: Release ==='); + await benchmarkJourneyBuild(adapter, 'release', deviceID, fixture, runs, timeoutSeconds, paths.release); + console.log('=== Journey benchmark phase 2/2: PGO optimized ==='); + await benchmarkJourneyBuild(adapter, 'optimized', deviceID, fixture, runs, timeoutSeconds, paths.optimized); + await compareJourneyBenchmarks(paths, runs); +} + +async function compareJourneyBenchmarks(paths: Record, runs: number): Promise { + if (!existsSync(paths.release) || !existsSync(paths.optimized)) { + fail('Missing paired journey samples in this benchmark batch.'); + } + const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(paths.release), readBenchmarkSamples(paths.optimized)]); + const release = benchmarkMetrics(releaseSamples, JOURNEY_SPAN_NAMES); + const optimized = benchmarkMetrics(optimizedSamples, JOURNEY_SPAN_NAMES); + + console.log('=== Journey benchmark comparison (positive percentages are faster) ==='); + for (const span of JOURNEY_SPAN_NAMES) { + const baseline = release[span]?.stats; + const pgo = optimized[span]?.stats; + if (!baseline || !pgo || baseline.runs !== runs || pgo.runs !== runs) { + fail(`Expected ${runs} paired ${span} samples in this journey benchmark batch.`); + } + const improvement = ((baseline.average - pgo.average) / baseline.average) * 100; + console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); + } + console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); +} + +async function sha256Artifact(path: string): Promise { + if (lstatSync(path).isDirectory()) { + return sha256Directory(path); + } + const hash = capture('shasum', ['-a', '256', path]).trim().split(/\s+/).at(0); + if (!hash || !/^[\da-f]{64}$/.test(hash)) { + fail(`Could not record the benchmark artifact SHA-256 for ${path}.`); + } + return hash; +} + +/** Include every bundle file, relative path, directory and symlink in a deterministic iOS artifact digest. */ +async function sha256Directory(directory: string): Promise { + const hash = createHash('sha256'); + async function visit(path: string): Promise { + for (const name of readdirSync(path).toSorted()) { + const child = join(path, name); + const details = lstatSync(child); + hash.update(relative(directory, child)).update('\0'); + if (details.isDirectory()) { + hash.update('directory\0'); + await visit(child); + } else if (details.isSymbolicLink()) { + hash.update('link\0').update(readlinkSync(child)).update('\0'); + } else if (details.isFile()) { + hash.update('file\0').update(String(details.size)).update('\0'); + for await (const chunk of createReadStream(child)) { + const bytes: unknown = chunk; + if (!Buffer.isBuffer(bytes)) { + fail(`Artifact file did not stream bytes: ${child}`); + } + hash.update(bytes); + } + } else { + fail(`Unsupported artifact entry in ${directory}: ${child}`); + } + } + } + await visit(directory); + return hash.digest('hex'); +} + +export {benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; diff --git a/scripts/pgo/journeyBenchmarkAndroid.ts b/scripts/pgo/journeyBenchmarkAndroid.ts new file mode 100644 index 000000000000..9aa305e40acf --- /dev/null +++ b/scripts/pgo/journeyBenchmarkAndroid.ts @@ -0,0 +1,52 @@ +/** Collects completed interactive benchmark spans from the current Android app process. */ + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {POLL_INTERVAL_MS, parseBenchmarkLogEvents, sleep} from '../lib/nativeAppBenchmark/shared'; +import {capture, run} from '../lib/scriptUtils'; + +function createAndroidJourneyBenchmarkReader(deviceID: string, appID: string) { + const adbArgs = (args: string[]) => ['-s', deviceID, ...args]; + let consumedEventCount = 0; + + function begin(): void { + run('adb', adbArgs(['logcat', '-c'])); + consumedEventCount = 0; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + if (spanNames.length === 0) { + throw new Error('Select at least one Android journey benchmark span.'); + } + const processID = capture('adb', adbArgs(['shell', 'pidof', appID])) + .trim() + .split(/\s+/) + .find((candidate) => /^\d+$/.test(candidate)); + if (!processID) { + throw new Error(`Unable to find the running Android process for ${appID}.`); + } + + const deadline = Date.now() + timeoutSeconds * 1000; + let observedSpanNames: string[] = []; + while (Date.now() < deadline) { + const logs = capture('adb', adbArgs(['logcat', `--pid=${processID}`, '-d', '-v', 'raw'])); + const events = parseBenchmarkLogEvents(logs); + if (events.length < consumedEventCount) { + throw new Error('Android benchmark logcat events were discarded during collection. Restart the journey benchmark.'); + } + const freshEvents = events.slice(consumedEventCount); + const collected = spanNames.map((spanName) => freshEvents.findLast((event) => event.span === spanName)); + observedSpanNames = [...new Set(freshEvents.map((event) => event.span))]; + if (collected.every((event) => event !== undefined)) { + consumedEventCount = events.length; + return collected.filter((event): event is BenchmarkLogEvent => event !== undefined); + } + await sleep(POLL_INTERVAL_MS); + } + throw new Error(`Timed out after ${timeoutSeconds}s waiting for Android journey benchmark spans: ${spanNames.join(', ')}. Observed: ${observedSpanNames.join(', ') || 'none'}.`); + } + + return {begin, collect}; +} + +export default createAndroidJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyBenchmarkIOS.ts b/scripts/pgo/journeyBenchmarkIOS.ts new file mode 100644 index 000000000000..f6287014ed46 --- /dev/null +++ b/scripts/pgo/journeyBenchmarkIOS.ts @@ -0,0 +1,102 @@ +/** Collects completed interactive benchmark spans from an iOS app's cache markers. */ + +// cspell:ignore devicectl + +import {existsSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {iOSBenchmarkMarkerPath, parseBenchmarkLogEvents} from '../lib/nativeAppBenchmark'; +import {runAllowFailure, sleep} from '../lib/scriptUtils'; + +const POLL_INTERVAL_MS = 250; + +/** Snapshots markers before an action, then waits for a fresh event for every requested span. */ +function createIOSJourneyBenchmarkReader(deviceID: string, appID: string) { + let previousMarkers: Map | undefined; + + function begin(spanNames: readonly string[]): void { + if (spanNames.length === 0) { + throw new Error('At least one iOS benchmark span is required.'); + } + const markers = new Map(); + for (const spanName of new Set(spanNames)) { + markers.set(spanName, readMarker(deviceID, appID, spanName)); + } + previousMarkers = markers; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + const snapshot = previousMarkers; + if (spanNames.length === 0 || !snapshot || spanNames.some((spanName) => !snapshot.has(spanName))) { + throw new Error('Snapshot every requested iOS benchmark span with begin() before the UI action.'); + } + if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) { + throw new Error('The iOS benchmark span timeout must be positive.'); + } + + const uniqueSpanNames = [...new Set(spanNames)]; + const deadline = Date.now() + timeoutSeconds * 1000; + const events = new Map(); + while (Date.now() < deadline) { + for (const spanName of uniqueSpanNames) { + if (events.has(spanName)) { + continue; + } + const marker = readMarker(deviceID, appID, spanName); + if (marker === undefined || marker === snapshot.get(spanName)) { + continue; + } + const event = parseBenchmarkLogEvents(marker).findLast((candidate) => candidate.span === spanName); + if (event) { + events.set(spanName, event); + } + } + if (events.size === uniqueSpanNames.length) { + return spanNames.map((spanName) => { + const event = events.get(spanName); + if (!event) { + throw new Error(`Missing collected iOS benchmark span ${spanName}.`); + } + return event; + }); + } + await sleep(POLL_INTERVAL_MS); + } + const missing = uniqueSpanNames.filter((spanName) => !events.has(spanName)); + throw new Error(`Timed out after ${timeoutSeconds}s waiting for iOS benchmark span(s): ${missing.join(', ')}.`); + } + + return {begin, collect}; +} + +function readMarker(deviceID: string, appID: string, spanName: string): string | undefined { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-journey-benchmark-ios-')); + const localPath = join(temporaryDirectory, 'benchmark.log'); + try { + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceID, + '--source', + iOSBenchmarkMarkerPath(spanName), + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + appID, + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8') : undefined; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } +} + +export default createIOSJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index d807dac61693..5d23ead14775 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -145,6 +145,24 @@ async function showTab(device: JourneyDevice, tab: string): Promise { throw new Error(`Cannot return to the ${tab} tab.`); } +/** Wait for a mounted bottom tab without navigating away from a partially loaded screen. */ +async function waitForTab(device: Pick, tab: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + if (findTabNode(device.snapshot(), tab)) { + return; + } + } catch (error) { + if (!(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + } + await sleep(300); + } + throw new Error(`Timed out waiting for the ${tab} bottom tab to mount.`); +} + function selectAllInboxFilter(device: JourneyDevice): void { const tapPoint = allFilterTapPoint(device.snapshot()); if (tapPoint) { @@ -350,6 +368,9 @@ export { prepareJourney, runJourneyWorkload, scrollDistance, + selectSpendSection, + showTab, spendSectionTapPoint, verifyJourneyAccount, + waitForTab, }; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 486c3660bce9..2343adaa03fb 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -5,7 +5,8 @@ import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; -import {existsSync, mkdirSync, writeFileSync} from 'node:fs'; +import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {resolve} from 'node:path'; import type {BenchmarkStats} from '../lib/benchmarkStatistics'; import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; @@ -17,6 +18,8 @@ import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppB import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; +import {benchmarkJourneyAll} from './journeyBenchmark'; +import {parseJourneyFixture} from './journeyConfig'; import {STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; @@ -34,6 +37,7 @@ const WORKFLOW_COMMANDS = [ 'benchmark-optimized', 'benchmark', 'compare-benchmarks', + 'benchmark-journey', 'dump', 'pull', 'merge', @@ -58,15 +62,15 @@ async function main(): Promise { }, { name: 'runs', - description: 'Number of measured startup runs', + description: 'Number of measured startup or journey runs', default: DEFAULT_STARTUP_RUNS, - parse: (value) => parsePositiveInteger(value, 'Startup run count'), + parse: (value) => parsePositiveInteger(value, 'Run count'), }, { name: 'timeout', - description: `Seconds to wait for the ${STARTUP_SPAN_NAME} benchmark span`, + description: `Seconds to wait for the ${STARTUP_SPAN_NAME} or journey benchmark spans`, default: DEFAULT_STARTUP_WAIT_SECONDS, - parse: (value) => parsePositiveInteger(value, 'Startup span timeout'), + parse: (value) => parsePositiveInteger(value, 'Span timeout'), }, ], namedArgs: { @@ -78,6 +82,14 @@ async function main(): Promise { description: 'Device identifier to use (adb serial on Android; CoreDevice identifier, UDID, serial number, or device name on iOS)', required: false, }, + fixture: { + description: 'Approved account fixture for journey benchmarks', + required: false, + }, + profile: { + description: 'Immutable merged profile to use when building the optimized app', + required: false, + }, }, }); /* eslint-enable @typescript-eslint/naming-convention */ @@ -86,14 +98,25 @@ async function main(): Promise { await runWorkflow( parseChoice(String(platformName), PLATFORM_NAMES, 'Platform'), parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), - requirePositiveInteger(Number(runs), 'Startup run count'), - requirePositiveInteger(Number(timeout), 'Startup span timeout'), + requirePositiveInteger(Number(runs), 'Run count'), + requirePositiveInteger(Number(timeout), 'Span timeout'), cli.namedArgs['app-id'], cli.namedArgs.device, + cli.namedArgs.fixture, + cli.namedArgs.profile, ); } -async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number, appID?: string, deviceIdentifier?: string): Promise { +async function runWorkflow( + platformName: PlatformName, + workflow: WorkflowCommand, + runs: number, + timeoutSeconds: number, + appID?: string, + deviceIdentifier?: string, + fixturePath?: string, + profilePath?: string, +): Promise { const adapter = getAdapter(platformName, appID, deviceIdentifier); switch (workflow) { @@ -104,7 +127,7 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand adapter.build('instrumented'); return; case 'build-optimized': - adapter.build('optimized'); + adapter.build('optimized', profilePath ? resolve(profilePath) : undefined); return; case 'verify-instrumented': adapter.verifyInstrumentation(); @@ -135,6 +158,11 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand case 'compare-benchmarks': await compareBenchmarks(adapter); return; + case 'benchmark-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds); + return; + } case 'dump': await adapter.dumpProfiles(); return; @@ -149,6 +177,14 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand } } +function readJourneyFixture(fixturePath?: string): ReturnType { + if (!fixturePath) { + fail('Journey benchmarks require --fixture with the approved signed-in account.'); + } + const input: unknown = JSON.parse(readFileSync(resolve(fixturePath), 'utf8')); + return parseJourneyFixture(input); +} + function mergeProfiles(adapter: PlatformAdapter): void { mkdirSync(adapter.profileDirectory, {recursive: true}); const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index c8c9daccf786..ed48e4666d92 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -3,7 +3,8 @@ import type {PlatformName} from '../lib/nativeAppBenchmark'; const STARTUP_SPAN_NAME = 'ManualAppStartup'; -const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${STARTUP_SPAN_NAME}`; +const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; +const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES].join(',')}`; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; @@ -20,7 +21,7 @@ type PlatformAdapter = { readonly artifactPaths: BuildArtifactPaths; readonly profileFormat?: string; appID: () => string; - build: (kind: BuildKind) => void; + build: (kind: BuildKind, profilePath?: string) => void; install: (kind: BuildKind) => void; verifyInstrumentation: () => void; clearDeviceProfiles: () => Promise; @@ -29,5 +30,5 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -export {BENCHMARK_SPANS_ENVIRONMENT, STARTUP_SPAN_NAME}; +export {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index de4d031da0d5..2d4324694d97 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -1,8 +1,14 @@ import {describe, expect, it} from 'bun:test'; +import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; +import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint, waitForTab} from '@scripts/pgo/journeyWorkload'; +import {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; const fixture = { accountEmail: 'heavy@example.com', @@ -13,6 +19,31 @@ const fixture = { }; describe('PGO journey safeguards', () => { + it('logs the held-out navigation spans in both PGO builds without replacing startup samples', () => { + // Given release and optimized artifacts share one build-time benchmark allowlist. + const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); + // When / Then the startup span and every short-journey span remain available. + expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES]); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1')).toEqual({ + release: '/pgo/ios/benchmarks/journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/journey/batch-1/pgo-optimized.csv', + }); + }); + + it('hashes iOS bundle resources as well as the app executable', async () => { + const app = mkdtempSync(join(tmpdir(), 'pgo-test-app-')); + try { + mkdirSync(join(app, 'Frameworks')); + writeFileSync(join(app, 'Expensify'), 'same executable'); + writeFileSync(join(app, 'main.jsbundle'), 'first bundle'); + const before = await sha256Artifact(app); + writeFileSync(join(app, 'main.jsbundle'), 'second bundle'); + expect(await sha256Artifact(app)).not.toBe(before); + } finally { + rmSync(app, {recursive: true, force: true}); + } + }); + it('requires explicit message approval in the fixture', () => { // Given account approval alone does not authorize a particular message destination. for (const allowMessages of [undefined, false, 'true']) { @@ -151,6 +182,21 @@ describe('PGO journey safeguards', () => { expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); }); + it('waits for a status-bearing iOS Account tab without requiring an exact label', async () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + const device = {snapshot: () => nodes}; + await expect(waitForTab(device, 'Account')).resolves.toBeUndefined(); + }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; const snapshot = (tabs: Array) => From 21f2d6c5ad01fc1e74165ccc0b58fca87744449e Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 24 Sep 2026 16:16:09 +0200 Subject: [PATCH 41/47] perf(pgo): add heavy report benchmark --- scripts/pgo/JOURNEY.md | 13 ++ scripts/pgo/README.md | 2 + scripts/pgo/journeyBenchmark.ts | 212 ++++++++++++++++++++++++++----- scripts/pgo/journeyWorkload.ts | 15 ++- scripts/pgo/pgo.ts | 22 +++- scripts/pgo/shared.ts | 5 +- tests/tooling/pgoJourney.test.ts | 9 +- 7 files changed, 235 insertions(+), 43 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index bd6ff35593bd..ce89443ae32e 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -88,3 +88,16 @@ scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id BUN ``` Build both the release and optimized artifacts with the current span allowlist first. `build-optimized --profile /absolute/path/to/merged.profdata` selects an immutable archived profile; without `--profile`, it uses `newdot.profdata`. The output uses the existing startup benchmark's raw-sample and summary CSV format in a unique `.pgo//benchmarks/journey//` directory. The two phases run together, so a failed or repeated phase cannot silently pair with stale data. Compare each interactive span separately; adding their durations together does not give a full journey time because there is automation time between taps. Keep this benchmark out of the profile-training batch, and avoid using its result as a release gate until build-order drift and device temperature have been characterized. + +## Heavy report benchmark + +Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes up to six fast swipes toward older messages and six back toward recent messages. It fails if the list does not expose changing content. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. + +```bash +scripts/pgo/pgo.ts android benchmark-heavy-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-heavy-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Rebuild both app variants after adding the `/r/*` span to the build-time benchmark allowlist. The earlier artifacts cannot emit this metric. Each batch goes to `.pgo//benchmarks/heavy-journey//` with raw and summary CSVs, artifact hashes, and a run manifest. On Android, an additional frame CSV counts rendered and missed-deadline frames during the fast-scroll window. Compare its missed-frame share separately from report-open latency; it is not elapsed scroll time. The current physical-iPhone frame probe could not consistently attach Xcode's Animation Hitches trace to the app process, so the iOS runner validates scroll movement but deliberately reports no scroll-frame score. A reliable iOS scrolling claim needs an XCTest performance test with a signposted scroll interval or another native frame trace that works on the CI device. + +Neither the six-minute collector nor this benchmark's total execution time measures app speed. Android frame counts still include any app rendering triggered during the capture window, and the report-open span can be affected by cached data, network, and device load. Keep the account dataset and device state stable, and warm both variants the same way. Run at least two batches with opposite phase orders by passing `--benchmark-order optimized-first` to the second batch; the default is `release-first`. Report distributions and confidence intervals, and only claim a benefit for the specified device and workload when the improvement exceeds run-to-run variation. No PGO profile can guarantee every launch or interaction will be faster. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index f9ec34b76b04..868d86a0aa0e 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -112,6 +112,8 @@ scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id com Each variant gets one discarded warm-up and 20 measured fresh-process journeys. Before each sample, the runner selects Spend → Expenses and returns to Inbox; it then times a warm Inbox → Spend → Inbox tab switch. Expenses can restore the account's saved query and filters, so keep that state fixed across both phases. The CSVs under a unique `.pgo//benchmarks/journey//` directory contain `ManualNavigateToReportsFirstPaint`, `ManualNavigateToReportsContentLoad`, and `ManualNavigateToInboxTab` samples, alongside a run manifest with artifact hashes. These are app-defined tap-to-paint timings, so automation overhead is outside the measurement. The journey sends no messages and leaves account data unchanged. Both phases run in one command and are compared only within that batch. Rebuild both archived variants with this version of the tooling before benchmarking; older PGO builds only logged startup. +For a populated-chat, fast-scroll benchmark, use `benchmark-heavy-journey` with the same arguments. It records the app's report-open span on both platforms and Android's rendered/missed scroll frames in a separate CSV. iOS scroll-frame measurement is not currently reliable on the connected phone; the runner verifies that scrolling moves through real content but does not report a scroll performance number. See [JOURNEY.md](./JOURNEY.md) for the exact measured interval, setup, and limitations. Both variants must be rebuilt to include the additional report-open marker. + The stages can also run independently: ```bash diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts index 2783f0221c01..2f8324e3c78b 100644 --- a/scripts/pgo/journeyBenchmark.ts +++ b/scripts/pgo/journeyBenchmark.ts @@ -1,7 +1,11 @@ -/** Repeats a small, read-only signed-in journey and records app-defined navigation spans. */ +/** Runs separate read-only signed-in navigation and heavy-report benchmarks. */ + +// cspell:ignore gfxinfo + +import {isRecord} from '@libs/ObjectUtils'; import {createHash} from 'node:crypto'; -import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readlinkSync, writeFileSync} from 'node:fs'; +import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, writeFileSync} from 'node:fs'; import {join, relative} from 'node:path'; import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; @@ -14,16 +18,33 @@ import {capture, fail} from '../lib/scriptUtils'; import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; import {createJourneyDevice} from './journeyDevice'; -import {selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; -import {JOURNEY_SPAN_NAMES} from './shared'; +import {openReport, scrollBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; type JourneyBenchmarkReader = { begin: (spanNames: readonly string[]) => void; collect: (spanNames: readonly string[], timeoutSeconds: number) => Promise; }; -function journeyBenchmarkPaths(adapter: Pick, batchID: string): Record { - const directory = join(adapter.benchmarkDirectory, 'journey', batchID); +type FrameHealth = {totalFrames: number; droppedFrames: number; droppedPercent: number; sampleWindowMs: number}; +type JourneyMeasurement = {events: BenchmarkLogEvent[]; frameHealth?: FrameHealth}; +type BenchmarkOrder = 'release-first' | 'optimized-first'; + +type JourneyScenario = { + directory: 'journey' | 'heavy-journey'; + spanNames: readonly string[]; + measure: (device: ReturnType, reader: JourneyBenchmarkReader, fixture: JourneyFixture, timeoutSeconds: number) => Promise; +}; + +const SHORT_JOURNEY: JourneyScenario = {directory: 'journey', spanNames: JOURNEY_SPAN_NAMES, measure: measureJourney}; +const HEAVY_JOURNEY: JourneyScenario = {directory: 'heavy-journey', spanNames: HEAVY_JOURNEY_SPAN_NAMES, measure: measureHeavyJourney}; + +function journeyBenchmarkPaths( + adapter: Pick, + batchID: string, + scenarioDirectory: JourneyScenario['directory'] = 'journey', +): Record { + const directory = join(adapter.benchmarkDirectory, scenarioDirectory, batchID); return {release: join(directory, 'release.csv'), optimized: join(directory, 'pgo-optimized.csv')}; } @@ -36,27 +57,40 @@ async function benchmarkJourneyBuild( runs: number, timeoutSeconds: number, outputPath: string, + scenario: JourneyScenario, ): Promise { console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); adapter.install(kind); - const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-journey-benchmark-${adapter.name}-${kind}-${Date.now()}`}); + const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-${scenario.directory}-${adapter.name}-${kind}-${Date.now()}`}); const reader: JourneyBenchmarkReader = adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); try { device.open(false); await waitForTab(device, 'Account'); await verifyJourneyAccount(device, fixture); - console.log(`Running one unmeasured ${kind} journey warm-up.`); - await measureJourney(device, reader, timeoutSeconds); + console.log(`Running one unmeasured ${kind} ${scenario.directory} warm-up.`); + await scenario.measure(device, reader, fixture, timeoutSeconds); - const recorder = createBenchmarkRecorder({spanNames: [...JOURNEY_SPAN_NAMES], runs, outputPath}); + const recorder = createBenchmarkRecorder({spanNames: [...scenario.spanNames], runs, outputPath}); + const frameRows: string[] = []; for (let runNumber = 1; runNumber <= runs; runNumber += 1) { - const events = await measureJourney(device, reader, timeoutSeconds); - console.log(`${kind} journey ${runNumber}/${runs}: ${recorder.record(events, runNumber).join(', ')}`); + const measurement = await scenario.measure(device, reader, fixture, timeoutSeconds); + console.log(`${kind} ${scenario.directory} ${runNumber}/${runs}: ${recorder.record(measurement.events, runNumber).join(', ')}`); + if (measurement.frameHealth) { + const {totalFrames, droppedFrames, droppedPercent, sampleWindowMs} = measurement.frameHealth; + frameRows.push(`${runNumber},${totalFrames},${droppedFrames},${droppedPercent},${sampleWindowMs}`); + console.log(`Android scroll frames: ${droppedFrames}/${totalFrames} missed deadlines (${droppedPercent.toFixed(1)}%).`); + } } await verifyJourneyAccount(device, fixture); - await recorder.complete(`${kind} journey metrics`); + await recorder.complete(`${kind} ${scenario.directory} metrics`); + if (frameRows.length > 0) { + if (frameRows.length !== runs) { + fail(`Only ${frameRows.length}/${runs} Android scroll frame samples were recorded.`); + } + writeFileSync(outputPath.replace(/\.csv$/, '-frames.csv'), `run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs\n${frameRows.join('\n')}\n`); + } } finally { try { device.command('close'); @@ -67,7 +101,7 @@ async function benchmarkJourneyBuild( } /** Uses a new app process for each sample; automation latency is outside the timed spans. */ -async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, timeoutSeconds: number): Promise { +async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, _fixture: JourneyFixture, timeoutSeconds: number): Promise { device.open(true); await waitForTab(device, 'Account'); // Normalize the saved Spend view before timing either tab transition. This makes the measured taps warm navigations from a known Expenses state. @@ -88,13 +122,87 @@ async function measureJourney(device: ReturnType, re await showTab(device, 'Inbox'); device.waitLabel('All'); const inboxEvents = await reader.collect(inboxSpans, timeoutSeconds); - return [...spendEvents, ...inboxEvents]; + return {events: [...spendEvents, ...inboxEvents]}; } -async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, fixture: JourneyFixture, runs: number, timeoutSeconds: number): Promise { +/** Opens a populated chat and fast-scrolls its history; device-command time is never a benchmark sample. */ +async function measureHeavyJourney( + device: ReturnType, + reader: JourneyBenchmarkReader, + fixture: JourneyFixture, + timeoutSeconds: number, +): Promise { + device.open(true); + await waitForTab(device, 'Account'); + await showTab(device, 'Inbox'); + await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); + const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); + if (device.platform === 'android') { + // Android gfxinfo resets after each read. Exclude search and report opening from this frame window. + readAndroidFrameHealth(device, false); + } + await scrollBothWays(device, Math.min(fixture.scrolls, 6), true, 'up', true); + return {events, frameHealth: device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined}; +} + +function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealth | undefined { + const result = device.command('perf', 'frames'); + if (!isRecord(result) || !isRecord(result.metrics) || !isRecord(result.metrics.fps) || !isRecord(result.sampling) || !isRecord(result.sampling.fps)) { + throw new Error('Android frame sampler returned an unexpected result.'); + } + if (result.sampling.fps.resetsAfterRead !== true || result.metrics.fps.available !== true) { + throw new Error('Android frame sampler is unavailable or did not reset its counter.'); + } + const {totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs} = result.metrics.fps; + if ( + typeof totalFrameCount !== 'number' || + typeof droppedFrameCount !== 'number' || + typeof droppedFramePercent !== 'number' || + typeof sampleWindowMs !== 'number' || + ![totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs].every(Number.isFinite) + ) { + throw new Error('Android frame sampler did not return finite frame counts.'); + } + if (requireFrames && totalFrameCount === 0) { + throw new Error('The Android scroll produced no measured frames. Discard this run.'); + } + return {totalFrames: totalFrameCount, droppedFrames: droppedFrameCount, droppedPercent: droppedFramePercent, sampleWindowMs}; +} + +async function benchmarkJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, SHORT_JOURNEY, order); +} + +async function benchmarkHeavyJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, HEAVY_JOURNEY, order); +} + +async function benchmarkJourneyScenario( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + scenario: JourneyScenario, + order: BenchmarkOrder, +): Promise { const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); - const paths = journeyBenchmarkPaths(adapter, batchID); - const directory = join(adapter.benchmarkDirectory, 'journey', batchID); + const paths = journeyBenchmarkPaths(adapter, batchID, scenario.directory); + const directory = join(adapter.benchmarkDirectory, scenario.directory, batchID); if (existsSync(directory)) { fail(`Journey benchmark batch already exists: ${directory}`); } @@ -109,7 +217,18 @@ async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, f accountEmail: fixture.accountEmail, runs, timeoutSeconds, - spans: JOURNEY_SPAN_NAMES, + order, + scenario: scenario.directory, + spans: scenario.spanNames, + ...(scenario === HEAVY_JOURNEY + ? { + scrollsPerDirection: Math.min(fixture.scrolls, 6), + scrollDurationMs: 120, + scrollTimeMeasured: false, + androidScrollFrameHealthMeasured: true, + iosScrollFrameHealthMeasured: false, + } + : {}), sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), artifacts: adapter.artifactPaths, @@ -122,24 +241,25 @@ async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, f 2, )}\n`, ); - console.log(`Journey benchmark batch: ${directory}`); - console.log('=== Journey benchmark phase 1/2: Release ==='); - await benchmarkJourneyBuild(adapter, 'release', deviceID, fixture, runs, timeoutSeconds, paths.release); - console.log('=== Journey benchmark phase 2/2: PGO optimized ==='); - await benchmarkJourneyBuild(adapter, 'optimized', deviceID, fixture, runs, timeoutSeconds, paths.optimized); - await compareJourneyBenchmarks(paths, runs); + console.log(`${scenario.directory} benchmark batch: ${directory}`); + const phases: BenchmarkKind[] = order === 'release-first' ? ['release', 'optimized'] : ['optimized', 'release']; + for (const [index, kind] of phases.entries()) { + console.log(`=== ${scenario.directory} benchmark phase ${index + 1}/2: ${kind} ===`); + await benchmarkJourneyBuild(adapter, kind, deviceID, fixture, runs, timeoutSeconds, paths[kind], scenario); + } + await compareJourneyBenchmarks(paths, runs, scenario, adapter.name); } -async function compareJourneyBenchmarks(paths: Record, runs: number): Promise { +async function compareJourneyBenchmarks(paths: Record, runs: number, scenario: JourneyScenario, platform: PlatformAdapter['name']): Promise { if (!existsSync(paths.release) || !existsSync(paths.optimized)) { fail('Missing paired journey samples in this benchmark batch.'); } const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(paths.release), readBenchmarkSamples(paths.optimized)]); - const release = benchmarkMetrics(releaseSamples, JOURNEY_SPAN_NAMES); - const optimized = benchmarkMetrics(optimizedSamples, JOURNEY_SPAN_NAMES); + const release = benchmarkMetrics(releaseSamples, scenario.spanNames); + const optimized = benchmarkMetrics(optimizedSamples, scenario.spanNames); - console.log('=== Journey benchmark comparison (positive percentages are faster) ==='); - for (const span of JOURNEY_SPAN_NAMES) { + console.log(`=== ${scenario.directory} benchmark comparison (positive percentages are faster) ===`); + for (const span of scenario.spanNames) { const baseline = release[span]?.stats; const pgo = optimized[span]?.stats; if (!baseline || !pgo || baseline.runs !== runs || pgo.runs !== runs) { @@ -148,9 +268,38 @@ async function compareJourneyBenchmarks(paths: Record, ru const improvement = ((baseline.average - pgo.average) / baseline.average) * 100; console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); } + if (scenario === HEAVY_JOURNEY && platform === 'android') { + const releaseFrames = readAndroidFrameSamples(paths.release.replace(/\.csv$/, '-frames.csv'), runs); + const optimizedFrames = readAndroidFrameSamples(paths.optimized.replace(/\.csv$/, '-frames.csv'), runs); + console.log(`Android scroll missed-frame share: release ${releaseFrames.toFixed(2)}%, PGO ${optimizedFrames.toFixed(2)}% (lower is better; separate from report-open latency).`); + } console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); } +function readAndroidFrameSamples(path: string, runs: number): number { + if (!existsSync(path)) { + fail(`Missing Android scroll frame samples: ${path}`); + } + const rows = readFileSync(path, 'utf8').trim().split('\n'); + if (rows.shift() !== 'run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs' || rows.length !== runs) { + fail(`Expected ${runs} Android scroll frame samples in ${path}.`); + } + let totalFrames = 0; + let droppedFrames = 0; + for (const [index, row] of rows.entries()) { + const values = row.split(',').map(Number); + const runNumber = values.at(0) ?? NaN; + const frames = values.at(1) ?? NaN; + const dropped = values.at(2) ?? NaN; + if (values.length !== 5 || values.some((value) => !Number.isFinite(value)) || runNumber !== index + 1 || frames <= 0 || dropped < 0 || dropped > frames) { + fail(`Invalid Android scroll frame sample ${index + 1} in ${path}.`); + } + totalFrames += frames; + droppedFrames += dropped; + } + return (droppedFrames / totalFrames) * 100; +} + async function sha256Artifact(path: string): Promise { if (lstatSync(path).isDirectory()) { return sha256Directory(path); @@ -193,4 +342,5 @@ async function sha256Directory(directory: string): Promise { return hash.digest('hex'); } -export {benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; +export {benchmarkHeavyJourneyAll, benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; +export type {BenchmarkOrder}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 5d23ead14775..ceebb55ee402 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -109,7 +109,7 @@ async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): device.waitLabel('Priority mode, Most recent'); } -async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false): Promise { +async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false, beforePress?: () => void): Promise { await showTab(device, 'Inbox'); device.press(idSelector('searchButton')); device.fill('search-autocomplete-text-input', report.query); @@ -124,6 +124,7 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson if (!resultLabel) { throw new Error(`No search result matched ${report.resultLabelPrefix}.`); } + beforePress?.(); device.pressLabel(resultLabel); device.wait(idSelector('composer')); if (!device.hasLabel(report.title)) { @@ -284,15 +285,15 @@ function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined return first; } -async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down', fast = false): Promise { const initialNodes = device.snapshot(); - const pixels = scrollDistance(initialNodes); + const pixels = scrollDistance(initialNodes, fast); let previous = contentSignature(initialNodes); let changed = 0; for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { let stationary = 0; for (let scroll = 0; scroll < count; scroll += 1) { - device.command('scroll', direction, '--pixels', String(pixels), '--settle'); + device.command('scroll', direction, '--pixels', String(pixels), ...(fast ? ['--duration-ms', '120'] : []), '--settle'); const current = contentSignature(device.snapshot()); if (current !== previous) { changed += 1; @@ -335,13 +336,13 @@ function contentSignature(nodes: JourneyNode[]): string { } /** Keep both ends of the gesture inside the center of portrait lists, below Spend's fixed header. */ -function scrollDistance(nodes: JourneyNode[]): number { +function scrollDistance(nodes: JourneyNode[], fast = false): number { const scrollAreas = nodes.filter((node) => /ScrollView|RecyclerView|ScrollArea/i.test(node.type) && node.width > 0 && node.height > 0); const viewport = scrollAreas.toSorted((left, right) => right.width * right.height - left.width * left.height).at(0); if (!viewport) { throw new Error('Cannot determine scroll bounds on this device.'); } - return Math.round(Math.min(viewport.height * 0.25, viewport.width * 0.6)); + return Math.round(Math.min(viewport.height * (fast ? 0.65 : 0.25), viewport.width * (fast ? 1.5 : 0.6))); } /** Match stable report identity while allowing the last-message preview to change. */ @@ -365,8 +366,10 @@ export { findReportResult, findTabNode, inAppBackTapPoint, + openReport, prepareJourney, runJourneyWorkload, + scrollBothWays, scrollDistance, selectSpendSection, showTab, diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 2343adaa03fb..88af0b143175 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -10,6 +10,7 @@ import {resolve} from 'node:path'; import type {BenchmarkStats} from '../lib/benchmarkStatistics'; import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; +import type {BenchmarkOrder} from './journeyBenchmark'; import type {BenchmarkKind, PlatformAdapter} from './shared'; import {benchmarkStartups as runBenchmarkStartups} from '../lib/benchmarkAppStartup'; @@ -18,12 +19,13 @@ import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppB import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; -import {benchmarkJourneyAll} from './journeyBenchmark'; +import {benchmarkHeavyJourneyAll, benchmarkJourneyAll} from './journeyBenchmark'; import {parseJourneyFixture} from './journeyConfig'; import {STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_STARTUP_WAIT_SECONDS = 30; +const BENCHMARK_ORDERS = ['release-first', 'optimized-first'] as const; const WORKFLOW_COMMANDS = [ 'build-release', 'build-instrumented', @@ -38,6 +40,7 @@ const WORKFLOW_COMMANDS = [ 'benchmark', 'compare-benchmarks', 'benchmark-journey', + 'benchmark-heavy-journey', 'dump', 'pull', 'merge', @@ -86,6 +89,10 @@ async function main(): Promise { description: 'Approved account fixture for journey benchmarks', required: false, }, + 'benchmark-order': { + description: 'Journey phase order; alternate it across paired batches to check build-order drift', + required: false, + }, profile: { description: 'Immutable merged profile to use when building the optimized app', required: false, @@ -104,6 +111,7 @@ async function main(): Promise { cli.namedArgs.device, cli.namedArgs.fixture, cli.namedArgs.profile, + cli.namedArgs['benchmark-order'], ); } @@ -116,6 +124,7 @@ async function runWorkflow( deviceIdentifier?: string, fixturePath?: string, profilePath?: string, + benchmarkOrder?: string, ): Promise { const adapter = getAdapter(platformName, appID, deviceIdentifier); @@ -160,7 +169,12 @@ async function runWorkflow( return; case 'benchmark-journey': { const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); - await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds); + await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + } + case 'benchmark-heavy-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkHeavyJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); return; } case 'dump': @@ -177,6 +191,10 @@ async function runWorkflow( } } +function readBenchmarkOrder(value?: string): BenchmarkOrder { + return value ? parseChoice(value, BENCHMARK_ORDERS, 'Journey benchmark order') : 'release-first'; +} + function readJourneyFixture(fixturePath?: string): ReturnType { if (!fixturePath) { fail('Journey benchmarks require --fixture with the approved signed-in account.'); diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index ed48e4666d92..bc50f922d7fe 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -4,7 +4,8 @@ import type {PlatformName} from '../lib/nativeAppBenchmark'; const STARTUP_SPAN_NAME = 'ManualAppStartup'; const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; -const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES].join(',')}`; +const HEAVY_JOURNEY_SPAN_NAMES = ['/r/*'] as const; +const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES].join(',')}`; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; @@ -30,5 +31,5 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -export {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; +export {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 2d4324694d97..6ec82d67b3ff 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -4,7 +4,7 @@ import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchma import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint, waitForTab} from '@scripts/pgo/journeyWorkload'; -import {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; +import {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; @@ -23,11 +23,15 @@ describe('PGO journey safeguards', () => { // Given release and optimized artifacts share one build-time benchmark allowlist. const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); // When / Then the startup span and every short-journey span remain available. - expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES]); + expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES]); expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1')).toEqual({ release: '/pgo/ios/benchmarks/journey/batch-1/release.csv', optimized: '/pgo/ios/benchmarks/journey/batch-1/pgo-optimized.csv', }); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1', 'heavy-journey')).toEqual({ + release: '/pgo/ios/benchmarks/heavy-journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/heavy-journey/batch-1/pgo-optimized.csv', + }); }); it('hashes iOS bundle resources as well as the app executable', async () => { @@ -159,6 +163,7 @@ describe('PGO journey safeguards', () => { const distance = scrollDistance(nodes); expect(distance).toBeGreaterThan(0); expect(distance).toBeLessThanOrEqual((height ?? 0) / 2); + expect(scrollDistance(nodes, true)).toBeGreaterThan(distance); } // Then missing viewport information must fail instead of falling back to a screen-edge gesture. expect(() => scrollDistance([])).toThrow('scroll bounds'); From e83fa480eea4dc032a6ad474a200b805d46cb8b2 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Fri, 25 Sep 2026 15:20:29 +0200 Subject: [PATCH 42/47] fix(pgo): reset heavy benchmark navigation between runs --- scripts/pgo/JOURNEY.md | 2 +- scripts/pgo/journeyBenchmark.ts | 14 +++++++------ scripts/pgo/journeyWorkload.ts | 24 +++++++++++++++++++--- tests/tooling/pgoJourney.test.ts | 35 +++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index ce89443ae32e..b532bb792455 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -91,7 +91,7 @@ Build both the release and optimized artifacts with the current span allowlist f ## Heavy report benchmark -Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes up to six fast swipes toward older messages and six back toward recent messages. It fails if the list does not expose changing content. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. +Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes four fast swipes toward older messages and four back toward recent messages. It checks that content changes after each direction; there is no accessibility snapshot between individual flings. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. ```bash scripts/pgo/pgo.ts android benchmark-heavy-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts index 2f8324e3c78b..40294812c581 100644 --- a/scripts/pgo/journeyBenchmark.ts +++ b/scripts/pgo/journeyBenchmark.ts @@ -18,7 +18,7 @@ import {capture, fail} from '../lib/scriptUtils'; import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; import {createJourneyDevice} from './journeyDevice'; -import {openReport, scrollBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {openReport, scrollFastBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; type JourneyBenchmarkReader = { @@ -67,7 +67,7 @@ async function benchmarkJourneyBuild( adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); try { device.open(false); - await waitForTab(device, 'Account'); + await showTab(device, 'Inbox'); await verifyJourneyAccount(device, fixture); console.log(`Running one unmeasured ${kind} ${scenario.directory} warm-up.`); await scenario.measure(device, reader, fixture, timeoutSeconds); @@ -133,16 +133,18 @@ async function measureHeavyJourney( timeoutSeconds: number, ): Promise { device.open(true); - await waitForTab(device, 'Account'); await showTab(device, 'Inbox'); + await waitForTab(device, 'Account'); await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); if (device.platform === 'android') { // Android gfxinfo resets after each read. Exclude search and report opening from this frame window. readAndroidFrameHealth(device, false); } - await scrollBothWays(device, Math.min(fixture.scrolls, 6), true, 'up', true); - return {events, frameHealth: device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined}; + scrollFastBothWays(device, Math.min(fixture.scrolls, 4)); + const frameHealth = device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined; + await showTab(device, 'Inbox'); + return {events, frameHealth}; } function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealth | undefined { @@ -222,7 +224,7 @@ async function benchmarkJourneyScenario( spans: scenario.spanNames, ...(scenario === HEAVY_JOURNEY ? { - scrollsPerDirection: Math.min(fixture.scrolls, 6), + scrollsPerDirection: Math.min(fixture.scrolls, 4), scrollDurationMs: 120, scrollTimeMeasured: false, androidScrollFrameHealthMeasured: true, diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index ceebb55ee402..c0307cd69232 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -285,15 +285,15 @@ function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined return first; } -async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down', fast = false): Promise { +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { const initialNodes = device.snapshot(); - const pixels = scrollDistance(initialNodes, fast); + const pixels = scrollDistance(initialNodes); let previous = contentSignature(initialNodes); let changed = 0; for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { let stationary = 0; for (let scroll = 0; scroll < count; scroll += 1) { - device.command('scroll', direction, '--pixels', String(pixels), ...(fast ? ['--duration-ms', '120'] : []), '--settle'); + device.command('scroll', direction, '--pixels', String(pixels), '--settle'); const current = contentSignature(device.snapshot()); if (current !== previous) { changed += 1; @@ -312,6 +312,23 @@ async function scrollBothWays(device: JourneyDevice, count: number, requireMovem } } +/** Fixed rapid swipes with one movement check per direction, keeping accessibility polling out of each fling. */ +function scrollFastBothWays(device: Pick, count: number): void { + const initialNodes = device.snapshot(); + const pixels = String(scrollDistance(initialNodes, true)); + let previous = contentSignature(initialNodes); + for (const direction of ['up', 'down']) { + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', pixels, '--duration-ms', '120', ...(scroll === count - 1 ? ['--settle'] : [])); + } + const current = contentSignature(device.snapshot()); + if (current === previous) { + throw new Error(`The populated report did not move during fast ${direction} scrolling. Discard this benchmark run.`); + } + previous = current; + } +} + async function waitForMessage(device: JourneyDevice, token: string): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { @@ -370,6 +387,7 @@ export { prepareJourney, runJourneyWorkload, scrollBothWays, + scrollFastBothWays, scrollDistance, selectSpendSection, showTab, diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 6ec82d67b3ff..80be75cb3a83 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -3,7 +3,17 @@ import {describe, expect, it} from 'bun:test'; import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint, waitForTab} from '@scripts/pgo/journeyWorkload'; +import { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + scrollDistance, + scrollFastBothWays, + spendSectionTapPoint, + waitForTab, +} from '@scripts/pgo/journeyWorkload'; import {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; @@ -169,6 +179,29 @@ describe('PGO journey safeguards', () => { expect(() => scrollDistance([])).toThrow('scroll bounds'); }); + it('uses fixed rapid swipes and validates movement in both directions', () => { + // Given a report with distinct content at the recent and older scroll positions. + const snapshots = ['recent', 'older', 'recent'].map((label) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width: 390, height: 844}}, {label}]}, 'test.app'), + ); + const commands: string[][] = []; + let read = 0; + // When the heavy benchmark scrolls in both directions. + scrollFastBothWays( + { + snapshot: () => snapshots.at(read++) ?? [], + command: (...args) => commands.push(args), + }, + 4, + ); + // Then it sends fixed-duration gestures and checks content after each direction. + expect(commands).toHaveLength(8); + expect(commands.every((args) => args.includes('120'))).toBe(true); + expect(commands.filter((args) => args.includes('--settle'))).toHaveLength(2); + expect(commands.at(0)?.at(1)).toBe('up'); + expect(commands.at(-1)?.at(1)).toBe('down'); + }); + it('selects iOS status-bearing bottom tabs without confusing them with page headings', () => { const nodes = parseJourneySnapshot( { From d0cfd6aa2911bd752aa917354668af89748f2b56 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Fri, 25 Sep 2026 16:56:18 +0200 Subject: [PATCH 43/47] fix(pgo): wait for navigation after Android relaunch --- scripts/pgo/journeyWorkload.ts | 41 +++++++++++++++++++------------- tests/tooling/pgoJourney.test.ts | 35 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index c0307cd69232..c126050d20d2 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -133,15 +133,21 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson } async function showTab(device: JourneyDevice, tab: string): Promise { - // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. - for (let attempt = 0; attempt < 5; attempt += 1) { - const tabNode = findTabNode(device.snapshot(), tab); + const deadline = Date.now() + 30_000; + let backPresses = 0; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const tabNode = findTabNode(nodes, tab); if (tabNode) { device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } - pressInAppBack(device); - await sleep(300); + // A relaunch can expose a loading view before tabs mount. Only navigate back from a real nested page. + if (backPresses < 5 && inAppBackTapPoint(nodes)) { + pressInAppBack(device, nodes); + backPresses += 1; + } + await sleep(500); } throw new Error(`Cannot return to the ${tab} tab.`); } @@ -183,26 +189,29 @@ function pressPreferences(device: JourneyDevice): void { device.pressLabel('Preferences'); } -function pressInAppBack(device: JourneyDevice): void { - if (device.platform === 'ios') { - const tapPoint = inAppBackTapPoint(device.snapshot()); - if (tapPoint) { - device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); - return; - } +function pressInAppBack(device: JourneyDevice, nodes = device.snapshot()): void { + const tapPoint = inAppBackTapPoint(nodes); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; } device.back(); } -/** The iOS search Back button remains visible but is reported non-hittable while the keyboard is open. */ +/** Use the app's top-left Back control; it can be non-hittable on iOS while the keyboard is open. */ function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { - const screen = nodes.find((node) => node.type === 'Application'); + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); if (!screen) { return undefined; } const candidates = nodes.filter( (node) => - node.type === 'Button' && + (node.type === 'Button' || node.type === 'android.widget.Button') && normalizeLabel(node.label) === 'Back' && node.enabled && node.width > 0 && @@ -213,7 +222,7 @@ function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undef node.y < screen.height * 0.2, ); if (candidates.length > 1) { - throw new Error('Ambiguous in-app Back control on iOS.'); + throw new Error('Ambiguous in-app Back control.'); } const back = candidates.at(0); return back ? {x: Math.round(back.x + back.width / 2), y: Math.round(back.y + back.height / 2)} : undefined; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 80be75cb3a83..0c3c6044c56b 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -11,6 +11,7 @@ import { inAppBackTapPoint, scrollDistance, scrollFastBothWays, + showTab, spendSectionTapPoint, waitForTab, } from '@scripts/pgo/journeyWorkload'; @@ -235,6 +236,40 @@ describe('PGO journey safeguards', () => { await expect(waitForTab(device, 'Account')).resolves.toBeUndefined(); }); + it('waits through Android relaunch loading before navigating from a restored page', async () => { + // Given a loading view, then a restored nested page, then the Inbox tab after app Back. + const screen = {type: 'android.widget.ScrollView', rect: {x: 0, y: 0, width: 1080, height: 2280}}; + const snapshots = [ + [screen], + [screen, {type: 'android.widget.Button', label: 'Back', rect: {x: 21, y: 144, width: 67, height: 105}}], + [screen, {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}], + ].map((nodes) => parseJourneySnapshot({appBundleId: 'test.app', nodes}, 'test.app')); + const commands: string[][] = []; + let read = 0; + const device: Parameters[0] = { + platform: 'android', + snapshot: () => snapshots[Math.min(read++, snapshots.length - 1)], + command: (...args: string[]) => commands.push(args), + wait: () => {}, + press: () => {}, + hasLabel: () => false, + pressLabel: () => {}, + waitLabel: () => {}, + fill: () => {}, + back: () => commands.push(['hardware back']), + open: () => {}, + }; + + // When the benchmark navigates to Inbox after relaunch. + await showTab(device, 'Inbox'); + + // Then loading does not trigger Back, and the app's Back control is used once. + expect(commands).toEqual([ + ['press', '55', '197', '--settle'], + ['press', '324', '2061', '--settle'], + ]); + }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; const snapshot = (tabs: Array) => From 69a8d678893868fbe5c51eae78ec01b88bdbe300 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 15:27:18 +0100 Subject: [PATCH 44/47] fix(pgo): align iOS IR profiling and Rock builds --- Mobile-Expensify | 2 +- ....86.0+044+add-local-pgo-build-switch.patch | 6 +- scripts/pgo/ios.ts | 57 +++++++++++-------- tests/tooling/pgoIOSRock.test.ts | 17 ++++++ 4 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 tests/tooling/pgoIOSRock.test.ts diff --git a/Mobile-Expensify b/Mobile-Expensify index 8b0d0a2977b0..7f8d598e6da7 160000 --- a/Mobile-Expensify +++ b/Mobile-Expensify @@ -1 +1 @@ -Subproject commit 8b0d0a2977b0b9d18eefacb9b6449dd5cfd57eca +Subproject commit 7f8d598e6da7931f3c66b8bcdb394f429d76a259 diff --git a/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch index dbff79c05e16..d7e9802f6fd1 100644 --- a/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch +++ b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch @@ -149,14 +149,14 @@ diff --git a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xco +pgo_compile_flags="" +pgo_linker_flags="" +if [[ "${EXPENSIFY_PGO_MODE:-off}" == "generate" ]]; then -+ pgo_compile_flags="-fprofile-instr-generate" -+ pgo_linker_flags="-fprofile-instr-generate;LINKER:-alias,___llvm_profile_set_filename,_expensify_llvm_profile_set_filename;LINKER:-alias,___llvm_profile_write_file,_expensify_llvm_profile_write_file;LINKER:-alias,___llvm_profile_reset_counters,_expensify_llvm_profile_reset_counters" ++ pgo_compile_flags="-fprofile-generate" ++ pgo_linker_flags="-fprofile-generate;LINKER:-alias,___llvm_profile_set_filename,_expensify_llvm_profile_set_filename;LINKER:-alias,___llvm_profile_write_file,_expensify_llvm_profile_write_file;LINKER:-alias,___llvm_profile_reset_counters,_expensify_llvm_profile_reset_counters" +elif [[ "${EXPENSIFY_PGO_MODE:-off}" == "use" ]]; then + if [[ -z "${EXPENSIFY_PGO_PROFILE:-}" ]]; then + echo "EXPENSIFY_PGO_PROFILE is required when EXPENSIFY_PGO_MODE=use" >&2 + exit 1 + fi -+ pgo_compile_flags="-fprofile-instr-use=${EXPENSIFY_PGO_PROFILE}" ++ pgo_compile_flags="-fprofile-use=${EXPENSIFY_PGO_PROFILE}" +fi + boost_context_flag="" diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index bc2ba74c7a1c..77ea09670c40 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled +// cspell:ignore Xfrontend hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ @@ -14,7 +14,7 @@ import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllo import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { - const profileFormat = 'ios-clang-frontend-swift-ir-v1'; + const profileFormat = 'ios-clang-swift-ir-v3'; const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); const rawProfileDirectory = join(profileDirectory, 'raw'); const appDirectory = join(profileDirectory, 'app'); @@ -181,19 +181,19 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str return [ ...baseSettings, 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', - 'OTHER_CFLAGS=$(inherited) -fprofile-instr-generate', - 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-generate', - 'OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-generate', - 'OTHER_LDFLAGS=$(inherited) -fprofile-instr-generate', + 'OTHER_CFLAGS=$(inherited) -fprofile-generate', + 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-generate', + 'OTHER_LDFLAGS=$(inherited) -fprofile-generate', ]; } if (kind === 'optimized') { - const clangProfileFlags = `-fprofile-instr-use=${profilePath} -Wno-error=profile-instr-unprofiled`; + const clangProfileFlags = `-fprofile-use=${profilePath} -Wno-error=profile-instr-unprofiled`; return [ ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, - `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${profilePath}`, + `OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-use=${profilePath}`, ]; } return baseSettings; @@ -208,7 +208,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str const profileFormatPath = `${profilePath}.format`; const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; if (recordedFormat !== profileFormat) { - fail('The merged iOS profile predates the current Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); + fail('The merged iOS profile does not match the current Clang and Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); } } @@ -230,28 +230,24 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), ] : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; - const xcodeArguments = [ - 'xcodebuild', - '-workspace', - 'Expensify.xcworkspace', - '-scheme', - 'Expensify', - '-configuration', - 'Release', - '-destination', - 'generic/platform=iOS', - '-derivedDataPath', - derivedDataDirectory, - ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), - '-quiet', + const rockArguments = iosRockBuildArguments(derivedDataDirectory, [ 'build', 'ARCHS=arm64', 'ONLY_ACTIVE_ARCH=YES', 'COMPILER_INDEX_STORE_ENABLE=NO', + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), ...signingSettings, ...pgoBuildSettings(kind, profilePath), - ]; - withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), ...xcodeArguments], iosDirectory)); + ]); + withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], join(rootDirectory, 'Mobile-Expensify'))); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -472,4 +468,15 @@ function replaceOnce(content: string, previous: string, next: string): string { return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; } +/** Preserve each Xcode setting as one argument through Rock's string argument parser. */ +function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { + return [ + 'build:ios', '--local', '--verbose', '--configuration', 'Release', + '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', + '--build-folder', derivedDataDirectory, '--no-install-pods', + '--extra-params', settings.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + +export {iosRockBuildArguments}; export default createIOSPgoAdapter; diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts new file mode 100644 index 000000000000..3eeaaec3a56a --- /dev/null +++ b/tests/tooling/pgoIOSRock.test.ts @@ -0,0 +1,17 @@ +// cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata + +import {describe, expect, it} from 'bun:test'; +import {iosRockBuildArguments} from '@scripts/pgo/ios'; +import {parseArgs} from '@rock-js/tools'; + +describe('PGO iOS Rock build arguments', () => { + it('selects a local device Release build and preserves exact compiler settings', () => { + // Given compiler settings containing spaces and inherited Xcode variables. + const settings = ['build', 'ARCHS=arm64', 'CODE_SIGN_IDENTITY=Apple Development', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-use=/tmp/profile directory/journey.profdata']; + // When the adapter creates a Rock invocation. + const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); + // Then the local Release destination and existing pod preparation are explicit. + expect(args.slice(0, 12)).toEqual(['build:ios', '--local', '--verbose', '--configuration', 'Release', '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', '--build-folder', '/tmp/derived data/optimized', '--no-install-pods']); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); +}); From e9dec6c3f6f39caa250f7f2179cf002464441326 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 18:49:58 +0100 Subject: [PATCH 45/47] fix(pgo): resolve foundation validation failures --- knip.json | 2 +- package-lock.json | 1 + package.json | 1 + scripts/pgo/JOURNEY.md | 2 +- scripts/pgo/ios.ts | 42 ++++++++++++++++++++++---------- tests/tooling/pgoIOSRock.test.ts | 17 ++++++++++++- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/knip.json b/knip.json index 8fba9bd02c97..793f2d7d1612 100644 --- a/knip.json +++ b/knip.json @@ -89,5 +89,5 @@ "diff-so-fancy", "@babel/plugin-proposal-class-properties" ], - "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl"] + "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl", "agent-device"] } diff --git a/package-lock.json b/package-lock.json index 39800039ca3c..4f918febbe79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -192,6 +192,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/package.json b/package.json index b5f2bed6fd9c..1e2873c0c1a9 100644 --- a/package.json +++ b/package.json @@ -269,6 +269,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index b532bb792455..c29d117fc601 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -8,7 +8,7 @@ Use a dedicated, owner-approved heavy account with a populated Inbox, a long chat history, and enough expenses and expense reports to scroll. Keep the account and its data between runs. Installing over the existing app preserves authentication; uninstalling or clearing app data does not. -The runner never enters credentials, retrieves a magic code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. +The runner never enters credentials, retrieves a security code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. Before every run, it opens Inbox to allow the app's automatic large-account preference update, then opens Account → Preferences and ensures Priority mode is **Most recent**. If it is **#focus**, the runner changes it and verifies the result. It checks the preference again at the end and rejects the run if #focus became enabled. diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 77ea09670c40..204855eaca7c 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -239,15 +239,21 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ...signingSettings, ...pgoBuildSettings(kind, profilePath), ]); - withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [ - ...sourceBuildEnvironment(mode, profilePath), - 'IS_HYBRID_APP=true', - 'BUNDLER=metro', - `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, - 'node', - join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), - ...rockArguments, - ], join(rootDirectory, 'Mobile-Expensify'))); + withLocalUploadPhasesDisabled(() => + run( + '/usr/bin/env', + [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], + join(rootDirectory, 'Mobile-Expensify'), + ), + ); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -471,10 +477,20 @@ function replaceOnce(content: string, previous: string, next: string): string { /** Preserve each Xcode setting as one argument through Rock's string argument parser. */ function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { return [ - 'build:ios', '--local', '--verbose', '--configuration', 'Release', - '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', - '--build-folder', derivedDataDirectory, '--no-install-pods', - '--extra-params', settings.map((argument) => JSON.stringify(argument)).join(' '), + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + derivedDataDirectory, + '--no-install-pods', + '--extra-params', + settings.map((argument) => JSON.stringify(argument)).join(' '), ]; } diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts index 3eeaaec3a56a..9809f0dc38ed 100644 --- a/tests/tooling/pgoIOSRock.test.ts +++ b/tests/tooling/pgoIOSRock.test.ts @@ -1,7 +1,9 @@ // cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata import {describe, expect, it} from 'bun:test'; + import {iosRockBuildArguments} from '@scripts/pgo/ios'; + import {parseArgs} from '@rock-js/tools'; describe('PGO iOS Rock build arguments', () => { @@ -11,7 +13,20 @@ describe('PGO iOS Rock build arguments', () => { // When the adapter creates a Rock invocation. const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); // Then the local Release destination and existing pod preparation are explicit. - expect(args.slice(0, 12)).toEqual(['build:ios', '--local', '--verbose', '--configuration', 'Release', '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', '--build-folder', '/tmp/derived data/optimized', '--no-install-pods']); + expect(args.slice(0, 12)).toEqual([ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + '/tmp/derived data/optimized', + '--no-install-pods', + ]); expect(parseArgs(args.at(13) ?? '')).toEqual(settings); }); }); From 16db8945c87888a4f6ead6deea8ba6d2ca8fa0d7 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 13 Oct 2026 14:33:39 +0100 Subject: [PATCH 46/47] fix(pgo): reject partial Android profiles and resolve Linux NDK paths --- scripts/pgo/android.ts | 36 ++++++--- tests/tooling/pgoAndroid.test.ts | 126 +++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 tests/tooling/pgoAndroid.test.ts diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index e0ef640aa0a8..d802c2d3f22b 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -14,9 +14,9 @@ import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; +const INSTRUMENTED_LIBRARIES = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { - const ndkVersion = '27.1.12297006'; const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); const rawProfileDirectory = join(profileDirectory, 'raw'); const apkDirectory = join(profileDirectory, 'apk'); @@ -56,12 +56,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent } function llvmTool(name: string): string { - const androidNdkHome: unknown = process.env.ANDROID_NDK_HOME; - const androidNdkRoot: unknown = process.env.ANDROID_NDK_ROOT; - const configuredNdkRoot = [androidNdkHome, androidNdkRoot].find((value): value is string => typeof value === 'string'); - const ndkRoot = configuredNdkRoot ?? join(homedir(), `Library/Android/sdk/ndk/${ndkVersion}`); - const hostTag = platform() === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; - const toolPath = join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); + const toolPath = androidLlvmToolPath(name); if (!existsSync(toolPath)) { fail(`Missing ${name} at ${toolPath}. Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build.`); } @@ -113,8 +108,13 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const deadline = Date.now() + profileDumpTimeoutSeconds * 1000; while (Date.now() < deadline) { const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); - const profileCount = /Wrote ([1-9][0-9]*) LLVM PGO profile/.exec(profileLogs)?.[1]; - if (profileCount) { + const profileCount = /Wrote ([0-9]+) LLVM PGO profile/.exec(profileLogs)?.[1]; + if (profileCount !== undefined && Number(profileCount) !== 0) { + if (Number(profileCount) !== INSTRUMENTED_LIBRARIES.length) { + fail( + `The PGO receiver wrote ${profileCount} profiles, but expected ${INSTRUMENTED_LIBRARIES.length}. All instrumented native libraries must write successfully before recording profiles.`, + ); + } console.log(`Wrote ${profileCount} LLVM PGO profile(s).`); return; } @@ -157,9 +157,8 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const llvmReadelf = llvmTool('llvm-readelf'); const llvmNm = llvmTool('llvm-nm'); const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); - const libraries = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; try { - for (const library of libraries) { + for (const library of INSTRUMENTED_LIBRARIES) { const apkEntry = `lib/arm64-v8a/${library}`; if (!apkEntries.has(apkEntry)) { fail(`Missing expected arm64 library in APK: ${apkEntry}`); @@ -206,6 +205,19 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent }; } +/** Resolve LLVM tools from the build's explicit NDK or host-specific SDK installation. */ +function androidLlvmToolPath( + name: string, + {hostPlatform = platform(), homeDirectory = homedir(), environment = process.env}: {hostPlatform?: NodeJS.Platform; homeDirectory?: string; environment?: NodeJS.ProcessEnv} = {}, +): string { + const configuredNdkRoot = [environment.ANDROID_NDK_HOME, environment.ANDROID_NDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const configuredSdkRoot = [environment.ANDROID_HOME, environment.ANDROID_SDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const sdkRoot = configuredSdkRoot ?? join(homeDirectory, hostPlatform === 'linux' ? 'Android/Sdk' : 'Library/Android/sdk'); + const ndkRoot = configuredNdkRoot ?? join(sdkRoot, 'ndk/27.1.12297006'); + const hostTag = hostPlatform === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; + return join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); +} + function androidApplicationID(buildGradle: string, configuredAppID?: string): string { if (configuredAppID) { return configuredAppID; @@ -222,4 +234,4 @@ function androidProfileReceiverComponent(appID: string): string { } export default createAndroidPgoAdapter; -export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent}; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidLlvmToolPath, androidProfileReceiverComponent}; diff --git a/tests/tooling/pgoAndroid.test.ts b/tests/tooling/pgoAndroid.test.ts new file mode 100644 index 000000000000..da67e2ab1f01 --- /dev/null +++ b/tests/tooling/pgoAndroid.test.ts @@ -0,0 +1,126 @@ +// cspell:ignore profdata + +import {beforeEach, describe, expect, it, mock} from 'bun:test'; + +import * as scriptUtils from '@scripts/lib/scriptUtils'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; +import process from 'node:process'; + +let profileLogs = ''; +const runCommand = mock(() => undefined); +const captureCommand = mock(() => profileLogs); +// Stub the device boundary while keeping the adapter's broadcast, polling and validation real. +mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, run: runCommand, capture: captureCommand})); +const {default: createAndroidPgoAdapter, androidLlvmToolPath} = await import('@scripts/pgo/android'); + +describe('PGO Android profile dumps', () => { + beforeEach(() => { + profileLogs = ''; + runCommand.mockClear(); + captureCommand.mockClear(); + }); + + it('accepts a dump only when all four native libraries wrote profiles', async () => { + // Given a complete writer response from the installed instrumented APK. + profileLogs = 'I/PgoProfileReceiver: Wrote 4 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then profile collection can continue. + await expect(dump).resolves.toBeUndefined(); + expect(runCommand.mock.calls).toEqual([ + ['adb', ['logcat', '-c']], + ['adb', ['shell', 'am', 'broadcast', '-a', 'com.expensify.chat.action.WRITE_PGO_PROFILES', '-n', 'com.example.pgo/org.me.mobiexpensifyg.PgoProfileReceiver']], + ]); + }); + + it.each([1, 2, 3, 5])('rejects a dump reporting %i profiles', async (count) => { + // Given a partial or unexpected writer response despite successful broadcast delivery. + profileLogs = `I/PgoProfileReceiver: Wrote ${count} LLVM PGO profile(s).`; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then training cannot silently accept an incomplete set of native counters. + await expect(dump).rejects.toThrow(`wrote ${count} profiles, but expected 4`); + }); + + it('keeps the diagnostic for a dump with no instrumented libraries', async () => { + // Given a receiver that could not find any profile writers. + profileLogs = 'I/PgoProfileReceiver: Wrote 0 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it explains that the instrumented APK must be rebuilt. + await expect(dump).rejects.toThrow('found no instrumented native libraries'); + }); + + it('rejects a non-instrumented APK', async () => { + // Given a receiver explicitly rejecting profiling in a normal release build. + profileLogs = 'Ignoring PGO profile request in a non-instrumented build.'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it directs the caller to install the instrumented build. + await expect(dump).rejects.toThrow('installed APK is not instrumented'); + }); +}); + +describe('PGO Android LLVM tools', () => { + it.each([ + ['linux', '/home/test/Android/Sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'], + ['darwin', '/home/test/Library/Android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-profdata'], + ] satisfies Array<[NodeJS.Platform, string]>)('uses the default SDK installation on %s', (hostPlatform, expected) => { + // Given an unconfigured host with its normal Android Studio SDK location. + // When LLVM tools are resolved without explicit environment settings. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform, homeDirectory: '/home/test', environment: {}}); + // Then the selected path uses that host's SDK directory and toolchain. + expect(path).toBe(expected); + }); + + it.each(['ANDROID_HOME', 'ANDROID_SDK_ROOT'])('honors %s on Linux', (variable) => { + // Given a Linux runner with its SDK installed outside the default home directory. + // When the SDK environment setting is used to resolve LLVM. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/android/sdk'}}); + // Then the pinned NDK is resolved within the configured SDK. + expect(path).toBe('/opt/android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it.each(['ANDROID_NDK_HOME', 'ANDROID_NDK_ROOT'])('prefers %s over SDK settings', (variable) => { + // Given an explicit NDK override and a different SDK location. + // When the tool path is resolved. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/custom-ndk', ANDROID_HOME: '/opt/android/sdk'}}); + // Then an explicitly selected build toolchain wins over the pinned SDK fallback. + expect(path).toBe('/opt/custom-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('prefers ANDROID_HOME over the deprecated SDK root setting and ignores empty NDK overrides', () => { + // Given both SDK settings and empty NDK settings. + const environment = {ANDROID_HOME: '/opt/current-sdk', ANDROID_SDK_ROOT: '/opt/old-sdk', ANDROID_NDK_HOME: '', ANDROID_NDK_ROOT: ''}; + // When LLVM tools are resolved on Linux. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment}); + // Then empty settings do not mask the SDK selected by ANDROID_HOME. + expect(path).toBe('/opt/current-sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('uses the resolved tool path in the adapter', () => { + // Given an explicit temporary NDK with a real tool file. + const directory = mkdtempSync(join(tmpdir(), 'pgo-ndk-test-')); + const previousNdk = process.env.ANDROID_NDK_HOME; + process.env.ANDROID_NDK_HOME = directory; + try { + const expected = androidLlvmToolPath('llvm-profdata'); + mkdirSync(dirname(expected), {recursive: true}); + writeFileSync(expected, ''); + // When the adapter checks tool availability. + const path = createAndroidPgoAdapter().llvmTool('llvm-profdata'); + // Then it returns the existing file in the selected NDK. + expect(path).toBe(expected); + } finally { + if (previousNdk === undefined) { + delete process.env.ANDROID_NDK_HOME; + } else { + process.env.ANDROID_NDK_HOME = previousNdk; + } + rmSync(directory, {recursive: true, force: true}); + } + }); +}); From 6aab15b779106cad60a2c817eeb97d002aaff8e9 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 13 Oct 2026 15:53:09 +0100 Subject: [PATCH 47/47] fix(pgo): skip Gradle lookup for explicit Android app IDs --- scripts/pgo/android.ts | 5 ++++- tests/tooling/pgoAndroid.test.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index d802c2d3f22b..c296c680791d 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -39,7 +39,10 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent let cachedAppID: string | undefined; function appID(): string { - cachedAppID ??= androidApplicationID(readFileSync(buildGradlePath, 'utf8'), configuredAppID); + if (configuredAppID) { + return configuredAppID; + } + cachedAppID ??= androidApplicationID(readFileSync(buildGradlePath, 'utf8')); return cachedAppID; } diff --git a/tests/tooling/pgoAndroid.test.ts b/tests/tooling/pgoAndroid.test.ts index da67e2ab1f01..ab82b96c3f1b 100644 --- a/tests/tooling/pgoAndroid.test.ts +++ b/tests/tooling/pgoAndroid.test.ts @@ -1,6 +1,6 @@ // cspell:ignore profdata -import {beforeEach, describe, expect, it, mock} from 'bun:test'; +import {afterAll, beforeEach, describe, expect, it, mock} from 'bun:test'; import * as scriptUtils from '@scripts/lib/scriptUtils'; @@ -10,12 +10,16 @@ import {dirname, join} from 'node:path'; import process from 'node:process'; let profileLogs = ''; +// Reproduce the CI checkout, which does not initialize the native submodule. +const rootDirectory = mkdtempSync(join(tmpdir(), 'pgo-android-test-')); const runCommand = mock(() => undefined); const captureCommand = mock(() => profileLogs); // Stub the device boundary while keeping the adapter's broadcast, polling and validation real. -mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, run: runCommand, capture: captureCommand})); +mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, rootDirectory, run: runCommand, capture: captureCommand})); const {default: createAndroidPgoAdapter, androidLlvmToolPath} = await import('@scripts/pgo/android'); +afterAll(() => rmSync(rootDirectory, {recursive: true, force: true})); + describe('PGO Android profile dumps', () => { beforeEach(() => { profileLogs = '';