From 3fa458a4a3fdad61d9aa8ab6c9faf85627b839e4 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 11:30:39 +0200 Subject: [PATCH 01/60] build: create script for PGO builds --- scripts/pgo/README.md | 40 ++++++++++++++ scripts/pgo/android-local-proof.sh | 83 ++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 scripts/pgo/README.md create mode 100755 scripts/pgo/android-local-proof.sh diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md new file mode 100644 index 000000000000..96bf6281f7b9 --- /dev/null +++ b/scripts/pgo/README.md @@ -0,0 +1,40 @@ +# Local Android PGO proof + +This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the project's `release` variant: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. + +## Preconditions + +- A physical arm64 Android device connected through `adb`. +- `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. +- A signed-in, seeded test account and the agreed manual NewDot journey. +- Dependencies have had the repository's patches applied. + +## Collect one profile + +1. Build and install the instrumented application: + + ```bash + scripts/pgo/android-local-proof.sh build-instrumented + adb install -r Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release.apk + ``` + +2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. +3. Persist the counters before force-stopping the application: + + ```bash + scripts/pgo/android-local-proof.sh dump + scripts/pgo/android-local-proof.sh pull + scripts/pgo/android-local-proof.sh merge + ``` + +4. Build the matched optimized application: + + ```bash + scripts/pgo/android-local-proof.sh build-optimized + ``` + +The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. + +## Compare + +Install the non-PGO and PGO APKs separately. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh new file mode 100755 index 000000000000..cf28b86b0138 --- /dev/null +++ b/scripts/pgo/android-local-proof.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euo pipefail + +readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" +readonly PACKAGE_NAME="org.me.mobiexpensifyg" +readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly BUILD_VARIANT="Release" + +function usage() { + echo "Usage: $0 {build-instrumented|dump|pull|merge|build-optimized}" +} + +function ndk_tool() { + local tool_name="$1" + local ndk_root="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" + if [[ -z "$ndk_root" ]]; then + echo "Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build." >&2 + exit 1 + fi + + local host_tag="darwin-x86_64" + if [[ "$(uname -s)" == "Linux" ]]; then + host_tag="linux-x86_64" + fi + echo "$ndk_root/toolchains/llvm/prebuilt/$host_tag/bin/$tool_name" +} + +function gradle() { + ( + cd "$ANDROID_DIR" + ./gradlew "$@" + ) +} + +case "${1:-}" in + build-instrumented) + gradle ":assemble$BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=generate + ;; + dump) + adb shell am broadcast \ + -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$PACKAGE_NAME/.PgoProfileReceiver" + ;; + pull) + rm -rf "$PROFILE_DIR/raw" + mkdir -p "$PROFILE_DIR/raw" + adb exec-out run-as "$PACKAGE_NAME" tar -C cache -cf - . | tar -C "$PROFILE_DIR/raw" -xf - + find "$PROFILE_DIR/raw" -name '*.profraw' -print + ;; + merge) + mkdir -p "$PROFILE_DIR" + profiles=() + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found. Run dump and pull first." >&2 + exit 1 + fi + "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" + "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" + ;; + build-optimized) + if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then + echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 + exit 1 + fi + gradle ":app:assemble$BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=use \ + -PpgoProfile="$PROFILE_DIR/newdot.profdata" + ;; + *) + usage + exit 1 + ;; +esac From 3295d3865472f07a4650f028202682e87f2ac8d6 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 11:30:55 +0200 Subject: [PATCH 02/60] build: configure PGO build --- android/app/build.gradle | 11 ++ android/app/src/main/AndroidManifest.xml | 8 ++ .../expensify/chat/PgoProfileReceiver.java | 27 +++++ .../com/expensify/chat/PgoProfileWriter.java | 19 +++ .../android/CMakeLists.txt | 25 +++- .../ExpensifyNitroUtils/android/build.gradle | 19 ++- .../android/src/main/cpp/PgoProfileWriter.cpp | 53 +++++++++ patches/react-native/details.md | 7 ++ ....85.3+040+add-local-pgo-build-switch.patch | 112 ++++++++++++++++++ 9 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 android/app/src/main/java/com/expensify/chat/PgoProfileReceiver.java create mode 100644 android/app/src/main/java/com/expensify/chat/PgoProfileWriter.java create mode 100644 modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp create mode 100644 patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch diff --git a/android/app/build.gradle b/android/app/build.gradle index 319f661b7eab..2eb448716d3b 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -81,6 +81,16 @@ project.ext.envConfigFiles = [ * Set this to true to Run Proguard on Release builds to minify the Java bytecode. */ def enableProguardInReleaseBuilds = true +def pgoMode = providers.gradleProperty('pgoMode').getOrElse('off') +def pgoProfile = providers.gradleProperty('pgoProfile').orNull + +if (!(pgoMode in ['off', 'generate', 'use'])) { + throw new GradleException("Unsupported pgoMode '${pgoMode}'. Use off, generate, or use.") +} + +if (pgoMode == 'use' && !pgoProfile) { + throw new GradleException('pgoProfile is required when pgoMode=use.') +} /** * The preferred build flavor of JavaScriptCore (JSC) @@ -116,6 +126,7 @@ android { // Supported language variants must be declared here to avoid from being removed during the compilation. // This also helps us to not include unnecessary language variants in the APK. resConfigs "en", "es" + buildConfigField 'boolean', 'PGO_PROFILE_GENERATION', (pgoMode == 'generate').toString() } flavorDimensions "default" diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 2cf668b50b25..d2bbf6ad3914 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -27,6 +27,14 @@ android:theme="@style/AppTheme" tools:replace="android:supportsRtl"> + + + + + + +#include + +#include +#include + +namespace { +using SetFilename = int (*)(const char *); +using WriteFile = int (*)(); + +constexpr std::array kInstrumentedLibraries = { + "libhermesvm.so", + "libreactnative.so", + "libjsi.so", + "libExpensifyNitroUtils.so", +}; + +bool writeProfile(void *handle, const std::string &profilePath) { + const auto setFilename = reinterpret_cast(dlsym(handle, "__llvm_profile_set_filename")); + const auto writeFile = reinterpret_cast(dlsym(handle, "__llvm_profile_write_file")); + if (setFilename == nullptr || writeFile == nullptr) { + return false; + } + + return setFilename(profilePath.c_str()) == 0 && writeFile() == 0; +} +} // namespace + +extern "C" JNIEXPORT jint JNICALL +Java_com_expensify_chat_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { + const char *directoryChars = env->GetStringUTFChars(directory, nullptr); + if (directoryChars == nullptr) { + return 0; + } + + const std::string profilePath = std::string(directoryChars) + "/newdot-%m.profraw"; + env->ReleaseStringUTFChars(directory, directoryChars); + + jint writtenProfiles = 0; + for (const char *library : kInstrumentedLibraries) { + void *handle = dlopen(library, RTLD_NOW | RTLD_NOLOAD); + if (handle == nullptr) { + continue; + } + + if (writeProfile(handle, profilePath)) { + ++writtenProfiles; + } + dlclose(handle); + } + + return writtenProfiles; +} diff --git a/patches/react-native/details.md b/patches/react-native/details.md index f59d0a5ec3d0..0fcb8da72a13 100644 --- a/patches/react-native/details.md +++ b/patches/react-native/details.md @@ -1,5 +1,12 @@ # `react-native` patches +### [react-native+0.85.3+040+add-local-pgo-build-switch.patch](react-native+0.85.3+040+add-local-pgo-build-switch.patch) + +- Reason: Adds an opt-in local LLVM PGO mode to the React Native Android runtime and Hermes source builds. The mode is disabled by default and is used only by the local PGO proof tooling. +- Upstream PR/issue: 🛑 +- E/App issue: 🛑 +- PR Introducing Patch: 🛑 + ### [react-native+0.85.3+001+initial.patch](react-native+0.85.3+001+initial.patch) - Reason: Fixes keyboard flickering issue when opening/closing modals. When an input is blurred and a modal is opened, the `rootView` becomes the `firstResponder`, causing the system to retain an incorrect keyboard state. This leads to keyboard flickering when the modal is closed. The patch adds code to resign the `rootView`'s `firstResponder` status before presenting the modal to prevent this issue. diff --git a/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch new file mode 100644 index 000000000000..57b7b6fe96df --- /dev/null +++ b/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch @@ -0,0 +1,112 @@ +diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modules/react-native/ReactAndroid/build.gradle.kts +--- a/node_modules/react-native/ReactAndroid/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/build.gradle.kts +@@ -40,6 +40,15 @@ val reactNativeRootDir = projectDir.parent + val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} + + // We put the publishing version from gradle.properties inside ext. so other + // subprojects can access it as well. +@@ -579,6 +588,10 @@ android { + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW", ++ "-DPGO_MODE=$pgoMode", + ) ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } + + if (hermesV1Enabled) { +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +--- a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +@@ -52,6 +52,15 @@ val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} + val reactNativeRootDir = project(":packages:react-native:ReactAndroid").projectDir.parent + val customDownloadDir = System.getenv("REACT_NATIVE_DOWNLOADS_DIR") + val downloadsDir = +@@ -383,6 +392,11 @@ android { + // We intentionally build Hermes with Intl support only. This is to simplify + // the build setup and to avoid overcomplicating the build-type matrix. + "-DHERMES_ENABLE_INTL=True", ++ "-DPGO_MODE=$pgoMode", + ) ++ ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } + + if (hermesV1Enabled) { +diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +--- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt ++++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +@@ -8,6 +8,14 @@ set(CMAKE_VERBOSE_MAKEFILE on) + + project(ReactAndroid) ++ ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() + + include(CheckIPOSupported) + check_ipo_supported(RESULT IPO_SUPPORT) +diff --git a/node_modules/react-native/sdks/hermes/CMakeLists.txt b/node_modules/react-native/sdks/hermes/CMakeLists.txt +--- a/node_modules/react-native/sdks/hermes/CMakeLists.txt ++++ b/node_modules/react-native/sdks/hermes/CMakeLists.txt +@@ -476,6 +476,12 @@ if (HERMES_ENABLE_CODE_COVERAGE) + endif() + endif() ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ append("-fprofile-generate" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) ++ append("-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -Wl,--export-dynamic-symbol=__llvm_profile_write_file" CMAKE_SHARED_LINKER_FLAGS) ++elseif(PGO_MODE STREQUAL "use") ++ append("-fprofile-use=${PGO_PROFILE}" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) ++endif() + if(HERMES_FACEBOOK_BUILD) + add_definitions(-DHERMES_FACEBOOK_BUILD) + endif() From e1747b547df47c7f5e2bfd90d3fdb4b6b6900894 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 11:47:28 +0200 Subject: [PATCH 03/60] build: add `verify-instrumented` tool --- scripts/pgo/android-local-proof.sh | 56 ++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index cf28b86b0138..e84932dcbabd 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -2,14 +2,19 @@ set -euo pipefail +readonly NDK_VERSION="27.1.12297006" +readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" + readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly PACKAGE_NAME="org.me.mobiexpensifyg" -readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly BUILD_VARIANT="Release" +readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" +readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" + function usage() { - echo "Usage: $0 {build-instrumented|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -34,6 +39,48 @@ function gradle() { ) } +function verify_pgo_instrumentation() ( + local apk_path="${1:-$APK_PATH}" + if [[ ! -f "$apk_path" ]]; then + echo "Missing APK at $apk_path. Build the instrumented release first." >&2 + exit 1 + fi + + local extracted_dir + extracted_dir="$(mktemp -d "${TMPDIR:-/tmp}/expensify-pgo-apk.XXXXXX")" + trap 'rm -rf "$extracted_dir"' EXIT + + local llvm_readelf + if ! llvm_readelf="$(ndk_tool llvm-readelf)"; then + exit 1 + fi + + local libraries=( + libreactnative.so + libhermesvm.so + libjsi.so + libExpensifyNitroUtils.so + ) + local library + for library in "${libraries[@]}"; do + local apk_entry="lib/arm64-v8a/$library" + if ! zipinfo -1 "$apk_path" | rg -Fx "$apk_entry" >/dev/null; then + echo "Missing expected arm64 library in APK: $apk_entry" >&2 + exit 1 + fi + + local extracted_library="$extracted_dir/$library" + unzip -p "$apk_path" "$apk_entry" > "$extracted_library" + local section_headers + section_headers="$("$llvm_readelf" -SW "$extracted_library")" + if [[ "$section_headers" != *"__llvm_prf_data"* || "$section_headers" != *"__llvm_prf_cnts"* || "$section_headers" != *"__llvm_prf_names"* ]]; then + echo "LLVM PGO instrumentation is missing from $apk_entry." >&2 + exit 1 + fi + echo "Verified LLVM PGO instrumentation: $apk_entry" + done +) + case "${1:-}" in build-instrumented) gradle ":assemble$BUILD_VARIANT" \ @@ -41,6 +88,9 @@ case "${1:-}" in -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate ;; + verify-instrumented) + verify_pgo_instrumentation + ;; dump) adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ From 96518ea7e01c7f12537005ab5a113bfd4926b771 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 12:30:04 +0200 Subject: [PATCH 04/60] chore: update RN patch --- ...atch => react-native+0.85.3+040+pgo.patch} | 91 ++++++++++++------- 1 file changed, 56 insertions(+), 35 deletions(-) rename patches/react-native/{react-native+0.85.3+040+add-local-pgo-build-switch.patch => react-native+0.85.3+040+pgo.patch} (68%) diff --git a/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.85.3+040+pgo.patch similarity index 68% rename from patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch rename to patches/react-native/react-native+0.85.3+040+pgo.patch index 57b7b6fe96df..7132cc63a984 100644 --- a/patches/react-native/react-native+0.85.3+040+add-local-pgo-build-switch.patch +++ b/patches/react-native/react-native+0.85.3+040+pgo.patch @@ -1,9 +1,11 @@ diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modules/react-native/ReactAndroid/build.gradle.kts +index bae034d..94d11e1 100644 --- a/node_modules/react-native/ReactAndroid/build.gradle.kts +++ b/node_modules/react-native/ReactAndroid/build.gradle.kts -@@ -40,6 +40,15 @@ val reactNativeRootDir = projectDir.parent +@@ -42,6 +42,16 @@ val reactNativeRootDir = projectDir.parent val hermesV1Enabled = rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() + +val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" +val pgoProfile = project.findProperty("pgoProfile")?.toString() + @@ -13,26 +15,36 @@ diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modu +if (pgoMode == "use") { + check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } +} - ++ // We put the publishing version from gradle.properties inside ext. so other // subprojects can access it as well. -@@ -579,6 +588,10 @@ android { + extra["publishing_version"] = project.findProperty("VERSION_NAME")?.toString()!! +@@ -579,8 +589,13 @@ android { + "-DANDROID_TOOLCHAIN=clang", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW", + "-DPGO_MODE=$pgoMode", ) + + if (pgoMode == "use") { + arguments("-DPGO_PROFILE=$pgoProfile") + } - ++ if (hermesV1Enabled) { + arguments("-DHERMES_V1_ENABLED=1") + } diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +index 6610782..853b937 100644 --- a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +++ b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts -@@ -52,6 +52,15 @@ val hermesV1Enabled = +@@ -52,6 +52,18 @@ fun getSDKManagerPath(): String { + + val hermesV1Enabled = rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++ +val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" +val pgoProfile = project.findProperty("pgoProfile")?.toString() ++val pgoCmakePath = File(projectDir, "pgo.cmake") + +check(pgoMode in setOf("off", "generate", "use")) { + "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." @@ -40,28 +52,60 @@ diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.k +if (pgoMode == "use") { + check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } +} ++ val reactNativeRootDir = project(":packages:react-native:ReactAndroid").projectDir.parent val customDownloadDir = System.getenv("REACT_NATIVE_DOWNLOADS_DIR") val downloadsDir = -@@ -383,6 +392,11 @@ android { +@@ -415,8 +427,14 @@ android { // We intentionally build Hermes with Intl support only. This is to simplify // the build setup and to avoid overcomplicating the build-type matrix. "-DHERMES_ENABLE_INTL=True", + "-DPGO_MODE=$pgoMode", ++ "-DCMAKE_PROJECT_Hermes_INCLUDE=${pgoCmakePath.absolutePath}", ) -+ + + if (pgoMode == "use") { + arguments("-DPGO_PROFILE=$pgoProfile") + } - ++ if (hermesV1Enabled) { + arguments("-DHERMESVM_HEAP_HV_MODE=HEAP_HV_PREFER32") + } +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +new file mode 100644 +index 0000000..bcc335d +--- /dev/null ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +@@ -0,0 +1,21 @@ ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +index 8125e02..d466798 100644 --- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt -@@ -8,6 +8,14 @@ set(CMAKE_VERBOSE_MAKEFILE on) +@@ -8,6 +8,28 @@ set(CMAKE_VERBOSE_MAKEFILE on) project(ReactAndroid) -+ + +if(PGO_MODE STREQUAL "generate") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir @@ -83,30 +127,7 @@ diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt + add_compile_options("-fprofile-use=${PGO_PROFILE}") + add_link_options("-fprofile-use=${PGO_PROFILE}") +endif() - ++ include(CheckIPOSupported) check_ipo_supported(RESULT IPO_SUPPORT) -diff --git a/node_modules/react-native/sdks/hermes/CMakeLists.txt b/node_modules/react-native/sdks/hermes/CMakeLists.txt ---- a/node_modules/react-native/sdks/hermes/CMakeLists.txt -+++ b/node_modules/react-native/sdks/hermes/CMakeLists.txt -@@ -476,6 +476,12 @@ if (HERMES_ENABLE_CODE_COVERAGE) - endif() - endif() -+if(PGO_MODE STREQUAL "generate") -+ execute_process( -+ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir -+ OUTPUT_VARIABLE CLANG_RESOURCE_DIR -+ OUTPUT_STRIP_TRAILING_WHITESPACE -+ ) -+ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") -+ if(NOT EXISTS "${PGO_RUNTIME}") -+ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") -+ endif() -+ append("-fprofile-generate" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) -+ append("-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -Wl,--export-dynamic-symbol=__llvm_profile_write_file" CMAKE_SHARED_LINKER_FLAGS) -+elseif(PGO_MODE STREQUAL "use") -+ append("-fprofile-use=${PGO_PROFILE}" CMAKE_CXX_FLAGS CMAKE_C_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) -+endif() - if(HERMES_FACEBOOK_BUILD) - add_definitions(-DHERMES_FACEBOOK_BUILD) - endif() + if (IPO_SUPPORT) From 27c7035fae767217ccba087983192dbf63060817 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:36:36 +0200 Subject: [PATCH 05/60] chore: add `.pgo` dir to `.gitignore` --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e547a16b7ca4..b483094cf1da 100644 --- a/.gitignore +++ b/.gitignore @@ -134,6 +134,9 @@ config/rsbuild/*.pem # Rspack build profiling output (RSPACK_PROFILE=OVERVIEW|ALL) .rspack-profile-*/ +# Local native PGO profiles +.pgo/ + # Expo .expo dist/ From e8d11043bab2f9bbb9da76593c52562a5a489c9d Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:37:34 +0200 Subject: [PATCH 06/60] fix: PGO merging --- .../android/CMakeLists.txt | 8 ++++-- .../android/src/main/cpp/PgoProfileWriter.cpp | 11 ++++---- .../react-native+0.85.3+040+pgo.patch | 20 +++++++++---- scripts/pgo/android-local-proof.sh | 28 +++++++++++++++++-- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index 4fa4657d5931..0936f76325d7 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -19,8 +19,12 @@ if(EXPENSIFY_PGO_MODE STREQUAL "generate") add_link_options( -fprofile-generate "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" - -Wl,--export-dynamic-symbol=__llvm_profile_set_filename - -Wl,--export-dynamic-symbol=__llvm_profile_write_file + -Wl,--undefined=__llvm_profile_set_filename + -Wl,--undefined=__llvm_profile_write_file + -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename + -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file + -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename + -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file ) elseif(EXPENSIFY_PGO_MODE STREQUAL "use") add_compile_options("-fprofile-use=${EXPENSIFY_PGO_PROFILE}") diff --git a/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp index 32e7477301b4..d4a0357f7f2f 100644 --- a/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp +++ b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp @@ -5,7 +5,7 @@ #include namespace { -using SetFilename = int (*)(const char *); +using SetFilename = void (*)(const char *); using WriteFile = int (*)(); constexpr std::array kInstrumentedLibraries = { @@ -16,18 +16,19 @@ constexpr std::array kInstrumentedLibraries = { }; bool writeProfile(void *handle, const std::string &profilePath) { - const auto setFilename = reinterpret_cast(dlsym(handle, "__llvm_profile_set_filename")); - const auto writeFile = reinterpret_cast(dlsym(handle, "__llvm_profile_write_file")); + const auto setFilename = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_set_filename")); + const auto writeFile = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_write_file")); if (setFilename == nullptr || writeFile == nullptr) { return false; } - return setFilename(profilePath.c_str()) == 0 && writeFile() == 0; + setFilename(profilePath.c_str()); + return writeFile() == 0; } } // namespace extern "C" JNIEXPORT jint JNICALL -Java_com_expensify_chat_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { +Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { const char *directoryChars = env->GetStringUTFChars(directory, nullptr); if (directoryChars == nullptr) { return 0; diff --git a/patches/react-native/react-native+0.85.3+040+pgo.patch b/patches/react-native/react-native+0.85.3+040+pgo.patch index 7132cc63a984..14ca0c45eb91 100644 --- a/patches/react-native/react-native+0.85.3+040+pgo.patch +++ b/patches/react-native/react-native+0.85.3+040+pgo.patch @@ -76,7 +76,7 @@ new file mode 100644 index 0000000..bcc335d --- /dev/null +++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake -@@ -0,0 +1,21 @@ +@@ -0,0 +1,25 @@ +if(PGO_MODE STREQUAL "generate") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir @@ -91,8 +91,12 @@ index 0000000..bcc335d + add_link_options( + -fprofile-generate + "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" -+ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -+ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file + ) +elseif(PGO_MODE STREQUAL "use") + add_compile_options("-fprofile-use=${PGO_PROFILE}") @@ -102,7 +106,7 @@ diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt index 8125e02..d466798 100644 --- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt -@@ -8,6 +8,28 @@ set(CMAKE_VERBOSE_MAKEFILE on) +@@ -8,6 +8,32 @@ set(CMAKE_VERBOSE_MAKEFILE on) project(ReactAndroid) @@ -120,8 +124,12 @@ index 8125e02..d466798 100644 + add_link_options( + -fprofile-generate + "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" -+ -Wl,--export-dynamic-symbol=__llvm_profile_set_filename -+ -Wl,--export-dynamic-symbol=__llvm_profile_write_file ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file + ) +elseif(PGO_MODE STREQUAL "use") + add_compile_options("-fprofile-use=${PGO_PROFILE}") diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index e84932dcbabd..56dc140520b4 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -12,6 +12,7 @@ readonly BUILD_VARIANT="Release" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" function usage() { echo "Usage: $0 {build-instrumented|verify-instrumented|dump|pull|merge|build-optimized}" @@ -77,6 +78,18 @@ function verify_pgo_instrumentation() ( echo "LLVM PGO instrumentation is missing from $apk_entry." >&2 exit 1 fi + + local dynamic_symbols + dynamic_symbols="$("$llvm_nm" -D --defined-only "$extracted_library")" + if [[ "$dynamic_symbols" != *"expensify_llvm_profile_set_filename"* || "$dynamic_symbols" != *"expensify_llvm_profile_write_file"* ]]; then + echo "LLVM PGO profile-writing APIs are not exported from $apk_entry." >&2 + exit 1 + fi + + if [[ "$library" == "libExpensifyNitroUtils.so" && "$dynamic_symbols" != *"Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles"* ]]; then + echo "The PGO JNI writer is missing from $apk_entry. Rebuild the native module before installing." >&2 + exit 1 + fi echo "Verified LLVM PGO instrumentation: $apk_entry" done ) @@ -99,8 +112,16 @@ case "${1:-}" in pull) rm -rf "$PROFILE_DIR/raw" mkdir -p "$PROFILE_DIR/raw" - adb exec-out run-as "$PACKAGE_NAME" tar -C cache -cf - . | tar -C "$PROFILE_DIR/raw" -xf - - find "$PROFILE_DIR/raw" -name '*.profraw' -print + adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" + profiles=() + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 + exit 1 + fi + printf '%s\n' "${profiles[@]}" ;; merge) mkdir -p "$PROFILE_DIR" @@ -114,13 +135,14 @@ case "${1:-}" in fi "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" + echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" ;; build-optimized) if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 exit 1 fi - gradle ":app:assemble$BUILD_VARIANT" \ + gradle ":assemble$BUILD_VARIANT" \ -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ From 2b114fdb3aa2ad5ac485132746a9b665b43e8140 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:38:02 +0200 Subject: [PATCH 07/60] build: add `install-instrumented` command --- scripts/pgo/android-local-proof.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 56dc140520b4..1f5bb99b59a5 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -15,7 +15,7 @@ readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|install|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -40,6 +40,15 @@ function gradle() { ) } +function install_apk() { + if [[ ! -f "$APK_PATH" ]]; then + echo "Missing APK at $APK_PATH. Build the APK first." >&2 + exit 1 + fi + + adb install -r "$APK_PATH" +} + function verify_pgo_instrumentation() ( local apk_path="${1:-$APK_PATH}" if [[ ! -f "$apk_path" ]]; then @@ -104,6 +113,9 @@ case "${1:-}" in verify-instrumented) verify_pgo_instrumentation ;; + install) + install_apk + ;; dump) adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ From 9322dba115d31c188998ab60e1c3b25de4b462a7 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:38:11 +0200 Subject: [PATCH 08/60] Update README.md --- scripts/pgo/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 96bf6281f7b9..2c53d2e6fac9 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -15,7 +15,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android ```bash scripts/pgo/android-local-proof.sh build-instrumented - adb install -r Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release.apk + scripts/pgo/android-local-proof.sh install ``` 2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. @@ -27,6 +27,8 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh merge ``` + Profiles are written to the app's external cache so they can be retrieved from a non-debuggable release APK with `adb pull`. + 4. Build the matched optimized application: ```bash From 47f9d5bcd9a3e3ffdd6ed175b58a3d9ed7fec4ee Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 14:38:14 +0200 Subject: [PATCH 09/60] Update android-local-proof.sh --- scripts/pgo/android-local-proof.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 1f5bb99b59a5..f11ea9b9eeb5 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -64,6 +64,10 @@ function verify_pgo_instrumentation() ( if ! llvm_readelf="$(ndk_tool llvm-readelf)"; then exit 1 fi + local llvm_nm + if ! llvm_nm="$(ndk_tool llvm-nm)"; then + exit 1 + fi local libraries=( libreactnative.so From 6ba45d606aef7076dfe485e859d03b2735d5144c Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:08:12 +0200 Subject: [PATCH 10/60] build: create record-startups command --- scripts/pgo/README.md | 18 +++++++++++++++ scripts/pgo/android-local-proof.sh | 36 +++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 2c53d2e6fac9..6c36f0375d85 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,6 +35,24 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` +### Collect a startup-focused profile + +To replace the manual journey with ten cold-process startup runs, then pull and merge the accumulated profiles automatically, run: + +```bash +scripts/pgo/android-local-proof.sh record-startups +``` + +The optional arguments set the number of runs and the fixed settling time after `am start -W` respectively: + +```bash +scripts/pgo/android-local-proof.sh record-startups 10 10 +``` + +The command clears only existing `newdot-*.profraw` files on the device before the first run. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. + +The settling time is currently a fallback because the release app does not expose a device-visible NewDot-ready signal. A benchmark-only build should emit a native Logcat marker and call Android's `reportFullyDrawn()` from `Expensify.tsx`'s `onSplashHide`, after the splash exit animation and startup gates have completed. Keep that measurement marker on the separate metrics branch. + The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. ## Compare diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index f11ea9b9eeb5..842e64ae9892 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -13,9 +13,10 @@ readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" +readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|install|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|install|record-startups [runs] [settle-seconds]|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -49,6 +50,36 @@ function install_apk() { adb install -r "$APK_PATH" } +function record_startups() { + local runs="${1:-10}" + local settle_seconds="${2:-10}" + + if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then + echo "Startup run count must be a positive integer, received: $runs" >&2 + exit 1 + fi + if [[ ! "$settle_seconds" =~ ^[0-9]+$ ]]; then + echo "Startup settle time must be a non-negative integer, received: $settle_seconds" >&2 + exit 1 + fi + + echo "Clearing previous device PGO profiles from $DEVICE_PROFILE_DIR." + adb shell "rm -f '$DEVICE_PROFILE_DIR'/newdot-*.profraw" + + local run + for ((run = 1; run <= runs; run++)); do + echo "Recording cold-process startup $run/$runs." + adb shell am force-stop "$PACKAGE_NAME" + adb shell am start -W -n "$START_ACTIVITY" + echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." + sleep "$settle_seconds" + "$0" dump + done + + "$0" pull + "$0" merge +} + function verify_pgo_instrumentation() ( local apk_path="${1:-$APK_PATH}" if [[ ! -f "$apk_path" ]]; then @@ -120,6 +151,9 @@ case "${1:-}" in install) install_apk ;; + record-startups) + record_startups "${2:-10}" "${3:-10}" + ;; dump) adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ From 62808e8d919a90a4a73308927cfda4f47d07c6c1 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:25:01 +0200 Subject: [PATCH 11/60] fix: prevent startup profile merge scope error --- scripts/pgo/android-local-proof.sh | 79 ++++++++++++++++++------------ 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 842e64ae9892..41c0ab08d3d8 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -50,6 +50,49 @@ function install_apk() { adb install -r "$APK_PATH" } +function dump_profiles() { + adb shell am broadcast \ + -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$PACKAGE_NAME/.PgoProfileReceiver" +} + +function pull_profiles() { + rm -rf "$PROFILE_DIR/raw" + mkdir -p "$PROFILE_DIR/raw" + adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" + + local -a profiles=() + local profile + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 + exit 1 + fi + printf '%s\n' "${profiles[@]}" +} + +function merge_profiles() { + mkdir -p "$PROFILE_DIR" + + local -a profiles=() + local profile + while IFS= read -r profile; do + profiles+=("$profile") + done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) + + if [[ "${#profiles[@]}" -eq 0 ]]; then + echo "No .profraw files found. Run dump and pull first." >&2 + exit 1 + fi + + "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" + "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" + echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" +} + function record_startups() { local runs="${1:-10}" local settle_seconds="${2:-10}" @@ -73,11 +116,11 @@ function record_startups() { adb shell am start -W -n "$START_ACTIVITY" echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." sleep "$settle_seconds" - "$0" dump + dump_profiles done - "$0" pull - "$0" merge + pull_profiles + merge_profiles } function verify_pgo_instrumentation() ( @@ -155,37 +198,13 @@ case "${1:-}" in record_startups "${2:-10}" "${3:-10}" ;; dump) - adb shell am broadcast \ - -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$PACKAGE_NAME/.PgoProfileReceiver" + dump_profiles ;; pull) - rm -rf "$PROFILE_DIR/raw" - mkdir -p "$PROFILE_DIR/raw" - adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" - profiles=() - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 - exit 1 - fi - printf '%s\n' "${profiles[@]}" + pull_profiles ;; merge) - mkdir -p "$PROFILE_DIR" - profiles=() - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found. Run dump and pull first." >&2 - exit 1 - fi - "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" - "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" - echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" + merge_profiles ;; build-optimized) if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then From 0da014af9d201d2b61bf984e3123a89299c9e4c9 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 16:12:37 +0200 Subject: [PATCH 12/60] build: add dedicated PGO build variants --- scripts/pgo/README.md | 10 +++- scripts/pgo/android-local-proof.sh | 83 ++++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 6c36f0375d85..2230e0576ce7 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,6 +1,6 @@ # Local Android PGO proof -This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the project's `release` variant: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the release-derived `pgoInstrumented` and `pgoOptimized` variants: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. ## Preconditions @@ -15,7 +15,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android ```bash scripts/pgo/android-local-proof.sh build-instrumented - scripts/pgo/android-local-proof.sh install + scripts/pgo/android-local-proof.sh install-instrumented ``` 2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. @@ -35,6 +35,12 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` + The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. Install the optimized APK with: + + ```bash + scripts/pgo/android-local-proof.sh install-optimized + ``` + ### Collect a startup-focused profile To replace the manual journey with ten cold-process startup runs, then pull and merge the accumulated profiles automatically, run: diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 41c0ab08d3d8..18bc7dc48da7 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -7,16 +7,24 @@ readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" readonly PACKAGE_NAME="org.me.mobiexpensifyg" -readonly BUILD_VARIANT="Release" +readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" +readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" -readonly APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" +readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" +readonly APP_READY_LOG_TAG="NewDotStartup" +readonly APP_READY_LOG_MESSAGE="APP_READY" +readonly DEFAULT_STARTUP_RUNS=10 +readonly DEFAULT_APP_READY_TIMEOUT_SECONDS=30 +readonly STARTUP_RELAUNCH_DELAY_SECONDS=0.5 +readonly PROFILE_DUMP_TIMEOUT_SECONDS=5 function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|install|record-startups [runs] [settle-seconds]|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-instrumented|verify-instrumented|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|dump|pull|merge|build-optimized}" } function ndk_tool() { @@ -41,19 +49,71 @@ function gradle() { ) } +function build_instrumented() { + gradle ":assemble$INSTRUMENTED_BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=generate +} + +function build_optimized() { + if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then + echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 + exit 1 + fi + + gradle ":assemble$OPTIMIZED_BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=use \ + -PpgoProfile="$PROFILE_DIR/newdot.profdata" +} + function install_apk() { - if [[ ! -f "$APK_PATH" ]]; then - echo "Missing APK at $APK_PATH. Build the APK first." >&2 + local apk_path="$1" + if [[ ! -f "$apk_path" ]]; then + echo "Missing APK at $apk_path. Build the APK first." >&2 exit 1 fi - adb install -r "$APK_PATH" + adb install -r "$apk_path" } function dump_profiles() { + adb logcat -c adb shell am broadcast \ -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ -n "$PACKAGE_NAME/.PgoProfileReceiver" + + wait_for_profile_dump +} + +function wait_for_profile_dump() { + local started_at="$SECONDS" + + while ((SECONDS - started_at < PROFILE_DUMP_TIMEOUT_SECONDS)); do + local profile_logs + profile_logs="$(adb logcat -d -s 'PgoProfileReceiver:I' '*:S')" + + if [[ "$profile_logs" =~ Wrote\ ([1-9][0-9]*)\ LLVM\ PGO\ profile ]]; then + echo "${BASH_REMATCH[0]}(s)." + return 0 + fi + if [[ "$profile_logs" == *"Ignoring PGO profile request in a non-instrumented build."* ]]; then + echo "The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files." >&2 + return 1 + fi + if [[ "$profile_logs" == *"Wrote 0 LLVM PGO profile(s)."* ]]; then + echo "The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles." >&2 + return 1 + fi + + sleep 0.1 + done + + echo "The PGO receiver did not confirm a profile write within ${PROFILE_DUMP_TIMEOUT_SECONDS}s." >&2 + adb logcat -d -s 'PgoProfileReceiver:I' '*:S' >&2 + return 1 } function pull_profiles() { @@ -113,6 +173,8 @@ function record_startups() { for ((run = 1; run <= runs; run++)); do echo "Recording cold-process startup $run/$runs." adb shell am force-stop "$PACKAGE_NAME" + sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" + adb logcat -c adb shell am start -W -n "$START_ACTIVITY" echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." sleep "$settle_seconds" @@ -124,7 +186,7 @@ function record_startups() { } function verify_pgo_instrumentation() ( - local apk_path="${1:-$APK_PATH}" + local apk_path="${1:-$INSTRUMENTED_APK_PATH}" if [[ ! -f "$apk_path" ]]; then echo "Missing APK at $apk_path. Build the instrumented release first." >&2 exit 1 @@ -191,8 +253,11 @@ case "${1:-}" in verify-instrumented) verify_pgo_instrumentation ;; - install) - install_apk + install | install-instrumented) + install_apk "$INSTRUMENTED_APK_PATH" + ;; + install-optimized) + install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) record_startups "${2:-10}" "${3:-10}" From ea4b7fb6631933b05198f71b42d0f7b254641be2 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:28:19 +0200 Subject: [PATCH 13/60] Wait for native app-ready signal during PGO startup recording --- Mobile-Expensify | 2 +- scripts/pgo/README.md | 8 +++---- scripts/pgo/android-local-proof.sh | 31 ++++++++++++++++++++----- src/Expensify.tsx | 2 ++ src/libs/BootSplash/types.ts | 1 + src/libs/StartupMetrics/index.native.ts | 7 ++++++ src/libs/StartupMetrics/index.ts | 3 +++ 7 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 src/libs/StartupMetrics/index.native.ts create mode 100644 src/libs/StartupMetrics/index.ts diff --git a/Mobile-Expensify b/Mobile-Expensify index 3d7ad2f66ba5..9c1320c6911b 160000 --- a/Mobile-Expensify +++ b/Mobile-Expensify @@ -1 +1 @@ -Subproject commit 3d7ad2f66ba5fe2ccb45c4d1b173fd41d922775d +Subproject commit 9c1320c6911b7106c7fdaf8648d3910609d99087 diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 2230e0576ce7..0739cd7c0cb8 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -49,15 +49,15 @@ To replace the manual journey with ten cold-process startup runs, then pull and scripts/pgo/android-local-proof.sh record-startups ``` -The optional arguments set the number of runs and the fixed settling time after `am start -W` respectively: +The optional arguments set the number of runs and the timeout for NewDot's native app-ready signal respectively: ```bash -scripts/pgo/android-local-proof.sh record-startups 10 10 +scripts/pgo/android-local-proof.sh record-startups 10 30 ``` -The command clears only existing `newdot-*.profraw` files on the device before the first run. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. +The command clears only existing `newdot-*.profraw` files on the device before the first run. It clears Logcat before each launch, waits for `NewDotStartup: APP_READY`, and fails rather than recording an incomplete startup if the marker does not appear before the timeout. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. -The settling time is currently a fallback because the release app does not expose a device-visible NewDot-ready signal. A benchmark-only build should emit a native Logcat marker and call Android's `reportFullyDrawn()` from `Expensify.tsx`'s `onSplashHide`, after the splash exit animation and startup gates have completed. Keep that measurement marker on the separate metrics branch. +The metrics build calls Android's `reportFullyDrawn()` and emits the native marker from `Expensify.tsx`'s `onSplashHide`, immediately after the `ManualAppStartup` Sentry span ends and after the splash exit animation and startup gates have completed. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 18bc7dc48da7..99e47e78bad8 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -155,14 +155,14 @@ function merge_profiles() { function record_startups() { local runs="${1:-10}" - local settle_seconds="${2:-10}" + local ready_timeout_seconds="${2:-30}" if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then echo "Startup run count must be a positive integer, received: $runs" >&2 exit 1 fi - if [[ ! "$settle_seconds" =~ ^[0-9]+$ ]]; then - echo "Startup settle time must be a non-negative integer, received: $settle_seconds" >&2 + if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 exit 1 fi @@ -176,8 +176,7 @@ function record_startups() { sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" adb logcat -c adb shell am start -W -n "$START_ACTIVITY" - echo "Waiting ${settle_seconds}s for NewDot to become ready before dumping." - sleep "$settle_seconds" + wait_for_app_ready "$ready_timeout_seconds" dump_profiles done @@ -185,6 +184,26 @@ function record_startups() { merge_profiles } +function wait_for_app_ready() { + local timeout_seconds="$1" + local started_at="$SECONDS" + + echo "Waiting up to ${timeout_seconds}s for $APP_READY_LOG_TAG: $APP_READY_LOG_MESSAGE." + while ((SECONDS - started_at < timeout_seconds)); do + local startup_logs + startup_logs="$(adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S')" + if [[ "$startup_logs" == *"$APP_READY_LOG_MESSAGE"* ]]; then + echo "NewDot reported APP_READY." + return 0 + fi + sleep 0.25 + done + + echo "NewDot did not report APP_READY within ${timeout_seconds}s." >&2 + adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S' >&2 + return 1 +} + function verify_pgo_instrumentation() ( local apk_path="${1:-$INSTRUMENTED_APK_PATH}" if [[ ! -f "$apk_path" ]]; then @@ -260,7 +279,7 @@ case "${1:-}" in install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) - record_startups "${2:-10}" "${3:-10}" + record_startups "${2:-10}" "${3:-30}" ;; dump) dump_profiles diff --git a/src/Expensify.tsx b/src/Expensify.tsx index 245c3a6a2c26..93ac0c72074c 100644 --- a/src/Expensify.tsx +++ b/src/Expensify.tsx @@ -34,6 +34,7 @@ import './libs/NetworkState'; import Navigation from './libs/Navigation/Navigation'; import NavigationRoot from './libs/Navigation/NavigationRoot'; import PushNotification from './libs/Notification/PushNotification'; +import StartupMetrics from './libs/StartupMetrics'; import {endSpan, getSpan, startSpan} from './libs/telemetry/activeSpans'; import {startBootsplashMonitor} from './libs/telemetry/bootsplashTelemetry'; import {cleanupTelemetryTrackers, initializeTelemetryTrackers} from './libs/telemetry/TelemetrySynchronizer'; @@ -208,6 +209,7 @@ function Expensify() { endSpan(CONST.TELEMETRY.SPAN_APP_STARTUP); endSpan(CONST.TELEMETRY.SPAN_BOOTSPLASH.ROOT); endSpan(CONST.TELEMETRY.SPAN_BOOTSPLASH.SPLASH_HIDER); + StartupMetrics.reportFullyDrawn(); }, [setSplashScreenState]); useLayoutEffect(() => { diff --git a/src/libs/BootSplash/types.ts b/src/libs/BootSplash/types.ts index 8fa41b2d1af2..3f20032d575b 100644 --- a/src/libs/BootSplash/types.ts +++ b/src/libs/BootSplash/types.ts @@ -2,6 +2,7 @@ type BootSplashModule = { logoSizeRatio: number; navigationBarHeight: number; hide: () => Promise; + reportFullyDrawn?: () => void; }; // eslint-disable-next-line import/prefer-default-export diff --git a/src/libs/StartupMetrics/index.native.ts b/src/libs/StartupMetrics/index.native.ts new file mode 100644 index 000000000000..b9c684246afa --- /dev/null +++ b/src/libs/StartupMetrics/index.native.ts @@ -0,0 +1,7 @@ +import {NativeModules} from 'react-native'; + +function reportFullyDrawn(): void { + NativeModules.BootSplash.reportFullyDrawn?.(); +} + +export default {reportFullyDrawn}; diff --git a/src/libs/StartupMetrics/index.ts b/src/libs/StartupMetrics/index.ts new file mode 100644 index 000000000000..f879a0dc9d99 --- /dev/null +++ b/src/libs/StartupMetrics/index.ts @@ -0,0 +1,3 @@ +const reportFullyDrawn = (): void => undefined; + +export default {reportFullyDrawn}; From 151f4f1f7f001782c2fe27fa81ec1b6333fc6e14 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 15:39:10 +0200 Subject: [PATCH 14/60] chore: improve script --- scripts/pgo/android-local-proof.sh | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 99e47e78bad8..47a437a6d2c7 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -154,8 +154,8 @@ function merge_profiles() { } function record_startups() { - local runs="${1:-10}" - local ready_timeout_seconds="${2:-30}" + local runs="${1:-$DEFAULT_STARTUP_RUNS}" + local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then echo "Startup run count must be a positive integer, received: $runs" >&2 @@ -264,10 +264,7 @@ function verify_pgo_instrumentation() ( case "${1:-}" in build-instrumented) - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=generate + build_instrumented ;; verify-instrumented) verify_pgo_instrumentation @@ -279,7 +276,7 @@ case "${1:-}" in install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) - record_startups "${2:-10}" "${3:-30}" + record_startups "${2:-}" "${3:-}" ;; dump) dump_profiles @@ -291,15 +288,7 @@ case "${1:-}" in merge_profiles ;; build-optimized) - if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then - echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 - exit 1 - fi - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=use \ - -PpgoProfile="$PROFILE_DIR/newdot.profdata" + build_optimized ;; *) usage From 192125505df87d12fa69105975c14ec6a512dc0c Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 16:27:40 +0200 Subject: [PATCH 15/60] chore: add PGO benchmarking suite --- scripts/pgo/README.md | 39 ++++++- scripts/pgo/android-local-proof.sh | 176 +++++++++++++++++++++++++++-- 2 files changed, 204 insertions(+), 11 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 0739cd7c0cb8..2fd8629fcf4d 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -57,7 +57,44 @@ scripts/pgo/android-local-proof.sh record-startups 10 30 The command clears only existing `newdot-*.profraw` files on the device before the first run. It clears Logcat before each launch, waits for `NewDotStartup: APP_READY`, and fails rather than recording an incomplete startup if the marker does not appear before the timeout. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. -The metrics build calls Android's `reportFullyDrawn()` and emits the native marker from `Expensify.tsx`'s `onSplashHide`, immediately after the `ManualAppStartup` Sentry span ends and after the splash exit animation and startup gates have completed. +The metrics build calls Android's `reportFullyDrawn()` and emits the machine-readable `NewDotStartup: APP_READY durationMs=` native marker from `Expensify.tsx`'s `onSplashHide`, immediately after the `ManualAppStartup` Sentry span ends and after the splash exit animation and startup gates have completed. The duration starts at the same native timestamp used by the Sentry span. + +## Benchmark startup + +Build the release and PGO-optimized APK from the same source revision, compiler, NDK, and ABI: + +```bash +scripts/pgo/android-local-proof.sh build-release +scripts/pgo/android-local-proof.sh build-optimized +``` + +Collect each benchmark independently. Each command installs its APK without clearing application data, performs one unmeasured warm-up, then records ten cold-process startup samples by default: + +```bash +scripts/pgo/android-local-proof.sh benchmark-release +scripts/pgo/android-local-proof.sh benchmark-optimized +``` + +The optional arguments select the measured run count and app-ready timeout: + +```bash +scripts/pgo/android-local-proof.sh benchmark-release 20 30 +scripts/pgo/android-local-proof.sh benchmark-optimized 20 30 +``` + +Compare previously collected samples: + +```bash +scripts/pgo/android-local-proof.sh compare-benchmarks +``` + +Or run the release benchmark first, then the optimized benchmark, and compare them in one command: + +```bash +scripts/pgo/android-local-proof.sh benchmark 10 30 +``` + +Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the mean, median, minimum, maximum, and percentage improvement. Positive improvement percentages mean the optimized build was faster. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 47a437a6d2c7..db00a15d658c 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -7,13 +7,18 @@ readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" readonly PACKAGE_NAME="org.me.mobiexpensifyg" +readonly RELEASE_BUILD_VARIANT="Release" readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" +readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" +readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" +readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" readonly APP_READY_LOG_TAG="NewDotStartup" @@ -23,8 +28,10 @@ readonly DEFAULT_APP_READY_TIMEOUT_SECONDS=30 readonly STARTUP_RELAUNCH_DELAY_SECONDS=0.5 readonly PROFILE_DUMP_TIMEOUT_SECONDS=5 +APP_READY_DURATION_MS="" + function usage() { - echo "Usage: $0 {build-instrumented|verify-instrumented|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|dump|pull|merge|build-optimized}" + echo "Usage: $0 {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}" } function ndk_tool() { @@ -56,6 +63,13 @@ function build_instrumented() { -PpgoMode=generate } +function build_release() { + gradle ":assemble$RELEASE_BUILD_VARIANT" \ + -PpatchedArtifacts.forceBuildFromSource=true \ + -PreactNativeArchitectures=arm64-v8a \ + -PpgoMode=off +} + function build_optimized() { if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 @@ -157,14 +171,7 @@ function record_startups() { local runs="${1:-$DEFAULT_STARTUP_RUNS}" local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then - echo "Startup run count must be a positive integer, received: $runs" >&2 - exit 1 - fi - if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then - echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 - exit 1 - fi + validate_startup_arguments "$runs" "$ready_timeout_seconds" echo "Clearing previous device PGO profiles from $DEVICE_PROFILE_DIR." adb shell "rm -f '$DEVICE_PROFILE_DIR'/newdot-*.profraw" @@ -184,16 +191,147 @@ function record_startups() { merge_profiles } +function validate_startup_arguments() { + local runs="$1" + local ready_timeout_seconds="$2" + + if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then + echo "Startup run count must be a positive integer, received: $runs" >&2 + exit 1 + fi + if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 + exit 1 + fi +} + +function measure_startup() { + local ready_timeout_seconds="$1" + + adb shell am force-stop "$PACKAGE_NAME" + sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" + adb logcat -c + adb shell am start -W -n "$START_ACTIVITY" + wait_for_app_ready "$ready_timeout_seconds" + + if [[ -z "$APP_READY_DURATION_MS" ]]; then + echo "APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes." >&2 + return 1 + fi +} + +function benchmark_startups() { + local label="$1" + local apk_path="$2" + local benchmark_path="$3" + local runs="$4" + local ready_timeout_seconds="$5" + + validate_startup_arguments "$runs" "$ready_timeout_seconds" + install_apk "$apk_path" + mkdir -p "$BENCHMARK_DIR" + + echo "Running one unmeasured warm-up startup for $label." + measure_startup "$ready_timeout_seconds" + + printf 'run,duration_ms\n' > "$benchmark_path" + local run + for ((run = 1; run <= runs; run++)); do + echo "Benchmarking $label cold-process startup $run/$runs." + measure_startup "$ready_timeout_seconds" + printf '%d,%s\n' "$run" "$APP_READY_DURATION_MS" >> "$benchmark_path" + done + + echo "Recorded $runs $label startup samples: $benchmark_path" +} + +function benchmark_stats() { + local benchmark_path="$1" + + awk -F, ' + NR > 1 && $2 ~ /^[0-9]+$/ { + values[++count] = $2 + sum += $2 + } + END { + if (count == 0) { + exit 1 + } + for (i = 1; i <= count; i++) { + for (j = i + 1; j <= count; j++) { + if (values[i] > values[j]) { + temporary = values[i] + values[i] = values[j] + values[j] = temporary + } + } + } + if (count % 2 == 1) { + median = values[(count + 1) / 2] + } else { + median = (values[count / 2] + values[count / 2 + 1]) / 2 + } + printf "%d %.2f %.2f %d %d\n", count, sum / count, median, values[1], values[count] + } + ' "$benchmark_path" +} + +function compare_benchmarks() { + if [[ ! -f "$RELEASE_BENCHMARK_PATH" || ! -f "$OPTIMIZED_BENCHMARK_PATH" ]]; then + echo "Missing benchmark data. Run benchmark-release and benchmark-optimized first." >&2 + exit 1 + fi + + local release_count release_mean release_median release_min release_max + read -r release_count release_mean release_median release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") + local optimized_count optimized_mean optimized_median optimized_min optimized_max + read -r optimized_count optimized_mean optimized_median optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") + + local mean_improvement median_improvement + mean_improvement="$(awk -v release="$release_mean" -v optimized="$optimized_mean" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" + median_improvement="$(awk -v release="$release_median" -v optimized="$optimized_median" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" + + printf '%-18s %6s %10s %10s %8s %8s\n' 'Build' 'Runs' 'Mean ms' 'Median ms' 'Min ms' 'Max ms' + printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'Release' "$release_count" "$release_mean" "$release_median" "$release_min" "$release_max" + printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_mean" "$optimized_median" "$optimized_min" "$optimized_max" + echo "PGO mean startup improvement: ${mean_improvement}%" + echo "PGO median startup improvement: ${median_improvement}%" + echo "Positive percentages are faster; negative percentages are regressions." +} + +function benchmark_release() { + benchmark_startups "release" "$RELEASE_APK_PATH" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" +} + +function benchmark_optimized() { + benchmark_startups "PGO optimized" "$OPTIMIZED_APK_PATH" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" +} + +function benchmark_all() { + local runs="${1:-$DEFAULT_STARTUP_RUNS}" + local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" + + benchmark_release "$runs" "$ready_timeout_seconds" + benchmark_optimized "$runs" "$ready_timeout_seconds" + compare_benchmarks +} + function wait_for_app_ready() { local timeout_seconds="$1" local started_at="$SECONDS" + APP_READY_DURATION_MS="" echo "Waiting up to ${timeout_seconds}s for $APP_READY_LOG_TAG: $APP_READY_LOG_MESSAGE." while ((SECONDS - started_at < timeout_seconds)); do local startup_logs startup_logs="$(adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S')" if [[ "$startup_logs" == *"$APP_READY_LOG_MESSAGE"* ]]; then - echo "NewDot reported APP_READY." + if [[ "$startup_logs" =~ APP_READY\ durationMs=([0-9]+) ]]; then + APP_READY_DURATION_MS="${BASH_REMATCH[1]}" + echo "NewDot reported APP_READY after ${APP_READY_DURATION_MS}ms." + else + echo "NewDot reported APP_READY without a numeric duration marker." + fi return 0 fi sleep 0.25 @@ -263,6 +401,9 @@ function verify_pgo_instrumentation() ( ) case "${1:-}" in + build-release) + build_release + ;; build-instrumented) build_instrumented ;; @@ -272,6 +413,9 @@ case "${1:-}" in install | install-instrumented) install_apk "$INSTRUMENTED_APK_PATH" ;; + install-release) + install_apk "$RELEASE_APK_PATH" + ;; install-optimized) install_apk "$OPTIMIZED_APK_PATH" ;; @@ -290,6 +434,18 @@ case "${1:-}" in build-optimized) build_optimized ;; + benchmark-release) + benchmark_release "${2:-}" "${3:-}" + ;; + benchmark-optimized) + benchmark_optimized "${2:-}" "${3:-}" + ;; + benchmark) + benchmark_all "${2:-}" "${3:-}" + ;; + compare-benchmarks) + compare_benchmarks + ;; *) usage exit 1 From 13df28ae16f7f88ca78ea1929d3d6dcda457c78c Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 16:36:43 +0200 Subject: [PATCH 16/60] build: add dedicated PGO build variants --- scripts/pgo/README.md | 4 ++-- scripts/pgo/android-local-proof.sh | 35 ++++++++++++------------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 2230e0576ce7..b22567559abc 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. Install the optimized APK with: + The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. They use the application IDs `org.me.mobiexpensifyg.pgo.instrumented` and `org.me.mobiexpensifyg.pgo.optimized`, with the launcher names “Expensify PGO Instrumented” and “Expensify PGO Optimized”, so both can remain installed simultaneously. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -63,4 +63,4 @@ The `.pgo/` directory is intentionally local-only. Never apply this profile to a ## Compare -Install the non-PGO and PGO APKs separately. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. +Install the instrumented and optimized APKs side by side. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 18bc7dc48da7..1a7923df9cad 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -6,7 +6,9 @@ readonly NDK_VERSION="27.1.12297006" readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly PACKAGE_NAME="org.me.mobiexpensifyg" +readonly BASE_PACKAGE_NAME="org.me.mobiexpensifyg" +readonly INSTRUMENTED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.instrumented" +readonly OPTIMIZED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.optimized" readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" @@ -14,8 +16,8 @@ readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" -readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" -readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" +readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$INSTRUMENTED_PACKAGE_NAME/cache" +readonly START_ACTIVITY="$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.ExpensifyActivityBase" readonly APP_READY_LOG_TAG="NewDotStartup" readonly APP_READY_LOG_MESSAGE="APP_READY" readonly DEFAULT_STARTUP_RUNS=10 @@ -71,19 +73,21 @@ function build_optimized() { function install_apk() { local apk_path="$1" + local package_name="$2" if [[ ! -f "$apk_path" ]]; then echo "Missing APK at $apk_path. Build the APK first." >&2 exit 1 fi adb install -r "$apk_path" + echo "Installed $package_name." } function dump_profiles() { adb logcat -c adb shell am broadcast \ - -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$PACKAGE_NAME/.PgoProfileReceiver" + -a "$INSTRUMENTED_PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.PgoProfileReceiver" wait_for_profile_dump } @@ -172,7 +176,7 @@ function record_startups() { local run for ((run = 1; run <= runs; run++)); do echo "Recording cold-process startup $run/$runs." - adb shell am force-stop "$PACKAGE_NAME" + adb shell am force-stop "$INSTRUMENTED_PACKAGE_NAME" sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" adb logcat -c adb shell am start -W -n "$START_ACTIVITY" @@ -245,19 +249,16 @@ function verify_pgo_instrumentation() ( case "${1:-}" in build-instrumented) - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=generate + build_instrumented ;; verify-instrumented) verify_pgo_instrumentation ;; install | install-instrumented) - install_apk "$INSTRUMENTED_APK_PATH" + install_apk "$INSTRUMENTED_APK_PATH" "$INSTRUMENTED_PACKAGE_NAME" ;; install-optimized) - install_apk "$OPTIMIZED_APK_PATH" + install_apk "$OPTIMIZED_APK_PATH" "$OPTIMIZED_PACKAGE_NAME" ;; record-startups) record_startups "${2:-10}" "${3:-10}" @@ -272,15 +273,7 @@ case "${1:-}" in merge_profiles ;; build-optimized) - if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then - echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 - exit 1 - fi - gradle ":assemble$BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=use \ - -PpgoProfile="$PROFILE_DIR/newdot.profdata" + build_optimized ;; *) usage From 7bf73a043a20e2ea4be59f3aa5e474a5a176b728 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 17:45:35 +0200 Subject: [PATCH 17/60] revert: separate build variants for PGO --- scripts/pgo/README.md | 12 +++--- scripts/pgo/android-local-proof.sh | 64 +++++++++++++++++------------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 8049f1f44b64..46ec0f9d5546 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,6 +1,6 @@ # Local Android PGO proof -This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 build. It uses the release-derived `pgoInstrumented` and `pgoOptimized` variants: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 release build: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. ## Preconditions @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The instrumented and optimized APKs remain available separately at `Mobile-Expensify/Android/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk`. They use the application IDs `org.me.mobiexpensifyg.pgo.instrumented` and `org.me.mobiexpensifyg.pgo.optimized`, with the launcher names “Expensify PGO Instrumented” and “Expensify PGO Optimized”, so both can remain installed simultaneously. Install the optimized APK with: + The script renames the instrumented and optimized release outputs to `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-instrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-optimized.apk`. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -61,11 +61,11 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the release and PGO-optimized APK from the same source revision, compiler, NDK, and ABI: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is renamed before the release APK is built: ```bash -scripts/pgo/android-local-proof.sh build-release scripts/pgo/android-local-proof.sh build-optimized +scripts/pgo/android-local-proof.sh build-release ``` Collect each benchmark independently. Each command installs its APK without clearing application data, performs one unmeasured warm-up, then records ten cold-process startup samples by default: @@ -88,7 +88,7 @@ Compare previously collected samples: scripts/pgo/android-local-proof.sh compare-benchmarks ``` -Or run the release benchmark first, then the optimized benchmark, and compare them in one command: +Or install and benchmark the release APK first, then install and benchmark the renamed optimized APK, and compare them in one command: ```bash scripts/pgo/android-local-proof.sh benchmark 10 30 @@ -100,4 +100,4 @@ The `.pgo/` directory is intentionally local-only. Never apply this profile to a ## Compare -Install the instrumented and optimized APKs side by side. For each build, sign in once, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. +Install the release and optimized APKs consecutively. For each build, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 867137080be8..5f5e9922ae7f 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -6,23 +6,19 @@ readonly NDK_VERSION="27.1.12297006" readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly BASE_PACKAGE_NAME="org.me.mobiexpensifyg" -readonly INSTRUMENTED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.instrumented" -readonly OPTIMIZED_PACKAGE_NAME="$BASE_PACKAGE_NAME.pgo.optimized" +readonly PACKAGE_NAME="org.me.mobiexpensifyg" readonly RELEASE_BUILD_VARIANT="Release" -readonly INSTRUMENTED_BUILD_VARIANT="PgoInstrumented" -readonly OPTIMIZED_BUILD_VARIANT="PgoOptimized" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" -readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoInstrumented/Expensify-pgoInstrumented.apk" -readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/pgoOptimized/Expensify-pgoOptimized.apk" +readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-instrumented.apk" +readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-optimized.apk" readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" -readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$INSTRUMENTED_PACKAGE_NAME/cache" -readonly START_ACTIVITY="$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.ExpensifyActivityBase" +readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" +readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" readonly APP_READY_LOG_TAG="NewDotStartup" readonly APP_READY_LOG_MESSAGE="APP_READY" readonly DEFAULT_STARTUP_RUNS=10 @@ -59,10 +55,11 @@ function gradle() { } function build_instrumented() { - gradle ":assemble$INSTRUMENTED_BUILD_VARIANT" \ + gradle ":assemble$RELEASE_BUILD_VARIANT" \ -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate + rename_release_apk "$INSTRUMENTED_APK_PATH" } function build_release() { @@ -78,30 +75,41 @@ function build_optimized() { exit 1 fi - gradle ":assemble$OPTIMIZED_BUILD_VARIANT" \ + gradle ":assemble$RELEASE_BUILD_VARIANT" \ -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ -PpgoProfile="$PROFILE_DIR/newdot.profdata" + rename_release_apk "$OPTIMIZED_APK_PATH" +} + +function rename_release_apk() { + local renamed_apk_path="$1" + if [[ ! -f "$RELEASE_APK_PATH" ]]; then + echo "Missing built release APK at $RELEASE_APK_PATH." >&2 + exit 1 + fi + + mv -f "$RELEASE_APK_PATH" "$renamed_apk_path" + echo "Renamed release APK: $renamed_apk_path" } function install_apk() { local apk_path="$1" - local package_name="$2" if [[ ! -f "$apk_path" ]]; then echo "Missing APK at $apk_path. Build the APK first." >&2 exit 1 fi adb install -r "$apk_path" - echo "Installed $package_name." + echo "Installed APK: $apk_path" } function dump_profiles() { adb logcat -c adb shell am broadcast \ - -a "$INSTRUMENTED_PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$INSTRUMENTED_PACKAGE_NAME/$BASE_PACKAGE_NAME.PgoProfileReceiver" + -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ + -n "$PACKAGE_NAME/.PgoProfileReceiver" wait_for_profile_dump } @@ -183,7 +191,7 @@ function record_startups() { local run for ((run = 1; run <= runs; run++)); do echo "Recording cold-process startup $run/$runs." - adb shell am force-stop "$INSTRUMENTED_PACKAGE_NAME" + adb shell am force-stop "$PACKAGE_NAME" sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" adb logcat -c adb shell am start -W -n "$START_ACTIVITY" @@ -226,13 +234,11 @@ function measure_startup() { function benchmark_startups() { local label="$1" - local apk_path="$2" - local benchmark_path="$3" - local runs="$4" - local ready_timeout_seconds="$5" + local benchmark_path="$2" + local runs="$3" + local ready_timeout_seconds="$4" validate_startup_arguments "$runs" "$ready_timeout_seconds" - install_apk "$apk_path" mkdir -p "$BENCHMARK_DIR" echo "Running one unmeasured warm-up startup for $label." @@ -304,19 +310,23 @@ function compare_benchmarks() { } function benchmark_release() { - benchmark_startups "release" "$RELEASE_APK_PATH" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" + install_apk "$RELEASE_APK_PATH" + benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" } function benchmark_optimized() { - benchmark_startups "PGO optimized" "$OPTIMIZED_APK_PATH" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" + install_apk "$OPTIMIZED_APK_PATH" + benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" } function benchmark_all() { local runs="${1:-$DEFAULT_STARTUP_RUNS}" local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - benchmark_release "$runs" "$ready_timeout_seconds" - benchmark_optimized "$runs" "$ready_timeout_seconds" + install_apk "$RELEASE_APK_PATH" + benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" + install_apk "$OPTIMIZED_APK_PATH" + benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" compare_benchmarks } @@ -415,13 +425,13 @@ case "${1:-}" in verify_pgo_instrumentation ;; install | install-instrumented) - install_apk "$INSTRUMENTED_APK_PATH" "$INSTRUMENTED_PACKAGE_NAME" + install_apk "$INSTRUMENTED_APK_PATH" ;; install-release) install_apk "$RELEASE_APK_PATH" ;; install-optimized) - install_apk "$OPTIMIZED_APK_PATH" "$OPTIMIZED_PACKAGE_NAME" + install_apk "$OPTIMIZED_APK_PATH" ;; record-startups) record_startups "${2:-}" "${3:-}" From 4ce4248b88cd0aef8314dd3e35f870fefa1af689 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:04:07 +0200 Subject: [PATCH 18/60] chore: move apk outputs to `.pgo` folder --- scripts/pgo/README.md | 4 ++-- scripts/pgo/android-local-proof.sh | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 46ec0f9d5546..18e7f64286c6 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The script renames the instrumented and optimized release outputs to `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-instrumented.apk` and `Mobile-Expensify/Android/build/outputs/apk/release/Expensify-release-optimized.apk`. All builds use the release application ID and are installed consecutively. Install the optimized APK with: + The script moves the instrumented and optimized release outputs to `.pgo/android/arm64-v8a/apk/Expensify-release-instrumented.apk` and `.pgo/android/arm64-v8a/apk/Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -61,7 +61,7 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is renamed before the release APK is built: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is moved into `.pgo/android/arm64-v8a/apk/` before the release APK is built: ```bash scripts/pgo/android-local-proof.sh build-optimized diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 5f5e9922ae7f..7c7da32ed25b 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -10,11 +10,12 @@ readonly PACKAGE_NAME="org.me.mobiexpensifyg" readonly RELEASE_BUILD_VARIANT="Release" readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" +readonly APK_DIR="$PROFILE_DIR/apk" readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" -readonly INSTRUMENTED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-instrumented.apk" -readonly OPTIMIZED_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release-optimized.apk" +readonly INSTRUMENTED_APK_PATH="$APK_DIR/Expensify-release-instrumented.apk" +readonly OPTIMIZED_APK_PATH="$APK_DIR/Expensify-release-optimized.apk" readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" @@ -59,7 +60,7 @@ function build_instrumented() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate - rename_release_apk "$INSTRUMENTED_APK_PATH" + archive_release_apk "$INSTRUMENTED_APK_PATH" } function build_release() { @@ -80,18 +81,19 @@ function build_optimized() { -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ -PpgoProfile="$PROFILE_DIR/newdot.profdata" - rename_release_apk "$OPTIMIZED_APK_PATH" + archive_release_apk "$OPTIMIZED_APK_PATH" } -function rename_release_apk() { - local renamed_apk_path="$1" +function archive_release_apk() { + local archived_apk_path="$1" if [[ ! -f "$RELEASE_APK_PATH" ]]; then echo "Missing built release APK at $RELEASE_APK_PATH." >&2 exit 1 fi - mv -f "$RELEASE_APK_PATH" "$renamed_apk_path" - echo "Renamed release APK: $renamed_apk_path" + mkdir -p "$APK_DIR" + mv -f "$RELEASE_APK_PATH" "$archived_apk_path" + echo "Archived release APK: $archived_apk_path" } function install_apk() { From c267adb2cfb4059328466266a78fcb0b48662f50 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:12:33 +0200 Subject: [PATCH 19/60] chore: archive release build --- scripts/pgo/README.md | 6 +++--- scripts/pgo/android-local-proof.sh | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 18e7f64286c6..c22b3a422872 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -35,7 +35,7 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android scripts/pgo/android-local-proof.sh build-optimized ``` - The script moves the instrumented and optimized release outputs to `.pgo/android/arm64-v8a/apk/Expensify-release-instrumented.apk` and `.pgo/android/arm64-v8a/apk/Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: + The script archives all release outputs under `.pgo/android/arm64-v8a/apk/`: `Expensify-release.apk`, `Expensify-release-instrumented.apk`, and `Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash scripts/pgo/android-local-proof.sh install-optimized @@ -61,7 +61,7 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Both modes initially use the normal release output path, so the optimized output is moved into `.pgo/android/arm64-v8a/apk/` before the release APK is built: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode initially uses Gradle's normal release output path, then the tool moves it into `.pgo/android/arm64-v8a/apk/`: ```bash scripts/pgo/android-local-proof.sh build-optimized @@ -88,7 +88,7 @@ Compare previously collected samples: scripts/pgo/android-local-proof.sh compare-benchmarks ``` -Or install and benchmark the release APK first, then install and benchmark the renamed optimized APK, and compare them in one command: +Or install and benchmark the archived release APK first, then install and benchmark the archived optimized APK, and compare them in one command: ```bash scripts/pgo/android-local-proof.sh benchmark 10 30 diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 7c7da32ed25b..8bdcf25afbe0 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -13,7 +13,8 @@ readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" readonly APK_DIR="$PROFILE_DIR/apk" readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" -readonly RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly GRADLE_RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" +readonly RELEASE_APK_PATH="$APK_DIR/Expensify-release.apk" readonly INSTRUMENTED_APK_PATH="$APK_DIR/Expensify-release-instrumented.apk" readonly OPTIMIZED_APK_PATH="$APK_DIR/Expensify-release-optimized.apk" readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" @@ -68,6 +69,7 @@ function build_release() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=off + archive_release_apk "$RELEASE_APK_PATH" } function build_optimized() { @@ -86,13 +88,13 @@ function build_optimized() { function archive_release_apk() { local archived_apk_path="$1" - if [[ ! -f "$RELEASE_APK_PATH" ]]; then - echo "Missing built release APK at $RELEASE_APK_PATH." >&2 + if [[ ! -f "$GRADLE_RELEASE_APK_PATH" ]]; then + echo "Missing built release APK at $GRADLE_RELEASE_APK_PATH." >&2 exit 1 fi mkdir -p "$APK_DIR" - mv -f "$RELEASE_APK_PATH" "$archived_apk_path" + mv -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" echo "Archived release APK: $archived_apk_path" } From 43425ff725d3a2c296c1660c8a2bef85016e3cef Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:24:32 +0200 Subject: [PATCH 20/60] chore: copy generated release file instead of moving it --- scripts/pgo/README.md | 2 +- scripts/pgo/android-local-proof.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index c22b3a422872..644bc125d79b 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -61,7 +61,7 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea ## Benchmark startup -Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode initially uses Gradle's normal release output path, then the tool moves it into `.pgo/android/arm64-v8a/apk/`: +Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode uses Gradle's normal release output path, then the tool copies it into `.pgo/android/arm64-v8a/apk/`. The currently built APK remains in Gradle's release directory: ```bash scripts/pgo/android-local-proof.sh build-optimized diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 8bdcf25afbe0..406fe4fd61c1 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -61,7 +61,7 @@ function build_instrumented() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=generate - archive_release_apk "$INSTRUMENTED_APK_PATH" + copy_release_apk "$INSTRUMENTED_APK_PATH" } function build_release() { @@ -69,7 +69,7 @@ function build_release() { -PpatchedArtifacts.forceBuildFromSource=true \ -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=off - archive_release_apk "$RELEASE_APK_PATH" + copy_release_apk "$RELEASE_APK_PATH" } function build_optimized() { @@ -83,10 +83,10 @@ function build_optimized() { -PreactNativeArchitectures=arm64-v8a \ -PpgoMode=use \ -PpgoProfile="$PROFILE_DIR/newdot.profdata" - archive_release_apk "$OPTIMIZED_APK_PATH" + copy_release_apk "$OPTIMIZED_APK_PATH" } -function archive_release_apk() { +function copy_release_apk() { local archived_apk_path="$1" if [[ ! -f "$GRADLE_RELEASE_APK_PATH" ]]; then echo "Missing built release APK at $GRADLE_RELEASE_APK_PATH." >&2 @@ -94,8 +94,8 @@ function archive_release_apk() { fi mkdir -p "$APK_DIR" - mv -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" - echo "Archived release APK: $archived_apk_path" + cp -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" + echo "Copied release APK: $archived_apk_path" } function install_apk() { From aa96634e2366c371be6de4bea25f85af37ef5ff6 Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:29:25 +0200 Subject: [PATCH 21/60] chore: extend PGO benchmark suite --- scripts/pgo/README.md | 2 +- scripts/pgo/android-local-proof.sh | 59 ++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 644bc125d79b..7ad4f9c868a3 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -94,7 +94,7 @@ Or install and benchmark the archived release APK first, then install and benchm scripts/pgo/android-local-proof.sh benchmark 10 30 ``` -Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the mean, median, minimum, maximum, and percentage improvement. Positive improvement percentages mean the optimized build was faster. +Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the average, P50, P75, P90, P95, P99, minimum, maximum, and percentage improvements. Percentiles use linear interpolation, and positive improvement percentages mean the optimized build was faster. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 406fe4fd61c1..994b9ac5d180 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -263,6 +263,15 @@ function benchmark_stats() { local benchmark_path="$1" awk -F, ' + function percentile(fraction, position, lower_index, remainder) { + position = 1 + (count - 1) * fraction + lower_index = int(position) + remainder = position - lower_index + if (lower_index >= count) { + return values[count] + } + return values[lower_index] + remainder * (values[lower_index + 1] - values[lower_index]) + } NR > 1 && $2 ~ /^[0-9]+$/ { values[++count] = $2 sum += $2 @@ -280,36 +289,46 @@ function benchmark_stats() { } } } - if (count % 2 == 1) { - median = values[(count + 1) / 2] - } else { - median = (values[count / 2] + values[count / 2 + 1]) / 2 - } - printf "%d %.2f %.2f %d %d\n", count, sum / count, median, values[1], values[count] + printf "%d %.2f %.2f %.2f %.2f %.2f %.2f %d %d\n", count, sum / count, percentile(0.50), percentile(0.75), percentile(0.90), percentile(0.95), percentile(0.99), values[1], values[count] } ' "$benchmark_path" } +function percentage_improvement() { + local release_value="$1" + local optimized_value="$2" + + awk -v release="$release_value" -v optimized="$optimized_value" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }' +} + function compare_benchmarks() { if [[ ! -f "$RELEASE_BENCHMARK_PATH" || ! -f "$OPTIMIZED_BENCHMARK_PATH" ]]; then echo "Missing benchmark data. Run benchmark-release and benchmark-optimized first." >&2 exit 1 fi - local release_count release_mean release_median release_min release_max - read -r release_count release_mean release_median release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") - local optimized_count optimized_mean optimized_median optimized_min optimized_max - read -r optimized_count optimized_mean optimized_median optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") - - local mean_improvement median_improvement - mean_improvement="$(awk -v release="$release_mean" -v optimized="$optimized_mean" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" - median_improvement="$(awk -v release="$release_median" -v optimized="$optimized_median" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }')" - - printf '%-18s %6s %10s %10s %8s %8s\n' 'Build' 'Runs' 'Mean ms' 'Median ms' 'Min ms' 'Max ms' - printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'Release' "$release_count" "$release_mean" "$release_median" "$release_min" "$release_max" - printf '%-18s %6d %10.2f %10.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_mean" "$optimized_median" "$optimized_min" "$optimized_max" - echo "PGO mean startup improvement: ${mean_improvement}%" - echo "PGO median startup improvement: ${median_improvement}%" + local release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max + read -r release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") + local optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max + read -r optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") + + local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement + average_improvement="$(percentage_improvement "$release_average" "$optimized_average")" + p50_improvement="$(percentage_improvement "$release_p50" "$optimized_p50")" + p75_improvement="$(percentage_improvement "$release_p75" "$optimized_p75")" + p90_improvement="$(percentage_improvement "$release_p90" "$optimized_p90")" + p95_improvement="$(percentage_improvement "$release_p95" "$optimized_p95")" + p99_improvement="$(percentage_improvement "$release_p99" "$optimized_p99")" + + printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'Build' 'Runs' 'Average' 'P50' 'P75' 'P90' 'P95' 'P99' 'Min' 'Max' + printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'Release' "$release_count" "$release_average" "$release_p50" "$release_p75" "$release_p90" "$release_p95" "$release_p99" "$release_min" "$release_max" + printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_average" "$optimized_p50" "$optimized_p75" "$optimized_p90" "$optimized_p95" "$optimized_p99" "$optimized_min" "$optimized_max" + echo "PGO average startup improvement: ${average_improvement}%" + echo "PGO P50 startup improvement: ${p50_improvement}%" + echo "PGO P75 startup improvement: ${p75_improvement}%" + echo "PGO P90 startup improvement: ${p90_improvement}%" + echo "PGO P95 startup improvement: ${p95_improvement}%" + echo "PGO P99 startup improvement: ${p99_improvement}%" echo "Positive percentages are faster; negative percentages are regressions." } From dfceb966d93a24a8e327e6031f78a800b4c9ba2a Mon Sep 17 00:00:00 2001 From: chrispader Date: Tue, 4 Aug 2026 18:49:00 +0200 Subject: [PATCH 22/60] chore: reformat benchmark output --- scripts/pgo/android-local-proof.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh index 994b9ac5d180..95e89d1f9b25 100755 --- a/scripts/pgo/android-local-proof.sh +++ b/scripts/pgo/android-local-proof.sh @@ -312,24 +312,21 @@ function compare_benchmarks() { local optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max read -r optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") - local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement + local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement min_improvement max_improvement average_improvement="$(percentage_improvement "$release_average" "$optimized_average")" p50_improvement="$(percentage_improvement "$release_p50" "$optimized_p50")" p75_improvement="$(percentage_improvement "$release_p75" "$optimized_p75")" p90_improvement="$(percentage_improvement "$release_p90" "$optimized_p90")" p95_improvement="$(percentage_improvement "$release_p95" "$optimized_p95")" p99_improvement="$(percentage_improvement "$release_p99" "$optimized_p99")" + min_improvement="$(percentage_improvement "$release_min" "$optimized_min")" + max_improvement="$(percentage_improvement "$release_max" "$optimized_max")" + echo "Positive percentages are faster; negative percentages are regressions." printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'Build' 'Runs' 'Average' 'P50' 'P75' 'P90' 'P95' 'P99' 'Min' 'Max' printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'Release' "$release_count" "$release_average" "$release_p50" "$release_p75" "$release_p90" "$release_p95" "$release_p99" "$release_min" "$release_max" printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_average" "$optimized_p50" "$optimized_p75" "$optimized_p90" "$optimized_p95" "$optimized_p99" "$optimized_min" "$optimized_max" - echo "PGO average startup improvement: ${average_improvement}%" - echo "PGO P50 startup improvement: ${p50_improvement}%" - echo "PGO P75 startup improvement: ${p75_improvement}%" - echo "PGO P90 startup improvement: ${p90_improvement}%" - echo "PGO P95 startup improvement: ${p95_improvement}%" - echo "PGO P99 startup improvement: ${p99_improvement}%" - echo "Positive percentages are faster; negative percentages are regressions." + printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'PGO improvement' '-' "${average_improvement}%" "${p50_improvement}%" "${p75_improvement}%" "${p90_improvement}%" "${p95_improvement}%" "${p99_improvement}%" "${min_improvement}%" "${max_improvement}%" } function benchmark_release() { From e25759f4c53b104296d17c85df870b8c17c2ac5b Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 10:48:08 +0200 Subject: [PATCH 23/60] refactor: migrate script to TypeScript --- scripts/pgo/README.md | 38 +- scripts/pgo/android-local-proof.sh | 487 ------------------------- scripts/pgo/local-proof.ts | 561 +++++++++++++++++++++++++++++ 3 files changed, 580 insertions(+), 506 deletions(-) delete mode 100755 scripts/pgo/android-local-proof.sh create mode 100755 scripts/pgo/local-proof.ts diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 7ad4f9c868a3..ab4e91407ebb 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,6 +1,6 @@ -# Local Android PGO proof +# Local PGO proof -This is an experimental, local-only LLVM PGO workflow for the standalone Android arm64 release build: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow. The TypeScript tool separates platform-specific operations from the shared build, profile, and benchmark workflows so iOS support can be added without duplicating the orchestration. Currently only the standalone Android arm64 release build is implemented: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. ## Preconditions @@ -14,17 +14,17 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android 1. Build and install the instrumented application: ```bash - scripts/pgo/android-local-proof.sh build-instrumented - scripts/pgo/android-local-proof.sh install-instrumented + scripts/pgo/local-proof.ts android build-instrumented + scripts/pgo/local-proof.ts android install-instrumented ``` 2. Run the manual authenticated journey once: open and scroll chats, send a message, visit and modify reversible workspace settings, attach an image and document, view attachments, then use search. 3. Persist the counters before force-stopping the application: ```bash - scripts/pgo/android-local-proof.sh dump - scripts/pgo/android-local-proof.sh pull - scripts/pgo/android-local-proof.sh merge + scripts/pgo/local-proof.ts android dump + scripts/pgo/local-proof.ts android pull + scripts/pgo/local-proof.ts android merge ``` Profiles are written to the app's external cache so they can be retrieved from a non-debuggable release APK with `adb pull`. @@ -32,13 +32,13 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android 4. Build the matched optimized application: ```bash - scripts/pgo/android-local-proof.sh build-optimized + scripts/pgo/local-proof.ts android build-optimized ``` The script archives all release outputs under `.pgo/android/arm64-v8a/apk/`: `Expensify-release.apk`, `Expensify-release-instrumented.apk`, and `Expensify-release-optimized.apk`. This keeps them safe when Gradle replaces the contents of its release output directory. The install and benchmark commands use these archived APKs. All builds use the release application ID and are installed consecutively. Install the optimized APK with: ```bash - scripts/pgo/android-local-proof.sh install-optimized + scripts/pgo/local-proof.ts android install-optimized ``` ### Collect a startup-focused profile @@ -46,13 +46,13 @@ This is an experimental, local-only LLVM PGO workflow for the standalone Android To replace the manual journey with ten cold-process startup runs, then pull and merge the accumulated profiles automatically, run: ```bash -scripts/pgo/android-local-proof.sh record-startups +scripts/pgo/local-proof.ts android record-startups ``` The optional arguments set the number of runs and the timeout for NewDot's native app-ready signal respectively: ```bash -scripts/pgo/android-local-proof.sh record-startups 10 30 +scripts/pgo/local-proof.ts android record-startups 10 30 ``` The command clears only existing `newdot-*.profraw` files on the device before the first run. It clears Logcat before each launch, waits for `NewDotStartup: APP_READY`, and fails rather than recording an incomplete startup if the marker does not appear before the timeout. LLVM's `%m` filename pattern merges each process into the same per-library raw profiles. Each process is dumped exactly once, and the final host-side `merge` converts those raw profiles into `.pgo/android/arm64-v8a/newdot.profdata`. @@ -64,34 +64,34 @@ The metrics build calls Android's `reportFullyDrawn()` and emits the machine-rea Build the PGO-optimized APK and then the release APK from the same source revision, compiler, NDK, and ABI. Every mode uses Gradle's normal release output path, then the tool copies it into `.pgo/android/arm64-v8a/apk/`. The currently built APK remains in Gradle's release directory: ```bash -scripts/pgo/android-local-proof.sh build-optimized -scripts/pgo/android-local-proof.sh build-release +scripts/pgo/local-proof.ts android build-optimized +scripts/pgo/local-proof.ts android build-release ``` Collect each benchmark independently. Each command installs its APK without clearing application data, performs one unmeasured warm-up, then records ten cold-process startup samples by default: ```bash -scripts/pgo/android-local-proof.sh benchmark-release -scripts/pgo/android-local-proof.sh benchmark-optimized +scripts/pgo/local-proof.ts android benchmark-release +scripts/pgo/local-proof.ts android benchmark-optimized ``` The optional arguments select the measured run count and app-ready timeout: ```bash -scripts/pgo/android-local-proof.sh benchmark-release 20 30 -scripts/pgo/android-local-proof.sh benchmark-optimized 20 30 +scripts/pgo/local-proof.ts android benchmark-release 20 30 +scripts/pgo/local-proof.ts android benchmark-optimized 20 30 ``` Compare previously collected samples: ```bash -scripts/pgo/android-local-proof.sh compare-benchmarks +scripts/pgo/local-proof.ts android compare-benchmarks ``` Or install and benchmark the archived release APK first, then install and benchmark the archived optimized APK, and compare them in one command: ```bash -scripts/pgo/android-local-proof.sh benchmark 10 30 +scripts/pgo/local-proof.ts android benchmark 10 30 ``` Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the average, P50, P75, P90, P95, P99, minimum, maximum, and percentage improvements. Percentiles use linear interpolation, and positive improvement percentages mean the optimized build was faster. diff --git a/scripts/pgo/android-local-proof.sh b/scripts/pgo/android-local-proof.sh deleted file mode 100755 index 95e89d1f9b25..000000000000 --- a/scripts/pgo/android-local-proof.sh +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -readonly NDK_VERSION="27.1.12297006" -readonly ANDROID_NDK_HOME="/Users/chris/Library/Android/sdk/ndk/$NDK_VERSION" - -readonly ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly PACKAGE_NAME="org.me.mobiexpensifyg" -readonly RELEASE_BUILD_VARIANT="Release" - -readonly PROFILE_DIR="$ROOT_DIR/.pgo/android/arm64-v8a" -readonly APK_DIR="$PROFILE_DIR/apk" -readonly BENCHMARK_DIR="$ROOT_DIR/.pgo/android/benchmarks" -readonly ANDROID_DIR="$ROOT_DIR/Mobile-Expensify/Android" -readonly GRADLE_RELEASE_APK_PATH="$ANDROID_DIR/build/outputs/apk/release/Expensify-release.apk" -readonly RELEASE_APK_PATH="$APK_DIR/Expensify-release.apk" -readonly INSTRUMENTED_APK_PATH="$APK_DIR/Expensify-release-instrumented.apk" -readonly OPTIMIZED_APK_PATH="$APK_DIR/Expensify-release-optimized.apk" -readonly RELEASE_BENCHMARK_PATH="$BENCHMARK_DIR/release.csv" -readonly OPTIMIZED_BENCHMARK_PATH="$BENCHMARK_DIR/pgo-optimized.csv" -readonly DEVICE_PROFILE_DIR="/sdcard/Android/data/$PACKAGE_NAME/cache" -readonly START_ACTIVITY="$PACKAGE_NAME/.ExpensifyActivityBase" -readonly APP_READY_LOG_TAG="NewDotStartup" -readonly APP_READY_LOG_MESSAGE="APP_READY" -readonly DEFAULT_STARTUP_RUNS=10 -readonly DEFAULT_APP_READY_TIMEOUT_SECONDS=30 -readonly STARTUP_RELAUNCH_DELAY_SECONDS=0.5 -readonly PROFILE_DUMP_TIMEOUT_SECONDS=5 - -APP_READY_DURATION_MS="" - -function usage() { - echo "Usage: $0 {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}" -} - -function ndk_tool() { - local tool_name="$1" - local ndk_root="${ANDROID_NDK_HOME:-${ANDROID_NDK_ROOT:-}}" - if [[ -z "$ndk_root" ]]; then - echo "Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build." >&2 - exit 1 - fi - - local host_tag="darwin-x86_64" - if [[ "$(uname -s)" == "Linux" ]]; then - host_tag="linux-x86_64" - fi - echo "$ndk_root/toolchains/llvm/prebuilt/$host_tag/bin/$tool_name" -} - -function gradle() { - ( - cd "$ANDROID_DIR" - ./gradlew "$@" - ) -} - -function build_instrumented() { - gradle ":assemble$RELEASE_BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=generate - copy_release_apk "$INSTRUMENTED_APK_PATH" -} - -function build_release() { - gradle ":assemble$RELEASE_BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=off - copy_release_apk "$RELEASE_APK_PATH" -} - -function build_optimized() { - if [[ ! -f "$PROFILE_DIR/newdot.profdata" ]]; then - echo "Missing $PROFILE_DIR/newdot.profdata. Run merge first." >&2 - exit 1 - fi - - gradle ":assemble$RELEASE_BUILD_VARIANT" \ - -PpatchedArtifacts.forceBuildFromSource=true \ - -PreactNativeArchitectures=arm64-v8a \ - -PpgoMode=use \ - -PpgoProfile="$PROFILE_DIR/newdot.profdata" - copy_release_apk "$OPTIMIZED_APK_PATH" -} - -function copy_release_apk() { - local archived_apk_path="$1" - if [[ ! -f "$GRADLE_RELEASE_APK_PATH" ]]; then - echo "Missing built release APK at $GRADLE_RELEASE_APK_PATH." >&2 - exit 1 - fi - - mkdir -p "$APK_DIR" - cp -f "$GRADLE_RELEASE_APK_PATH" "$archived_apk_path" - echo "Copied release APK: $archived_apk_path" -} - -function install_apk() { - local apk_path="$1" - if [[ ! -f "$apk_path" ]]; then - echo "Missing APK at $apk_path. Build the APK first." >&2 - exit 1 - fi - - adb install -r "$apk_path" - echo "Installed APK: $apk_path" -} - -function dump_profiles() { - adb logcat -c - adb shell am broadcast \ - -a "$PACKAGE_NAME.action.WRITE_PGO_PROFILES" \ - -n "$PACKAGE_NAME/.PgoProfileReceiver" - - wait_for_profile_dump -} - -function wait_for_profile_dump() { - local started_at="$SECONDS" - - while ((SECONDS - started_at < PROFILE_DUMP_TIMEOUT_SECONDS)); do - local profile_logs - profile_logs="$(adb logcat -d -s 'PgoProfileReceiver:I' '*:S')" - - if [[ "$profile_logs" =~ Wrote\ ([1-9][0-9]*)\ LLVM\ PGO\ profile ]]; then - echo "${BASH_REMATCH[0]}(s)." - return 0 - fi - if [[ "$profile_logs" == *"Ignoring PGO profile request in a non-instrumented build."* ]]; then - echo "The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files." >&2 - return 1 - fi - if [[ "$profile_logs" == *"Wrote 0 LLVM PGO profile(s)."* ]]; then - echo "The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles." >&2 - return 1 - fi - - sleep 0.1 - done - - echo "The PGO receiver did not confirm a profile write within ${PROFILE_DUMP_TIMEOUT_SECONDS}s." >&2 - adb logcat -d -s 'PgoProfileReceiver:I' '*:S' >&2 - return 1 -} - -function pull_profiles() { - rm -rf "$PROFILE_DIR/raw" - mkdir -p "$PROFILE_DIR/raw" - adb pull "$DEVICE_PROFILE_DIR/." "$PROFILE_DIR/raw" - - local -a profiles=() - local profile - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found in $DEVICE_PROFILE_DIR. Run dump first." >&2 - exit 1 - fi - printf '%s\n' "${profiles[@]}" -} - -function merge_profiles() { - mkdir -p "$PROFILE_DIR" - - local -a profiles=() - local profile - while IFS= read -r profile; do - profiles+=("$profile") - done < <(find "$PROFILE_DIR/raw" -name '*.profraw' -type f) - - if [[ "${#profiles[@]}" -eq 0 ]]; then - echo "No .profraw files found. Run dump and pull first." >&2 - exit 1 - fi - - "$(ndk_tool llvm-profdata)" merge --output="$PROFILE_DIR/newdot.profdata" "${profiles[@]}" - "$(ndk_tool llvm-profdata)" show --all-functions "$PROFILE_DIR/newdot.profdata" > "$PROFILE_DIR/newdot.profdata.txt" - echo "Merged PGO profile: $PROFILE_DIR/newdot.profdata" -} - -function record_startups() { - local runs="${1:-$DEFAULT_STARTUP_RUNS}" - local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - - validate_startup_arguments "$runs" "$ready_timeout_seconds" - - echo "Clearing previous device PGO profiles from $DEVICE_PROFILE_DIR." - adb shell "rm -f '$DEVICE_PROFILE_DIR'/newdot-*.profraw" - - local run - for ((run = 1; run <= runs; run++)); do - echo "Recording cold-process startup $run/$runs." - adb shell am force-stop "$PACKAGE_NAME" - sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" - adb logcat -c - adb shell am start -W -n "$START_ACTIVITY" - wait_for_app_ready "$ready_timeout_seconds" - dump_profiles - done - - pull_profiles - merge_profiles -} - -function validate_startup_arguments() { - local runs="$1" - local ready_timeout_seconds="$2" - - if [[ ! "$runs" =~ ^[1-9][0-9]*$ ]]; then - echo "Startup run count must be a positive integer, received: $runs" >&2 - exit 1 - fi - if [[ ! "$ready_timeout_seconds" =~ ^[1-9][0-9]*$ ]]; then - echo "App-ready timeout must be a positive integer, received: $ready_timeout_seconds" >&2 - exit 1 - fi -} - -function measure_startup() { - local ready_timeout_seconds="$1" - - adb shell am force-stop "$PACKAGE_NAME" - sleep "$STARTUP_RELAUNCH_DELAY_SECONDS" - adb logcat -c - adb shell am start -W -n "$START_ACTIVITY" - wait_for_app_ready "$ready_timeout_seconds" - - if [[ -z "$APP_READY_DURATION_MS" ]]; then - echo "APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes." >&2 - return 1 - fi -} - -function benchmark_startups() { - local label="$1" - local benchmark_path="$2" - local runs="$3" - local ready_timeout_seconds="$4" - - validate_startup_arguments "$runs" "$ready_timeout_seconds" - mkdir -p "$BENCHMARK_DIR" - - echo "Running one unmeasured warm-up startup for $label." - measure_startup "$ready_timeout_seconds" - - printf 'run,duration_ms\n' > "$benchmark_path" - local run - for ((run = 1; run <= runs; run++)); do - echo "Benchmarking $label cold-process startup $run/$runs." - measure_startup "$ready_timeout_seconds" - printf '%d,%s\n' "$run" "$APP_READY_DURATION_MS" >> "$benchmark_path" - done - - echo "Recorded $runs $label startup samples: $benchmark_path" -} - -function benchmark_stats() { - local benchmark_path="$1" - - awk -F, ' - function percentile(fraction, position, lower_index, remainder) { - position = 1 + (count - 1) * fraction - lower_index = int(position) - remainder = position - lower_index - if (lower_index >= count) { - return values[count] - } - return values[lower_index] + remainder * (values[lower_index + 1] - values[lower_index]) - } - NR > 1 && $2 ~ /^[0-9]+$/ { - values[++count] = $2 - sum += $2 - } - END { - if (count == 0) { - exit 1 - } - for (i = 1; i <= count; i++) { - for (j = i + 1; j <= count; j++) { - if (values[i] > values[j]) { - temporary = values[i] - values[i] = values[j] - values[j] = temporary - } - } - } - printf "%d %.2f %.2f %.2f %.2f %.2f %.2f %d %d\n", count, sum / count, percentile(0.50), percentile(0.75), percentile(0.90), percentile(0.95), percentile(0.99), values[1], values[count] - } - ' "$benchmark_path" -} - -function percentage_improvement() { - local release_value="$1" - local optimized_value="$2" - - awk -v release="$release_value" -v optimized="$optimized_value" 'BEGIN { printf "%.2f", ((release - optimized) / release) * 100 }' -} - -function compare_benchmarks() { - if [[ ! -f "$RELEASE_BENCHMARK_PATH" || ! -f "$OPTIMIZED_BENCHMARK_PATH" ]]; then - echo "Missing benchmark data. Run benchmark-release and benchmark-optimized first." >&2 - exit 1 - fi - - local release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max - read -r release_count release_average release_p50 release_p75 release_p90 release_p95 release_p99 release_min release_max < <(benchmark_stats "$RELEASE_BENCHMARK_PATH") - local optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max - read -r optimized_count optimized_average optimized_p50 optimized_p75 optimized_p90 optimized_p95 optimized_p99 optimized_min optimized_max < <(benchmark_stats "$OPTIMIZED_BENCHMARK_PATH") - - local average_improvement p50_improvement p75_improvement p90_improvement p95_improvement p99_improvement min_improvement max_improvement - average_improvement="$(percentage_improvement "$release_average" "$optimized_average")" - p50_improvement="$(percentage_improvement "$release_p50" "$optimized_p50")" - p75_improvement="$(percentage_improvement "$release_p75" "$optimized_p75")" - p90_improvement="$(percentage_improvement "$release_p90" "$optimized_p90")" - p95_improvement="$(percentage_improvement "$release_p95" "$optimized_p95")" - p99_improvement="$(percentage_improvement "$release_p99" "$optimized_p99")" - min_improvement="$(percentage_improvement "$release_min" "$optimized_min")" - max_improvement="$(percentage_improvement "$release_max" "$optimized_max")" - - echo "Positive percentages are faster; negative percentages are regressions." - printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'Build' 'Runs' 'Average' 'P50' 'P75' 'P90' 'P95' 'P99' 'Min' 'Max' - printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'Release' "$release_count" "$release_average" "$release_p50" "$release_p75" "$release_p90" "$release_p95" "$release_p99" "$release_min" "$release_max" - printf '%-18s %5d %10.2f %9.2f %9.2f %9.2f %9.2f %9.2f %8d %8d\n' 'PGO optimized' "$optimized_count" "$optimized_average" "$optimized_p50" "$optimized_p75" "$optimized_p90" "$optimized_p95" "$optimized_p99" "$optimized_min" "$optimized_max" - printf '%-18s %5s %10s %9s %9s %9s %9s %9s %8s %8s\n' 'PGO improvement' '-' "${average_improvement}%" "${p50_improvement}%" "${p75_improvement}%" "${p90_improvement}%" "${p95_improvement}%" "${p99_improvement}%" "${min_improvement}%" "${max_improvement}%" -} - -function benchmark_release() { - install_apk "$RELEASE_APK_PATH" - benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" -} - -function benchmark_optimized() { - install_apk "$OPTIMIZED_APK_PATH" - benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "${1:-$DEFAULT_STARTUP_RUNS}" "${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" -} - -function benchmark_all() { - local runs="${1:-$DEFAULT_STARTUP_RUNS}" - local ready_timeout_seconds="${2:-$DEFAULT_APP_READY_TIMEOUT_SECONDS}" - - install_apk "$RELEASE_APK_PATH" - benchmark_startups "release" "$RELEASE_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" - install_apk "$OPTIMIZED_APK_PATH" - benchmark_startups "PGO optimized" "$OPTIMIZED_BENCHMARK_PATH" "$runs" "$ready_timeout_seconds" - compare_benchmarks -} - -function wait_for_app_ready() { - local timeout_seconds="$1" - local started_at="$SECONDS" - - APP_READY_DURATION_MS="" - echo "Waiting up to ${timeout_seconds}s for $APP_READY_LOG_TAG: $APP_READY_LOG_MESSAGE." - while ((SECONDS - started_at < timeout_seconds)); do - local startup_logs - startup_logs="$(adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S')" - if [[ "$startup_logs" == *"$APP_READY_LOG_MESSAGE"* ]]; then - if [[ "$startup_logs" =~ APP_READY\ durationMs=([0-9]+) ]]; then - APP_READY_DURATION_MS="${BASH_REMATCH[1]}" - echo "NewDot reported APP_READY after ${APP_READY_DURATION_MS}ms." - else - echo "NewDot reported APP_READY without a numeric duration marker." - fi - return 0 - fi - sleep 0.25 - done - - echo "NewDot did not report APP_READY within ${timeout_seconds}s." >&2 - adb logcat -d -s "$APP_READY_LOG_TAG:I" '*:S' >&2 - return 1 -} - -function verify_pgo_instrumentation() ( - local apk_path="${1:-$INSTRUMENTED_APK_PATH}" - if [[ ! -f "$apk_path" ]]; then - echo "Missing APK at $apk_path. Build the instrumented release first." >&2 - exit 1 - fi - - local extracted_dir - extracted_dir="$(mktemp -d "${TMPDIR:-/tmp}/expensify-pgo-apk.XXXXXX")" - trap 'rm -rf "$extracted_dir"' EXIT - - local llvm_readelf - if ! llvm_readelf="$(ndk_tool llvm-readelf)"; then - exit 1 - fi - local llvm_nm - if ! llvm_nm="$(ndk_tool llvm-nm)"; then - exit 1 - fi - - local libraries=( - libreactnative.so - libhermesvm.so - libjsi.so - libExpensifyNitroUtils.so - ) - local library - for library in "${libraries[@]}"; do - local apk_entry="lib/arm64-v8a/$library" - if ! zipinfo -1 "$apk_path" | rg -Fx "$apk_entry" >/dev/null; then - echo "Missing expected arm64 library in APK: $apk_entry" >&2 - exit 1 - fi - - local extracted_library="$extracted_dir/$library" - unzip -p "$apk_path" "$apk_entry" > "$extracted_library" - local section_headers - section_headers="$("$llvm_readelf" -SW "$extracted_library")" - if [[ "$section_headers" != *"__llvm_prf_data"* || "$section_headers" != *"__llvm_prf_cnts"* || "$section_headers" != *"__llvm_prf_names"* ]]; then - echo "LLVM PGO instrumentation is missing from $apk_entry." >&2 - exit 1 - fi - - local dynamic_symbols - dynamic_symbols="$("$llvm_nm" -D --defined-only "$extracted_library")" - if [[ "$dynamic_symbols" != *"expensify_llvm_profile_set_filename"* || "$dynamic_symbols" != *"expensify_llvm_profile_write_file"* ]]; then - echo "LLVM PGO profile-writing APIs are not exported from $apk_entry." >&2 - exit 1 - fi - - if [[ "$library" == "libExpensifyNitroUtils.so" && "$dynamic_symbols" != *"Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles"* ]]; then - echo "The PGO JNI writer is missing from $apk_entry. Rebuild the native module before installing." >&2 - exit 1 - fi - echo "Verified LLVM PGO instrumentation: $apk_entry" - done -) - -case "${1:-}" in - build-release) - build_release - ;; - build-instrumented) - build_instrumented - ;; - verify-instrumented) - verify_pgo_instrumentation - ;; - install | install-instrumented) - install_apk "$INSTRUMENTED_APK_PATH" - ;; - install-release) - install_apk "$RELEASE_APK_PATH" - ;; - install-optimized) - install_apk "$OPTIMIZED_APK_PATH" - ;; - record-startups) - record_startups "${2:-}" "${3:-}" - ;; - dump) - dump_profiles - ;; - pull) - pull_profiles - ;; - merge) - merge_profiles - ;; - build-optimized) - build_optimized - ;; - benchmark-release) - benchmark_release "${2:-}" "${3:-}" - ;; - benchmark-optimized) - benchmark_optimized "${2:-}" "${3:-}" - ;; - benchmark) - benchmark_all "${2:-}" "${3:-}" - ;; - compare-benchmarks) - compare_benchmarks - ;; - *) - usage - exit 1 - ;; -esac diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts new file mode 100755 index 000000000000..07ab17346b11 --- /dev/null +++ b/scripts/pgo/local-proof.ts @@ -0,0 +1,561 @@ +#!/usr/bin/env -S node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON + +import {spawnSync} from 'node:child_process'; +import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; +import {homedir, platform, tmpdir} from 'node:os'; +import {basename, dirname, join, resolve} from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +const DEFAULT_STARTUP_RUNS = 10; +const DEFAULT_APP_READY_TIMEOUT_SECONDS = 30; +const STARTUP_RELAUNCH_DELAY_MS = 500; + +type BuildKind = 'release' | 'instrumented' | 'optimized'; +type BenchmarkKind = Extract; + +type BuildArtifactPaths = Record; + +type BenchmarkStats = { + count: number; + average: number; + p50: number; + p75: number; + p90: number; + p95: number; + p99: number; + min: number; + max: number; +}; + +type PlatformAdapter = { + readonly name: string; + readonly profileDirectory: string; + readonly rawProfileDirectory: string; + readonly mergedProfilePath: string; + readonly benchmarkDirectory: string; + readonly benchmarkPaths: Record; + readonly artifactPaths: BuildArtifactPaths; + build: (kind: BuildKind) => void; + install: (kind: BuildKind) => void; + verifyInstrumentation: () => void; + clearDeviceProfiles: () => void; + dumpProfiles: () => Promise; + pullProfiles: () => void; + forceStop: () => void; + clearSystemLogs: () => void; + launch: () => void; + waitForAppReady: (timeoutSeconds: number) => Promise; + llvmTool: (name: string) => string; +}; + +const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const rootDirectory = resolve(scriptDirectory, '../..'); + +function fail(message: string): never { + throw new Error(message); +} + +function run(command: string, args: string[], cwd = rootDirectory): void { + const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } +} + +function capture(command: string, args: string[], cwd = rootDirectory): string { + const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const stderr = result.stderr.trim(); + fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { + const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolvePromise) => { + setTimeout(resolvePromise, milliseconds); + }); +} + +function findFiles(directory: string, extension: string): string[] { + if (!existsSync(directory)) { + return []; + } + + return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return findFiles(path, extension); + } + return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; + }); +} + +function requirePositiveInteger(rawValue: string | undefined, defaultValue: number, label: string): number { + if (rawValue === undefined || rawValue.length === 0) { + return defaultValue; + } + + const value = Number(rawValue); + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive integer, received: ${rawValue}`); + } + return value; +} + +function valueAt(values: T[], index: number): T { + const value = values.at(index); + if (value === undefined) { + fail(`Expected a value at index ${index}.`); + } + return value; +} + +function createAndroidAdapter(): PlatformAdapter { + const ndkVersion = '27.1.12297006'; + const packageName = 'org.me.mobiexpensifyg'; + const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const apkDirectory = join(profileDirectory, 'apk'); + const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); + const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); + const gradleReleaseApkPath = join(androidDirectory, 'build/outputs/apk/release/Expensify-release.apk'); + const deviceProfileDirectory = `/sdcard/Android/data/${packageName}/cache`; + const startActivity = `${packageName}/.ExpensifyActivityBase`; + const appReadyLogTag = 'NewDotStartup'; + const appReadyLogMessage = 'APP_READY'; + const profileDumpTimeoutSeconds = 5; + + const artifactPaths: BuildArtifactPaths = { + release: join(apkDirectory, 'Expensify-release.apk'), + instrumented: join(apkDirectory, 'Expensify-release-instrumented.apk'), + optimized: join(apkDirectory, 'Expensify-release-optimized.apk'), + }; + + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + function adb(args: string[]): void { + run('adb', args); + } + + function adbCapture(args: string[]): string { + return capture('adb', args); + } + + function llvmTool(name: string): string { + const androidNdkHome: unknown = process.env.ANDROID_NDK_HOME; + const androidNdkRoot: unknown = process.env.ANDROID_NDK_ROOT; + const configuredNdkRoot = [androidNdkHome, androidNdkRoot].find((value): value is string => typeof value === 'string'); + const ndkRoot = configuredNdkRoot ?? join(homedir(), `Library/Android/sdk/ndk/${ndkVersion}`); + const hostTag = platform() === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; + const toolPath = join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); + if (!existsSync(toolPath)) { + fail(`Missing ${name} at ${toolPath}. Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build.`); + } + return toolPath; + } + + function archiveReleaseApk(destination: string): void { + if (!existsSync(gradleReleaseApkPath)) { + fail(`Missing built release APK at ${gradleReleaseApkPath}.`); + } + + mkdirSync(apkDirectory, {recursive: true}); + copyFileSync(gradleReleaseApkPath, destination); + console.log(`Copied release APK: ${destination}`); + } + + function build(kind: BuildKind): void { + if (kind === 'optimized' && !existsSync(join(profileDirectory, 'newdot.profdata'))) { + fail(`Missing ${join(profileDirectory, 'newdot.profdata')}. Run merge first.`); + } + + const pgoModes: Record = { + release: 'off', + instrumented: 'generate', + optimized: 'use', + }; + const pgoMode = pgoModes[kind]; + const gradleArguments = [':assembleRelease', '-PpatchedArtifacts.forceBuildFromSource=true', '-PreactNativeArchitectures=arm64-v8a', `-PpgoMode=${pgoMode}`]; + if (kind === 'optimized') { + gradleArguments.push(`-PpgoProfile=${join(profileDirectory, 'newdot.profdata')}`); + } + + run('./gradlew', gradleArguments, androidDirectory); + archiveReleaseApk(artifactPaths[kind]); + } + + function install(kind: BuildKind): void { + const apkPath = artifactPaths[kind]; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the APK first.`); + } + + adb(['install', '-r', apkPath]); + console.log(`Installed APK: ${apkPath}`); + } + + async function waitForProfileDump(): Promise { + const deadline = Date.now() + profileDumpTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + const profileCount = /Wrote ([1-9][0-9]*) LLVM PGO profile/.exec(profileLogs)?.[1]; + if (profileCount) { + console.log(`Wrote ${profileCount} LLVM PGO profile(s).`); + return; + } + if (profileLogs.includes('Ignoring PGO profile request in a non-instrumented build.')) { + fail('The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files.'); + } + if (profileLogs.includes('Wrote 0 LLVM PGO profile(s).')) { + fail('The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles.'); + } + await sleep(100); + } + + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + fail(`The PGO receiver did not confirm a profile write within ${profileDumpTimeoutSeconds}s.\n${profileLogs}`); + } + + async function dumpProfiles(): Promise { + adb(['logcat', '-c']); + adb(['shell', 'am', 'broadcast', '-a', `${packageName}.action.WRITE_PGO_PROFILES`, '-n', `${packageName}/.PgoProfileReceiver`]); + await waitForProfileDump(); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(rawProfileDirectory, {recursive: true}); + adb(['pull', `${deviceProfileDirectory}/.`, rawProfileDirectory]); + + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in ${deviceProfileDirectory}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + async function waitForAppReady(timeoutSeconds: number): Promise { + const deadline = Date.now() + timeoutSeconds * 1000; + console.log(`Waiting up to ${timeoutSeconds}s for ${appReadyLogTag}: ${appReadyLogMessage}.`); + while (Date.now() < deadline) { + const startupLogs = adbCapture(['logcat', '-d', '-s', `${appReadyLogTag}:I`, '*:S']); + if (startupLogs.includes(appReadyLogMessage)) { + const duration = /APP_READY durationMs=([0-9]+)/.exec(startupLogs)?.[1]; + if (duration) { + console.log(`NewDot reported APP_READY after ${duration}ms.`); + return Number(duration); + } + console.log('NewDot reported APP_READY without a numeric duration marker.'); + return null; + } + await sleep(250); + } + + const startupLogs = adbCapture(['logcat', '-d', '-s', `${appReadyLogTag}:I`, '*:S']); + fail(`NewDot did not report APP_READY within ${timeoutSeconds}s.\n${startupLogs}`); + } + + function verifyInstrumentation(): void { + const apkPath = artifactPaths.instrumented; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the instrumented release first.`); + } + + const extractedDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-apk-')); + const llvmReadelf = llvmTool('llvm-readelf'); + const llvmNm = llvmTool('llvm-nm'); + const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); + const libraries = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; + + try { + for (const library of libraries) { + const apkEntry = `lib/arm64-v8a/${library}`; + if (!apkEntries.has(apkEntry)) { + fail(`Missing expected arm64 library in APK: ${apkEntry}`); + } + + const extractedLibrary = join(extractedDirectory, library); + writeFileSync(extractedLibrary, captureBinary('unzip', ['-p', apkPath, apkEntry])); + const sectionHeaders = capture(llvmReadelf, ['-SW', extractedLibrary]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => sectionHeaders.includes(section))) { + fail(`LLVM PGO instrumentation is missing from ${apkEntry}.`); + } + + const dynamicSymbols = capture(llvmNm, ['-D', '--defined-only', extractedLibrary]); + if (!['expensify_llvm_profile_set_filename', 'expensify_llvm_profile_write_file'].every((symbol) => dynamicSymbols.includes(symbol))) { + fail(`LLVM PGO profile-writing APIs are not exported from ${apkEntry}.`); + } + if (library === 'libExpensifyNitroUtils.so' && !dynamicSymbols.includes('Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles')) { + fail(`The PGO JNI writer is missing from ${apkEntry}. Rebuild the native module before installing.`); + } + console.log(`Verified LLVM PGO instrumentation: ${apkEntry}`); + } + } finally { + rmSync(extractedDirectory, {recursive: true, force: true}); + } + } + + return { + name: 'android', + profileDirectory, + rawProfileDirectory, + mergedProfilePath: join(profileDirectory, 'newdot.profdata'), + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + build, + install, + verifyInstrumentation, + clearDeviceProfiles: () => { + console.log(`Clearing previous device PGO profiles from ${deviceProfileDirectory}.`); + adb(['shell', `rm -f '${deviceProfileDirectory}'/newdot-*.profraw`]); + }, + dumpProfiles, + pullProfiles, + forceStop: () => adb(['shell', 'am', 'force-stop', packageName]), + clearSystemLogs: () => adb(['logcat', '-c']), + launch: () => adb(['shell', 'am', 'start', '-W', '-n', startActivity]), + waitForAppReady, + llvmTool, + }; +} + +function mergeProfiles(adapter: PlatformAdapter): void { + mkdirSync(adapter.profileDirectory, {recursive: true}); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail('No .profraw files found. Run dump and pull first.'); + } + + const llvmProfdata = adapter.llvmTool('llvm-profdata'); + run(llvmProfdata, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + const profileReport = capture(llvmProfdata, ['show', '--all-functions', adapter.mergedProfilePath]); + writeFileSync(`${adapter.mergedProfilePath}.txt`, profileReport); + console.log(`Merged PGO profile: ${adapter.mergedProfilePath}`); +} + +async function measureStartup(adapter: PlatformAdapter, readyTimeoutSeconds: number): Promise { + adapter.forceStop(); + await sleep(STARTUP_RELAUNCH_DELAY_MS); + adapter.clearSystemLogs(); + adapter.launch(); + const duration = await adapter.waitForAppReady(readyTimeoutSeconds); + if (duration === null) { + fail('APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes.'); + } + return duration; +} + +async function recordStartups(adapter: PlatformAdapter, runs: number, readyTimeoutSeconds: number): Promise { + adapter.clearDeviceProfiles(); + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + console.log(`Recording cold-process startup ${runNumber}/${runs}.`); + await measureStartup(adapter, readyTimeoutSeconds); + await adapter.dumpProfiles(); + } + + adapter.pullProfiles(); + mergeProfiles(adapter); +} + +async function benchmarkStartups(adapter: PlatformAdapter, kind: BenchmarkKind, runs: number, readyTimeoutSeconds: number): Promise { + const label = kind === 'release' ? 'release' : 'PGO optimized'; + const benchmarkPath = adapter.benchmarkPaths[kind]; + mkdirSync(adapter.benchmarkDirectory, {recursive: true}); + + console.log(`Running one unmeasured warm-up startup for ${label}.`); + await measureStartup(adapter, readyTimeoutSeconds); + + const samples: number[] = []; + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + console.log(`Benchmarking ${label} cold-process startup ${runNumber}/${runs}.`); + samples.push(await measureStartup(adapter, readyTimeoutSeconds)); + } + + const csv = ['run,duration_ms', ...samples.map((duration, index) => `${index + 1},${duration}`), ''].join('\n'); + writeFileSync(benchmarkPath, csv); + console.log(`Recorded ${runs} ${label} startup samples: ${benchmarkPath}`); +} + +function percentile(sortedValues: number[], fraction: number): number { + const position = (sortedValues.length - 1) * fraction; + const lowerIndex = Math.floor(position); + const upperIndex = Math.ceil(position); + const remainder = position - lowerIndex; + const lowerValue = valueAt(sortedValues, lowerIndex); + return lowerValue + remainder * (valueAt(sortedValues, upperIndex) - lowerValue); +} + +function benchmarkStats(path: string): BenchmarkStats { + const values = readFileSync(path, 'utf8') + .trim() + .split('\n') + .slice(1) + .map((line) => Number(line.split(',').at(1))) + .filter(Number.isFinite) + .sort((left, right) => left - right); + if (values.length === 0) { + fail(`No benchmark samples found in ${path}.`); + } + + return { + count: values.length, + average: values.reduce((sum, value) => sum + value, 0) / values.length, + p50: percentile(values, 0.5), + p75: percentile(values, 0.75), + p90: percentile(values, 0.9), + p95: percentile(values, 0.95), + p99: percentile(values, 0.99), + min: valueAt(values, 0), + max: valueAt(values, -1), + }; +} + +function percentageImprovement(releaseValue: number, optimizedValue: number): number { + return ((releaseValue - optimizedValue) / releaseValue) * 100; +} + +function compareBenchmarks(adapter: PlatformAdapter): void { + const releasePath = adapter.benchmarkPaths.release; + const optimizedPath = adapter.benchmarkPaths.optimized; + if (!existsSync(releasePath) || !existsSync(optimizedPath)) { + fail('Missing benchmark data. Run benchmark-release and benchmark-optimized first.'); + } + + const release = benchmarkStats(releasePath); + const optimized = benchmarkStats(optimizedPath); + const numericColumns: Array> = ['average', 'p50', 'p75', 'p90', 'p95', 'p99', 'min', 'max']; + const improvements = numericColumns.map((key) => percentageImprovement(release[key], optimized[key])); + + const formatLabel = (value: string) => value.padEnd(18); + const formatCount = (value: string) => value.padStart(5); + const formatMetric = (value: string) => value.padStart(10); + const row = (label: string, count: string, values: string[]) => [formatLabel(label), formatCount(count), ...values.map(formatMetric)].join(' '); + const statsRow = (label: string, stats: BenchmarkStats) => + row( + label, + String(stats.count), + [stats.average, stats.p50, stats.p75, stats.p90, stats.p95, stats.p99, stats.min, stats.max].map((value) => value.toFixed(2)), + ); + + console.log('Positive percentages are faster; negative percentages are regressions.'); + console.log(row('Build', 'Runs', ['Average', 'P50', 'P75', 'P90', 'P95', 'P99', 'Min', 'Max'])); + console.log(statsRow('Release', release)); + console.log(statsRow('PGO optimized', optimized)); + console.log( + row( + 'PGO improvement', + '-', + improvements.map((improvement) => `${improvement.toFixed(2)}%`), + ), + ); +} + +function printUsage(): void { + console.log( + `Usage: ${basename(process.argv.at(1) ?? 'local-proof.ts')} android {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}`, + ); +} + +function getAdapter(platformName: string | undefined): PlatformAdapter { + if (platformName === 'android') { + return createAndroidAdapter(); + } + if (platformName === 'ios') { + fail('The iOS PGO workflow is not implemented yet.'); + } + + printUsage(); + fail(`Unsupported or missing platform: ${platformName ?? '(none)'}`); +} + +async function main(): Promise { + const [, , platformName, command, runsArgument, timeoutArgument] = process.argv; + const adapter = getAdapter(platformName); + const runs = requirePositiveInteger(runsArgument, DEFAULT_STARTUP_RUNS, 'Startup run count'); + const timeoutSeconds = requirePositiveInteger(timeoutArgument, DEFAULT_APP_READY_TIMEOUT_SECONDS, 'App-ready timeout'); + + switch (command) { + case 'build-release': + adapter.build('release'); + return; + case 'build-instrumented': + adapter.build('instrumented'); + return; + case 'build-optimized': + adapter.build('optimized'); + return; + case 'verify-instrumented': + adapter.verifyInstrumentation(); + return; + case 'install-release': + adapter.install('release'); + return; + case 'install-instrumented': + adapter.install('instrumented'); + return; + case 'install-optimized': + adapter.install('optimized'); + return; + case 'record-startups': + await recordStartups(adapter, runs, timeoutSeconds); + return; + case 'benchmark-release': + adapter.install('release'); + await benchmarkStartups(adapter, 'release', runs, timeoutSeconds); + return; + case 'benchmark-optimized': + adapter.install('optimized'); + await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); + return; + case 'benchmark': + adapter.install('release'); + await benchmarkStartups(adapter, 'release', runs, timeoutSeconds); + adapter.install('optimized'); + await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); + compareBenchmarks(adapter); + return; + case 'compare-benchmarks': + compareBenchmarks(adapter); + return; + case 'dump': + await adapter.dumpProfiles(); + return; + case 'pull': + adapter.pullProfiles(); + return; + case 'merge': + mergeProfiles(adapter); + return; + default: + printUsage(); + fail(`Unsupported or missing command: ${command ?? '(none)'}`); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); From 04f20ae06dea0c930fb97f5070ae66faa1917d8c Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 11:10:33 +0200 Subject: [PATCH 24/60] refactor: use CLI tool in the PGO tool --- scripts/pgo/local-proof.ts | 118 +++++++++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 07ab17346b11..55f18fd02c46 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -1,11 +1,13 @@ #!/usr/bin/env -S node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON +import type {TupleToUnion} from 'type-fest'; + +import CLI from 'expensify-common/CLI'; import {spawnSync} from 'node:child_process'; import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; import {homedir, platform, tmpdir} from 'node:os'; -import {basename, dirname, join, resolve} from 'node:path'; +import {dirname, join, resolve} from 'node:path'; import process from 'node:process'; -import {fileURLToPath} from 'node:url'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_APP_READY_TIMEOUT_SECONDS = 30; @@ -13,6 +15,8 @@ const STARTUP_RELAUNCH_DELAY_MS = 500; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; +type PlatformName = TupleToUnion; +type WorkflowCommand = TupleToUnion; type BuildArtifactPaths = Record; @@ -49,7 +53,30 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const PLATFORM_NAMES = ['android', 'ios'] as const; +const WORKFLOW_COMMANDS = [ + 'build-release', + 'build-instrumented', + 'build-optimized', + 'verify-instrumented', + 'install-release', + 'install-instrumented', + 'install-optimized', + 'record-startups', + 'benchmark-release', + 'benchmark-optimized', + 'benchmark', + 'compare-benchmarks', + 'dump', + 'pull', + 'merge', +] as const; + +const scriptPath = process.argv.at(1); +if (!scriptPath) { + throw new Error('Unable to resolve the PGO script path.'); +} +const scriptDirectory = dirname(resolve(scriptPath)); const rootDirectory = resolve(scriptDirectory, '../..'); function fail(message: string): never { @@ -109,18 +136,25 @@ function findFiles(directory: string, extension: string): string[] { }); } -function requirePositiveInteger(rawValue: string | undefined, defaultValue: number, label: string): number { - if (rawValue === undefined || rawValue.length === 0) { - return defaultValue; - } - - const value = Number(rawValue); +function requirePositiveInteger(value: number, label: string): number { if (!Number.isSafeInteger(value) || value <= 0) { - fail(`${label} must be a positive integer, received: ${rawValue}`); + fail(`${label} must be a positive integer, received: ${value}`); } return value; } +function parsePositiveInteger(rawValue: string, label: string): number { + return requirePositiveInteger(Number(rawValue), label); +} + +function parseChoice(rawValue: string, choices: readonly T[], label: string): T { + const choice = choices.find((candidate) => candidate === rawValue); + if (!choice) { + fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); + } + return choice; +} + function valueAt(values: T[], index: number): T { const value = values.at(index); if (value === undefined) { @@ -473,31 +507,17 @@ function compareBenchmarks(adapter: PlatformAdapter): void { ); } -function printUsage(): void { - console.log( - `Usage: ${basename(process.argv.at(1) ?? 'local-proof.ts')} android {build-release|build-instrumented|build-optimized|verify-instrumented|install-release|install-instrumented|install-optimized|record-startups [runs] [ready-timeout-seconds]|benchmark-release [runs] [ready-timeout-seconds]|benchmark-optimized [runs] [ready-timeout-seconds]|benchmark [runs] [ready-timeout-seconds]|compare-benchmarks|dump|pull|merge}`, - ); -} - -function getAdapter(platformName: string | undefined): PlatformAdapter { +function getAdapter(platformName: PlatformName): PlatformAdapter { if (platformName === 'android') { return createAndroidAdapter(); } - if (platformName === 'ios') { - fail('The iOS PGO workflow is not implemented yet.'); - } - - printUsage(); - fail(`Unsupported or missing platform: ${platformName ?? '(none)'}`); + fail('The iOS PGO workflow is not implemented yet.'); } -async function main(): Promise { - const [, , platformName, command, runsArgument, timeoutArgument] = process.argv; +async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number): Promise { const adapter = getAdapter(platformName); - const runs = requirePositiveInteger(runsArgument, DEFAULT_STARTUP_RUNS, 'Startup run count'); - const timeoutSeconds = requirePositiveInteger(timeoutArgument, DEFAULT_APP_READY_TIMEOUT_SECONDS, 'App-ready timeout'); - switch (command) { + switch (workflow) { case 'build-release': adapter.build('release'); return; @@ -548,13 +568,49 @@ async function main(): Promise { return; case 'merge': mergeProfiles(adapter); - return; + break; default: - printUsage(); - fail(`Unsupported or missing command: ${command ?? '(none)'}`); + fail('Unsupported workflow.'); } } +async function main(): Promise { + const cli = new CLI({ + positionalArgs: [ + { + name: 'platform', + description: `Native platform to target (${PLATFORM_NAMES.join(', ')})`, + parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform'), + }, + { + name: 'workflow', + description: `PGO workflow to run (${WORKFLOW_COMMANDS.join(', ')})`, + parse: (value): WorkflowCommand => parseChoice(value, WORKFLOW_COMMANDS, 'Workflow'), + }, + { + name: 'runs', + description: 'Number of measured startup runs', + default: DEFAULT_STARTUP_RUNS, + parse: (value) => parsePositiveInteger(value, 'Startup run count'), + }, + { + name: 'timeout', + description: 'Seconds to wait for the app-ready marker', + default: DEFAULT_APP_READY_TIMEOUT_SECONDS, + parse: (value) => parsePositiveInteger(value, 'App-ready timeout'), + }, + ], + }); + + const {platform: platformName, workflow, runs, timeout} = cli.positionalArgs; + await runWorkflow( + parseChoice(String(platformName), PLATFORM_NAMES, 'Platform'), + parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), + requirePositiveInteger(Number(runs), 'Startup run count'), + requirePositiveInteger(Number(timeout), 'App-ready timeout'), + ); +} + main().catch((error: unknown) => { console.error(error instanceof Error ? error.message : error); process.exitCode = 1; From c68accc8a96721d442ab6c60b96157225fdfe3a1 Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 15:31:36 +0200 Subject: [PATCH 25/60] feat: implement iOS PGO --- ios/NewExpensify/RCTBootSplash.h | 1 + ios/NewExpensify/RCTBootSplash.mm | 112 +++++ .../react-native+0.85.3+040+pgo.patch | 43 ++ scripts/pgo/README.md | 60 ++- scripts/pgo/local-proof.ts | 440 +++++++++++++++++- scripts/pod-install.sh | 5 +- 6 files changed, 651 insertions(+), 10 deletions(-) diff --git a/ios/NewExpensify/RCTBootSplash.h b/ios/NewExpensify/RCTBootSplash.h index f499fe59f916..29b22858a561 100644 --- a/ios/NewExpensify/RCTBootSplash.h +++ b/ios/NewExpensify/RCTBootSplash.h @@ -4,6 +4,7 @@ + (void)initWithStoryboard:(NSString * _Nonnull)storyboardName rootView:(UIView * _Nullable)rootView; ++ (void)initializePGOProfileCollection; + (void)hide:(BOOL)fade; + (void)bringSubviewToFrontIfInitialized; diff --git a/ios/NewExpensify/RCTBootSplash.mm b/ios/NewExpensify/RCTBootSplash.mm index c2cff4c5b833..0ba896813d56 100644 --- a/ios/NewExpensify/RCTBootSplash.mm +++ b/ios/NewExpensify/RCTBootSplash.mm @@ -13,6 +13,22 @@ #import #import +#ifndef EXPENSIFY_PGO_GENERATE +#define EXPENSIFY_PGO_GENERATE 0 +#endif + +#if EXPENSIFY_PGO_GENERATE +extern "C" void __llvm_profile_set_filename(const char *FilenamePat); +extern "C" int __llvm_profile_write_file(void); +extern "C" void __llvm_profile_reset_counters(void); +#endif + +static NSString *const PGO_DIRECTORY_NAME = @"ExpensifyPGO"; +static NSString *const PGO_APP_READY_MARKER = @"app-ready.txt"; +static NSString *const PGO_STATUS_MARKER = @"profile-status.txt"; +static CFStringRef const PGO_WRITE_NOTIFICATION = CFSTR("com.expensify.pgo.write-profiles"); +static CFStringRef const PGO_CLEAR_NOTIFICATION = CFSTR("com.expensify.pgo.clear-profiles"); + static RCTSurfaceHostingProxyRootView *_rootView = nil; static UIView *_loadingView = nil; @@ -20,6 +36,29 @@ static bool _fade = false; static bool _nativeHidden = false; +static long long CurrentTimeMilliseconds(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000); +} + +@interface RCTBootSplash () ++ (NSURL *)pgoDirectoryURL; ++ (void)writePGOStatus:(NSString *)status result:(int)result; ++ (void)writePGOProfiles; ++ (void)clearPGOProfiles; +@end + +static void HandlePGONotification(__unused CFNotificationCenterRef center, + __unused void *observer, + CFStringRef name, + __unused const void *object, + __unused CFDictionaryRef userInfo) { + if (CFEqual(name, PGO_WRITE_NOTIFICATION)) { + [RCTBootSplash writePGOProfiles]; + } else if (CFEqual(name, PGO_CLEAR_NOTIFICATION)) { + [RCTBootSplash clearPGOProfiles]; + } +} + @implementation RCTBootSplash RCT_EXPORT_MODULE(); @@ -32,6 +71,69 @@ + (BOOL)requiresMainQueueSetup { return NO; } ++ (NSURL *)pgoDirectoryURL { + NSURL *cachesURL = [[[NSFileManager defaultManager] URLsForDirectory:NSCachesDirectory + inDomains:NSUserDomainMask] firstObject]; + NSURL *directoryURL = [cachesURL URLByAppendingPathComponent:PGO_DIRECTORY_NAME isDirectory:YES]; + [[NSFileManager defaultManager] createDirectoryAtURL:directoryURL + withIntermediateDirectories:YES + attributes:nil + error:nil]; + return directoryURL; +} + ++ (void)writePGOStatus:(NSString *)status result:(int)result { + NSString *contents = [NSString stringWithFormat:@"%lld,%@,%d", CurrentTimeMilliseconds(), status, result]; + NSURL *statusURL = [[self pgoDirectoryURL] URLByAppendingPathComponent:PGO_STATUS_MARKER]; + [contents writeToURL:statusURL atomically:YES encoding:NSUTF8StringEncoding error:nil]; +} + ++ (void)initializePGOProfileCollection { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + CFNotificationCenterRef notificationCenter = CFNotificationCenterGetDarwinNotifyCenter(); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_WRITE_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_CLEAR_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + +#if EXPENSIFY_PGO_GENERATE + NSString *profilePattern = [[[self pgoDirectoryURL] URLByAppendingPathComponent:@"newdot-%m.profraw"] path]; + __llvm_profile_set_filename(profilePattern.fileSystemRepresentation); +#endif + }); +} + ++ (void)writePGOProfiles { +#if EXPENSIFY_PGO_GENERATE + int result = __llvm_profile_write_file(); + [self writePGOStatus:@"written" result:result]; + NSLog(@"ExpensifyPGO: wrote LLVM profile with result=%d", result); +#else + [self writePGOStatus:@"not-instrumented" result:1]; + NSLog(@"ExpensifyPGO: ignored profile write in a non-instrumented build"); +#endif +} + ++ (void)clearPGOProfiles { + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSURL *directoryURL = [self pgoDirectoryURL]; + NSArray *files = [fileManager contentsOfDirectoryAtURL:directoryURL + includingPropertiesForKeys:nil + options:0 + error:nil]; + for (NSURL *fileURL in files) { + if ([fileURL.lastPathComponent hasSuffix:@".profraw"] || [fileURL.lastPathComponent isEqualToString:PGO_STATUS_MARKER]) { + [fileManager removeItemAtURL:fileURL error:nil]; + } + } + +#if EXPENSIFY_PGO_GENERATE + __llvm_profile_reset_counters(); + [self writePGOStatus:@"cleared" result:0]; +#else + [self writePGOStatus:@"not-instrumented" result:1]; +#endif +} + + (bool)isLoadingViewVisible { return _loadingView != nil && ![_loadingView isHidden]; } @@ -202,4 +304,14 @@ - (void)hideImpl:(BOOL)fade [self hideImpl:0 resolve:resolve]; } +RCT_EXPORT_METHOD(reportFullyDrawn) { + double startedAt = [[NSUserDefaults standardUserDefaults] doubleForKey:@"AppStartTime"]; + long long readyAt = CurrentTimeMilliseconds(); + long long duration = startedAt > 0 ? MAX(0, readyAt - (long long)startedAt) : 0; + NSString *contents = [NSString stringWithFormat:@"%lld,%lld", readyAt, duration]; + NSURL *markerURL = [[RCTBootSplash pgoDirectoryURL] URLByAppendingPathComponent:PGO_APP_READY_MARKER]; + [contents writeToURL:markerURL atomically:YES encoding:NSUTF8StringEncoding error:nil]; + NSLog(@"NewDotStartup: APP_READY durationMs=%lld", duration); +} + @end diff --git a/patches/react-native/react-native+0.85.3+040+pgo.patch b/patches/react-native/react-native+0.85.3+040+pgo.patch index 14ca0c45eb91..ae4cd4becbf6 100644 --- a/patches/react-native/react-native+0.85.3+040+pgo.patch +++ b/patches/react-native/react-native+0.85.3+040+pgo.patch @@ -139,3 +139,46 @@ index 8125e02..d466798 100644 include(CheckIPOSupported) check_ipo_supported(RESULT IPO_SUPPORT) if (IPO_SUPPORT) +diff --git a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +--- a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh ++++ b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +@@ -59,6 +59,19 @@ if [[ $xcode_major_version -ge 15 ]]; then + xcode_15_flags="LINKER:-ld_classic" + fi + ++pgo_compile_flags="" ++pgo_linker_flags="" ++if [[ "${EXPENSIFY_PGO_MODE:-off}" == "generate" ]]; then ++ pgo_compile_flags="-fprofile-instr-generate" ++ pgo_linker_flags="-fprofile-instr-generate" ++elif [[ "${EXPENSIFY_PGO_MODE:-off}" == "use" ]]; then ++ if [[ -z "${EXPENSIFY_PGO_PROFILE:-}" ]]; then ++ echo "EXPENSIFY_PGO_PROFILE is required when EXPENSIFY_PGO_MODE=use" >&2 ++ exit 1 ++ fi ++ pgo_compile_flags="-fprofile-instr-use=${EXPENSIFY_PGO_PROFILE} -Wno-profile-instr-unprofiled -Wno-profile-instr-out-of-date" ++fi ++ + boost_context_flag="" + if [[ $PLATFORM_NAME == "catalyst" ]]; then + boost_context_flag="-DHERMES_ALLOW_BOOST_CONTEXT=0" +@@ -70,7 +84,7 @@ echo "Configure Apple framework" + "$CMAKE_BINARY" \ + -S "${PODS_ROOT}/hermes-engine" \ + -B "${PODS_ROOT}/hermes-engine/build/${PLATFORM_NAME}" \ +- -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags" \ ++ -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags;$pgo_linker_flags" \ + -DHERMES_APPLE_TARGET_PLATFORM:STRING="$PLATFORM_NAME" \ + -DCMAKE_OSX_ARCHITECTURES:STRING="$architectures" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING="$deployment_target" \ +@@ -82,8 +96,8 @@ echo "Configure Apple framework" + -DHERMES_ENABLE_BITCODE:BOOLEAN=false \ + -DHERMES_BUILD_APPLE_FRAMEWORK:BOOLEAN=true \ + -DHERMES_BUILD_SHARED_JSI:BOOLEAN=false \ +- -DCMAKE_CXX_FLAGS:STRING="-gdwarf" \ +- -DCMAKE_C_FLAGS:STRING="-gdwarf" \ ++ -DCMAKE_CXX_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ ++ -DCMAKE_C_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ + -DIMPORT_HOST_COMPILERS:PATH="${hermesc_path}" \ + -DJSI_DIR="$jsi_path" \ + -DHERMES_RELEASE_VERSION="for RN $release_version" \ diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index ab4e91407ebb..d9bf17e07284 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,14 +1,24 @@ # Local PGO proof -This is an experimental, local-only LLVM PGO workflow. The TypeScript tool separates platform-specific operations from the shared build, profile, and benchmark workflows so iOS support can be added without duplicating the orchestration. Currently only the standalone Android arm64 release build is implemented: optimized native code and a bundled production JavaScript bundle, signed with the local debug key so it can be installed without release credentials. It does not publish artifacts and it does not add application metrics. +This is an experimental, local-only LLVM PGO workflow for Android and iOS. The TypeScript tool shares the build, profile, merge, and benchmark orchestration while each platform adapter handles its native build system and physical-device tooling. It builds React Native, Hermes, Nitro modules, and other source-based native dependencies from source so they can participate in PGO. Precompiled vendored frameworks cannot be instrumented retroactively. + +The workflow does not publish artifacts. It uses the existing NewDot `APP_READY` point immediately after the `ManualAppStartup` Sentry span ends as both the profile boundary and startup benchmark marker. ## Preconditions -- A physical arm64 Android device connected through `adb`. -- `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. +- A physical arm64 Android or iOS device. Simulators are not suitable for collecting a production-device PGO profile. +- For Android, `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. +- For iOS, Xcode command-line tools, a connected and unlocked device, and locally valid Apple Development signing settings for the app and its remaining extensions. The tool respects the project development team by default; override it with `IOS_DEVELOPMENT_TEAM` and select a device with `IOS_DEVICE_ID` when necessary. It reads the locally signed bundle identifier from the archived `.app`; `IOS_BUNDLE_IDENTIFIER` is available as an explicit override. - A signed-in, seeded test account and the agreed manual NewDot journey. - Dependencies have had the repository's patches applied. +Every command starts with a platform: + +```bash +scripts/pgo/local-proof.ts android --help +scripts/pgo/local-proof.ts ios --help +``` + ## Collect one profile 1. Build and install the instrumented application: @@ -101,3 +111,47 @@ The `.pgo/` directory is intentionally local-only. Never apply this profile to a ## Compare Install the release and optimized APKs consecutively. For each build, exclude the first post-install run, force-stop before every subsequent run, and record ten repetitions of the same journey with `am start -W` and Perfetto. Keep attachment-upload latency diagnostic only; use cold start and local interaction/frame timing as the primary decision metrics. + +## iOS workflow + +The iOS commands mirror Android: + +```bash +scripts/pgo/local-proof.ts ios build-instrumented +scripts/pgo/local-proof.ts ios verify-instrumented +scripts/pgo/local-proof.ts ios install-instrumented +scripts/pgo/local-proof.ts ios record-startups 10 30 +scripts/pgo/local-proof.ts ios build-optimized +scripts/pgo/local-proof.ts ios build-release +scripts/pgo/local-proof.ts ios benchmark 10 30 +``` + +`record-startups` launches the installed instrumented app, clears old counters, performs the requested cold-process startups, explicitly flushes LLVM counters after every `APP_READY`, copies the raw profiles from the app data container, and merges them into `.pgo/ios/arm64/newdot.profdata`. The app must remain in the foreground until each dump completes; force-terminating an iOS app is not a reliable profile-flush mechanism. + +For a broader manual profile, use the same instrumented build, perform the agreed authenticated NewDot journey, and then run: + +```bash +scripts/pgo/local-proof.ts ios dump +scripts/pgo/local-proof.ts ios pull +scripts/pgo/local-proof.ts ios merge +scripts/pgo/local-proof.ts ios build-optimized +``` + +The iOS adapter uses an arm64 Release device build with local development signing. It forces React Native core, React Native dependencies, and Hermes to build from source. Xcode-level Clang and Swift frontend instrumentation covers app and source-based CocoaPods targets; a React Native patch forwards the matching flags into Hermes' nested CMake build. Release, instrumented, and optimized `.app` bundles are archived under `.pgo/ios/arm64/app/` and benchmarks under `.pgo/ios/benchmarks/`. + +The source-build preparation also validates the source roots CocoaPods needs for Hermes, `libdav1d`, and `libwebp`. If an existing generated checkout is incomplete, the tool removes only that Pod directory and lets the locked `pod install` restore it before compiling. This prevents missing private-header failures caused by a partially materialized CocoaPods sandbox. + +If signing is temporarily unavailable, `IOS_CODE_SIGNING_ALLOWED=NO scripts/pgo/local-proof.ts ios build-instrumented` builds an unsigned device artifact for compilation and instrumentation verification only. Unsigned apps cannot be installed, profiled, or benchmarked; the connected device and all app-extension bundle identifiers must be covered by valid development provisioning profiles for the complete workflow. + +The three-stage procedure is the same on both platforms, but the mechanics differ: + +| Stage | Android | iOS | +| --- | --- | --- | +| Build | Gradle/NDK | CocoaPods + `xcodebuild` | +| Instrument | Clang `-fprofile-generate` | Clang `-fprofile-instr-generate`, Swift `-profile-generate` | +| Flush | Android broadcast/JNI | Darwin notification/native LLVM runtime | +| Retrieve | `adb pull` from external cache | CoreDevice copy from the app data container | +| Merge | NDK `llvm-profdata` | Xcode `llvm-profdata` | +| Consume | Clang `-fprofile-use` | Clang `-fprofile-instr-use`, Swift `-profile-use` | + +Profiles are tied to the exact compiler, architecture, source, dependencies, and build configuration that produced them. Rebuild all three app variants from the same revision and Xcode version, do not reuse an Android profile on iOS, and regenerate the iOS profile after meaningful native-source or toolchain changes. The release and optimized benchmarks preserve app data and use the same bundle identifier so the authenticated state and data set remain comparable. diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 55f18fd02c46..2385734cc56c 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -4,7 +4,7 @@ import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; import {spawnSync} from 'node:child_process'; -import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; +import {copyFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; import {homedir, platform, tmpdir} from 'node:os'; import {dirname, join, resolve} from 'node:path'; import process from 'node:process'; @@ -15,6 +15,7 @@ const STARTUP_RELAUNCH_DELAY_MS = 500; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; +type PgoMode = 'off' | 'generate' | 'use'; type PlatformName = TupleToUnion; type WorkflowCommand = TupleToUnion; @@ -43,7 +44,8 @@ type PlatformAdapter = { build: (kind: BuildKind) => void; install: (kind: BuildKind) => void; verifyInstrumentation: () => void; - clearDeviceProfiles: () => void; + prepareProfileCollection?: (timeoutSeconds: number) => Promise; + clearDeviceProfiles: () => Promise; dumpProfiles: () => Promise; pullProfiles: () => void; forceStop: () => void; @@ -116,6 +118,11 @@ function captureBinary(command: string, args: string[], cwd = rootDirectory): Bu return result.stdout; } +function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { + const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); + return !result.error && result.status === 0; +} + function sleep(milliseconds: number): Promise { return new Promise((resolvePromise) => { setTimeout(resolvePromise, milliseconds); @@ -155,6 +162,15 @@ function parseChoice(rawValue: string, choices: readonly T[], return choice; } +function environmentString(name: string): string | undefined { + const value: unknown = process.env[name]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + function valueAt(values: T[], index: number): T { const value = values.at(index); if (value === undefined) { @@ -362,7 +378,7 @@ function createAndroidAdapter(): PlatformAdapter { build, install, verifyInstrumentation, - clearDeviceProfiles: () => { + clearDeviceProfiles: async () => { console.log(`Clearing previous device PGO profiles from ${deviceProfileDirectory}.`); adb(['shell', `rm -f '${deviceProfileDirectory}'/newdot-*.profraw`]); }, @@ -376,6 +392,415 @@ function createAndroidAdapter(): PlatformAdapter { }; } +function createIosAdapter(): PlatformAdapter { + const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const appDirectory = join(profileDirectory, 'app'); + const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); + const markerDirectory = join(profileDirectory, 'markers'); + const iosDirectory = join(rootDirectory, 'Mobile-Expensify/iOS'); + const hermesVersionPath = join(rootDirectory, 'node_modules/react-native/sdks/.hermesv1version'); + const remotePgoDirectory = 'Library/Caches/ExpensifyPGO'; + const profileStatusPath = `${remotePgoDirectory}/profile-status.txt`; + const appReadyMarkerPath = `${remotePgoDirectory}/app-ready.txt`; + const profileWriteNotification = 'com.expensify.pgo.write-profiles'; + const profileClearNotification = 'com.expensify.pgo.clear-profiles'; + const profileOperationTimeoutSeconds = 10; + const developmentTeam = environmentString('IOS_DEVELOPMENT_TEAM'); + const codeSigningAllowed = !['0', 'false', 'no'].includes((environmentString('IOS_CODE_SIGNING_ALLOWED') ?? 'yes').toLowerCase()); + const configuredBundleIdentifier = environmentString('IOS_BUNDLE_IDENTIFIER'); + const mergedProfilePath = join(profileDirectory, 'newdot.profdata'); + + const artifactPaths: BuildArtifactPaths = { + release: join(appDirectory, 'Expensify-release.app'), + instrumented: join(appDirectory, 'Expensify-release-instrumented.app'), + optimized: join(appDirectory, 'Expensify-release-optimized.app'), + }; + + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + let cachedDeviceIdentifier: string | undefined; + let cachedBundleIdentifier: string | undefined; + let previousAppReadyMarker: string | undefined; + + function bundleIdentifier(): string { + if (cachedBundleIdentifier) { + return cachedBundleIdentifier; + } + if (configuredBundleIdentifier) { + cachedBundleIdentifier = configuredBundleIdentifier; + return configuredBundleIdentifier; + } + + const archivedApp = [artifactPaths.instrumented, artifactPaths.optimized, artifactPaths.release].find((path) => existsSync(join(path, 'Info.plist'))); + if (!archivedApp) { + fail('No archived iOS app is available to determine its bundle identifier. Build an app first, or set IOS_BUNDLE_IDENTIFIER.'); + } + cachedBundleIdentifier = capture('/usr/libexec/PlistBuddy', ['-c', 'Print :CFBundleIdentifier', join(archivedApp, 'Info.plist')]).trim(); + if (!cachedBundleIdentifier) { + fail(`The archived app at ${archivedApp} has no CFBundleIdentifier.`); + } + console.log(`Using iOS bundle identifier ${cachedBundleIdentifier}.`); + return cachedBundleIdentifier; + } + + function deviceIdentifier(): string { + if (cachedDeviceIdentifier) { + return cachedDeviceIdentifier; + } + + const configuredIdentifier = environmentString('IOS_DEVICE_ID'); + if (configuredIdentifier) { + cachedDeviceIdentifier = configuredIdentifier; + return configuredIdentifier; + } + + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-ios-devices-')); + const jsonPath = join(temporaryDirectory, 'devices.json'); + try { + run('xcrun', ['devicectl', 'list', 'devices', '--json-output', jsonPath]); + const parsedDevices: unknown = JSON.parse(readFileSync(jsonPath, 'utf8')); + if (!isRecord(parsedDevices) || !isRecord(parsedDevices.result) || !Array.isArray(parsedDevices.result.devices)) { + fail('CoreDevice returned an unexpected device-list response.'); + } + + const physicalDevices = parsedDevices.result.devices.flatMap((device) => { + if (!isRecord(device) || !isRecord(device.hardwareProperties) || !isRecord(device.deviceProperties)) { + return []; + } + const {hardwareProperties, deviceProperties} = device; + if (hardwareProperties.platform !== 'iOS' || hardwareProperties.reality !== 'physical' || deviceProperties.bootState !== 'booted') { + return []; + } + return typeof hardwareProperties.udid === 'string' ? [{name: String(deviceProperties.name), udid: hardwareProperties.udid}] : []; + }); + + if (physicalDevices.length === 0) { + fail('No booted physical iOS device is connected. Connect and unlock a device, or set IOS_DEVICE_ID.'); + } + if (physicalDevices.length > 1) { + fail(`Multiple physical iOS devices are connected (${physicalDevices.map(({name}) => name).join(', ')}). Set IOS_DEVICE_ID to the desired UDID.`); + } + + const selectedDevice = valueAt(physicalDevices, 0); + cachedDeviceIdentifier = selectedDevice.udid; + console.log(`Using iOS device ${selectedDevice.name} (${selectedDevice.udid}).`); + return selectedDevice.udid; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } + } + + function llvmTool(name: string): string { + return capture('xcrun', ['--find', name]).trim(); + } + + function sourceBuildEnvironment(mode: PgoMode): string[] { + const environment = ['RCT_USE_RN_DEP=0', 'RCT_USE_PREBUILT_RNCORE=0', 'RCT_BUILD_HERMES_FROM_SOURCE=true', `EXPENSIFY_PGO_MODE=${mode}`]; + if (mode === 'use') { + environment.push(`EXPENSIFY_PGO_PROFILE=${mergedProfilePath}`); + } + return environment; + } + + function prepareSourcePods(mode: PgoMode): void { + if (!existsSync(hermesVersionPath)) { + fail(`Missing React Native's pinned Hermes version file at ${hermesVersionPath}.`); + } + + const requiredPodSources = [ + {directory: join(iosDirectory, 'Pods/hermes-engine'), requiredFile: 'CMakeLists.txt'}, + {directory: join(iosDirectory, 'Pods/libdav1d'), requiredFile: 'dav1d/include/common/intops.h'}, + {directory: join(iosDirectory, 'Pods/libwebp'), requiredFile: 'src/webp/types.h'}, + ]; + for (const {directory, requiredFile} of requiredPodSources) { + if (existsSync(directory) && !existsSync(join(directory, requiredFile))) { + console.log(`Removing incomplete CocoaPods checkout: ${directory}`); + rmSync(directory, {recursive: true, force: true}); + } + } + + run('/usr/bin/env', [...sourceBuildEnvironment(mode), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); + + for (const {directory, requiredFile} of requiredPodSources) { + if (!existsSync(join(directory, requiredFile))) { + fail(`CocoaPods did not install the required source file ${join(directory, requiredFile)}.`); + } + } + + const hermesBuildDirectory = join(iosDirectory, 'Pods/hermes-engine/build'); + rmSync(hermesBuildDirectory, {recursive: true, force: true}); + } + + function pgoBuildSettings(kind: BuildKind): string[] { + if (kind === 'instrumented') { + return [ + 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', + 'OTHER_CFLAGS=$(inherited) -fprofile-instr-generate', + 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -profile-generate', + 'OTHER_LDFLAGS=$(inherited) -fprofile-instr-generate', + ]; + } + if (kind === 'optimized') { + const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-profile-instr-unprofiled -Wno-profile-instr-out-of-date`; + return [ + 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_SWIFT_FLAGS=$(inherited) -profile-use=${mergedProfilePath}`, + ]; + } + return ['CLANG_USE_OPTIMIZATION_PROFILE=NO']; + } + + function build(kind: BuildKind): void { + if (kind === 'optimized' && !existsSync(mergedProfilePath)) { + fail(`Missing ${mergedProfilePath}. Run merge first.`); + } + + let mode: PgoMode = 'off'; + if (kind === 'instrumented') { + mode = 'generate'; + } else if (kind === 'optimized') { + mode = 'use'; + } + prepareSourcePods(mode); + + const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); + const builtAppPath = join(derivedDataDirectory, 'Build/Products/Release-iphoneos/Expensify.app'); + const signingSettings = codeSigningAllowed + ? [ + 'CODE_SIGN_STYLE=Automatic', + 'CODE_SIGN_IDENTITY=Apple Development', + 'PROVISIONING_PROFILE=', + 'PROVISIONING_PROFILE_SPECIFIER=', + ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), + ] + : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; + const xcodeArguments = [ + 'xcodebuild', + '-workspace', + 'Expensify.xcworkspace', + '-scheme', + 'Expensify', + '-configuration', + 'Release', + '-destination', + `id=${deviceIdentifier()}`, + '-derivedDataPath', + derivedDataDirectory, + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), + '-quiet', + 'build', + 'ARCHS=arm64', + 'ONLY_ACTIVE_ARCH=YES', + 'COMPILER_INDEX_STORE_ENABLE=NO', + ...signingSettings, + ...pgoBuildSettings(kind), + ]; + run('/usr/bin/env', [...sourceBuildEnvironment(mode), ...xcodeArguments], iosDirectory); + + if (!existsSync(builtAppPath)) { + fail(`Missing built iOS app at ${builtAppPath}.`); + } + mkdirSync(appDirectory, {recursive: true}); + rmSync(artifactPaths[kind], {recursive: true, force: true}); + cpSync(builtAppPath, artifactPaths[kind], {recursive: true, preserveTimestamps: true}); + console.log(`Copied release app: ${artifactPaths[kind]}`); + } + + function install(kind: BuildKind): void { + const appPath = artifactPaths[kind]; + if (!existsSync(appPath)) { + fail(`Missing app at ${appPath}. Build it first.`); + } + if (!existsSync(join(appPath, '_CodeSignature/CodeResources'))) { + fail(`The app at ${appPath} is unsigned. Rebuild without IOS_CODE_SIGNING_ALLOWED=NO before installing it on a device.`); + } + run('xcrun', ['devicectl', 'device', 'install', 'app', '--device', deviceIdentifier(), appPath]); + console.log(`Installed app: ${appPath}`); + } + + function tryReadDeviceFile(remotePath: string, localName: string): string | undefined { + mkdirSync(markerDirectory, {recursive: true}); + const localPath = join(markerDirectory, localName); + rmSync(localPath, {recursive: true, force: true}); + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePath, + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8').trim() : undefined; + } + + function postNotification(name: string): void { + run('xcrun', ['devicectl', 'device', 'notification', 'post', '--device', deviceIdentifier(), '--name', name]); + } + + async function waitForProfileStatus(expectedStatus: string, previousStatus: string | undefined): Promise { + const deadline = Date.now() + profileOperationTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const statusContents = tryReadDeviceFile(profileStatusPath, 'profile-status.txt'); + if (statusContents && statusContents !== previousStatus) { + const [, status, rawResult] = statusContents.split(','); + if (status === 'not-instrumented') { + fail('The installed iOS app is not instrumented. Build and install build-instrumented before recording profiles.'); + } + if (status === expectedStatus && Number(rawResult) === 0) { + console.log(`iOS PGO profiles ${expectedStatus}.`); + return; + } + fail(`The iOS PGO profile operation failed: ${statusContents}`); + } + await sleep(250); + } + fail(`The iOS app did not confirm that PGO profiles were ${expectedStatus} within ${profileOperationTimeoutSeconds}s.`); + } + + async function clearDeviceProfiles(): Promise { + console.log(`Clearing previous iOS PGO profiles from ${remotePgoDirectory}.`); + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-clear.txt'); + postNotification(profileClearNotification); + await waitForProfileStatus('cleared', previousStatus); + } + + async function dumpProfiles(): Promise { + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-write.txt'); + postNotification(profileWriteNotification); + await waitForProfileStatus('written', previousStatus); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(dirname(rawProfileDirectory), {recursive: true}); + run('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePgoDirectory, + '--destination', + rawProfileDirectory, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + ]); + + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in the iOS app container at ${remotePgoDirectory}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + function forceStop(): void { + runAllowFailure('xcrun', ['devicectl', 'device', 'process', 'terminate', '--device', deviceIdentifier(), bundleIdentifier()]); + } + + function launch(): void { + previousAppReadyMarker = tryReadDeviceFile(appReadyMarkerPath, 'app-ready-before-launch.txt'); + run('xcrun', ['devicectl', 'device', 'process', 'launch', '--device', deviceIdentifier(), '--terminate-existing', bundleIdentifier()]); + } + + async function waitForAppReady(timeoutSeconds: number): Promise { + const deadline = Date.now() + timeoutSeconds * 1000; + console.log(`Waiting up to ${timeoutSeconds}s for the iOS NewDot APP_READY marker.`); + while (Date.now() < deadline) { + const marker = tryReadDeviceFile(appReadyMarkerPath, 'app-ready.txt'); + if (marker && marker !== previousAppReadyMarker) { + const [, rawDuration] = marker.split(','); + const duration = Number(rawDuration); + if (Number.isFinite(duration) && duration >= 0) { + console.log(`NewDot reported APP_READY after ${duration}ms.`); + return duration; + } + return null; + } + await sleep(250); + } + fail(`NewDot did not report APP_READY on iOS within ${timeoutSeconds}s.`); + } + + function verifyInstrumentation(): void { + const appPath = artifactPaths.instrumented; + const executablePath = join(appPath, 'Expensify'); + if (!existsSync(executablePath)) { + fail(`Missing instrumented iOS executable at ${executablePath}.`); + } + + const sectionHeaders = capture('xcrun', ['size', '-m', executablePath]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => sectionHeaders.includes(section))) { + fail('LLVM PGO instrumentation is missing from the iOS app executable.'); + } + const executableStrings = capture('strings', ['-a', executablePath]); + if (!['ExpensifyPGO: wrote LLVM profile', 'NewDotStartup: APP_READY'].every((marker) => executableStrings.includes(marker))) { + fail('The iOS app executable is missing the PGO flush or APP_READY marker.'); + } + console.log(`Verified LLVM PGO instrumentation: ${executablePath}`); + + const hermesExecutable = join(appPath, 'Frameworks/hermesvm.framework/hermesvm'); + if (!existsSync(hermesExecutable)) { + fail(`Missing embedded source-built Hermes framework at ${hermesExecutable}.`); + } + const hermesSections = capture('xcrun', ['size', '-m', hermesExecutable]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => hermesSections.includes(section))) { + fail('LLVM PGO instrumentation is missing from the source-built iOS Hermes framework.'); + } + console.log(`Verified LLVM PGO instrumentation: ${hermesExecutable}`); + } + + async function prepareProfileCollection(timeoutSeconds: number): Promise { + console.log('Launching the instrumented iOS app once so its PGO notification handlers are active.'); + forceStop(); + await sleep(STARTUP_RELAUNCH_DELAY_MS); + launch(); + await waitForAppReady(timeoutSeconds); + } + + return { + name: 'ios', + profileDirectory, + rawProfileDirectory, + mergedProfilePath, + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + build, + install, + verifyInstrumentation, + prepareProfileCollection, + clearDeviceProfiles, + dumpProfiles, + pullProfiles, + forceStop, + clearSystemLogs: () => undefined, + launch, + waitForAppReady, + llvmTool, + }; +} + function mergeProfiles(adapter: PlatformAdapter): void { mkdirSync(adapter.profileDirectory, {recursive: true}); const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); @@ -397,13 +822,16 @@ async function measureStartup(adapter: PlatformAdapter, readyTimeoutSeconds: num adapter.launch(); const duration = await adapter.waitForAppReady(readyTimeoutSeconds); if (duration === null) { - fail('APP_READY did not contain a numeric durationMs marker. Rebuild the APK with the startup metric changes.'); + fail(`APP_READY did not contain a numeric durationMs marker. Rebuild the ${adapter.name} app with the startup metric changes.`); } return duration; } async function recordStartups(adapter: PlatformAdapter, runs: number, readyTimeoutSeconds: number): Promise { - adapter.clearDeviceProfiles(); + if (adapter.prepareProfileCollection) { + await adapter.prepareProfileCollection(readyTimeoutSeconds); + } + await adapter.clearDeviceProfiles(); for (let runNumber = 1; runNumber <= runs; runNumber += 1) { console.log(`Recording cold-process startup ${runNumber}/${runs}.`); await measureStartup(adapter, readyTimeoutSeconds); @@ -511,7 +939,7 @@ function getAdapter(platformName: PlatformName): PlatformAdapter { if (platformName === 'android') { return createAndroidAdapter(); } - fail('The iOS PGO workflow is not implemented yet.'); + return createIosAdapter(); } async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number): Promise { diff --git a/scripts/pod-install.sh b/scripts/pod-install.sh index b0739b28d4a4..fbbb1ce890cb 100755 --- a/scripts/pod-install.sh +++ b/scripts/pod-install.sh @@ -54,7 +54,10 @@ if [[ "$IS_HYBRID_APP_REPO" == "true" && "$NEW_DOT_FLAG" == "false" ]]; then # Navigate to the OldDot repository, and run bundle install and pod install cd Mobile-Expensify/ios bundle install - RCT_USE_RN_DEP=0 RCT_USE_PREBUILT_RNCORE=0 bundle exec pod install + RCT_USE_RN_DEP=0\ + RCT_USE_PREBUILT_RNCORE=0\ + RCT_BUILD_HERMES_FROM_SOURCE=true\ + bundle exec pod install exit 0 fi From fa69456727635b15cf1ff87bd16845895a628d20 Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 18:20:23 +0200 Subject: [PATCH 26/60] fix: ios optimized build --- scripts/pgo/README.md | 6 +++--- scripts/pgo/local-proof.ts | 26 ++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index d9bf17e07284..c7fc898dfcff 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -137,7 +137,7 @@ scripts/pgo/local-proof.ts ios merge scripts/pgo/local-proof.ts ios build-optimized ``` -The iOS adapter uses an arm64 Release device build with local development signing. It forces React Native core, React Native dependencies, and Hermes to build from source. Xcode-level Clang and Swift frontend instrumentation covers app and source-based CocoaPods targets; a React Native patch forwards the matching flags into Hermes' nested CMake build. Release, instrumented, and optimized `.app` bundles are archived under `.pgo/ios/arm64/app/` and benchmarks under `.pgo/ios/benchmarks/`. +The iOS adapter uses an arm64 Release device build with local development signing. It forces React Native core, React Native dependencies, and Hermes to build from source. Xcode-level Clang frontend instrumentation and Swift IR instrumentation cover app and source-based CocoaPods targets; a React Native patch forwards the matching flags into Hermes' nested CMake build. Release, instrumented, and optimized `.app` bundles are archived under `.pgo/ios/arm64/app/` and benchmarks under `.pgo/ios/benchmarks/`. The source-build preparation also validates the source roots CocoaPods needs for Hermes, `libdav1d`, and `libwebp`. If an existing generated checkout is incomplete, the tool removes only that Pod directory and lets the locked `pod install` restore it before compiling. This prevents missing private-header failures caused by a partially materialized CocoaPods sandbox. @@ -148,10 +148,10 @@ The three-stage procedure is the same on both platforms, but the mechanics diffe | Stage | Android | iOS | | --- | --- | --- | | Build | Gradle/NDK | CocoaPods + `xcodebuild` | -| Instrument | Clang `-fprofile-generate` | Clang `-fprofile-instr-generate`, Swift `-profile-generate` | +| Instrument | Clang `-fprofile-generate` | Clang `-fprofile-instr-generate`, Swift `-ir-profile-generate` | | Flush | Android broadcast/JNI | Darwin notification/native LLVM runtime | | Retrieve | `adb pull` from external cache | CoreDevice copy from the app data container | | Merge | NDK `llvm-profdata` | Xcode `llvm-profdata` | -| Consume | Clang `-fprofile-use` | Clang `-fprofile-instr-use`, Swift `-profile-use` | +| Consume | Clang `-fprofile-use` | Clang `-fprofile-instr-use`, Swift `-ir-profile-use` | Profiles are tied to the exact compiler, architecture, source, dependencies, and build configuration that produced them. Rebuild all three app variants from the same revision and Xcode version, do not reuse an Android profile on iOS, and regenerate the iOS profile after meaningful native-source or toolchain changes. The release and optimized benchmarks preserve app data and use the same bundle identifier so the authenticated state and data set remain comparable. diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 2385734cc56c..32b8c7a44c14 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -41,6 +41,7 @@ type PlatformAdapter = { readonly benchmarkDirectory: string; readonly benchmarkPaths: Record; readonly artifactPaths: BuildArtifactPaths; + readonly profileFormat?: string; build: (kind: BuildKind) => void; install: (kind: BuildKind) => void; verifyInstrumentation: () => void; @@ -393,6 +394,7 @@ function createAndroidAdapter(): PlatformAdapter { } function createIosAdapter(): PlatformAdapter { + const profileFormat = 'ios-clang-frontend-swift-ir-v1'; const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); const rawProfileDirectory = join(profileDirectory, 'raw'); const appDirectory = join(profileDirectory, 'app'); @@ -536,32 +538,40 @@ function createIosAdapter(): PlatformAdapter { } function pgoBuildSettings(kind: BuildKind): string[] { + const baseSettings = ['ENABLE_CODE_COVERAGE=NO', 'CLANG_COVERAGE_MAPPING=NO', 'CLANG_USE_OPTIMIZATION_PROFILE=NO']; if (kind === 'instrumented') { return [ - 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + ...baseSettings, 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', 'OTHER_CFLAGS=$(inherited) -fprofile-instr-generate', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-generate', - 'OTHER_SWIFT_FLAGS=$(inherited) -profile-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-generate', 'OTHER_LDFLAGS=$(inherited) -fprofile-instr-generate', ]; } if (kind === 'optimized') { const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-profile-instr-unprofiled -Wno-profile-instr-out-of-date`; return [ - 'CLANG_USE_OPTIMIZATION_PROFILE=NO', + ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, - `OTHER_SWIFT_FLAGS=$(inherited) -profile-use=${mergedProfilePath}`, + `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${mergedProfilePath}`, ]; } - return ['CLANG_USE_OPTIMIZATION_PROFILE=NO']; + return baseSettings; } function build(kind: BuildKind): void { if (kind === 'optimized' && !existsSync(mergedProfilePath)) { fail(`Missing ${mergedProfilePath}. Run merge first.`); } + if (kind === 'optimized') { + const profileFormatPath = `${mergedProfilePath}.format`; + const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; + if (recordedFormat !== profileFormat) { + fail('The merged iOS profile predates the current Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); + } + } let mode: PgoMode = 'off'; if (kind === 'instrumented') { @@ -591,7 +601,7 @@ function createIosAdapter(): PlatformAdapter { '-configuration', 'Release', '-destination', - `id=${deviceIdentifier()}`, + 'generic/platform=iOS', '-derivedDataPath', derivedDataDirectory, ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), @@ -786,6 +796,7 @@ function createIosAdapter(): PlatformAdapter { benchmarkDirectory, benchmarkPaths, artifactPaths, + profileFormat, build, install, verifyInstrumentation, @@ -810,6 +821,9 @@ function mergeProfiles(adapter: PlatformAdapter): void { const llvmProfdata = adapter.llvmTool('llvm-profdata'); run(llvmProfdata, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + if (adapter.profileFormat) { + writeFileSync(`${adapter.mergedProfilePath}.format`, `${adapter.profileFormat}\n`); + } const profileReport = capture(llvmProfdata, ['show', '--all-functions', adapter.mergedProfilePath]); writeFileSync(`${adapter.mergedProfilePath}.txt`, profileReport); console.log(`Merged PGO profile: ${adapter.mergedProfilePath}`); From 5ddd179edffa7463a9e4f31d2840b4c26d735eff Mon Sep 17 00:00:00 2001 From: chrispader Date: Wed, 5 Aug 2026 18:28:06 +0200 Subject: [PATCH 27/60] fix: benchmark release first --- scripts/pgo/local-proof.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 32b8c7a44c14..7e263a05cd3e 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -875,6 +875,21 @@ async function benchmarkStartups(adapter: PlatformAdapter, kind: BenchmarkKind, console.log(`Recorded ${runs} ${label} startup samples: ${benchmarkPath}`); } +async function benchmarkAll(adapter: PlatformAdapter, runs: number, readyTimeoutSeconds: number): Promise { + console.log('=== Benchmark phase 1/2: Release ==='); + console.log(`Installing release artifact: ${adapter.artifactPaths.release}`); + adapter.install('release'); + await benchmarkStartups(adapter, 'release', runs, readyTimeoutSeconds); + + console.log('=== Benchmark phase 2/2: PGO optimized ==='); + console.log(`Installing PGO optimized artifact: ${adapter.artifactPaths.optimized}`); + adapter.install('optimized'); + await benchmarkStartups(adapter, 'optimized', runs, readyTimeoutSeconds); + + console.log('=== Benchmark comparison ==='); + compareBenchmarks(adapter); +} + function percentile(sortedValues: number[], fraction: number): number { const position = (sortedValues.length - 1) * fraction; const lowerIndex = Math.floor(position); @@ -993,11 +1008,7 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); return; case 'benchmark': - adapter.install('release'); - await benchmarkStartups(adapter, 'release', runs, timeoutSeconds); - adapter.install('optimized'); - await benchmarkStartups(adapter, 'optimized', runs, timeoutSeconds); - compareBenchmarks(adapter); + await benchmarkAll(adapter, runs, timeoutSeconds); return; case 'compare-benchmarks': compareBenchmarks(adapter); From 22a19cd57bf91882be7468a80a825ae886c2a633 Mon Sep 17 00:00:00 2001 From: chrispader Date: Thu, 6 Aug 2026 15:07:47 +0200 Subject: [PATCH 28/60] feat: allow defining device --- scripts/pgo/local-proof.ts | 61 +++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/scripts/pgo/local-proof.ts b/scripts/pgo/local-proof.ts index 7e263a05cd3e..6823d63bffec 100755 --- a/scripts/pgo/local-proof.ts +++ b/scripts/pgo/local-proof.ts @@ -180,7 +180,7 @@ function valueAt(values: T[], index: number): T { return value; } -function createAndroidAdapter(): PlatformAdapter { +function createAndroidAdapter(configuredDeviceIdentifier?: string): PlatformAdapter { const ndkVersion = '27.1.12297006'; const packageName = 'org.me.mobiexpensifyg'; const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); @@ -207,11 +207,11 @@ function createAndroidAdapter(): PlatformAdapter { }; function adb(args: string[]): void { - run('adb', args); + run('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); } function adbCapture(args: string[]): string { - return capture('adb', args); + return capture('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); } function llvmTool(name: string): string { @@ -393,7 +393,7 @@ function createAndroidAdapter(): PlatformAdapter { }; } -function createIosAdapter(): PlatformAdapter { +function createIosAdapter(cliDeviceIdentifier?: string): PlatformAdapter { const profileFormat = 'ios-clang-frontend-swift-ir-v1'; const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); const rawProfileDirectory = join(profileDirectory, 'raw'); @@ -454,12 +454,7 @@ function createIosAdapter(): PlatformAdapter { return cachedDeviceIdentifier; } - const configuredIdentifier = environmentString('IOS_DEVICE_ID'); - if (configuredIdentifier) { - cachedDeviceIdentifier = configuredIdentifier; - return configuredIdentifier; - } - + const configuredIdentifier = cliDeviceIdentifier ?? environmentString('IOS_DEVICE_ID'); const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-ios-devices-')); const jsonPath = join(temporaryDirectory, 'devices.json'); try { @@ -477,17 +472,40 @@ function createIosAdapter(): PlatformAdapter { if (hardwareProperties.platform !== 'iOS' || hardwareProperties.reality !== 'physical' || deviceProperties.bootState !== 'booted') { return []; } - return typeof hardwareProperties.udid === 'string' ? [{name: String(deviceProperties.name), udid: hardwareProperties.udid}] : []; + if (typeof hardwareProperties.udid !== 'string' || typeof deviceProperties.name !== 'string') { + return []; + } + const name = deviceProperties.name.trim(); + const osVersion = typeof deviceProperties.osVersionNumber === 'string' ? deviceProperties.osVersionNumber : undefined; + const aliases = [name, hardwareProperties.udid]; + if (typeof device.identifier === 'string') { + aliases.push(device.identifier); + } + if (typeof hardwareProperties.serialNumber === 'string') { + aliases.push(hardwareProperties.serialNumber); + } + if (osVersion) { + aliases.push(`${name} (${osVersion})`, `${name} (${osVersion}) (${hardwareProperties.udid})`); + } + return [{aliases, name, udid: hardwareProperties.udid}]; }); if (physicalDevices.length === 0) { fail('No booted physical iOS device is connected. Connect and unlock a device, or set IOS_DEVICE_ID.'); } - if (physicalDevices.length > 1) { + + const normalizeDeviceSelector = (value: string) => value.trim().replaceAll(/\s+/g, ' ').toLowerCase(); + const matchingDevices = configuredIdentifier + ? physicalDevices.filter(({aliases}) => aliases.some((alias) => normalizeDeviceSelector(alias) === normalizeDeviceSelector(configuredIdentifier))) + : physicalDevices; + if (matchingDevices.length === 0) { + fail(`No connected physical iOS device matches "${configuredIdentifier}". Available devices: ${physicalDevices.map(({name, udid}) => `${name} (${udid})`).join(', ')}.`); + } + if (matchingDevices.length > 1) { fail(`Multiple physical iOS devices are connected (${physicalDevices.map(({name}) => name).join(', ')}). Set IOS_DEVICE_ID to the desired UDID.`); } - const selectedDevice = valueAt(physicalDevices, 0); + const selectedDevice = valueAt(matchingDevices, 0); cachedDeviceIdentifier = selectedDevice.udid; console.log(`Using iOS device ${selectedDevice.name} (${selectedDevice.udid}).`); return selectedDevice.udid; @@ -964,15 +982,15 @@ function compareBenchmarks(adapter: PlatformAdapter): void { ); } -function getAdapter(platformName: PlatformName): PlatformAdapter { +function getAdapter(platformName: PlatformName, deviceIdentifier?: string): PlatformAdapter { if (platformName === 'android') { - return createAndroidAdapter(); + return createAndroidAdapter(deviceIdentifier); } - return createIosAdapter(); + return createIosAdapter(deviceIdentifier); } -async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number): Promise { - const adapter = getAdapter(platformName); +async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number, deviceIdentifier?: string): Promise { + const adapter = getAdapter(platformName, deviceIdentifier); switch (workflow) { case 'build-release': @@ -1053,6 +1071,12 @@ async function main(): Promise { parse: (value) => parsePositiveInteger(value, 'App-ready timeout'), }, ], + namedArgs: { + device: { + description: 'Device identifier to use (adb serial on Android; CoreDevice identifier, UDID, serial number, or device name on iOS)', + required: false, + }, + }, }); const {platform: platformName, workflow, runs, timeout} = cli.positionalArgs; @@ -1061,6 +1085,7 @@ async function main(): Promise { parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), requirePositiveInteger(Number(runs), 'Startup run count'), requirePositiveInteger(Number(timeout), 'App-ready timeout'), + cli.namedArgs.device, ); } From d35b7f5ef5c44f582c210ab0fe251c1f8ed402c0 Mon Sep 17 00:00:00 2001 From: chrispader Date: Thu, 6 Aug 2026 15:09:26 +0200 Subject: [PATCH 29/60] Update README.md --- scripts/pgo/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index c7fc898dfcff..981b5b803c6a 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -8,7 +8,7 @@ The workflow does not publish artifacts. It uses the existing NewDot `APP_READY` - A physical arm64 Android or iOS device. Simulators are not suitable for collecting a production-device PGO profile. - For Android, `ANDROID_NDK_HOME` points at NDK `27.1.12297006`, the version pinned in `Mobile-Expensify/Android/build.gradle`. -- For iOS, Xcode command-line tools, a connected and unlocked device, and locally valid Apple Development signing settings for the app and its remaining extensions. The tool respects the project development team by default; override it with `IOS_DEVELOPMENT_TEAM` and select a device with `IOS_DEVICE_ID` when necessary. It reads the locally signed bundle identifier from the archived `.app`; `IOS_BUNDLE_IDENTIFIER` is available as an explicit override. +- For iOS, Xcode command-line tools, a connected and unlocked device, and locally valid Apple Development signing settings for the app and its remaining extensions. The tool respects the project development team by default; override it with `IOS_DEVELOPMENT_TEAM`. Select a device with `--device`; `IOS_DEVICE_ID` remains available as an environment fallback. It reads the locally signed bundle identifier from the archived `.app`; `IOS_BUNDLE_IDENTIFIER` is available as an explicit override. - A signed-in, seeded test account and the agreed manual NewDot journey. - Dependencies have had the repository's patches applied. @@ -104,6 +104,18 @@ Or install and benchmark the archived release APK first, then install and benchm scripts/pgo/local-proof.ts android benchmark 10 30 ``` +Select a specific connected device for install or benchmark commands with `--device`. Use an adb serial for Android and any identifier accepted by CoreDevice for iOS, such as a CoreDevice identifier, UDID, serial number, or device name: + +```bash +scripts/pgo/local-proof.ts --device emulator-5554 android install-optimized +scripts/pgo/local-proof.ts --device emulator-5554 android benchmark 10 30 +scripts/pgo/local-proof.ts --device Chris14Pro ios install-optimized +scripts/pgo/local-proof.ts --device Chris14Pro ios benchmark 10 30 +scripts/pgo/local-proof.ts --device "Chris14Pro (26.6) (00008120-00065D541E3B401E)" ios benchmark 10 30 +``` + +Quote selectors containing spaces or parentheses. The iOS resolver accepts either the plain device name or the complete physical-device entry printed under `== Devices ==` by `xcrun xctrace list devices`; it does not select entries from the simulator section. + Raw samples are stored in `.pgo/android/benchmarks/release.csv` and `.pgo/android/benchmarks/pgo-optimized.csv`. The comparison reports the average, P50, P75, P90, P95, P99, minimum, maximum, and percentage improvements. Percentiles use linear interpolation, and positive improvement percentages mean the optimized build was faster. The `.pgo/` directory is intentionally local-only. Never apply this profile to another ABI, build mode, NDK version, or substantially different source revision. A production-release comparison additionally needs the repository's R8/SafetyNet dependency issue fixed; this local proof deliberately does not change that unrelated configuration. From 9befe34fdb832f100984705b5e69c5f6c4b8a389 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 12:59:42 +0200 Subject: [PATCH 30/60] chore: allow native PGO terminology in spelling checks --- modules/ExpensifyNitroUtils/android/CMakeLists.txt | 2 ++ scripts/pgo/README.md | 2 ++ scripts/pgo/android.ts | 2 ++ scripts/pgo/ios.ts | 2 ++ scripts/pgo/pgo.ts | 2 ++ tests/tooling/pgo.test.ts | 1 + 6 files changed, 11 insertions(+) diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index 0936f76325d7..ab05b09f3c0c 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -1,4 +1,6 @@ project(ExpensifyNitroUtils) +# cspell:ignore STREQUAL libclang fprofile defsym + cmake_minimum_required(VERSION 3.9.0) set(PACKAGE_NAME ExpensifyNitroUtils) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index ad7122bb583c..54729c66c1ee 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -1,5 +1,7 @@ # Native profile-guided optimization + + This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. The command enables the existing `ManualAppStartup` benchmark span in every app it builds. Profile collection waits for that span to finish, then explicitly writes the LLVM counters before the app process is stopped. Benchmarking uses the same native span tooling as the repository's general startup benchmark. diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 55b0d532c76d..0a35e42a7770 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -1,3 +1,5 @@ +// cspell:ignore sdcard profdata Ppatched Ppgo profraw Readelf readelf zipinfo libreactnative libhermesvm libjsi cnts + /** Android-specific release builds, native profile persistence, retrieval, and instrumentation checks. */ import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index cd76d60c9dce..c262db94059d 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -1,3 +1,5 @@ +// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm + /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index e64a854d2d1d..1b2767c7bb6e 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -1,5 +1,7 @@ #!/usr/bin/env bun +// cspell:ignore profraw Profdata profdata + import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; diff --git a/tests/tooling/pgo.test.ts b/tests/tooling/pgo.test.ts index 5b18494f2ed0..8839e05fc7c7 100644 --- a/tests/tooling/pgo.test.ts +++ b/tests/tooling/pgo.test.ts @@ -1,4 +1,5 @@ import {describe, expect, it} from 'bun:test'; +// cspell:ignore chrispader import {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; import {percentageImprovement} from '@scripts/pgo/pgo'; From 169f09f7a45876333679dd42e20628344d697d77 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 13:12:31 +0200 Subject: [PATCH 31/60] docs: clarify PGO scenario isolation and artifact matching --- scripts/pgo/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 54729c66c1ee..ecf487661453 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -85,6 +85,8 @@ scripts/pgo/pgo.ts android merge Use the equivalent `ios` commands on iOS. A representative journey should exercise the common signed-in path: open and scroll chats, send a message, visit a workspace setting, attach and view a file, and run a search. Keep account state, data size, and network conditions stable between collections. +Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The current command automates this lifecycle for startup collection; a mixed-scenario CI runner still needs that orchestration. + ## Build and benchmark the optimized app Build the optimized app only after recording a fresh profile. Build the baseline release from the same source revision and toolchain: @@ -129,6 +131,8 @@ The current `record-startups` command accumulates and merges the startup runs au Run profile generation as a scheduled or release-candidate job, not on every pull request. Use pinned self-hosted runners or a device farm, preferably with physical arm64 devices, the production compiler versions, stable thermal and power conditions, a seeded account, and deterministic local fixtures where possible. Build one instrumented artifact per platform, run the fixed startup and interaction suites against that exact artifact, flush after every successful scenario, merge the separate outputs with documented weights, then build the optimized artifact without changing the checkout or toolchain. +Store both repository revisions, dependency locks, compiler and SDK versions, ABI, build settings, scenario version, and test-data fixture with each profile. Reject mismatched artifacts before the optimized build. Keep the instrumented build and raw profiles so a failed collection can be investigated without rerunning it. + Benchmark both variants on the same device after a cooldown, discard post-install warm-ups, and retain raw samples. The `benchmark` workflow runs all baseline samples before all optimized samples, so thermal or time-dependent drift can bias the comparison. The two archived builds intentionally share an identifier to preserve app data, which prevents side-by-side alternation. CI can reduce drift with ABBA install blocks that preserve the shared identifier and discard each post-install warm-up. Another option is to give the builds distinct identifiers, seed matched state, and use the repository's alternating startup benchmark. Evaluate both startup and held-out interactive journeys that were excluded from training. Gate on several releases of data rather than a single noisy run. Hosted CI is usually insufficient for the complete workflow because hosted runners rarely expose stable physical devices. Android emulators can validate automation and may contribute useful same-ABI counts when the workload is representative, but final validation should use target devices. iOS device collection also needs signing and physical-device access. A small self-hosted device pool or managed device farm is the realistic route to automation. From 66a65c1b7e593f6d377554b9b5d28ff89f09e7e8 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 13:38:35 +0200 Subject: [PATCH 32/60] fix: support Xcode 27 benchmark process discovery --- scripts/lib/nativeAppBenchmark.ts | 2 ++ scripts/lib/nativeAppBenchmark/ios.ts | 32 +++++++++++------------ tests/tooling/benchmarkAppStartup.test.ts | 32 ++++++++++++++++++++++- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/scripts/lib/nativeAppBenchmark.ts b/scripts/lib/nativeAppBenchmark.ts index bce54b015663..434fc2c038fe 100644 --- a/scripts/lib/nativeAppBenchmark.ts +++ b/scripts/lib/nativeAppBenchmark.ts @@ -6,6 +6,7 @@ import {assertAndroidAppInstalled, createAndroidAdapter, parseAndroidProcessIden import { createIOSAdapter, iOSBenchmarkMarkerPath, + iOSProcessListArguments, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, @@ -27,6 +28,7 @@ export { createNativeAppBenchmarkAdapter, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseBenchmarkLogEvents, diff --git a/scripts/lib/nativeAppBenchmark/ios.ts b/scripts/lib/nativeAppBenchmark/ios.ts index fa82833d1719..d5a82d60b543 100644 --- a/scripts/lib/nativeAppBenchmark/ios.ts +++ b/scripts/lib/nativeAppBenchmark/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore BEGINSWITH devicectl +// cspell:ignore devicectl /** Implements iOS benchmark setup, launch, process control, and marker collection through CoreDevice. */ @@ -67,20 +67,7 @@ async function createIOSAdapter({rootDirectory, deviceIdentifier, appID}: Omit 0 ? processIdentifier : undefined; } +/** Lists every process so client-side matching works across CoreDevice versions with different filter field names. */ +function iOSProcessListArguments(deviceIdentifier: string, outputPath: string): string[] { + return ['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']; +} + /** Maps a span name to its encoded marker path inside the iOS app data container. */ function iOSBenchmarkMarkerPath(spanName: string): string { return `${IOS_BENCHMARK_DIRECTORY}/${encodeURIComponent(spanName)}.log`; @@ -283,5 +275,13 @@ function isCoreDeviceInstalledApp(value: JsonValue): value is CoreDeviceInstalle return isJSONObject(value) && typeof value.bundleIdentifier === 'string' && typeof value.url === 'string'; } -export {createIOSAdapter, iOSBenchmarkMarkerPath, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, parseIOSRunningAppProcessIdentifier}; +export { + createIOSAdapter, + iOSBenchmarkMarkerPath, + iOSProcessListArguments, + parseIOSInstalledAppURL, + parseIOSInstalledAppsResponse, + parseIOSLaunchProcessIdentifier, + parseIOSRunningAppProcessIdentifier, +}; export type {CoreDeviceInstalledAppsResponse}; diff --git a/tests/tooling/benchmarkAppStartup.test.ts b/tests/tooling/benchmarkAppStartup.test.ts index 0385db56a12a..4c213eb7ef12 100644 --- a/tests/tooling/benchmarkAppStartup.test.ts +++ b/tests/tooling/benchmarkAppStartup.test.ts @@ -1,4 +1,4 @@ -// cspell:ignore appex +// cspell:ignore appex devicectl import {describe, expect, it, jest} from 'bun:test'; @@ -7,6 +7,7 @@ import { assertAndroidAppInstalled, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseIOSInstalledAppURL, @@ -78,6 +79,35 @@ describe('benchmarkAppStartup', () => { expect(processIdentifier).toBe(456); }); + it('lists iOS processes without a version-specific CoreDevice filter', () => { + // Given + const deviceIdentifier = '00008120-EXAMPLE'; + const outputPath = '/tmp/processes.json'; + + // When + const args = iOSProcessListArguments(deviceIdentifier, outputPath); + + // Then + expect(args).toEqual(['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']); + expect(args).not.toContain('--filter'); + }); + + it('ignores an extension process when the main iOS app is not running', () => { + // Given + const appURL = 'file:///private/var/containers/Bundle/Application/EXAMPLE/Expensify.app/'; + const response = { + result: { + runningProcesses: [{executable: `${appURL}PlugIns/LiveActivityExtension.appex/LiveActivityExtension`, processIdentifier: 20702}], + }, + }; + + // When + const processIdentifier = parseIOSRunningAppProcessIdentifier(response, appURL); + + // Then + expect(processIdentifier).toBeUndefined(); + }); + it('rejects malformed CoreDevice installed-app responses before using them', () => { expect(() => parseIOSInstalledAppsResponse({result: {apps: [{bundleIdentifier: 'com.example.app'}]}})).toThrow('unexpected installed-app response'); }); From d94d768870b7bb22352e68c22c64a26dc20fe41d Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 21 Sep 2026 13:55:06 +0200 Subject: [PATCH 33/60] docs: clarify iOS PGO profile coverage --- scripts/pgo/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index ecf487661453..b9f78fd5a8c2 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -141,6 +141,7 @@ Hosted CI is usually insufficient for the complete workflow because hosted runne - LLVM PGO improves compiled native code. It can optimize a source-built Hermes engine, but it does not directly reorder JavaScript or replace JavaScript startup analysis. - Precompiled vendored frameworks cannot be instrumented after the fact. +- The iOS flush currently exports the app executable's profile runtime and Hermes's separate runtime. Other dynamically linked frameworks need their own export path before their instrumentation can contribute to the merged profile. - Profiles become stale after native source, dependency, compiler, SDK, ABI, or important build-setting changes. Regenerate them instead of accepting out-of-date warnings. - Instrumented builds have overhead. Never use them as the performance baseline. - Local bootstrap settings disable Android minification so synthetic identifiers and debug signing work. A production decision still needs a production-like, minified CI build with the repository's signing and dependency issues resolved. From c4bd27d2eb7fd2533da1229abe40be83be52ea0d Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 22 Sep 2026 10:17:11 +0200 Subject: [PATCH 34/60] refactor: share PGO script utilities --- scripts/lib/scriptUtils.ts | 104 ++++++++++++++++++++++++++++++ scripts/pgo/android.ts | 6 +- scripts/pgo/ios.ts | 5 +- scripts/pgo/pgo.ts | 7 ++- scripts/pgo/shared.ts | 125 +------------------------------------ tests/tooling/pgo.test.ts | 8 ++- 6 files changed, 124 insertions(+), 131 deletions(-) create mode 100644 scripts/lib/scriptUtils.ts diff --git a/scripts/lib/scriptUtils.ts b/scripts/lib/scriptUtils.ts new file mode 100644 index 000000000000..9a665b430b86 --- /dev/null +++ b/scripts/lib/scriptUtils.ts @@ -0,0 +1,104 @@ +/** Reusable helpers for repository tooling scripts. */ + +import {spawnSync} from 'node:child_process'; +import {existsSync, readdirSync} from 'node:fs'; +import {join, resolve} from 'node:path'; +import process from 'node:process'; + +const rootDirectory = resolve(import.meta.dirname, '../..'); + +function fail(message: string): never { + throw new Error(message); +} + +function run(command: string, args: string[], cwd = rootDirectory): void { + const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } +} + +function capture(command: string, args: string[], cwd = rootDirectory): string { + const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const stderr = result.stderr.trim(); + fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { + const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { + const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); + return !result.error && result.status === 0; +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolvePromise) => { + setTimeout(resolvePromise, milliseconds); + }); +} + +function findFiles(directory: string, extension: string): string[] { + if (!existsSync(directory)) { + return []; + } + + return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return findFiles(path, extension); + } + return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; + }); +} + +function requirePositiveInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive integer, received: ${value}`); + } + return value; +} + +function parsePositiveInteger(rawValue: string, label: string): number { + return requirePositiveInteger(Number(rawValue), label); +} + +function parseChoice(rawValue: string, choices: readonly T[], label: string): T { + const choice = choices.find((candidate) => candidate === rawValue); + if (!choice) { + fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); + } + return choice; +} + +function environmentString(name: string): string | undefined { + const value: unknown = process.env[name]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function valueAt(values: readonly T[], index: number): T { + const value = values.at(index); + if (value === undefined) { + fail(`Expected a value at index ${index}.`); + } + return value; +} + +export {capture, captureBinary, environmentString, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run, runAllowFailure, sleep, valueAt}; diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 0a35e42a7770..0463bf8073d4 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -9,7 +9,8 @@ import process from 'node:process'; import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PlatformAdapter} from './shared'; -import {BENCHMARK_SPANS_ENVIRONMENT, capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from './shared'; +import {capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from '../lib/scriptUtils'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; @@ -212,4 +213,5 @@ function androidProfileReceiverComponent(appID: string): string { return `${appID}/${ANDROID_JAVA_NAMESPACE}.PgoProfileReceiver`; } -export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent, createAndroidPgoAdapter}; +export default createAndroidPgoAdapter; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent}; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index c262db94059d..765684d6d349 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -2,13 +2,16 @@ /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ +import {isRecord} from '@libs/ObjectUtils'; + import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {dirname, join} from 'node:path'; import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter} from './shared'; -import {BENCHMARK_SPANS_ENVIRONMENT, capture, environmentString, fail, findFiles, isRecord, rootDirectory, run, runAllowFailure, sleep, valueAt} from './shared'; +import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllowFailure, sleep, valueAt} from '../lib/scriptUtils'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { const profileFormat = 'ios-clang-frontend-swift-ir-v1'; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 1b2767c7bb6e..486c3660bce9 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -14,9 +14,10 @@ import type {BenchmarkKind, PlatformAdapter} from './shared'; import {benchmarkStartups as runBenchmarkStartups} from '../lib/benchmarkAppStartup'; import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppBenchmark'; -import {createAndroidPgoAdapter} from './android'; +import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; -import {STARTUP_SPAN_NAME, capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from './shared'; +import {STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_STARTUP_WAIT_SECONDS = 30; @@ -281,4 +282,4 @@ if (import.meta.main) { }); } -export {main, percentageImprovement}; +export default percentageImprovement; diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index c7522ce26d74..c8c9daccf786 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -1,15 +1,9 @@ -/** Shared PGO workflow types, paths, subprocess helpers, and argument validation. */ - -import {spawnSync} from 'node:child_process'; -import {existsSync, readdirSync} from 'node:fs'; -import {join, resolve} from 'node:path'; -import process from 'node:process'; +/** Shared PGO workflow types and constants. */ import type {PlatformName} from '../lib/nativeAppBenchmark'; const STARTUP_SPAN_NAME = 'ManualAppStartup'; const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${STARTUP_SPAN_NAME}`; -const rootDirectory = resolve(import.meta.dirname, '../..'); type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; @@ -35,120 +29,5 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -function fail(message: string): never { - throw new Error(message); -} - -function run(command: string, args: string[], cwd = rootDirectory): void { - const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); - if (result.error) { - fail(`Failed to run ${command}: ${result.error.message}`); - } - if (result.status !== 0) { - fail(`${command} exited with status ${result.status ?? 'unknown'}.`); - } -} - -function capture(command: string, args: string[], cwd = rootDirectory): string { - const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); - if (result.error) { - fail(`Failed to run ${command}: ${result.error.message}`); - } - if (result.status !== 0) { - const stderr = result.stderr.trim(); - fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); - } - return result.stdout; -} - -function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { - const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); - if (result.error) { - fail(`Failed to run ${command}: ${result.error.message}`); - } - if (result.status !== 0) { - fail(`${command} exited with status ${result.status ?? 'unknown'}.`); - } - return result.stdout; -} - -function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { - const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); - return !result.error && result.status === 0; -} - -function sleep(milliseconds: number): Promise { - return new Promise((resolvePromise) => { - setTimeout(resolvePromise, milliseconds); - }); -} - -function findFiles(directory: string, extension: string): string[] { - if (!existsSync(directory)) { - return []; - } - - return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { - const path = join(directory, entry.name); - if (entry.isDirectory()) { - return findFiles(path, extension); - } - return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; - }); -} - -function requirePositiveInteger(value: number, label: string): number { - if (!Number.isSafeInteger(value) || value <= 0) { - fail(`${label} must be a positive integer, received: ${value}`); - } - return value; -} - -function parsePositiveInteger(rawValue: string, label: string): number { - return requirePositiveInteger(Number(rawValue), label); -} - -function parseChoice(rawValue: string, choices: readonly T[], label: string): T { - const choice = choices.find((candidate) => candidate === rawValue); - if (!choice) { - fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); - } - return choice; -} - -function environmentString(name: string): string | undefined { - const value: unknown = process.env[name]; - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function valueAt(values: T[], index: number): T { - const value = values.at(index); - if (value === undefined) { - fail(`Expected a value at index ${index}.`); - } - return value; -} - -export { - BENCHMARK_SPANS_ENVIRONMENT, - STARTUP_SPAN_NAME, - capture, - captureBinary, - environmentString, - fail, - findFiles, - isRecord, - parseChoice, - parsePositiveInteger, - requirePositiveInteger, - rootDirectory, - run, - runAllowFailure, - sleep, - valueAt, -}; +export {BENCHMARK_SPANS_ENVIRONMENT, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgo.test.ts b/tests/tooling/pgo.test.ts index 8839e05fc7c7..ac3ff4ba66e0 100644 --- a/tests/tooling/pgo.test.ts +++ b/tests/tooling/pgo.test.ts @@ -1,10 +1,14 @@ import {describe, expect, it} from 'bun:test'; // cspell:ignore chrispader -import {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; -import {percentageImprovement} from '@scripts/pgo/pgo'; +import createAndroidPgoAdapter, {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; +import percentageImprovement from '@scripts/pgo/pgo'; describe('PGO tooling', () => { + it('exports the Android adapter as the default export', () => { + expect(createAndroidPgoAdapter).toBeFunction(); + }); + it('reads the bootstrapped release application ID', () => { // Given const buildGradle = ` From 6aa10315586deab1d2e09bccc0ac5a7f51552ddd Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 11:45:36 +0200 Subject: [PATCH 35/60] fix: allow partial iOS PGO profiles --- scripts/pgo/ios.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 765684d6d349..8fd607438ebc 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm +// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ @@ -186,7 +186,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ]; } if (kind === 'optimized') { - const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath}`; + const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-error=profile-instr-unprofiled`; return [ ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, From f6978f17f1ef3953008b839294d59d8f45ef397c Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 14:00:36 +0200 Subject: [PATCH 36/60] perf: automate native PGO journeys --- package.json | 1 + scripts/pgo/JOURNEY.md | 77 +++++++++++ scripts/pgo/README.md | 4 +- scripts/pgo/journey.ts | 154 ++++++++++++++++++++++ scripts/pgo/journeyConfig.ts | 71 ++++++++++ scripts/pgo/journeyDevice.ts | 143 ++++++++++++++++++++ scripts/pgo/journeyWorkload.ts | 215 +++++++++++++++++++++++++++++++ tests/tooling/pgoJourney.test.ts | 135 +++++++++++++++++++ 8 files changed, 799 insertions(+), 1 deletion(-) create mode 100644 scripts/pgo/JOURNEY.md create mode 100755 scripts/pgo/journey.ts create mode 100644 scripts/pgo/journeyConfig.ts create mode 100644 scripts/pgo/journeyDevice.ts create mode 100644 scripts/pgo/journeyWorkload.ts create mode 100644 tests/tooling/pgoJourney.test.ts diff --git a/package.json b/package.json index 0e4259356074..bbc1b65e7393 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build-staging": "ENV=staging bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "build-adhoc": "ENV=adhoc bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "benchmark-app-startup": "bun scripts/benchmarkAppStartup.ts", + "pgo:journey": "bun scripts/pgo/journey.ts", "bootstrap-device": "bun scripts/bootstrapForDevice.ts", "createDocsRoutes": "bun .github/scripts/createDocsRoutes.ts", "generateAllowedUrls": "bun .github/scripts/generateAllowedUrls.ts", diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md new file mode 100644 index 000000000000..34757f14e5e6 --- /dev/null +++ b/scripts/pgo/JOURNEY.md @@ -0,0 +1,77 @@ +# Automated native PGO journey + + + +`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full Android journey, including message sending and native profile export, has been validated on a physical Samsung phone. iOS execution still needs live validation. + +## Account and device setup + +Use a dedicated, owner-approved heavy account with a populated Inbox, a long chat history, and enough expenses and expense reports to scroll. Keep the account and its data between runs. Installing over the existing app preserves authentication; uninstalling or clearing app data does not. + +The runner never enters credentials, retrieves a magic code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. + +Before every run, it opens Inbox to allow the app's automatic large-account preference update, then opens Account → Preferences and ensures Priority mode is **Most recent**. If it is **#focus**, the runner changes it and verifies the result. It checks the preference again at the end and rejects the run if #focus became enabled. + +Install `agent-device` on the runner and pin its version. Android needs USB debugging and a connected, authorized device. The runner temporarily enables the tool's test keyboard for reliable text replacement and restores the previous keyboard when its session closes. iOS requires macOS, Xcode, a trusted device with Developer Mode enabled, and signing for the tool's XCTest runner. The existing iOS profile collector supports physical devices; simulator profiles are not inputs to a device build. + +Create an ignored fixture, for example `.pgo/journey-fixture.json`: + +```json +{ + "accountEmail": "heavy@example.com", + "accountClass": "heavy", + "description": "Owner-approved high-traffic test account with populated chats, expenses, and reports.", + "report": { + "query": "busy", + "resultLabelPrefix": "#busy, Test workspace", + "title": "#busy" + }, + "personalChat": { + "query": "heavy@example.com", + "resultLabelPrefix": "Test User (you), heavy@example.com", + "title": "Test User (you)" + }, + "allowMessages": false, + "scrolls": 8, + "tabCycles": 3 +} +``` + +Choose the report name and workspace from the actual account. Search matching permits a changing message preview after the configured identity, and rejects ambiguous results. The personal-chat query must be the exact approved account email. That result can show either the email or the latest message beneath its `(you)` title, so both forms are supported for repeated runs. Confirm the personal chat with Chris before setting `allowMessages` to `true`. Messages are restricted to that account's `(you)` chat; each run sends two uniquely marked test messages and leaves them in place. An existing composer draft causes a failure. + +## Validate and record + +First check the account and #focus setting without sending messages: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --preflight +``` + +Use `--navigation-only` instead of `--preflight` to exercise all navigation and scrolling without sending messages. This mode cannot collect a training profile. Omit both flags for the complete journey once message permission is recorded in the fixture. + +The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes and edits two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. + +After installing and verifying an instrumented build with the existing PGO commands, collect three independent repetitions: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +scripts/pgo/journey.ts ios --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +``` + +Start with three repetitions per platform. Use the recorded duration and resulting native counter coverage to adjust the workload; extra repetitions of the same data do not add new paths. Most activity here exercises Hermes, Fabric mounting, layout, text, native input, and scrolling. This trains the compiled native implementation used by JavaScript; it does not compile the application's JavaScript with LLVM. Avoid spending most of the run waiting on network responses or looping over trivial settings screens. + +On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. + +The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. + +Each batch lives under `.pgo//journeys//` and includes `result.json`, per-run raw profiles, `journey.profdata`, and a function summary. The result records the fixture, app and device identifiers, both repository revisions, automation version, completed runs, and durations. These artifacts contain account identifiers and must stay in restricted CI storage. + +The journey profile is kept separate from the startup profile. The existing optimized-build command reads `newdot.profdata` from its platform directory: CI must explicitly select or weight the successful scenario profiles, write that final input, and preserve the iOS `.format` marker. Do not silently replace the startup input with an incomplete navigation run. + +## Release pipeline integration + +Use an exclusive device lease on a persistent runner: Linux or macOS with an arm64 Android device, and macOS with an arm64 iPhone. Build the instrumented artifact from the release candidate, install it over the signed-in app, run preflight and the full collection, then build the optimized artifact from the same source and toolchain. Gate publication on a separate performance and correctness check. The instrumented artifact is a CI input, not the store upload. + +Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. + +This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows, validating iOS, and measuring held-out interactive performance remain separate integration steps. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index b9f78fd5a8c2..6c01fac3d088 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -75,6 +75,8 @@ The app must remain in the foreground until each profile write completes. Force- ## Collect an interactive profile +For the automated heavy-account workload, see [Automated native PGO journey](./JOURNEY.md). It checks the signed-in account, disables #focus, drives the critical user flows, and can archive and merge profiles from each successful run. + For a broader profile, install the instrumented app, perform the agreed journey, and write the counters while the app is still running: ```bash @@ -85,7 +87,7 @@ scripts/pgo/pgo.ts android merge Use the equivalent `ios` commands on iOS. A representative journey should exercise the common signed-in path: open and scroll chats, send a message, visit a workspace setting, attach and view a file, and run a search. Keep account state, data size, and network conditions stable between collections. -Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The current command automates this lifecycle for startup collection; a mixed-scenario CI runner still needs that orchestration. +Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The startup recorder and journey runner each handle that lifecycle; CI must still choose and combine the scenario profiles. ## Build and benchmark the optimized app diff --git a/scripts/pgo/journey.ts b/scripts/pgo/journey.ts new file mode 100755 index 000000000000..624f67f2c56e --- /dev/null +++ b/scripts/pgo/journey.ts @@ -0,0 +1,154 @@ +#!/usr/bin/env bun + +// cspell:ignore profraw profdata + +import CLI from 'expensify-common/CLI'; +import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {dirname, join, relative, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {PLATFORM_NAMES} from '../lib/nativeAppBenchmark'; +import {capture, findFiles, parseChoice, parsePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {parseJourneyFixture} from './journeyConfig'; +import {createJourneyDevice} from './journeyDevice'; +import {prepareJourney, runJourneyWorkload, verifyJourneyAccount} from './journeyWorkload'; + +/** Run the same UI workload on a selected Android or iOS device; never automate authentication. */ +async function main(): Promise { + /* eslint-disable @typescript-eslint/naming-convention */ + const cli = new CLI({ + positionalArgs: [{name: 'platform', description: 'android or ios', parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform')}], + namedArgs: { + fixture: {description: 'JSON fixture identifying the approved heavy account and reports'}, + device: {description: 'Android serial or iOS UDID'}, + 'app-id': {description: 'Installed application or bundle identifier'}, + runs: {description: 'Complete journeys to run', default: 1, parse: (value) => parsePositiveInteger(value, 'Runs')}, + }, + flags: { + collect: {description: 'Flush, archive, and merge LLVM profiles from an already-installed instrumented app'}, + preflight: {description: 'Verify account and disable #focus, then stop without running the workload'}, + 'navigation-only': {description: 'Validate navigation without sending messages; cannot collect training profiles'}, + }, + }); + /* eslint-enable @typescript-eslint/naming-convention */ + const platform = parseChoice(String(cli.positionalArgs.platform), PLATFORM_NAMES, 'Platform'); + const fixtureInput: unknown = JSON.parse(readFileSync(resolve(cli.namedArgs.fixture), 'utf8')); + const fixture = parseJourneyFixture(fixtureInput); + if (cli.flags['navigation-only'] && cli.flags.collect) { + throw new Error('Navigation-only validation cannot produce a complete training profile.'); + } + if (!cli.flags.preflight && !cli.flags['navigation-only'] && !fixture.allowMessages) { + throw new Error('Confirm the personal chat with Chris before setting allowMessages=true in the fixture. Use --navigation-only to validate without sending.'); + } + const runs = parsePositiveInteger(String(cli.namedArgs.runs), 'Runs'); + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const batchDirectory = join(rootDirectory, '.pgo', platform, 'journeys', batchID); + mkdirSync(batchDirectory, {recursive: true}); + const device = createJourneyDevice({platform, device: cli.namedArgs.device, appID: cli.namedArgs['app-id'], session: `pgo-${platform}-${batchID}`}); + const adapter = platform === 'android' ? createAndroidPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device) : createIOSPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device); + const completedProfiles: string[] = []; + const evidence = { + platform, + device: cli.namedArgs.device, + appID: cli.namedArgs['app-id'], + accountEmail: fixture.accountEmail, + accountClass: fixture.accountClass, + fixture, + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + automationVersion: capture('agent-device', ['--version']).trim(), + requestedRuns: runs, + collectProfiles: cli.flags.collect, + navigationOnly: cli.flags['navigation-only'], + completedRuns: 0, + runDurationsSeconds: [] as number[], + status: 'running', + failure: '', + }; + const saveEvidence = () => writeFileSync(join(batchDirectory, 'result.json'), `${JSON.stringify(evidence, null, 2)}\n`); + saveEvidence(); + try { + device.open(false); + await prepareJourney(device, fixture); + if (cli.flags.preflight) { + evidence.status = 'preflight-passed'; + return; + } + if (cli.flags.collect) { + // Verify the installed build's writer before spending time on the workload; discard these setup counters. + await adapter.dumpProfiles(); + await adapter.clearDeviceProfiles(); + } + for (let index = 1; index <= runs; index += 1) { + const runID = `${batchID}-${index}`; + const runDirectory = join(batchDirectory, `run-${index}`); + mkdirSync(runDirectory); + if (index > 1) { + await prepareJourney(device, fixture); + if (cli.flags.collect) { + await adapter.clearDeviceProfiles(); + } + } + console.log(`Starting journey ${index}/${runs}.`); + const startedAt = Date.now(); + device.open(true); + await runJourneyWorkload(device, fixture, runID, cli.flags['navigation-only']); + await verifyJourneyAccount(device, fixture); + if (cli.flags.collect) { + // Flush once per fresh process. Never merge partial or failed journeys. + await adapter.dumpProfiles(); + adapter.pullProfiles(); + const rawDirectory = join(runDirectory, 'raw'); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + throw new Error('The completed journey produced no LLVM profiles.'); + } + for (const profile of profiles) { + // Archive counters only, not unrelated files pulled from the app's cache directory. + const archivedProfile = join(rawDirectory, relative(adapter.rawProfileDirectory, profile)); + mkdirSync(dirname(archivedProfile), {recursive: true}); + copyFileSync(profile, archivedProfile); + completedProfiles.push(archivedProfile); + } + } + evidence.completedRuns = index; + evidence.runDurationsSeconds.push(Math.round((Date.now() - startedAt) / 1000)); + saveEvidence(); + } + if (cli.flags.collect) { + const mergedProfile = join(batchDirectory, 'journey.profdata'); + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${mergedProfile}`, ...completedProfiles]); + if (adapter.profileFormat) { + writeFileSync(`${mergedProfile}.format`, `${adapter.profileFormat}\n`); + } + writeFileSync(`${mergedProfile}.txt`, capture(adapter.llvmTool('llvm-profdata'), ['show', '--all-functions', mergedProfile])); + console.log(`Journey profile: ${mergedProfile}`); + } + evidence.status = cli.flags['navigation-only'] ? 'navigation-passed' : 'passed'; + } catch (error) { + evidence.status = 'failed'; + evidence.failure = error instanceof Error ? error.message : String(error); + throw error; + } finally { + saveEvidence(); + // Release the device and restore Android's original keyboard, including on failed attempts. + // Closing a session leaves persisted authentication intact. + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + console.log(`Journey result: ${join(batchDirectory, 'result.json')}`); + } +} + +if (import.meta.main) { + main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = message.startsWith('SIGN_IN_REQUIRED:') ? 2 : 1; + }); +} diff --git a/scripts/pgo/journeyConfig.ts b/scripts/pgo/journeyConfig.ts new file mode 100644 index 000000000000..8a6f0cb0fe6a --- /dev/null +++ b/scripts/pgo/journeyConfig.ts @@ -0,0 +1,71 @@ +import {isRecord} from '@libs/ObjectUtils'; + +type JourneyReport = { + query: string; + resultLabelPrefix: string; + title: string; +}; + +type JourneyFixture = { + accountEmail: string; + accountClass: 'heavy'; + description: string; + report: JourneyReport; + personalChat: JourneyReport; + allowMessages: boolean; + scrolls: number; + tabCycles: number; +}; + +/** Require an explicitly selected heavy account instead of silently training on whoever is signed in. */ +function parseJourneyFixture(input: unknown): JourneyFixture { + if (!isRecord(input) || input.accountClass !== 'heavy') { + throw new Error('The journey requires a fixture with accountClass="heavy", approved by the account owner.'); + } + const accountEmail = requiredString(input.accountEmail, 'accountEmail'); + if (!accountEmail.includes('@')) { + throw new Error('accountEmail must identify the approved heavy account.'); + } + const personalChat = parseReport(input.personalChat, 'personalChat'); + if (personalChat.query !== accountEmail || !personalChat.title.endsWith('(you)') || personalChat.resultLabelPrefix !== `${personalChat.title}, ${accountEmail}`) { + throw new Error('personalChat must be the approved account\'s own "(you)" chat, with its email as query and its title and email in resultLabelPrefix.'); + } + return { + accountEmail, + accountClass: 'heavy', + description: requiredString(input.description, 'description'), + report: parseReport(input.report, 'report'), + personalChat, + allowMessages: input.allowMessages === true, + scrolls: boundedInteger(input.scrolls ?? 8, 'scrolls', 4, 30), + tabCycles: boundedInteger(input.tabCycles ?? 3, 'tabCycles', 2, 10), + }; +} + +function parseReport(input: unknown, name: string): JourneyReport { + if (!isRecord(input)) { + throw new Error(`${name} must specify query, resultLabelPrefix, and title.`); + } + return { + query: requiredString(input.query, `${name}.query`), + resultLabelPrefix: requiredString(input.resultLabelPrefix, `${name}.resultLabelPrefix`), + title: requiredString(input.title, `${name}.title`), + }; +} + +function requiredString(input: unknown, name: string): string { + if (typeof input !== 'string' || input.trim().length === 0) { + throw new Error(`${name} must be a nonempty string.`); + } + return input.trim(); +} + +function boundedInteger(input: unknown, name: string, minimum: number, maximum: number): number { + if (typeof input !== 'number' || !Number.isSafeInteger(input) || input < minimum || input > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`); + } + return input; +} + +export {parseJourneyFixture}; +export type {JourneyFixture, JourneyReport}; diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts new file mode 100644 index 000000000000..d5e888639f3d --- /dev/null +++ b/scripts/pgo/journeyDevice.ts @@ -0,0 +1,143 @@ +import {isRecord} from '@libs/ObjectUtils'; + +import {spawnSync} from 'node:child_process'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +type JourneyNode = { + label: string; + value: string; + identifier: string; + type: string; + enabled: boolean; + width: number; + height: number; +}; + +type JourneyDevice = ReturnType; + +/** Execute deterministic device commands. Authentication is always handled by the user. */ +function createJourneyDevice(options: {platform: PlatformName; device: string; appID: string; session: string}) { + const targetArguments = ['--platform', options.platform, options.platform === 'android' ? '--serial' : '--udid', options.device, '--session', options.session, '--json']; + + function command(...args: string[]): unknown { + const result = spawnSync('agent-device', [...args, ...targetArguments], {encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024}); + if (result.error) { + throw new Error(`Device command ${args.at(0)} failed: ${result.error.message}`); + } + let response: unknown; + try { + response = JSON.parse(result.stdout); + } catch { + throw new Error(`Device command ${args.at(0)} did not return JSON: ${result.stderr.trim()}`); + } + if (!isRecord(response) || response.success !== true || result.status !== 0) { + const error = isRecord(response) && isRecord(response.error) ? response.error : undefined; + throw new Error(`Device command ${args.at(0)} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); + } + return response.data; + } + + function snapshot(): JourneyNode[] { + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const nodes = parseJourneySnapshot(command('snapshot', '-i', '--force-full'), options.appID); + assertSignedIn(nodes); + return nodes; + } catch (error) { + // The Android accessibility service can briefly return only system UI while a heavy list mounts. + // Retry this read alone; repeating a tap or send could perform the action twice. + if (attempt === 2 || !(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + command('wait', '500'); + } + } + throw new Error('Unable to read the app accessibility tree.'); + } + + function wait(selector: string): void { + try { + command('wait', selector, '30000'); + } catch (error) { + // Give sign-in a specific, actionable failure instead of continuing through an expired session. + snapshot(); + throw error; + } + } + + function press(selector: string): void { + wait(selector); + command('press', selector, '--settle'); + } + + function labels(): string[] { + return snapshot().map((node) => normalizeLabel(node.label)); + } + + function hasLabel(label: string): boolean { + return labels().includes(normalizeLabel(label)); + } + + return { + command, + snapshot, + wait, + press, + hasLabel, + pressLabel: (label: string) => press(labelSelector(label)), + waitLabel: (label: string) => wait(labelSelector(label)), + fill: (identifier: string, text: string) => { + wait(idSelector(identifier)); + command('fill', idSelector(identifier), text, '--settle'); + }, + back: () => command('back', '--settle'), + open: (relaunch: boolean) => command('open', options.appID, ...(relaunch ? ['--relaunch'] : []), ...(options.platform === 'android' ? ['--test-ime'] : [])), + }; +} + +function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { + if (!isRecord(data) || !Array.isArray(data.nodes)) { + throw new Error('The device did not return an accessibility tree.'); + } + if (data.appBundleId !== appID) { + throw new Error('The automation is no longer attached to the expected app.'); + } + return data.nodes + .filter(isRecord) + .filter((node) => !node.bundleId || node.bundleId === appID) + .map((node) => ({ + label: typeof node.label === 'string' ? node.label : '', + value: typeof node.value === 'string' ? node.value : '', + identifier: typeof node.identifier === 'string' ? node.identifier : '', + type: typeof node.type === 'string' ? node.type : '', + enabled: node.enabled !== false, + width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, + height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + })); +} + +function assertSignedIn(nodes: JourneyNode[]): void { + const signInLabels = new Set(['Sign in', 'Email or phone number', 'Enter your email or phone number', 'Enter your magic code']); + if (nodes.some((node) => ['username', 'validateCode'].includes(node.identifier) || signInLabels.has(normalizeLabel(node.label)))) { + throw new Error('SIGN_IN_REQUIRED: Ask Chris to sign into the approved heavy account, then restart the journey. Automated sign-in is disabled.'); + } +} + +function normalizeLabel(label: string): string { + return label + .replaceAll(/[\u2066-\u2069]/g, '') + .replaceAll(/\s+/g, ' ') + .trim(); +} + +function labelSelector(label: string): string { + return `label=${JSON.stringify(label)}`; +} + +function idSelector(identifier: string): string { + return `id=${JSON.stringify(identifier)}`; +} + +export {assertSignedIn, createJourneyDevice, idSelector, labelSelector, normalizeLabel, parseJourneySnapshot}; +export type {JourneyDevice, JourneyNode}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts new file mode 100644 index 000000000000..7592cc6fcbb2 --- /dev/null +++ b/scripts/pgo/journeyWorkload.ts @@ -0,0 +1,215 @@ +import type {JourneyFixture, JourneyReport} from './journeyConfig'; +import type {JourneyDevice, JourneyNode} from './journeyDevice'; + +import {sleep} from '../lib/scriptUtils'; +import {idSelector, normalizeLabel} from './journeyDevice'; + +/** Run only after the user has signed into and approved the fixture's heavy account. */ +async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture, runID: string, navigationOnly = false): Promise { + console.log('Scrolling the full inbox.'); + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + device.pressLabel('All'); + await scrollBothWays(device, fixture.scrolls, true); + + console.log('Opening and scrolling a populated report.'); + await openReport(device, fixture.report); + await scrollBothWays(device, fixture.scrolls, true, 'up'); + + console.log(navigationOnly ? 'Opening the personal chat without sending.' : 'Composing and sending messages in the personal chat.'); + await openReport(device, fixture.personalChat, true); + for (let messageNumber = 1; messageNumber <= (navigationOnly ? 0 : 2); messageNumber += 1) { + // Check the destination immediately before every write. Search-result matching alone is insufficient. + device.waitLabel(fixture.personalChat.title); + const composer = device.snapshot().find((node) => node.identifier === 'composer'); + if (!composer) { + throw new Error('The personal chat composer is missing.'); + } + if (composer.value && composer.value !== 'Write something...') { + throw new Error('The personal chat contains an existing draft. Clear or send it manually before running the journey.'); + } + const token = `PGO ${runID} message ${messageNumber}`; + device.fill('composer', `${token} draft`); + device.fill('composer', `${token}. Answered by Codex, instructed by Chris.`); + device.pressLabel('Send'); + await waitForMessage(device, token); + } + + console.log('Scrolling Spend expenses and reports.'); + await showTab(device, 'Spend'); + device.pressLabel('Expenses'); + device.waitLabel('Filters'); + await scrollBothWays(device, fixture.scrolls, true); + device.pressLabel('Reports'); + device.waitLabel('Filters'); + await scrollBothWays(device, Math.max(4, Math.floor(fixture.scrolls / 2)), true); + device.pressLabel('Expenses'); + + console.log('Switching tabs and reopening the report.'); + for (let cycle = 0; cycle < fixture.tabCycles; cycle += 1) { + for (const tab of ['Inbox', 'Spend', 'Workspaces', 'Inbox']) { + await showTab(device, tab); + } + } + await openReport(device, fixture.report); + await showTab(device, 'Inbox'); +} + +/** Check identity before changing preferences; never sign in or switch accounts automatically. */ +async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): Promise { + // Entering Inbox first allows the app's automatic large-account focus prompt to appear before we check preferences. + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + try { + device.waitLabel(fixture.accountEmail); + } catch { + throw new Error(`SIGN_IN_REQUIRED: Ask Chris to sign into ${fixture.accountEmail}. The journey cannot run on another account.`); + } + device.pressLabel('Preferences'); + await ensureFocusDisabled(device, true); + device.back(); + await showTab(device, 'Inbox'); + console.log('Verified the approved heavy account and disabled #focus.'); +} + +/** End-of-run verification rejects a profile if focus was enabled during the workload. */ +async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixture): Promise { + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + device.waitLabel(fixture.accountEmail); + device.pressLabel('Preferences'); + await ensureFocusDisabled(device, false); + device.back(); + await showTab(device, 'Inbox'); +} + +async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): Promise { + device.waitLabel('Priority mode'); + if (device.hasLabel('Priority mode, Most recent')) { + return; + } + if (!device.hasLabel('Priority mode, #focus')) { + throw new Error('Cannot verify Priority mode. The journey requires English UI labels and #focus disabled.'); + } + if (!allowChange) { + throw new Error('#focus became enabled during the journey. Discard this run and restart after disabling it.'); + } + device.pressLabel('Priority mode, #focus'); + device.pressLabel('Most recent'); + device.pressLabel('Save'); + device.waitLabel('Priority mode, Most recent'); +} + +async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false): Promise { + await showTab(device, 'Inbox'); + device.press(idSelector('searchButton')); + device.fill('search-autocomplete-text-input', report.query); + const deadline = Date.now() + 30_000; + let resultLabel: string | undefined; + while (!resultLabel && Date.now() < deadline) { + resultLabel = findReportResult(device.snapshot(), report.resultLabelPrefix, isPersonalChat ? report.title : undefined); + if (!resultLabel) { + await sleep(500); + } + } + if (!resultLabel) { + throw new Error(`No search result matched ${report.resultLabelPrefix}.`); + } + device.pressLabel(resultLabel); + device.wait(idSelector('composer')); + device.waitLabel(report.title); +} + +async function showTab(device: JourneyDevice, tab: string): Promise { + // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. + for (let attempt = 0; attempt < 5; attempt += 1) { + if (device.hasLabel(tab)) { + device.pressLabel(tab); + return; + } + device.back(); + await sleep(300); + } + throw new Error(`Cannot return to the ${tab} tab.`); +} + +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { + const initialNodes = device.snapshot(); + const pixels = scrollDistance(initialNodes); + let previous = contentSignature(initialNodes); + let changed = 0; + for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { + let stationary = 0; + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', String(pixels), '--settle'); + const current = contentSignature(device.snapshot()); + if (current !== previous) { + changed += 1; + stationary = 0; + } else { + stationary += 1; + } + previous = current; + if (stationary === 2) { + break; + } + } + } + if (requireMovement && changed < 2) { + throw new Error('The list did not expose enough changing content. Use a populated heavy-account fixture; do not collect this run.'); + } +} + +async function waitForMessage(device: JourneyDevice, token: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const composer = nodes.find((node) => node.identifier === 'composer'); + const visibleMessage = nodes.some((node) => node.identifier !== 'composer' && !/EditText|TextField|TextViewInput/i.test(node.type) && node.label.includes(token)); + if (composer && !composer.value.includes(token) && visibleMessage) { + return; + } + await sleep(500); + } + throw new Error('The sent message did not appear with an empty composer. Do not retry the send automatically.'); +} + +function contentSignature(nodes: JourneyNode[]): string { + return nodes + .map((node) => normalizeLabel(node.label)) + .filter(Boolean) + .join('\n'); +} + +/** Keep both ends of the gesture inside the center of portrait lists, below Spend's fixed header. */ +function scrollDistance(nodes: JourneyNode[]): number { + const scrollAreas = nodes.filter((node) => /ScrollView|RecyclerView|ScrollArea/i.test(node.type) && node.width > 0 && node.height > 0); + const viewport = scrollAreas.toSorted((left, right) => right.width * right.height - left.width * left.height).at(0); + if (!viewport) { + throw new Error('Cannot determine scroll bounds on this device.'); + } + return Math.round(Math.min(viewport.height * 0.25, viewport.width * 0.6)); +} + +/** Match stable report identity while allowing the last-message preview to change. */ +function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitle?: string): string | undefined { + const identity = normalizeLabel(prefix); + // The fixture binds this title to the signed-in account and an exact email search. A populated self-chat replaces the email subtitle with its last message. + const personalPrefix = personalChatTitle ? `${normalizeLabel(personalChatTitle)}, ` : undefined; + const matches = nodes.filter((node) => { + const label = normalizeLabel(node.label); + return label === identity || label.startsWith(`${identity} •`) || (personalPrefix !== undefined && label.startsWith(personalPrefix)); + }); + if (matches.length > 1) { + throw new Error(`Ambiguous report search: ${prefix}. Use a more specific fixture.`); + } + return matches.at(0)?.label; +} + +export {findReportResult, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts new file mode 100644 index 000000000000..49c9c1476c84 --- /dev/null +++ b/tests/tooling/pgoJourney.test.ts @@ -0,0 +1,135 @@ +import {describe, expect, it} from 'bun:test'; + +import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; +import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; +import {findReportResult, scrollDistance} from '@scripts/pgo/journeyWorkload'; + +const fixture = { + accountEmail: 'heavy@example.com', + accountClass: 'heavy', + description: 'Owner-approved account with a populated inbox, message history, expenses, and reports.', + report: {query: 'busy', resultLabelPrefix: '#busy, Test workspace', title: '#busy'}, + personalChat: {query: 'heavy@example.com', resultLabelPrefix: 'Test User (you), heavy@example.com', title: 'Test User (you)'}, +}; + +describe('PGO journey safeguards', () => { + it('requires explicit message approval in the fixture', () => { + // Given account approval alone does not authorize a particular message destination. + for (const allowMessages of [undefined, false, 'true']) { + // When / Then full journeys remain unavailable until approval is recorded explicitly. + expect(parseJourneyFixture({...fixture, allowMessages}).allowMessages).toBe(false); + } + expect(parseJourneyFixture({...fixture, allowMessages: true}).allowMessages).toBe(true); + }); + + it('requires the account owner to select a heavy account explicitly', () => { + // Given an unclassified or ordinary account, a populated-looking screen is insufficient evidence. + for (const accountClass of [undefined, 'ordinary', '']) { + // When / Then the fixture cannot start a training run. + expect(() => parseJourneyFixture({...fixture, accountClass})).toThrow('accountClass="heavy"'); + } + }); + + it("rejects message destinations that are not the approved account's personal chat", () => { + // Given a fixture that would send messages to another person or to a different account's personal chat. + for (const personalChat of [ + {...fixture.personalChat, query: 'another@example.com'}, + {...fixture.personalChat, title: 'Another person'}, + {...fixture.personalChat, resultLabelPrefix: 'Other User (you), other@example.com'}, + {...fixture.personalChat, resultLabelPrefix: 'Test User (you), heavy@example.com.invalid'}, + ]) { + // When / Then no device action is permitted for that fixture. + expect(() => parseJourneyFixture({...fixture, personalChat})).toThrow('own "(you)" chat'); + } + }); + + it('rejects an empty workload or an unbounded scrolling loop', () => { + // Given invalid workload sizes that would create an unrepresentative or runaway recording. + for (const scrolls of [0, 1, 31, NaN, '8']) { + // When / Then parsing fails before the app is touched. + expect(() => parseJourneyFixture({...fixture, scrolls})).toThrow('scrolls'); + } + }); + + it('filters system keyboard content out of app assertions', () => { + // Given Android's keyboard can display labels unrelated to the foreground app. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {bundleId: 'keyboard.app', label: 'Sign in'}, + {bundleId: 'test.app', label: 'Inbox'}, + ], + }, + 'test.app', + ); + // When / Then keyboard content cannot falsely trigger a sign-in failure. + expect(nodes).toHaveLength(1); + expect(() => assertSignedIn(nodes)).not.toThrow(); + }); + + it('stops for manual sign-in on both platforms', () => { + // Given either platform has reached an authentication or code-entry screen. + for (const identifier of ['username', 'validateCode']) { + for (const type of ['android.widget.EditText', 'TextField']) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{identifier, type}]}, 'test.app'); + // When / Then the runner requests user input instead of filling credentials. + expect(() => assertSignedIn(nodes)).toThrow('SIGN_IN_REQUIRED'); + } + } + }); + + it('rejects a snapshot from the wrong app', () => { + // Given device focus has moved to an unexpected application. + const snapshot = {appBundleId: 'other.app', nodes: []}; + // When / Then its labels cannot satisfy the journey's assertions. + expect(() => parseJourneySnapshot(snapshot, 'test.app')).toThrow('expected app'); + }); + + it('normalizes accessibility text direction markers without changing its words', () => { + // Given translated UI labels include bidirectional isolation and non-breaking spaces. + const label = '\u2066Most\u00a0recent\u2069'; + // When / Then the same label can be compared across native platforms. + expect(normalizeLabel(label)).toBe('Most recent'); + }); + + it('matches report identity independently of the changing last-message preview', () => { + // Given the workspace has similarly named reports and a new message changes the preview. + const label = '#busy, Test workspace • New message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: '#busy, Test workspace copy • Other message'}]}, 'test.app'); + // When / Then only the specified workspace can be selected. + expect(findReportResult(nodes, '#busy, Test workspace')).toBe(label); + }); + + it('refuses ambiguous report matches', () => { + // Given two results share the fixture identity. + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label: '#busy, Test workspace'}, {label: '#busy, Test workspace • Message'}]}, 'test.app'); + // When / Then choosing the first result would risk opening an unintended destination. + expect(() => findReportResult(nodes, '#busy, Test workspace')).toThrow('Ambiguous'); + }); + + it('finds the approved personal chat after its email subtitle becomes a message preview', () => { + // Given a previous run sent a message, while another account has a similarly named chat. + const label = 'Test User (you), PGO previous run message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: 'Test User (you)'}, {label: 'Test User, Message from another account'}]}, 'test.app'); + // When / Then the validated personal-chat title matches only the full result row. + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix, fixture.personalChat.title)).toBe(label); + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix)).toBeUndefined(); + }); + + it('sizes scrolls for different screens without starting inside the fixed header', () => { + // Given Android uses physical pixels while iOS accessibility exposes logical screen dimensions. + for (const [width, height] of [ + [1080, 2280], + [390, 844], + ]) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width, height}}]}, 'test.app'); + // When the gesture is centered, both ends stay within the middle half of the portrait viewport. + const distance = scrollDistance(nodes); + expect(distance).toBeGreaterThan(0); + expect(distance).toBeLessThanOrEqual((height ?? 0) / 2); + } + // Then missing viewport information must fail instead of falling back to a screen-edge gesture. + expect(() => scrollDistance([])).toThrow('scroll bounds'); + }); +}); From a9cae54bf29e740d49971d2d6b62b96ec6d46daf Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 14:25:19 +0200 Subject: [PATCH 37/60] fix: match iOS PGO tab status labels --- scripts/pgo/journeyDevice.ts | 2 ++ scripts/pgo/journeyWorkload.ts | 21 ++++++++++++++++++--- tests/tooling/pgoJourney.test.ts | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts index d5e888639f3d..bee0f7d4f9c0 100644 --- a/scripts/pgo/journeyDevice.ts +++ b/scripts/pgo/journeyDevice.ts @@ -12,6 +12,7 @@ type JourneyNode = { enabled: boolean; width: number; height: number; + y: number; }; type JourneyDevice = ReturnType; @@ -114,6 +115,7 @@ function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { enabled: node.enabled !== false, width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + y: isRecord(node.rect) && typeof node.rect.y === 'number' ? node.rect.y : 0, })); } diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 7592cc6fcbb2..19274cad56dc 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -129,8 +129,9 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson async function showTab(device: JourneyDevice, tab: string): Promise { // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. for (let attempt = 0; attempt < 5; attempt += 1) { - if (device.hasLabel(tab)) { - device.pressLabel(tab); + const tabLabel = findTabLabel(device.snapshot(), tab); + if (tabLabel) { + device.pressLabel(tabLabel); return; } device.back(); @@ -139,6 +140,20 @@ async function showTab(device: JourneyDevice, tab: string): Promise { throw new Error(`Cannot return to the ${tab} tab.`); } +/** iOS appends unread/review status to tab accessibility labels. */ +function findTabLabel(nodes: JourneyNode[], tab: string): string | undefined { + const screenHeight = nodes.find((node) => node.type === 'Application')?.height; + const candidates = nodes.filter((node) => { + const label = normalizeLabel(node.label); + const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); + return matches && (!screenHeight || node.y >= screenHeight * 0.7); + }); + if (candidates.length > 1) { + throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); + } + return candidates.at(0)?.label; +} + async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { const initialNodes = device.snapshot(); const pixels = scrollDistance(initialNodes); @@ -212,4 +227,4 @@ function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitl return matches.at(0)?.label; } -export {findReportResult, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; +export {findReportResult, findTabLabel, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 49c9c1476c84..eb6e90f68c27 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -2,7 +2,7 @@ import {describe, expect, it} from 'bun:test'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {findReportResult, scrollDistance} from '@scripts/pgo/journeyWorkload'; +import {findReportResult, findTabLabel, scrollDistance} from '@scripts/pgo/journeyWorkload'; const fixture = { accountEmail: 'heavy@example.com', @@ -132,4 +132,21 @@ describe('PGO journey safeguards', () => { // Then missing viewport information must fail instead of falling back to a screen-edge gesture. expect(() => scrollDistance([])).toThrow('scroll bounds'); }); + + it('selects iOS status-bearing bottom tabs without confusing them with page headings', () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Inbox', rect: {width: 390, height: 50, y: 80}}, + {label: 'Inbox. Your review is required', rect: {width: 78, height: 71, y: 749}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + expect(findTabLabel(nodes, 'Inbox')).toBe('Inbox. Your review is required'); + expect(findTabLabel(nodes, 'Account')).toBe('Account, My settings. Your review is required.'); + }); }); From 394ac25641bd04cb47bfdcc85baa51da1bec7ddf Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 15:22:41 +0200 Subject: [PATCH 38/60] fix: complete iOS PGO journey on physical device --- scripts/pgo/JOURNEY.md | 4 +- scripts/pgo/journeyDevice.ts | 5 +- scripts/pgo/journeyWorkload.ts | 114 ++++++++++++++++++++++++------- tests/tooling/pgoJourney.test.ts | 68 +++++++++++++++++- 4 files changed, 162 insertions(+), 29 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index 34757f14e5e6..6e410d5002e2 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -49,7 +49,7 @@ scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture Use `--navigation-only` instead of `--preflight` to exercise all navigation and scrolling without sending messages. This mode cannot collect a training profile. Omit both flags for the complete journey once message permission is recorded in the fixture. -The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes and edits two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. +The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. Android also edits each draft before sending; iOS enters the final text once because replacing non-empty text in its composer was unreliable during live validation. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. After installing and verifying an instrumented build with the existing PGO commands, collect three independent repetitions: @@ -62,6 +62,8 @@ Start with three repetitions per platform. Use the recorded duration and resulti On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. +On the same day, a complete instrumented iOS recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. It sent two approved messages to the account's personal chat, exported two raw profiles, and produced a 15 MB merged profile with 23,822 functions showing nonzero execution counts. The final check confirmed the heavy account and #focus off. This is one validation recording; the three-repetition training batch remains to be collected. + The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. Each batch lives under `.pgo//journeys//` and includes `result.json`, per-run raw profiles, `journey.profdata`, and a function summary. The result records the fixture, app and device identifiers, both repository revisions, automation version, completed runs, and durations. These artifacts contain account identifiers and must stay in restricted CI storage. diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts index bee0f7d4f9c0..89dc251fa1e2 100644 --- a/scripts/pgo/journeyDevice.ts +++ b/scripts/pgo/journeyDevice.ts @@ -12,6 +12,7 @@ type JourneyNode = { enabled: boolean; width: number; height: number; + x: number; y: number; }; @@ -34,7 +35,7 @@ function createJourneyDevice(options: {platform: PlatformName; device: string; a } if (!isRecord(response) || response.success !== true || result.status !== 0) { const error = isRecord(response) && isRecord(response.error) ? response.error : undefined; - throw new Error(`Device command ${args.at(0)} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); + throw new Error(`Device command ${args.at(0)} ${args.at(1) ?? ''} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); } return response.data; } @@ -81,6 +82,7 @@ function createJourneyDevice(options: {platform: PlatformName; device: string; a } return { + platform: options.platform, command, snapshot, wait, @@ -115,6 +117,7 @@ function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { enabled: node.enabled !== false, width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + x: isRecord(node.rect) && typeof node.rect.x === 'number' ? node.rect.x : 0, y: isRecord(node.rect) && typeof node.rect.y === 'number' ? node.rect.y : 0, })); } diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 19274cad56dc..919c2d3f7be4 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -11,7 +11,7 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture if (device.hasLabel('No thanks')) { device.pressLabel('No thanks'); } - device.pressLabel('All'); + selectAllInboxFilter(device); await scrollBothWays(device, fixture.scrolls, true); console.log('Opening and scrolling a populated report.'); @@ -22,7 +22,9 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture await openReport(device, fixture.personalChat, true); for (let messageNumber = 1; messageNumber <= (navigationOnly ? 0 : 2); messageNumber += 1) { // Check the destination immediately before every write. Search-result matching alone is insufficient. - device.waitLabel(fixture.personalChat.title); + if (!device.hasLabel(fixture.personalChat.title)) { + throw new Error('The personal chat title is not visible before sending.'); + } const composer = device.snapshot().find((node) => node.identifier === 'composer'); if (!composer) { throw new Error('The personal chat composer is missing.'); @@ -31,7 +33,9 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture throw new Error('The personal chat contains an existing draft. Clear or send it manually before running the journey.'); } const token = `PGO ${runID} message ${messageNumber}`; - device.fill('composer', `${token} draft`); + if (device.platform === 'android') { + device.fill('composer', `${token} draft`); + } device.fill('composer', `${token}. Answered by Codex, instructed by Chris.`); device.pressLabel('Send'); await waitForMessage(device, token); @@ -39,13 +43,13 @@ async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture console.log('Scrolling Spend expenses and reports.'); await showTab(device, 'Spend'); - device.pressLabel('Expenses'); + selectSpendSection(device, 'Expenses'); device.waitLabel('Filters'); await scrollBothWays(device, fixture.scrolls, true); - device.pressLabel('Reports'); + selectSpendSection(device, 'Reports'); device.waitLabel('Filters'); await scrollBothWays(device, Math.max(4, Math.floor(fixture.scrolls / 2)), true); - device.pressLabel('Expenses'); + selectSpendSection(device, 'Expenses'); console.log('Switching tabs and reopening the report.'); for (let cycle = 0; cycle < fixture.tabCycles; cycle += 1) { @@ -66,12 +70,10 @@ async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): P } await showTab(device, 'Account'); device.command('scroll', 'top', '--settle'); - try { - device.waitLabel(fixture.accountEmail); - } catch { + if (!device.hasLabel(fixture.accountEmail)) { throw new Error(`SIGN_IN_REQUIRED: Ask Chris to sign into ${fixture.accountEmail}. The journey cannot run on another account.`); } - device.pressLabel('Preferences'); + pressPreferences(device); await ensureFocusDisabled(device, true); device.back(); await showTab(device, 'Inbox'); @@ -82,15 +84,16 @@ async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): P async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixture): Promise { await showTab(device, 'Account'); device.command('scroll', 'top', '--settle'); - device.waitLabel(fixture.accountEmail); - device.pressLabel('Preferences'); + if (!device.hasLabel(fixture.accountEmail)) { + throw new Error(`The account changed during the journey. Expected ${fixture.accountEmail}; discard this run.`); + } + pressPreferences(device); await ensureFocusDisabled(device, false); device.back(); await showTab(device, 'Inbox'); } async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): Promise { - device.waitLabel('Priority mode'); if (device.hasLabel('Priority mode, Most recent')) { return; } @@ -123,15 +126,17 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson } device.pressLabel(resultLabel); device.wait(idSelector('composer')); - device.waitLabel(report.title); + if (!device.hasLabel(report.title)) { + throw new Error(`The opened report does not show the expected title ${report.title}.`); + } } async function showTab(device: JourneyDevice, tab: string): Promise { // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. for (let attempt = 0; attempt < 5; attempt += 1) { - const tabLabel = findTabLabel(device.snapshot(), tab); - if (tabLabel) { - device.pressLabel(tabLabel); + const tabNode = findTabNode(device.snapshot(), tab); + if (tabNode) { + device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } device.back(); @@ -140,18 +145,77 @@ async function showTab(device: JourneyDevice, tab: string): Promise { throw new Error(`Cannot return to the ${tab} tab.`); } +function selectAllInboxFilter(device: JourneyDevice): void { + const tapPoint = allFilterTapPoint(device.snapshot()); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel('All'); +} + +function pressPreferences(device: JourneyDevice): void { + const cell = device.snapshot().filter((node) => normalizeLabel(node.label) === 'Preferences' && node.type === 'Cell' && node.width > 0 && node.height > 0); + const preferenceCell = cell.length === 1 ? cell.at(0) : undefined; + if (preferenceCell) { + device.command('press', String(Math.round(preferenceCell.x + preferenceCell.width / 2)), String(Math.round(preferenceCell.y + preferenceCell.height / 2)), '--settle'); + return; + } + device.pressLabel('Preferences'); +} + +function selectSpendSection(device: JourneyDevice, section: 'Expenses' | 'Reports'): void { + const tapPoint = spendSectionTapPoint(device.snapshot(), section); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel(section); +} + +/** On iOS, Expenses labels the whole horizontal strip while Reports has its own bounds. */ +function spendSectionTapPoint(nodes: JourneyNode[], section: 'Expenses' | 'Reports'): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'Expenses' && node.type === 'Cell' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const reports = nodes.find((node) => normalizeLabel(node.label) === 'Reports' && node.x > 0 && node.height > 0); + if (!reports) { + throw new Error('Cannot locate the Expenses and Reports segments in Spend.'); + } + return {x: Math.round(section === 'Reports' ? reports.x + reports.width / 2 : reports.x / 2), y: Math.round(reports.y + reports.height / 2)}; +} + +/** iOS exposes the whole filter strip as the All cell; its center lands on Unread. */ +function allFilterTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'All' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const unread = nodes.find((node) => normalizeLabel(node.label) === 'Unread' && node.x > 0 && node.height > 0); + if (!unread) { + throw new Error('Cannot locate the All filter inside the iOS Inbox strip.'); + } + return {x: Math.round(unread.x / 2), y: Math.round(unread.y + unread.height / 2)}; +} + /** iOS appends unread/review status to tab accessibility labels. */ -function findTabLabel(nodes: JourneyNode[], tab: string): string | undefined { - const screenHeight = nodes.find((node) => node.type === 'Application')?.height; +function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined { + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); const candidates = nodes.filter((node) => { const label = normalizeLabel(node.label); const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); - return matches && (!screenHeight || node.y >= screenHeight * 0.7); + return matches && (!screen || (node.y >= screen.height * 0.7 && node.width >= screen.width * 0.1 && node.height >= screen.height * 0.05)); }); if (candidates.length > 1) { throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); } - return candidates.at(0)?.label; + return candidates.at(0); } async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { @@ -196,9 +260,11 @@ async function waitForMessage(device: JourneyDevice, token: string): Promise node.type === 'Application')?.height; return nodes - .map((node) => normalizeLabel(node.label)) - .filter(Boolean) + .filter((node) => !screenHeight || (node.y + node.height > screenHeight * 0.13 && node.y < screenHeight * 0.88)) + .filter((node) => !!normalizeLabel(node.label)) + .map((node) => `${normalizeLabel(node.label)}:${Math.round(node.y / 8)}`) .join('\n'); } @@ -227,4 +293,4 @@ function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitl return matches.at(0)?.label; } -export {findReportResult, findTabLabel, prepareJourney, runJourneyWorkload, scrollDistance, verifyJourneyAccount}; +export {allFilterTapPoint, contentSignature, findReportResult, findTabNode, prepareJourney, runJourneyWorkload, scrollDistance, spendSectionTapPoint, verifyJourneyAccount}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index eb6e90f68c27..06da11572f7b 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -2,7 +2,7 @@ import {describe, expect, it} from 'bun:test'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {findReportResult, findTabLabel, scrollDistance} from '@scripts/pgo/journeyWorkload'; +import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; const fixture = { accountEmail: 'heavy@example.com', @@ -141,12 +141,74 @@ describe('PGO journey safeguards', () => { {type: 'Application', rect: {width: 390, height: 844, y: 0}}, {label: 'Inbox', rect: {width: 390, height: 50, y: 80}}, {label: 'Inbox. Your review is required', rect: {width: 78, height: 71, y: 749}}, + {label: 'Inbox', type: 'StaticText', rect: {width: 26, height: 14, y: 793}}, {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, ], }, 'test.app', ); - expect(findTabLabel(nodes, 'Inbox')).toBe('Inbox. Your review is required'); - expect(findTabLabel(nodes, 'Account')).toBe('Account, My settings. Your review is required.'); + expect(findTabNode(nodes, 'Inbox')?.label).toBe('Inbox. Your review is required'); + expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); + }); + + it('targets the visible All filter rather than the iOS strip cell center over Unread', () => { + // Given an iOS accessibility cell whose bounds cover the whole Inbox filter strip. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'All', type: 'Cell', rect: {width: 390, height: 52, x: 0, y: 115}}, + {label: 'Unread', type: 'Other', rect: {width: 136, height: 40, x: 102, y: 115}}, + ], + }, + 'test.app', + ); + // When the journey chooses the All filter's tap point. + const tapPoint = allFilterTapPoint(nodes); + // Then it stays left of Unread. + expect(tapPoint).toEqual({x: 51, y: 135}); + }); + + it('detects iOS report scrolling when offscreen accessibility labels stay loaded', () => { + // Given the same report labels at different vertical positions after a scroll. + const reportNodes = (firstMessageY: number) => + parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Old message', rect: {width: 390, height: 40, y: firstMessageY - 80}}, + {label: 'Recent message', rect: {width: 390, height: 40, y: firstMessageY}}, + ], + }, + 'test.app', + ); + // When content signatures are compared. + const before = contentSignature(reportNodes(500)); + const after = contentSignature(reportNodes(266)); + // Then movement is visible despite identical labels. + expect(before).not.toBe(after); + }); + + it('targets the visible iOS Spend segments despite a wide Expenses strip cell', () => { + // Given an Expenses cell that covers the full segment strip and a bounded Reports control. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Expenses', type: 'Cell', rect: {width: 1426, height: 52, x: 0, y: 115}}, + {label: 'Reports', type: 'Other', rect: {width: 94, height: 40, x: 125, y: 115}}, + ], + }, + 'test.app', + ); + // When each section is selected. + const expenses = spendSectionTapPoint(nodes, 'Expenses'); + const reports = spendSectionTapPoint(nodes, 'Reports'); + // Then both tap points land inside their visible segments. + expect(expenses).toEqual({x: 63, y: 135}); + expect(reports).toEqual({x: 172, y: 135}); }); }); From b7b4105a2c53327f82d2f635406ff1843eed9524 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 23 Sep 2026 17:20:21 +0200 Subject: [PATCH 39/60] fix(pgo): handle mobile navigation tree edge cases --- scripts/pgo/JOURNEY.md | 6 +-- scripts/pgo/journeyWorkload.ts | 71 +++++++++++++++++++++++++++++--- tests/tooling/pgoJourney.test.ts | 30 +++++++++++++- 3 files changed, 97 insertions(+), 10 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index 6e410d5002e2..aa32aa722d49 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -2,7 +2,7 @@ -`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full Android journey, including message sending and native profile export, has been validated on a physical Samsung phone. iOS execution still needs live validation. +`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full journey, including message sending and native profile export, has been validated on a physical Samsung phone and iPhone 13 Pro. ## Account and device setup @@ -62,7 +62,7 @@ Start with three repetitions per platform. Use the recorded duration and resulti On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. -On the same day, a complete instrumented iOS recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. It sent two approved messages to the account's personal chat, exported two raw profiles, and produced a 15 MB merged profile with 23,822 functions showing nonzero execution counts. The final check confirmed the heavy account and #focus off. This is one validation recording; the three-repetition training batch remains to be collected. +On the same day, a complete instrumented iOS validation recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. A subsequent three-repetition training batch passed in 6 minutes 41 seconds, 6 minutes 35 seconds, and 6 minutes 37 seconds per journey. The batch exported six raw profiles and produced a 15 MB merged profile containing 77,590 functions. Every repetition sent two approved messages to the account's personal chat, then confirmed the heavy account and #focus off. These timings include automation overhead; they are not app performance measurements. The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. @@ -76,4 +76,4 @@ Use an exclusive device lease on a persistent runner: Linux or macOS with an arm Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. -This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows, validating iOS, and measuring held-out interactive performance remain separate integration steps. +This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows and measuring held-out interactive performance remain separate integration steps. diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 919c2d3f7be4..d807dac61693 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -75,7 +75,7 @@ async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): P } pressPreferences(device); await ensureFocusDisabled(device, true); - device.back(); + pressInAppBack(device); await showTab(device, 'Inbox'); console.log('Verified the approved heavy account and disabled #focus.'); } @@ -89,7 +89,7 @@ async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixtu } pressPreferences(device); await ensureFocusDisabled(device, false); - device.back(); + pressInAppBack(device); await showTab(device, 'Inbox'); } @@ -139,7 +139,7 @@ async function showTab(device: JourneyDevice, tab: string): Promise { device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } - device.back(); + pressInAppBack(device); await sleep(300); } throw new Error(`Cannot return to the ${tab} tab.`); @@ -164,6 +164,42 @@ function pressPreferences(device: JourneyDevice): void { device.pressLabel('Preferences'); } +function pressInAppBack(device: JourneyDevice): void { + if (device.platform === 'ios') { + const tapPoint = inAppBackTapPoint(device.snapshot()); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + } + device.back(); +} + +/** The iOS search Back button remains visible but is reported non-hittable while the keyboard is open. */ +function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screen = nodes.find((node) => node.type === 'Application'); + if (!screen) { + return undefined; + } + const candidates = nodes.filter( + (node) => + node.type === 'Button' && + normalizeLabel(node.label) === 'Back' && + node.enabled && + node.width > 0 && + node.height > 0 && + node.x >= 0 && + node.y >= 0 && + node.x < screen.width * 0.25 && + node.y < screen.height * 0.2, + ); + if (candidates.length > 1) { + throw new Error('Ambiguous in-app Back control on iOS.'); + } + const back = candidates.at(0); + return back ? {x: Math.round(back.x + back.width / 2), y: Math.round(back.y + back.height / 2)} : undefined; +} + function selectSpendSection(device: JourneyDevice, section: 'Expenses' | 'Reports'): void { const tapPoint = spendSectionTapPoint(device.snapshot(), section); if (tapPoint) { @@ -212,10 +248,22 @@ function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); return matches && (!screen || (node.y >= screen.height * 0.7 && node.width >= screen.width * 0.1 && node.height >= screen.height * 0.05)); }); - if (candidates.length > 1) { + const first = candidates.at(0); + const duplicatedNode = + first && + candidates.every( + (node) => + node.type === first.type && + normalizeLabel(node.label) === normalizeLabel(first.label) && + node.x === first.x && + node.y === first.y && + node.width === first.width && + node.height === first.height, + ); + if (candidates.length > 1 && !duplicatedNode) { throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); } - return candidates.at(0); + return first; } async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { @@ -293,4 +341,15 @@ function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitl return matches.at(0)?.label; } -export {allFilterTapPoint, contentSignature, findReportResult, findTabNode, prepareJourney, runJourneyWorkload, scrollDistance, spendSectionTapPoint, verifyJourneyAccount}; +export { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + prepareJourney, + runJourneyWorkload, + scrollDistance, + spendSectionTapPoint, + verifyJourneyAccount, +}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 06da11572f7b..de4d031da0d5 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -2,7 +2,7 @@ import {describe, expect, it} from 'bun:test'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; +import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; const fixture = { accountEmail: 'heavy@example.com', @@ -151,6 +151,15 @@ describe('PGO journey safeguards', () => { expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { + const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; + const snapshot = (tabs: Array) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'Application', rect: {x: 0, y: 0, width: 1080, height: 2280}}, ...tabs]}, 'test.app'); + + expect(findTabNode(snapshot([tab, tab]), 'Inbox')?.x).toBe(216); + expect(() => findTabNode(snapshot([tab, {...tab, rect: {...tab.rect, x: 432}}]), 'Inbox')).toThrow('Ambiguous Inbox tab'); + }); + it('targets the visible All filter rather than the iOS strip cell center over Unread', () => { // Given an iOS accessibility cell whose bounds cover the whole Inbox filter strip. const nodes = parseJourneySnapshot( @@ -170,6 +179,25 @@ describe('PGO journey safeguards', () => { expect(tapPoint).toEqual({x: 51, y: 135}); }); + it('taps the iOS search Back button even when the keyboard marks it non-hittable', () => { + // Given the search screen's bounded Back button beneath an open keyboard. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Back', type: 'Button', hittable: false, rect: {width: 40, height: 40, x: 8, y: 59}}, + {label: 'Back', type: 'Button', rect: {width: 40, height: 40, x: 300, y: 700}}, + ], + }, + 'test.app', + ); + // When the journey looks for the in-app Back control. + const tapPoint = inAppBackTapPoint(nodes); + // Then it chooses the top-left button rather than the keyboard area. + expect(tapPoint).toEqual({x: 28, y: 79}); + }); + it('detects iOS report scrolling when offscreen accessibility labels stay loaded', () => { // Given the same report labels at different vertical positions after a scroll. const reportNodes = (firstMessageY: number) => From 7673df96f954a4b7eae33c6e282ffd0b0d26b693 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 24 Sep 2026 15:08:13 +0200 Subject: [PATCH 40/60] perf(pgo): benchmark short signed-in journeys --- scripts/pgo/JOURNEY.md | 11 ++ scripts/pgo/README.md | 13 +- scripts/pgo/android.ts | 22 ++- scripts/pgo/ios.ts | 94 ++++++++++-- scripts/pgo/journeyBenchmark.ts | 196 +++++++++++++++++++++++++ scripts/pgo/journeyBenchmarkAndroid.ts | 52 +++++++ scripts/pgo/journeyBenchmarkIOS.ts | 102 +++++++++++++ scripts/pgo/journeyWorkload.ts | 21 +++ scripts/pgo/pgo.ts | 54 +++++-- scripts/pgo/shared.ts | 7 +- tests/tooling/pgoJourney.test.ts | 48 +++++- 11 files changed, 584 insertions(+), 36 deletions(-) create mode 100644 scripts/pgo/journeyBenchmark.ts create mode 100644 scripts/pgo/journeyBenchmarkAndroid.ts create mode 100644 scripts/pgo/journeyBenchmarkIOS.ts diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index aa32aa722d49..bd6ff35593bd 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -77,3 +77,14 @@ Use an exclusive device lease on a persistent runner: Linux or macOS with an arm Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows and measuring held-out interactive performance remain separate integration steps. + +## Held-out interaction benchmark + +The PGO workflow has a shorter, read-only benchmark that is separate from the six-minute profile-collection journey. Each sample relaunches the signed-in app, selects Spend → Expenses, returns to Inbox, then times a warm Inbox → Spend → Inbox switch. The app's existing first-paint, content-load, and Inbox navigation spans measure the UI work. One complete warm-up is discarded for each build, followed by the requested number of fresh-process samples. The benchmark checks the same approved account and #focus preference, but sends no messages. + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Build both the release and optimized artifacts with the current span allowlist first. `build-optimized --profile /absolute/path/to/merged.profdata` selects an immutable archived profile; without `--profile`, it uses `newdot.profdata`. The output uses the existing startup benchmark's raw-sample and summary CSV format in a unique `.pgo//benchmarks/journey//` directory. The two phases run together, so a failed or repeated phase cannot silently pair with stale data. Compare each interactive span separately; adding their durations together does not give a full journey time because there is automation time between taps. Keep this benchmark out of the profile-training batch, and avoid using its result as a release gate until build-order drift and device temperature have been characterized. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 6c01fac3d088..f9ec34b76b04 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -4,7 +4,7 @@ This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. -The command enables the existing `ManualAppStartup` benchmark span in every app it builds. Profile collection waits for that span to finish, then explicitly writes the LLVM counters before the app process is stopped. Benchmarking uses the same native span tooling as the repository's general startup benchmark. +The command enables `ManualAppStartup` and the existing Spend/Inbox navigation spans in every app it builds. Startup profile collection waits for `ManualAppStartup` to finish, then explicitly writes the LLVM counters before the app process is stopped. Both benchmark modes use app-defined span durations rather than device-automation wall time. ## Prepare local release identifiers @@ -99,8 +99,19 @@ scripts/pgo/pgo.ts android build-release scripts/pgo/pgo.ts android benchmark 20 45 ``` +Pass `--profile /absolute/path/to/merged.profdata` to `build-optimized` when comparing an archived profile. The path becomes part of the compiler input; keep the file immutable for the build. Without it, the command uses the platform's `newdot.profdata`. Local PGO builds disable Sentry uploads. iOS temporarily skips the FullStory and Sentry upload phases and restores the original Xcode project afterward. Android excludes `fullstoryRelease` and archives the pre-FullStory APK, so its local benchmarks are internally paired but do not represent the fully post-processed production APK. Do not compare their absolute timings directly with earlier or store builds that used FullStory processing. + The benchmark installs the archived baseline, runs one warm-up and the requested `ManualAppStartup` samples, then repeats the process with the optimized app. Both builds use the same application identifier, so installing the second artifact preserves the seeded account and data. Results are stored under `.pgo//benchmarks/` in the repository benchmark CSV format. +For a separate, read-only interaction benchmark, use the approved signed-in account fixture described in [JOURNEY.md](./JOURNEY.md): + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id com.chrispader.expensify.pgo --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id com.chrispader.expensify.expensifylite.pgo --fixture .pgo/journey-fixture.json +``` + +Each variant gets one discarded warm-up and 20 measured fresh-process journeys. Before each sample, the runner selects Spend → Expenses and returns to Inbox; it then times a warm Inbox → Spend → Inbox tab switch. Expenses can restore the account's saved query and filters, so keep that state fixed across both phases. The CSVs under a unique `.pgo//benchmarks/journey//` directory contain `ManualNavigateToReportsFirstPaint`, `ManualNavigateToReportsContentLoad`, and `ManualNavigateToInboxTab` samples, alongside a run manifest with artifact hashes. These are app-defined tap-to-paint timings, so automation overhead is outside the measurement. The journey sends no messages and leaves account data unchanged. Both phases run in one command and are compared only within that batch. Rebuild both archived variants with this version of the tooling before benchmarking; older PGO builds only logged startup. + The stages can also run independently: ```bash diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 0463bf8073d4..e0ef640aa0a8 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -23,7 +23,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); const buildGradlePath = join(androidDirectory, 'build.gradle'); - const gradleReleaseApkPath = join(androidDirectory, 'build/outputs/apk/release/Expensify-release.apk'); + const gradleReleaseApkPath = join(androidDirectory, 'build/intermediates/apk/release/packageRelease/Expensify-release.apk'); const profileDumpTimeoutSeconds = 5; const artifactPaths: BuildArtifactPaths = { @@ -77,18 +77,26 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent console.log(`Copied release APK: ${destination}`); } - function build(kind: BuildKind): void { - if (kind === 'optimized' && !existsSync(join(profileDirectory, 'newdot.profdata'))) { - fail(`Missing ${join(profileDirectory, 'newdot.profdata')}. Run merge first.`); + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? join(profileDirectory, 'newdot.profdata'); + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); } const pgoModes: Record = {release: 'off', instrumented: 'generate', optimized: 'use'}; - const gradleArguments = [':assembleRelease', '-PpatchedArtifacts.forceBuildFromSource=true', '-PreactNativeArchitectures=arm64-v8a', `-PpgoMode=${pgoModes[kind]}`]; + const gradleArguments = [ + ':packageRelease', + '-PpatchedArtifacts.forceBuildFromSource=true', + '-PreactNativeArchitectures=arm64-v8a', + `-PpgoMode=${pgoModes[kind]}`, + '-x', + 'fullstoryRelease', + ]; if (kind === 'optimized') { - gradleArguments.push(`-PpgoProfile=${join(profileDirectory, 'newdot.profdata')}`); + gradleArguments.push(`-PpgoProfile=${profilePath}`); } - run('/usr/bin/env', [BENCHMARK_SPANS_ENVIRONMENT, './gradlew', ...gradleArguments], androidDirectory); + run('/usr/bin/env', [BENCHMARK_SPANS_ENVIRONMENT, 'SENTRY_DISABLE_AUTO_UPLOAD=true', './gradlew', ...gradleArguments], androidDirectory); archiveReleaseApk(artifactPaths[kind]); } diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 8fd607438ebc..bc2ba74c7a1c 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -4,7 +4,7 @@ import {isRecord} from '@libs/ObjectUtils'; -import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {dirname, join} from 'node:path'; @@ -21,6 +21,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); const markerDirectory = join(profileDirectory, 'markers'); const iosDirectory = join(rootDirectory, 'Mobile-Expensify/iOS'); + const projectPath = join(iosDirectory, 'Expensify.xcodeproj/project.pbxproj'); const hermesVersionPath = join(rootDirectory, 'node_modules/react-native/sdks/.hermesv1version'); const remotePgoDirectory = 'Library/Caches/ExpensifyPGO'; const profileStatusPath = `${remotePgoDirectory}/profile-status.txt`; @@ -134,22 +135,23 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str return capture('xcrun', ['--find', name]).trim(); } - function sourceBuildEnvironment(mode: PgoMode): string[] { + function sourceBuildEnvironment(mode: PgoMode, profilePath = mergedProfilePath): string[] { const environment = [ 'BUILD_RN_FROM_SOURCE=1', 'RCT_USE_RN_DEP=0', 'RCT_USE_PREBUILT_RNCORE=0', 'RCT_BUILD_HERMES_FROM_SOURCE=true', BENCHMARK_SPANS_ENVIRONMENT, + 'SENTRY_DISABLE_AUTO_UPLOAD=true', `EXPENSIFY_PGO_MODE=${mode}`, ]; if (mode === 'use') { - environment.push(`EXPENSIFY_PGO_PROFILE=${mergedProfilePath}`); + environment.push(`EXPENSIFY_PGO_PROFILE=${profilePath}`); } return environment; } - function prepareSourcePods(mode: PgoMode): void { + function prepareSourcePods(mode: PgoMode, profilePath: string): void { if (!existsSync(hermesVersionPath)) { fail(`Missing React Native's pinned Hermes version file at ${hermesVersionPath}.`); } @@ -164,7 +166,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str rmSync(directory, {recursive: true, force: true}); } } - run('/usr/bin/env', [...sourceBuildEnvironment(mode), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); + run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); for (const {directory, requiredFile} of requiredPodSources) { if (!existsSync(join(directory, requiredFile))) { fail(`CocoaPods did not install the required source file ${join(directory, requiredFile)}.`); @@ -173,7 +175,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str rmSync(join(iosDirectory, 'Pods/hermes-engine/build'), {recursive: true, force: true}); } - function pgoBuildSettings(kind: BuildKind): string[] { + function pgoBuildSettings(kind: BuildKind, profilePath: string): string[] { const baseSettings = ['ENABLE_CODE_COVERAGE=NO', 'CLANG_COVERAGE_MAPPING=NO', 'CLANG_USE_OPTIMIZATION_PROFILE=NO']; if (kind === 'instrumented') { return [ @@ -186,23 +188,24 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ]; } if (kind === 'optimized') { - const clangProfileFlags = `-fprofile-instr-use=${mergedProfilePath} -Wno-error=profile-instr-unprofiled`; + const clangProfileFlags = `-fprofile-instr-use=${profilePath} -Wno-error=profile-instr-unprofiled`; return [ ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, - `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${mergedProfilePath}`, + `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${profilePath}`, ]; } return baseSettings; } - function build(kind: BuildKind): void { - if (kind === 'optimized' && !existsSync(mergedProfilePath)) { - fail(`Missing ${mergedProfilePath}. Run merge first.`); + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? mergedProfilePath; + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); } if (kind === 'optimized') { - const profileFormatPath = `${mergedProfilePath}.format`; + const profileFormatPath = `${profilePath}.format`; const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; if (recordedFormat !== profileFormat) { fail('The merged iOS profile predates the current Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); @@ -215,7 +218,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str } else if (kind === 'optimized') { mode = 'use'; } - prepareSourcePods(mode); + prepareSourcePods(mode, profilePath); const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); const builtAppPath = join(derivedDataDirectory, 'Build/Products/Release-iphoneos/Expensify.app'); const signingSettings = codeSigningAllowed @@ -246,9 +249,9 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str 'ONLY_ACTIVE_ARCH=YES', 'COMPILER_INDEX_STORE_ENABLE=NO', ...signingSettings, - ...pgoBuildSettings(kind), + ...pgoBuildSettings(kind, profilePath), ]; - run('/usr/bin/env', [...sourceBuildEnvironment(mode), ...xcodeArguments], iosDirectory); + withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), ...xcodeArguments], iosDirectory)); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -258,6 +261,59 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str console.log(`Copied release app: ${artifactPaths[kind]}`); } + /** Prevent local release builds from publishing source maps or FullStory assets; restore the existing project text even if Xcode fails. */ + function withLocalUploadPhasesDisabled(buildApp: () => void): void { + const replacements = [ + // Xcode expands these literal variables inside the project build phase. + { + // eslint-disable-next-line no-template-curly-in-string + command: '\\"${PODS_ROOT}/FullStory/tools/FullStoryCommandLine\\" \\"${CONFIGURATION_BUILD_DIR}/${WRAPPER_NAME}\\"', + disabled: 'echo Skipping FullStory upload for local PGO build', + }, + {command: 'source \\"$PROJECT_ROOT/tools/upload-sentry-source-maps.sh\\"', disabled: 'echo Skipping Sentry source-map upload for local PGO build'}, + ]; + const originalProject = readFileSync(projectPath, 'utf8'); + const guardedProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, command, disabled), originalProject); + let restored = false; + let guarded = false; + const restoreProject = () => { + if (restored) { + return; + } + if (!guarded) { + writeFileSync(projectPath, originalProject); + restored = true; + return; + } + const currentProject = readFileSync(projectPath, 'utf8'); + const restoredProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, disabled, command), currentProject); + writeFileSync(projectPath, restoredProject); + restored = true; + }; + const onInterrupt = () => { + restoreProject(); + process.exit(130); + }; + const onTermination = () => { + restoreProject(); + process.exit(143); + }; + process.once('SIGINT', onInterrupt); + process.once('SIGTERM', onTermination); + try { + writeFileSync(projectPath, guardedProject); + guarded = true; + buildApp(); + } finally { + try { + restoreProject(); + } finally { + process.off('SIGINT', onInterrupt); + process.off('SIGTERM', onTermination); + } + } + } + function install(kind: BuildKind): void { const appPath = artifactPaths[kind]; if (!existsSync(appPath)) { @@ -408,4 +464,12 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str }; } +function replaceOnce(content: string, previous: string, next: string): string { + const offset = content.indexOf(previous); + if (offset < 0 || content.indexOf(previous, offset + previous.length) >= 0) { + fail('The iOS project upload phases changed. Refusing to build until the local PGO upload guard is updated.'); + } + return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; +} + export default createIOSPgoAdapter; diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts new file mode 100644 index 000000000000..2783f0221c01 --- /dev/null +++ b/scripts/pgo/journeyBenchmark.ts @@ -0,0 +1,196 @@ +/** Repeats a small, read-only signed-in journey and records app-defined navigation spans. */ + +import {createHash} from 'node:crypto'; +import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readlinkSync, writeFileSync} from 'node:fs'; +import {join, relative} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; +import type {JourneyFixture} from './journeyConfig'; +import type {PlatformAdapter, BenchmarkKind} from './shared'; + +import {createBenchmarkRecorder} from '../lib/benchmarkAppStartup'; +import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; +import {capture, fail} from '../lib/scriptUtils'; +import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; +import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; +import {createJourneyDevice} from './journeyDevice'; +import {selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {JOURNEY_SPAN_NAMES} from './shared'; + +type JourneyBenchmarkReader = { + begin: (spanNames: readonly string[]) => void; + collect: (spanNames: readonly string[], timeoutSeconds: number) => Promise; +}; + +function journeyBenchmarkPaths(adapter: Pick, batchID: string): Record { + const directory = join(adapter.benchmarkDirectory, 'journey', batchID); + return {release: join(directory, 'release.csv'), optimized: join(directory, 'pgo-optimized.csv')}; +} + +/** Installs the selected build, discards one complete warm-up, then records every requested run. */ +async function benchmarkJourneyBuild( + adapter: PlatformAdapter, + kind: BenchmarkKind, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + outputPath: string, +): Promise { + console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); + adapter.install(kind); + + const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-journey-benchmark-${adapter.name}-${kind}-${Date.now()}`}); + const reader: JourneyBenchmarkReader = + adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); + try { + device.open(false); + await waitForTab(device, 'Account'); + await verifyJourneyAccount(device, fixture); + console.log(`Running one unmeasured ${kind} journey warm-up.`); + await measureJourney(device, reader, timeoutSeconds); + + const recorder = createBenchmarkRecorder({spanNames: [...JOURNEY_SPAN_NAMES], runs, outputPath}); + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + const events = await measureJourney(device, reader, timeoutSeconds); + console.log(`${kind} journey ${runNumber}/${runs}: ${recorder.record(events, runNumber).join(', ')}`); + } + await verifyJourneyAccount(device, fixture); + await recorder.complete(`${kind} journey metrics`); + } finally { + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +/** Uses a new app process for each sample; automation latency is outside the timed spans. */ +async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, timeoutSeconds: number): Promise { + device.open(true); + await waitForTab(device, 'Account'); + // Normalize the saved Spend view before timing either tab transition. This makes the measured taps warm navigations from a known Expenses state. + await showTab(device, 'Spend'); + selectSpendSection(device, 'Expenses'); + device.waitLabel('Filters'); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + + const spendSpans = JOURNEY_SPAN_NAMES.slice(0, 2); + const inboxSpans = [JOURNEY_SPAN_NAMES[2]]; + reader.begin(spendSpans); + await showTab(device, 'Spend'); + device.waitLabel('Filters'); + const spendEvents = await reader.collect(spendSpans, timeoutSeconds); + + reader.begin(inboxSpans); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + const inboxEvents = await reader.collect(inboxSpans, timeoutSeconds); + return [...spendEvents, ...inboxEvents]; +} + +async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, fixture: JourneyFixture, runs: number, timeoutSeconds: number): Promise { + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const paths = journeyBenchmarkPaths(adapter, batchID); + const directory = join(adapter.benchmarkDirectory, 'journey', batchID); + if (existsSync(directory)) { + fail(`Journey benchmark batch already exists: ${directory}`); + } + mkdirSync(directory, {recursive: true}); + writeFileSync( + join(directory, 'run.json'), + `${JSON.stringify( + { + platform: adapter.name, + deviceID, + appID: adapter.appID(), + accountEmail: fixture.accountEmail, + runs, + timeoutSeconds, + spans: JOURNEY_SPAN_NAMES, + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + artifacts: adapter.artifactPaths, + artifactSHA256: { + release: await sha256Artifact(adapter.artifactPaths.release), + optimized: await sha256Artifact(adapter.artifactPaths.optimized), + }, + }, + null, + 2, + )}\n`, + ); + console.log(`Journey benchmark batch: ${directory}`); + console.log('=== Journey benchmark phase 1/2: Release ==='); + await benchmarkJourneyBuild(adapter, 'release', deviceID, fixture, runs, timeoutSeconds, paths.release); + console.log('=== Journey benchmark phase 2/2: PGO optimized ==='); + await benchmarkJourneyBuild(adapter, 'optimized', deviceID, fixture, runs, timeoutSeconds, paths.optimized); + await compareJourneyBenchmarks(paths, runs); +} + +async function compareJourneyBenchmarks(paths: Record, runs: number): Promise { + if (!existsSync(paths.release) || !existsSync(paths.optimized)) { + fail('Missing paired journey samples in this benchmark batch.'); + } + const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(paths.release), readBenchmarkSamples(paths.optimized)]); + const release = benchmarkMetrics(releaseSamples, JOURNEY_SPAN_NAMES); + const optimized = benchmarkMetrics(optimizedSamples, JOURNEY_SPAN_NAMES); + + console.log('=== Journey benchmark comparison (positive percentages are faster) ==='); + for (const span of JOURNEY_SPAN_NAMES) { + const baseline = release[span]?.stats; + const pgo = optimized[span]?.stats; + if (!baseline || !pgo || baseline.runs !== runs || pgo.runs !== runs) { + fail(`Expected ${runs} paired ${span} samples in this journey benchmark batch.`); + } + const improvement = ((baseline.average - pgo.average) / baseline.average) * 100; + console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); + } + console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); +} + +async function sha256Artifact(path: string): Promise { + if (lstatSync(path).isDirectory()) { + return sha256Directory(path); + } + const hash = capture('shasum', ['-a', '256', path]).trim().split(/\s+/).at(0); + if (!hash || !/^[\da-f]{64}$/.test(hash)) { + fail(`Could not record the benchmark artifact SHA-256 for ${path}.`); + } + return hash; +} + +/** Include every bundle file, relative path, directory and symlink in a deterministic iOS artifact digest. */ +async function sha256Directory(directory: string): Promise { + const hash = createHash('sha256'); + async function visit(path: string): Promise { + for (const name of readdirSync(path).toSorted()) { + const child = join(path, name); + const details = lstatSync(child); + hash.update(relative(directory, child)).update('\0'); + if (details.isDirectory()) { + hash.update('directory\0'); + await visit(child); + } else if (details.isSymbolicLink()) { + hash.update('link\0').update(readlinkSync(child)).update('\0'); + } else if (details.isFile()) { + hash.update('file\0').update(String(details.size)).update('\0'); + for await (const chunk of createReadStream(child)) { + const bytes: unknown = chunk; + if (!Buffer.isBuffer(bytes)) { + fail(`Artifact file did not stream bytes: ${child}`); + } + hash.update(bytes); + } + } else { + fail(`Unsupported artifact entry in ${directory}: ${child}`); + } + } + } + await visit(directory); + return hash.digest('hex'); +} + +export {benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; diff --git a/scripts/pgo/journeyBenchmarkAndroid.ts b/scripts/pgo/journeyBenchmarkAndroid.ts new file mode 100644 index 000000000000..9aa305e40acf --- /dev/null +++ b/scripts/pgo/journeyBenchmarkAndroid.ts @@ -0,0 +1,52 @@ +/** Collects completed interactive benchmark spans from the current Android app process. */ + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {POLL_INTERVAL_MS, parseBenchmarkLogEvents, sleep} from '../lib/nativeAppBenchmark/shared'; +import {capture, run} from '../lib/scriptUtils'; + +function createAndroidJourneyBenchmarkReader(deviceID: string, appID: string) { + const adbArgs = (args: string[]) => ['-s', deviceID, ...args]; + let consumedEventCount = 0; + + function begin(): void { + run('adb', adbArgs(['logcat', '-c'])); + consumedEventCount = 0; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + if (spanNames.length === 0) { + throw new Error('Select at least one Android journey benchmark span.'); + } + const processID = capture('adb', adbArgs(['shell', 'pidof', appID])) + .trim() + .split(/\s+/) + .find((candidate) => /^\d+$/.test(candidate)); + if (!processID) { + throw new Error(`Unable to find the running Android process for ${appID}.`); + } + + const deadline = Date.now() + timeoutSeconds * 1000; + let observedSpanNames: string[] = []; + while (Date.now() < deadline) { + const logs = capture('adb', adbArgs(['logcat', `--pid=${processID}`, '-d', '-v', 'raw'])); + const events = parseBenchmarkLogEvents(logs); + if (events.length < consumedEventCount) { + throw new Error('Android benchmark logcat events were discarded during collection. Restart the journey benchmark.'); + } + const freshEvents = events.slice(consumedEventCount); + const collected = spanNames.map((spanName) => freshEvents.findLast((event) => event.span === spanName)); + observedSpanNames = [...new Set(freshEvents.map((event) => event.span))]; + if (collected.every((event) => event !== undefined)) { + consumedEventCount = events.length; + return collected.filter((event): event is BenchmarkLogEvent => event !== undefined); + } + await sleep(POLL_INTERVAL_MS); + } + throw new Error(`Timed out after ${timeoutSeconds}s waiting for Android journey benchmark spans: ${spanNames.join(', ')}. Observed: ${observedSpanNames.join(', ') || 'none'}.`); + } + + return {begin, collect}; +} + +export default createAndroidJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyBenchmarkIOS.ts b/scripts/pgo/journeyBenchmarkIOS.ts new file mode 100644 index 000000000000..f6287014ed46 --- /dev/null +++ b/scripts/pgo/journeyBenchmarkIOS.ts @@ -0,0 +1,102 @@ +/** Collects completed interactive benchmark spans from an iOS app's cache markers. */ + +// cspell:ignore devicectl + +import {existsSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {iOSBenchmarkMarkerPath, parseBenchmarkLogEvents} from '../lib/nativeAppBenchmark'; +import {runAllowFailure, sleep} from '../lib/scriptUtils'; + +const POLL_INTERVAL_MS = 250; + +/** Snapshots markers before an action, then waits for a fresh event for every requested span. */ +function createIOSJourneyBenchmarkReader(deviceID: string, appID: string) { + let previousMarkers: Map | undefined; + + function begin(spanNames: readonly string[]): void { + if (spanNames.length === 0) { + throw new Error('At least one iOS benchmark span is required.'); + } + const markers = new Map(); + for (const spanName of new Set(spanNames)) { + markers.set(spanName, readMarker(deviceID, appID, spanName)); + } + previousMarkers = markers; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + const snapshot = previousMarkers; + if (spanNames.length === 0 || !snapshot || spanNames.some((spanName) => !snapshot.has(spanName))) { + throw new Error('Snapshot every requested iOS benchmark span with begin() before the UI action.'); + } + if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) { + throw new Error('The iOS benchmark span timeout must be positive.'); + } + + const uniqueSpanNames = [...new Set(spanNames)]; + const deadline = Date.now() + timeoutSeconds * 1000; + const events = new Map(); + while (Date.now() < deadline) { + for (const spanName of uniqueSpanNames) { + if (events.has(spanName)) { + continue; + } + const marker = readMarker(deviceID, appID, spanName); + if (marker === undefined || marker === snapshot.get(spanName)) { + continue; + } + const event = parseBenchmarkLogEvents(marker).findLast((candidate) => candidate.span === spanName); + if (event) { + events.set(spanName, event); + } + } + if (events.size === uniqueSpanNames.length) { + return spanNames.map((spanName) => { + const event = events.get(spanName); + if (!event) { + throw new Error(`Missing collected iOS benchmark span ${spanName}.`); + } + return event; + }); + } + await sleep(POLL_INTERVAL_MS); + } + const missing = uniqueSpanNames.filter((spanName) => !events.has(spanName)); + throw new Error(`Timed out after ${timeoutSeconds}s waiting for iOS benchmark span(s): ${missing.join(', ')}.`); + } + + return {begin, collect}; +} + +function readMarker(deviceID: string, appID: string, spanName: string): string | undefined { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-journey-benchmark-ios-')); + const localPath = join(temporaryDirectory, 'benchmark.log'); + try { + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceID, + '--source', + iOSBenchmarkMarkerPath(spanName), + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + appID, + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8') : undefined; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } +} + +export default createIOSJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index d807dac61693..5d23ead14775 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -145,6 +145,24 @@ async function showTab(device: JourneyDevice, tab: string): Promise { throw new Error(`Cannot return to the ${tab} tab.`); } +/** Wait for a mounted bottom tab without navigating away from a partially loaded screen. */ +async function waitForTab(device: Pick, tab: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + if (findTabNode(device.snapshot(), tab)) { + return; + } + } catch (error) { + if (!(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + } + await sleep(300); + } + throw new Error(`Timed out waiting for the ${tab} bottom tab to mount.`); +} + function selectAllInboxFilter(device: JourneyDevice): void { const tapPoint = allFilterTapPoint(device.snapshot()); if (tapPoint) { @@ -350,6 +368,9 @@ export { prepareJourney, runJourneyWorkload, scrollDistance, + selectSpendSection, + showTab, spendSectionTapPoint, verifyJourneyAccount, + waitForTab, }; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 486c3660bce9..2343adaa03fb 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -5,7 +5,8 @@ import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; -import {existsSync, mkdirSync, writeFileSync} from 'node:fs'; +import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {resolve} from 'node:path'; import type {BenchmarkStats} from '../lib/benchmarkStatistics'; import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; @@ -17,6 +18,8 @@ import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppB import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; +import {benchmarkJourneyAll} from './journeyBenchmark'; +import {parseJourneyFixture} from './journeyConfig'; import {STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; @@ -34,6 +37,7 @@ const WORKFLOW_COMMANDS = [ 'benchmark-optimized', 'benchmark', 'compare-benchmarks', + 'benchmark-journey', 'dump', 'pull', 'merge', @@ -58,15 +62,15 @@ async function main(): Promise { }, { name: 'runs', - description: 'Number of measured startup runs', + description: 'Number of measured startup or journey runs', default: DEFAULT_STARTUP_RUNS, - parse: (value) => parsePositiveInteger(value, 'Startup run count'), + parse: (value) => parsePositiveInteger(value, 'Run count'), }, { name: 'timeout', - description: `Seconds to wait for the ${STARTUP_SPAN_NAME} benchmark span`, + description: `Seconds to wait for the ${STARTUP_SPAN_NAME} or journey benchmark spans`, default: DEFAULT_STARTUP_WAIT_SECONDS, - parse: (value) => parsePositiveInteger(value, 'Startup span timeout'), + parse: (value) => parsePositiveInteger(value, 'Span timeout'), }, ], namedArgs: { @@ -78,6 +82,14 @@ async function main(): Promise { description: 'Device identifier to use (adb serial on Android; CoreDevice identifier, UDID, serial number, or device name on iOS)', required: false, }, + fixture: { + description: 'Approved account fixture for journey benchmarks', + required: false, + }, + profile: { + description: 'Immutable merged profile to use when building the optimized app', + required: false, + }, }, }); /* eslint-enable @typescript-eslint/naming-convention */ @@ -86,14 +98,25 @@ async function main(): Promise { await runWorkflow( parseChoice(String(platformName), PLATFORM_NAMES, 'Platform'), parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), - requirePositiveInteger(Number(runs), 'Startup run count'), - requirePositiveInteger(Number(timeout), 'Startup span timeout'), + requirePositiveInteger(Number(runs), 'Run count'), + requirePositiveInteger(Number(timeout), 'Span timeout'), cli.namedArgs['app-id'], cli.namedArgs.device, + cli.namedArgs.fixture, + cli.namedArgs.profile, ); } -async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand, runs: number, timeoutSeconds: number, appID?: string, deviceIdentifier?: string): Promise { +async function runWorkflow( + platformName: PlatformName, + workflow: WorkflowCommand, + runs: number, + timeoutSeconds: number, + appID?: string, + deviceIdentifier?: string, + fixturePath?: string, + profilePath?: string, +): Promise { const adapter = getAdapter(platformName, appID, deviceIdentifier); switch (workflow) { @@ -104,7 +127,7 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand adapter.build('instrumented'); return; case 'build-optimized': - adapter.build('optimized'); + adapter.build('optimized', profilePath ? resolve(profilePath) : undefined); return; case 'verify-instrumented': adapter.verifyInstrumentation(); @@ -135,6 +158,11 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand case 'compare-benchmarks': await compareBenchmarks(adapter); return; + case 'benchmark-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds); + return; + } case 'dump': await adapter.dumpProfiles(); return; @@ -149,6 +177,14 @@ async function runWorkflow(platformName: PlatformName, workflow: WorkflowCommand } } +function readJourneyFixture(fixturePath?: string): ReturnType { + if (!fixturePath) { + fail('Journey benchmarks require --fixture with the approved signed-in account.'); + } + const input: unknown = JSON.parse(readFileSync(resolve(fixturePath), 'utf8')); + return parseJourneyFixture(input); +} + function mergeProfiles(adapter: PlatformAdapter): void { mkdirSync(adapter.profileDirectory, {recursive: true}); const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index c8c9daccf786..ed48e4666d92 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -3,7 +3,8 @@ import type {PlatformName} from '../lib/nativeAppBenchmark'; const STARTUP_SPAN_NAME = 'ManualAppStartup'; -const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${STARTUP_SPAN_NAME}`; +const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; +const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES].join(',')}`; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; @@ -20,7 +21,7 @@ type PlatformAdapter = { readonly artifactPaths: BuildArtifactPaths; readonly profileFormat?: string; appID: () => string; - build: (kind: BuildKind) => void; + build: (kind: BuildKind, profilePath?: string) => void; install: (kind: BuildKind) => void; verifyInstrumentation: () => void; clearDeviceProfiles: () => Promise; @@ -29,5 +30,5 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -export {BENCHMARK_SPANS_ENVIRONMENT, STARTUP_SPAN_NAME}; +export {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index de4d031da0d5..2d4324694d97 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -1,8 +1,14 @@ import {describe, expect, it} from 'bun:test'; +import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint} from '@scripts/pgo/journeyWorkload'; +import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint, waitForTab} from '@scripts/pgo/journeyWorkload'; +import {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; const fixture = { accountEmail: 'heavy@example.com', @@ -13,6 +19,31 @@ const fixture = { }; describe('PGO journey safeguards', () => { + it('logs the held-out navigation spans in both PGO builds without replacing startup samples', () => { + // Given release and optimized artifacts share one build-time benchmark allowlist. + const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); + // When / Then the startup span and every short-journey span remain available. + expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES]); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1')).toEqual({ + release: '/pgo/ios/benchmarks/journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/journey/batch-1/pgo-optimized.csv', + }); + }); + + it('hashes iOS bundle resources as well as the app executable', async () => { + const app = mkdtempSync(join(tmpdir(), 'pgo-test-app-')); + try { + mkdirSync(join(app, 'Frameworks')); + writeFileSync(join(app, 'Expensify'), 'same executable'); + writeFileSync(join(app, 'main.jsbundle'), 'first bundle'); + const before = await sha256Artifact(app); + writeFileSync(join(app, 'main.jsbundle'), 'second bundle'); + expect(await sha256Artifact(app)).not.toBe(before); + } finally { + rmSync(app, {recursive: true, force: true}); + } + }); + it('requires explicit message approval in the fixture', () => { // Given account approval alone does not authorize a particular message destination. for (const allowMessages of [undefined, false, 'true']) { @@ -151,6 +182,21 @@ describe('PGO journey safeguards', () => { expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); }); + it('waits for a status-bearing iOS Account tab without requiring an exact label', async () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + const device = {snapshot: () => nodes}; + await expect(waitForTab(device, 'Account')).resolves.toBeUndefined(); + }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; const snapshot = (tabs: Array) => From 21f2d6c5ad01fc1e74165ccc0b58fca87744449e Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 24 Sep 2026 16:16:09 +0200 Subject: [PATCH 41/60] perf(pgo): add heavy report benchmark --- scripts/pgo/JOURNEY.md | 13 ++ scripts/pgo/README.md | 2 + scripts/pgo/journeyBenchmark.ts | 212 ++++++++++++++++++++++++++----- scripts/pgo/journeyWorkload.ts | 15 ++- scripts/pgo/pgo.ts | 22 +++- scripts/pgo/shared.ts | 5 +- tests/tooling/pgoJourney.test.ts | 9 +- 7 files changed, 235 insertions(+), 43 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index bd6ff35593bd..ce89443ae32e 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -88,3 +88,16 @@ scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id BUN ``` Build both the release and optimized artifacts with the current span allowlist first. `build-optimized --profile /absolute/path/to/merged.profdata` selects an immutable archived profile; without `--profile`, it uses `newdot.profdata`. The output uses the existing startup benchmark's raw-sample and summary CSV format in a unique `.pgo//benchmarks/journey//` directory. The two phases run together, so a failed or repeated phase cannot silently pair with stale data. Compare each interactive span separately; adding their durations together does not give a full journey time because there is automation time between taps. Keep this benchmark out of the profile-training batch, and avoid using its result as a release gate until build-order drift and device temperature have been characterized. + +## Heavy report benchmark + +Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes up to six fast swipes toward older messages and six back toward recent messages. It fails if the list does not expose changing content. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. + +```bash +scripts/pgo/pgo.ts android benchmark-heavy-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-heavy-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Rebuild both app variants after adding the `/r/*` span to the build-time benchmark allowlist. The earlier artifacts cannot emit this metric. Each batch goes to `.pgo//benchmarks/heavy-journey//` with raw and summary CSVs, artifact hashes, and a run manifest. On Android, an additional frame CSV counts rendered and missed-deadline frames during the fast-scroll window. Compare its missed-frame share separately from report-open latency; it is not elapsed scroll time. The current physical-iPhone frame probe could not consistently attach Xcode's Animation Hitches trace to the app process, so the iOS runner validates scroll movement but deliberately reports no scroll-frame score. A reliable iOS scrolling claim needs an XCTest performance test with a signposted scroll interval or another native frame trace that works on the CI device. + +Neither the six-minute collector nor this benchmark's total execution time measures app speed. Android frame counts still include any app rendering triggered during the capture window, and the report-open span can be affected by cached data, network, and device load. Keep the account dataset and device state stable, and warm both variants the same way. Run at least two batches with opposite phase orders by passing `--benchmark-order optimized-first` to the second batch; the default is `release-first`. Report distributions and confidence intervals, and only claim a benefit for the specified device and workload when the improvement exceeds run-to-run variation. No PGO profile can guarantee every launch or interaction will be faster. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index f9ec34b76b04..868d86a0aa0e 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -112,6 +112,8 @@ scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id com Each variant gets one discarded warm-up and 20 measured fresh-process journeys. Before each sample, the runner selects Spend → Expenses and returns to Inbox; it then times a warm Inbox → Spend → Inbox tab switch. Expenses can restore the account's saved query and filters, so keep that state fixed across both phases. The CSVs under a unique `.pgo//benchmarks/journey//` directory contain `ManualNavigateToReportsFirstPaint`, `ManualNavigateToReportsContentLoad`, and `ManualNavigateToInboxTab` samples, alongside a run manifest with artifact hashes. These are app-defined tap-to-paint timings, so automation overhead is outside the measurement. The journey sends no messages and leaves account data unchanged. Both phases run in one command and are compared only within that batch. Rebuild both archived variants with this version of the tooling before benchmarking; older PGO builds only logged startup. +For a populated-chat, fast-scroll benchmark, use `benchmark-heavy-journey` with the same arguments. It records the app's report-open span on both platforms and Android's rendered/missed scroll frames in a separate CSV. iOS scroll-frame measurement is not currently reliable on the connected phone; the runner verifies that scrolling moves through real content but does not report a scroll performance number. See [JOURNEY.md](./JOURNEY.md) for the exact measured interval, setup, and limitations. Both variants must be rebuilt to include the additional report-open marker. + The stages can also run independently: ```bash diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts index 2783f0221c01..2f8324e3c78b 100644 --- a/scripts/pgo/journeyBenchmark.ts +++ b/scripts/pgo/journeyBenchmark.ts @@ -1,7 +1,11 @@ -/** Repeats a small, read-only signed-in journey and records app-defined navigation spans. */ +/** Runs separate read-only signed-in navigation and heavy-report benchmarks. */ + +// cspell:ignore gfxinfo + +import {isRecord} from '@libs/ObjectUtils'; import {createHash} from 'node:crypto'; -import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readlinkSync, writeFileSync} from 'node:fs'; +import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, writeFileSync} from 'node:fs'; import {join, relative} from 'node:path'; import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; @@ -14,16 +18,33 @@ import {capture, fail} from '../lib/scriptUtils'; import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; import {createJourneyDevice} from './journeyDevice'; -import {selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; -import {JOURNEY_SPAN_NAMES} from './shared'; +import {openReport, scrollBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; type JourneyBenchmarkReader = { begin: (spanNames: readonly string[]) => void; collect: (spanNames: readonly string[], timeoutSeconds: number) => Promise; }; -function journeyBenchmarkPaths(adapter: Pick, batchID: string): Record { - const directory = join(adapter.benchmarkDirectory, 'journey', batchID); +type FrameHealth = {totalFrames: number; droppedFrames: number; droppedPercent: number; sampleWindowMs: number}; +type JourneyMeasurement = {events: BenchmarkLogEvent[]; frameHealth?: FrameHealth}; +type BenchmarkOrder = 'release-first' | 'optimized-first'; + +type JourneyScenario = { + directory: 'journey' | 'heavy-journey'; + spanNames: readonly string[]; + measure: (device: ReturnType, reader: JourneyBenchmarkReader, fixture: JourneyFixture, timeoutSeconds: number) => Promise; +}; + +const SHORT_JOURNEY: JourneyScenario = {directory: 'journey', spanNames: JOURNEY_SPAN_NAMES, measure: measureJourney}; +const HEAVY_JOURNEY: JourneyScenario = {directory: 'heavy-journey', spanNames: HEAVY_JOURNEY_SPAN_NAMES, measure: measureHeavyJourney}; + +function journeyBenchmarkPaths( + adapter: Pick, + batchID: string, + scenarioDirectory: JourneyScenario['directory'] = 'journey', +): Record { + const directory = join(adapter.benchmarkDirectory, scenarioDirectory, batchID); return {release: join(directory, 'release.csv'), optimized: join(directory, 'pgo-optimized.csv')}; } @@ -36,27 +57,40 @@ async function benchmarkJourneyBuild( runs: number, timeoutSeconds: number, outputPath: string, + scenario: JourneyScenario, ): Promise { console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); adapter.install(kind); - const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-journey-benchmark-${adapter.name}-${kind}-${Date.now()}`}); + const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-${scenario.directory}-${adapter.name}-${kind}-${Date.now()}`}); const reader: JourneyBenchmarkReader = adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); try { device.open(false); await waitForTab(device, 'Account'); await verifyJourneyAccount(device, fixture); - console.log(`Running one unmeasured ${kind} journey warm-up.`); - await measureJourney(device, reader, timeoutSeconds); + console.log(`Running one unmeasured ${kind} ${scenario.directory} warm-up.`); + await scenario.measure(device, reader, fixture, timeoutSeconds); - const recorder = createBenchmarkRecorder({spanNames: [...JOURNEY_SPAN_NAMES], runs, outputPath}); + const recorder = createBenchmarkRecorder({spanNames: [...scenario.spanNames], runs, outputPath}); + const frameRows: string[] = []; for (let runNumber = 1; runNumber <= runs; runNumber += 1) { - const events = await measureJourney(device, reader, timeoutSeconds); - console.log(`${kind} journey ${runNumber}/${runs}: ${recorder.record(events, runNumber).join(', ')}`); + const measurement = await scenario.measure(device, reader, fixture, timeoutSeconds); + console.log(`${kind} ${scenario.directory} ${runNumber}/${runs}: ${recorder.record(measurement.events, runNumber).join(', ')}`); + if (measurement.frameHealth) { + const {totalFrames, droppedFrames, droppedPercent, sampleWindowMs} = measurement.frameHealth; + frameRows.push(`${runNumber},${totalFrames},${droppedFrames},${droppedPercent},${sampleWindowMs}`); + console.log(`Android scroll frames: ${droppedFrames}/${totalFrames} missed deadlines (${droppedPercent.toFixed(1)}%).`); + } } await verifyJourneyAccount(device, fixture); - await recorder.complete(`${kind} journey metrics`); + await recorder.complete(`${kind} ${scenario.directory} metrics`); + if (frameRows.length > 0) { + if (frameRows.length !== runs) { + fail(`Only ${frameRows.length}/${runs} Android scroll frame samples were recorded.`); + } + writeFileSync(outputPath.replace(/\.csv$/, '-frames.csv'), `run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs\n${frameRows.join('\n')}\n`); + } } finally { try { device.command('close'); @@ -67,7 +101,7 @@ async function benchmarkJourneyBuild( } /** Uses a new app process for each sample; automation latency is outside the timed spans. */ -async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, timeoutSeconds: number): Promise { +async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, _fixture: JourneyFixture, timeoutSeconds: number): Promise { device.open(true); await waitForTab(device, 'Account'); // Normalize the saved Spend view before timing either tab transition. This makes the measured taps warm navigations from a known Expenses state. @@ -88,13 +122,87 @@ async function measureJourney(device: ReturnType, re await showTab(device, 'Inbox'); device.waitLabel('All'); const inboxEvents = await reader.collect(inboxSpans, timeoutSeconds); - return [...spendEvents, ...inboxEvents]; + return {events: [...spendEvents, ...inboxEvents]}; } -async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, fixture: JourneyFixture, runs: number, timeoutSeconds: number): Promise { +/** Opens a populated chat and fast-scrolls its history; device-command time is never a benchmark sample. */ +async function measureHeavyJourney( + device: ReturnType, + reader: JourneyBenchmarkReader, + fixture: JourneyFixture, + timeoutSeconds: number, +): Promise { + device.open(true); + await waitForTab(device, 'Account'); + await showTab(device, 'Inbox'); + await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); + const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); + if (device.platform === 'android') { + // Android gfxinfo resets after each read. Exclude search and report opening from this frame window. + readAndroidFrameHealth(device, false); + } + await scrollBothWays(device, Math.min(fixture.scrolls, 6), true, 'up', true); + return {events, frameHealth: device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined}; +} + +function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealth | undefined { + const result = device.command('perf', 'frames'); + if (!isRecord(result) || !isRecord(result.metrics) || !isRecord(result.metrics.fps) || !isRecord(result.sampling) || !isRecord(result.sampling.fps)) { + throw new Error('Android frame sampler returned an unexpected result.'); + } + if (result.sampling.fps.resetsAfterRead !== true || result.metrics.fps.available !== true) { + throw new Error('Android frame sampler is unavailable or did not reset its counter.'); + } + const {totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs} = result.metrics.fps; + if ( + typeof totalFrameCount !== 'number' || + typeof droppedFrameCount !== 'number' || + typeof droppedFramePercent !== 'number' || + typeof sampleWindowMs !== 'number' || + ![totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs].every(Number.isFinite) + ) { + throw new Error('Android frame sampler did not return finite frame counts.'); + } + if (requireFrames && totalFrameCount === 0) { + throw new Error('The Android scroll produced no measured frames. Discard this run.'); + } + return {totalFrames: totalFrameCount, droppedFrames: droppedFrameCount, droppedPercent: droppedFramePercent, sampleWindowMs}; +} + +async function benchmarkJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, SHORT_JOURNEY, order); +} + +async function benchmarkHeavyJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, HEAVY_JOURNEY, order); +} + +async function benchmarkJourneyScenario( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + scenario: JourneyScenario, + order: BenchmarkOrder, +): Promise { const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); - const paths = journeyBenchmarkPaths(adapter, batchID); - const directory = join(adapter.benchmarkDirectory, 'journey', batchID); + const paths = journeyBenchmarkPaths(adapter, batchID, scenario.directory); + const directory = join(adapter.benchmarkDirectory, scenario.directory, batchID); if (existsSync(directory)) { fail(`Journey benchmark batch already exists: ${directory}`); } @@ -109,7 +217,18 @@ async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, f accountEmail: fixture.accountEmail, runs, timeoutSeconds, - spans: JOURNEY_SPAN_NAMES, + order, + scenario: scenario.directory, + spans: scenario.spanNames, + ...(scenario === HEAVY_JOURNEY + ? { + scrollsPerDirection: Math.min(fixture.scrolls, 6), + scrollDurationMs: 120, + scrollTimeMeasured: false, + androidScrollFrameHealthMeasured: true, + iosScrollFrameHealthMeasured: false, + } + : {}), sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), artifacts: adapter.artifactPaths, @@ -122,24 +241,25 @@ async function benchmarkJourneyAll(adapter: PlatformAdapter, deviceID: string, f 2, )}\n`, ); - console.log(`Journey benchmark batch: ${directory}`); - console.log('=== Journey benchmark phase 1/2: Release ==='); - await benchmarkJourneyBuild(adapter, 'release', deviceID, fixture, runs, timeoutSeconds, paths.release); - console.log('=== Journey benchmark phase 2/2: PGO optimized ==='); - await benchmarkJourneyBuild(adapter, 'optimized', deviceID, fixture, runs, timeoutSeconds, paths.optimized); - await compareJourneyBenchmarks(paths, runs); + console.log(`${scenario.directory} benchmark batch: ${directory}`); + const phases: BenchmarkKind[] = order === 'release-first' ? ['release', 'optimized'] : ['optimized', 'release']; + for (const [index, kind] of phases.entries()) { + console.log(`=== ${scenario.directory} benchmark phase ${index + 1}/2: ${kind} ===`); + await benchmarkJourneyBuild(adapter, kind, deviceID, fixture, runs, timeoutSeconds, paths[kind], scenario); + } + await compareJourneyBenchmarks(paths, runs, scenario, adapter.name); } -async function compareJourneyBenchmarks(paths: Record, runs: number): Promise { +async function compareJourneyBenchmarks(paths: Record, runs: number, scenario: JourneyScenario, platform: PlatformAdapter['name']): Promise { if (!existsSync(paths.release) || !existsSync(paths.optimized)) { fail('Missing paired journey samples in this benchmark batch.'); } const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(paths.release), readBenchmarkSamples(paths.optimized)]); - const release = benchmarkMetrics(releaseSamples, JOURNEY_SPAN_NAMES); - const optimized = benchmarkMetrics(optimizedSamples, JOURNEY_SPAN_NAMES); + const release = benchmarkMetrics(releaseSamples, scenario.spanNames); + const optimized = benchmarkMetrics(optimizedSamples, scenario.spanNames); - console.log('=== Journey benchmark comparison (positive percentages are faster) ==='); - for (const span of JOURNEY_SPAN_NAMES) { + console.log(`=== ${scenario.directory} benchmark comparison (positive percentages are faster) ===`); + for (const span of scenario.spanNames) { const baseline = release[span]?.stats; const pgo = optimized[span]?.stats; if (!baseline || !pgo || baseline.runs !== runs || pgo.runs !== runs) { @@ -148,9 +268,38 @@ async function compareJourneyBenchmarks(paths: Record, ru const improvement = ((baseline.average - pgo.average) / baseline.average) * 100; console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); } + if (scenario === HEAVY_JOURNEY && platform === 'android') { + const releaseFrames = readAndroidFrameSamples(paths.release.replace(/\.csv$/, '-frames.csv'), runs); + const optimizedFrames = readAndroidFrameSamples(paths.optimized.replace(/\.csv$/, '-frames.csv'), runs); + console.log(`Android scroll missed-frame share: release ${releaseFrames.toFixed(2)}%, PGO ${optimizedFrames.toFixed(2)}% (lower is better; separate from report-open latency).`); + } console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); } +function readAndroidFrameSamples(path: string, runs: number): number { + if (!existsSync(path)) { + fail(`Missing Android scroll frame samples: ${path}`); + } + const rows = readFileSync(path, 'utf8').trim().split('\n'); + if (rows.shift() !== 'run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs' || rows.length !== runs) { + fail(`Expected ${runs} Android scroll frame samples in ${path}.`); + } + let totalFrames = 0; + let droppedFrames = 0; + for (const [index, row] of rows.entries()) { + const values = row.split(',').map(Number); + const runNumber = values.at(0) ?? NaN; + const frames = values.at(1) ?? NaN; + const dropped = values.at(2) ?? NaN; + if (values.length !== 5 || values.some((value) => !Number.isFinite(value)) || runNumber !== index + 1 || frames <= 0 || dropped < 0 || dropped > frames) { + fail(`Invalid Android scroll frame sample ${index + 1} in ${path}.`); + } + totalFrames += frames; + droppedFrames += dropped; + } + return (droppedFrames / totalFrames) * 100; +} + async function sha256Artifact(path: string): Promise { if (lstatSync(path).isDirectory()) { return sha256Directory(path); @@ -193,4 +342,5 @@ async function sha256Directory(directory: string): Promise { return hash.digest('hex'); } -export {benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; +export {benchmarkHeavyJourneyAll, benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; +export type {BenchmarkOrder}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index 5d23ead14775..ceebb55ee402 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -109,7 +109,7 @@ async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): device.waitLabel('Priority mode, Most recent'); } -async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false): Promise { +async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false, beforePress?: () => void): Promise { await showTab(device, 'Inbox'); device.press(idSelector('searchButton')); device.fill('search-autocomplete-text-input', report.query); @@ -124,6 +124,7 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson if (!resultLabel) { throw new Error(`No search result matched ${report.resultLabelPrefix}.`); } + beforePress?.(); device.pressLabel(resultLabel); device.wait(idSelector('composer')); if (!device.hasLabel(report.title)) { @@ -284,15 +285,15 @@ function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined return first; } -async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down', fast = false): Promise { const initialNodes = device.snapshot(); - const pixels = scrollDistance(initialNodes); + const pixels = scrollDistance(initialNodes, fast); let previous = contentSignature(initialNodes); let changed = 0; for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { let stationary = 0; for (let scroll = 0; scroll < count; scroll += 1) { - device.command('scroll', direction, '--pixels', String(pixels), '--settle'); + device.command('scroll', direction, '--pixels', String(pixels), ...(fast ? ['--duration-ms', '120'] : []), '--settle'); const current = contentSignature(device.snapshot()); if (current !== previous) { changed += 1; @@ -335,13 +336,13 @@ function contentSignature(nodes: JourneyNode[]): string { } /** Keep both ends of the gesture inside the center of portrait lists, below Spend's fixed header. */ -function scrollDistance(nodes: JourneyNode[]): number { +function scrollDistance(nodes: JourneyNode[], fast = false): number { const scrollAreas = nodes.filter((node) => /ScrollView|RecyclerView|ScrollArea/i.test(node.type) && node.width > 0 && node.height > 0); const viewport = scrollAreas.toSorted((left, right) => right.width * right.height - left.width * left.height).at(0); if (!viewport) { throw new Error('Cannot determine scroll bounds on this device.'); } - return Math.round(Math.min(viewport.height * 0.25, viewport.width * 0.6)); + return Math.round(Math.min(viewport.height * (fast ? 0.65 : 0.25), viewport.width * (fast ? 1.5 : 0.6))); } /** Match stable report identity while allowing the last-message preview to change. */ @@ -365,8 +366,10 @@ export { findReportResult, findTabNode, inAppBackTapPoint, + openReport, prepareJourney, runJourneyWorkload, + scrollBothWays, scrollDistance, selectSpendSection, showTab, diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 2343adaa03fb..88af0b143175 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -10,6 +10,7 @@ import {resolve} from 'node:path'; import type {BenchmarkStats} from '../lib/benchmarkStatistics'; import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; +import type {BenchmarkOrder} from './journeyBenchmark'; import type {BenchmarkKind, PlatformAdapter} from './shared'; import {benchmarkStartups as runBenchmarkStartups} from '../lib/benchmarkAppStartup'; @@ -18,12 +19,13 @@ import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppB import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; -import {benchmarkJourneyAll} from './journeyBenchmark'; +import {benchmarkHeavyJourneyAll, benchmarkJourneyAll} from './journeyBenchmark'; import {parseJourneyFixture} from './journeyConfig'; import {STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_STARTUP_WAIT_SECONDS = 30; +const BENCHMARK_ORDERS = ['release-first', 'optimized-first'] as const; const WORKFLOW_COMMANDS = [ 'build-release', 'build-instrumented', @@ -38,6 +40,7 @@ const WORKFLOW_COMMANDS = [ 'benchmark', 'compare-benchmarks', 'benchmark-journey', + 'benchmark-heavy-journey', 'dump', 'pull', 'merge', @@ -86,6 +89,10 @@ async function main(): Promise { description: 'Approved account fixture for journey benchmarks', required: false, }, + 'benchmark-order': { + description: 'Journey phase order; alternate it across paired batches to check build-order drift', + required: false, + }, profile: { description: 'Immutable merged profile to use when building the optimized app', required: false, @@ -104,6 +111,7 @@ async function main(): Promise { cli.namedArgs.device, cli.namedArgs.fixture, cli.namedArgs.profile, + cli.namedArgs['benchmark-order'], ); } @@ -116,6 +124,7 @@ async function runWorkflow( deviceIdentifier?: string, fixturePath?: string, profilePath?: string, + benchmarkOrder?: string, ): Promise { const adapter = getAdapter(platformName, appID, deviceIdentifier); @@ -160,7 +169,12 @@ async function runWorkflow( return; case 'benchmark-journey': { const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); - await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds); + await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + } + case 'benchmark-heavy-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkHeavyJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); return; } case 'dump': @@ -177,6 +191,10 @@ async function runWorkflow( } } +function readBenchmarkOrder(value?: string): BenchmarkOrder { + return value ? parseChoice(value, BENCHMARK_ORDERS, 'Journey benchmark order') : 'release-first'; +} + function readJourneyFixture(fixturePath?: string): ReturnType { if (!fixturePath) { fail('Journey benchmarks require --fixture with the approved signed-in account.'); diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index ed48e4666d92..bc50f922d7fe 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -4,7 +4,8 @@ import type {PlatformName} from '../lib/nativeAppBenchmark'; const STARTUP_SPAN_NAME = 'ManualAppStartup'; const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; -const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES].join(',')}`; +const HEAVY_JOURNEY_SPAN_NAMES = ['/r/*'] as const; +const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES].join(',')}`; type BuildKind = 'release' | 'instrumented' | 'optimized'; type BenchmarkKind = Extract; @@ -30,5 +31,5 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -export {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; +export {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 2d4324694d97..6ec82d67b3ff 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -4,7 +4,7 @@ import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchma import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint, waitForTab} from '@scripts/pgo/journeyWorkload'; -import {BENCHMARK_SPANS_ENVIRONMENT, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; +import {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; @@ -23,11 +23,15 @@ describe('PGO journey safeguards', () => { // Given release and optimized artifacts share one build-time benchmark allowlist. const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); // When / Then the startup span and every short-journey span remain available. - expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES]); + expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES]); expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1')).toEqual({ release: '/pgo/ios/benchmarks/journey/batch-1/release.csv', optimized: '/pgo/ios/benchmarks/journey/batch-1/pgo-optimized.csv', }); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1', 'heavy-journey')).toEqual({ + release: '/pgo/ios/benchmarks/heavy-journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/heavy-journey/batch-1/pgo-optimized.csv', + }); }); it('hashes iOS bundle resources as well as the app executable', async () => { @@ -159,6 +163,7 @@ describe('PGO journey safeguards', () => { const distance = scrollDistance(nodes); expect(distance).toBeGreaterThan(0); expect(distance).toBeLessThanOrEqual((height ?? 0) / 2); + expect(scrollDistance(nodes, true)).toBeGreaterThan(distance); } // Then missing viewport information must fail instead of falling back to a screen-edge gesture. expect(() => scrollDistance([])).toThrow('scroll bounds'); From e83fa480eea4dc032a6ad474a200b805d46cb8b2 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Fri, 25 Sep 2026 15:20:29 +0200 Subject: [PATCH 42/60] fix(pgo): reset heavy benchmark navigation between runs --- scripts/pgo/JOURNEY.md | 2 +- scripts/pgo/journeyBenchmark.ts | 14 +++++++------ scripts/pgo/journeyWorkload.ts | 24 +++++++++++++++++++--- tests/tooling/pgoJourney.test.ts | 35 +++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index ce89443ae32e..b532bb792455 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -91,7 +91,7 @@ Build both the release and optimized artifacts with the current span allowlist f ## Heavy report benchmark -Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes up to six fast swipes toward older messages and six back toward recent messages. It fails if the list does not expose changing content. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. +Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes four fast swipes toward older messages and four back toward recent messages. It checks that content changes after each direction; there is no accessibility snapshot between individual flings. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. ```bash scripts/pgo/pgo.ts android benchmark-heavy-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts index 2f8324e3c78b..40294812c581 100644 --- a/scripts/pgo/journeyBenchmark.ts +++ b/scripts/pgo/journeyBenchmark.ts @@ -18,7 +18,7 @@ import {capture, fail} from '../lib/scriptUtils'; import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; import {createJourneyDevice} from './journeyDevice'; -import {openReport, scrollBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {openReport, scrollFastBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; type JourneyBenchmarkReader = { @@ -67,7 +67,7 @@ async function benchmarkJourneyBuild( adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); try { device.open(false); - await waitForTab(device, 'Account'); + await showTab(device, 'Inbox'); await verifyJourneyAccount(device, fixture); console.log(`Running one unmeasured ${kind} ${scenario.directory} warm-up.`); await scenario.measure(device, reader, fixture, timeoutSeconds); @@ -133,16 +133,18 @@ async function measureHeavyJourney( timeoutSeconds: number, ): Promise { device.open(true); - await waitForTab(device, 'Account'); await showTab(device, 'Inbox'); + await waitForTab(device, 'Account'); await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); if (device.platform === 'android') { // Android gfxinfo resets after each read. Exclude search and report opening from this frame window. readAndroidFrameHealth(device, false); } - await scrollBothWays(device, Math.min(fixture.scrolls, 6), true, 'up', true); - return {events, frameHealth: device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined}; + scrollFastBothWays(device, Math.min(fixture.scrolls, 4)); + const frameHealth = device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined; + await showTab(device, 'Inbox'); + return {events, frameHealth}; } function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealth | undefined { @@ -222,7 +224,7 @@ async function benchmarkJourneyScenario( spans: scenario.spanNames, ...(scenario === HEAVY_JOURNEY ? { - scrollsPerDirection: Math.min(fixture.scrolls, 6), + scrollsPerDirection: Math.min(fixture.scrolls, 4), scrollDurationMs: 120, scrollTimeMeasured: false, androidScrollFrameHealthMeasured: true, diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index ceebb55ee402..c0307cd69232 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -285,15 +285,15 @@ function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined return first; } -async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down', fast = false): Promise { +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { const initialNodes = device.snapshot(); - const pixels = scrollDistance(initialNodes, fast); + const pixels = scrollDistance(initialNodes); let previous = contentSignature(initialNodes); let changed = 0; for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { let stationary = 0; for (let scroll = 0; scroll < count; scroll += 1) { - device.command('scroll', direction, '--pixels', String(pixels), ...(fast ? ['--duration-ms', '120'] : []), '--settle'); + device.command('scroll', direction, '--pixels', String(pixels), '--settle'); const current = contentSignature(device.snapshot()); if (current !== previous) { changed += 1; @@ -312,6 +312,23 @@ async function scrollBothWays(device: JourneyDevice, count: number, requireMovem } } +/** Fixed rapid swipes with one movement check per direction, keeping accessibility polling out of each fling. */ +function scrollFastBothWays(device: Pick, count: number): void { + const initialNodes = device.snapshot(); + const pixels = String(scrollDistance(initialNodes, true)); + let previous = contentSignature(initialNodes); + for (const direction of ['up', 'down']) { + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', pixels, '--duration-ms', '120', ...(scroll === count - 1 ? ['--settle'] : [])); + } + const current = contentSignature(device.snapshot()); + if (current === previous) { + throw new Error(`The populated report did not move during fast ${direction} scrolling. Discard this benchmark run.`); + } + previous = current; + } +} + async function waitForMessage(device: JourneyDevice, token: string): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { @@ -370,6 +387,7 @@ export { prepareJourney, runJourneyWorkload, scrollBothWays, + scrollFastBothWays, scrollDistance, selectSpendSection, showTab, diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 6ec82d67b3ff..80be75cb3a83 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -3,7 +3,17 @@ import {describe, expect, it} from 'bun:test'; import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; -import {allFilterTapPoint, contentSignature, findReportResult, findTabNode, inAppBackTapPoint, scrollDistance, spendSectionTapPoint, waitForTab} from '@scripts/pgo/journeyWorkload'; +import { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + scrollDistance, + scrollFastBothWays, + spendSectionTapPoint, + waitForTab, +} from '@scripts/pgo/journeyWorkload'; import {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; @@ -169,6 +179,29 @@ describe('PGO journey safeguards', () => { expect(() => scrollDistance([])).toThrow('scroll bounds'); }); + it('uses fixed rapid swipes and validates movement in both directions', () => { + // Given a report with distinct content at the recent and older scroll positions. + const snapshots = ['recent', 'older', 'recent'].map((label) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width: 390, height: 844}}, {label}]}, 'test.app'), + ); + const commands: string[][] = []; + let read = 0; + // When the heavy benchmark scrolls in both directions. + scrollFastBothWays( + { + snapshot: () => snapshots.at(read++) ?? [], + command: (...args) => commands.push(args), + }, + 4, + ); + // Then it sends fixed-duration gestures and checks content after each direction. + expect(commands).toHaveLength(8); + expect(commands.every((args) => args.includes('120'))).toBe(true); + expect(commands.filter((args) => args.includes('--settle'))).toHaveLength(2); + expect(commands.at(0)?.at(1)).toBe('up'); + expect(commands.at(-1)?.at(1)).toBe('down'); + }); + it('selects iOS status-bearing bottom tabs without confusing them with page headings', () => { const nodes = parseJourneySnapshot( { From d0cfd6aa2911bd752aa917354668af89748f2b56 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Fri, 25 Sep 2026 16:56:18 +0200 Subject: [PATCH 43/60] fix(pgo): wait for navigation after Android relaunch --- scripts/pgo/journeyWorkload.ts | 41 +++++++++++++++++++------------- tests/tooling/pgoJourney.test.ts | 35 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index c0307cd69232..c126050d20d2 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -133,15 +133,21 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson } async function showTab(device: JourneyDevice, tab: string): Promise { - // Use app back navigation instead of Android's hardware Back, which can dismiss the keyboard or leave the app. - for (let attempt = 0; attempt < 5; attempt += 1) { - const tabNode = findTabNode(device.snapshot(), tab); + const deadline = Date.now() + 30_000; + let backPresses = 0; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const tabNode = findTabNode(nodes, tab); if (tabNode) { device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } - pressInAppBack(device); - await sleep(300); + // A relaunch can expose a loading view before tabs mount. Only navigate back from a real nested page. + if (backPresses < 5 && inAppBackTapPoint(nodes)) { + pressInAppBack(device, nodes); + backPresses += 1; + } + await sleep(500); } throw new Error(`Cannot return to the ${tab} tab.`); } @@ -183,26 +189,29 @@ function pressPreferences(device: JourneyDevice): void { device.pressLabel('Preferences'); } -function pressInAppBack(device: JourneyDevice): void { - if (device.platform === 'ios') { - const tapPoint = inAppBackTapPoint(device.snapshot()); - if (tapPoint) { - device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); - return; - } +function pressInAppBack(device: JourneyDevice, nodes = device.snapshot()): void { + const tapPoint = inAppBackTapPoint(nodes); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; } device.back(); } -/** The iOS search Back button remains visible but is reported non-hittable while the keyboard is open. */ +/** Use the app's top-left Back control; it can be non-hittable on iOS while the keyboard is open. */ function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { - const screen = nodes.find((node) => node.type === 'Application'); + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); if (!screen) { return undefined; } const candidates = nodes.filter( (node) => - node.type === 'Button' && + (node.type === 'Button' || node.type === 'android.widget.Button') && normalizeLabel(node.label) === 'Back' && node.enabled && node.width > 0 && @@ -213,7 +222,7 @@ function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undef node.y < screen.height * 0.2, ); if (candidates.length > 1) { - throw new Error('Ambiguous in-app Back control on iOS.'); + throw new Error('Ambiguous in-app Back control.'); } const back = candidates.at(0); return back ? {x: Math.round(back.x + back.width / 2), y: Math.round(back.y + back.height / 2)} : undefined; diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 80be75cb3a83..0c3c6044c56b 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -11,6 +11,7 @@ import { inAppBackTapPoint, scrollDistance, scrollFastBothWays, + showTab, spendSectionTapPoint, waitForTab, } from '@scripts/pgo/journeyWorkload'; @@ -235,6 +236,40 @@ describe('PGO journey safeguards', () => { await expect(waitForTab(device, 'Account')).resolves.toBeUndefined(); }); + it('waits through Android relaunch loading before navigating from a restored page', async () => { + // Given a loading view, then a restored nested page, then the Inbox tab after app Back. + const screen = {type: 'android.widget.ScrollView', rect: {x: 0, y: 0, width: 1080, height: 2280}}; + const snapshots = [ + [screen], + [screen, {type: 'android.widget.Button', label: 'Back', rect: {x: 21, y: 144, width: 67, height: 105}}], + [screen, {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}], + ].map((nodes) => parseJourneySnapshot({appBundleId: 'test.app', nodes}, 'test.app')); + const commands: string[][] = []; + let read = 0; + const device: Parameters[0] = { + platform: 'android', + snapshot: () => snapshots[Math.min(read++, snapshots.length - 1)], + command: (...args: string[]) => commands.push(args), + wait: () => {}, + press: () => {}, + hasLabel: () => false, + pressLabel: () => {}, + waitLabel: () => {}, + fill: () => {}, + back: () => commands.push(['hardware back']), + open: () => {}, + }; + + // When the benchmark navigates to Inbox after relaunch. + await showTab(device, 'Inbox'); + + // Then loading does not trigger Back, and the app's Back control is used once. + expect(commands).toEqual([ + ['press', '55', '197', '--settle'], + ['press', '324', '2061', '--settle'], + ]); + }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; const snapshot = (tabs: Array) => From 5e0a7d1d9b2cf9e0b952309407515ddbc9e5a413 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 13:43:45 +0100 Subject: [PATCH 44/60] feat(pgo): add native evaluation tooling --- package.json | 1 + scripts/pgo/EVALUATION.md | 36 +++ scripts/pgo/android.ts | 92 +++++- scripts/pgo/androidRelease.init.gradle | 22 ++ scripts/pgo/candidateResolve.ts | 186 +++++++++++ scripts/pgo/evaluationReport.ts | 330 +++++++++++++++++++ scripts/pgo/evaluationRun.ts | 371 ++++++++++++++++++++++ scripts/pgo/evaluationSchema.ts | 110 +++++++ scripts/pgo/ios.ts | 55 +++- scripts/pgo/journey.ts | 4 + scripts/pgo/journeyBenchmark.ts | 148 ++++++--- scripts/pgo/journeyWorkload.ts | 11 +- scripts/pgo/localProof.ts | 116 +++++++ scripts/pgo/pgo.ts | 56 +++- scripts/pgo/pgoCandidate.ts | 194 +++++++++++ scripts/pgo/pgoEvaluation.ts | 45 +++ scripts/pgo/proof.ts | 99 ++++++ scripts/pgo/shared.ts | 20 +- tests/tooling/pgoAndroidRelease.test.ts | 70 ++++ tests/tooling/pgoCandidate.test.ts | 166 ++++++++++ tests/tooling/pgoEvaluationReport.test.ts | 173 ++++++++++ tests/tooling/pgoEvaluationRun.test.ts | 204 ++++++++++++ tests/tooling/pgoIOSRock.test.ts | 32 ++ tests/tooling/pgoJourney.test.ts | 80 ++++- 24 files changed, 2524 insertions(+), 97 deletions(-) create mode 100644 scripts/pgo/EVALUATION.md create mode 100644 scripts/pgo/androidRelease.init.gradle create mode 100644 scripts/pgo/candidateResolve.ts create mode 100644 scripts/pgo/evaluationReport.ts create mode 100644 scripts/pgo/evaluationRun.ts create mode 100644 scripts/pgo/evaluationSchema.ts create mode 100644 scripts/pgo/localProof.ts create mode 100755 scripts/pgo/pgoCandidate.ts create mode 100755 scripts/pgo/pgoEvaluation.ts create mode 100644 scripts/pgo/proof.ts create mode 100644 tests/tooling/pgoAndroidRelease.test.ts create mode 100644 tests/tooling/pgoCandidate.test.ts create mode 100644 tests/tooling/pgoEvaluationReport.test.ts create mode 100644 tests/tooling/pgoEvaluationRun.test.ts create mode 100644 tests/tooling/pgoIOSRock.test.ts diff --git a/package.json b/package.json index 63675e657e66..0423ee4c7614 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "server:vcr:dev": "npm run dev -w @expensify/victory-chart-renderer --", "server:vcr:test": "npm run test -w @expensify/victory-chart-renderer --", "test:bun": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./server ./tests/tooling", + "test:pgo-ci": "TZ=utc bun test --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./tests/tooling/pgoCandidate.test.ts ./tests/tooling/pgoEvaluationReport.test.ts ./tests/tooling/pgoEvaluationRun.test.ts ./tests/tooling/pgoAndroidRelease.test.ts ./tests/tooling/pgoIOSRock.test.ts ./tests/tooling/pgoJourney.test.ts", "eval:proposal-police": "bun test --path-ignore-patterns='**/__never__/**' ./evals/proposalPolice/", "server:vcr:build:linux": "npm run build:linux -w @expensify/victory-chart-renderer --", "server:vcr:build:linux-arm": "npm run build:linux-arm -w @expensify/victory-chart-renderer --", diff --git a/scripts/pgo/EVALUATION.md b/scripts/pgo/EVALUATION.md new file mode 100644 index 000000000000..553d9f38149d --- /dev/null +++ b/scripts/pgo/EVALUATION.md @@ -0,0 +1,36 @@ +# Native PGO evaluation + +The evaluation workflow measures a native candidate without selecting a release artifact or changing deployment. It records the actual App checkout, Mobile gitlink, version and producer attempt before evaluating each platform whose native build succeeded. A failed web deployment does not invalidate an otherwise successful native build. + +Each platform uses a dedicated physical runner. Its concurrency queue preserves up to 100 pending evaluations instead of replacing the previous pending candidate. Monitor the queue and rerun candidates if that limit is exceeded. Configure the `pgo-android` and `pgo-ios` runner labels and these settings before enabling device jobs: + +| Variable | Required runner configuration | +| --- | --- | +| `PGO_ANDROID_DEVICE_ID` | The approved physical Android serial | +| `PGO_ANDROID_APP_ID` | The dedicated evaluation application identifier | +| `PGO_ANDROID_FIXTURE_PATH` | An absolute path to the privately provisioned approved account fixture | +| `PGO_ANDROID_ENVFILE` | An absolute path to the fixed merged Android Release environment file | +| `PGO_IOS_DEVICE_ID` | The approved physical iPhone UDID | +| `PGO_IOS_APP_ID` | The dedicated evaluation bundle identifier | +| `PGO_IOS_FIXTURE_PATH` | An absolute path to the privately provisioned approved account fixture | +| `PGO_IOS_ENVFILE` | An absolute path to the fixed merged iOS Release environment file | +| `IOS_CODE_SIGNING_ALLOWED` | `true` for physical-device iOS evaluation | +| `IOS_DEVELOPMENT_TEAM` | The authorized Apple development team identifier | + +The iOS runner also needs an installed development certificate and a provisioning profile that permits the selected physical iPhone and dedicated bundle identifier. Configure the signing variables as repository variables. Keep certificate material and signing credentials private on the runner. + +The configured application and bundle identifiers must match the identifiers produced by the candidate's native configuration, environment and provisioning profile. These selectors do not rewrite native identifiers. A dedicated physical runner can use an existing matching identifier. + +The account must already be signed in to the dedicated application. Authentication is never automated. The fixture must explicitly authorize the heavy account and any training messages to its personal chat. Other reports are read-only. Keep account fixtures, environment files and signing credentials on the protected runner. Store device serials and private fixture/environment paths in repository secrets so the Actions environment banner masks them. Application identifiers and nonsecret signing settings can use repository variables. Do not store fixture or environment contents in variables or artifacts. + +The runner builds a baseline and instrumented Release, verifies instrumentation, then collects three complete journeys. It validates the archived profiles before building the optimized candidate. Android uses R8 while disabling FullStory build processing and upload phases. The runtime SDK may remain present. These dedicated evaluation builds are not store-equivalent releases. + +The runner records twenty samples per binary for heavy report navigation, separate warm navigation and cold startup. It repeats each workload with the opposite binary order. Each binary gets a five-second cool-down followed by one discarded warm-up before the measured samples. Thermal state is not verified. Warm navigation paths overlap training, and the heavy report first-layout span may stop at a skeleton rather than fully loaded content. + +Android frame telemetry is optional. Reports retain available native janky-frame counts and explicitly count unavailable observations. Those counters cover automation windows, with unverified reset and isolation boundaries. They do not establish isolated scrolling performance. No iOS frame metric is reported. + +Before and after each accepted stage, the runner checks the candidate commits, evaluated source contents, toolchain, fixed environment, fixture, profile and artifact hashes. The reviewed measurement code is an explicit overlay on the candidate, with its own hash. For iOS, generated CocoaPods lock files must match across all three build modes. The runner retains those locks privately and restores only the original tracked lockfile bytes before checking source consistency. + +Public artifacts contain numeric observations, distributions and hashes. Raw device output, screenshots, fixtures, environment values and detailed errors remain private on the runner. Private evidence lives under the candidate's ignored evaluation directory and needs a runner retention policy. Compiler consumption remains unverified in the public result unless execution evidence is independently established. + +Automatic evaluations follow completed trusted staging or production deploy runs. A manual rerun selects a producer run ID and its current immutable attempt. The exact approved same-repository preview branch can run before merge. Forks and arbitrary heads cannot access device jobs. Preview reports stay in workflow summaries and artifacts. A deployment report can update only its version-matching labelled historical checklist; otherwise the workflow visibly skips the comment. Evaluation never promotes or replaces the ordinary native artifact. diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index e0ef640aa0a8..b656d6e13903 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -10,20 +10,22 @@ import process from 'node:process'; import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PlatformAdapter} from './shared'; import {capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from '../lib/scriptUtils'; +import {androidBuildEnvironment} from './proof'; import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { + const releasePipeline = androidReleasePipeline(process.env.PGO_ANDROID_RELEASE_PIPELINE, process.env.PGO_ANDROID_SHRINK); const ndkVersion = '27.1.12297006'; const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); const rawProfileDirectory = join(profileDirectory, 'raw'); - const apkDirectory = join(profileDirectory, 'apk'); + const apkDirectory = join(profileDirectory, releasePipeline.artifactDirectory); const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); const buildGradlePath = join(androidDirectory, 'build.gradle'); - const gradleReleaseApkPath = join(androidDirectory, 'build/intermediates/apk/release/packageRelease/Expensify-release.apk'); + const gradleReleaseApkPath = join(androidDirectory, `build/intermediates/apk/release/${releasePipeline.apkTask}/Expensify-release.apk`); const profileDumpTimeoutSeconds = 5; const artifactPaths: BuildArtifactPaths = { @@ -78,6 +80,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent } function build(kind: BuildKind, profileOverride?: string): void { + const buildEnvironment = androidBuildEnvironment(); const profilePath = profileOverride ?? join(profileDirectory, 'newdot.profdata'); if (kind === 'optimized' && !existsSync(profilePath)) { fail(`Missing ${profilePath}. Run merge first or pass --profile.`); @@ -85,19 +88,45 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const pgoModes: Record = {release: 'off', instrumented: 'generate', optimized: 'use'}; const gradleArguments = [ - ':packageRelease', + releasePipeline.task, '-PpatchedArtifacts.forceBuildFromSource=true', '-PreactNativeArchitectures=arm64-v8a', `-PpgoMode=${pgoModes[kind]}`, - '-x', - 'fullstoryRelease', + ...releasePipeline.arguments, ]; + if (releasePipeline.enabled) { + console.log( + 'Android release pipeline rehearsal enables shrinking and FullStory processing, including external asset uploads. The existing local app ID and signing configuration still apply.', + ); + } else if (releasePipeline.shrinking) { + console.log('Android shrinking rehearsal enables release ProGuard rules and disables FullStory tasks. The existing local app ID and signing configuration still apply.'); + } if (kind === 'optimized') { gradleArguments.push(`-PpgoProfile=${profilePath}`); } - run('/usr/bin/env', [BENCHMARK_SPANS_ENVIRONMENT, 'SENTRY_DISABLE_AUTO_UPLOAD=true', './gradlew', ...gradleArguments], androidDirectory); + run( + '/usr/bin/env', + [ + BENCHMARK_SPANS_ENVIRONMENT, + 'SENTRY_DISABLE_AUTO_UPLOAD=true', + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...androidRockBuildArguments(releasePipeline.task, gradleArguments.slice(1)), + ], + join(rootDirectory, 'Mobile-Expensify'), + ); + if (JSON.stringify(androidBuildEnvironment()) !== JSON.stringify(buildEnvironment)) { + fail('Build environment changed during the Android build. Restore the selected environment file and rebuild.'); + } archiveReleaseApk(artifactPaths[kind]); + writeFileSync( + `${artifactPaths[kind]}.build-settings.json`, + `${JSON.stringify({kind, releasePipeline: releasePipeline.enabled, appID: appID(), sourceBuiltRuntime: true, shrinking: releasePipeline.shrinking, fullStory: releasePipeline.enabled, signing: 'existing-project-configuration', storeEquivalent: false, gradleArguments, buildEnvironment}, null, 2)}\n`, + ); } function install(kind: BuildKind): void { @@ -206,6 +235,55 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent }; } +/** Keep release processing separate from bootstrap artifacts without rewriting local signing or identifiers. */ +function androidReleasePipeline(value?: string, shrinkValue?: string): {enabled: boolean; shrinking: boolean; artifactDirectory: string; task: string; apkTask: string; arguments: string[]} { + if (value && value !== 'true' && value !== 'false') { + fail('PGO_ANDROID_RELEASE_PIPELINE must be true or false.'); + } + if (shrinkValue && shrinkValue !== 'true' && shrinkValue !== 'false') { + fail('PGO_ANDROID_SHRINK must be true or false.'); + } + if (value === 'true' && shrinkValue === 'true') { + fail('Choose PGO_ANDROID_RELEASE_PIPELINE or PGO_ANDROID_SHRINK, not both.'); + } + if (shrinkValue === 'true') { + return { + enabled: false, + shrinking: true, + artifactDirectory: 'apk-shrunk', + task: ':packageRelease', + apkTask: 'packageRelease', + arguments: ['-x', 'fullstoryRelease', '-PpgoSkipFullStory=true', '--init-script', join(rootDirectory, 'scripts/pgo/androidRelease.init.gradle')], + }; + } + if (value !== 'true') { + return {enabled: false, shrinking: false, artifactDirectory: 'apk', task: ':packageRelease', apkTask: 'packageRelease', arguments: ['-x', 'fullstoryRelease']}; + } + return { + enabled: true, + shrinking: true, + artifactDirectory: 'apk-release-pipeline', + task: ':fullstoryRelease', + apkTask: 'fullstoryRelease', + arguments: ['--init-script', join(rootDirectory, 'scripts/pgo/androidRelease.init.gradle')], + }; +} + +/** Rock prefixes the application module and parses the extra Gradle arguments as one quoted argument list. */ +function androidRockBuildArguments(task: string, gradleArguments: string[]): string[] { + return [ + 'build:android', + '--local', + '--verbose', + '--variant', + 'release', + '--tasks', + task.replace(/^:/, ''), + '--extra-params', + gradleArguments.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + function androidApplicationID(buildGradle: string, configuredAppID?: string): string { if (configuredAppID) { return configuredAppID; @@ -222,4 +300,4 @@ function androidProfileReceiverComponent(appID: string): string { } export default createAndroidPgoAdapter; -export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent}; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent, androidReleasePipeline, androidRockBuildArguments}; diff --git a/scripts/pgo/androidRelease.init.gradle b/scripts/pgo/androidRelease.init.gradle new file mode 100644 index 000000000000..ed3bdaf1c414 --- /dev/null +++ b/scripts/pgo/androidRelease.init.gradle @@ -0,0 +1,22 @@ +// Apply release processing without changing the owner's local application ID or signing configuration. +gradle.beforeProject { application -> + def skipFullStory = application.providers.gradleProperty('pgoSkipFullStory').getOrElse('false') == 'true' + if (skipFullStory) { + application.tasks.configureEach { task -> + if (task.name.toLowerCase().startsWith('fullstory')) { + task.enabled = false + } + } + } + application.plugins.withId('com.android.application') { + application.extensions.getByName('androidComponents').finalizeDsl { android -> + def release = android.buildTypes.findByName('release') + if (release == null) { + throw new GradleException('The PGO release pipeline requires a release build type.') + } + release.minifyEnabled = true + release.proguardFiles android.getDefaultProguardFile('proguard-android-optimize.txt'), application.file('proguard-rules.pro') + application.logger.lifecycle("PGO shrinking enabled; FullStory tasks ${skipFullStory ? 'disabled' : 'enabled'}; existing signing and application ID preserved.") + } + } +} diff --git a/scripts/pgo/candidateResolve.ts b/scripts/pgo/candidateResolve.ts new file mode 100644 index 000000000000..aa0dbcad3b76 --- /dev/null +++ b/scripts/pgo/candidateResolve.ts @@ -0,0 +1,186 @@ +// cspell:ignore chrispader +/** Resolve immutable trusted candidates; no candidate code runs on the hosted resolver. */ +import type {EvaluationPlan, EvaluationPlatform} from './evaluationSchema'; + +import {isGitID, isRecord, parseCandidate} from './evaluationSchema'; + +type GitHubRead = (path: string) => Promise; +const repository = 'Expensify/App'; +const previewBranch = '@chrispader/perf/pgo-ci-evaluation'; +const producerName = 'Deploy code to staging or production'; + +async function resolveProducer(metadata: unknown, run: unknown, jobs: unknown, toolingSha: string, read: GitHubRead, platform?: EvaluationPlatform): Promise { + if ( + !isRecord(metadata) || + metadata.schema !== 1 || + metadata.repository !== repository || + !isRecord(run) || + run.name !== producerName || + run.path !== '.github/workflows/deploy.yml' || + run.status !== 'completed' || + !isRecord(run.repository) || + run.repository.full_name !== repository || + !isRecord(jobs) || + !Array.isArray(jobs.jobs) || + !isGitID(toolingSha) + ) { + throw new Error('Untrusted or incomplete producer run.'); + } + if ( + !validID(metadata.runID) || + !validID(metadata.runAttempt) || + !validID(run.id) || + !validID(run.run_attempt) || + !['staging', 'production'].includes(String(run.head_branch)) || + !['push', 'workflow_dispatch'].includes(String(run.event)) || + metadata.runID !== run.id || + metadata.runAttempt !== run.run_attempt || + typeof metadata.nativeEligible !== 'boolean' + ) { + throw new Error('Producer metadata does not match its immutable run attempt.'); + } + const candidate = parseCandidate(metadata.candidate); + if (candidate.environment === 'preview') { + throw new Error('Producer environment must be staging or production.'); + } + await verifyRepositoryCandidate(candidate.appSha, candidate.mobileSha, candidate.version, read); + const comparison = await read(`/repos/${repository}/compare/${candidate.appSha}...${candidate.environment}`); + if (!isRecord(comparison) || !['ahead', 'identical'].includes(String(comparison.status))) { + throw new Error('Producer candidate is not an ancestor of its environment.'); + } + const nativeJobs = jobs.jobs; + const platforms: EvaluationPlatform[] = ['android', 'ios'].filter((value): value is EvaluationPlatform => { + if (!metadata.nativeEligible || (platform && value !== platform)) { + return false; + } + const names = value === 'android' ? ['Build Android HybridApp', 'Build Android HybridApp / Build Android app'] : ['Build iOS HybridApp', 'Build iOS HybridApp / Build iOS HybridApp']; + return nativeJobs.some((job: unknown) => isRecord(job) && names.includes(String(job.name)) && job.conclusion === 'success'); + }); + return { + schema: 1, + repository, + toolingSha, + candidate, + producerRunID: Number(metadata.runID), + producerRunAttempt: Number(metadata.runAttempt), + platforms, + checklistIssueNumber: await matchingChecklist(candidate.version, read), + }; +} + +async function resolvePreview(event: unknown, toolingSha: string, read: GitHubRead, platform?: EvaluationPlatform): Promise { + if ( + !isRecord(event) || + !isRecord(event.pull_request) || + !isRecord(event.pull_request.head) || + !isRecord(event.pull_request.head.repo) || + !isRecord(event.pull_request.base) || + !isRecord(event.pull_request.base.repo) || + event.pull_request.head.repo.full_name !== repository || + event.pull_request.base.repo.full_name !== repository || + event.pull_request.head.ref !== previewBranch || + !isGitID(event.pull_request.head.sha) || + !isGitID(toolingSha) + ) { + throw new Error('Credential devices require the approved same-repository preview branch.'); + } + const sha = event.pull_request.head.sha; + const current = await read(`/repos/${repository}/pulls/${Number(event.number)}`); + if (!isRecord(current) || !isRecord(current.head) || current.head.sha !== sha || current.state !== 'open') { + throw new Error('Preview head changed or pull request is closed.'); + } + const candidate = await repositoryCandidate(sha, read); + return { + schema: 1, + repository, + toolingSha, + candidate: {...candidate, environment: 'preview'}, + producerRunID: null, + producerRunAttempt: null, + platforms: platform ? [platform] : ['android', 'ios'], + checklistIssueNumber: null, + }; +} + +async function repositoryCandidate(sha: string, read: GitHubRead): Promise<{appSha: string; mobileSha: string; version: string}> { + const tree = await read(`/repos/${repository}/git/trees/${sha}`); + const packageFile = await read(`/repos/${repository}/contents/package.json?ref=${sha}`); + if (!isRecord(tree) || !Array.isArray(tree.tree) || !isRecord(packageFile) || typeof packageFile.content !== 'string') { + throw new Error('Missing candidate tree or package metadata.'); + } + const mobile: unknown = tree.tree.find((entry: unknown) => isRecord(entry) && entry.path === 'Mobile-Expensify' && entry.mode === '160000'); + if (!isRecord(mobile) || !isGitID(mobile.sha)) { + throw new Error('Candidate has no immutable Mobile-Expensify gitlink.'); + } + const pkg: unknown = JSON.parse(Buffer.from(packageFile.content, 'base64').toString('utf8')); + if (!isRecord(pkg) || typeof pkg.version !== 'string') { + throw new Error('Missing candidate version.'); + } + return parseCandidate({appSha: sha, mobileSha: mobile.sha, version: pkg.version, environment: 'preview'}); +} + +async function verifyRepositoryCandidate(sha: string, mobile: string, version: string, read: GitHubRead): Promise { + const actual = await repositoryCandidate(sha, read); + if (actual.mobileSha !== mobile || actual.version !== version) { + throw new Error('Producer metadata differs from the actual candidate checkout.'); + } +} + +async function matchingChecklist(version: string, read: GitHubRead): Promise { + const matches: number[] = []; + for (let page = 1; page <= 100; page += 1) { + const issues = await read(`/repos/${repository}/issues?labels=StagingDeployCash&state=all&per_page=100&page=${page}`); + if (!Array.isArray(issues)) { + throw new Error('Invalid deployment checklist response.'); + } + for (const issue of issues) { + if (!checklistMatches(issue, version) || !isRecord(issue) || !validID(issue.number)) { + continue; + } + matches.push(issue.number); + } + if (matches.length > 1) { + throw new Error('Multiple deployment checklists match the exact candidate version.'); + } + if (issues.length < 100) { + return matches.at(0) ?? null; + } + } + throw new Error('Historical checklist search exceeds its bounded pagination limit.'); +} + +function assertProducerEventAttempt(event: unknown, run: unknown): void { + if (isRecord(event) && isRecord(event.workflow_run) && isRecord(run) && event.workflow_run.id === run.id && event.workflow_run.run_attempt === run.run_attempt) { + return; + } + throw new Error('Producer event was superseded by another run or attempt.'); +} + +function selectProducerArtifact(input: unknown, runID: number, runAttempt: number): Record & {id: number} { + if (!isRecord(input) || !Array.isArray(input.artifacts) || typeof input.total_count !== 'number' || input.total_count > 100) { + throw new Error('Invalid producer artifact enumeration.'); + } + const artifacts: unknown[] = input.artifacts.filter((item: unknown) => isRecord(item) && item.name === `pgo-native-candidate-${runAttempt}` && item.expired === false); + const artifact = artifacts.at(0); + if (artifacts.length !== 1 || !isRecord(artifact) || !isRecord(artifact.workflow_run) || artifact.workflow_run.id !== runID || !validID(artifact.id)) { + throw new Error('Missing or ambiguous immutable producer metadata.'); + } + return {...artifact, id: artifact.id}; +} + +function validID(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +function checklistMatches(issue: unknown, version: string): boolean { + if (!isRecord(issue) || typeof issue.body !== 'string' || !Array.isArray(issue.labels) || issue.pull_request) { + return false; + } + return ( + issue.labels.some((label: unknown) => label === 'StagingDeployCash' || (isRecord(label) && label.name === 'StagingDeployCash')) && + issue.body.match(/\*\*Release Version:\*\*\s*`([^`]+)`/)?.at(1) === `${version}-staging` + ); +} + +export {assertProducerEventAttempt, selectProducerArtifact, checklistMatches, matchingChecklist, repositoryCandidate, resolvePreview, resolveProducer}; +export type {GitHubRead}; diff --git a/scripts/pgo/evaluationReport.ts b/scripts/pgo/evaluationReport.ts new file mode 100644 index 000000000000..64eef208ae21 --- /dev/null +++ b/scripts/pgo/evaluationReport.ts @@ -0,0 +1,330 @@ +// cspell:ignore cooldown +/** Validate and summarize public numeric evaluation evidence without selecting a release binary. */ +import {isDeepStrictEqual} from 'node:util'; + +import type {BenchmarkSample} from '../lib/benchmarkStatistics'; +import type {GitHubRead} from './candidateResolve'; +import type {EvaluationPlan, EvaluationPlatform} from './evaluationSchema'; + +import {benchmarkStats, percentile} from '../lib/benchmarkStatistics'; +import {checklistMatches} from './candidateResolve'; +import {candidateKey, isRecord, isSha256} from './evaluationSchema'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; + +// The hyphenated scenario name is the persisted benchmark schema. +// eslint-disable-next-line @typescript-eslint/naming-convention +const scenarios = {startup: ['ManualAppStartup'], journey: [...JOURNEY_SPAN_NAMES], 'heavy-journey': [...HEAVY_JOURNEY_SPAN_NAMES]}; +const publicToolchainKeys = [ + 'compilerVersion', + 'profileToolVersion', + 'sdkVersion', + 'buildToolVersion', + 'identitySha256', + 'reviewedOverlaySha256', + 'fixtureSha256', + 'environmentSha256', + 'releasePodLockSha256', + 'instrumentedPodLockSha256', + 'optimizedPodLockSha256', +]; + +async function evaluationReport(plan: EvaluationPlan, inputs: unknown[], read: GitHubRead) { + const platforms = plan.platforms.map((platform) => { + const candidates = inputs.filter((input) => isRecord(input) && input.platform === platform); + if (candidates.length > 1) { + throw new Error('Duplicate platform evaluation evidence.'); + } + const input = candidates.at(0); + if (!input) { + return {platform, status: 'missing' as const, cohorts: undefined}; + } + return summarizePlatform(plan, platform, input); + }); + const status = platforms.every((entry) => entry.status === 'passed') ? 'complete' : 'incomplete'; + const limitations = [ + 'Evaluation only: canonical baseline artifacts and deployment selection are unchanged.', + 'Measurements retain every sample, including IQR outliers; summaries are descriptive, not a significance or causal benefit claim.', + 'Heavy report first layout can contain a skeleton and does not prove content readiness. Warm navigation and the heavy fixture overlap training.', + 'Device temperature, CI repeatability, store equivalence and isolated scrolling improvement are unverified.', + 'Native artifacts are local evaluation builds with the reviewed tooling overlay; the source fingerprint is not a claim of a pristine candidate checkout.', + 'Compiler configuration and binary/profile hashes alone do not establish actual profile consumption or full coverage.', + ]; + const summary = { + schema: 1, + status, + candidate: plan.candidate, + toolingSha: plan.toolingSha, + producerRunID: plan.producerRunID, + producerRunAttempt: plan.producerRunAttempt, + platforms, + limitations, + }; + const heading = `PGO evaluation for ${plan.candidate.version} (${plan.candidate.environment})`; + const rows = platforms.flatMap((entry) => + entry.cohorts + ? entry.cohorts.flatMap((cohort) => + cohort.statistics.map( + (metric) => + `| ${entry.platform} | ${cohort.scenario} | ${cohort.order} | ${metric.span} | ${metric.release.average.toFixed(2)} | ${metric.optimized.average.toFixed(2)} | ${metric.meanEffectPercent?.toFixed(2) ?? 'N/A'}% |`, + ), + ) + : [`| ${entry.platform} | ${entry.status} | — | — | — | — | — |`], + ); + const markdown = [ + heading, + '', + `Candidate App: ${plan.candidate.appSha}; Mobile: ${plan.candidate.mobileSha}. Tooling: ${plan.toolingSha}.`, + '', + `Evidence status: ${status}. Positive effects mean (baseline − optimized) / baseline × 100.`, + '', + '| Platform | Workload | Order | Span | Baseline mean ms | Optimized mean ms | Effect |', + '| --- | --- | --- | --- | ---: | ---: | ---: |', + ...rows, + '', + 'The attached JSON contains all sanitized raw samples, mean, median, p10/p90, min/max, quartiles, IQR outlier counts and provenance. Both orders are reported separately.', + '', + ...limitations.map((text) => `- ${text}`), + '', + ].join('\n'); + let targetIssueNumber: number | null = null; + if (plan.checklistIssueNumber !== null) { + const issue = await read(`/repos/Expensify/App/issues/${plan.checklistIssueNumber}`); + if (checklistMatches(issue, plan.candidate.version)) { + targetIssueNumber = plan.checklistIssueNumber; + } + } + const marker = ``; + const runLink = process.env.GITHUB_RUN_ID ? `https://github.com/Expensify/App/actions/runs/${process.env.GITHUB_RUN_ID}` : 'Workflow run link unavailable.'; + const body = `Chris's AI co-pilot reporting the CI evaluation. 🤖\n\n${marker}\n${markdown}\nEvaluation run: ${runLink}\n`; + if (body.length > 32000) { + throw new Error('Evaluation comment exceeds the bounded report size.'); + } + return { + summary, + markdown, + comment: { + marker, + body, + targetIssueNumber, + version: plan.candidate.version, + skippedReason: targetIssueNumber === null ? 'No exact historical deployment checklist matches this candidate.' : null, + }, + }; +} + +function summarizePlatform(plan: EvaluationPlan, platform: EvaluationPlatform, input: unknown) { + if (!isRecord(input) || input.schema !== 1 || input.platform !== platform || !isDeepStrictEqual(input.candidate, plan.candidate) || input.toolingSha !== plan.toolingSha) { + throw new Error('Platform result differs from the exact evaluation candidate.'); + } + if (input.status === 'failed') { + const stages = [ + 'candidate-guard', + 'account-preflight', + 'build-release', + 'build-instrumented', + 'verify-instrumented', + 'install-instrumented', + 'training', + 'validate-training', + 'build-optimized', + ...['startup', 'journey', 'heavy-journey'].flatMap((scenario) => ['release-first', 'optimized-first'].map((order) => `${scenario}-${order}`)), + ]; + if (input.failureStage !== undefined && (typeof input.failureStage !== 'string' || !stages.includes(input.failureStage))) { + throw new Error('Invalid categorical failure stage.'); + } + return {platform, status: 'failed' as const, failureStage: input.failureStage}; + } + if ( + input.status !== 'passed' || + !isRecord(input.artifactSha256) || + !['release', 'instrumented', 'optimized'].every((kind) => isSha256(input.artifactSha256 && isRecord(input.artifactSha256) ? input.artifactSha256[kind] : undefined)) || + !isSha256(input.profileSha256) || + !isSha256(input.sourceIdentity) || + !isRecord(input.toolchain) || + !Array.isArray(input.cohorts) || + input.cohorts.length !== 6 || + !['verified', 'unverified'].includes(String(input.compilerConsumption)) + ) { + throw new Error('Incomplete platform provenance.'); + } + const toolchain = Object.fromEntries( + publicToolchainKeys.flatMap((key) => { + const value = isRecord(input.toolchain) ? input.toolchain[key] : undefined; + if (value === undefined) { + return []; + } + if (typeof value !== 'string' || value.length > 2000 || (key.endsWith('Sha256') ? !isSha256(value) : !/^[\w .():,+#-]+$/.test(value))) { + throw new Error('Toolchain field is not a public version or digest.'); + } + return [[key, value]]; + }), + ); + const cohorts = input.cohorts.map(summarizeCohort); + for (const scenario of Object.keys(scenarios)) { + const orders = cohorts + .filter((cohort) => cohort.scenario === scenario) + .map((cohort) => cohort.order) + .sort(); + if (!isDeepStrictEqual(orders, ['optimized-first', 'release-first'])) { + throw new Error('Expected complete opposite orders for each workload.'); + } + } + return { + platform, + status: 'passed' as const, + artifactSha256: {release: String(input.artifactSha256.release), instrumented: String(input.artifactSha256.instrumented), optimized: String(input.artifactSha256.optimized)}, + profileSha256: input.profileSha256, + sourceIdentity: input.sourceIdentity, + toolchain, + compilerConsumption: input.compilerConsumption, + cohorts, + }; +} + +function summarizeCohort(input: unknown) { + if ( + !isRecord(input) || + (input.scenario !== 'startup' && input.scenario !== 'journey' && input.scenario !== 'heavy-journey') || + !['release-first', 'optimized-first'].includes(String(input.order)) || + input.runs !== 20 || + input.warmUpRunsPerBinary !== 1 || + input.cooldownSecondsPerBinary !== 5 || + !isSha256(input.releaseCsvSha256) || + !isSha256(input.optimizedCsvSha256) + ) { + throw new Error('Invalid evaluation cohort metadata.'); + } + const scenario = input.scenario; + const spans = scenarios[scenario]; + if (!isDeepStrictEqual(input.spans, spans)) { + throw new Error('Unexpected evaluation spans.'); + } + const release = numericSamples(input.release, spans); + const optimized = numericSamples(input.optimized, spans); + const statistics = spans.map((span) => { + const baseline = distribution(release.filter((sample) => sample.span === span).map((sample) => sample.durationMs)); + const pgo = distribution(optimized.filter((sample) => sample.span === span).map((sample) => sample.durationMs)); + return {span, release: baseline, optimized: pgo, meanEffectPercent: baseline.average > 0 ? (100 * (baseline.average - pgo.average)) / baseline.average : null}; + }); + return { + scenario, + order: String(input.order), + runs: 20, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: 5, + releaseCsvSha256: input.releaseCsvSha256, + optimizedCsvSha256: input.optimizedCsvSha256, + release, + optimized, + statistics, + frames: frameCoverage(input.frames), + }; +} + +function numericSamples(input: unknown, spans: readonly string[]): BenchmarkSample[] { + if (!Array.isArray(input) || input.length !== 20 * spans.length) { + throw new Error('Incomplete or extra timing samples.'); + } + const samples = input.map((sample: unknown) => { + if ( + !isRecord(sample) || + !Number.isSafeInteger(sample.run) || + typeof sample.span !== 'string' || + !spans.includes(sample.span) || + typeof sample.durationMs !== 'number' || + !Number.isFinite(sample.durationMs) || + sample.durationMs < 0 + ) { + throw new Error('Invalid timing sample.'); + } + return {run: Number(sample.run), span: sample.span, durationMs: sample.durationMs}; + }); + for (const span of spans) { + if ( + !isDeepStrictEqual( + samples.filter((sample) => sample.span === span).map((sample) => sample.run), + Array.from({length: 20}, (_, index) => index + 1), + ) + ) { + throw new Error('Timing samples must be exactly ordered 1 through 20.'); + } + } + return samples; +} + +function frameCoverage(input: unknown) { + if (input === undefined) { + return {status: 'unverified' as const}; + } + if (!isRecord(input)) { + throw new Error('Invalid numeric frame coverage.'); + } + return { + status: 'recorded' as const, + release: summarizeFrames(input.release), + optimized: summarizeFrames(input.optimized), + meaning: 'Native janky frames, not FPS or literal dropped frames; variable automation window and reset isolation are unverified.', + }; +} + +function summarizeFrames(input: unknown) { + if (!Array.isArray(input) || input.length !== 20) { + throw new Error('Frame coverage must record every measured run.'); + } + const samples = input.map((sample: unknown, index) => { + if (!isRecord(sample) || sample.run !== index + 1) { + throw new Error('Frame coverage is not ordered.'); + } + if (sample.status === 'unavailable') { + return {run: index + 1, status: 'unavailable' as const}; + } + const {totalFrames, jankyFrames, originalPercent, sampleWindowMs} = sample; + if ( + sample.status !== 'available' || + typeof totalFrames !== 'number' || + typeof jankyFrames !== 'number' || + typeof originalPercent !== 'number' || + typeof sampleWindowMs !== 'number' || + ![totalFrames, jankyFrames, originalPercent, sampleWindowMs].every(Number.isFinite) || + !Number.isInteger(totalFrames) || + !Number.isInteger(jankyFrames) || + totalFrames <= 0 || + jankyFrames < 0 || + jankyFrames > totalFrames || + originalPercent < 0 || + originalPercent > 100 || + sampleWindowMs <= 0 + ) { + throw new Error('Invalid available native frame sample.'); + } + return {run: index + 1, status: 'available' as const, totalFrames, jankyFrames, originalPercent, sampleWindowMs}; + }); + const available = samples.filter((sample) => sample.status === 'available'); + const sumTotalFrames = available.reduce((total, sample) => total + sample.totalFrames, 0); + const sumJankyFrames = available.reduce((total, sample) => total + sample.jankyFrames, 0); + return { + requestedSamples: 20, + availableSamples: available.length, + unavailableSamples: 20 - available.length, + samples, + summary: available.length + ? { + sumTotalFrames, + sumJankyFrames, + weightedJankyPercent: (100 * sumJankyFrames) / sumTotalFrames, + originalPercent: distribution(available.map((sample) => sample.originalPercent)), + sampleWindowMs: distribution(available.map((sample) => sample.sampleWindowMs)), + } + : null, + }; +} + +function distribution(values: readonly number[]) { + const stats = benchmarkStats(values); + const q1 = percentile(values, 0.25); + const q3 = stats.p75; + const iqr = q3 - q1; + return {...stats, median: stats.p50, p10: percentile(values, 0.1), q1, q3, iqr, iqrOutliers: values.filter((value) => value < q1 - 1.5 * iqr || value > q3 + 1.5 * iqr).length}; +} + +export {frameCoverage, summarizeFrames, distribution, evaluationReport, numericSamples, summarizeCohort, summarizePlatform}; diff --git a/scripts/pgo/evaluationRun.ts b/scripts/pgo/evaluationRun.ts new file mode 100644 index 000000000000..2ba20dbbdbc7 --- /dev/null +++ b/scripts/pgo/evaluationRun.ts @@ -0,0 +1,371 @@ +// cspell:ignore cooldown PodLock profdata + +import {spawnSync} from 'node:child_process'; +/** Executes an isolated evaluation; private device evidence never enters the public result. */ +import {createHash} from 'node:crypto'; +import {appendFileSync, closeSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, writeFileSync} from 'node:fs'; +import {join, resolve} from 'node:path'; +import {pathToFileURL} from 'node:url'; + +import type {BenchmarkSample} from '../lib/benchmarkStatistics'; +import type {EvaluationCohort, EvaluationPlan, EvaluationPlatform, EvaluationResult, FrameObservation} from './evaluationSchema'; +import type {PlatformAdapter} from './shared'; + +import {isRecord} from './evaluationSchema'; + +type ProofModule = {sourceIdentity: () => string; toolchainIdentity: (adapter: PlatformAdapter) => string; fileHash: (path: string) => string}; +type AdapterModule = {default: (appID: string, device: string) => PlatformAdapter}; +type TrainingModule = {validateTraining: (platform: EvaluationPlatform, batch: string) => {profile: string}}; + +const OVERLAY_FILES = ['android.ts', 'ios.ts', 'shared.ts', 'pgo.ts', 'journey.ts', 'journeyBenchmark.ts', 'journeyWorkload.ts', 'proof.ts', 'localProof.ts', 'androidRelease.init.gradle']; +const SCENARIOS = [ + {name: 'heavy-journey', action: 'benchmark-heavy-journey', spans: ['/r/*']}, + {name: 'journey', action: 'benchmark-journey', spans: ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab']}, + {name: 'startup', action: 'benchmark', spans: ['ManualAppStartup']}, +] as const; + +async function executeEvaluation(plan: EvaluationPlan, platform: EvaluationPlatform, publicDirectory: string): Promise { + const candidate = requiredDirectory('PGO_CANDIDATE_DIRECTORY'); + const tooling = requiredDirectory('PGO_TOOLING_DIRECTORY'); + const privateDirectory = join(candidate, '.pgo', 'evaluation', `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`, platform); + mkdirSync(privateDirectory, {recursive: true}); + mkdirSync(publicDirectory, {recursive: true}); + const result: EvaluationResult = { + schema: 1, + platform, + candidate: plan.candidate, + toolingSha: plan.toolingSha, + status: 'failed', + artifactSha256: {release: '', instrumented: '', optimized: ''}, + profileSha256: '', + sourceIdentity: '', + toolchain: {}, + compilerConsumption: 'unverified', + cohorts: [], + }; + let stage = 'candidate-guard'; + const runPrivate = (file: string, args: string[]) => privateCommand(process.execPath, [join(candidate, 'scripts/pgo', file), ...args], candidate, privateDirectory, stage); + try { + verifyCandidate(plan, candidate, tooling); + if (!plan.platforms.includes(platform)) { + throw new Error('Platform not eligible.'); + } + const device = requiredValue(`PGO_${platform.toUpperCase()}_DEVICE_ID`); + const appID = requiredValue(`PGO_${platform.toUpperCase()}_APP_ID`); + const fixture = requiredValue(`PGO_${platform.toUpperCase()}_FIXTURE_PATH`); + // The fixture is privately provisioned on the runner; never print or copy its contents. + const fixtureHash = sha256(readFileSync(fixture)); + process.env.ENVFILE = resolve(candidate, 'Mobile-Expensify', requiredValue(`PGO_${platform.toUpperCase()}_ENVFILE`)); + appendFileSync(join(candidate, '.git/info/exclude'), '\n/evaluation-tooling/\n/pgo-plan/\n/pgo-public/\n'); + const overlay = OVERLAY_FILES.map((file) => { + const source = join(tooling, 'scripts/pgo', file); + const hash = sha256(readFileSync(source)); + cpSync(source, join(candidate, 'scripts/pgo', file)); + return {file, sha256: hash}; + }); + result.toolchain.reviewedOverlaySha256 = sha256(Buffer.from(JSON.stringify(overlay))); + configureSafeEvaluationOptions(candidate); + const proof: unknown = await import(pathToFileURL(join(candidate, 'scripts/pgo/proof.ts')).href); + const adapterModule: unknown = await import(pathToFileURL(join(candidate, `scripts/pgo/${platform}.ts`)).href); + if (!isProofModule(proof) || !isAdapterModule(adapterModule)) { + throw new Error('Reviewed evaluation module contract is missing.'); + } + const adapter = adapterModule.default(appID, device); + result.sourceIdentity = proof.sourceIdentity(); + const toolchain = proof.toolchainIdentity(adapter); + result.toolchain.identitySha256 = sha256(Buffer.from(toolchain)); + result.toolchain.fixtureSha256 = fixtureHash; + const environmentFile = process.env.ENVFILE; + result.toolchain.environmentSha256 = sha256(readFileSync(environmentFile)); + let profilePath: string | undefined; + const guard = () => { + verifyCandidate(plan, candidate, tooling); + if (proof.sourceIdentity() !== result.sourceIdentity || proof.toolchainIdentity(adapter) !== toolchain || sha256(readFileSync(fixture)) !== fixtureHash) { + throw new Error('Frozen evaluation inputs changed.'); + } + if (sha256(readFileSync(environmentFile)) !== result.toolchain.environmentSha256) { + throw new Error('Build environment changed.'); + } + if (profilePath && proof.fileHash(profilePath) !== result.profileSha256) { + throw new Error('Frozen training profile changed.'); + } + for (const kind of ['release', 'instrumented', 'optimized'] as const) { + if (result.artifactSha256[kind] && proof.fileHash(adapter.artifactPaths[kind]) !== result.artifactSha256[kind]) { + throw new Error('Frozen artifact changed.'); + } + } + }; + const deviceArgs = ['--device', device, '--app-id', appID]; + const podLockPath = join(candidate, 'Mobile-Expensify/iOS/Podfile.lock'); + const originalPodLock = platform === 'ios' ? readFileSync(podLockPath) : undefined; + let generatedPodLockHash: string | undefined; + const build = (kind: 'release' | 'instrumented' | 'optimized', profile?: string) => { + try { + runPrivate('pgo.ts', [platform, `build-${kind}`, ...deviceArgs, ...(profile ? ['--profile', profile] : [])]); + if (originalPodLock) { + const generated = readFileSync(podLockPath); + const hash = sha256(generated); + writeFileSync(join(privateDirectory, `${kind}-Podfile.lock`), generated, {mode: 0o600}); + result.toolchain[`${kind}PodLockSha256`] = hash; + if (generatedPodLockHash && hash !== generatedPodLockHash) { + throw new Error('Compiled pod dependency lock changed between build modes.'); + } + generatedPodLockHash = hash; + } + } finally { + if (originalPodLock) { + writeFileSync(podLockPath, originalPodLock); + } + } + }; + stage = 'account-preflight'; + runPrivate('journey.ts', [platform, '--fixture', fixture, ...deviceArgs, '--preflight']); + for (const kind of ['release', 'instrumented'] as const) { + stage = `build-${kind}`; + guard(); + build(kind); + result.artifactSha256[kind] = proof.fileHash(adapter.artifactPaths[kind]); + guard(); + } + stage = 'verify-instrumented'; + runPrivate('pgo.ts', [platform, 'verify-instrumented', ...deviceArgs]); + stage = 'install-instrumented'; + runPrivate('pgo.ts', [platform, 'install-instrumented', ...deviceArgs]); + process.env.PGO_INSTRUMENTED_BINARY_SHA256 = result.artifactSha256.instrumented; + stage = 'training'; + const journeyRoot = join(candidate, '.pgo', platform, 'journeys'); + const previousJourneys = directoryNames(journeyRoot); + runPrivate('journey.ts', [platform, '--fixture', fixture, ...deviceArgs, '--runs', '3', '--collect']); + guard(); + const trainingBatch = newBatch(journeyRoot, previousJourneys); + stage = 'validate-training'; + const validation: unknown = await import(pathToFileURL(join(candidate, 'scripts/pgo/localProof.ts')).href); + if (!isTrainingModule(validation)) { + throw new Error('Reviewed training validator is missing.'); + } + const training = validation.validateTraining(platform, trainingBatch); + profilePath = training.profile; + result.profileSha256 = proof.fileHash(training.profile); + stage = 'build-optimized'; + build('optimized', training.profile); + result.artifactSha256.optimized = proof.fileHash(adapter.artifactPaths.optimized); + guard(); + for (const {order, scenario} of benchmarkStagePlan()) { + stage = `${scenario.name}-${order}`; + const root = join(adapter.benchmarkDirectory, scenario.name); + const before = directoryNames(root); + runGuardedStage(guard, () => runPrivate('pgo.ts', [platform, scenario.action, '20', '60', ...deviceArgs, '--fixture', fixture, '--benchmark-order', order])); + result.cohorts.push(readCohort(newBatch(root, before), scenario.name, order, [...scenario.spans], device, appID)); + } + result.status = 'passed'; + } catch { + result.failureStage = stage; + // The detailed error and raw device output stay private on the runner. + console.error(`PGO evaluation failed at ${stage}. Private runner evidence retained.`); + } finally { + writeFileSync(join(publicDirectory, `evaluation-${platform}.json`), `${JSON.stringify(result, null, 2)}\n`); + } + if (result.status !== 'passed') { + throw new Error('PGO evaluation did not complete.'); + } +} + +function readCohort(batch: string, scenario: EvaluationCohort['scenario'], order: EvaluationCohort['order'], spans: string[], device: string, appID: string): EvaluationCohort { + const metadata: unknown = JSON.parse(readFileSync(join(batch, 'run.json'), 'utf8')); + if ( + !isRecord(metadata) || + metadata.deviceID !== device || + metadata.appID !== appID || + metadata.order !== order || + metadata.runs !== 20 || + metadata.warmUpRunsPerBinary !== 1 || + metadata.cooldownSecondsPerBinary !== 5 + ) { + throw new Error('Benchmark context does not match the verified plan.'); + } + const releasePath = join(batch, 'release.csv'); + const optimizedPath = join(batch, scenario === 'startup' ? 'optimized.csv' : 'pgo-optimized.csv'); + return { + scenario, + order, + spans, + runs: 20, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: 5, + release: readSamples(releasePath, spans), + optimized: readSamples(optimizedPath, spans), + releaseCsvSha256: sha256(readFileSync(releasePath)), + optimizedCsvSha256: sha256(readFileSync(optimizedPath)), + ...(scenario === 'heavy-journey' && metadata.platform === 'android' + ? { + frames: { + release: readPublicFrames(join(batch, 'release-frames.json')), + optimized: readPublicFrames(join(batch, 'pgo-optimized-frames.json')), + }, + } + : {}), + }; +} + +function readPublicFrames(path: string): FrameObservation[] { + if (!existsSync(path)) { + return Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable'})); + } + const input: unknown = JSON.parse(readFileSync(path, 'utf8')); + if (!isRecord(input) || input.schema !== 1 || input.requestedRuns !== 20 || !Array.isArray(input.samples) || input.samples.length !== 20) { + throw new Error('Unexpected optional frame coverage.'); + } + return input.samples.map((sample: unknown, index) => { + if (!isRecord(sample) || sample.run !== index + 1) { + throw new Error('Optional frame run identity changed.'); + } + if (sample.status === 'unavailable') { + return {run: sample.run, status: 'unavailable'}; + } + const metrics = sample.metrics; + if ( + sample.status !== 'available' || + !isRecord(metrics) || + typeof metrics.totalFrames !== 'number' || + !Number.isSafeInteger(metrics.totalFrames) || + metrics.totalFrames <= 0 || + typeof metrics.droppedFrames !== 'number' || + !Number.isSafeInteger(metrics.droppedFrames) || + metrics.droppedFrames < 0 || + metrics.droppedFrames > metrics.totalFrames || + typeof metrics.droppedPercent !== 'number' || + !Number.isFinite(metrics.droppedPercent) || + metrics.droppedPercent < 0 || + metrics.droppedPercent > 100 || + typeof metrics.sampleWindowMs !== 'number' || + !Number.isFinite(metrics.sampleWindowMs) || + metrics.sampleWindowMs <= 0 + ) { + throw new Error('Invalid optional frame metrics.'); + } + return { + run: sample.run, + status: 'available', + totalFrames: metrics.totalFrames, + jankyFrames: metrics.droppedFrames, + originalPercent: metrics.droppedPercent, + sampleWindowMs: metrics.sampleWindowMs, + }; + }); +} + +function readSamples(path: string, spans: string[]): BenchmarkSample[] { + const lines = readFileSync(path, 'utf8').trim().split('\n'); + if (lines.shift() !== 'run,span,duration_ms') { + throw new Error('Unexpected benchmark CSV format.'); + } + const samples = lines.map((line) => { + const [run, span, duration] = line.trim().split(','); + return {run: Number(run), span, durationMs: Number(duration)}; + }); + if ( + samples.length !== 20 * spans.length || + samples.some( + (sample) => !Number.isInteger(sample.run) || sample.run < 1 || sample.run > 20 || !spans.includes(sample.span) || !Number.isFinite(sample.durationMs) || sample.durationMs < 0, + ) || + new Set(samples.map((sample) => `${sample.run}:${sample.span}`)).size !== samples.length + ) { + throw new Error('Expected twenty complete samples for every span.'); + } + return samples; +} + +function verifyCandidate(plan: EvaluationPlan, candidate: string, tooling: string): void { + const git = (cwd: string, args: string[]) => { + const result = spawnSync('git', args, {cwd, encoding: 'utf8'}); + if (result.status !== 0) { + throw new Error('Cannot verify immutable checkout.'); + } + return result.stdout.trim(); + }; + const packageManifest: unknown = JSON.parse(readFileSync(join(candidate, 'package.json'), 'utf8')); + if ( + !isRecord(packageManifest) || + packageManifest.version !== plan.candidate.version || + git(candidate, ['rev-parse', 'HEAD']) !== plan.candidate.appSha || + git(join(candidate, 'Mobile-Expensify'), ['rev-parse', 'HEAD']) !== plan.candidate.mobileSha || + git(tooling, ['rev-parse', 'HEAD']) !== plan.toolingSha || + git(candidate, ['ls-tree', 'HEAD', 'Mobile-Expensify']).split(/\s+/).at(2) !== plan.candidate.mobileSha + ) { + throw new Error('Immutable candidate pins do not match.'); + } +} + +function privateCommand(command: string, args: string[], cwd: string, directory: string, stage: string): void { + const descriptor = openSync(join(directory, `${stage}.log`), 'a', 0o600); + try { + const result = spawnSync(command, args, {cwd, env: process.env, stdio: ['ignore', descriptor, descriptor]}); + if (result.status !== 0) { + throw new Error('Private evaluation stage failed.'); + } + } finally { + closeSync(descriptor); + } +} + +function requiredDirectory(name: string): string { + const value = resolve(requiredValue(name)); + if (!existsSync(value)) { + throw new Error('Missing evaluation checkout.'); + } + return value; +} +function requiredValue(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error('Missing provisioned evaluation configuration.'); + } + return value; +} +function directoryNames(path: string): Set { + return new Set(existsSync(path) ? readdirSync(path) : []); +} +function newBatch(path: string, previous: Set): string { + const added = readdirSync(path).filter((name) => !previous.has(name)); + if (added.length !== 1) { + throw new Error('Expected one isolated evidence batch.'); + } + const name = added.at(0); + if (!name) { + throw new Error('Missing isolated evidence batch.'); + } + return join(path, name); +} +function sha256(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function isProofModule(input: unknown): input is ProofModule { + return isRecord(input) && typeof input.sourceIdentity === 'function' && typeof input.toolchainIdentity === 'function' && typeof input.fileHash === 'function'; +} +function isAdapterModule(input: unknown): input is AdapterModule { + return isRecord(input) && typeof input.default === 'function'; +} +function isTrainingModule(input: unknown): input is TrainingModule { + return isRecord(input) && typeof input.validateTraining === 'function'; +} + +function benchmarkStagePlan() { + return (['release-first', 'optimized-first'] as const).flatMap((order) => SCENARIOS.map((scenario) => ({order, scenario}))); +} + +function configureSafeEvaluationOptions(candidate: string): void { + process.env.PGO_ANDROID_RELEASE_PIPELINE = 'false'; + process.env.PGO_ANDROID_SHRINK = 'true'; + process.env.PGO_BENCHMARK_COOLDOWN_SECONDS = '5'; + process.env.PATH = `${join(candidate, 'node_modules/.bin')}:${process.env.PATH ?? ''}`; +} + +function runGuardedStage(guard: () => void, execute: () => T): T { + guard(); + const result = execute(); + guard(); + return result; +} + +export {benchmarkStagePlan, configureSafeEvaluationOptions, executeEvaluation, readCohort, readPublicFrames, readSamples, runGuardedStage}; diff --git a/scripts/pgo/evaluationSchema.ts b/scripts/pgo/evaluationSchema.ts new file mode 100644 index 000000000000..d0ad7c2e8253 --- /dev/null +++ b/scripts/pgo/evaluationSchema.ts @@ -0,0 +1,110 @@ +// cspell:ignore cooldown +/** Public identities and measurement records for evaluation-only PGO jobs. */ +import {createHash} from 'node:crypto'; + +import type {BenchmarkSample} from '../lib/benchmarkStatistics'; + +type EvaluationPlatform = 'android' | 'ios'; +type CandidateIdentity = {appSha: string; mobileSha: string; version: string; environment: 'staging' | 'production' | 'preview'}; +type CandidateMetadata = {schema: 1; repository: 'Expensify/App'; runID: number; runAttempt: number; nativeEligible: boolean; candidate: CandidateIdentity}; +type EvaluationPlan = { + schema: 1; + repository: 'Expensify/App'; + toolingSha: string; + candidate: CandidateIdentity; + producerRunID: number | null; + producerRunAttempt: number | null; + platforms: EvaluationPlatform[]; + checklistIssueNumber: number | null; +}; +type FrameObservation = {run: number; status: 'unavailable'} | {run: number; status: 'available'; totalFrames: number; jankyFrames: number; originalPercent: number; sampleWindowMs: number}; +type EvaluationCohort = { + scenario: 'startup' | 'journey' | 'heavy-journey'; + order: 'release-first' | 'optimized-first'; + spans: string[]; + runs: number; + warmUpRunsPerBinary: number; + cooldownSecondsPerBinary: number; + release: BenchmarkSample[]; + optimized: BenchmarkSample[]; + releaseCsvSha256: string; + optimizedCsvSha256: string; + frames?: {release: FrameObservation[]; optimized: FrameObservation[]}; +}; +type EvaluationResult = { + schema: 1; + platform: EvaluationPlatform; + candidate: CandidateIdentity; + toolingSha: string; + status: 'passed' | 'failed'; + failureStage?: string; + artifactSha256: {release: string; instrumented: string; optimized: string}; + profileSha256: string; + sourceIdentity: string; + toolchain: Record; + compilerConsumption: 'unverified' | 'verified'; + cohorts: EvaluationCohort[]; +}; + +function candidateKey(candidate: CandidateIdentity): string { + return createHash('sha256') + .update(JSON.stringify([candidate.appSha, candidate.mobileSha, candidate.version, candidate.environment])) + .digest('hex'); +} + +function parsePlan(input: unknown): EvaluationPlan { + if (!isRecord(input) || input.schema !== 1 || input.repository !== 'Expensify/App' || !isGitID(input.toolingSha) || !isRecord(input.candidate) || !Array.isArray(input.platforms)) { + throw new Error('Invalid PGO evaluation plan.'); + } + const candidate = parseCandidate(input.candidate); + const platforms = input.platforms.filter((value): value is EvaluationPlatform => value === 'android' || value === 'ios'); + if ( + platforms.length !== input.platforms.length || + new Set(platforms).size !== platforms.length || + !nullableID(input.producerRunID) || + !nullableID(input.producerRunAttempt) || + !nullableID(input.checklistIssueNumber) + ) { + throw new Error('Invalid PGO evaluation plan identities.'); + } + return { + schema: 1, + repository: 'Expensify/App', + toolingSha: input.toolingSha, + candidate, + platforms, + producerRunID: input.producerRunID, + producerRunAttempt: input.producerRunAttempt, + checklistIssueNumber: input.checklistIssueNumber, + }; +} + +function parseCandidate(input: unknown): CandidateIdentity { + if ( + !isRecord(input) || + !isGitID(input.appSha) || + !isGitID(input.mobileSha) || + typeof input.version !== 'string' || + !/^\d+\.\d+\.\d+(?:-\d+)?$/.test(input.version) || + (input.environment !== 'staging' && input.environment !== 'production' && input.environment !== 'preview') + ) { + throw new Error('Invalid immutable PGO candidate identity.'); + } + return {appSha: input.appSha, mobileSha: input.mobileSha, version: input.version, environment: input.environment}; +} + +function isRecord(input: unknown): input is Record { + return typeof input === 'object' && input !== null && !Array.isArray(input); +} +function isGitID(input: unknown): input is string { + return typeof input === 'string' && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(input); +} +function isSha256(input: unknown): input is string { + return typeof input === 'string' && /^[a-f0-9]{64}$/.test(input); +} +function nullableID(input: unknown): input is number | null { + return input === null || (typeof input === 'number' && Number.isSafeInteger(input) && input > 0); +} + +export {candidateKey, isGitID, isRecord, isSha256, parseCandidate, parsePlan}; +export type {CandidateIdentity, CandidateMetadata, EvaluationCohort, EvaluationPlan, EvaluationPlatform, EvaluationResult, FrameObservation}; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index bc2ba74c7a1c..0bfc476f8440 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -230,28 +230,30 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), ] : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; - const xcodeArguments = [ - 'xcodebuild', - '-workspace', - 'Expensify.xcworkspace', - '-scheme', - 'Expensify', - '-configuration', - 'Release', - '-destination', - 'generic/platform=iOS', - '-derivedDataPath', - derivedDataDirectory, - ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), - '-quiet', + const rockArguments = iosRockBuildArguments(derivedDataDirectory, [ 'build', 'ARCHS=arm64', 'ONLY_ACTIVE_ARCH=YES', 'COMPILER_INDEX_STORE_ENABLE=NO', + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), ...signingSettings, ...pgoBuildSettings(kind, profilePath), - ]; - withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), ...xcodeArguments], iosDirectory)); + ]); + withLocalUploadPhasesDisabled(() => + run( + '/usr/bin/env', + [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], + join(rootDirectory, 'Mobile-Expensify'), + ), + ); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -472,4 +474,25 @@ function replaceOnce(content: string, previous: string, next: string): string { return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; } +/** Preserve each Xcode setting as one argument through Rock's string argument parser. */ +function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { + return [ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + derivedDataDirectory, + '--no-install-pods', + '--extra-params', + settings.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + +export {iosRockBuildArguments}; export default createIOSPgoAdapter; diff --git a/scripts/pgo/journey.ts b/scripts/pgo/journey.ts index 624f67f2c56e..48a52380ace3 100755 --- a/scripts/pgo/journey.ts +++ b/scripts/pgo/journey.ts @@ -15,6 +15,7 @@ import createIOSPgoAdapter from './ios'; import {parseJourneyFixture} from './journeyConfig'; import {createJourneyDevice} from './journeyDevice'; import {prepareJourney, runJourneyWorkload, verifyJourneyAccount} from './journeyWorkload'; +import {sourceIdentity, toolchainIdentity} from './proof'; /** Run the same UI workload on a selected Android or iOS device; never automate authentication. */ async function main(): Promise { @@ -59,6 +60,9 @@ async function main(): Promise { fixture, sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + sourceIdentity: sourceIdentity(), + toolchain: toolchainIdentity(adapter), + instrumentedBinarySha256: process.env.PGO_INSTRUMENTED_BINARY_SHA256, automationVersion: capture('agent-device', ['--version']).trim(), requestedRuns: runs, collectProfiles: cli.flags.collect, diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts index 40294812c581..593c35c1011f 100644 --- a/scripts/pgo/journeyBenchmark.ts +++ b/scripts/pgo/journeyBenchmark.ts @@ -1,3 +1,5 @@ +// cspell:ignore cooldown Cooldown + /** Runs separate read-only signed-in navigation and heavy-report benchmarks. */ // cspell:ignore gfxinfo @@ -19,7 +21,7 @@ import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; import {createJourneyDevice} from './journeyDevice'; import {openReport, scrollFastBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; -import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; +import {benchmarkCooldownSeconds, cooldownBeforeBenchmark, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; type JourneyBenchmarkReader = { begin: (spanNames: readonly string[]) => void; @@ -27,7 +29,8 @@ type JourneyBenchmarkReader = { }; type FrameHealth = {totalFrames: number; droppedFrames: number; droppedPercent: number; sampleWindowMs: number}; -type JourneyMeasurement = {events: BenchmarkLogEvent[]; frameHealth?: FrameHealth}; +type FrameHealthResult = {status: 'available'; metrics: FrameHealth; raw: unknown} | {status: 'unavailable'; reason: string; raw: unknown}; +type JourneyMeasurement = {events: BenchmarkLogEvent[]; frameTelemetry?: FrameHealthResult}; type BenchmarkOrder = 'release-first' | 'optimized-first'; type JourneyScenario = { @@ -61,6 +64,7 @@ async function benchmarkJourneyBuild( ): Promise { console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); adapter.install(kind); + await cooldownBeforeBenchmark(); const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-${scenario.directory}-${adapter.name}-${kind}-${Date.now()}`}); const reader: JourneyBenchmarkReader = @@ -73,23 +77,35 @@ async function benchmarkJourneyBuild( await scenario.measure(device, reader, fixture, timeoutSeconds); const recorder = createBenchmarkRecorder({spanNames: [...scenario.spanNames], runs, outputPath}); - const frameRows: string[] = []; + const frameSamples: Array = []; for (let runNumber = 1; runNumber <= runs; runNumber += 1) { const measurement = await scenario.measure(device, reader, fixture, timeoutSeconds); console.log(`${kind} ${scenario.directory} ${runNumber}/${runs}: ${recorder.record(measurement.events, runNumber).join(', ')}`); - if (measurement.frameHealth) { - const {totalFrames, droppedFrames, droppedPercent, sampleWindowMs} = measurement.frameHealth; - frameRows.push(`${runNumber},${totalFrames},${droppedFrames},${droppedPercent},${sampleWindowMs}`); - console.log(`Android scroll frames: ${droppedFrames}/${totalFrames} missed deadlines (${droppedPercent.toFixed(1)}%).`); + if (measurement.frameTelemetry) { + frameSamples.push({run: runNumber, ...measurement.frameTelemetry}); + if (measurement.frameTelemetry.status === 'available') { + const {totalFrames, droppedFrames, droppedPercent} = measurement.frameTelemetry.metrics; + console.log(`Android frame window: ${droppedFrames}/${totalFrames} native janky frames (${droppedPercent.toFixed(1)}%).`); + } else { + console.log(`Android frame telemetry unavailable for run ${runNumber}: ${measurement.frameTelemetry.reason}`); + } } } await verifyJourneyAccount(device, fixture); await recorder.complete(`${kind} ${scenario.directory} metrics`); - if (frameRows.length > 0) { - if (frameRows.length !== runs) { - fail(`Only ${frameRows.length}/${runs} Android scroll frame samples were recorded.`); - } - writeFileSync(outputPath.replace(/\.csv$/, '-frames.csv'), `run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs\n${frameRows.join('\n')}\n`); + if (frameSamples.length > 0) { + writeFileSync( + outputPath.replace(/\.csv$/, '-frames.json'), + `${JSON.stringify({schema: 1, requestedRuns: runs, samples: frameSamples, resetVerified: false, isolatedScrollWindowVerified: false}, null, 2)}\n`, + ); + const rows = frameSamples.flatMap((sample) => { + if (sample.status !== 'available') { + return []; + } + const {totalFrames, droppedFrames, droppedPercent, sampleWindowMs} = sample.metrics; + return [`${sample.run},${totalFrames},${droppedFrames},${droppedPercent},${sampleWindowMs}`]; + }); + writeFileSync(outputPath.replace(/\.csv$/, '-frames.csv'), `run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs\n${rows.join('\n')}\n`); } } finally { try { @@ -137,23 +153,44 @@ async function measureHeavyJourney( await waitForTab(device, 'Account'); await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); - if (device.platform === 'android') { - // Android gfxinfo resets after each read. Exclude search and report opening from this frame window. - readAndroidFrameHealth(device, false); - } + // The sampler requests a reset after reading; actual reset success is not verified. + const beforeFrames = device.platform === 'android' ? readAndroidFrameHealth(device, false) : undefined; scrollFastBothWays(device, Math.min(fixture.scrolls, 4)); - const frameHealth = device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined; + const afterFrames = device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined; + const frameTelemetry = beforeFrames && afterFrames ? frameWindowResult(beforeFrames, afterFrames) : undefined; await showTab(device, 'Inbox'); - return {events, frameHealth}; + return {events, frameTelemetry}; } -function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealth | undefined { - const result = device.command('perf', 'frames'); +/** Ancillary frame telemetry cannot erase otherwise complete native timing samples. */ +function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealthResult { + try { + return parseAndroidFrameHealth(device.command('perf', 'frames'), requireFrames); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + return {status: 'unavailable', reason, raw: {commandError: reason}}; + } +} + +function frameWindowResult(before: FrameHealthResult, after: FrameHealthResult): FrameHealthResult { + const raw = {before: before.raw, after: after.raw}; + if (before.status === 'unavailable') { + return {status: 'unavailable', reason: `Pre-scroll sampler unavailable; window boundary unverified: ${before.reason}`, raw}; + } + return {...after, raw}; +} + +function parseAndroidFrameHealth(result: unknown, requireFrames: boolean): FrameHealthResult { + const unavailable = (reason: string): FrameHealthResult => ({status: 'unavailable', reason, raw: result}); if (!isRecord(result) || !isRecord(result.metrics) || !isRecord(result.metrics.fps) || !isRecord(result.sampling) || !isRecord(result.sampling.fps)) { - throw new Error('Android frame sampler returned an unexpected result.'); + return unavailable('Android frame sampler returned an unexpected result.'); } - if (result.sampling.fps.resetsAfterRead !== true || result.metrics.fps.available !== true) { - throw new Error('Android frame sampler is unavailable or did not reset its counter.'); + if (result.sampling.fps.resetsAfterRead !== true) { + return unavailable('Android frame sampler does not declare reset-after-read support; reset success is not verified.'); + } + if (result.metrics.fps.available !== true) { + const reason = typeof result.metrics.fps.reason === 'string' ? result.metrics.fps.reason : 'No sampler reason provided.'; + return unavailable(`Android frame sampler unavailable: ${reason}`); } const {totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs} = result.metrics.fps; if ( @@ -161,14 +198,22 @@ function readAndroidFrameHealth(device: ReturnType, typeof droppedFrameCount !== 'number' || typeof droppedFramePercent !== 'number' || typeof sampleWindowMs !== 'number' || - ![totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs].every(Number.isFinite) + ![totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs].every(Number.isFinite) || + !Number.isInteger(totalFrameCount) || + !Number.isInteger(droppedFrameCount) || + totalFrameCount < 0 || + droppedFrameCount < 0 || + droppedFrameCount > totalFrameCount || + droppedFramePercent < 0 || + droppedFramePercent > 100 || + sampleWindowMs < 0 ) { - throw new Error('Android frame sampler did not return finite frame counts.'); + return unavailable('Android frame sampler returned invalid frame counts or window.'); } - if (requireFrames && totalFrameCount === 0) { - throw new Error('The Android scroll produced no measured frames. Discard this run.'); + if (requireFrames && (totalFrameCount === 0 || sampleWindowMs === 0)) { + return unavailable('No frames or positive frame window were available for this scroll sample.'); } - return {totalFrames: totalFrameCount, droppedFrames: droppedFrameCount, droppedPercent: droppedFramePercent, sampleWindowMs}; + return {status: 'available', metrics: {totalFrames: totalFrameCount, droppedFrames: droppedFrameCount, droppedPercent: droppedFramePercent, sampleWindowMs}, raw: result}; } async function benchmarkJourneyAll( @@ -220,6 +265,9 @@ async function benchmarkJourneyScenario( runs, timeoutSeconds, order, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: benchmarkCooldownSeconds(), + thermalStateVerified: false, scenario: scenario.directory, spans: scenario.spanNames, ...(scenario === HEAVY_JOURNEY @@ -227,7 +275,8 @@ async function benchmarkJourneyScenario( scrollsPerDirection: Math.min(fixture.scrolls, 4), scrollDurationMs: 120, scrollTimeMeasured: false, - androidScrollFrameHealthMeasured: true, + androidScrollFrameHealthMeasured: false, + androidFrameTelemetry: 'optional; per-sample availability recorded; reset and isolated window unverified', iosScrollFrameHealthMeasured: false, } : {}), @@ -271,35 +320,30 @@ async function compareJourneyBenchmarks(paths: Record, ru console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); } if (scenario === HEAVY_JOURNEY && platform === 'android') { - const releaseFrames = readAndroidFrameSamples(paths.release.replace(/\.csv$/, '-frames.csv'), runs); - const optimizedFrames = readAndroidFrameSamples(paths.optimized.replace(/\.csv$/, '-frames.csv'), runs); - console.log(`Android scroll missed-frame share: release ${releaseFrames.toFixed(2)}%, PGO ${optimizedFrames.toFixed(2)}% (lower is better; separate from report-open latency).`); + const releaseFrames = readAndroidFrameSamples(paths.release.replace(/\.csv$/, '-frames.json'), runs); + const optimizedFrames = readAndroidFrameSamples(paths.optimized.replace(/\.csv$/, '-frames.json'), runs); + console.log( + `Android frame telemetry availability: release ${releaseFrames.available}/${runs}, PGO ${optimizedFrames.available}/${runs}. Reset and isolated scroll window are unverified; no scrolling gain is inferred.`, + ); } console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); } -function readAndroidFrameSamples(path: string, runs: number): number { - if (!existsSync(path)) { - fail(`Missing Android scroll frame samples: ${path}`); - } - const rows = readFileSync(path, 'utf8').trim().split('\n'); - if (rows.shift() !== 'run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs' || rows.length !== runs) { - fail(`Expected ${runs} Android scroll frame samples in ${path}.`); +function readAndroidFrameSamples(path: string, runs: number): {available: number} { + const evidence: unknown = JSON.parse(readFileSync(path, 'utf8')); + if (!isRecord(evidence) || !Array.isArray(evidence.samples) || evidence.samples.length !== runs) { + fail(`Expected ${runs} Android frame telemetry statuses in ${path}.`); } - let totalFrames = 0; - let droppedFrames = 0; - for (const [index, row] of rows.entries()) { - const values = row.split(',').map(Number); - const runNumber = values.at(0) ?? NaN; - const frames = values.at(1) ?? NaN; - const dropped = values.at(2) ?? NaN; - if (values.length !== 5 || values.some((value) => !Number.isFinite(value)) || runNumber !== index + 1 || frames <= 0 || dropped < 0 || dropped > frames) { - fail(`Invalid Android scroll frame sample ${index + 1} in ${path}.`); + let available = 0; + for (const [index, sample] of evidence.samples.entries()) { + if (!isRecord(sample) || sample.run !== index + 1 || !['available', 'unavailable'].includes(String(sample.status))) { + fail(`Invalid Android frame telemetry status ${index + 1} in ${path}.`); + } + if (sample.status === 'available') { + available += 1; } - totalFrames += frames; - droppedFrames += dropped; } - return (droppedFrames / totalFrames) * 100; + return {available}; } async function sha256Artifact(path: string): Promise { @@ -344,5 +388,5 @@ async function sha256Directory(directory: string): Promise { return hash.digest('hex'); } -export {benchmarkHeavyJourneyAll, benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; +export {benchmarkHeavyJourneyAll, benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact, parseAndroidFrameHealth, frameWindowResult}; export type {BenchmarkOrder}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts index c126050d20d2..c6a61e948860 100644 --- a/scripts/pgo/journeyWorkload.ts +++ b/scripts/pgo/journeyWorkload.ts @@ -132,16 +132,20 @@ async function openReport(device: JourneyDevice, report: JourneyReport, isPerson } } -async function showTab(device: JourneyDevice, tab: string): Promise { - const deadline = Date.now() + 30_000; +async function showTab(device: JourneyDevice, tab: string, now = Date.now): Promise { + const deadline = now() + 30_000; let backPresses = 0; - while (Date.now() < deadline) { + for (;;) { const nodes = device.snapshot(); const tabNode = findTabNode(nodes, tab); if (tabNode) { device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); return; } + // A completed Back may use the remaining budget. Inspect its result before timing out. + if (now() >= deadline) { + throw new Error(`Cannot return to the ${tab} tab.`); + } // A relaunch can expose a loading view before tabs mount. Only navigate back from a real nested page. if (backPresses < 5 && inAppBackTapPoint(nodes)) { pressInAppBack(device, nodes); @@ -149,7 +153,6 @@ async function showTab(device: JourneyDevice, tab: string): Promise { } await sleep(500); } - throw new Error(`Cannot return to the ${tab} tab.`); } /** Wait for a mounted bottom tab without navigating away from a partially loaded screen. */ diff --git a/scripts/pgo/localProof.ts b/scripts/pgo/localProof.ts new file mode 100644 index 000000000000..d751222f3b01 --- /dev/null +++ b/scripts/pgo/localProof.ts @@ -0,0 +1,116 @@ +#!/usr/bin/env bun + +// cspell:ignore profdata profraw fprofile unbuilt + +import {existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {capture, fail, findFiles, run, valueAt} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {fileHash, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney} from './proof'; + +/** Validate the immutable collected profile before optimization. */ +function validateTraining(platform: PlatformName, batch: string) { + const adapter = platform === 'android' ? createAndroidPgoAdapter() : createIOSPgoAdapter(); + const toolchain = toolchainIdentity(adapter); + const identity = sourceIdentity(); + const result: unknown = JSON.parse(readFileSync(join(batch, 'result.json'), 'utf8')); + validateJourney(result, platform, identity, toolchain); + const instrumentedBinarySha256 = fileHash(adapter.artifactPaths.instrumented); + if (!isRecord(result) || result.instrumentedBinarySha256 !== instrumentedBinarySha256) { + fail('The collected journey is not bound to the current instrumented artifact.'); + } + const profile = join(batch, 'journey.profdata'); + const counts = profileCounts(capture(adapter.llvmTool('llvm-profdata'), ['show', profile])); + const runs = [1, 2, 3].map((runNumber) => { + const paths = findFiles(join(batch, `run-${runNumber}`, 'raw'), '.profraw').sort(); + if (paths.length === 0) { + fail(`Journey ${runNumber} has no archived raw profiles.`); + } + return paths.map((path) => ({path, sha256: fileHash(path)})); + }); + const temporary = mkdtempSync(join(tmpdir(), 'pgo-proof-')); + try { + const merged = join(temporary, 'merged.profdata'); + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${merged}`, ...runs.flatMap((profiles) => profiles.map(({path}) => path))]); + if (fileHash(merged) !== fileHash(profile)) { + fail('The selected profile does not match the three archived journeys.'); + } + for (const profiles of runs) { + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${merged}`, ...profiles.map(({path}) => path)]); + profileCounts(capture(adapter.llvmTool('llvm-profdata'), ['show', merged])); + } + } finally { + rmSync(temporary, {recursive: true, force: true}); + } + if (adapter.profileFormat && readFileSync(`${profile}.format`, 'utf8').trim() !== adapter.profileFormat) { + fail('The profile format does not match the current instrumentation.'); + } + return {toolchain, identity, counts, runs, profile, instrumentedBinarySha256}; +} + +function createProof(platform: PlatformName, batch: string, binary: string, commands: string, output: string): void { + const {toolchain, identity, counts, runs, profile, instrumentedBinarySha256} = validateTraining(platform, batch); + const compilation: unknown = JSON.parse(readFileSync(commands, 'utf8')); + const {objects, configuredUnbuilt} = compilerEvidence(compilation, profile); + writeFileSync( + output, + `${JSON.stringify({schema: 1, platform, sourceIdentity: identity, toolchain, instrumentedBinarySha256, counts, runs, profile: {path: profile, sha256: fileHash(profile)}, binary: {path: binary, sha256: fileHash(binary)}, compilation: {status: 'requested-configuration', consumption: 'unverified', path: commands, sha256: fileHash(commands), objects, configuredUnbuilt}}, null, 2)}\n`, + ); +} + +/** Commands are evidence of requested flags; existing object hashes bind their outputs separately. */ +function compilerObjects(input: unknown, profile: string): Array<{path: string; sha256: string}> { + return compilerEvidence(input, profile).objects; +} + +/** Separate unbuilt configured targets from existing output hashes without claiming execution. */ +function compilerEvidence(input: unknown, profile: string): {objects: Array<{path: string; sha256: string}>; configuredUnbuilt: string[]} { + if (!Array.isArray(input)) { + fail('Expected a compilation database.'); + } + const configuredUnbuilt: string[] = []; + const objects = input.flatMap((entry: unknown) => { + if (!isRecord(entry) || typeof entry.directory !== 'string') { + fail('Invalid compilation database entry.'); + } + const args = Array.isArray(entry.arguments) && entry.arguments.every((arg: unknown) => typeof arg === 'string') ? entry.arguments.join(' ') : entry.command; + if (typeof args !== 'string' || !/-fprofile-(?:instr-)?use=/.test(args)) { + return []; + } + const selected = /-fprofile-(?:instr-)?use=(?:"([^"]+)"|'([^']+)'|([^\s]+))/.exec(args); + const selectedPath = selected?.[1] ?? selected?.[2] ?? selected?.[3]; + if (!selectedPath || resolve(entry.directory, selectedPath) !== resolve(profile)) { + fail('Compiler command consumes a different PGO profile.'); + } + const outputMatch = /(?:^|\s)-o\s+(?:"([^"]+)"|'([^']+)'|([^\s]+))/.exec(args); + const objectPath = typeof entry.output === 'string' ? entry.output : (outputMatch?.[1] ?? outputMatch?.[2] ?? outputMatch?.[3]); + if (!objectPath || !args.includes(' -c ')) { + fail('PGO evidence must include a compilation command and its output object.'); + } + const path = resolve(entry.directory, objectPath); + if (!existsSync(path)) { + configuredUnbuilt.push(path); + return []; + } + return [{path, sha256: fileHash(path)}]; + }); + if (objects.length === 0) { + fail('No matching compiler commands have existing object outputs for the selected PGO profile.'); + } + return {objects, configuredUnbuilt}; +} + +if (import.meta.main) { + const [platform, ...args] = process.argv.slice(2); + if ((platform !== 'android' && platform !== 'ios') || args.length !== 4) { + fail('Expected .'); + } + createProof(platform, resolve(valueAt(args, 0)), resolve(valueAt(args, 1)), resolve(valueAt(args, 2)), resolve(valueAt(args, 3))); +} + +export {compilerEvidence, compilerObjects, createProof, validateTraining}; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts index 88af0b143175..cc58511ab68e 100755 --- a/scripts/pgo/pgo.ts +++ b/scripts/pgo/pgo.ts @@ -1,12 +1,14 @@ #!/usr/bin/env bun +// cspell:ignore cooldown Cooldown + // cspell:ignore profraw Profdata profdata import type {TupleToUnion} from 'type-fest'; import CLI from 'expensify-common/CLI'; -import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; -import {resolve} from 'node:path'; +import {copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {join, resolve} from 'node:path'; import type {BenchmarkStats} from '../lib/benchmarkStatistics'; import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; @@ -21,7 +23,8 @@ import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; import {benchmarkHeavyJourneyAll, benchmarkJourneyAll} from './journeyBenchmark'; import {parseJourneyFixture} from './journeyConfig'; -import {STARTUP_SPAN_NAME} from './shared'; +import {androidBuildEnvironment, fileHash, sourceIdentity, toolchainIdentity} from './proof'; +import {benchmarkCooldownSeconds, cooldownBeforeBenchmark, STARTUP_SPAN_NAME} from './shared'; const DEFAULT_STARTUP_RUNS = 10; const DEFAULT_STARTUP_WAIT_SECONDS = 30; @@ -162,7 +165,7 @@ async function runWorkflow( await benchmarkBuild(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), 'optimized', runs, timeoutSeconds); return; case 'benchmark': - await benchmarkAll(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), runs, timeoutSeconds); + await benchmarkAll(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); return; case 'compare-benchmarks': await compareBenchmarks(adapter); @@ -251,6 +254,7 @@ async function recordStartups(adapter: PlatformAdapter, benchmarkAdapter: Native async function benchmarkBuild(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, kind: BenchmarkKind, runs: number, waitTimeSeconds: number): Promise { mkdirSync(adapter.benchmarkDirectory, {recursive: true}); + await cooldownBeforeBenchmark(); await runBenchmarkStartups(benchmarkAdapter, { mode: 'process', spanNames: [STARTUP_SPAN_NAME], @@ -261,17 +265,39 @@ async function benchmarkBuild(adapter: PlatformAdapter, benchmarkAdapter: Native }); } -async function benchmarkAll(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, runs: number, waitTimeSeconds: number): Promise { - console.log('=== Benchmark phase 1/2: Release ==='); - console.log(`Installing release artifact: ${adapter.artifactPaths.release}`); - adapter.install('release'); - await benchmarkBuild(adapter, benchmarkAdapter, 'release', runs, waitTimeSeconds); - - console.log('=== Benchmark phase 2/2: PGO optimized ==='); - await benchmarkAdapter.prepareStartup('process'); - console.log(`Installing PGO optimized artifact: ${adapter.artifactPaths.optimized}`); - adapter.install('optimized'); - await benchmarkBuild(adapter, benchmarkAdapter, 'optimized', runs, waitTimeSeconds); +async function benchmarkAll(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, runs: number, waitTimeSeconds: number, order: BenchmarkOrder): Promise { + const batch = join(adapter.benchmarkDirectory, 'startup', new Date().toISOString().replaceAll(/[:.]/g, '-')); + mkdirSync(batch, {recursive: true}); + writeFileSync( + join(batch, 'run.json'), + `${JSON.stringify( + { + platform: adapter.name, + deviceID: benchmarkAdapter.deviceIdentifier, + appID: adapter.appID(), + order, + runs, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: benchmarkCooldownSeconds(), + thermalStateVerified: false, + measurementDriverIdentity: sourceIdentity(), + toolchain: toolchainIdentity(adapter), + buildEnvironment: adapter.name === 'android' ? androidBuildEnvironment() : null, + artifacts: {release: adapter.artifactPaths.release, optimized: adapter.artifactPaths.optimized}, + artifactSHA256: {release: fileHash(adapter.artifactPaths.release), optimized: fileHash(adapter.artifactPaths.optimized)}, + }, + null, + 2, + )}\n`, + ); + const phases: BenchmarkKind[] = order === 'release-first' ? ['release', 'optimized'] : ['optimized', 'release']; + for (const kind of phases) { + await benchmarkAdapter.prepareStartup('process'); + console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); + adapter.install(kind); + await benchmarkBuild(adapter, benchmarkAdapter, kind, runs, waitTimeSeconds); + copyFileSync(adapter.benchmarkPaths[kind], join(batch, `${kind}.csv`)); + } console.log('=== Benchmark comparison ==='); await compareBenchmarks(adapter); diff --git a/scripts/pgo/pgoCandidate.ts b/scripts/pgo/pgoCandidate.ts new file mode 100755 index 000000000000..a85cb8ff9a15 --- /dev/null +++ b/scripts/pgo/pgoCandidate.ts @@ -0,0 +1,194 @@ +#!/usr/bin/env bun +/* eslint-disable @typescript-eslint/naming-convention -- CLI and GitHub output keys use their external hyphenated schema. */ +/** Capture deployment identity or resolve a trusted evaluation plan without device access. */ +import {createHash} from 'node:crypto'; +import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join, resolve} from 'node:path'; +import {parseArgs} from 'node:util'; + +import type {GitHubRead} from './candidateResolve'; +import type {CandidateMetadata} from './evaluationSchema'; + +import {capture} from '../lib/scriptUtils'; +import {assertProducerEventAttempt, resolvePreview, resolveProducer, selectProducerArtifact} from './candidateResolve'; +import {isRecord, parseCandidate} from './evaluationSchema'; + +async function main(): Promise { + const {values, positionals} = parseArgs({ + allowPositionals: true, + options: { + output: {type: 'string'}, + environment: {type: 'string'}, + 'native-eligible': {type: 'string'}, + 'source-run-id': {type: 'string'}, + platform: {type: 'string', default: 'both'}, + }, + }); + const output = values.output; + if (!output) { + throw new Error('An explicit output path is required.'); + } + if (positionals.at(0) === 'capture') { + if (!['true', 'false'].includes(values['native-eligible'] ?? '')) { + throw new Error('Explicit native eligibility is required.'); + } + const candidate = captureCandidate(values.environment ?? '', values['native-eligible'] === 'true'); + writeJSON(output, candidate); + return; + } + if (positionals.at(0) !== 'prepare' || process.env.GITHUB_REPOSITORY !== 'Expensify/App') { + throw new Error('Expected a trusted prepare invocation in Expensify/App.'); + } + const read = createGitHubReader(); + const event: unknown = JSON.parse(readFileSync(requiredEnvironment('GITHUB_EVENT_PATH'), 'utf8')); + const toolingSha = capture('git', ['rev-parse', 'HEAD']).trim(); + const platform = values.platform === 'both' ? undefined : values.platform; + if (platform !== undefined && platform !== 'android' && platform !== 'ios') { + throw new Error('Invalid evaluation platform.'); + } + let plan; + if (process.env.GITHUB_EVENT_NAME === 'pull_request') { + plan = await resolvePreview(event, toolingSha, read, platform); + } else { + let runID = 0; + if (values['source-run-id']) { + runID = positiveID(values['source-run-id']); + } else if (isRecord(event) && isRecord(event.workflow_run)) { + runID = positiveID(String(event.workflow_run.id)); + } + if (runID === 0) { + throw new Error('An immutable producer run ID is required.'); + } + const run = await read(`/repos/Expensify/App/actions/runs/${runID}`); + if (!isRecord(run)) { + throw new Error('Invalid producer run.'); + } + const jobs = await read(`/repos/Expensify/App/actions/runs/${runID}/attempts/${positiveID(String(run.run_attempt))}/jobs?per_page=100`); + if (!isRecord(jobs) || Number(jobs.total_count) > 100) { + throw new Error('Producer jobs require complete bounded enumeration.'); + } + if (process.env.GITHUB_EVENT_NAME === 'workflow_run') { + assertProducerEventAttempt(event, run); + } + plan = await resolveProducer(await producerMetadata(runID, positiveID(String(run.run_attempt)), read), run, jobs, toolingSha, read, platform); + } + writeJSON(output, plan); + appendOutputs({ + eligible: String(plan.platforms.length > 0), + 'app-sha': plan.candidate.appSha, + 'mobile-sha': plan.candidate.mobileSha, + version: plan.candidate.version, + platforms: plan.platforms.join(','), + 'platform-matrix': JSON.stringify(plan.platforms), + }); +} + +function captureCandidate(environment: string, nativeEligible: boolean, readSource = capture): CandidateMetadata { + if (process.env.GITHUB_REPOSITORY !== 'Expensify/App' || !['staging', 'production'].includes(environment)) { + throw new Error('Invalid producer repository or environment.'); + } + const pkg: unknown = JSON.parse(readFileSync('package.json', 'utf8')); + if (!isRecord(pkg)) { + throw new Error('Missing deployment package metadata.'); + } + const mobileSha = readSource('git', ['rev-parse', 'HEAD:Mobile-Expensify']).trim(); + if (readSource('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim() !== mobileSha) { + throw new Error('Native checkout differs from the committed deployment gitlink.'); + } + const candidate = parseCandidate({appSha: readSource('git', ['rev-parse', 'HEAD']).trim(), mobileSha, version: pkg.version, environment}); + return { + schema: 1, + repository: 'Expensify/App', + runID: positiveID(requiredEnvironment('GITHUB_RUN_ID')), + runAttempt: positiveID(requiredEnvironment('GITHUB_RUN_ATTEMPT')), + nativeEligible, + candidate, + }; +} + +async function producerMetadata(runID: number, runAttempt: number, read: GitHubRead): Promise { + const response = await read(`/repos/Expensify/App/actions/runs/${runID}/artifacts?per_page=100`); + const artifact = selectProducerArtifact(response, runID, runAttempt); + const responseZip = await fetch(`https://api.github.com/repos/Expensify/App/actions/artifacts/${artifact.id}/zip`, {headers: githubHeaders()}); + if (!responseZip.ok) { + throw new Error('Producer metadata download failed.'); + } + const bytes = Buffer.from(await responseZip.arrayBuffer()); + if (bytes.length > 1024 * 1024) { + throw new Error('Producer metadata archive exceeds its size bound.'); + } + if (typeof artifact.digest === 'string' && artifact.digest !== `sha256:${createHash('sha256').update(bytes).digest('hex')}`) { + throw new Error('Producer metadata digest differs.'); + } + const directory = mkdtempSync(join(tmpdir(), 'pgo-candidate-')); + try { + const archive = join(directory, 'candidate.zip'); + writeFileSync(archive, bytes); + const entries = capture('unzip', ['-Z1', archive]).trim().split('\n'); + if (entries.length !== 1 || entries.at(0) !== 'candidate.json') { + throw new Error('Unexpected producer metadata archive contents.'); + } + return JSON.parse(capture('unzip', ['-p', archive, 'candidate.json'])) as unknown; + } finally { + rmSync(directory, {recursive: true, force: true}); + } +} + +function createGitHubReader(): GitHubRead { + return async (path) => { + if (!path.startsWith('/repos/Expensify/App/')) { + throw new Error('GitHub read escaped the approved repository.'); + } + const response = await fetch(`https://api.github.com${path}`, {headers: githubHeaders()}); + if (!response.ok) { + throw new Error(`GitHub metadata request failed (${response.status}).`); + } + return response.json(); + }; +} + +function githubHeaders(): Record { + return {Authorization: `Bearer ${requiredEnvironment('GITHUB_TOKEN')}`, Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'}; +} +function positiveID(value: string): number { + const id = Number(value); + if (!Number.isSafeInteger(id) || id <= 0) { + throw new Error('Invalid immutable run identity.'); + } + return id; +} +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing ${name}.`); + } + return value; +} +function writeJSON(path: string, value: unknown): void { + mkdirSync(dirname(resolve(path)), {recursive: true}); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} +function appendOutputs(values: Record): void { + if (!process.env.GITHUB_OUTPUT) { + return; + } + writeFileSync( + process.env.GITHUB_OUTPUT, + Object.entries(values) + .map(([key, value]) => `${key}=${value}\n`) + .join(''), + {flag: 'a'}, + ); +} + +if (import.meta.main) { + try { + await main(); + } catch { + appendOutputs({eligible: 'false', reason: 'Candidate metadata or trust validation failed.'}); + console.error('PGO candidate metadata or trust validation failed; no device evaluation is eligible.'); + process.exitCode = 1; + } +} +export {captureCandidate, createGitHubReader, main}; diff --git a/scripts/pgo/pgoEvaluation.ts b/scripts/pgo/pgoEvaluation.ts new file mode 100755 index 000000000000..f4721d68c18a --- /dev/null +++ b/scripts/pgo/pgoEvaluation.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env bun +/** Evaluation-only CLI: native execution and hosted public reporting are separate operations. */ +import {mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {basename, join} from 'node:path'; +import {parseArgs} from 'node:util'; + +import {findFiles} from '../lib/scriptUtils'; +import {evaluationReport} from './evaluationReport'; +import {parsePlan} from './evaluationSchema'; +import {createGitHubReader} from './pgoCandidate'; + +async function main(): Promise { + const {values, positionals} = parseArgs({allowPositionals: true, options: {plan: {type: 'string'}, output: {type: 'string'}, results: {type: 'string'}}}); + if (!values.plan || !values.output) { + throw new Error('Explicit plan and output paths are required.'); + } + const plan = parsePlan(JSON.parse(readFileSync(values.plan, 'utf8'))); + mkdirSync(values.output, {recursive: true}); + if (positionals.at(0) === 'execute') { + const platform = positionals.at(1); + if ((platform !== 'android' && platform !== 'ios') || !plan.platforms.includes(platform)) { + throw new Error('Platform is not eligible for this immutable plan.'); + } + const {executeEvaluation} = await import('./evaluationRun'); + await executeEvaluation(plan, platform, values.output); + return; + } + if (positionals.at(0) !== 'report' || !values.results) { + throw new Error('Expected execute or report with explicit results.'); + } + const files = findFiles(values.results, '.json').filter((path) => /^evaluation-(android|ios)\.json$/.test(basename(path))); + const report = await evaluationReport( + plan, + files.map((path) => JSON.parse(readFileSync(path, 'utf8')) as unknown), + createGitHubReader(), + ); + writeFileSync(join(values.output, 'summary.json'), `${JSON.stringify(report.summary, null, 2)}\n`); + writeFileSync(join(values.output, 'summary.md'), report.markdown); + writeFileSync(join(values.output, 'comment.json'), `${JSON.stringify(report.comment, null, 2)}\n`); +} + +if (import.meta.main) { + await main(); +} +export default main; diff --git a/scripts/pgo/proof.ts b/scripts/pgo/proof.ts new file mode 100644 index 000000000000..2bc5a8b7a59b --- /dev/null +++ b/scripts/pgo/proof.ts @@ -0,0 +1,99 @@ +// cspell:ignore profdata profraw fprofile + +import {createHash} from 'node:crypto'; +import {readFileSync, readdirSync, realpathSync, statSync} from 'node:fs'; +import {join, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; +import type {PlatformAdapter} from './shared'; + +import {capture, captureBinary, fail, rootDirectory} from '../lib/scriptUtils'; + +function toolchainIdentity(adapter: PlatformAdapter): string { + return JSON.stringify({ + profileTool: capture(adapter.llvmTool('llvm-profdata'), ['--version']).trim(), + compiler: capture(adapter.llvmTool('clang'), ['--version']).trim(), + platform: adapter.name, + buildTool: adapter.name === 'ios' ? capture('xcodebuild', ['-version']).trim() : capture('java', ['--version']).trim(), + sdk: adapter.name === 'ios' ? capture('xcrun', ['--sdk', 'iphoneos', '--show-sdk-version']).trim() : adapter.llvmTool('clang'), + androidReleasePipeline: process.env.PGO_ANDROID_RELEASE_PIPELINE ?? 'false', + androidShrink: process.env.PGO_ANDROID_SHRINK ?? 'false', + buildEnvironment: adapter.name === 'android' ? androidBuildEnvironment() : null, + }); +} + +/** Resolve ENVFILE exactly relative to the hybrid Android project's parent directory. */ +function androidBuildEnvironment(envFile = process.env.ENVFILE): {path: string; sha256: string} | null { + if (!envFile) { + return null; + } + const path = realpathSync(resolve(rootDirectory, 'Mobile-Expensify', envFile)); + return {path, sha256: createHash('sha256').update(readFileSync(path)).digest('hex')}; +} + +/** Bind tracked changes and untracked source files without changing either checkout. */ +function sourceIdentity(): string { + const hash = createHash('sha256'); + for (const directory of [rootDirectory, join(rootDirectory, 'Mobile-Expensify')]) { + hash.update(capture('git', ['rev-parse', 'HEAD'], directory)); + hash.update(captureBinary('git', ['diff', 'HEAD', '--binary'], directory)); + const paths = capture('git', ['ls-files', '--others', '--exclude-standard', '-z'], directory).split('\0').filter(Boolean).sort(); + for (const path of paths) { + hash.update(path); + hash.update('\0'); + hash.update(readFileSync(join(directory, path))); + } + } + return hash.digest('hex'); +} + +/** Reject partial batches and profiles collected from different source contents. */ +function validateJourney(input: unknown, platform: PlatformName, identity: string, toolchain: string): void { + if ( + isRecord(input) && + input.platform === platform && + input.status === 'passed' && + input.accountClass === 'heavy' && + input.requestedRuns === 3 && + input.completedRuns === 3 && + input.collectProfiles === true && + input.navigationOnly === false && + input.sourceIdentity === identity && + input.toolchain === toolchain + ) { + return; + } + fail('PGO proof requires three complete collected heavy-account journeys from the current source and toolchain.'); +} + +function profileCounts(summary: string): {functions: number; count: string} { + const functions = Number(/Total functions:\s*([0-9]+)/.exec(summary)?.[1]); + const count = /(?:Total count|Maximum function count):\s*([0-9]+)/.exec(summary)?.[1]; + if (!Number.isSafeInteger(functions) || functions <= 0 || !count || BigInt(count) <= 0n) { + fail('The LLVM profile has no executed functions.'); + } + return {functions, count}; +} + +function fileHash(path: string, allowEmpty = false): string { + const stat = statSync(path); + if (stat.isDirectory()) { + const hash = createHash('sha256'); + for (const name of readdirSync(path).sort()) { + hash.update(name); + hash.update('\0'); + hash.update(fileHash(join(path, name), true)); + } + return hash.digest('hex'); + } + if (stat.size === 0 && !allowEmpty) { + fail(`Empty PGO evidence file: ${path}`); + } + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export {androidBuildEnvironment, fileHash, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney}; diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts index bc50f922d7fe..c02c63c7f802 100644 --- a/scripts/pgo/shared.ts +++ b/scripts/pgo/shared.ts @@ -1,7 +1,11 @@ +// cspell:ignore cooldown Cooldown + /** Shared PGO workflow types and constants. */ import type {PlatformName} from '../lib/nativeAppBenchmark'; +import {fail, sleep} from '../lib/scriptUtils'; + const STARTUP_SPAN_NAME = 'ManualAppStartup'; const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; const HEAVY_JOURNEY_SPAN_NAMES = ['/r/*'] as const; @@ -31,5 +35,19 @@ type PlatformAdapter = { llvmTool: (name: string) => string; }; -export {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; +function benchmarkCooldownSeconds(): number { + const seconds = Number(process.env.PGO_BENCHMARK_COOLDOWN_SECONDS ?? 0); + if (!Number.isSafeInteger(seconds) || seconds < 0 || seconds > 60) { + fail('Benchmark cooldown seconds must be an integer between 0 and 60.'); + } + return seconds; +} + +async function cooldownBeforeBenchmark(): Promise { + const seconds = benchmarkCooldownSeconds(); + console.log(`Waiting ${seconds}s before the discarded warm-up. This delay does not verify thermal state.`); + await sleep(seconds * 1000); +} + +export {benchmarkCooldownSeconds, cooldownBeforeBenchmark, BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/pgoAndroidRelease.test.ts b/tests/tooling/pgoAndroidRelease.test.ts new file mode 100644 index 000000000000..1e9576a78754 --- /dev/null +++ b/tests/tooling/pgoAndroidRelease.test.ts @@ -0,0 +1,70 @@ +// cspell:ignore Ppgo profdata + +import {describe, expect, it} from 'bun:test'; + +import {androidReleasePipeline, androidRockBuildArguments} from '@scripts/pgo/android'; + +import {parseArgs} from '@rock-js/tools'; + +describe('PGO Android release processing', () => { + it('builds locally through Rock with the exact optimized profile and FullStory exclusions', () => { + // Given a shrinking build and an immutable profile in a directory containing spaces. + const pipeline = androidReleasePipeline(undefined, 'true'); + const parameters = ['-PpgoMode=use', '-PpgoProfile=/tmp/profile batch/journey.profdata', ...pipeline.arguments]; + // When the adapter forwards this build to Rock. + const args = androidRockBuildArguments(pipeline.task, parameters); + const forwarded = parseArgs(args.at(-1) ?? ''); + // Then Rock cannot reuse a cached build or lose profile and upload-prevention arguments. + expect(args.slice(0, 7)).toEqual(['build:android', '--local', '--verbose', '--variant', 'release', '--tasks', 'packageRelease']); + expect(forwarded).toEqual(parameters); + expect(forwarded).toContain('-PpgoSkipFullStory=true'); + expect(forwarded).toContain('fullstoryRelease'); + }); + it('keeps the bootstrap build separate from FullStory output', () => { + // Given no explicit release pipeline request. + // When the build selects its task and artifact. + const options = androidReleasePipeline(); + // Then bootstrap processing does not masquerade as the final release pipeline. + expect(options).toEqual({enabled: false, shrinking: false, artifactDirectory: 'apk', task: ':packageRelease', apkTask: 'packageRelease', arguments: ['-x', 'fullstoryRelease']}); + }); + + it('selects the processed APK without overriding local credentials', () => { + // Given an explicit request to exercise shrinking and FullStory. + // When the build selects its task and artifact. + const options = androidReleasePipeline('true'); + // Then it targets the final processed APK and supplies no signing or identity overrides. + expect(options.task).toBe(':fullstoryRelease'); + expect(options.apkTask).toBe('fullstoryRelease'); + expect(options.arguments.at(0)).toBe('--init-script'); + expect(options.arguments.join(' ')).not.toMatch(/signing|password|applicationId/i); + }); + + it('rejects ambiguous opt-in settings', () => { + // Given a mistyped environment setting. + // When the build decides whether release processing is required. + const select = () => androidReleasePipeline('yes'); + // Then it fails instead of silently substituting the bootstrap build. + expect(select).toThrow('must be true or false'); + }); + + it('shrinks locally while excluding external FullStory processing', () => { + // Given a request for release shrinking without asset uploads. + // When the build selects its tasks and output directory. + const options = androidReleasePipeline(undefined, 'true'); + // Then the APK remains distinct and FullStory is both excluded and disabled. + expect(options.shrinking).toBe(true); + expect(options.enabled).toBe(false); + expect(options.artifactDirectory).toBe('apk-shrunk'); + expect(options.task).toBe(':packageRelease'); + expect(options.arguments).toContain('-PpgoSkipFullStory=true'); + expect(options.arguments).toContain('-x'); + }); + + it('rejects simultaneous shrinking and upload pipeline requests', () => { + // Given conflicting local-only and external processing choices. + // When the build selects a pipeline. + const select = () => androidReleasePipeline('true', 'true'); + // Then it cannot silently choose the upload-capable task. + expect(select).toThrow('not both'); + }); +}); diff --git a/tests/tooling/pgoCandidate.test.ts b/tests/tooling/pgoCandidate.test.ts new file mode 100644 index 000000000000..2f3b7e862319 --- /dev/null +++ b/tests/tooling/pgoCandidate.test.ts @@ -0,0 +1,166 @@ +// cspell:ignore chrispader +/* eslint-disable @typescript-eslint/naming-convention -- Fixtures reproduce the GitHub API snake_case schema. */ +import {describe, expect, it} from 'bun:test'; + +import {execFileSync} from 'node:child_process'; + +import type {GitHubRead} from '../../scripts/pgo/candidateResolve'; + +import {assertProducerEventAttempt, checklistMatches, matchingChecklist, resolvePreview, resolveProducer, selectProducerArtifact} from '../../scripts/pgo/candidateResolve'; +import {captureCandidate} from '../../scripts/pgo/pgoCandidate'; + +const appSha = 'a'.repeat(40); +const mobileSha = 'b'.repeat(40); +const toolingSha = 'c'.repeat(40); +const candidate = {appSha, mobileSha, version: '9.4.94-0', environment: 'staging'}; +const metadata = {schema: 1, repository: 'Expensify/App', runID: 10, runAttempt: 2, nativeEligible: true, candidate}; +const run = { + id: 10, + run_attempt: 2, + repository: {full_name: 'Expensify/App'}, + name: 'Deploy code to staging or production', + path: '.github/workflows/deploy.yml', + status: 'completed', + conclusion: 'failure', + event: 'workflow_dispatch', + head_branch: 'staging', + head_sha: toolingSha, +}; +const jobs = { + jobs: [ + {name: 'Build Android HybridApp / Build Android app', conclusion: 'success'}, + {name: 'Build iOS HybridApp / Build iOS HybridApp', conclusion: 'failure'}, + ], +}; +const read: GitHubRead = async (path) => { + if (path.includes('/git/trees/')) { + return {tree: [{path: 'Mobile-Expensify', mode: '160000', sha: mobileSha}]}; + } + if (path.includes('/contents/package.json')) { + return {content: Buffer.from(JSON.stringify({version: candidate.version})).toString('base64')}; + } + if (path.includes('/compare/')) { + return {status: 'ahead'}; + } + if (path.includes('/pulls/')) { + return {head: {sha: appSha}, state: 'open'}; + } + if (path.includes('/issues?')) { + return [{number: 42, body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]}]; + } + throw new Error('Unexpected metadata request.'); +}; + +function preview(repo = 'Expensify/App', sha = appSha) { + return {number: 3, pull_request: {head: {sha, ref: '@chrispader/perf/pgo-ci-evaluation', repo: {full_name: repo}}, base: {repo: {full_name: 'Expensify/App'}}}}; +} + +describe('PGO immutable candidate resolution', () => { + it('uses actual producer metadata rather than dispatch head and preserves independent platform success', async () => { + // Given a completed deployment with web/iOS failure and a different dispatch head. + // When the exact producer metadata and nested native jobs are resolved. + const plan = await resolveProducer(metadata, run, jobs, toolingSha, read); + // Then the successful Android build alone is eligible at the actual immutable source. + expect(plan.platforms).toEqual(['android']); + expect(plan.candidate.appSha).toBe(appSha); + expect(plan.candidate.appSha).not.toBe(run.head_sha); + expect(plan.checklistIssueNumber).toBe(42); + }); + + it('rejects missing metadata, retry drift, wrong environment and candidate gitlink drift', async () => { + // Given metadata tied to one repository, run attempt and native checkout. + const changedRead: GitHubRead = async (path) => (path.includes('/git/trees/') ? {tree: [{path: 'Mobile-Expensify', mode: '160000', sha: 'd'.repeat(40)}]} : read(path)); + // When a required identity differs or metadata is absent. + // Then no credential-device plan may be created. + await expect(resolveProducer(undefined, run, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer({...metadata, runAttempt: 1}, run, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer(metadata, {...run, head_branch: 'feature'}, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer(metadata, run, jobs, toolingSha, changedRead)).rejects.toThrow(); + await expect(resolveProducer({...metadata, runID: '10'}, run, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer({...metadata, repository: 'fork/App'}, run, jobs, toolingSha, read)).rejects.toThrow(); + }); + + it('does not confuse unrelated successful jobs or non-native production promotions with eligible builds', async () => { + // Given a spoofed job suffix and a promotion that reuses prior native binaries. + // When platform eligibility is resolved. + const unrelated = await resolveProducer(metadata, run, {jobs: [{name: 'Unrelated / Build Android app', conclusion: 'success'}]}, toolingSha, read); + const reused = await resolveProducer({...metadata, nativeEligible: false}, run, jobs, toolingSha, read); + // Then neither can initiate evaluation on credential devices. + expect(unrelated.platforms).toEqual([]); + expect(reused.platforms).toEqual([]); + }); + + it('requires an open exact same-repository preview head and rejects forks or stale heads', async () => { + // Given the one approved preview branch in Expensify/App. + // When a fork, stale revision or closed pull request is supplied. + const plan = await resolvePreview(preview(), toolingSha, read); + // Then only the current trusted head can evaluate; normal forks receive no device authorization. + expect(plan.platforms).toEqual(['android', 'ios']); + await expect(resolvePreview(preview('fork/App'), toolingSha, read)).rejects.toThrow(); + await expect(resolvePreview(preview('Expensify/App', toolingSha), toolingSha, read)).rejects.toThrow(); + await expect(resolvePreview(preview(), toolingSha, async (path) => (path.includes('/pulls/') ? {head: {sha: appSha}, state: 'closed'} : read(path)))).rejects.toThrow(); + }); + + it('matches historical release headers exactly instead of a latest issue or version substring', () => { + // Given one labelled historical checklist and unrelated version text. + const issue = {number: 42, body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]}; + // When the immutable candidate version is compared. + // Then only its exact labelled release header qualifies. + expect(checklistMatches(issue, '9.4.94-0')).toBe(true); + expect(checklistMatches(issue, '9.4.94')).toBe(false); + expect(checklistMatches({...issue, body: 'A link mentions 9.4.94-0'}, '9.4.94-0')).toBe(false); + expect(checklistMatches({...issue, labels: []}, '9.4.94-0')).toBe(false); + }); + + it('paginates historical checklists and rejects ambiguous exact versions', async () => { + // Given more than one page of historical deployment checklists. + const otherIssues = Array.from({length: 100}, () => ({body: '**Release Version:** `1.0.0-staging`', labels: [{name: 'StagingDeployCash'}], number: 1})); + const historical = {number: 42, body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]}; + // When the matching older version is on the second page. + const found = await matchingChecklist('9.4.94-0', async (path) => (path.endsWith('page=1') ? otherIssues : [historical])); + // Then the exact historical issue is used, and duplicate matches fail closed. + expect(found).toBe(42); + await expect(matchingChecklist('9.4.94-0', async () => [historical, {...historical, number: 43}])).rejects.toThrow(); + }); + + it('rejects a queued producer event superseded by a newer attempt', () => { + // Given a workflow event emitted before its producer was rerun. + const oldEvent = {workflow_run: {id: 10, run_attempt: 1}}; + // When current metadata refers to completed attempt two. + // Then the queued event cannot silently evaluate the newer producer. + expect(() => assertProducerEventAttempt(oldEvent, run)).toThrow(); + expect(() => assertProducerEventAttempt({workflow_run: {id: 10, run_attempt: 2}}, run)).not.toThrow(); + }); + + it('selects only metadata from the current immutable producer attempt', () => { + // Given retained artifacts from two attempts of the same workflow run. + const old = {id: 1, name: 'pgo-native-candidate-1', expired: false, workflow_run: {id: 10}}; + const current = {id: 2, name: 'pgo-native-candidate-2', expired: false, workflow_run: {id: 10}}; + // When manual evaluation resolves the latest completed attempt. + const selected = selectProducerArtifact({total_count: 2, artifacts: [old, current]}, 10, 2); + // Then stale or cross-run metadata cannot substitute for its exact artifact. + expect(selected.id).toBe(2); + expect(() => selectProducerArtifact({total_count: 1, artifacts: [old]}, 10, 2)).toThrow(); + expect(() => selectProducerArtifact({total_count: 1, artifacts: [{...current, workflow_run: {id: 11}}]}, 10, 2)).toThrow(); + }); + + it('captures the actual local deployment HEAD and gitlink even when the workflow context SHA differs', () => { + // Given a checked-out deployment and an unrelated workflow-context revision. + const previous = {...process.env}; + Object.assign(process.env, {GITHUB_REPOSITORY: 'Expensify/App', GITHUB_RUN_ID: '10', GITHUB_RUN_ATTEMPT: '2', GITHUB_SHA: appSha}); + try { + // When the producer captures its actual repository state. + const actualHead = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim(); + const actualLink = execFileSync('git', ['rev-parse', 'HEAD:Mobile-Expensify'], {encoding: 'utf8'}).trim(); + const readSource = (_command: string, args: string[]) => (args.includes('-C') || args.includes('HEAD:Mobile-Expensify') ? actualLink : actualHead); + const captured = captureCandidate('staging', true, readSource); + expect(() => captureCandidate('staging', true, (_command, args) => (args.includes('-C') ? appSha : readSource(_command, args)))).toThrow(); + // Then HEAD and the committed native gitlink are bound independently of GITHUB_SHA. + expect(captured.candidate.appSha).toBe(execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim()); + expect(captured.candidate.mobileSha).toBe(execFileSync('git', ['rev-parse', 'HEAD:Mobile-Expensify'], {encoding: 'utf8'}).trim()); + expect(captured.runAttempt).toBe(2); + } finally { + process.env = previous; + } + }); +}); diff --git a/tests/tooling/pgoEvaluationReport.test.ts b/tests/tooling/pgoEvaluationReport.test.ts new file mode 100644 index 000000000000..951ea217faeb --- /dev/null +++ b/tests/tooling/pgoEvaluationReport.test.ts @@ -0,0 +1,173 @@ +// cspell:ignore cooldown +import {describe, expect, it} from 'bun:test'; + +import type {EvaluationCohort, EvaluationPlan, EvaluationResult} from '../../scripts/pgo/evaluationSchema'; + +import {distribution, evaluationReport, summarizePlatform, summarizeFrames} from '../../scripts/pgo/evaluationReport'; +import {candidateKey, parsePlan} from '../../scripts/pgo/evaluationSchema'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from '../../scripts/pgo/shared'; + +const hash = 'a'.repeat(64); +const plan: EvaluationPlan = { + schema: 1, + repository: 'Expensify/App', + toolingSha: 'c'.repeat(40), + candidate: {appSha: 'a'.repeat(40), mobileSha: 'b'.repeat(40), version: '9.4.94-0', environment: 'staging'}, + producerRunID: 10, + producerRunAttempt: 2, + platforms: ['android'], + checklistIssueNumber: 42, +}; +function result(): EvaluationResult { + const definitions = [ + ['startup', ['ManualAppStartup']], + ['journey', [...JOURNEY_SPAN_NAMES]], + ['heavy-journey', [...HEAVY_JOURNEY_SPAN_NAMES]], + ] as const; + const cohorts: EvaluationCohort[] = definitions.flatMap(([scenario, spans]) => + (['release-first', 'optimized-first'] as const).map((order) => ({ + scenario, + order, + spans: [...spans], + runs: 20, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: 5, + releaseCsvSha256: hash, + optimizedCsvSha256: hash, + release: Array.from({length: 20}, (_, index) => spans.map((span) => ({run: index + 1, span, durationMs: 100}))).flat(), + optimized: Array.from({length: 20}, (_, index) => spans.map((span) => ({run: index + 1, span, durationMs: 90}))).flat(), + })), + ); + return { + schema: 1, + platform: 'android', + candidate: plan.candidate, + toolingSha: plan.toolingSha, + status: 'passed', + artifactSha256: {release: hash, instrumented: hash, optimized: hash}, + profileSha256: hash, + sourceIdentity: hash, + toolchain: {compilerVersion: 'Clang 17'}, + compilerConsumption: 'unverified', + cohorts, + }; +} + +describe('PGO public evaluation reports', () => { + it('retains all observations and computes distributions and positive-faster effects without selection', async () => { + // Given complete opposite-order cohorts for every workload. + const input = result(); + // When the report computes numeric summaries. + const report = await evaluationReport(plan, [input], async () => ({body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]})); + const summary = report.summary; + // Then all samples remain and the mean effect is ten percent without choosing a deployment artifact. + expect(summary.status).toBe('complete'); + expect(summary.platforms.at(0)?.cohorts?.at(0)?.statistics.at(0)?.meanEffectPercent).toBe(10); + expect(report.markdown).toContain('canonical baseline artifacts and deployment selection are unchanged'); + expect(report.comment.targetIssueNumber).toBe(42); + expect(report.comment.body).toStartWith("Chris's AI"); + }); + + it('rejects incomplete, extra, reordered and cross-candidate measurements rather than salvaging subsets', () => { + // Given a complete result bound to the candidate. + const input = result(); + const cohort = input.cohorts.at(0); + if (!cohort) { + throw new Error('Missing synthetic cohort.'); + } + // When any mandatory measurement or identity is changed. + // Then the entire claimed complete platform evidence is rejected. + expect(() => summarizePlatform(plan, 'android', {...input, candidate: {...plan.candidate, appSha: 'd'.repeat(40)}})).toThrow(); + expect(() => summarizePlatform(plan, 'android', {...input, cohorts: input.cohorts.slice(1)})).toThrow(); + for (const release of [cohort.release.slice(1), [...cohort.release, {...cohort.release.at(0)}], [...cohort.release].reverse()]) { + expect(() => summarizePlatform(plan, 'android', {...input, cohorts: [{...cohort, release}, ...input.cohorts.slice(1)]})).toThrow(); + } + expect(() => summarizePlatform(plan, 'android', {...input, cohorts: [{...cohort, warmUpRunsPerBinary: 0}, ...input.cohorts.slice(1)]})).toThrow(); + }); + + it('whitelists public numeric provenance and excludes private fields and raw errors', async () => { + // Given private extra fields that must never enter uploaded or commented report data. + const input = {...result(), accountEmail: 'private@example.com', fixture: '/private/fixture.json', rawError: 'private error', deviceID: 'secret-device'}; + // When explicit output fields are selected and an unavailable platform is represented. + const report = await evaluationReport(plan, [input], async () => ({body: 'wrong version', labels: [{name: 'StagingDeployCash'}]})); + const serialized = JSON.stringify(report); + // Then secrets are omitted and an obsolete checklist cannot receive a report for another version. + expect(serialized).not.toContain('private@example.com'); + expect(serialized).not.toContain('secret-device'); + expect(serialized).not.toContain('/private/fixture.json'); + expect(report.comment.targetIssueNumber).toBeNull(); + expect(report.comment.skippedReason).toContain('No exact historical'); + }); + + it('keeps missing and failed platforms explicit while candidate-specific markers remain stable across reruns', async () => { + // Given a missing result or a failed cohort for the same immutable candidate. + const first = await evaluationReport(plan, [], async () => ({})); + const second = await evaluationReport( + plan, + [{schema: 1, candidate: plan.candidate, toolingSha: plan.toolingSha, platform: 'android', status: 'failed', rawError: 'secret'}], + async () => ({}), + ); + // When the version or App commit changes. + const other = {...plan.candidate, appSha: 'd'.repeat(40)}; + // Then the old failed result is not converted to numeric success, and other candidates have separate markers. + expect(JSON.stringify(first.summary)).toContain('missing'); + expect(JSON.stringify(second.summary)).toContain('failed'); + expect(first.comment.marker).toBe(second.comment.marker); + expect(candidateKey(other)).not.toBe(candidateKey(plan.candidate)); + expect(JSON.stringify(second)).not.toContain('secret'); + }); + + it('retains hashed overlay/config provenance and reports missing frames without zero filling', () => { + // Given approved digest fields and twenty unavailable ancillary samples. + const input = {...result(), toolchain: {identitySha256: hash, reviewedOverlaySha256: hash, environmentSha256: hash, fixtureSha256: hash}}; + const missing = Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable', raw: 'private'})); + // When numeric coverage and the public hash whitelist are summarized. + const publicResult = summarizePlatform(plan, 'android', input); + const coverage = summarizeFrames(missing); + // Then provenance survives while unavailable frames have no fabricated summary or raw data. + expect(publicResult.toolchain?.reviewedOverlaySha256).toBe(hash); + expect(coverage.availableSamples).toBe(0); + expect(coverage.unavailableSamples).toBe(20); + expect(coverage.summary).toBeNull(); + expect(JSON.stringify(coverage)).not.toContain('private'); + expect(() => summarizeFrames(missing.slice(1))).toThrow(); + }); + + it('weights native jank by total frames while preserving original percentages and windows', () => { + // Given two available windows with unequal total frame counts. + const samples = Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable'})); + const available = [ + {run: 1, status: 'available', totalFrames: 10, jankyFrames: 5, originalPercent: 50, sampleWindowMs: 100}, + {run: 2, status: 'available', totalFrames: 90, jankyFrames: 0, originalPercent: 0, sampleWindowMs: 200}, + ]; + // When total-frame weighted jank and original per-window values are calculated. + const coverage = summarizeFrames([...available, ...samples.slice(2)]); + // Then the weighted percentage is five rather than the sample-mean twenty-five. + expect(coverage.summary?.weightedJankyPercent).toBe(5); + expect(coverage.summary?.originalPercent.average).toBe(25); + expect(coverage.unavailableSamples).toBe(18); + expect(coverage.summary?.sampleWindowMs.average).toBe(150); + }); + + it('uses known interpolated statistics while retaining IQR outliers', () => { + // Given a small known distribution containing an outlier. + // When all values are summarized without trimming. + const stats = distribution([1, 2, 3, 100]); + // Then the arithmetic and retained outlier count are explicit. + expect(stats.runs).toBe(4); + expect(stats.average).toBe(26.5); + expect(stats.median).toBe(2.5); + expect(stats.p10).toBe(1.3); + expect(stats.iqrOutliers).toBe(1); + }); + + it('rejects malformed plan identities before execution or reporting', () => { + // Given a valid immutable plan. + // When a platform or artifact identity is malformed. + // Then no permissive fallback can interpret it as another candidate. + expect(parsePlan(plan)).toEqual(plan); + expect(() => parsePlan({...plan, toolingSha: 'short'})).toThrow(); + expect(() => parsePlan({...plan, platforms: ['android', 'android']})).toThrow(); + expect(() => summarizePlatform(plan, 'android', {...result(), artifactSha256: {release: 'a'.repeat(40), optimized: hash, instrumented: hash}})).toThrow(); + }); +}); diff --git a/tests/tooling/pgoEvaluationRun.test.ts b/tests/tooling/pgoEvaluationRun.test.ts new file mode 100644 index 000000000000..53457b972411 --- /dev/null +++ b/tests/tooling/pgoEvaluationRun.test.ts @@ -0,0 +1,204 @@ +// cspell:ignore cooldown + +import {describe, expect, it} from 'bun:test'; + +import createAndroidPgoAdapter from '@scripts/pgo/android'; +import {benchmarkStagePlan, configureSafeEvaluationOptions, readCohort, readPublicFrames, readSamples, runGuardedStage} from '@scripts/pgo/evaluationRun'; +import {profileCounts, validateJourney} from '@scripts/pgo/proof'; + +import {mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +describe('PGO evaluation cohort acceptance', () => { + it('selects the shrunk artifact directory before constructing the Android adapter', () => { + // Given inherited settings that would select the ordinary bootstrap directory. + const names = ['PGO_ANDROID_RELEASE_PIPELINE', 'PGO_ANDROID_SHRINK', 'PGO_BENCHMARK_COOLDOWN_SECONDS', 'PATH']; + const previous = names.map((name) => [name, process.env[name]] as const); + process.env.PGO_ANDROID_SHRINK = 'false'; + try { + // When safe evaluation options are applied before adapter construction. + configureSafeEvaluationOptions(process.cwd()); + const adapter = createAndroidPgoAdapter('evaluation.app', 'physical'); + // Then the parent and child build both refer to the shrunk artifact path. + expect(adapter.artifactPaths.instrumented).toContain('/apk-shrunk/'); + expect(process.env.PGO_ANDROID_RELEASE_PIPELINE).toBe('false'); + } finally { + for (const [name, value] of previous) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + } + }); + it('publishes frame availability without private responses or invented zeros', () => { + // Given twenty unavailable frame windows containing private raw responses. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + const path = join(directory, 'frames.json'); + writeFileSync( + path, + JSON.stringify({ + schema: 1, + requestedRuns: 20, + samples: Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable', raw: 'private UI response', reason: 'private path'})), + }), + ); + try { + // When frame observations pass into the public result. + const frames = readPublicFrames(path); + // Then unavailable observations retain their run identity and no private fields or numeric replacements. + expect(frames).toHaveLength(20); + expect(frames.at(0)).toEqual({run: 1, status: 'unavailable'}); + expect(JSON.stringify(frames)).not.toContain('private'); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + it('rejects partial and foreign training before optimizing', () => { + // Given a complete training result and its expected source and toolchain. + const result = { + platform: 'android', + status: 'passed', + accountClass: 'heavy', + requestedRuns: 3, + completedRuns: 3, + collectProfiles: true, + navigationOnly: false, + sourceIdentity: 'source', + toolchain: 'tools', + }; + // When a completed run is missing or the source differs. + // Then neither result can be accepted for optimization. + expect(() => validateJourney({...result, completedRuns: 2}, 'android', 'source', 'tools')).toThrow(); + expect(() => validateJourney(result, 'android', 'foreign', 'tools')).toThrow(); + expect(() => validateJourney(result, 'android', 'source', 'tools')).not.toThrow(); + }); + + it('retains large LLVM counters and rejects a zero execution maximum', () => { + // Given LLVM summaries using Maximum function count rather than Total count. + const positive = 'Total functions: 2\nMaximum function count: 4294967297\n'; + // When counts exceed the unsigned 32-bit range. + const counts = profileCounts(positive); + // Then the exact count survives and a zero maximum remains invalid. + expect(counts.count).toBe('4294967297'); + expect(() => profileCounts('Total functions: 2\nMaximum function count: 0\n')).toThrow(); + }); + it('executes fresh paired workloads in both orders using supported CLI actions', () => { + // Given the evaluation stage plan used by the physical runner. + const stages = benchmarkStagePlan(); + // When its actions and orders are inspected without touching a device. + const actions = stages.map(({order, scenario}) => `${scenario.action}:${order}`); + // Then all three workload types have distinct cohorts in each order. + expect(actions).toEqual([ + 'benchmark-heavy-journey:release-first', + 'benchmark-journey:release-first', + 'benchmark:release-first', + 'benchmark-heavy-journey:optimized-first', + 'benchmark-journey:optimized-first', + 'benchmark:optimized-first', + ]); + }); + + it('stops before launching a workload when immutable inputs changed', () => { + // Given a failed source or profile guard and a workload that has not started. + let launched = false; + // When the guarded execution boundary is entered. + const execute = () => + runGuardedStage( + () => { + throw new Error('changed'); + }, + () => { + launched = true; + }, + ); + // Then no device operation is launched. + expect(execute).toThrow('changed'); + expect(launched).toBe(false); + }); + + it('rejects completed work if its inputs changed during execution', () => { + // Given a guard that succeeds before work and fails afterward. + let checks = 0; + // When the workload completes against changing inputs. + const execute = () => + runGuardedStage( + () => { + checks += 1; + if (checks === 2) { + throw new Error('changed'); + } + }, + () => 'completed', + ); + // Then the stage cannot be accepted as a successful cohort. + expect(execute).toThrow('changed'); + expect(checks).toBe(2); + }); + + it('rejects a cohort from the wrong physical device before reading samples', () => { + // Given metadata from a different device despite a successful command exit. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + writeFileSync(join(directory, 'run.json'), JSON.stringify({deviceID: 'other', appID: 'app', order: 'release-first', runs: 20, warmUpRunsPerBinary: 1, cooldownSecondsPerBinary: 5})); + try { + // When the expected cohort is bound to the approved device. + // Then metadata mismatch prevents accepting unrelated CSV files. + expect(() => readCohort(directory, 'startup', 'release-first', ['ManualAppStartup'], 'approved', 'app')).toThrow('context'); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + + it('accepts actual startup batch filenames and preserves every native timing', () => { + // Given a completed startup batch using the startup producer's optimized.csv name. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + writeFileSync( + join(directory, 'run.json'), + JSON.stringify({platform: 'android', deviceID: 'approved', appID: 'app', order: 'optimized-first', runs: 20, warmUpRunsPerBinary: 1, cooldownSecondsPerBinary: 5}), + ); + const csv = `run,span,duration_ms\n${Array.from({length: 20}, (_, index) => `${index + 1},ManualAppStartup,${100 + index}`).join('\n')}\n`; + writeFileSync(join(directory, 'release.csv'), csv); + writeFileSync(join(directory, 'optimized.csv'), csv); + try { + // When the startup cohort reader binds the physical device and run order. + const cohort = readCohort(directory, 'startup', 'optimized-first', ['ManualAppStartup'], 'approved', 'app'); + // Then both complete observations and their actual file hashes are retained. + expect(cohort.release).toHaveLength(20); + expect(cohort.optimized).toHaveLength(20); + expect(cohort.optimizedCsvSha256).toBe(cohort.releaseCsvSha256); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + it('retains zero duration samples and requires twenty distinct runs', () => { + // Given a complete native span cohort including a zero duration observation. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + const path = join(directory, 'samples.csv'); + writeFileSync(path, `run,span,duration_ms\n${Array.from({length: 20}, (_, index) => `${index + 1},startup,${index}`).join('\n')}\n`); + try { + // When the exact expected span is read. + const samples = readSamples(path, ['startup']); + // Then all observations are retained without inventing an exclusion threshold. + expect(samples).toHaveLength(20); + expect(samples.at(0)?.durationMs).toBe(0); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + + it('rejects duplicate runs even when the total row count is twenty', () => { + // Given a CSV whose final row repeats run nineteen instead of run twenty. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + const path = join(directory, 'samples.csv'); + writeFileSync(path, `run,span,duration_ms\n${Array.from({length: 20}, (_, index) => `${Math.min(index + 1, 19)},startup,1`).join('\n')}\n`); + try { + // When cohort validation checks each run and span pair. + // Then the incomplete cohort cannot become a successful public evaluation. + expect(() => readSamples(path, ['startup'])).toThrow('twenty complete samples'); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); +}); diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts new file mode 100644 index 000000000000..0184be56e9c9 --- /dev/null +++ b/tests/tooling/pgoIOSRock.test.ts @@ -0,0 +1,32 @@ +// cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata + +import {describe, expect, it} from 'bun:test'; + +import {iosRockBuildArguments} from '@scripts/pgo/ios'; + +import {parseArgs} from '@rock-js/tools'; + +describe('PGO iOS Rock build arguments', () => { + it('selects a local device Release build and preserves exact compiler settings', () => { + // Given compiler settings containing spaces and inherited Xcode variables. + const settings = ['build', 'ARCHS=arm64', 'CODE_SIGN_IDENTITY=Apple Development', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-use=/tmp/profile directory/journey.profdata']; + // When the adapter creates a Rock invocation. + const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); + // Then the local Release destination and existing pod preparation are explicit. + expect(args.slice(0, 12)).toEqual([ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + '/tmp/derived data/optimized', + '--no-install-pods', + ]); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); +}); diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts index 0c3c6044c56b..6adee4af2183 100644 --- a/tests/tooling/pgoJourney.test.ts +++ b/tests/tooling/pgoJourney.test.ts @@ -1,6 +1,6 @@ import {describe, expect, it} from 'bun:test'; -import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; +import {frameWindowResult, journeyBenchmarkPaths, parseAndroidFrameHealth, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; import { @@ -30,7 +30,46 @@ const fixture = { }; describe('PGO journey safeguards', () => { - it('logs the held-out navigation spans in both PGO builds without replacing startup samples', () => { + it('retains unavailable ancillary telemetry without inventing a zero frame sample', () => { + // Given native telemetry is unavailable while workload timing remains independently recorded. + const raw = {sampling: {fps: {resetsAfterRead: true}}, metrics: {fps: {available: false, reason: 'No graphics counters'}}}; + // When the response is classified. + const result = parseAndroidFrameHealth(raw, true); + // Then the raw reason survives and no numeric frame sample is fabricated. + expect(result).toEqual({status: 'unavailable', reason: 'Android frame sampler unavailable: No graphics counters', raw}); + expect('metrics' in result).toBe(false); + }); + + it('requires declared reset support and valid positive post-scroll counters', () => { + // Given a sampler response with valid counters and a declared reset capability. + const raw = {sampling: {fps: {resetsAfterRead: true}}, metrics: {fps: {available: true, totalFrameCount: 10, droppedFrameCount: 2, droppedFramePercent: 20, sampleWindowMs: 500}}}; + // When valid, malformed, unsupported and empty windows are classified. + expect(parseAndroidFrameHealth(raw, true).status).toBe('available'); + expect(parseAndroidFrameHealth({...raw, sampling: {fps: {resetsAfterRead: false}}}, true).status).toBe('unavailable'); + expect(parseAndroidFrameHealth({metrics: null}, true).status).toBe('unavailable'); + for (const invalid of [{totalFrameCount: -1}, {droppedFrameCount: 11}, {sampleWindowMs: Number.NaN}, {totalFrameCount: 0, droppedFrameCount: 0, sampleWindowMs: 0}]) { + // Then invalid telemetry is unavailable rather than a zero-valued observation. + expect(parseAndroidFrameHealth({...raw, metrics: {fps: {...raw.metrics.fps, ...invalid}}}, true).status).toBe('unavailable'); + } + expect(parseAndroidFrameHealth({...raw, metrics: {fps: {...raw.metrics.fps, totalFrameCount: 0, droppedFrameCount: 0, sampleWindowMs: 0}}}, false).status).toBe('available'); + }); + + it('preserves both responses when the pre-scroll boundary is unavailable', () => { + // Given an unavailable boundary and otherwise valid post-scroll counters. + const before = parseAndroidFrameHealth({unavailable: true}, false); + const after = parseAndroidFrameHealth( + {sampling: {fps: {resetsAfterRead: true}}, metrics: {fps: {available: true, totalFrameCount: 10, droppedFrameCount: 1, droppedFramePercent: 10, sampleWindowMs: 500}}}, + true, + ); + // When the window is assembled without another destructive sampler read. + const result = frameWindowResult(before, after); + // Then the window remains unavailable with its complete evidence. + expect(result.status).toBe('unavailable'); + expect(result.raw).toEqual({before: before.raw, after: after.raw}); + expect('metrics' in result).toBe(false); + }); + + it('logs the separate navigation spans in both PGO builds without replacing startup samples', () => { // Given release and optimized artifacts share one build-time benchmark allowlist. const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); // When / Then the startup span and every short-journey span remain available. @@ -270,6 +309,43 @@ describe('PGO journey safeguards', () => { ]); }); + it.each([true, false])('checks a slow Back result before enforcing the tab deadline (tab appears: %j)', async (tabAppears) => { + // Given a successful Back command that completes after the navigation deadline. + const screen = {type: 'Application', rect: {x: 0, y: 0, width: 1080, height: 2280}}; + const back = {type: 'android.widget.Button', label: 'Back', rect: {x: 21, y: 144, width: 67, height: 105}}; + const inbox = {type: 'android.view.View', label: 'Inbox', rect: {x: 216, y: 1968, width: 216, height: 186}}; + let elapsed = 0; + let reads = 0; + const commands: string[][] = []; + const device: Parameters[0] = { + platform: 'android', + snapshot: () => parseJourneySnapshot({appBundleId: 'test.app', nodes: [screen, reads++ > 0 && tabAppears ? inbox : back]}, 'test.app'), + command: (...args: string[]) => { + commands.push(args); + elapsed = 31_000; + }, + wait: () => {}, + press: () => {}, + hasLabel: () => false, + pressLabel: () => {}, + waitLabel: () => {}, + fill: () => {}, + back: () => {}, + open: () => {}, + }; + // When the loop checks the completed Back's result. + const navigation = showTab(device, 'Inbox', () => elapsed); + // Then a mounted tab succeeds, while an unchanged screen fails without another Back. + if (tabAppears) { + await expect(navigation).resolves.toBeUndefined(); + expect(commands).toHaveLength(2); + } else { + await expect(navigation).rejects.toThrow('Cannot return to the Inbox tab.'); + expect(commands).toHaveLength(1); + } + expect(reads).toBe(2); + }); + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; const snapshot = (tabs: Array) => From 932b1a7d903e83ac51992224b3c0e12248ddc503 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 13:45:41 +0100 Subject: [PATCH 45/60] ci(pgo): enable report-only native evaluation --- .github/workflows/deploy.yml | 18 ++ .github/workflows/pgoEvaluation.yml | 219 ++++++++++++++++++++ package.json | 2 +- tests/tooling/pgoEvaluationWorkflow.test.ts | 108 ++++++++++ 4 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pgoEvaluation.yml create mode 100644 tests/tooling/pgoEvaluationWorkflow.test.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a1e6219d0b9e..5d75747d4e5a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -180,6 +180,24 @@ jobs: isCherryPick, ); + - name: Record native evaluation candidate + id: pgoCandidate + continue-on-error: true + env: + PGO_CANDIDATE_ENVIRONMENT: ${{ env.DEPLOY_ENV }} + PGO_NATIVE_ELIGIBLE: ${{ env.DEPLOY_ENV == 'staging' || fromJSON(steps.isCherryPick.outputs.IS_CHERRY_PICK) }} + run: ./node_modules/.bin/bun scripts/pgo/pgoCandidate.ts capture --output pgo-candidate/candidate.json --environment "$PGO_CANDIDATE_ENVIRONMENT" --native-eligible "$PGO_NATIVE_ELIGIBLE" + + - name: Upload native evaluation candidate metadata + if: steps.pgoCandidate.outcome == 'success' + continue-on-error: true + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: pgo-native-candidate-${{ github.run_attempt }} + path: pgo-candidate/candidate.json + if-no-files-found: error + retention-days: 30 + - name: Get Android native version id: getAndroidVersion run: echo "VERSION_CODE=$(grep -oP 'android:versionCode="\K[0-9]+' Mobile-Expensify/Android/AndroidManifest.xml)" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml new file mode 100644 index 000000000000..f04f78a9fa8a --- /dev/null +++ b/.github/workflows/pgoEvaluation.yml @@ -0,0 +1,219 @@ +name: Evaluate native PGO candidate + +on: + workflow_run: + workflows: [Deploy code to staging or production] + types: [completed] + branches: [staging, production] + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + producer_run_id: + description: 'Immutable deploy run to evaluate' + required: true + type: string + platform: + description: 'Platforms to evaluate' + type: choice + default: both + options: [both, android, ios] + +# Evaluation never publishes or selects a deployable native artifact. +permissions: + contents: read + +jobs: + prepare: + if: >- + github.repository == 'Expensify/App' && + (github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == 'Expensify/App' && + github.event.pull_request.head.ref == '@chrispader/perf/pgo-ci-evaluation')) + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + pull-requests: read + outputs: + eligible: ${{ steps.candidate.outputs.eligible }} + app-sha: ${{ steps.candidate.outputs.app-sha }} + mobile-sha: ${{ steps.candidate.outputs.mobile-sha }} + platforms: ${{ steps.candidate.outputs.platforms }} + tooling-sha: ${{ steps.tooling.outputs.sha }} + platform-matrix: ${{ steps.candidate.outputs.platform-matrix }} + steps: + - name: Checkout reviewed evaluation tooling + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Record reviewed tooling revision + id: tooling + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Setup repository tooling + uses: ./.github/actions/composite/setupNode + - name: Resolve and validate immutable candidate + id: candidate + env: + GITHUB_TOKEN: ${{ github.token }} + PGO_PRODUCER_RUN_ID: ${{ inputs.producer_run_id }} + PGO_PLATFORM: ${{ inputs.platform || 'both' }} + run: | + args=() + if [[ -n "$PGO_PRODUCER_RUN_ID" ]]; then + args+=(--source-run-id "$PGO_PRODUCER_RUN_ID") + fi + ./node_modules/.bin/bun scripts/pgo/pgoCandidate.ts prepare --output pgo-plan/plan.json --platform "$PGO_PLATFORM" "${args[@]}" + - name: Preserve verified evaluation plan + if: steps.candidate.outputs.eligible == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: pgo-evaluation-plan + path: pgo-plan/plan.json + if-no-files-found: error + retention-days: 30 + + evaluate: + needs: prepare + if: needs.prepare.outputs.eligible == 'true' + strategy: + fail-fast: false + matrix: + platform: ${{ fromJSON(needs.prepare.outputs.platform-matrix) }} + # A single platform process owns build, training and both benchmark orders. + runs-on: ${{ matrix.platform == 'android' && 'pgo-android' || 'pgo-ios' }} + concurrency: + group: pgo-evaluation-physical-${{ matrix.platform }} + cancel-in-progress: false + queue: max + timeout-minutes: 360 + permissions: + contents: read + actions: read + steps: + - name: Checkout pinned native candidate + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + ref: ${{ needs.prepare.outputs.app-sha }} + submodules: recursive + persist-credentials: false + - name: Checkout reviewed evaluation tooling + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + ref: ${{ needs.prepare.outputs.tooling-sha }} + path: evaluation-tooling + persist-credentials: false + - name: Setup pinned candidate dependencies + uses: ./evaluation-tooling/.github/actions/composite/setupNode + with: + IS_HYBRID_BUILD: 'true' + - name: Expose pinned dependencies to reviewed execution overlay + run: ln -s "$GITHUB_WORKSPACE/node_modules" evaluation-tooling/node_modules + - name: Download verified plan + if: contains(needs.prepare.outputs.platforms, matrix.platform) + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: pgo-evaluation-plan + path: pgo-plan + - name: Verify candidate and native gitlink before device access + if: contains(needs.prepare.outputs.platforms, matrix.platform) + env: + APP_SHA: ${{ needs.prepare.outputs.app-sha }} + MOBILE_SHA: ${{ needs.prepare.outputs.mobile-sha }} + run: | + test "$(git rev-parse HEAD)" = "$APP_SHA" + test "$(git -C Mobile-Expensify rev-parse HEAD)" = "$MOBILE_SHA" + test "$(git ls-tree HEAD Mobile-Expensify | awk '{print $3}')" = "$MOBILE_SHA" + - name: Evaluate pinned candidate on its physical device + if: contains(needs.prepare.outputs.platforms, matrix.platform) + env: + PGO_CANDIDATE_DIRECTORY: ${{ github.workspace }} + PGO_TOOLING_DIRECTORY: ${{ github.workspace }}/evaluation-tooling + PGO_PLATFORM: ${{ matrix.platform }} + PGO_ANDROID_DEVICE_ID: ${{ secrets.PGO_ANDROID_DEVICE_ID }} + PGO_ANDROID_APP_ID: ${{ vars.PGO_ANDROID_APP_ID }} + PGO_ANDROID_FIXTURE_PATH: ${{ secrets.PGO_ANDROID_FIXTURE_PATH }} + PGO_ANDROID_ENVFILE: ${{ secrets.PGO_ANDROID_ENVFILE }} + PGO_IOS_DEVICE_ID: ${{ secrets.PGO_IOS_DEVICE_ID }} + PGO_IOS_APP_ID: ${{ vars.PGO_IOS_APP_ID }} + PGO_IOS_FIXTURE_PATH: ${{ secrets.PGO_IOS_FIXTURE_PATH }} + PGO_IOS_ENVFILE: ${{ secrets.PGO_IOS_ENVFILE }} + IOS_CODE_SIGNING_ALLOWED: ${{ vars.IOS_CODE_SIGNING_ALLOWED }} + IOS_DEVELOPMENT_TEAM: ${{ vars.IOS_DEVELOPMENT_TEAM }} + run: ./node_modules/.bin/bun evaluation-tooling/scripts/pgo/pgoEvaluation.ts execute "$PGO_PLATFORM" --plan pgo-plan/plan.json --output pgo-public + - name: Preserve public numeric result and provenance + if: always() && contains(needs.prepare.outputs.platforms, matrix.platform) + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: pgo-evaluation-${{ matrix.platform }} + path: pgo-public/evaluation-${{ matrix.platform }}.json + if-no-files-found: warn + retention-days: 30 + + report: + needs: [prepare, evaluate] + if: always() && needs.prepare.outputs.eligible == 'true' + runs-on: ubuntu-latest + concurrency: + group: pgo-evaluation-report-${{ needs.prepare.outputs.app-sha }} + cancel-in-progress: false + queue: max + permissions: + contents: read + actions: read + issues: write + steps: + - name: Checkout reviewed reporting tooling + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + ref: ${{ needs.prepare.outputs.tooling-sha }} + persist-credentials: false + - name: Setup repository tooling + uses: ./.github/actions/composite/setupNode + - name: Download public evaluation artifacts + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: pgo-evaluation-* + path: pgo-results + - name: Render report for this immutable candidate + env: + GITHUB_TOKEN: ${{ github.token }} + run: ./node_modules/.bin/bun scripts/pgo/pgoEvaluation.ts report --plan pgo-results/pgo-evaluation-plan/plan.json --results pgo-results --output pgo-report + - name: Publish report summary + run: cat pgo-report/summary.md >> "$GITHUB_STEP_SUMMARY" + - name: Preserve public report + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: pgo-evaluation-report + path: | + pgo-report/summary.json + pgo-report/summary.md + if-no-files-found: error + retention-days: 30 + - name: Attach report to its matching deploy checklist + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd + with: + script: | + const fs = require('node:fs'); + const comment = JSON.parse(fs.readFileSync('pgo-report/comment.json', 'utf8')); + if (!Number.isSafeInteger(comment.targetIssueNumber) || !comment.marker || !comment.body) { + core.notice('No matching report target was resolved.'); + return; + } + const {data: issue} = await github.rest.issues.get({...context.repo, issue_number: comment.targetIssueNumber}); + { + const labels = issue.labels.map((label) => typeof label === 'string' ? label : label.name); + const tag = issue.body?.match(/\*\*Release Version:\*\*\s*`([^`]+)`/)?.[1]; + if (!labels.includes('StagingDeployCash') || !comment.version || tag !== `${comment.version}-staging`) { + core.notice('Historical checklist changed; report remains available as an artifact.'); + return; + } + } + const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: comment.targetIssueNumber, per_page: 100}); + const existing = comments.find((item) => item.user?.type === 'Bot' && item.body?.includes(comment.marker)); + if (existing) { + await github.rest.issues.updateComment({...context.repo, comment_id: existing.id, body: comment.body}); + } else { + await github.rest.issues.createComment({...context.repo, issue_number: comment.targetIssueNumber, body: comment.body}); + } diff --git a/package.json b/package.json index 0423ee4c7614..3bcb94be69af 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "server:vcr:dev": "npm run dev -w @expensify/victory-chart-renderer --", "server:vcr:test": "npm run test -w @expensify/victory-chart-renderer --", "test:bun": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./server ./tests/tooling", - "test:pgo-ci": "TZ=utc bun test --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./tests/tooling/pgoCandidate.test.ts ./tests/tooling/pgoEvaluationReport.test.ts ./tests/tooling/pgoEvaluationRun.test.ts ./tests/tooling/pgoAndroidRelease.test.ts ./tests/tooling/pgoIOSRock.test.ts ./tests/tooling/pgoJourney.test.ts", + "test:pgo-ci": "TZ=utc bun test --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./tests/tooling/pgoCandidate.test.ts ./tests/tooling/pgoEvaluationReport.test.ts ./tests/tooling/pgoEvaluationRun.test.ts ./tests/tooling/pgoAndroidRelease.test.ts ./tests/tooling/pgoIOSRock.test.ts ./tests/tooling/pgoJourney.test.ts ./tests/tooling/pgoEvaluationWorkflow.test.ts", "eval:proposal-police": "bun test --path-ignore-patterns='**/__never__/**' ./evals/proposalPolice/", "server:vcr:build:linux": "npm run build:linux -w @expensify/victory-chart-renderer --", "server:vcr:build:linux-arm": "npm run build:linux-arm -w @expensify/victory-chart-renderer --", diff --git a/tests/tooling/pgoEvaluationWorkflow.test.ts b/tests/tooling/pgoEvaluationWorkflow.test.ts new file mode 100644 index 000000000000..55d07c91b6ce --- /dev/null +++ b/tests/tooling/pgoEvaluationWorkflow.test.ts @@ -0,0 +1,108 @@ +import {describe, expect, it} from 'bun:test'; + +import {isRecord} from '@scripts/pgo/evaluationSchema'; + +import {load} from 'js-yaml'; +import {readFileSync} from 'node:fs'; +import {runInNewContext} from 'node:vm'; + +describe('PGO evaluation workflow contract', () => { + it('keeps producer metadata failures outside canonical deployment eligibility', () => { + // Given the ordinary deploy workflow and its evaluation metadata additions. + const job = workflowJob('.github/workflows/deploy.yml', 'prep'); + // When both metadata steps are inspected. + const capture = workflowStep(job, 'Record native evaluation candidate'); + const upload = workflowStep(job, 'Upload native evaluation candidate metadata'); + // Then either failure is non-blocking and reruns retain separate attempt artifacts. + expect(capture?.['continue-on-error']).toBe(true); + expect(upload?.['continue-on-error']).toBe(true); + expect(isRecord(upload.with) && upload.with.name).toBe(expression('pgo-native-candidate-', 'github.run_attempt')); + }); + + it('schedules eligible platforms only and masks private device configuration', () => { + // Given the standalone evaluation workflow. + const job = workflowJob('.github/workflows/pgoEvaluation.yml', 'evaluate'); + // When runner selection, permissions and its execution environment are read. + const environment = workflowStep(job, 'Evaluate pinned candidate on its physical device').env; + expect(isRecord(environment)).toBe(true); + if (!isRecord(environment) || !isRecord(job.strategy) || !isRecord(job.strategy.matrix) || !isRecord(job.permissions) || !isRecord(job.concurrency)) { + throw new Error('Missing workflow job contract.'); + } + // Then an ineligible platform never requests a device runner and private paths are masked. + expect(job.strategy.matrix.platform).toContain('fromJSON(needs.prepare.outputs.platform-matrix)'); + expect(job.permissions.issues).toBeUndefined(); + expect(environment.PGO_ANDROID_DEVICE_ID).toBe(expression('', 'secrets.PGO_ANDROID_DEVICE_ID')); + expect(environment.PGO_IOS_FIXTURE_PATH).toBe(expression('', 'secrets.PGO_IOS_FIXTURE_PATH')); + expect(environment.PGO_IOS_ENVFILE).toBe(expression('', 'secrets.PGO_IOS_ENVFILE')); + expect(job.concurrency.queue).toBe('max'); + const report = workflowJob('.github/workflows/pgoEvaluation.yml', 'report'); + if (!isRecord(report.concurrency)) { + throw new Error('Missing report concurrency.'); + } + expect(report.concurrency.group).toContain('needs.prepare.outputs.app-sha'); + expect(report.concurrency.queue).toBe('max'); + }); + + it.each([ + ['9.4.94-0-staging', 1], + ['9.4.94-01-staging', 0], + ])('posts only to the exact historical checklist tag %s', async (tag, expectedPosts) => { + // Given the actual YAML posting script and a closed historical checklist. + const step = workflowStep(workflowJob('.github/workflows/pgoEvaluation.yml', 'report'), 'Attach report to its matching deploy checklist'); + const script = isRecord(step.with) ? step.with.script : undefined; + expect(script).toBeDefined(); + let posts = 0; + const issue = {state: 'closed', labels: [{name: 'StagingDeployCash'}], body: `**Release Version:** \`${tag}\``}; + const github = { + rest: { + issues: { + get: async () => ({data: issue}), + listComments: async () => [], + createComment: async () => { + posts += 1; + }, + updateComment: async () => { + posts += 1; + }, + }, + }, + paginate: async () => [], + }; + const requireFixture = () => ({readFileSync: () => JSON.stringify({targetIssueNumber: 42, marker: '', body: 'Public numeric report', version: '9.4.94-0'})}); + // When the trusted inline script rechecks the live checklist before posting. + // This executes only the repository's static workflow script with mocked GitHub and filesystem inputs. + if (typeof script !== 'string') { + throw new Error('Missing posting script.'); + } + const completion: unknown = runInNewContext(`(async () => {${script}})()`, { + require: requireFixture, + core: {notice: () => undefined}, + github, + context: {repo: {owner: 'Expensify', repo: 'App'}}, + }); + await completion; + // Then a closed matching issue works while a similar version cannot receive the report. + expect(posts).toBe(expectedPosts); + }); +}); + +function workflowJob(path: string, name: string): Record { + const input: unknown = load(readFileSync(path, 'utf8')); + if (!isRecord(input) || !isRecord(input.jobs) || !isRecord(input.jobs[name])) { + throw new Error('Missing workflow job.'); + } + return input.jobs[name]; +} +function workflowStep(job: Record, name: string): Record { + if (!Array.isArray(job.steps)) { + throw new Error('Missing workflow steps.'); + } + const step: unknown = job.steps.find((input: unknown) => isRecord(input) && input.name === name); + if (!isRecord(step)) { + throw new Error('Missing workflow step.'); + } + return step; +} +function expression(prefix: string, content: string): string { + return `${prefix}${String.fromCharCode(36)}{{ ${content} }}`; +} From 69a8d678893868fbe5c51eae78ec01b88bdbe300 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 15:27:18 +0100 Subject: [PATCH 46/60] fix(pgo): align iOS IR profiling and Rock builds --- Mobile-Expensify | 2 +- ....86.0+044+add-local-pgo-build-switch.patch | 6 +- scripts/pgo/ios.ts | 57 +++++++++++-------- tests/tooling/pgoIOSRock.test.ts | 17 ++++++ 4 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 tests/tooling/pgoIOSRock.test.ts diff --git a/Mobile-Expensify b/Mobile-Expensify index 8b0d0a2977b0..7f8d598e6da7 160000 --- a/Mobile-Expensify +++ b/Mobile-Expensify @@ -1 +1 @@ -Subproject commit 8b0d0a2977b0b9d18eefacb9b6449dd5cfd57eca +Subproject commit 7f8d598e6da7931f3c66b8bcdb394f429d76a259 diff --git a/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch index dbff79c05e16..d7e9802f6fd1 100644 --- a/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch +++ b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch @@ -149,14 +149,14 @@ diff --git a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xco +pgo_compile_flags="" +pgo_linker_flags="" +if [[ "${EXPENSIFY_PGO_MODE:-off}" == "generate" ]]; then -+ pgo_compile_flags="-fprofile-instr-generate" -+ pgo_linker_flags="-fprofile-instr-generate;LINKER:-alias,___llvm_profile_set_filename,_expensify_llvm_profile_set_filename;LINKER:-alias,___llvm_profile_write_file,_expensify_llvm_profile_write_file;LINKER:-alias,___llvm_profile_reset_counters,_expensify_llvm_profile_reset_counters" ++ pgo_compile_flags="-fprofile-generate" ++ pgo_linker_flags="-fprofile-generate;LINKER:-alias,___llvm_profile_set_filename,_expensify_llvm_profile_set_filename;LINKER:-alias,___llvm_profile_write_file,_expensify_llvm_profile_write_file;LINKER:-alias,___llvm_profile_reset_counters,_expensify_llvm_profile_reset_counters" +elif [[ "${EXPENSIFY_PGO_MODE:-off}" == "use" ]]; then + if [[ -z "${EXPENSIFY_PGO_PROFILE:-}" ]]; then + echo "EXPENSIFY_PGO_PROFILE is required when EXPENSIFY_PGO_MODE=use" >&2 + exit 1 + fi -+ pgo_compile_flags="-fprofile-instr-use=${EXPENSIFY_PGO_PROFILE}" ++ pgo_compile_flags="-fprofile-use=${EXPENSIFY_PGO_PROFILE}" +fi + boost_context_flag="" diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index bc2ba74c7a1c..77ea09670c40 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled +// cspell:ignore Xfrontend hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled /** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ @@ -14,7 +14,7 @@ import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllo import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { - const profileFormat = 'ios-clang-frontend-swift-ir-v1'; + const profileFormat = 'ios-clang-swift-ir-v3'; const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); const rawProfileDirectory = join(profileDirectory, 'raw'); const appDirectory = join(profileDirectory, 'app'); @@ -181,19 +181,19 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str return [ ...baseSettings, 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', - 'OTHER_CFLAGS=$(inherited) -fprofile-instr-generate', - 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-instr-generate', - 'OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-generate', - 'OTHER_LDFLAGS=$(inherited) -fprofile-instr-generate', + 'OTHER_CFLAGS=$(inherited) -fprofile-generate', + 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-generate', + 'OTHER_LDFLAGS=$(inherited) -fprofile-generate', ]; } if (kind === 'optimized') { - const clangProfileFlags = `-fprofile-instr-use=${profilePath} -Wno-error=profile-instr-unprofiled`; + const clangProfileFlags = `-fprofile-use=${profilePath} -Wno-error=profile-instr-unprofiled`; return [ ...baseSettings, `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, - `OTHER_SWIFT_FLAGS=$(inherited) -ir-profile-use=${profilePath}`, + `OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-use=${profilePath}`, ]; } return baseSettings; @@ -208,7 +208,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str const profileFormatPath = `${profilePath}.format`; const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; if (recordedFormat !== profileFormat) { - fail('The merged iOS profile predates the current Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); + fail('The merged iOS profile does not match the current Clang and Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); } } @@ -230,28 +230,24 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), ] : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; - const xcodeArguments = [ - 'xcodebuild', - '-workspace', - 'Expensify.xcworkspace', - '-scheme', - 'Expensify', - '-configuration', - 'Release', - '-destination', - 'generic/platform=iOS', - '-derivedDataPath', - derivedDataDirectory, - ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), - '-quiet', + const rockArguments = iosRockBuildArguments(derivedDataDirectory, [ 'build', 'ARCHS=arm64', 'ONLY_ACTIVE_ARCH=YES', 'COMPILER_INDEX_STORE_ENABLE=NO', + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), ...signingSettings, ...pgoBuildSettings(kind, profilePath), - ]; - withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), ...xcodeArguments], iosDirectory)); + ]); + withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], join(rootDirectory, 'Mobile-Expensify'))); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -472,4 +468,15 @@ function replaceOnce(content: string, previous: string, next: string): string { return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; } +/** Preserve each Xcode setting as one argument through Rock's string argument parser. */ +function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { + return [ + 'build:ios', '--local', '--verbose', '--configuration', 'Release', + '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', + '--build-folder', derivedDataDirectory, '--no-install-pods', + '--extra-params', settings.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + +export {iosRockBuildArguments}; export default createIOSPgoAdapter; diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts new file mode 100644 index 000000000000..3eeaaec3a56a --- /dev/null +++ b/tests/tooling/pgoIOSRock.test.ts @@ -0,0 +1,17 @@ +// cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata + +import {describe, expect, it} from 'bun:test'; +import {iosRockBuildArguments} from '@scripts/pgo/ios'; +import {parseArgs} from '@rock-js/tools'; + +describe('PGO iOS Rock build arguments', () => { + it('selects a local device Release build and preserves exact compiler settings', () => { + // Given compiler settings containing spaces and inherited Xcode variables. + const settings = ['build', 'ARCHS=arm64', 'CODE_SIGN_IDENTITY=Apple Development', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-use=/tmp/profile directory/journey.profdata']; + // When the adapter creates a Rock invocation. + const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); + // Then the local Release destination and existing pod preparation are explicit. + expect(args.slice(0, 12)).toEqual(['build:ios', '--local', '--verbose', '--configuration', 'Release', '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', '--build-folder', '/tmp/derived data/optimized', '--no-install-pods']); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); +}); From c4f2c5e072eec968ef389fc4d42430b5a866dc64 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 15:27:59 +0100 Subject: [PATCH 47/60] fix(pgo): require executed Swift profiles in evaluation --- scripts/pgo/localProof.ts | 5 ++++- scripts/pgo/proof.ts | 10 +++++++-- tests/tooling/pgoEvaluationRun.test.ts | 29 ++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/pgo/localProof.ts b/scripts/pgo/localProof.ts index d751222f3b01..3b09f2112735 100644 --- a/scripts/pgo/localProof.ts +++ b/scripts/pgo/localProof.ts @@ -11,7 +11,7 @@ import type {PlatformName} from '../lib/nativeAppBenchmark'; import {capture, fail, findFiles, run, valueAt} from '../lib/scriptUtils'; import createAndroidPgoAdapter from './android'; import createIOSPgoAdapter from './ios'; -import {fileHash, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney} from './proof'; +import {fileHash, hasExecutedSwiftFunctions, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney} from './proof'; /** Validate the immutable collected profile before optimization. */ function validateTraining(platform: PlatformName, batch: string) { @@ -26,6 +26,9 @@ function validateTraining(platform: PlatformName, batch: string) { } const profile = join(batch, 'journey.profdata'); const counts = profileCounts(capture(adapter.llvmTool('llvm-profdata'), ['show', profile])); + if (platform === 'ios' && !hasExecutedSwiftFunctions(capture(adapter.llvmTool('llvm-profdata'), ['show', '--all-functions', '--counts', profile]))) { + fail('The merged iOS profile has no executed Swift functions. Rebuild with Swift IR instrumentation and collect fresh profiles.'); + } const runs = [1, 2, 3].map((runNumber) => { const paths = findFiles(join(batch, `run-${runNumber}`, 'raw'), '.profraw').sort(); if (paths.length === 0) { diff --git a/scripts/pgo/proof.ts b/scripts/pgo/proof.ts index 2bc5a8b7a59b..59678c53c269 100644 --- a/scripts/pgo/proof.ts +++ b/scripts/pgo/proof.ts @@ -68,13 +68,19 @@ function validateJourney(input: unknown, platform: PlatformName, identity: strin function profileCounts(summary: string): {functions: number; count: string} { const functions = Number(/Total functions:\s*([0-9]+)/.exec(summary)?.[1]); - const count = /(?:Total count|Maximum function count):\s*([0-9]+)/.exec(summary)?.[1]; + const count = /Total count:\s*([0-9]+)/.exec(summary)?.[1] ?? /Maximum function count:\s*([0-9]+)/.exec(summary)?.[1]; if (!Number.isSafeInteger(functions) || functions <= 0 || !count || BigInt(count) <= 0n) { fail('The LLVM profile has no executed functions.'); } return {functions, count}; } +/** Require Swift execution without borrowing counters from a neighboring Clang function. */ +function hasExecutedSwiftFunctions(report: string): boolean { + const functions = report.matchAll(/^ {2}(\S[^\n]*):\r?\n((?: {4}[^\n]*\r?\n)*)/gm); + return Array.from(functions).some((entry) => /(?:^|:)\$s/.test(entry[1] ?? '') && /(?:Function count:\s*|Block counts:\s*\[[^\]\n]*)[1-9][0-9]*/.test(entry[2] ?? '')); +} + function fileHash(path: string, allowEmpty = false): string { const stat = statSync(path); if (stat.isDirectory()) { @@ -96,4 +102,4 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -export {androidBuildEnvironment, fileHash, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney}; +export {androidBuildEnvironment, fileHash, hasExecutedSwiftFunctions, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney}; diff --git a/tests/tooling/pgoEvaluationRun.test.ts b/tests/tooling/pgoEvaluationRun.test.ts index 53457b972411..c2b2157a1f6f 100644 --- a/tests/tooling/pgoEvaluationRun.test.ts +++ b/tests/tooling/pgoEvaluationRun.test.ts @@ -1,10 +1,10 @@ -// cspell:ignore cooldown +// cspell:ignore cooldown probey import {describe, expect, it} from 'bun:test'; import createAndroidPgoAdapter from '@scripts/pgo/android'; import {benchmarkStagePlan, configureSafeEvaluationOptions, readCohort, readPublicFrames, readSamples, runGuardedStage} from '@scripts/pgo/evaluationRun'; -import {profileCounts, validateJourney} from '@scripts/pgo/proof'; +import {hasExecutedSwiftFunctions, profileCounts, validateJourney} from '@scripts/pgo/proof'; import {mkdtempSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; @@ -85,6 +85,31 @@ describe('PGO evaluation cohort acceptance', () => { expect(counts.count).toBe('4294967297'); expect(() => profileCounts('Total functions: 2\nMaximum function count: 0\n')).toThrow(); }); + it('prefers an explicit aggregate LLVM count over a positive maximum', () => { + // Given frontend and IR summaries with different supported execution-count fields. + const frontend = 'Total functions: 2\nMaximum function count: 42\nTotal count: 100'; + const ir = 'Total functions: 2\nMaximum function count: 42'; + // When both summaries are parsed, or an explicit aggregate count is zero. + expect(profileCounts(frontend).count).toBe('100'); + expect(profileCounts(ir).count).toBe('42'); + // Then an executed maximum cannot override a reported zero aggregate count. + expect(() => profileCounts(frontend.replace('Total count: 100', 'Total count: 0'))).toThrow('no executed functions'); + }); + + it('requires executed Swift counters without borrowing a neighboring function count', () => { + // Given a zero-count Swift function followed by an executed Clang function. + const report = 'Counters:\n $s5Probe5probeyS2iF:\n Hash: 0x0\n Counters: 1\n Function count: 0\n Block counts: []\n clangFunction:\n Function count: 42\n'; + // When the iOS execution guard reads frontend or IR function counter reports. + expect(hasExecutedSwiftFunctions(report)).toBe(false); + expect(hasExecutedSwiftFunctions(report.replace('Function count: 0', 'Function count: 3'))).toBe(true); + expect(hasExecutedSwiftFunctions(report.replace('$s5Probe5probeyS2iF', 'anotherClangFunction'))).toBe(false); + expect(hasExecutedSwiftFunctions('Total count: 42')).toBe(false); + // Then only Swift functions with their own positive execution counters pass. + expect(hasExecutedSwiftFunctions('Counters:\n $s5Probe5probeyS2iF:\n Hash: 0x12\n Counters: 3\n Block counts: [0, 102, 1]\n')).toBe(true); + expect(hasExecutedSwiftFunctions('Counters:\n $s5Probe5probeyS2iF:\n Block counts: [0, 0, 0]\n')).toBe(false); + expect(hasExecutedSwiftFunctions(report.replace('$s5Probe5probeyS2iF', '/tmp/Probe.swift:$s5Probe5probeyS2iF').replace('Function count: 0', 'Function count: 3'))).toBe(true); + }); + it('executes fresh paired workloads in both orders using supported CLI actions', () => { // Given the evaluation stage plan used by the physical runner. const stages = benchmarkStagePlan(); From e9dec6c3f6f39caa250f7f2179cf002464441326 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 18:49:58 +0100 Subject: [PATCH 48/60] fix(pgo): resolve foundation validation failures --- knip.json | 2 +- package-lock.json | 1 + package.json | 1 + scripts/pgo/JOURNEY.md | 2 +- scripts/pgo/ios.ts | 42 ++++++++++++++++++++++---------- tests/tooling/pgoIOSRock.test.ts | 17 ++++++++++++- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/knip.json b/knip.json index 8fba9bd02c97..793f2d7d1612 100644 --- a/knip.json +++ b/knip.json @@ -89,5 +89,5 @@ "diff-so-fancy", "@babel/plugin-proposal-class-properties" ], - "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl"] + "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl", "agent-device"] } diff --git a/package-lock.json b/package-lock.json index 39800039ca3c..4f918febbe79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -192,6 +192,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/package.json b/package.json index b5f2bed6fd9c..1e2873c0c1a9 100644 --- a/package.json +++ b/package.json @@ -269,6 +269,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md index b532bb792455..c29d117fc601 100644 --- a/scripts/pgo/JOURNEY.md +++ b/scripts/pgo/JOURNEY.md @@ -8,7 +8,7 @@ Use a dedicated, owner-approved heavy account with a populated Inbox, a long chat history, and enough expenses and expense reports to scroll. Keep the account and its data between runs. Installing over the existing app preserves authentication; uninstalling or clearing app data does not. -The runner never enters credentials, retrieves a magic code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. +The runner never enters credentials, retrieves a security code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. Before every run, it opens Inbox to allow the app's automatic large-account preference update, then opens Account → Preferences and ensures Priority mode is **Most recent**. If it is **#focus**, the runner changes it and verifies the result. It checks the preference again at the end and rejects the run if #focus became enabled. diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 77ea09670c40..204855eaca7c 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -239,15 +239,21 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ...signingSettings, ...pgoBuildSettings(kind, profilePath), ]); - withLocalUploadPhasesDisabled(() => run('/usr/bin/env', [ - ...sourceBuildEnvironment(mode, profilePath), - 'IS_HYBRID_APP=true', - 'BUNDLER=metro', - `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, - 'node', - join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), - ...rockArguments, - ], join(rootDirectory, 'Mobile-Expensify'))); + withLocalUploadPhasesDisabled(() => + run( + '/usr/bin/env', + [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], + join(rootDirectory, 'Mobile-Expensify'), + ), + ); if (!existsSync(builtAppPath)) { fail(`Missing built iOS app at ${builtAppPath}.`); } @@ -471,10 +477,20 @@ function replaceOnce(content: string, previous: string, next: string): string { /** Preserve each Xcode setting as one argument through Rock's string argument parser. */ function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { return [ - 'build:ios', '--local', '--verbose', '--configuration', 'Release', - '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', - '--build-folder', derivedDataDirectory, '--no-install-pods', - '--extra-params', settings.map((argument) => JSON.stringify(argument)).join(' '), + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + derivedDataDirectory, + '--no-install-pods', + '--extra-params', + settings.map((argument) => JSON.stringify(argument)).join(' '), ]; } diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts index 3eeaaec3a56a..9809f0dc38ed 100644 --- a/tests/tooling/pgoIOSRock.test.ts +++ b/tests/tooling/pgoIOSRock.test.ts @@ -1,7 +1,9 @@ // cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata import {describe, expect, it} from 'bun:test'; + import {iosRockBuildArguments} from '@scripts/pgo/ios'; + import {parseArgs} from '@rock-js/tools'; describe('PGO iOS Rock build arguments', () => { @@ -11,7 +13,20 @@ describe('PGO iOS Rock build arguments', () => { // When the adapter creates a Rock invocation. const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); // Then the local Release destination and existing pod preparation are explicit. - expect(args.slice(0, 12)).toEqual(['build:ios', '--local', '--verbose', '--configuration', 'Release', '--scheme', 'Expensify', '--destination', 'generic/platform=iOS', '--build-folder', '/tmp/derived data/optimized', '--no-install-pods']); + expect(args.slice(0, 12)).toEqual([ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + '/tmp/derived data/optimized', + '--no-install-pods', + ]); expect(parseArgs(args.at(13) ?? '')).toEqual(settings); }); }); From e7ce3420a2844ea3af1570c846ea4cbcd3cfa553 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Mon, 12 Oct 2026 18:59:30 +0100 Subject: [PATCH 49/60] fix(pgo): use built-in YAML parser in workflow tests --- tests/tooling/pgoEvaluationWorkflow.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tooling/pgoEvaluationWorkflow.test.ts b/tests/tooling/pgoEvaluationWorkflow.test.ts index 55d07c91b6ce..5f8b9d43b39f 100644 --- a/tests/tooling/pgoEvaluationWorkflow.test.ts +++ b/tests/tooling/pgoEvaluationWorkflow.test.ts @@ -2,7 +2,6 @@ import {describe, expect, it} from 'bun:test'; import {isRecord} from '@scripts/pgo/evaluationSchema'; -import {load} from 'js-yaml'; import {readFileSync} from 'node:fs'; import {runInNewContext} from 'node:vm'; @@ -87,7 +86,7 @@ describe('PGO evaluation workflow contract', () => { }); function workflowJob(path: string, name: string): Record { - const input: unknown = load(readFileSync(path, 'utf8')); + const input: unknown = Bun.YAML.parse(readFileSync(path, 'utf8')); if (!isRecord(input) || !isRecord(input.jobs) || !isRecord(input.jobs[name])) { throw new Error('Missing workflow job.'); } From 16db8945c87888a4f6ead6deea8ba6d2ca8fa0d7 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 13 Oct 2026 14:33:39 +0100 Subject: [PATCH 50/60] fix(pgo): reject partial Android profiles and resolve Linux NDK paths --- scripts/pgo/android.ts | 36 ++++++--- tests/tooling/pgoAndroid.test.ts | 126 +++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 tests/tooling/pgoAndroid.test.ts diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index e0ef640aa0a8..d802c2d3f22b 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -14,9 +14,9 @@ import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; +const INSTRUMENTED_LIBRARIES = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { - const ndkVersion = '27.1.12297006'; const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); const rawProfileDirectory = join(profileDirectory, 'raw'); const apkDirectory = join(profileDirectory, 'apk'); @@ -56,12 +56,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent } function llvmTool(name: string): string { - const androidNdkHome: unknown = process.env.ANDROID_NDK_HOME; - const androidNdkRoot: unknown = process.env.ANDROID_NDK_ROOT; - const configuredNdkRoot = [androidNdkHome, androidNdkRoot].find((value): value is string => typeof value === 'string'); - const ndkRoot = configuredNdkRoot ?? join(homedir(), `Library/Android/sdk/ndk/${ndkVersion}`); - const hostTag = platform() === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; - const toolPath = join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); + const toolPath = androidLlvmToolPath(name); if (!existsSync(toolPath)) { fail(`Missing ${name} at ${toolPath}. Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build.`); } @@ -113,8 +108,13 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const deadline = Date.now() + profileDumpTimeoutSeconds * 1000; while (Date.now() < deadline) { const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); - const profileCount = /Wrote ([1-9][0-9]*) LLVM PGO profile/.exec(profileLogs)?.[1]; - if (profileCount) { + const profileCount = /Wrote ([0-9]+) LLVM PGO profile/.exec(profileLogs)?.[1]; + if (profileCount !== undefined && Number(profileCount) !== 0) { + if (Number(profileCount) !== INSTRUMENTED_LIBRARIES.length) { + fail( + `The PGO receiver wrote ${profileCount} profiles, but expected ${INSTRUMENTED_LIBRARIES.length}. All instrumented native libraries must write successfully before recording profiles.`, + ); + } console.log(`Wrote ${profileCount} LLVM PGO profile(s).`); return; } @@ -157,9 +157,8 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const llvmReadelf = llvmTool('llvm-readelf'); const llvmNm = llvmTool('llvm-nm'); const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); - const libraries = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; try { - for (const library of libraries) { + for (const library of INSTRUMENTED_LIBRARIES) { const apkEntry = `lib/arm64-v8a/${library}`; if (!apkEntries.has(apkEntry)) { fail(`Missing expected arm64 library in APK: ${apkEntry}`); @@ -206,6 +205,19 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent }; } +/** Resolve LLVM tools from the build's explicit NDK or host-specific SDK installation. */ +function androidLlvmToolPath( + name: string, + {hostPlatform = platform(), homeDirectory = homedir(), environment = process.env}: {hostPlatform?: NodeJS.Platform; homeDirectory?: string; environment?: NodeJS.ProcessEnv} = {}, +): string { + const configuredNdkRoot = [environment.ANDROID_NDK_HOME, environment.ANDROID_NDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const configuredSdkRoot = [environment.ANDROID_HOME, environment.ANDROID_SDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const sdkRoot = configuredSdkRoot ?? join(homeDirectory, hostPlatform === 'linux' ? 'Android/Sdk' : 'Library/Android/sdk'); + const ndkRoot = configuredNdkRoot ?? join(sdkRoot, 'ndk/27.1.12297006'); + const hostTag = hostPlatform === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; + return join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); +} + function androidApplicationID(buildGradle: string, configuredAppID?: string): string { if (configuredAppID) { return configuredAppID; @@ -222,4 +234,4 @@ function androidProfileReceiverComponent(appID: string): string { } export default createAndroidPgoAdapter; -export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent}; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidLlvmToolPath, androidProfileReceiverComponent}; diff --git a/tests/tooling/pgoAndroid.test.ts b/tests/tooling/pgoAndroid.test.ts new file mode 100644 index 000000000000..da67e2ab1f01 --- /dev/null +++ b/tests/tooling/pgoAndroid.test.ts @@ -0,0 +1,126 @@ +// cspell:ignore profdata + +import {beforeEach, describe, expect, it, mock} from 'bun:test'; + +import * as scriptUtils from '@scripts/lib/scriptUtils'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; +import process from 'node:process'; + +let profileLogs = ''; +const runCommand = mock(() => undefined); +const captureCommand = mock(() => profileLogs); +// Stub the device boundary while keeping the adapter's broadcast, polling and validation real. +mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, run: runCommand, capture: captureCommand})); +const {default: createAndroidPgoAdapter, androidLlvmToolPath} = await import('@scripts/pgo/android'); + +describe('PGO Android profile dumps', () => { + beforeEach(() => { + profileLogs = ''; + runCommand.mockClear(); + captureCommand.mockClear(); + }); + + it('accepts a dump only when all four native libraries wrote profiles', async () => { + // Given a complete writer response from the installed instrumented APK. + profileLogs = 'I/PgoProfileReceiver: Wrote 4 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then profile collection can continue. + await expect(dump).resolves.toBeUndefined(); + expect(runCommand.mock.calls).toEqual([ + ['adb', ['logcat', '-c']], + ['adb', ['shell', 'am', 'broadcast', '-a', 'com.expensify.chat.action.WRITE_PGO_PROFILES', '-n', 'com.example.pgo/org.me.mobiexpensifyg.PgoProfileReceiver']], + ]); + }); + + it.each([1, 2, 3, 5])('rejects a dump reporting %i profiles', async (count) => { + // Given a partial or unexpected writer response despite successful broadcast delivery. + profileLogs = `I/PgoProfileReceiver: Wrote ${count} LLVM PGO profile(s).`; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then training cannot silently accept an incomplete set of native counters. + await expect(dump).rejects.toThrow(`wrote ${count} profiles, but expected 4`); + }); + + it('keeps the diagnostic for a dump with no instrumented libraries', async () => { + // Given a receiver that could not find any profile writers. + profileLogs = 'I/PgoProfileReceiver: Wrote 0 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it explains that the instrumented APK must be rebuilt. + await expect(dump).rejects.toThrow('found no instrumented native libraries'); + }); + + it('rejects a non-instrumented APK', async () => { + // Given a receiver explicitly rejecting profiling in a normal release build. + profileLogs = 'Ignoring PGO profile request in a non-instrumented build.'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it directs the caller to install the instrumented build. + await expect(dump).rejects.toThrow('installed APK is not instrumented'); + }); +}); + +describe('PGO Android LLVM tools', () => { + it.each([ + ['linux', '/home/test/Android/Sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'], + ['darwin', '/home/test/Library/Android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-profdata'], + ] satisfies Array<[NodeJS.Platform, string]>)('uses the default SDK installation on %s', (hostPlatform, expected) => { + // Given an unconfigured host with its normal Android Studio SDK location. + // When LLVM tools are resolved without explicit environment settings. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform, homeDirectory: '/home/test', environment: {}}); + // Then the selected path uses that host's SDK directory and toolchain. + expect(path).toBe(expected); + }); + + it.each(['ANDROID_HOME', 'ANDROID_SDK_ROOT'])('honors %s on Linux', (variable) => { + // Given a Linux runner with its SDK installed outside the default home directory. + // When the SDK environment setting is used to resolve LLVM. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/android/sdk'}}); + // Then the pinned NDK is resolved within the configured SDK. + expect(path).toBe('/opt/android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it.each(['ANDROID_NDK_HOME', 'ANDROID_NDK_ROOT'])('prefers %s over SDK settings', (variable) => { + // Given an explicit NDK override and a different SDK location. + // When the tool path is resolved. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/custom-ndk', ANDROID_HOME: '/opt/android/sdk'}}); + // Then an explicitly selected build toolchain wins over the pinned SDK fallback. + expect(path).toBe('/opt/custom-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('prefers ANDROID_HOME over the deprecated SDK root setting and ignores empty NDK overrides', () => { + // Given both SDK settings and empty NDK settings. + const environment = {ANDROID_HOME: '/opt/current-sdk', ANDROID_SDK_ROOT: '/opt/old-sdk', ANDROID_NDK_HOME: '', ANDROID_NDK_ROOT: ''}; + // When LLVM tools are resolved on Linux. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment}); + // Then empty settings do not mask the SDK selected by ANDROID_HOME. + expect(path).toBe('/opt/current-sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('uses the resolved tool path in the adapter', () => { + // Given an explicit temporary NDK with a real tool file. + const directory = mkdtempSync(join(tmpdir(), 'pgo-ndk-test-')); + const previousNdk = process.env.ANDROID_NDK_HOME; + process.env.ANDROID_NDK_HOME = directory; + try { + const expected = androidLlvmToolPath('llvm-profdata'); + mkdirSync(dirname(expected), {recursive: true}); + writeFileSync(expected, ''); + // When the adapter checks tool availability. + const path = createAndroidPgoAdapter().llvmTool('llvm-profdata'); + // Then it returns the existing file in the selected NDK. + expect(path).toBe(expected); + } finally { + if (previousNdk === undefined) { + delete process.env.ANDROID_NDK_HOME; + } else { + process.env.ANDROID_NDK_HOME = previousNdk; + } + rmSync(directory, {recursive: true, force: true}); + } + }); +}); From 83339c5c367ae6fe075c61ea7d1172e1d3d6d539 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 13 Oct 2026 15:53:09 +0100 Subject: [PATCH 51/60] fix(pgo): skip Gradle lookup for explicit Android app IDs --- scripts/pgo/android.ts | 5 ++++- tests/tooling/pgoAndroid.test.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index bb1eecb0d731..c6a043c495af 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -41,7 +41,10 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent let cachedAppID: string | undefined; function appID(): string { - cachedAppID ??= androidApplicationID(readFileSync(buildGradlePath, 'utf8'), configuredAppID); + if (configuredAppID) { + return configuredAppID; + } + cachedAppID ??= androidApplicationID(readFileSync(buildGradlePath, 'utf8')); return cachedAppID; } diff --git a/tests/tooling/pgoAndroid.test.ts b/tests/tooling/pgoAndroid.test.ts index da67e2ab1f01..ab82b96c3f1b 100644 --- a/tests/tooling/pgoAndroid.test.ts +++ b/tests/tooling/pgoAndroid.test.ts @@ -1,6 +1,6 @@ // cspell:ignore profdata -import {beforeEach, describe, expect, it, mock} from 'bun:test'; +import {afterAll, beforeEach, describe, expect, it, mock} from 'bun:test'; import * as scriptUtils from '@scripts/lib/scriptUtils'; @@ -10,12 +10,16 @@ import {dirname, join} from 'node:path'; import process from 'node:process'; let profileLogs = ''; +// Reproduce the CI checkout, which does not initialize the native submodule. +const rootDirectory = mkdtempSync(join(tmpdir(), 'pgo-android-test-')); const runCommand = mock(() => undefined); const captureCommand = mock(() => profileLogs); // Stub the device boundary while keeping the adapter's broadcast, polling and validation real. -mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, run: runCommand, capture: captureCommand})); +mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, rootDirectory, run: runCommand, capture: captureCommand})); const {default: createAndroidPgoAdapter, androidLlvmToolPath} = await import('@scripts/pgo/android'); +afterAll(() => rmSync(rootDirectory, {recursive: true, force: true})); + describe('PGO Android profile dumps', () => { beforeEach(() => { profileLogs = ''; From 4e5abf7c9769fe88b530e361b268eb48e455c844 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 Oct 2026 12:37:26 +0100 Subject: [PATCH 52/60] feat(pgo): prepare combined CI verification and runner handoff --- .github/workflows/pgoEvaluation.yml | 7 ++- scripts/pgo/EVALUATION.md | 47 +++++++++++++++++++++ scripts/pgo/android.ts | 2 +- scripts/pgo/androidRelease.init.gradle | 19 ++++++++- scripts/pgo/candidateResolve.ts | 2 +- tests/tooling/pgoCandidate.test.ts | 5 ++- tests/tooling/pgoEvaluationWorkflow.test.ts | 23 ++++++++++ 7 files changed, 99 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml index f04f78a9fa8a..9be2685f0c66 100644 --- a/.github/workflows/pgoEvaluation.yml +++ b/.github/workflows/pgoEvaluation.yml @@ -29,7 +29,7 @@ jobs: github.repository == 'Expensify/App' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == 'Expensify/App' && - github.event.pull_request.head.ref == '@chrispader/perf/pgo-ci-evaluation')) + github.event.pull_request.head.ref == '@chrispader/perf/pgo-ci-verification')) runs-on: ubuntu-latest permissions: contents: read @@ -97,6 +97,7 @@ jobs: with: ref: ${{ needs.prepare.outputs.app-sha }} submodules: recursive + token: ${{ secrets.OS_BOTIFY_TOKEN }} persist-credentials: false - name: Checkout reviewed evaluation tooling uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -135,6 +136,10 @@ jobs: PGO_ANDROID_APP_ID: ${{ vars.PGO_ANDROID_APP_ID }} PGO_ANDROID_FIXTURE_PATH: ${{ secrets.PGO_ANDROID_FIXTURE_PATH }} PGO_ANDROID_ENVFILE: ${{ secrets.PGO_ANDROID_ENVFILE }} + PGO_ANDROID_KEYSTORE_PATH: ${{ secrets.PGO_ANDROID_KEYSTORE_PATH }} + PGO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.PGO_ANDROID_KEYSTORE_PASSWORD }} + PGO_ANDROID_KEY_ALIAS: ${{ secrets.PGO_ANDROID_KEY_ALIAS }} + PGO_ANDROID_KEY_PASSWORD: ${{ secrets.PGO_ANDROID_KEY_PASSWORD }} PGO_IOS_DEVICE_ID: ${{ secrets.PGO_IOS_DEVICE_ID }} PGO_IOS_APP_ID: ${{ vars.PGO_IOS_APP_ID }} PGO_IOS_FIXTURE_PATH: ${{ secrets.PGO_IOS_FIXTURE_PATH }} diff --git a/scripts/pgo/EVALUATION.md b/scripts/pgo/EVALUATION.md index 553d9f38149d..f4b8005a0416 100644 --- a/scripts/pgo/EVALUATION.md +++ b/scripts/pgo/EVALUATION.md @@ -1,5 +1,7 @@ # Native PGO evaluation + + The evaluation workflow measures a native candidate without selecting a release artifact or changing deployment. It records the actual App checkout, Mobile gitlink, version and producer attempt before evaluating each platform whose native build succeeded. A failed web deployment does not invalidate an otherwise successful native build. Each platform uses a dedicated physical runner. Its concurrency queue preserves up to 100 pending evaluations instead of replacing the previous pending candidate. Monitor the queue and rerun candidates if that limit is exceeded. Configure the `pgo-android` and `pgo-ios` runner labels and these settings before enabling device jobs: @@ -19,6 +21,8 @@ Each platform uses a dedicated physical runner. Its concurrency queue preserves The iOS runner also needs an installed development certificate and a provisioning profile that permits the selected physical iPhone and dedicated bundle identifier. Configure the signing variables as repository variables. Keep certificate material and signing credentials private on the runner. +Android CI signing uses a private keystore on the runner. Set repository secrets `PGO_ANDROID_KEYSTORE_PATH`, `PGO_ANDROID_KEYSTORE_PASSWORD`, `PGO_ANDROID_KEY_ALIAS` and `PGO_ANDROID_KEY_PASSWORD`. The path must be absolute and outside the checkout so cleanup does not delete the key. The evaluation build uses this signing configuration without changing tracked native files. When no keystore path is supplied, local builds retain their existing signing configuration. Keep the same key across runs so updates preserve the signed-in app's data. + The configured application and bundle identifiers must match the identifiers produced by the candidate's native configuration, environment and provisioning profile. These selectors do not rewrite native identifiers. A dedicated physical runner can use an existing matching identifier. The account must already be signed in to the dedicated application. Authentication is never automated. The fixture must explicitly authorize the heavy account and any training messages to its personal chat. Other reports are read-only. Keep account fixtures, environment files and signing credentials on the protected runner. Store device serials and private fixture/environment paths in repository secrets so the Actions environment banner masks them. Application identifiers and nonsecret signing settings can use repository variables. Do not store fixture or environment contents in variables or artifacts. @@ -34,3 +38,46 @@ Before and after each accepted stage, the runner checks the candidate commits, e Public artifacts contain numeric observations, distributions and hashes. Raw device output, screenshots, fixtures, environment values and detailed errors remain private on the runner. Private evidence lives under the candidate's ignored evaluation directory and needs a runner retention policy. Compiler consumption remains unverified in the public result unless execution evidence is independently established. Automatic evaluations follow completed trusted staging or production deploy runs. A manual rerun selects a producer run ID and its current immutable attempt. The exact approved same-repository preview branch can run before merge. Forks and arbitrary heads cannot access device jobs. Preview reports stay in workflow summaries and artifacts. A deployment report can update only its version-matching labelled historical checklist; otherwise the workflow visibly skips the comment. Evaluation never promotes or replaces the ordinary native artifact. + +## Run the combined flow before merging + +The combined verification branch contains the App foundation, evaluation scripts and CI workflow. Its `Mobile-Expensify` gitlink pins the native PGO hooks together with the native revision from App main. The smaller review PRs remain separate. Use this branch to verify the complete report-only flow without starting a deployment or merging the prerequisites first. + +1. Set up protected self-hosted runners with labels `pgo-android` and `pgo-ios` in Expensify/App. Connect one authorized physical arm64 Android device and one trusted, unlocked physical iPhone. Reserve each device for its evaluation job. Android needs the repository's JDK, SDK and NDK, with `ANDROID_HOME` and `ANDROID_NDK_HOME` available to the runner service. iOS needs macOS, the repository's Xcode and Ruby toolchain, Bundler, CocoaPods, development signing for the app and extensions, and a signed `agent-device` XCTest runner. Pin `agent-device` to version `0.20.6` on both runners and expose it on the service's `PATH`. +2. Complete the repository's normal HybridApp environment setup on each runner, including private dependency credentials. Keep the approved fixture, fixed merged Release environment file and signing material outside the checkout. Set `STANDALONE_NEW_DOT=false`. The workflow installs both App and Mobile-Expensify dependencies; the PGO builds select HybridApp and build React Native and Hermes from source through Rock. +3. In **Expensify/App → Settings → Secrets and variables → Actions**, configure the secrets and variables below. These are repository settings; this workflow does not select a GitHub environment, so environment-only secrets are insufficient. Enable the approved workflow actions and grant the runners access to this repository. + + | Repository secrets | Value | + | --- | --- | + | `OS_BOTIFY_TOKEN` | A token authorized to read App and the private Mobile-Expensify repository, used for the recursive checkout | + | `PGO_ANDROID_DEVICE_ID`, `PGO_IOS_DEVICE_ID` | The corresponding physical device identifiers | + | `PGO_ANDROID_FIXTURE_PATH`, `PGO_IOS_FIXTURE_PATH` | Absolute paths to the approved JSON fixtures on the corresponding runners | + | `PGO_ANDROID_ENVFILE`, `PGO_IOS_ENVFILE` | Absolute paths to the fixed merged Release environment files | + | `PGO_ANDROID_KEYSTORE_PATH` | Absolute path to the Android evaluation keystore | + | `PGO_ANDROID_KEYSTORE_PASSWORD`, `PGO_ANDROID_KEY_ALIAS`, `PGO_ANDROID_KEY_PASSWORD` | Credentials for that evaluation key | + + | Repository variables | Value | + | --- | --- | + | `PGO_ANDROID_APP_ID` | The application ID produced by the checked-in Android Release configuration | + | `PGO_IOS_APP_ID` | The bundle ID produced by the checked-in iOS Release configuration | + | `IOS_CODE_SIGNING_ALLOWED` | `true` | + | `IOS_DEVELOPMENT_TEAM` | The Apple team authorized for the app, extensions and physical iPhone | + + The workflow's `GITHUB_TOKEN` supplies its own App metadata, artifact and issue-comment permissions. No additional reporting token is needed. Do not put fixture contents, environment contents or certificate material into these path settings. + +4. Install an app with the same identifier and signing identity on each dedicated device and sign into the approved high-traffic account manually. Follow [the fixture setup](./JOURNEY.md#account-and-device-setup) and set `allowMessages: true` only with the owner's approval. Training sends two test messages per journey to that account's personal chat. The fixture's report must contain enough content to scroll. Keep the account signed in between builds; the runner installs updates without clearing app data. If `SIGN_IN_REQUIRED` appears, sign in on the physical device and rerun the failed job. +5. Copy both combined branch tips into the Expensify repositories without squashing or cherry-picking them. The App gitlink must continue to point to the exact native commit. From a clone of Expensify/App with its native submodule available, run: + + ```bash + git -C Mobile-Expensify fetch https://github.com/margelo/Mobile-Expensify.git @chrispader/perf/pgo-ci-verification + git -C Mobile-Expensify push git@github.com:Expensify/Mobile-Expensify.git FETCH_HEAD:refs/heads/@chrispader/perf/pgo-ci-verification + git fetch https://github.com/margelo/Expensify.git @chrispader/perf/pgo-ci-verification + git push git@github.com:Expensify/App.git FETCH_HEAD:refs/heads/@chrispader/perf/pgo-ci-verification + ``` + + Preserve an existing upstream branch before replacing it. The fork PR intentionally skips device jobs. Copying the native branch makes the pinned native commit fetchable from the private upstream repository; it does not merge either PR. + +6. Open a PR **within Expensify/App**, from `@chrispader/perf/pgo-ci-verification` to `main`. That exact branch name is required by both the workflow and candidate resolver. The `opened`, `synchronize` and `reopened` events run **Evaluate native PGO candidate** from the PR head. After reviewing the copied code, allow its jobs onto the protected runners. A new commit or closing and reopening this same-repository PR triggers another attempt. No staging or production deployment is needed for this preview. +7. Open the run and verify `prepare`, both `evaluate` matrix jobs, and `report`. Each device job builds baseline and instrumented binaries, verifies native instrumentation, collects three complete training journeys, validates and merges their profiles, builds the optimized binary, and compares both binaries in both build orders. Inspect `pgo-evaluation-plan`, `pgo-evaluation-android`, `pgo-evaluation-ios` and `pgo-evaluation-report`. Confirm the plan matches the App head and native gitlink, both platform results report `status: passed`, and their samples cover startup, warm navigation and the heavy report. Inspect the private runner logs for unexpected JavaScript errors. A green report job alone does not mean device evaluation succeeded; failed platform results remain visible in the report. Preview runs publish artifacts and a summary without commenting on a deployment checklist. + +`workflow_run` and the manual **Run workflow** entry point require the workflow to exist on the default branch. They are separate from the PR trigger above. After the reviewed workflow and prerequisites land upstream, a new native staging or production deployment records its candidate metadata and starts evaluation automatically. To repeat one, select its completed deployment run ID as `producer_run_id` and choose `both`, `android` or `ios`. Older deployments without the `pgo-native-candidate-` artifact cannot be evaluated through that entry point. Results attach only to the matching deployment checklist, and evaluation never changes the binary selected for deployment. diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index c6a043c495af..e0d926666707 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -123,7 +123,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent archiveReleaseApk(artifactPaths[kind]); writeFileSync( `${artifactPaths[kind]}.build-settings.json`, - `${JSON.stringify({kind, releasePipeline: releasePipeline.enabled, appID: appID(), sourceBuiltRuntime: true, shrinking: releasePipeline.shrinking, fullStory: releasePipeline.enabled, signing: 'existing-project-configuration', storeEquivalent: false, gradleArguments, buildEnvironment}, null, 2)}\n`, + `${JSON.stringify({kind, releasePipeline: releasePipeline.enabled, appID: appID(), sourceBuiltRuntime: true, shrinking: releasePipeline.shrinking, fullStory: releasePipeline.enabled, signing: process.env.PGO_ANDROID_KEYSTORE_PATH ? 'dedicated-pgo-keystore' : 'existing-project-configuration', storeEquivalent: false, gradleArguments, buildEnvironment}, null, 2)}\n`, ); } diff --git a/scripts/pgo/androidRelease.init.gradle b/scripts/pgo/androidRelease.init.gradle index ed3bdaf1c414..75933f894099 100644 --- a/scripts/pgo/androidRelease.init.gradle +++ b/scripts/pgo/androidRelease.init.gradle @@ -1,4 +1,4 @@ -// Apply release processing without changing the owner's local application ID or signing configuration. +// Apply release processing with optional dedicated CI signing, preserving the application ID. gradle.beforeProject { application -> def skipFullStory = application.providers.gradleProperty('pgoSkipFullStory').getOrElse('false') == 'true' if (skipFullStory) { @@ -16,7 +16,22 @@ gradle.beforeProject { application -> } release.minifyEnabled = true release.proguardFiles android.getDefaultProguardFile('proguard-android-optimize.txt'), application.file('proguard-rules.pro') - application.logger.lifecycle("PGO shrinking enabled; FullStory tasks ${skipFullStory ? 'disabled' : 'enabled'}; existing signing and application ID preserved.") + def keystorePath = System.getenv('PGO_ANDROID_KEYSTORE_PATH') + if (keystorePath) { + def storePassword = System.getenv('PGO_ANDROID_KEYSTORE_PASSWORD') + def keyAlias = System.getenv('PGO_ANDROID_KEY_ALIAS') + def keyPassword = System.getenv('PGO_ANDROID_KEY_PASSWORD') + if (!storePassword || !keyAlias || !keyPassword) { + throw new GradleException('Dedicated PGO signing requires all Android keystore settings.') + } + def signing = android.signingConfigs.maybeCreate('pgoEvaluation') + signing.storeFile = application.file(keystorePath) + signing.storePassword = storePassword + signing.keyAlias = keyAlias + signing.keyPassword = keyPassword + release.signingConfig = signing + } + application.logger.lifecycle("PGO shrinking enabled; FullStory tasks ${skipFullStory ? 'disabled' : 'enabled'}; application ID preserved.") } } } diff --git a/scripts/pgo/candidateResolve.ts b/scripts/pgo/candidateResolve.ts index aa0dbcad3b76..91f2ed43650f 100644 --- a/scripts/pgo/candidateResolve.ts +++ b/scripts/pgo/candidateResolve.ts @@ -6,7 +6,7 @@ import {isGitID, isRecord, parseCandidate} from './evaluationSchema'; type GitHubRead = (path: string) => Promise; const repository = 'Expensify/App'; -const previewBranch = '@chrispader/perf/pgo-ci-evaluation'; +const previewBranch = '@chrispader/perf/pgo-ci-verification'; const producerName = 'Deploy code to staging or production'; async function resolveProducer(metadata: unknown, run: unknown, jobs: unknown, toolingSha: string, read: GitHubRead, platform?: EvaluationPlatform): Promise { diff --git a/tests/tooling/pgoCandidate.test.ts b/tests/tooling/pgoCandidate.test.ts index 2f3b7e862319..eab28e50462b 100644 --- a/tests/tooling/pgoCandidate.test.ts +++ b/tests/tooling/pgoCandidate.test.ts @@ -52,7 +52,7 @@ const read: GitHubRead = async (path) => { }; function preview(repo = 'Expensify/App', sha = appSha) { - return {number: 3, pull_request: {head: {sha, ref: '@chrispader/perf/pgo-ci-evaluation', repo: {full_name: repo}}, base: {repo: {full_name: 'Expensify/App'}}}}; + return {number: 3, pull_request: {head: {sha, ref: '@chrispader/perf/pgo-ci-verification', repo: {full_name: repo}}, base: {repo: {full_name: 'Expensify/App'}}}}; } describe('PGO immutable candidate resolution', () => { @@ -97,6 +97,9 @@ describe('PGO immutable candidate resolution', () => { // Then only the current trusted head can evaluate; normal forks receive no device authorization. expect(plan.platforms).toEqual(['android', 'ios']); await expect(resolvePreview(preview('fork/App'), toolingSha, read)).rejects.toThrow(); + const wrongBranch = preview(); + wrongBranch.pull_request.head.ref = 'other-preview'; + await expect(resolvePreview(wrongBranch, toolingSha, read)).rejects.toThrow(); await expect(resolvePreview(preview('Expensify/App', toolingSha), toolingSha, read)).rejects.toThrow(); await expect(resolvePreview(preview(), toolingSha, async (path) => (path.includes('/pulls/') ? {head: {sha: appSha}, state: 'closed'} : read(path)))).rejects.toThrow(); }); diff --git a/tests/tooling/pgoEvaluationWorkflow.test.ts b/tests/tooling/pgoEvaluationWorkflow.test.ts index 5f8b9d43b39f..28dc11349773 100644 --- a/tests/tooling/pgoEvaluationWorkflow.test.ts +++ b/tests/tooling/pgoEvaluationWorkflow.test.ts @@ -1,3 +1,5 @@ +// cspell:ignore chrispader +/* eslint-disable @typescript-eslint/naming-convention -- Fixtures reproduce GitHub's event schema. */ import {describe, expect, it} from 'bun:test'; import {isRecord} from '@scripts/pgo/evaluationSchema'; @@ -6,6 +8,23 @@ import {readFileSync} from 'node:fs'; import {runInNewContext} from 'node:vm'; describe('PGO evaluation workflow contract', () => { + it.each([ + ['Expensify/App', '@chrispader/perf/pgo-ci-verification', true], + ['margelo/Expensify', '@chrispader/perf/pgo-ci-verification', false], + ['Expensify/App', 'other-preview', false], + ])('gates physical-device access for %s on %s', (repository, branch, eligible) => { + // Given the actual prepare-job condition and a pull-request head. + const job = workflowJob('.github/workflows/pgoEvaluation.yml', 'prepare'); + if (typeof job.if !== 'string') { + throw new Error('Missing preview authorization condition.'); + } + const github = {repository: 'Expensify/App', event_name: 'pull_request', event: {pull_request: {head: {repo: {full_name: repository}, ref: branch}}}}; + // When the condition runs with the event metadata available to Actions. + const actual: unknown = runInNewContext(job.if, {github}); + // Then only the named upstream branch can request credential devices. + expect(actual).toBe(eligible); + }); + it('keeps producer metadata failures outside canonical deployment eligibility', () => { // Given the ordinary deploy workflow and its evaluation metadata additions. const job = workflowJob('.github/workflows/deploy.yml', 'prep'); @@ -33,6 +52,10 @@ describe('PGO evaluation workflow contract', () => { expect(environment.PGO_ANDROID_DEVICE_ID).toBe(expression('', 'secrets.PGO_ANDROID_DEVICE_ID')); expect(environment.PGO_IOS_FIXTURE_PATH).toBe(expression('', 'secrets.PGO_IOS_FIXTURE_PATH')); expect(environment.PGO_IOS_ENVFILE).toBe(expression('', 'secrets.PGO_IOS_ENVFILE')); + expect(environment.PGO_ANDROID_KEYSTORE_PATH).toBe(expression('', 'secrets.PGO_ANDROID_KEYSTORE_PATH')); + expect(environment.PGO_ANDROID_KEYSTORE_PASSWORD).toBe(expression('', 'secrets.PGO_ANDROID_KEYSTORE_PASSWORD')); + const checkout = workflowStep(job, 'Checkout pinned native candidate'); + expect(isRecord(checkout.with) && checkout.with.token).toBe(expression('', 'secrets.OS_BOTIFY_TOKEN')); expect(job.concurrency.queue).toBe('max'); const report = workflowJob('.github/workflows/pgoEvaluation.yml', 'report'); if (!isRecord(report.concurrency)) { From 7cd178a699f2115744f9626665ba62aaca30fad3 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 Oct 2026 18:59:22 +0100 Subject: [PATCH 53/60] feat(pgo): verify CI flow on hosted emulators --- .github/workflows/deploy.yml | 18 -- .github/workflows/pgoEvaluation.yml | 277 ++++++------------ .../android/CMakeLists.txt | 9 +- package.json | 3 +- patches/react-native/details.md | 6 + ...t-native+0.86.0+048+pgo-emulator-abi.patch | 38 +++ scripts/pgo/EVALUATION.md | 87 ++---- scripts/pgo/README.md | 4 +- scripts/pgo/android.ts | 20 +- scripts/pgo/androidEmulator.init.gradle | 18 ++ scripts/pgo/androidRelease.init.gradle | 19 +- scripts/pgo/copy-ci-preview.sh | 21 ++ scripts/pgo/emulator.ts | 166 +++++++++++ scripts/pgo/emulatorFixture.ts | 34 +++ scripts/pgo/ios.ts | 64 +++- scripts/pgo/iosSimulator.ts | 60 ++++ src/PgoVerificationApp.tsx | 78 +++++ tests/tooling/pgoEmulator.test.ts | 93 ++++++ tests/tooling/pgoEvaluationWorkflow.test.ts | 166 ++++------- tests/tooling/pgoIOSRock.test.ts | 9 + 20 files changed, 761 insertions(+), 429 deletions(-) create mode 100644 patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch create mode 100644 scripts/pgo/androidEmulator.init.gradle create mode 100755 scripts/pgo/copy-ci-preview.sh create mode 100644 scripts/pgo/emulator.ts create mode 100644 scripts/pgo/emulatorFixture.ts create mode 100644 scripts/pgo/iosSimulator.ts create mode 100644 src/PgoVerificationApp.tsx create mode 100644 tests/tooling/pgoEmulator.test.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a13fd0c19ca0..d2cebadd8ba8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -180,24 +180,6 @@ jobs: isCherryPick, ); - - name: Record native evaluation candidate - id: pgoCandidate - continue-on-error: true - env: - PGO_CANDIDATE_ENVIRONMENT: ${{ env.DEPLOY_ENV }} - PGO_NATIVE_ELIGIBLE: ${{ env.DEPLOY_ENV == 'staging' || fromJSON(steps.isCherryPick.outputs.IS_CHERRY_PICK) }} - run: ./node_modules/.bin/bun scripts/pgo/pgoCandidate.ts capture --output pgo-candidate/candidate.json --environment "$PGO_CANDIDATE_ENVIRONMENT" --native-eligible "$PGO_NATIVE_ELIGIBLE" - - - name: Upload native evaluation candidate metadata - if: steps.pgoCandidate.outcome == 'success' - continue-on-error: true - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f - with: - name: pgo-native-candidate-${{ github.run_attempt }} - path: pgo-candidate/candidate.json - if-no-files-found: error - retention-days: 30 - - name: Get Android native version id: getAndroidVersion run: echo "VERSION_CODE=$(grep -oP 'android:versionCode="\K[0-9]+' Mobile-Expensify/Android/AndroidManifest.xml)" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml index 9be2685f0c66..a58b122e1c8b 100644 --- a/.github/workflows/pgoEvaluation.yml +++ b/.github/workflows/pgoEvaluation.yml @@ -1,224 +1,115 @@ -name: Evaluate native PGO candidate +name: Verify native PGO on emulators on: - workflow_run: - workflows: [Deploy code to staging or production] - types: [completed] - branches: [staging, production] - pull_request: - types: [opened, synchronize, reopened] + push: + branches: ['@chrispader/perf/pgo-ci-verification'] workflow_dispatch: inputs: - producer_run_id: - description: 'Immutable deploy run to evaluate' - required: true - type: string platform: - description: 'Platforms to evaluate' + description: Platforms to verify type: choice default: both options: [both, android, ios] -# Evaluation never publishes or selects a deployable native artifact. permissions: contents: read -jobs: - prepare: - if: >- - github.repository == 'Expensify/App' && - (github.event_name != 'pull_request' || - (github.event.pull_request.head.repo.full_name == 'Expensify/App' && - github.event.pull_request.head.ref == '@chrispader/perf/pgo-ci-verification')) - runs-on: ubuntu-latest - permissions: - contents: read - actions: read - pull-requests: read - outputs: - eligible: ${{ steps.candidate.outputs.eligible }} - app-sha: ${{ steps.candidate.outputs.app-sha }} - mobile-sha: ${{ steps.candidate.outputs.mobile-sha }} - platforms: ${{ steps.candidate.outputs.platforms }} - tooling-sha: ${{ steps.tooling.outputs.sha }} - platform-matrix: ${{ steps.candidate.outputs.platform-matrix }} - steps: - - name: Checkout reviewed evaluation tooling - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - name: Record reviewed tooling revision - id: tooling - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Setup repository tooling - uses: ./.github/actions/composite/setupNode - - name: Resolve and validate immutable candidate - id: candidate - env: - GITHUB_TOKEN: ${{ github.token }} - PGO_PRODUCER_RUN_ID: ${{ inputs.producer_run_id }} - PGO_PLATFORM: ${{ inputs.platform || 'both' }} - run: | - args=() - if [[ -n "$PGO_PRODUCER_RUN_ID" ]]; then - args+=(--source-run-id "$PGO_PRODUCER_RUN_ID") - fi - ./node_modules/.bin/bun scripts/pgo/pgoCandidate.ts prepare --output pgo-plan/plan.json --platform "$PGO_PLATFORM" "${args[@]}" - - name: Preserve verified evaluation plan - if: steps.candidate.outputs.eligible == 'true' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f - with: - name: pgo-evaluation-plan - path: pgo-plan/plan.json - if-no-files-found: error - retention-days: 30 +concurrency: + group: pgo-emulator-${{ github.ref }} + cancel-in-progress: true + +env: + STANDALONE_NEW_DOT: 'false' + IS_HYBRID_APP: 'true' + BUNDLER: metro + SENTRY_DISABLE_AUTO_UPLOAD: 'true' + CMAKE_BUILD_PARALLEL_LEVEL: '2' - evaluate: - needs: prepare - if: needs.prepare.outputs.eligible == 'true' +jobs: + verify: + # Only the copied, reviewed branch in App receives private dependency access. + if: github.repository == 'Expensify/App' && github.ref == 'refs/heads/@chrispader/perf/pgo-ci-verification' + runs-on: ${{ matrix.platform == 'android' && 'ubuntu-24.04' || 'macos-26' }} + timeout-minutes: 360 strategy: fail-fast: false matrix: - platform: ${{ fromJSON(needs.prepare.outputs.platform-matrix) }} - # A single platform process owns build, training and both benchmark orders. - runs-on: ${{ matrix.platform == 'android' && 'pgo-android' || 'pgo-ios' }} - concurrency: - group: pgo-evaluation-physical-${{ matrix.platform }} - cancel-in-progress: false - queue: max - timeout-minutes: 360 - permissions: - contents: read - actions: read + platform: ${{ fromJSON(inputs.platform == 'android' && '["android"]' || inputs.platform == 'ios' && '["ios"]' || '["android","ios"]') }} steps: - - name: Checkout pinned native candidate + - name: Check existing repository secrets + env: + CHECKOUT_TOKEN: ${{ secrets.OS_BOTIFY_TOKEN }} + MAPBOX_TOKEN: ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }} + run: | + test -n "$CHECKOUT_TOKEN" || { echo '::error::Add the existing OS_BOTIFY_TOKEN repository secret with access to Mobile-Expensify.'; exit 1; } + test -n "$MAPBOX_TOKEN" || { echo '::error::Add the existing MAPBOX_SDK_DOWNLOAD_TOKEN repository secret.'; exit 1; } + - name: Checkout the candidate and its pinned native sources uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: - ref: ${{ needs.prepare.outputs.app-sha }} submodules: recursive token: ${{ secrets.OS_BOTIFY_TOKEN }} persist-credentials: false - - name: Checkout reviewed evaluation tooling - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: ${{ needs.prepare.outputs.tooling-sha }} - path: evaluation-tooling - persist-credentials: false - - name: Setup pinned candidate dependencies - uses: ./evaluation-tooling/.github/actions/composite/setupNode + - name: Verify native checkout + run: | + test "$(git -C Mobile-Expensify rev-parse HEAD)" = "$(git ls-tree HEAD Mobile-Expensify | awk '{print $3}')" + test "$(./scripts/is-hybrid-app.sh)" = true + - name: Setup HybridApp dependencies + uses: ./.github/actions/composite/setupNode with: IS_HYBRID_BUILD: 'true' - - name: Expose pinned dependencies to reviewed execution overlay - run: ln -s "$GITHUB_WORKSPACE/node_modules" evaluation-tooling/node_modules - - name: Download verified plan - if: contains(needs.prepare.outputs.platforms, matrix.platform) - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 - with: - name: pgo-evaluation-plan - path: pgo-plan - - name: Verify candidate and native gitlink before device access - if: contains(needs.prepare.outputs.platforms, matrix.platform) + - name: Setup private SDK downloads env: - APP_SHA: ${{ needs.prepare.outputs.app-sha }} - MOBILE_SHA: ${{ needs.prepare.outputs.mobile-sha }} + MAPBOX_TOKEN: ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }} + run: ./scripts/setup-mapbox-sdk.sh "$MAPBOX_TOKEN" + + - name: Read repository Java version + if: matrix.platform == 'android' + id: java + uses: ./.github/actions/composite/getJavaVersion + - name: Setup Java + if: matrix.platform == 'android' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 + with: + distribution: temurin + java-version: ${{ steps.java.outputs.version }} + - name: Setup Android build tools and emulator access + if: matrix.platform == 'android' run: | - test "$(git rev-parse HEAD)" = "$APP_SHA" - test "$(git -C Mobile-Expensify rev-parse HEAD)" = "$MOBILE_SHA" - test "$(git ls-tree HEAD Mobile-Expensify | awk '{print $3}')" = "$MOBILE_SHA" - - name: Evaluate pinned candidate on its physical device - if: contains(needs.prepare.outputs.platforms, matrix.platform) - env: - PGO_CANDIDATE_DIRECTORY: ${{ github.workspace }} - PGO_TOOLING_DIRECTORY: ${{ github.workspace }}/evaluation-tooling - PGO_PLATFORM: ${{ matrix.platform }} - PGO_ANDROID_DEVICE_ID: ${{ secrets.PGO_ANDROID_DEVICE_ID }} - PGO_ANDROID_APP_ID: ${{ vars.PGO_ANDROID_APP_ID }} - PGO_ANDROID_FIXTURE_PATH: ${{ secrets.PGO_ANDROID_FIXTURE_PATH }} - PGO_ANDROID_ENVFILE: ${{ secrets.PGO_ANDROID_ENVFILE }} - PGO_ANDROID_KEYSTORE_PATH: ${{ secrets.PGO_ANDROID_KEYSTORE_PATH }} - PGO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.PGO_ANDROID_KEYSTORE_PASSWORD }} - PGO_ANDROID_KEY_ALIAS: ${{ secrets.PGO_ANDROID_KEY_ALIAS }} - PGO_ANDROID_KEY_PASSWORD: ${{ secrets.PGO_ANDROID_KEY_PASSWORD }} - PGO_IOS_DEVICE_ID: ${{ secrets.PGO_IOS_DEVICE_ID }} - PGO_IOS_APP_ID: ${{ vars.PGO_IOS_APP_ID }} - PGO_IOS_FIXTURE_PATH: ${{ secrets.PGO_IOS_FIXTURE_PATH }} - PGO_IOS_ENVFILE: ${{ secrets.PGO_IOS_ENVFILE }} - IOS_CODE_SIGNING_ALLOWED: ${{ vars.IOS_CODE_SIGNING_ALLOWED }} - IOS_DEVELOPMENT_TEAM: ${{ vars.IOS_DEVELOPMENT_TEAM }} - run: ./node_modules/.bin/bun evaluation-tooling/scripts/pgo/pgoEvaluation.ts execute "$PGO_PLATFORM" --plan pgo-plan/plan.json --output pgo-public - - name: Preserve public numeric result and provenance - if: always() && contains(needs.prepare.outputs.platforms, matrix.platform) - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" 'platforms;android-37' 'build-tools;36.0.0' 'ndk;27.1.12297006' 'cmake;3.22.1' + echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/27.1.12297006" >> "$GITHUB_ENV" + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Verify Android PGO on a disposable emulator + if: matrix.platform == 'android' + uses: ReactiveCircus/android-emulator-runner@ed009f5318f15b1cf93a191b856c4f1748a2d4c1 with: - name: pgo-evaluation-${{ matrix.platform }} - path: pgo-public/evaluation-${{ matrix.platform }}.json - if-no-files-found: warn - retention-days: 30 + api-level: 36 + arch: x86_64 + target: google_apis + profile: pixel_7 + disable-animations: true + script: npm run pgo:verify-ci -- android - report: - needs: [prepare, evaluate] - if: always() && needs.prepare.outputs.eligible == 'true' - runs-on: ubuntu-latest - concurrency: - group: pgo-evaluation-report-${{ needs.prepare.outputs.app-sha }} - cancel-in-progress: false - queue: max - permissions: - contents: read - actions: read - issues: write - steps: - - name: Checkout reviewed reporting tooling - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - ref: ${{ needs.prepare.outputs.tooling-sha }} - persist-credentials: false - - name: Setup repository tooling - uses: ./.github/actions/composite/setupNode - - name: Download public evaluation artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - name: Select repository Xcode + if: matrix.platform == 'ios' + uses: ./.github/actions/composite/getXcodeVersion + - name: Setup Ruby and CocoaPods + if: matrix.platform == 'ios' + uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f with: - pattern: pgo-evaluation-* - path: pgo-results - - name: Render report for this immutable candidate - env: - GITHUB_TOKEN: ${{ github.token }} - run: ./node_modules/.bin/bun scripts/pgo/pgoEvaluation.ts report --plan pgo-results/pgo-evaluation-plan/plan.json --results pgo-results --output pgo-report - - name: Publish report summary - run: cat pgo-report/summary.md >> "$GITHUB_STEP_SUMMARY" - - name: Preserve public report + ruby-version: .ruby-version + bundler-cache: true + - name: Verify iOS PGO on a disposable simulator + if: matrix.platform == 'ios' + run: npm run pgo:verify-ci -- ios + + - name: Upload verification profiles and report + if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: pgo-evaluation-report - path: | - pgo-report/summary.json - pgo-report/summary.md - if-no-files-found: error - retention-days: 30 - - name: Attach report to its matching deploy checklist - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd - with: - script: | - const fs = require('node:fs'); - const comment = JSON.parse(fs.readFileSync('pgo-report/comment.json', 'utf8')); - if (!Number.isSafeInteger(comment.targetIssueNumber) || !comment.marker || !comment.body) { - core.notice('No matching report target was resolved.'); - return; - } - const {data: issue} = await github.rest.issues.get({...context.repo, issue_number: comment.targetIssueNumber}); - { - const labels = issue.labels.map((label) => typeof label === 'string' ? label : label.name); - const tag = issue.body?.match(/\*\*Release Version:\*\*\s*`([^`]+)`/)?.[1]; - if (!labels.includes('StagingDeployCash') || !comment.version || tag !== `${comment.version}-staging`) { - core.notice('Historical checklist changed; report remains available as an artifact.'); - return; - } - } - const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: comment.targetIssueNumber, per_page: 100}); - const existing = comments.find((item) => item.user?.type === 'Bot' && item.body?.includes(comment.marker)); - if (existing) { - await github.rest.issues.updateComment({...context.repo, comment_id: existing.id, body: comment.body}); - } else { - await github.rest.issues.createComment({...context.repo, issue_number: comment.targetIssueNumber, body: comment.body}); - } + name: pgo-emulator-${{ matrix.platform }} + path: .pgo/ci/${{ matrix.platform }} + include-hidden-files: true + if-no-files-found: warn + retention-days: 7 diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index ab05b09f3c0c..c766d89915cb 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -13,7 +13,14 @@ if(EXPENSIFY_PGO_MODE STREQUAL "generate") OUTPUT_VARIABLE CLANG_RESOURCE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE ) - set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") + if(ANDROID_ABI STREQUAL "arm64-v8a") + set(PGO_ARCH aarch64) + elseif(ANDROID_ABI STREQUAL "x86_64") + set(PGO_ARCH x86_64) + else() + message(FATAL_ERROR "Unsupported PGO ABI: ${ANDROID_ABI}") + endif() + set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-${PGO_ARCH}-android.a") if(NOT EXISTS "${PGO_RUNTIME}") message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") endif() diff --git a/package.json b/package.json index d367b6b03bcf..3dfd849ce7f9 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build-staging": "ENV=staging bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "build-adhoc": "ENV=adhoc bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "benchmark-app-startup": "bun scripts/benchmarkAppStartup.ts", + "pgo:verify-ci": "bun scripts/pgo/emulator.ts", "pgo:journey": "bun scripts/pgo/journey.ts", "bootstrap-device": "bun scripts/bootstrapForDevice.ts", "createDocsRoutes": "bun .github/scripts/createDocsRoutes.ts", @@ -85,7 +86,7 @@ "server:vcr:dev": "npm run dev -w @expensify/victory-chart-renderer --", "server:vcr:test": "npm run test -w @expensify/victory-chart-renderer --", "test:bun": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./server ./tests/tooling", - "test:pgo-ci": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./tests/tooling/pgoCandidate.test.ts ./tests/tooling/pgoEvaluationReport.test.ts ./tests/tooling/pgoEvaluationRun.test.ts ./tests/tooling/pgoAndroidRelease.test.ts ./tests/tooling/pgoAndroid.test.ts ./tests/tooling/pgoIOSRock.test.ts ./tests/tooling/pgoJourney.test.ts ./tests/tooling/pgoEvaluationWorkflow.test.ts", + "test:pgo-ci": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./tests/tooling/pgoCandidate.test.ts ./tests/tooling/pgoEvaluationReport.test.ts ./tests/tooling/pgoEvaluationRun.test.ts ./tests/tooling/pgoAndroidRelease.test.ts ./tests/tooling/pgoAndroid.test.ts ./tests/tooling/pgoIOSRock.test.ts ./tests/tooling/pgoJourney.test.ts ./tests/tooling/pgoEvaluationWorkflow.test.ts ./tests/tooling/pgoEmulator.test.ts", "eval:proposal-police": "bun test --path-ignore-patterns='**/__never__/**' ./evals/proposalPolice/", "server:vcr:build:linux": "npm run build:linux -w @expensify/victory-chart-renderer --", "server:vcr:build:linux-arm": "npm run build:linux-arm -w @expensify/victory-chart-renderer --", diff --git a/patches/react-native/details.md b/patches/react-native/details.md index 2b18be31ad06..506e3f165e15 100644 --- a/patches/react-native/details.md +++ b/patches/react-native/details.md @@ -397,3 +397,9 @@ - E/App issue: https://github.com/Expensify/App/issues/103369 - PR introducing patch: https://github.com/Expensify/App/pull/103447 - 0.86.0 migration note: the adopted release containing `3f553d7` is not enough to drop this patch. `0.87.0` still throws here and already has `checkNotNull(parentView)` right before this check. `0.88` soft-fails a non-`ViewGroup` parent but keeps that `checkNotNull`, so a null `parentView` (the placeholder `ViewState` from [react-native#58526](https://github.com/react/react-native/issues/58526)) still crashes. On the 0.88 upgrade, reduce it to soft-failing the `checkNotNull` only. Drop it once upstream handles a null `parentView` in `removeViewAt`. + +### [react-native+0.86.0+048+pgo-emulator-abi.patch](react-native+0.86.0+048+pgo-emulator-abi.patch) + +- Reason: Select the profiling runtime for the configured Android ABI so the PGO build switches also work on x86_64 emulators. ARM64 device builds retain the aarch64 runtime. +- E/App issue: https://github.com/Expensify/App/issues/98008 +- PR introducing patch: https://github.com/margelo/Expensify/pull/34 diff --git a/patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch b/patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch new file mode 100644 index 000000000000..7322b2232e4f --- /dev/null +++ b/patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch @@ -0,0 +1,38 @@ +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +--- a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +@@ -4,7 +4,14 @@ + OUTPUT_VARIABLE CLANG_RESOURCE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +- set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(ANDROID_ABI STREQUAL "arm64-v8a") ++ set(PGO_ARCH aarch64) ++ elseif(ANDROID_ABI STREQUAL "x86_64") ++ set(PGO_ARCH x86_64) ++ else() ++ message(FATAL_ERROR "Unsupported PGO ABI: ${ANDROID_ABI}") ++ endif() ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-${PGO_ARCH}-android.a") + if(NOT EXISTS "${PGO_RUNTIME}") + message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") + endif() +diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +--- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt ++++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +@@ -14,7 +14,14 @@ + OUTPUT_VARIABLE CLANG_RESOURCE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +- set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(ANDROID_ABI STREQUAL "arm64-v8a") ++ set(PGO_ARCH aarch64) ++ elseif(ANDROID_ABI STREQUAL "x86_64") ++ set(PGO_ARCH x86_64) ++ else() ++ message(FATAL_ERROR "Unsupported PGO ABI: ${ANDROID_ABI}") ++ endif() ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-${PGO_ARCH}-android.a") + if(NOT EXISTS "${PGO_RUNTIME}") + message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") + endif() diff --git a/scripts/pgo/EVALUATION.md b/scripts/pgo/EVALUATION.md index f4b8005a0416..2fe1d46fbdab 100644 --- a/scripts/pgo/EVALUATION.md +++ b/scripts/pgo/EVALUATION.md @@ -1,83 +1,34 @@ -# Native PGO evaluation +# Verify native PGO in CI -The evaluation workflow measures a native candidate without selecting a release artifact or changing deployment. It records the actual App checkout, Mobile gitlink, version and producer attempt before evaluating each platform whose native build succeeded. A failed web deployment does not invalidate an otherwise successful native build. +This workflow builds baseline and instrumented HybridApps, runs a bundled synthetic workload three times, merges the native profiles, and rebuilds with PGO. It then runs both baseline and optimized apps on disposable Android and iOS emulators. The workload requires no test account or API credentials. -Each platform uses a dedicated physical runner. Its concurrency queue preserves up to 100 pending evaluations instead of replacing the previous pending candidate. Monitor the queue and rerun candidates if that limit is exceeded. Configure the `pgo-android` and `pgo-ios` runner labels and these settings before enabling device jobs: +## Run the combined PR -| Variable | Required runner configuration | -| --- | --- | -| `PGO_ANDROID_DEVICE_ID` | The approved physical Android serial | -| `PGO_ANDROID_APP_ID` | The dedicated evaluation application identifier | -| `PGO_ANDROID_FIXTURE_PATH` | An absolute path to the privately provisioned approved account fixture | -| `PGO_ANDROID_ENVFILE` | An absolute path to the fixed merged Android Release environment file | -| `PGO_IOS_DEVICE_ID` | The approved physical iPhone UDID | -| `PGO_IOS_APP_ID` | The dedicated evaluation bundle identifier | -| `PGO_IOS_FIXTURE_PATH` | An absolute path to the privately provisioned approved account fixture | -| `PGO_IOS_ENVFILE` | An absolute path to the fixed merged iOS Release environment file | -| `IOS_CODE_SIGNING_ALLOWED` | `true` for physical-device iOS evaluation | -| `IOS_DEVELOPMENT_TEAM` | The authorized Apple development team identifier | +You need permission to push branches to Expensify/App and Expensify/Mobile-Expensify. -The iOS runner also needs an installed development certificate and a provisioning profile that permits the selected physical iPhone and dedicated bundle identifier. Configure the signing variables as repository variables. Keep certificate material and signing credentials private on the runner. +1. In **Expensify/App → Settings → Secrets and variables → Actions**, confirm these existing repository secrets are available: -Android CI signing uses a private keystore on the runner. Set repository secrets `PGO_ANDROID_KEYSTORE_PATH`, `PGO_ANDROID_KEYSTORE_PASSWORD`, `PGO_ANDROID_KEY_ALIAS` and `PGO_ANDROID_KEY_PASSWORD`. The path must be absolute and outside the checkout so cleanup does not delete the key. The evaluation build uses this signing configuration without changing tracked native files. When no keystore path is supplied, local builds retain their existing signing configuration. Keep the same key across runs so updates preserve the signed-in app's data. - -The configured application and bundle identifiers must match the identifiers produced by the candidate's native configuration, environment and provisioning profile. These selectors do not rewrite native identifiers. A dedicated physical runner can use an existing matching identifier. - -The account must already be signed in to the dedicated application. Authentication is never automated. The fixture must explicitly authorize the heavy account and any training messages to its personal chat. Other reports are read-only. Keep account fixtures, environment files and signing credentials on the protected runner. Store device serials and private fixture/environment paths in repository secrets so the Actions environment banner masks them. Application identifiers and nonsecret signing settings can use repository variables. Do not store fixture or environment contents in variables or artifacts. - -The runner builds a baseline and instrumented Release, verifies instrumentation, then collects three complete journeys. It validates the archived profiles before building the optimized candidate. Android uses R8 while disabling FullStory build processing and upload phases. The runtime SDK may remain present. These dedicated evaluation builds are not store-equivalent releases. - -The runner records twenty samples per binary for heavy report navigation, separate warm navigation and cold startup. It repeats each workload with the opposite binary order. Each binary gets a five-second cool-down followed by one discarded warm-up before the measured samples. Thermal state is not verified. Warm navigation paths overlap training, and the heavy report first-layout span may stop at a skeleton rather than fully loaded content. - -Android frame telemetry is optional. Reports retain available native janky-frame counts and explicitly count unavailable observations. Those counters cover automation windows, with unverified reset and isolation boundaries. They do not establish isolated scrolling performance. No iOS frame metric is reported. - -Before and after each accepted stage, the runner checks the candidate commits, evaluated source contents, toolchain, fixed environment, fixture, profile and artifact hashes. The reviewed measurement code is an explicit overlay on the candidate, with its own hash. For iOS, generated CocoaPods lock files must match across all three build modes. The runner retains those locks privately and restores only the original tracked lockfile bytes before checking source consistency. - -Public artifacts contain numeric observations, distributions and hashes. Raw device output, screenshots, fixtures, environment values and detailed errors remain private on the runner. Private evidence lives under the candidate's ignored evaluation directory and needs a runner retention policy. Compiler consumption remains unverified in the public result unless execution evidence is independently established. - -Automatic evaluations follow completed trusted staging or production deploy runs. A manual rerun selects a producer run ID and its current immutable attempt. The exact approved same-repository preview branch can run before merge. Forks and arbitrary heads cannot access device jobs. Preview reports stay in workflow summaries and artifacts. A deployment report can update only its version-matching labelled historical checklist; otherwise the workflow visibly skips the comment. Evaluation never promotes or replaces the ordinary native artifact. - -## Run the combined flow before merging - -The combined verification branch contains the App foundation, evaluation scripts and CI workflow. Its `Mobile-Expensify` gitlink pins the native PGO hooks together with the native revision from App main. The smaller review PRs remain separate. Use this branch to verify the complete report-only flow without starting a deployment or merging the prerequisites first. - -1. Set up protected self-hosted runners with labels `pgo-android` and `pgo-ios` in Expensify/App. Connect one authorized physical arm64 Android device and one trusted, unlocked physical iPhone. Reserve each device for its evaluation job. Android needs the repository's JDK, SDK and NDK, with `ANDROID_HOME` and `ANDROID_NDK_HOME` available to the runner service. iOS needs macOS, the repository's Xcode and Ruby toolchain, Bundler, CocoaPods, development signing for the app and extensions, and a signed `agent-device` XCTest runner. Pin `agent-device` to version `0.20.6` on both runners and expose it on the service's `PATH`. -2. Complete the repository's normal HybridApp environment setup on each runner, including private dependency credentials. Keep the approved fixture, fixed merged Release environment file and signing material outside the checkout. Set `STANDALONE_NEW_DOT=false`. The workflow installs both App and Mobile-Expensify dependencies; the PGO builds select HybridApp and build React Native and Hermes from source through Rock. -3. In **Expensify/App → Settings → Secrets and variables → Actions**, configure the secrets and variables below. These are repository settings; this workflow does not select a GitHub environment, so environment-only secrets are insufficient. Enable the approved workflow actions and grant the runners access to this repository. - - | Repository secrets | Value | - | --- | --- | - | `OS_BOTIFY_TOKEN` | A token authorized to read App and the private Mobile-Expensify repository, used for the recursive checkout | - | `PGO_ANDROID_DEVICE_ID`, `PGO_IOS_DEVICE_ID` | The corresponding physical device identifiers | - | `PGO_ANDROID_FIXTURE_PATH`, `PGO_IOS_FIXTURE_PATH` | Absolute paths to the approved JSON fixtures on the corresponding runners | - | `PGO_ANDROID_ENVFILE`, `PGO_IOS_ENVFILE` | Absolute paths to the fixed merged Release environment files | - | `PGO_ANDROID_KEYSTORE_PATH` | Absolute path to the Android evaluation keystore | - | `PGO_ANDROID_KEYSTORE_PASSWORD`, `PGO_ANDROID_KEY_ALIAS`, `PGO_ANDROID_KEY_PASSWORD` | Credentials for that evaluation key | - - | Repository variables | Value | + | Secret | Purpose | | --- | --- | - | `PGO_ANDROID_APP_ID` | The application ID produced by the checked-in Android Release configuration | - | `PGO_IOS_APP_ID` | The bundle ID produced by the checked-in iOS Release configuration | - | `IOS_CODE_SIGNING_ALLOWED` | `true` | - | `IOS_DEVELOPMENT_TEAM` | The Apple team authorized for the app, extensions and physical iPhone | - - The workflow's `GITHUB_TOKEN` supplies its own App metadata, artifact and issue-comment permissions. No additional reporting token is needed. Do not put fixture contents, environment contents or certificate material into these path settings. + | `OS_BOTIFY_TOKEN` | Read App and the private Mobile-Expensify repository | + | `MAPBOX_SDK_DOWNLOAD_TOKEN` | Download the native Mapbox SDK | -4. Install an app with the same identifier and signing identity on each dedicated device and sign into the approved high-traffic account manually. Follow [the fixture setup](./JOURNEY.md#account-and-device-setup) and set `allowMessages: true` only with the owner's approval. Training sends two test messages per journey to that account's personal chat. The fixture's report must contain enough content to scroll. Keep the account signed in between builds; the runner installs updates without clearing app data. If `SIGN_IN_REQUIRED` appears, sign in on the physical device and rerun the failed job. -5. Copy both combined branch tips into the Expensify repositories without squashing or cherry-picking them. The App gitlink must continue to point to the exact native commit. From a clone of Expensify/App with its native submodule available, run: +2. Check out the combined branch and run its copy script. The script copies the pinned native commit first, then pushes the App branch. It preserves both commits and their signatures. ```bash - git -C Mobile-Expensify fetch https://github.com/margelo/Mobile-Expensify.git @chrispader/perf/pgo-ci-verification - git -C Mobile-Expensify push git@github.com:Expensify/Mobile-Expensify.git FETCH_HEAD:refs/heads/@chrispader/perf/pgo-ci-verification - git fetch https://github.com/margelo/Expensify.git @chrispader/perf/pgo-ci-verification - git push git@github.com:Expensify/App.git FETCH_HEAD:refs/heads/@chrispader/perf/pgo-ci-verification + git clone --single-branch --branch @chrispader/perf/pgo-ci-verification git@github.com:margelo/Expensify.git expensify-pgo-ci + cd expensify-pgo-ci + ./scripts/pgo/copy-ci-preview.sh ``` - Preserve an existing upstream branch before replacing it. The fork PR intentionally skips device jobs. Copying the native branch makes the pinned native commit fetchable from the private upstream repository; it does not merge either PR. +3. Open **Expensify/App → Actions → Verify native PGO on emulators**. The App push starts both jobs automatically, before the workflow is merged into main. Confirm both jobs pass and download `pgo-emulator-android` and `pgo-emulator-ios` for their reports and profiles. Use **Re-run jobs** to repeat the run. + +The workflow installs the toolchains, creates the devices, uses Android's debug signing key, and disables iOS signing. You do not need device IDs, signing certificates, environment files, an AWS account, or a local dependency install. New pushes to the copied branch start another run. The Margelo fork skips the native jobs because it does not own Expensify's private dependency access. + +Each report records the App and native commits, artifact hashes, profile counts, and successful synthetic workload samples. A failed build, incomplete workload, or invalid profile fails the job and leaves the available evidence in its artifact. -6. Open a PR **within Expensify/App**, from `@chrispader/perf/pgo-ci-verification` to `main`. That exact branch name is required by both the workflow and candidate resolver. The `opened`, `synchronize` and `reopened` events run **Evaluate native PGO candidate** from the PR head. After reviewing the copied code, allow its jobs onto the protected runners. A new commit or closing and reopening this same-repository PR triggers another attempt. No staging or production deployment is needed for this preview. -7. Open the run and verify `prepare`, both `evaluate` matrix jobs, and `report`. Each device job builds baseline and instrumented binaries, verifies native instrumentation, collects three complete training journeys, validates and merges their profiles, builds the optimized binary, and compares both binaries in both build orders. Inspect `pgo-evaluation-plan`, `pgo-evaluation-android`, `pgo-evaluation-ios` and `pgo-evaluation-report`. Confirm the plan matches the App head and native gitlink, both platform results report `status: passed`, and their samples cover startup, warm navigation and the heavy report. Inspect the private runner logs for unexpected JavaScript errors. A green report job alone does not mean device evaluation succeeded; failed platform results remain visible in the report. Preview runs publish artifacts and a summary without commenting on a deployment checklist. +## Interpret the result -`workflow_run` and the manual **Run workflow** entry point require the workflow to exist on the default branch. They are separate from the PR trigger above. After the reviewed workflow and prerequisites land upstream, a new native staging or production deployment records its candidate metadata and starts evaluation automatically. To repeat one, select its completed deployment run ID as `producer_run_id` and choose `both`, `android` or `ios`. Older deployments without the `pgo-native-candidate-` artifact cannot be evaluated through that entry point. Results attach only to the matching deployment checklist, and evaluation never changes the binary selected for deployment. +Passing verifies the native profile generation, collection, merge, and optimized rebuild on the emulator target. The bundled workload renders synthetic report and expense lists instead of loading the signed-in Expensify app. Its timings do not establish phone performance, and its profiles must not be used for device releases. The workflow does not change deployments or choose a production binary. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md index 868d86a0aa0e..ba8a1538647a 100644 --- a/scripts/pgo/README.md +++ b/scripts/pgo/README.md @@ -2,6 +2,8 @@ +For the hosted emulator workflow and the Expensify engineer handoff, see [Verify native PGO in CI](./EVALUATION.md). + This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. The command enables `ManualAppStartup` and the existing Spend/Inbox navigation spans in every app it builds. Startup profile collection waits for `ManualAppStartup` to finish, then explicitly writes the LLVM counters before the app process is stopped. Both benchmark modes use app-defined span durations rather than device-automation wall time. @@ -142,7 +144,7 @@ A startup-only profile is useful, but it tends to overfit initialization and can The current `record-startups` command accumulates and merges the startup runs automatically. For a mixed CI profile, preserve each scenario's output independently, convert it to its own indexed profile, then combine the scenario profiles with `llvm-profdata merge --weighted-input`. Choose weights from measured production flow frequency and performance impact, not from a default startup-to-interaction ratio. A giant tour of every feature is less useful than a stable suite that reflects real traffic. -## A realistic CI design +## Collect profiles for device releases Run profile generation as a scheduled or release-candidate job, not on every pull request. Use pinned self-hosted runners or a device farm, preferably with physical arm64 devices, the production compiler versions, stable thermal and power conditions, a seeded account, and deterministic local fixtures where possible. Build one instrumented artifact per platform, run the fixed startup and interaction suites against that exact artifact, flush after every successful scenario, merge the separate outputs with documented weights, then build the optimized artifact without changing the checkout or toolchain. diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index e0d926666707..a8671484d6ed 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -1,4 +1,4 @@ -// cspell:ignore sdcard profdata Ppatched Ppgo profraw Readelf readelf zipinfo libreactnative libhermesvm libjsi cnts +// cspell:ignore Dorg jvmargs Metaspace sdcard profdata Ppatched Ppgo profraw Readelf readelf zipinfo libreactnative libhermesvm libjsi cnts /** Android-specific release builds, native profile persistence, retrieval, and instrumentation checks. */ @@ -18,8 +18,13 @@ const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; const INSTRUMENTED_LIBRARIES = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { + const architecture = process.env.PGO_ANDROID_ARCH ?? 'arm64-v8a'; + if (architecture !== 'arm64-v8a' && architecture !== 'x86_64') { + fail('PGO_ANDROID_ARCH must be arm64-v8a or x86_64.'); + } + const emulatorVerification = process.env.PGO_CI_EMULATOR === 'true'; const releasePipeline = androidReleasePipeline(process.env.PGO_ANDROID_RELEASE_PIPELINE, process.env.PGO_ANDROID_SHRINK); - const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); + const profileDirectory = join(rootDirectory, '.pgo/android', architecture); const rawProfileDirectory = join(profileDirectory, 'raw'); const apkDirectory = join(profileDirectory, releasePipeline.artifactDirectory); const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); @@ -88,9 +93,12 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const gradleArguments = [ releasePipeline.task, '-PpatchedArtifacts.forceBuildFromSource=true', - '-PreactNativeArchitectures=arm64-v8a', + `-PreactNativeArchitectures=${architecture}`, `-PpgoMode=${pgoModes[kind]}`, ...releasePipeline.arguments, + ...(emulatorVerification + ? ['--init-script', join(rootDirectory, 'scripts/pgo/androidEmulator.init.gradle'), '--max-workers=2', '-Dorg.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1024m'] + : []), ]; if (releasePipeline.enabled) { console.log( @@ -123,7 +131,7 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent archiveReleaseApk(artifactPaths[kind]); writeFileSync( `${artifactPaths[kind]}.build-settings.json`, - `${JSON.stringify({kind, releasePipeline: releasePipeline.enabled, appID: appID(), sourceBuiltRuntime: true, shrinking: releasePipeline.shrinking, fullStory: releasePipeline.enabled, signing: process.env.PGO_ANDROID_KEYSTORE_PATH ? 'dedicated-pgo-keystore' : 'existing-project-configuration', storeEquivalent: false, gradleArguments, buildEnvironment}, null, 2)}\n`, + `${JSON.stringify({kind, architecture, emulatorVerification, releasePipeline: releasePipeline.enabled, appID: appID(), sourceBuiltRuntime: true, shrinking: releasePipeline.shrinking, fullStory: releasePipeline.enabled, signing: emulatorVerification ? 'debug-keystore' : 'existing-project-configuration', storeEquivalent: false, gradleArguments, buildEnvironment}, null, 2)}\n`, ); } @@ -191,9 +199,9 @@ function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdent const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); try { for (const library of INSTRUMENTED_LIBRARIES) { - const apkEntry = `lib/arm64-v8a/${library}`; + const apkEntry = `lib/${architecture}/${library}`; if (!apkEntries.has(apkEntry)) { - fail(`Missing expected arm64 library in APK: ${apkEntry}`); + fail(`Missing expected ${architecture} library in APK: ${apkEntry}`); } const extractedLibrary = join(extractedDirectory, library); writeFileSync(extractedLibrary, captureBinary('unzip', ['-p', apkPath, apkEntry])); diff --git a/scripts/pgo/androidEmulator.init.gradle b/scripts/pgo/androidEmulator.init.gradle new file mode 100644 index 000000000000..1b68380e2387 --- /dev/null +++ b/scripts/pgo/androidEmulator.init.gradle @@ -0,0 +1,18 @@ +// Disposable verification builds use the bundled workload and the Android debug key. +gradle.beforeProject { application -> + application.plugins.withId('com.facebook.react') { + application.extensions.getByName('react').entryFile.set( + application.file('../../src/PgoVerificationApp.tsx') + ) + } + application.plugins.withId('com.android.application') { + application.extensions.getByName('androidComponents').finalizeDsl { android -> + android.buildTypes.getByName('release').signingConfig = android.signingConfigs.getByName('debug') + } + } + application.tasks.configureEach { task -> + if (task.name.toLowerCase().startsWith('fullstory')) { + task.enabled = false + } + } +} diff --git a/scripts/pgo/androidRelease.init.gradle b/scripts/pgo/androidRelease.init.gradle index 75933f894099..ed3bdaf1c414 100644 --- a/scripts/pgo/androidRelease.init.gradle +++ b/scripts/pgo/androidRelease.init.gradle @@ -1,4 +1,4 @@ -// Apply release processing with optional dedicated CI signing, preserving the application ID. +// Apply release processing without changing the owner's local application ID or signing configuration. gradle.beforeProject { application -> def skipFullStory = application.providers.gradleProperty('pgoSkipFullStory').getOrElse('false') == 'true' if (skipFullStory) { @@ -16,22 +16,7 @@ gradle.beforeProject { application -> } release.minifyEnabled = true release.proguardFiles android.getDefaultProguardFile('proguard-android-optimize.txt'), application.file('proguard-rules.pro') - def keystorePath = System.getenv('PGO_ANDROID_KEYSTORE_PATH') - if (keystorePath) { - def storePassword = System.getenv('PGO_ANDROID_KEYSTORE_PASSWORD') - def keyAlias = System.getenv('PGO_ANDROID_KEY_ALIAS') - def keyPassword = System.getenv('PGO_ANDROID_KEY_PASSWORD') - if (!storePassword || !keyAlias || !keyPassword) { - throw new GradleException('Dedicated PGO signing requires all Android keystore settings.') - } - def signing = android.signingConfigs.maybeCreate('pgoEvaluation') - signing.storeFile = application.file(keystorePath) - signing.storePassword = storePassword - signing.keyAlias = keyAlias - signing.keyPassword = keyPassword - release.signingConfig = signing - } - application.logger.lifecycle("PGO shrinking enabled; FullStory tasks ${skipFullStory ? 'disabled' : 'enabled'}; application ID preserved.") + application.logger.lifecycle("PGO shrinking enabled; FullStory tasks ${skipFullStory ? 'disabled' : 'enabled'}; existing signing and application ID preserved.") } } } diff --git a/scripts/pgo/copy-ci-preview.sh b/scripts/pgo/copy-ci-preview.sh new file mode 100755 index 000000000000..e1c10a04495d --- /dev/null +++ b/scripts/pgo/copy-ci-preview.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# cspell:ignore chrispader +set -euo pipefail + +# Push the pinned native revision first so the App push can start CI immediately. +branch='@chrispader/perf/pgo-ci-verification' +repository=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT + +app_sha=$(git -C "$repository" rev-parse HEAD) +mobile_sha=$(git -C "$repository" ls-tree HEAD Mobile-Expensify | awk '{print $3}') +test -n "$mobile_sha" || { echo 'Run this script from the combined PGO PR branch.' >&2; exit 1; } + +git init --quiet --bare "$temporary/native.git" +git -C "$temporary/native.git" fetch --no-tags git@github.com:margelo/Mobile-Expensify.git "$mobile_sha" +test "$(git -C "$temporary/native.git" rev-parse FETCH_HEAD)" = "$mobile_sha" +git -C "$temporary/native.git" push git@github.com:Expensify/Mobile-Expensify.git "FETCH_HEAD:refs/heads/$branch" +git -C "$repository" push git@github.com:Expensify/App.git "$app_sha:refs/heads/$branch" + +echo 'Open Expensify/App > Actions > Verify native PGO on emulators to see the run.' diff --git a/scripts/pgo/emulator.ts b/scripts/pgo/emulator.ts new file mode 100644 index 000000000000..42479101dd44 --- /dev/null +++ b/scripts/pgo/emulator.ts @@ -0,0 +1,166 @@ +// cspell:ignore sdcard profdata profraw simctl +import {appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync} from 'node:fs'; +import {join} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; +import type {VerificationMarker} from './emulatorFixture'; +import type {PlatformAdapter} from './shared'; + +import {capture, fail, findFiles, parseChoice, rootDirectory, run, runAllowFailure, sleep} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import {parseVerificationMarker, VERIFICATION_MARKER} from './emulatorFixture'; +import createIOSPgoAdapter from './ios'; +import {bootIOSVerificationSimulator, createIOSSimulatorDevice} from './iosSimulator'; +import {fileHash, profileCounts} from './proof'; + +/** Verify the real native PGO build cycle with a self-running synthetic workload. */ +async function verifyEmulator(platform: PlatformName): Promise { + const output = join(rootDirectory, '.pgo/ci', platform); + rmSync(output, {recursive: true, force: true}); + mkdirSync(output, {recursive: true}); + process.env.PGO_CI_EMULATOR = 'true'; + process.env.PGO_ANDROID_RELEASE_PIPELINE = 'false'; + process.env.PGO_ANDROID_SHRINK = 'false'; + let simulatorIdentifier: string | undefined; + try { + if (platform === 'android') { + process.env.PGO_ANDROID_ARCH = 'x86_64'; + } else { + simulatorIdentifier = bootIOSVerificationSimulator(); + process.env.PGO_IOS_SIMULATOR_ID = simulatorIdentifier; + } + const identifier = simulatorIdentifier ?? androidEmulatorIdentifier(); + const adapter = platform === 'android' ? createAndroidPgoAdapter(undefined, identifier) : createIOSPgoAdapter(); + + adapter.build('release'); + adapter.build('instrumented'); + adapter.verifyInstrumentation(); + adapter.install('instrumented'); + const workload = createEmulatorWorkload(adapter, identifier); + await workload(); + for (let repetition = 1; repetition <= 3; repetition++) { + await adapter.clearDeviceProfiles(); + await workload(); + await adapter.dumpProfiles(); + adapter.pullProfiles(); + cpSync(adapter.rawProfileDirectory, join(output, 'raw', String(repetition)), {recursive: true}); + } + + const profiles = findFiles(join(output, 'raw'), '.profraw').sort(); + if (profiles.length < 3) { + fail('Each emulator training repetition must produce a native profile.'); + } + const llvm = adapter.llvmTool('llvm-profdata'); + run(llvm, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + if (adapter.profileFormat) { + writeFileSync(`${adapter.mergedProfilePath}.format`, `${adapter.profileFormat}\n`); + } + const profileReport = capture(llvm, ['show', '--all-functions', '--counts', adapter.mergedProfilePath]); + const counts = profileCounts(profileReport); + writeFileSync(join(output, 'profile.txt'), profileReport); + cpSync(adapter.mergedProfilePath, join(output, 'verification.profdata')); + const profileHash = fileHash(adapter.mergedProfilePath); + adapter.build('optimized', adapter.mergedProfilePath); + if (fileHash(adapter.mergedProfilePath) !== profileHash) { + fail('The training profile changed during the optimized build.'); + } + + const samples: Array<{kind: 'release' | 'optimized'; workload: VerificationMarker}> = []; + for (const kind of ['release', 'optimized', 'optimized', 'release'] as const) { + adapter.install(kind); + samples.push({kind, workload: await workload()}); + } + writeFileSync( + join(output, 'result.json'), + `${JSON.stringify( + { + schema: 1, + status: 'passed', + platform, + target: platform === 'android' ? 'android-emulator-x86_64' : `ios-simulator-${process.arch}`, + workload: 'synthetic', + deployable: false, + appSHA: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileSHA: capture('git', ['rev-parse', 'HEAD'], join(rootDirectory, 'Mobile-Expensify')).trim(), + profile: {sha256: profileHash, ...counts}, + artifacts: Object.fromEntries(Object.entries(adapter.artifactPaths).map(([kind, path]) => [kind, fileHash(path)])), + samples, + }, + null, + 2, + )}\n`, + ); + publishSummary( + `Native PGO ${platform} emulator verification passed. Baseline, instrumentation, three training runs, profile merge and optimized rebuild completed. Both builds completed the synthetic workload. These samples do not establish phone performance or select a deployment artifact.\n`, + output, + ); + } catch (error) { + writeFileSync(join(output, 'result.json'), `${JSON.stringify({schema: 1, status: 'failed', platform, workload: 'synthetic', deployable: false}, null, 2)}\n`); + publishSummary(`Native PGO ${platform} emulator verification failed. Open the failed step's log for the build or workload error.\n`, output); + throw error; + } finally { + if (simulatorIdentifier) { + runAllowFailure('xcrun', ['simctl', 'shutdown', simulatorIdentifier]); + runAllowFailure('xcrun', ['simctl', 'delete', simulatorIdentifier]); + } + } +} + +function createEmulatorWorkload(adapter: PlatformAdapter, identifier: string): () => Promise { + const simulator = adapter.name === 'ios' ? createIOSSimulatorDevice(identifier, adapter.appID) : undefined; + const marker = `/sdcard/Android/data/${adapter.appID()}/cache/${VERIFICATION_MARKER}`; + const adb = (args: string[]) => run('adb', ['-s', identifier, ...args]); + return async () => { + if (simulator) { + simulator.removeFile(`Library/Caches/${VERIFICATION_MARKER}`); + simulator.launch(); + } else { + adb(['shell', 'am', 'force-stop', adapter.appID()]); + adb(['shell', 'rm', '-f', marker]); + adb(['shell', 'monkey', '-p', adapter.appID(), '-c', 'android.intent.category.LAUNCHER', '1']); + } + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + let text: string | undefined; + if (simulator) { + text = simulator.readFile(`Library/Caches/${VERIFICATION_MARKER}`); + } else if (runAllowFailure('adb', ['-s', identifier, 'shell', 'test', '-f', marker])) { + text = capture('adb', ['-s', identifier, 'shell', 'cat', marker]); + } + if (text) { + const input: unknown = JSON.parse(text); + return parseVerificationMarker(input); + } + await sleep(500); + } + fail('The bundled emulator workload did not finish within 90 seconds.'); + }; +} + +function androidEmulatorIdentifier(): string { + const devices = capture('adb', ['devices']) + .split('\n') + .flatMap((line) => { + const identifier = /^(emulator-\d+)\s+device$/.exec(line.trim())?.[1]; + return identifier ? [identifier] : []; + }); + const identifier = devices.length === 1 ? devices.at(0) : undefined; + if (!identifier || capture('adb', ['-s', identifier, 'shell', 'getprop', 'ro.product.cpu.abi']).trim() !== 'x86_64') { + fail('PGO CI requires one booted x86_64 Android emulator.'); + } + return identifier; +} + +function publishSummary(text: string, output: string): void { + writeFileSync(join(output, 'summary.md'), text); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); + } +} + +if (import.meta.main) { + const platform = parseChoice(process.argv.at(2) ?? '', ['android', 'ios'], 'Emulator platform'); + await verifyEmulator(platform); +} + +export default verifyEmulator; diff --git a/scripts/pgo/emulatorFixture.ts b/scripts/pgo/emulatorFixture.ts new file mode 100644 index 000000000000..1cd4a2ab4a9d --- /dev/null +++ b/scripts/pgo/emulatorFixture.ts @@ -0,0 +1,34 @@ +import {isRecord} from '@libs/ObjectUtils'; + +const VERIFICATION_SCREENS = ['Inbox', 'Report', 'Expenses', 'Search', 'Inbox'] as const; +const VERIFICATION_ROW_COUNT = 240; +const VERIFICATION_MARKER = 'pgo-ci-workload.json'; +type VerificationMarker = {schema: 1; status: 'passed'; workload: 'synthetic'; screens: number; rows: number; durationMs: number}; + +function parseVerificationMarker(input: unknown): VerificationMarker { + if ( + !isRecord(input) || + input.schema !== 1 || + input.status !== 'passed' || + input.workload !== 'synthetic' || + input.screens !== VERIFICATION_SCREENS.length || + input.rows !== VERIFICATION_ROW_COUNT || + typeof input.durationMs !== 'number' || + !Number.isFinite(input.durationMs) || + input.durationMs <= 0 + ) { + throw new Error('The emulator workload did not complete every synthetic screen.'); + } + return {schema: 1, status: 'passed', workload: 'synthetic', screens: input.screens, rows: input.rows, durationMs: input.durationMs}; +} + +function verificationRows(screen: number) { + return Array.from({length: VERIFICATION_ROW_COUNT}, (_, index) => ({ + id: index + 1, + title: `CI report ${index + 1}`, + amount: ((index * 173 + screen * 31) % 10000) / 100, + })).sort((a, b) => (screen % 2 === 0 ? a.id - b.id : b.amount - a.amount)); +} + +export {parseVerificationMarker, verificationRows, VERIFICATION_MARKER, VERIFICATION_ROW_COUNT, VERIFICATION_SCREENS}; +export type {VerificationMarker}; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts index 204855eaca7c..af4c40e3a43c 100644 --- a/scripts/pgo/ios.ts +++ b/scripts/pgo/ios.ts @@ -11,11 +11,14 @@ import {dirname, join} from 'node:path'; import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter} from './shared'; import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllowFailure, sleep, valueAt} from '../lib/scriptUtils'; +import {createIOSSimulatorDevice} from './iosSimulator'; import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { - const profileFormat = 'ios-clang-swift-ir-v3'; - const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); + const simulatorIdentifier = environmentString('PGO_IOS_SIMULATOR_ID'); + const architecture = simulatorIdentifier ? process.arch : 'arm64'; + const profileFormat = `ios-clang-swift-ir-v3${simulatorIdentifier ? `-simulator-${architecture}` : ''}`; + const profileDirectory = join(rootDirectory, '.pgo/ios', simulatorIdentifier ? `simulator-${architecture}` : architecture); const rawProfileDirectory = join(profileDirectory, 'raw'); const appDirectory = join(profileDirectory, 'app'); const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); @@ -29,7 +32,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str const profileClearNotification = 'com.expensify.pgo.clear-profiles'; const profileOperationTimeoutSeconds = 10; const developmentTeam = environmentString('IOS_DEVELOPMENT_TEAM'); - const codeSigningAllowed = !['0', 'false', 'no'].includes((environmentString('IOS_CODE_SIGNING_ALLOWED') ?? 'yes').toLowerCase()); + const codeSigningAllowed = !simulatorIdentifier && !['0', 'false', 'no'].includes((environmentString('IOS_CODE_SIGNING_ALLOWED') ?? 'yes').toLowerCase()); const configuredBundleIdentifier = configuredAppID ?? environmentString('IOS_BUNDLE_IDENTIFIER'); const mergedProfilePath = join(profileDirectory, 'newdot.profdata'); @@ -45,6 +48,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str let cachedDeviceIdentifier: string | undefined; let cachedBundleIdentifier: string | undefined; + const simulator = simulatorIdentifier ? createIOSSimulatorDevice(simulatorIdentifier, bundleIdentifier) : undefined; function bundleIdentifier(): string { if (cachedBundleIdentifier) { @@ -68,6 +72,9 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str } function deviceIdentifier(): string { + if (simulatorIdentifier) { + return simulatorIdentifier; + } if (cachedDeviceIdentifier) { return cachedDeviceIdentifier; } @@ -220,7 +227,7 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str } prepareSourcePods(mode, profilePath); const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); - const builtAppPath = join(derivedDataDirectory, 'Build/Products/Release-iphoneos/Expensify.app'); + const builtAppPath = join(derivedDataDirectory, `Build/Products/Release-${simulator ? 'iphonesimulator' : 'iphoneos'}/Expensify.app`); const signingSettings = codeSigningAllowed ? [ 'CODE_SIGN_STYLE=Automatic', @@ -230,15 +237,20 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), ] : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; - const rockArguments = iosRockBuildArguments(derivedDataDirectory, [ - 'build', - 'ARCHS=arm64', - 'ONLY_ACTIVE_ARCH=YES', - 'COMPILER_INDEX_STORE_ENABLE=NO', - ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), - ...signingSettings, - ...pgoBuildSettings(kind, profilePath), - ]); + const rockArguments = iosRockBuildArguments( + derivedDataDirectory, + [ + 'build', + `ARCHS=${architecture}`, + 'ONLY_ACTIVE_ARCH=YES', + 'COMPILER_INDEX_STORE_ENABLE=NO', + ...(simulator ? ['-sdk', 'iphonesimulator', '-jobs', '2', `ENTRY_FILE=${join(rootDirectory, 'src/PgoVerificationApp.tsx')}`] : []), + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), + ...signingSettings, + ...pgoBuildSettings(kind, profilePath), + ], + simulator ? 'generic/platform=iOS Simulator' : 'generic/platform=iOS', + ); withLocalUploadPhasesDisabled(() => run( '/usr/bin/env', @@ -260,6 +272,10 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str mkdirSync(appDirectory, {recursive: true}); rmSync(artifactPaths[kind], {recursive: true, force: true}); cpSync(builtAppPath, artifactPaths[kind], {recursive: true, preserveTimestamps: true}); + if (simulator) { + // Hosted runners keep only the installable between the three source builds. + rmSync(derivedDataDirectory, {recursive: true, force: true}); + } console.log(`Copied release app: ${artifactPaths[kind]}`); } @@ -321,6 +337,10 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str if (!existsSync(appPath)) { fail(`Missing app at ${appPath}. Build it first.`); } + if (simulator) { + simulator.install(appPath); + return; + } if (!existsSync(join(appPath, '_CodeSignature/CodeResources'))) { fail(`The app at ${appPath} is unsigned. Rebuild without IOS_CODE_SIGNING_ALLOWED=NO before installing it on a device.`); } @@ -329,6 +349,9 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str } function tryReadDeviceFile(remotePath: string, localName: string): string | undefined { + if (simulator) { + return simulator.readFile(remotePath); + } mkdirSync(markerDirectory, {recursive: true}); const localPath = join(markerDirectory, localName); rmSync(localPath, {recursive: true, force: true}); @@ -353,6 +376,10 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str } function postNotification(name: string): void { + if (simulator) { + simulator.notify(name); + return; + } run('xcrun', ['devicectl', 'device', 'notification', 'post', '--device', deviceIdentifier(), '--name', name]); } @@ -392,6 +419,13 @@ function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: str function pullProfiles(): void { rmSync(rawProfileDirectory, {recursive: true, force: true}); mkdirSync(dirname(rawProfileDirectory), {recursive: true}); + if (simulator) { + simulator.copyDirectory(remotePgoDirectory, rawProfileDirectory); + if (findFiles(rawProfileDirectory, '.profraw').length === 0) { + fail('The simulator did not produce any native PGO profiles.'); + } + return; + } run('xcrun', [ 'devicectl', 'device', @@ -475,7 +509,7 @@ function replaceOnce(content: string, previous: string, next: string): string { } /** Preserve each Xcode setting as one argument through Rock's string argument parser. */ -function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { +function iosRockBuildArguments(derivedDataDirectory: string, settings: string[], destination = 'generic/platform=iOS'): string[] { return [ 'build:ios', '--local', @@ -485,7 +519,7 @@ function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]) '--scheme', 'Expensify', '--destination', - 'generic/platform=iOS', + destination, '--build-folder', derivedDataDirectory, '--no-install-pods', diff --git a/scripts/pgo/iosSimulator.ts b/scripts/pgo/iosSimulator.ts new file mode 100644 index 000000000000..4d5eb6ed7cbe --- /dev/null +++ b/scripts/pgo/iosSimulator.ts @@ -0,0 +1,60 @@ +import {isRecord} from '@libs/ObjectUtils'; + +// cspell:ignore bootstatus simctl notifyutil UDID +import {cpSync, existsSync, readFileSync, rmSync} from 'node:fs'; +import {join} from 'node:path'; + +import {capture, fail, run} from '../lib/scriptUtils'; + +/** Create a disposable simulator from the selected Xcode's installed iOS runtime. */ +function bootIOSVerificationSimulator(): string { + const input: unknown = JSON.parse(capture('xcrun', ['simctl', 'list', 'runtimes', '--json'])); + const runtime = selectIOSVerificationRuntime(input); + const identifier = capture('xcrun', ['simctl', 'create', 'PGO verification', 'com.apple.CoreSimulator.SimDeviceType.iPhone-16', runtime]).trim(); + if (!/^[a-f0-9-]{36}$/i.test(identifier)) { + fail('Xcode did not return a simulator identifier.'); + } + run('xcrun', ['simctl', 'boot', identifier]); + run('xcrun', ['simctl', 'bootstatus', identifier, '-b']); + return identifier; +} + +function selectIOSVerificationRuntime(input: unknown): string { + if (!isRecord(input) || !Array.isArray(input.runtimes)) { + fail('Xcode returned an invalid simulator runtime list.'); + } + const runtimes = input.runtimes.flatMap((item: unknown) => { + if ( + !isRecord(item) || + item.isAvailable !== true || + typeof item.identifier !== 'string' || + typeof item.version !== 'string' || + !item.identifier.startsWith('com.apple.CoreSimulator.SimRuntime.iOS-') + ) { + return []; + } + return [{identifier: item.identifier, version: item.version}]; + }); + const runtime = runtimes.sort((a, b) => b.version.localeCompare(a.version, undefined, {numeric: true})).at(0); + if (!runtime) { + fail('The selected Xcode has no available iOS simulator runtime.'); + } + return runtime.identifier; +} + +function createIOSSimulatorDevice(identifier: string, appID: () => string) { + const containerPath = () => capture('xcrun', ['simctl', 'get_app_container', identifier, appID(), 'data']).trim(); + return { + install: (path: string) => run('xcrun', ['simctl', 'install', identifier, path]), + launch: () => run('xcrun', ['simctl', 'launch', '--terminate-running-process', identifier, appID()]), + notify: (name: string) => run('xcrun', ['simctl', 'spawn', identifier, 'notifyutil', '-p', name]), + removeFile: (path: string) => rmSync(join(containerPath(), path), {force: true}), + readFile: (path: string): string | undefined => { + const source = join(containerPath(), path); + return existsSync(source) ? readFileSync(source, 'utf8').trim() : undefined; + }, + copyDirectory: (path: string, destination: string) => cpSync(join(containerPath(), path), destination, {recursive: true}), + }; +} + +export {bootIOSVerificationSimulator, createIOSSimulatorDevice, selectIOSVerificationRuntime}; diff --git a/src/PgoVerificationApp.tsx b/src/PgoVerificationApp.tsx new file mode 100644 index 000000000000..be6b2c891e76 --- /dev/null +++ b/src/PgoVerificationApp.tsx @@ -0,0 +1,78 @@ +import type {TurboModule} from 'react-native'; + +// This entry is selected only by the emulator verification build, never by index.js. +import React, {useEffect, useRef, useState} from 'react'; +// eslint-disable-next-line no-restricted-imports -- This isolated CI entry runs without application providers. +import {AppRegistry, Platform, ScrollView, StyleSheet, Text, TurboModuleRegistry, View} from 'react-native'; +import RNFS from 'react-native-fs'; + +import {verificationRows, VERIFICATION_MARKER, VERIFICATION_ROW_COUNT, VERIFICATION_SCREENS} from '../scripts/pgo/emulatorFixture'; + +type BootSplashModule = TurboModule & {hide: () => Promise}; +const BootSplash = TurboModuleRegistry.getEnforcing('BootSplash'); +const startedAt = performance.now(); + +const styles = StyleSheet.create({ + container: {flex: 1, padding: 24, paddingTop: 64, backgroundColor: '#ffffff'}, + heading: {marginBottom: 12}, + row: {paddingVertical: 12, flexDirection: 'row', justifyContent: 'space-between'}, +}); + +/** Exercise Hermes, native layout and file I/O using public synthetic data. */ +function VerificationApp() { + const [screen, setScreen] = useState(0); + const scheduledScreen = useRef(-1); + const rows = verificationRows(screen); + + useEffect(() => { + BootSplash.hide(); + }, []); + + const advance = () => { + if (screen <= scheduledScreen.current) { + return; + } + scheduledScreen.current = screen; + if (screen < VERIFICATION_SCREENS.length - 1) { + setTimeout(() => setScreen((previous) => previous + 1), 500); + return; + } + const directory = Platform.OS === 'android' ? RNFS.ExternalCachesDirectoryPath : RNFS.CachesDirectoryPath; + const temporaryPath = `${directory}/${VERIFICATION_MARKER}.tmp`; + RNFS.writeFile( + temporaryPath, + JSON.stringify({ + schema: 1, + status: 'passed', + workload: 'synthetic', + screens: VERIFICATION_SCREENS.length, + rows: VERIFICATION_ROW_COUNT, + durationMs: performance.now() - startedAt, + }), + 'utf8', + ).then(() => RNFS.moveFile(temporaryPath, `${directory}/${VERIFICATION_MARKER}`)); + }; + + return ( + + PGO CI verification + {VERIFICATION_SCREENS.at(screen)} + + {rows.map((row) => ( + + {row.title} + {row.amount.toFixed(2)} + + ))} + + + ); +} + +AppRegistry.registerComponent('NewExpensify', () => VerificationApp); diff --git a/tests/tooling/pgoEmulator.test.ts b/tests/tooling/pgoEmulator.test.ts new file mode 100644 index 000000000000..e40dad5b64c5 --- /dev/null +++ b/tests/tooling/pgoEmulator.test.ts @@ -0,0 +1,93 @@ +// cspell:ignore esac simctl profdata +import {afterAll, describe, expect, it} from 'bun:test'; + +import createAndroidPgoAdapter from '@scripts/pgo/android'; +import {parseVerificationMarker, verificationRows, VERIFICATION_ROW_COUNT, VERIFICATION_SCREENS} from '@scripts/pgo/emulatorFixture'; +import {selectIOSVerificationRuntime} from '@scripts/pgo/iosSimulator'; + +import {spawnSync} from 'node:child_process'; +import {mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; + +const directory = mkdtempSync(join(tmpdir(), 'pgo-emulator-tests-')); +afterAll(() => rmSync(directory, {recursive: true, force: true})); + +describe('PGO emulator verification', () => { + it('accepts only a completed synthetic workload with finite timing', () => { + // Given the synthetic workload's explicit completion marker. + const marker = {schema: 1, status: 'passed', workload: 'synthetic', screens: VERIFICATION_SCREENS.length, rows: VERIFICATION_ROW_COUNT, durationMs: 1200}; + // When complete, partial and malformed results arrive from the app. + expect(parseVerificationMarker(marker).durationMs).toBe(1200); + // Then an incomplete render, an unapproved workload or invalid timing fails verification. + for (const change of [{screens: 1}, {rows: 0}, {workload: 'live'}, {durationMs: Number.NaN}, {durationMs: Number.POSITIVE_INFINITY}, {durationMs: 0}]) { + expect(() => parseVerificationMarker({...marker, ...change})).toThrow('did not complete'); + } + expect(new Set(verificationRows(1).map((row) => row.id)).size).toBe(VERIFICATION_ROW_COUNT); + }); + + it('selects an installed iOS runtime without choosing an unavailable or non-iOS runtime', () => { + // Given runtime versions that require numeric ordering and an unavailable newer runtime. + const runtimes = [ + {identifier: 'com.apple.CoreSimulator.SimRuntime.iOS-26-9', version: '26.9', isAvailable: true}, + {identifier: 'com.apple.CoreSimulator.SimRuntime.iOS-26-10', version: '26.10', isAvailable: true}, + {identifier: 'com.apple.CoreSimulator.SimRuntime.iOS-27', version: '27', isAvailable: false}, + {identifier: 'com.apple.CoreSimulator.SimRuntime.watchOS-27', version: '27', isAvailable: true}, + ]; + // When a disposable simulator's runtime is selected. + const selected = selectIOSVerificationRuntime({runtimes}); + // Then the latest installed iOS runtime is selected and missing runtimes fail clearly. + expect(selected).toBe('com.apple.CoreSimulator.SimRuntime.iOS-26-10'); + expect(() => selectIOSVerificationRuntime({runtimes: []})).toThrow('no available iOS'); + }); + + it('isolates emulator profiles from physical Android profiles', () => { + // Given the hosted emulator architecture instead of the physical-device default. + const previous = process.env.PGO_ANDROID_ARCH; + try { + process.env.PGO_ANDROID_ARCH = 'x86_64'; + // When its platform adapter chooses profile paths. + const adapter = createAndroidPgoAdapter('com.example.pgo'); + // Then it cannot overwrite or consume the physical ARM64 profile by default. + expect(adapter.mergedProfilePath).toEndWith('/android/x86_64/newdot.profdata'); + process.env.PGO_ANDROID_ARCH = '../arm64-v8a'; + expect(() => createAndroidPgoAdapter()).toThrow('must be arm64-v8a or x86_64'); + } finally { + if (previous === undefined) { + delete process.env.PGO_ANDROID_ARCH; + } else { + process.env.PGO_ANDROID_ARCH = previous; + } + } + }); + + it.each([0, 1])('publishes App only after the pinned native push succeeds with status %i', (nativeStatus) => { + // Given a fake Git transport that records pushes without contacting GitHub. + const log = join(directory, `copy-${nativeStatus}.log`); + writeFileSync( + join(directory, 'git'), + `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$PGO_COPY_LOG" +case "$*" in + *'ls-tree HEAD Mobile-Expensify'*) echo '160000 commit bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Mobile-Expensify';; + *'rev-parse FETCH_HEAD'*) echo 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb';; + *'rev-parse HEAD'*) echo 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';; + *'push git@github.com:Expensify/Mobile-Expensify.git'*) exit "$PGO_COPY_NATIVE_STATUS";; +esac +`, + {mode: 0o755}, + ); + // When the engineer runs the actual copy helper. + const result = spawnSync('bash', [resolve('scripts/pgo/copy-ci-preview.sh')], { + encoding: 'utf8', + env: {...process.env, PATH: `${directory}:${process.env.PATH}`, PGO_COPY_LOG: log, PGO_COPY_NATIVE_STATUS: String(nativeStatus)}, + }); + const commands = readFileSync(log, 'utf8'); + // Then a failed native upload never starts an App workflow against a missing gitlink. + expect(result.status).toBe(nativeStatus); + expect(commands.includes('push git@github.com:Expensify/App.git')).toBe(nativeStatus === 0); + if (nativeStatus === 0) { + expect(commands.indexOf('push git@github.com:Expensify/Mobile-Expensify.git')).toBeLessThan(commands.indexOf('push git@github.com:Expensify/App.git')); + } + }); +}); diff --git a/tests/tooling/pgoEvaluationWorkflow.test.ts b/tests/tooling/pgoEvaluationWorkflow.test.ts index 28dc11349773..04594038f3b7 100644 --- a/tests/tooling/pgoEvaluationWorkflow.test.ts +++ b/tests/tooling/pgoEvaluationWorkflow.test.ts @@ -1,5 +1,4 @@ // cspell:ignore chrispader -/* eslint-disable @typescript-eslint/naming-convention -- Fixtures reproduce GitHub's event schema. */ import {describe, expect, it} from 'bun:test'; import {isRecord} from '@scripts/pgo/evaluationSchema'; @@ -7,124 +6,73 @@ import {isRecord} from '@scripts/pgo/evaluationSchema'; import {readFileSync} from 'node:fs'; import {runInNewContext} from 'node:vm'; -describe('PGO evaluation workflow contract', () => { +const input: unknown = Bun.YAML.parse(readFileSync('.github/workflows/pgoEvaluation.yml', 'utf8')); +if (!isRecord(input) || !isRecord(input.jobs) || !isRecord(input.jobs.verify)) { + throw new Error('Missing emulator verification workflow.'); +} +const workflow = input; +const job = input.jobs.verify; + +describe('PGO emulator workflow', () => { it.each([ - ['Expensify/App', '@chrispader/perf/pgo-ci-verification', true], - ['margelo/Expensify', '@chrispader/perf/pgo-ci-verification', false], - ['Expensify/App', 'other-preview', false], - ])('gates physical-device access for %s on %s', (repository, branch, eligible) => { - // Given the actual prepare-job condition and a pull-request head. - const job = workflowJob('.github/workflows/pgoEvaluation.yml', 'prepare'); + ['Expensify/App', 'refs/heads/@chrispader/perf/pgo-ci-verification', true], + ['margelo/Expensify', 'refs/heads/@chrispader/perf/pgo-ci-verification', false], + ['Expensify/App', 'refs/heads/main', false], + ['Expensify/App', 'refs/pull/34/merge', false], + ])('limits private dependency access for %s at %s', (repository, ref, expected) => { + // Given the workflow's actual authorization expression. if (typeof job.if !== 'string') { - throw new Error('Missing preview authorization condition.'); + throw new Error('Missing authorization condition.'); } - const github = {repository: 'Expensify/App', event_name: 'pull_request', event: {pull_request: {head: {repo: {full_name: repository}, ref: branch}}}}; - // When the condition runs with the event metadata available to Actions. - const actual: unknown = runInNewContext(job.if, {github}); - // Then only the named upstream branch can request credential devices. - expect(actual).toBe(eligible); + // When Actions evaluates a repository and branch. + const result: unknown = runInNewContext(job.if, {github: {repository, ref}}); + // Then only the reviewed upstream preview branch receives credentials. + expect(result).toBe(expected); }); - it('keeps producer metadata failures outside canonical deployment eligibility', () => { - // Given the ordinary deploy workflow and its evaluation metadata additions. - const job = workflowJob('.github/workflows/deploy.yml', 'prep'); - // When both metadata steps are inspected. - const capture = workflowStep(job, 'Record native evaluation candidate'); - const upload = workflowStep(job, 'Upload native evaluation candidate metadata'); - // Then either failure is non-blocking and reruns retain separate attempt artifacts. - expect(capture?.['continue-on-error']).toBe(true); - expect(upload?.['continue-on-error']).toBe(true); - expect(isRecord(upload.with) && upload.with.name).toBe(expression('pgo-native-candidate-', 'github.run_attempt')); + it('starts on branch copy without requiring a deployment or a default-branch merge', () => { + // Given the verification workflow's event configuration. + const events = workflow.on; + // When its push trigger is inspected. + if (!isRecord(events) || !isRecord(events.push)) { + throw new Error('Missing push trigger.'); + } + // Then copying the branch starts verification and no deployment starts it. + expect(events.push.branches).toEqual(['@chrispader/perf/pgo-ci-verification']); + expect(events.workflow_run).toBeUndefined(); + expect(events.pull_request_target).toBeUndefined(); + expect(events.workflow_dispatch).toBeDefined(); }); - it('schedules eligible platforms only and masks private device configuration', () => { - // Given the standalone evaluation workflow. - const job = workflowJob('.github/workflows/pgoEvaluation.yml', 'evaluate'); - // When runner selection, permissions and its execution environment are read. - const environment = workflowStep(job, 'Evaluate pinned candidate on its physical device').env; - expect(isRecord(environment)).toBe(true); - if (!isRecord(environment) || !isRecord(job.strategy) || !isRecord(job.strategy.matrix) || !isRecord(job.permissions) || !isRecord(job.concurrency)) { - throw new Error('Missing workflow job contract.'); + it.each(['both', 'android', 'ios'])('runs the requested hosted platform selection %s', (platform) => { + // Given the actual platform matrix expression and hosted runner expression. + if (!isRecord(job.strategy) || !isRecord(job.strategy.matrix) || typeof job.strategy.matrix.platform !== 'string' || typeof job['runs-on'] !== 'string') { + throw new Error('Missing emulator matrix.'); } - // Then an ineligible platform never requests a device runner and private paths are masked. - expect(job.strategy.matrix.platform).toContain('fromJSON(needs.prepare.outputs.platform-matrix)'); - expect(job.permissions.issues).toBeUndefined(); - expect(environment.PGO_ANDROID_DEVICE_ID).toBe(expression('', 'secrets.PGO_ANDROID_DEVICE_ID')); - expect(environment.PGO_IOS_FIXTURE_PATH).toBe(expression('', 'secrets.PGO_IOS_FIXTURE_PATH')); - expect(environment.PGO_IOS_ENVFILE).toBe(expression('', 'secrets.PGO_IOS_ENVFILE')); - expect(environment.PGO_ANDROID_KEYSTORE_PATH).toBe(expression('', 'secrets.PGO_ANDROID_KEYSTORE_PATH')); - expect(environment.PGO_ANDROID_KEYSTORE_PASSWORD).toBe(expression('', 'secrets.PGO_ANDROID_KEYSTORE_PASSWORD')); - const checkout = workflowStep(job, 'Checkout pinned native candidate'); - expect(isRecord(checkout.with) && checkout.with.token).toBe(expression('', 'secrets.OS_BOTIFY_TOKEN')); - expect(job.concurrency.queue).toBe('max'); - const report = workflowJob('.github/workflows/pgoEvaluation.yml', 'report'); - if (!isRecord(report.concurrency)) { - throw new Error('Missing report concurrency.'); - } - expect(report.concurrency.group).toContain('needs.prepare.outputs.app-sha'); - expect(report.concurrency.queue).toBe('max'); + const expression = (value: string) => value.slice(3, -2).trim(); + // When the selected platforms are expanded by Actions. + const platforms: unknown = runInNewContext(expression(job.strategy.matrix.platform), {inputs: {platform}, fromJSON: JSON.parse}); + // Then neither platform requests a self-hosted physical runner. + expect(platforms).toEqual(platform === 'both' ? ['android', 'ios'] : [platform]); + expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'android'}})).toBe('ubuntu-24.04'); + expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'ios'}})).toBe('macos-26'); }); - it.each([ - ['9.4.94-0-staging', 1], - ['9.4.94-01-staging', 0], - ])('posts only to the exact historical checklist tag %s', async (tag, expectedPosts) => { - // Given the actual YAML posting script and a closed historical checklist. - const step = workflowStep(workflowJob('.github/workflows/pgoEvaluation.yml', 'report'), 'Attach report to its matching deploy checklist'); - const script = isRecord(step.with) ? step.with.script : undefined; - expect(script).toBeDefined(); - let posts = 0; - const issue = {state: 'closed', labels: [{name: 'StagingDeployCash'}], body: `**Release Version:** \`${tag}\``}; - const github = { - rest: { - issues: { - get: async () => ({data: issue}), - listComments: async () => [], - createComment: async () => { - posts += 1; - }, - updateComment: async () => { - posts += 1; - }, - }, - }, - paginate: async () => [], - }; - const requireFixture = () => ({readFileSync: () => JSON.stringify({targetIssueNumber: 42, marker: '', body: 'Public numeric report', version: '9.4.94-0'})}); - // When the trusted inline script rechecks the live checklist before posting. - // This executes only the repository's static workflow script with mocked GitHub and filesystem inputs. - if (typeof script !== 'string') { - throw new Error('Missing posting script.'); + it('uses only the existing checkout and SDK secrets and preserves failure artifacts', () => { + // Given the complete published workflow. + const text = readFileSync('.github/workflows/pgoEvaluation.yml', 'utf8'); + // When private configuration references are collected. + const secrets = [...new Set(Array.from(text.matchAll(/secrets[.]([A-Z_]+)/g), (match) => match[1]))].sort(); + // Then account login, device IDs and signing secrets are unnecessary. + expect(secrets).toEqual(['MAPBOX_SDK_DOWNLOAD_TOKEN', 'OS_BOTIFY_TOKEN']); + expect(text).not.toContain('PGO_ANDROID_DEVICE_ID'); + expect(text).not.toContain('PGO_IOS_DEVICE_ID'); + expect(text).not.toContain('issues: write'); + if (!Array.isArray(job.steps)) { + throw new Error('Missing verification steps.'); } - const completion: unknown = runInNewContext(`(async () => {${script}})()`, { - require: requireFixture, - core: {notice: () => undefined}, - github, - context: {repo: {owner: 'Expensify', repo: 'App'}}, - }); - await completion; - // Then a closed matching issue works while a similar version cannot receive the report. - expect(posts).toBe(expectedPosts); + const upload: unknown = job.steps.find((step: unknown) => isRecord(step) && step.name === 'Upload verification profiles and report'); + expect(isRecord(upload) && upload.if).toBe('always()'); + expect(readFileSync('.github/workflows/deploy.yml', 'utf8')).not.toContain('pgoCandidate'); }); }); - -function workflowJob(path: string, name: string): Record { - const input: unknown = Bun.YAML.parse(readFileSync(path, 'utf8')); - if (!isRecord(input) || !isRecord(input.jobs) || !isRecord(input.jobs[name])) { - throw new Error('Missing workflow job.'); - } - return input.jobs[name]; -} -function workflowStep(job: Record, name: string): Record { - if (!Array.isArray(job.steps)) { - throw new Error('Missing workflow steps.'); - } - const step: unknown = job.steps.find((input: unknown) => isRecord(input) && input.name === name); - if (!isRecord(step)) { - throw new Error('Missing workflow step.'); - } - return step; -} -function expression(prefix: string, content: string): string { - return `${prefix}${String.fromCharCode(36)}{{ ${content} }}`; -} diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts index 9809f0dc38ed..85bc67f6d068 100644 --- a/tests/tooling/pgoIOSRock.test.ts +++ b/tests/tooling/pgoIOSRock.test.ts @@ -7,6 +7,15 @@ import {iosRockBuildArguments} from '@scripts/pgo/ios'; import {parseArgs} from '@rock-js/tools'; describe('PGO iOS Rock build arguments', () => { + it('selects a simulator without changing the default device destination', () => { + // Given an explicit verification destination and unsigned simulator settings. + const settings = ['build', '-sdk', 'iphonesimulator', 'CODE_SIGNING_ALLOWED=NO']; + // When the verification build is routed through Rock. + const args = iosRockBuildArguments('/tmp/pgo-simulator', settings, 'generic/platform=iOS Simulator'); + // Then the simulator SDK stays separate from device builds and settings remain intact. + expect(args.at(8)).toBe('generic/platform=iOS Simulator'); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); it('selects a local device Release build and preserves exact compiler settings', () => { // Given compiler settings containing spaces and inherited Xcode variables. const settings = ['build', 'ARCHS=arm64', 'CODE_SIGN_IDENTITY=Apple Development', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-use=/tmp/profile directory/journey.profdata']; From 05a029bb47f2d9091867e37088e1637d975ad39e Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 Oct 2026 19:09:38 +0100 Subject: [PATCH 54/60] fix(pgo): prepare hosted runner capacity for native builds --- .github/workflows/pgoEvaluation.yml | 5 ++++- tests/tooling/pgoEvaluationWorkflow.test.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml index a58b122e1c8b..173cfad0046a 100644 --- a/.github/workflows/pgoEvaluation.yml +++ b/.github/workflows/pgoEvaluation.yml @@ -29,7 +29,7 @@ jobs: verify: # Only the copied, reviewed branch in App receives private dependency access. if: github.repository == 'Expensify/App' && github.ref == 'refs/heads/@chrispader/perf/pgo-ci-verification' - runs-on: ${{ matrix.platform == 'android' && 'ubuntu-24.04' || 'macos-26' }} + runs-on: ${{ matrix.platform == 'android' && 'ubuntu-24.04' || 'blacksmith-12vcpu-macos-latest' }} timeout-minutes: 360 strategy: fail-fast: false @@ -53,6 +53,9 @@ jobs: run: | test "$(git -C Mobile-Expensify rev-parse HEAD)" = "$(git ls-tree HEAD Mobile-Expensify | awk '{print $3}')" test "$(./scripts/is-hybrid-app.sh)" = true + - name: Free space for Android source builds + if: matrix.platform == 'android' + run: sudo rm -rf /usr/share/dotnet /usr/local/.ghcup /opt/ghc /usr/local/share/powershell - name: Setup HybridApp dependencies uses: ./.github/actions/composite/setupNode with: diff --git a/tests/tooling/pgoEvaluationWorkflow.test.ts b/tests/tooling/pgoEvaluationWorkflow.test.ts index 04594038f3b7..3cee4b862a38 100644 --- a/tests/tooling/pgoEvaluationWorkflow.test.ts +++ b/tests/tooling/pgoEvaluationWorkflow.test.ts @@ -55,7 +55,7 @@ describe('PGO emulator workflow', () => { // Then neither platform requests a self-hosted physical runner. expect(platforms).toEqual(platform === 'both' ? ['android', 'ios'] : [platform]); expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'android'}})).toBe('ubuntu-24.04'); - expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'ios'}})).toBe('macos-26'); + expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'ios'}})).toBe('blacksmith-12vcpu-macos-latest'); }); it('uses only the existing checkout and SDK secrets and preserves failure artifacts', () => { From ff9418b1d32aa8478b21639711e440b82824be84 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 22 Oct 2026 14:20:12 +0100 Subject: [PATCH 55/60] fix(pgo): correct emulator CI dependency setup Co-authored-by: Codex --- .github/workflows/pgoEvaluation.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml index 173cfad0046a..ad29f720bdb7 100644 --- a/.github/workflows/pgoEvaluation.yml +++ b/.github/workflows/pgoEvaluation.yml @@ -78,7 +78,7 @@ jobs: - name: Setup Android build tools and emulator access if: matrix.platform == 'android' run: | - "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" 'platforms;android-37' 'build-tools;36.0.0' 'ndk;27.1.12297006' 'cmake;3.22.1' + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" 'platforms;android-37.0' 'build-tools;36.0.0' 'ndk;27.1.12297006' 'cmake;3.22.1' echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/27.1.12297006" >> "$GITHUB_ENV" echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules @@ -99,13 +99,18 @@ jobs: uses: ./.github/actions/composite/getXcodeVersion - name: Setup Ruby and CocoaPods if: matrix.platform == 'ios' + id: ruby uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f with: ruby-version: .ruby-version bundler-cache: true - name: Verify iOS PGO on a disposable simulator if: matrix.platform == 'ios' - run: npm run pgo:verify-ci -- ios + env: + BUNDLE_GEMFILE: ${{ github.workspace }}/Gemfile + run: | + export PATH="${{ steps.ruby.outputs.ruby-prefix }}/bin:$PATH" + npm run pgo:verify-ci -- ios - name: Upload verification profiles and report if: always() From 966029330b3244d1a9ba6d9b7f8b3521c85d3924 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 22 Oct 2026 14:54:10 +0100 Subject: [PATCH 56/60] fix(pgo): pin emulator runner to its v2.38.0 release The pinned android-emulator-runner commit came from its main branch, which does not ship node_modules, so the action failed to load @actions/core. Release commits bundle their dependencies. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pgoEvaluation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml index ad29f720bdb7..354afa3e2652 100644 --- a/.github/workflows/pgoEvaluation.yml +++ b/.github/workflows/pgoEvaluation.yml @@ -85,7 +85,7 @@ jobs: sudo udevadm trigger --name-match=kvm - name: Verify Android PGO on a disposable emulator if: matrix.platform == 'android' - uses: ReactiveCircus/android-emulator-runner@ed009f5318f15b1cf93a191b856c4f1748a2d4c1 + uses: ReactiveCircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 with: api-level: 36 arch: x86_64 From ae47c3663ca6a43e3b19589cf64540819bca4699 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 22 Oct 2026 16:32:52 +0100 Subject: [PATCH 57/60] fix(pgo): resolve emulator build dependencies from Maven Central The app project reaches Maven Central only through repo.grails.org. Hosted runners start with an empty Gradle cache, so the emulator build sent every lookup there and Gradle failed with HTTP 429 while resolving the Node.js download. The emulator init script now points that repository at Maven Central, which it only mirrors for this build. Co-Authored-By: Claude Opus 5.5 --- scripts/pgo/androidEmulator.init.gradle | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/pgo/androidEmulator.init.gradle b/scripts/pgo/androidEmulator.init.gradle index 1b68380e2387..e22d2ac29a46 100644 --- a/scripts/pgo/androidEmulator.init.gradle +++ b/scripts/pgo/androidEmulator.init.gradle @@ -15,4 +15,11 @@ gradle.beforeProject { application -> task.enabled = false } } + // Without a Gradle cache, the app sends every Maven Central lookup through repo.grails.org, which answers hosted runners with HTTP 429. + // Grails only mirrors Maven Central for this build, so ask Maven Central directly. + application.repositories.withType(MavenArtifactRepository).configureEach { repository -> + if (repository.url.host == 'repo.grails.org') { + repository.url = 'https://repo.maven.apache.org/maven2/' + } + } } From e0628be113f1a255dc3b7cd220a488d1ef9cb6d5 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 22 Oct 2026 19:08:31 +0100 Subject: [PATCH 58/60] fix(pgo): skip R8 in emulator verification builds R8 ran out of its 4 GB heap on the hosted Android runner, which also runs the emulator. It only shrinks Java and Kotlin code, which native PGO does not change, so the disposable emulator builds now skip it. Co-Authored-By: Claude Opus 5.5 --- scripts/pgo/androidEmulator.init.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/pgo/androidEmulator.init.gradle b/scripts/pgo/androidEmulator.init.gradle index e22d2ac29a46..a96c11cb1e8c 100644 --- a/scripts/pgo/androidEmulator.init.gradle +++ b/scripts/pgo/androidEmulator.init.gradle @@ -7,7 +7,10 @@ gradle.beforeProject { application -> } application.plugins.withId('com.android.application') { application.extensions.getByName('androidComponents').finalizeDsl { android -> - android.buildTypes.getByName('release').signingConfig = android.signingConfigs.getByName('debug') + def release = android.buildTypes.getByName('release') + release.signingConfig = android.signingConfigs.getByName('debug') + // R8 needs more heap than a hosted runner can spare beside the emulator, and it only shrinks Java and Kotlin code, which native PGO does not change. + release.minifyEnabled = false } } application.tasks.configureEach { task -> From c919a3198f63f3545589041f23c06d93e63f648f Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 22 Oct 2026 22:34:03 +0100 Subject: [PATCH 59/60] fix(pgo): skip the APK listing redirect when FullStory is excluded FullStory transforms the release APK, so its excluded task is the one that writes the final APK listing. On a clean checkout, AGP's createReleaseApkListingFileRedirect task then fails because that listing does not exist. Local builds passed only because an earlier build had left the file behind. The redirect only serves Android Studio, so builds that exclude FullStory now skip it too. Co-Authored-By: Claude Opus 5.5 --- scripts/pgo/android.ts | 6 ++++-- tests/tooling/pgoAndroidRelease.test.ts | 14 +++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index a8671484d6ed..52344e521a8c 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -16,6 +16,8 @@ import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; const INSTRUMENTED_LIBRARIES = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; +// FullStory's APK transform writes the final APK listing. Without it, AGP's IDE listing redirect fails on a clean build. +const FULLSTORY_EXCLUSIONS = ['-x', 'fullstoryRelease', '-x', 'createReleaseApkListingFileRedirect']; function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { const architecture = process.env.PGO_ANDROID_ARCH ?? 'arm64-v8a'; @@ -263,11 +265,11 @@ function androidReleasePipeline(value?: string, shrinkValue?: string): {enabled: artifactDirectory: 'apk-shrunk', task: ':packageRelease', apkTask: 'packageRelease', - arguments: ['-x', 'fullstoryRelease', '-PpgoSkipFullStory=true', '--init-script', join(rootDirectory, 'scripts/pgo/androidRelease.init.gradle')], + arguments: [...FULLSTORY_EXCLUSIONS, '-PpgoSkipFullStory=true', '--init-script', join(rootDirectory, 'scripts/pgo/androidRelease.init.gradle')], }; } if (value !== 'true') { - return {enabled: false, shrinking: false, artifactDirectory: 'apk', task: ':packageRelease', apkTask: 'packageRelease', arguments: ['-x', 'fullstoryRelease']}; + return {enabled: false, shrinking: false, artifactDirectory: 'apk', task: ':packageRelease', apkTask: 'packageRelease', arguments: [...FULLSTORY_EXCLUSIONS]}; } return { enabled: true, diff --git a/tests/tooling/pgoAndroidRelease.test.ts b/tests/tooling/pgoAndroidRelease.test.ts index 1e9576a78754..5f984f9f695a 100644 --- a/tests/tooling/pgoAndroidRelease.test.ts +++ b/tests/tooling/pgoAndroidRelease.test.ts @@ -24,8 +24,15 @@ describe('PGO Android release processing', () => { // Given no explicit release pipeline request. // When the build selects its task and artifact. const options = androidReleasePipeline(); - // Then bootstrap processing does not masquerade as the final release pipeline. - expect(options).toEqual({enabled: false, shrinking: false, artifactDirectory: 'apk', task: ':packageRelease', apkTask: 'packageRelease', arguments: ['-x', 'fullstoryRelease']}); + // Then bootstrap processing does not masquerade as the final release pipeline, and a clean build skips the IDE listing redirect that needs FullStory's output. + expect(options).toEqual({ + enabled: false, + shrinking: false, + artifactDirectory: 'apk', + task: ':packageRelease', + apkTask: 'packageRelease', + arguments: ['-x', 'fullstoryRelease', '-x', 'createReleaseApkListingFileRedirect'], + }); }); it('selects the processed APK without overriding local credentials', () => { @@ -51,13 +58,14 @@ describe('PGO Android release processing', () => { // Given a request for release shrinking without asset uploads. // When the build selects its tasks and output directory. const options = androidReleasePipeline(undefined, 'true'); - // Then the APK remains distinct and FullStory is both excluded and disabled. + // Then the APK remains distinct, FullStory is both excluded and disabled, and the IDE listing redirect that needs its output is skipped. expect(options.shrinking).toBe(true); expect(options.enabled).toBe(false); expect(options.artifactDirectory).toBe('apk-shrunk'); expect(options.task).toBe(':packageRelease'); expect(options.arguments).toContain('-PpgoSkipFullStory=true'); expect(options.arguments).toContain('-x'); + expect(options.arguments).toContain('createReleaseApkListingFileRedirect'); }); it('rejects simultaneous shrinking and upload pipeline requests', () => { From fd35e23d4f590599b6ab699ded25139f91cedfa9 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Fri, 23 Oct 2026 21:01:38 +0100 Subject: [PATCH 60/60] fix(pgo): bundle OldDot shared JS in Android PGO builds Mobile-Expensify only runs buildGrunt for install, assemble and bundle tasks. The PGO builds call packageRelease directly, so a clean checkout packaged the APK without app/shared/bundle.js and OldDot crashed at launch before NewDot could start the emulator workload. Co-Authored-By: Claude Opus 5.5 --- scripts/pgo/android.ts | 3 ++- tests/tooling/pgoAndroidRelease.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts index 52344e521a8c..e5d92873fc56 100644 --- a/scripts/pgo/android.ts +++ b/scripts/pgo/android.ts @@ -290,7 +290,8 @@ function androidRockBuildArguments(task: string, gradleArguments: string[]): str '--variant', 'release', '--tasks', - task.replace(/^:/, ''), + // Mobile-Expensify only runs buildGrunt, which bundles OldDot's shared JS into the APK assets, for install, assemble and bundle tasks. + `buildGrunt,${task.replace(/^:/, '')}`, '--extra-params', gradleArguments.map((argument) => JSON.stringify(argument)).join(' '), ]; diff --git a/tests/tooling/pgoAndroidRelease.test.ts b/tests/tooling/pgoAndroidRelease.test.ts index 5f984f9f695a..3be6ef3a94d5 100644 --- a/tests/tooling/pgoAndroidRelease.test.ts +++ b/tests/tooling/pgoAndroidRelease.test.ts @@ -14,8 +14,8 @@ describe('PGO Android release processing', () => { // When the adapter forwards this build to Rock. const args = androidRockBuildArguments(pipeline.task, parameters); const forwarded = parseArgs(args.at(-1) ?? ''); - // Then Rock cannot reuse a cached build or lose profile and upload-prevention arguments. - expect(args.slice(0, 7)).toEqual(['build:android', '--local', '--verbose', '--variant', 'release', '--tasks', 'packageRelease']); + // Then Rock cannot reuse a cached build or lose profile and upload-prevention arguments, and Gradle bundles OldDot's shared JS first because packaging alone skips it. + expect(args.slice(0, 7)).toEqual(['build:android', '--local', '--verbose', '--variant', 'release', '--tasks', 'buildGrunt,packageRelease']); expect(forwarded).toEqual(parameters); expect(forwarded).toContain('-PpgoSkipFullStory=true'); expect(forwarded).toContain('fullstoryRelease');