diff --git a/.github/workflows/pgoEvaluation.yml b/.github/workflows/pgoEvaluation.yml new file mode 100644 index 000000000000..354afa3e2652 --- /dev/null +++ b/.github/workflows/pgoEvaluation.yml @@ -0,0 +1,123 @@ +name: Verify native PGO on emulators + +on: + push: + branches: ['@chrispader/perf/pgo-ci-verification'] + workflow_dispatch: + inputs: + platform: + description: Platforms to verify + type: choice + default: both + options: [both, android, ios] + +permissions: + contents: read + +concurrency: + group: pgo-emulator-${{ github.ref }} + cancel-in-progress: true + +env: + STANDALONE_NEW_DOT: 'false' + IS_HYBRID_APP: 'true' + BUNDLER: metro + SENTRY_DISABLE_AUTO_UPLOAD: 'true' + CMAKE_BUILD_PARALLEL_LEVEL: '2' + +jobs: + verify: + # Only the copied, reviewed branch in App receives private dependency access. + if: github.repository == 'Expensify/App' && github.ref == 'refs/heads/@chrispader/perf/pgo-ci-verification' + runs-on: ${{ matrix.platform == 'android' && 'ubuntu-24.04' || 'blacksmith-12vcpu-macos-latest' }} + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + platform: ${{ fromJSON(inputs.platform == 'android' && '["android"]' || inputs.platform == 'ios' && '["ios"]' || '["android","ios"]') }} + steps: + - name: Check existing repository secrets + env: + CHECKOUT_TOKEN: ${{ secrets.OS_BOTIFY_TOKEN }} + MAPBOX_TOKEN: ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }} + run: | + test -n "$CHECKOUT_TOKEN" || { echo '::error::Add the existing OS_BOTIFY_TOKEN repository secret with access to Mobile-Expensify.'; exit 1; } + test -n "$MAPBOX_TOKEN" || { echo '::error::Add the existing MAPBOX_SDK_DOWNLOAD_TOKEN repository secret.'; exit 1; } + - name: Checkout the candidate and its pinned native sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + submodules: recursive + token: ${{ secrets.OS_BOTIFY_TOKEN }} + persist-credentials: false + - name: Verify native checkout + run: | + test "$(git -C Mobile-Expensify rev-parse HEAD)" = "$(git ls-tree HEAD Mobile-Expensify | awk '{print $3}')" + test "$(./scripts/is-hybrid-app.sh)" = true + - name: Free space for Android source builds + if: matrix.platform == 'android' + run: sudo rm -rf /usr/share/dotnet /usr/local/.ghcup /opt/ghc /usr/local/share/powershell + - name: Setup HybridApp dependencies + uses: ./.github/actions/composite/setupNode + with: + IS_HYBRID_BUILD: 'true' + - name: Setup private SDK downloads + env: + MAPBOX_TOKEN: ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }} + run: ./scripts/setup-mapbox-sdk.sh "$MAPBOX_TOKEN" + + - name: Read repository Java version + if: matrix.platform == 'android' + id: java + uses: ./.github/actions/composite/getJavaVersion + - name: Setup Java + if: matrix.platform == 'android' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 + with: + distribution: temurin + java-version: ${{ steps.java.outputs.version }} + - name: Setup Android build tools and emulator access + if: matrix.platform == 'android' + run: | + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" 'platforms;android-37.0' 'build-tools;36.0.0' 'ndk;27.1.12297006' 'cmake;3.22.1' + echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/27.1.12297006" >> "$GITHUB_ENV" + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Verify Android PGO on a disposable emulator + if: matrix.platform == 'android' + uses: ReactiveCircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + with: + api-level: 36 + arch: x86_64 + target: google_apis + profile: pixel_7 + disable-animations: true + script: npm run pgo:verify-ci -- android + + - name: Select repository Xcode + if: matrix.platform == 'ios' + uses: ./.github/actions/composite/getXcodeVersion + - name: Setup Ruby and CocoaPods + if: matrix.platform == 'ios' + id: ruby + uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f + with: + ruby-version: .ruby-version + bundler-cache: true + - name: Verify iOS PGO on a disposable simulator + if: matrix.platform == 'ios' + env: + BUNDLE_GEMFILE: ${{ github.workspace }}/Gemfile + run: | + export PATH="${{ steps.ruby.outputs.ruby-prefix }}/bin:$PATH" + npm run pgo:verify-ci -- ios + + - name: Upload verification profiles and report + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: pgo-emulator-${{ matrix.platform }} + path: .pgo/ci/${{ matrix.platform }} + include-hidden-files: true + if-no-files-found: warn + retention-days: 7 diff --git a/.gitignore b/.gitignore index 8c6d0691fdb9..dda47ce6a069 100644 --- a/.gitignore +++ b/.gitignore @@ -136,6 +136,9 @@ config/certificatePinning/roots/ # Rspack build profiling output (RSPACK_PROFILE=OVERVIEW|ALL) .rspack-profile-*/ +# Local native PGO profiles +.pgo/ + # Expo .expo dist/ diff --git a/Mobile-Expensify b/Mobile-Expensify index f8df56177f15..bea79f939c1d 160000 --- a/Mobile-Expensify +++ b/Mobile-Expensify @@ -1 +1 @@ -Subproject commit f8df56177f157a3c50f415d95750c9b5cc70ff0e +Subproject commit bea79f939c1d490c701ecce2f36f85711285aa95 diff --git a/android/app/build.gradle b/android/app/build.gradle index dafa2ca7fc2a..572f1f488d47 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -84,6 +84,16 @@ project.ext.envConfigFiles = [ * Set this to true to Run Proguard on Release builds to minify the Java bytecode. */ def enableProguardInReleaseBuilds = true +def pgoMode = providers.gradleProperty('pgoMode').getOrElse('off') +def pgoProfile = providers.gradleProperty('pgoProfile').orNull + +if (!(pgoMode in ['off', 'generate', 'use'])) { + throw new GradleException("Unsupported pgoMode '${pgoMode}'. Use off, generate, or use.") +} + +if (pgoMode == 'use' && !pgoProfile) { + throw new GradleException('pgoProfile is required when pgoMode=use.') +} /** * The preferred build flavor of JavaScriptCore (JSC) @@ -119,6 +129,7 @@ android { // Supported language variants must be declared here to avoid from being removed during the compilation. // This also helps us to not include unnecessary language variants in the APK. resConfigs "en", "es" + buildConfigField 'boolean', 'PGO_PROFILE_GENERATION', (pgoMode == 'generate').toString() } flavorDimensions "default" diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index c0d4ff50a0c2..4ea4287658b7 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -28,6 +28,14 @@ android:theme="@style/AppTheme" tools:replace="android:supportsRtl"> + + + + + + #import +#ifndef EXPENSIFY_PGO_GENERATE +#define EXPENSIFY_PGO_GENERATE 0 +#endif + +#if EXPENSIFY_PGO_GENERATE +#include + +extern "C" void __llvm_profile_set_filename(const char *FilenamePat); +extern "C" int __llvm_profile_write_file(void); +extern "C" void __llvm_profile_reset_counters(void); + +using PGOSetFilename = void (*)(const char *); +using PGOWriteFile = int (*)(); +using PGOResetCounters = void (*)(); + +template +static Function HermesPGOFunction(const char *symbol) { + return reinterpret_cast(dlsym(RTLD_DEFAULT, symbol)); +} +#endif + +static NSString *const PGO_DIRECTORY_NAME = @"ExpensifyPGO"; +static NSString *const PGO_STATUS_MARKER = @"profile-status.txt"; +static CFStringRef const PGO_WRITE_NOTIFICATION = CFSTR("com.expensify.pgo.write-profiles"); +static CFStringRef const PGO_CLEAR_NOTIFICATION = CFSTR("com.expensify.pgo.clear-profiles"); + static RCTSurfaceHostingProxyRootView *_rootView = nil; static UIView *_loadingView = nil; @@ -20,6 +46,29 @@ static bool _fade = false; static bool _nativeHidden = false; +static long long CurrentTimeMilliseconds(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000); +} + +@interface RCTBootSplash () ++ (NSURL *)pgoDirectoryURL; ++ (void)writePGOStatus:(NSString *)status result:(int)result; ++ (void)writePGOProfiles; ++ (void)clearPGOProfiles; +@end + +static void HandlePGONotification(__unused CFNotificationCenterRef center, + __unused void *observer, + CFStringRef name, + __unused const void *object, + __unused CFDictionaryRef userInfo) { + if (CFEqual(name, PGO_WRITE_NOTIFICATION)) { + [RCTBootSplash writePGOProfiles]; + } else if (CFEqual(name, PGO_CLEAR_NOTIFICATION)) { + [RCTBootSplash clearPGOProfiles]; + } +} + @implementation RCTBootSplash RCT_EXPORT_MODULE(); @@ -32,6 +81,81 @@ + (BOOL)requiresMainQueueSetup { return NO; } ++ (NSURL *)pgoDirectoryURL { + NSURL *cachesURL = [[[NSFileManager defaultManager] URLsForDirectory:NSCachesDirectory + inDomains:NSUserDomainMask] firstObject]; + NSURL *directoryURL = [cachesURL URLByAppendingPathComponent:PGO_DIRECTORY_NAME isDirectory:YES]; + [[NSFileManager defaultManager] createDirectoryAtURL:directoryURL + withIntermediateDirectories:YES + attributes:nil + error:nil]; + return directoryURL; +} + ++ (void)writePGOStatus:(NSString *)status result:(int)result { + NSString *contents = [NSString stringWithFormat:@"%lld,%@,%d", CurrentTimeMilliseconds(), status, result]; + NSURL *statusURL = [[self pgoDirectoryURL] URLByAppendingPathComponent:PGO_STATUS_MARKER]; + [contents writeToURL:statusURL atomically:YES encoding:NSUTF8StringEncoding error:nil]; +} + ++ (void)initializePGOProfileCollection { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + CFNotificationCenterRef notificationCenter = CFNotificationCenterGetDarwinNotifyCenter(); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_WRITE_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_CLEAR_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + +#if EXPENSIFY_PGO_GENERATE + NSString *profilePattern = [[[self pgoDirectoryURL] URLByAppendingPathComponent:@"newdot-%m.profraw"] path]; + __llvm_profile_set_filename(profilePattern.fileSystemRepresentation); + PGOSetFilename setHermesFilename = HermesPGOFunction("expensify_llvm_profile_set_filename"); + if (setHermesFilename != nullptr) { + setHermesFilename(profilePattern.fileSystemRepresentation); + } +#endif + }); +} + ++ (void)writePGOProfiles { +#if EXPENSIFY_PGO_GENERATE + int appResult = __llvm_profile_write_file(); + PGOWriteFile writeHermesFile = HermesPGOFunction("expensify_llvm_profile_write_file"); + int hermesResult = writeHermesFile != nullptr ? writeHermesFile() : 1; + int result = appResult == 0 && hermesResult == 0 ? 0 : 1; + [self writePGOStatus:@"written" result:result]; + NSLog(@"ExpensifyPGO: wrote LLVM profile with result=%d appResult=%d hermesResult=%d", result, appResult, hermesResult); +#else + [self writePGOStatus:@"not-instrumented" result:1]; + NSLog(@"ExpensifyPGO: ignored profile write in a non-instrumented build"); +#endif +} + ++ (void)clearPGOProfiles { + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSURL *directoryURL = [self pgoDirectoryURL]; + NSArray *files = [fileManager contentsOfDirectoryAtURL:directoryURL + includingPropertiesForKeys:nil + options:0 + error:nil]; + for (NSURL *fileURL in files) { + if ([fileURL.lastPathComponent hasSuffix:@".profraw"] || [fileURL.lastPathComponent isEqualToString:PGO_STATUS_MARKER]) { + [fileManager removeItemAtURL:fileURL error:nil]; + } + } + +#if EXPENSIFY_PGO_GENERATE + __llvm_profile_reset_counters(); + PGOResetCounters resetHermesCounters = HermesPGOFunction("expensify_llvm_profile_reset_counters"); + int result = resetHermesCounters != nullptr ? 0 : 1; + if (resetHermesCounters != nullptr) { + resetHermesCounters(); + } + [self writePGOStatus:@"cleared" result:result]; +#else + [self writePGOStatus:@"not-instrumented" result:1]; +#endif +} + + (bool)isLoadingViewVisible { return _loadingView != nil && ![_loadingView isHidden]; } diff --git a/knip.json b/knip.json index 730d221fe72e..01b1ecb6984e 100644 --- a/knip.json +++ b/knip.json @@ -91,5 +91,5 @@ "diff-so-fancy", "@babel/plugin-proposal-class-properties" ], - "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl"] + "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl", "agent-device"] } diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index 0f384e8dd874..c766d89915cb 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -1,13 +1,49 @@ project(ExpensifyNitroUtils) +# cspell:ignore STREQUAL libclang fprofile defsym + cmake_minimum_required(VERSION 3.9.0) set(PACKAGE_NAME ExpensifyNitroUtils) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_CXX_STANDARD 20) +if(EXPENSIFY_PGO_MODE STREQUAL "generate") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir + OUTPUT_VARIABLE CLANG_RESOURCE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(ANDROID_ABI STREQUAL "arm64-v8a") + set(PGO_ARCH aarch64) + elseif(ANDROID_ABI STREQUAL "x86_64") + set(PGO_ARCH x86_64) + else() + message(FATAL_ERROR "Unsupported PGO ABI: ${ANDROID_ABI}") + endif() + set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-${PGO_ARCH}-android.a") + if(NOT EXISTS "${PGO_RUNTIME}") + message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") + endif() + add_compile_options(-fprofile-generate) + add_link_options( + -fprofile-generate + "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" + -Wl,--undefined=__llvm_profile_set_filename + -Wl,--undefined=__llvm_profile_write_file + -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename + -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file + -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename + -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file + ) +elseif(EXPENSIFY_PGO_MODE STREQUAL "use") + add_compile_options("-fprofile-use=${EXPENSIFY_PGO_PROFILE}") + add_link_options("-fprofile-use=${EXPENSIFY_PGO_PROFILE}") +endif() + # Define C++ library and add all sources add_library(${PACKAGE_NAME} SHARED src/main/cpp/cpp-adapter.cpp + src/main/cpp/PgoProfileWriter.cpp ) # Add Nitrogen specs @@ -29,4 +65,4 @@ target_link_libraries( ${PACKAGE_NAME} ${LOG_LIB} android -) \ No newline at end of file +) diff --git a/modules/ExpensifyNitroUtils/android/build.gradle b/modules/ExpensifyNitroUtils/android/build.gradle index cef4d21f1e8f..692d58148fa6 100644 --- a/modules/ExpensifyNitroUtils/android/build.gradle +++ b/modules/ExpensifyNitroUtils/android/build.gradle @@ -34,6 +34,17 @@ def getExtOrIntegerDefault(name) { return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["ExpensifyNitroUtils_" + name]).toInteger() } +def pgoMode = rootProject.findProperty('pgoMode') ?: 'off' +def pgoProfile = rootProject.findProperty('pgoProfile') + +if (!(pgoMode in ['off', 'generate', 'use'])) { + throw new GradleException("Unsupported pgoMode '${pgoMode}'. Use off, generate, or use.") +} + +if (pgoMode == 'use' && !pgoProfile) { + throw new GradleException('pgoProfile is required when pgoMode=use.') +} + android { namespace "com.margelo.nitro.utils" ndkVersion getExtOrDefault("ndkVersion") @@ -47,7 +58,10 @@ android { externalNativeBuild { cmake { cppFlags "-O2 -frtti -fexceptions -Wall -fstack-protector-all" - arguments "-DANDROID_STL=c++_shared" + arguments "-DANDROID_STL=c++_shared", "-DEXPENSIFY_PGO_MODE=${pgoMode}" + if (pgoMode == 'use') { + arguments "-DEXPENSIFY_PGO_PROFILE=${file(pgoProfile).absolutePath}" + } abiFilters (*reactNativeArchitectures()) } } @@ -126,4 +140,3 @@ dependencies { // Add a dependency on NitroModules implementation project(":react-native-nitro-modules") } - diff --git a/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp new file mode 100644 index 000000000000..492d68e1f09e --- /dev/null +++ b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp @@ -0,0 +1,63 @@ +#include +#include + +#include +#include + +namespace { +using SetFilename = void (*)(const char *); +using WriteFile = int (*)(); + +constexpr std::array kInstrumentedLibraries = { + "libhermesvm.so", + "libreactnative.so", + "libjsi.so", + "libExpensifyNitroUtils.so", +}; + +bool writeProfile(void *handle, const std::string &profilePath) { + const auto setFilename = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_set_filename")); + const auto writeFile = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_write_file")); + if (setFilename == nullptr || writeFile == nullptr) { + return false; + } + + setFilename(profilePath.c_str()); + return writeFile() == 0; +} + +jint writeProfiles(JNIEnv *env, jstring directory) { + const char *directoryChars = env->GetStringUTFChars(directory, nullptr); + if (directoryChars == nullptr) { + return 0; + } + + const std::string profilePath = std::string(directoryChars) + "/newdot-%m.profraw"; + env->ReleaseStringUTFChars(directory, directoryChars); + + jint writtenProfiles = 0; + for (const char *library : kInstrumentedLibraries) { + void *handle = dlopen(library, RTLD_NOW | RTLD_NOLOAD); + if (handle == nullptr) { + continue; + } + + if (writeProfile(handle, profilePath)) { + ++writtenProfiles; + } + dlclose(handle); + } + + return writtenProfiles; +} +} // namespace + +extern "C" JNIEXPORT jint JNICALL +Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { + return writeProfiles(env, directory); +} + +extern "C" JNIEXPORT jint JNICALL +Java_com_expensify_chat_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { + return writeProfiles(env, directory); +} diff --git a/package-lock.json b/package-lock.json index 6cca7f501815..5486fe9c0e91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -193,6 +193,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/package.json b/package.json index 2023e9cdc908..3dfd849ce7f9 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,8 @@ "build-staging": "ENV=staging bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "build-adhoc": "ENV=adhoc bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "benchmark-app-startup": "bun scripts/benchmarkAppStartup.ts", + "pgo:verify-ci": "bun scripts/pgo/emulator.ts", + "pgo:journey": "bun scripts/pgo/journey.ts", "bootstrap-device": "bun scripts/bootstrapForDevice.ts", "createDocsRoutes": "bun .github/scripts/createDocsRoutes.ts", "generateAllowedUrls": "bun .github/scripts/generateAllowedUrls.ts", @@ -84,6 +86,7 @@ "server:vcr:dev": "npm run dev -w @expensify/victory-chart-renderer --", "server:vcr:test": "npm run test -w @expensify/victory-chart-renderer --", "test:bun": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./server ./tests/tooling", + "test:pgo-ci": "TZ=utc bun test --parallel --preload ./scripts/stubReactNative.js --preload ./tests/tooling/setup.ts ./tests/tooling/pgoCandidate.test.ts ./tests/tooling/pgoEvaluationReport.test.ts ./tests/tooling/pgoEvaluationRun.test.ts ./tests/tooling/pgoAndroidRelease.test.ts ./tests/tooling/pgoAndroid.test.ts ./tests/tooling/pgoIOSRock.test.ts ./tests/tooling/pgoJourney.test.ts ./tests/tooling/pgoEvaluationWorkflow.test.ts ./tests/tooling/pgoEmulator.test.ts", "eval:proposal-police": "bun test --path-ignore-patterns='**/__never__/**' ./evals/proposalPolice/", "server:vcr:build:linux": "npm run build:linux -w @expensify/victory-chart-renderer --", "server:vcr:build:linux-arm": "npm run build:linux-arm -w @expensify/victory-chart-renderer --", @@ -270,6 +273,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/patches/react-native/details.md b/patches/react-native/details.md index 0d83081b6d09..506e3f165e15 100644 --- a/patches/react-native/details.md +++ b/patches/react-native/details.md @@ -1,5 +1,12 @@ # `react-native` patches +### [react-native+0.86.0+044+add-local-pgo-build-switch.patch](react-native+0.86.0+044+add-local-pgo-build-switch.patch) + +- Reason: Adds an opt-in local LLVM PGO mode to the React Native Android runtime and Hermes source builds. The mode is disabled by default and is used only by the local PGO tooling. +- Upstream PR/issue: 🛑 +- E/App issue: 🛑 +- PR Introducing Patch: 🛑 + ### [react-native+0.86.0+001+initial.patch](react-native+0.86.0+001+initial.patch) - Reason: Fixes keyboard flickering issue when opening/closing modals. When an input is blurred and a modal is opened, the `rootView` becomes the `firstResponder`, causing the system to retain an incorrect keyboard state. This leads to keyboard flickering when the modal is closed. The patch adds code to resign the `rootView`'s `firstResponder` status before presenting the modal to prevent this issue. @@ -390,3 +397,9 @@ - E/App issue: https://github.com/Expensify/App/issues/103369 - PR introducing patch: https://github.com/Expensify/App/pull/103447 - 0.86.0 migration note: the adopted release containing `3f553d7` is not enough to drop this patch. `0.87.0` still throws here and already has `checkNotNull(parentView)` right before this check. `0.88` soft-fails a non-`ViewGroup` parent but keeps that `checkNotNull`, so a null `parentView` (the placeholder `ViewState` from [react-native#58526](https://github.com/react/react-native/issues/58526)) still crashes. On the 0.88 upgrade, reduce it to soft-failing the `checkNotNull` only. Drop it once upstream handles a null `parentView` in `removeViewAt`. + +### [react-native+0.86.0+048+pgo-emulator-abi.patch](react-native+0.86.0+048+pgo-emulator-abi.patch) + +- Reason: Select the profiling runtime for the configured Android ABI so the PGO build switches also work on x86_64 emulators. ARM64 device builds retain the aarch64 runtime. +- E/App issue: https://github.com/Expensify/App/issues/98008 +- PR introducing patch: https://github.com/margelo/Expensify/pull/34 diff --git a/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch new file mode 100644 index 000000000000..d7e9802f6fd1 --- /dev/null +++ b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch @@ -0,0 +1,184 @@ +diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modules/react-native/ReactAndroid/build.gradle.kts +index bae034d..94d11e1 100644 +--- a/node_modules/react-native/ReactAndroid/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/build.gradle.kts +@@ -42,6 +42,16 @@ val reactNativeRootDir = projectDir.parent + val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() + ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} ++ + // We put the publishing version from gradle.properties inside ext. so other + // subprojects can access it as well. + extra["publishing_version"] = project.findProperty("VERSION_NAME")?.toString()!! +@@ -579,8 +589,13 @@ android { + "-DANDROID_TOOLCHAIN=clang", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW", ++ "-DPGO_MODE=$pgoMode", + ) + ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } ++ + if (hermesV1Enabled) { + arguments("-DHERMES_V1_ENABLED=1") + } +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +index 6610782..853b937 100644 +--- a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +@@ -52,6 +52,18 @@ fun getSDKManagerPath(): String { + + val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++ ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++val pgoCmakePath = File(projectDir, "pgo.cmake") ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} ++ + val reactNativeRootDir = project(":packages:react-native:ReactAndroid").projectDir.parent + val customDownloadDir = System.getenv("REACT_NATIVE_DOWNLOADS_DIR") + val downloadsDir = +@@ -415,8 +427,14 @@ android { + // We intentionally build Hermes with Intl support only. This is to simplify + // the build setup and to avoid overcomplicating the build-type matrix. + "-DHERMES_ENABLE_INTL=True", ++ "-DPGO_MODE=$pgoMode", ++ "-DCMAKE_PROJECT_Hermes_INCLUDE=${pgoCmakePath.absolutePath}", + ) + ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } ++ + if (hermesV1Enabled) { + arguments("-DHERMESVM_HEAP_HV_MODE=HEAP_HV_PREFER32") + } +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +new file mode 100644 +index 0000000..bcc335d +--- /dev/null ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +@@ -0,0 +1,25 @@ ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() +diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +index 8125e02..d466798 100644 +--- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt ++++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +@@ -8,6 +8,32 @@ set(CMAKE_VERBOSE_MAKEFILE on) + + project(ReactAndroid) + ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() ++ + include(CheckIPOSupported) + check_ipo_supported(RESULT IPO_SUPPORT) + if (IPO_SUPPORT) +diff --git a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +--- a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh ++++ b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +@@ -59,6 +59,19 @@ if [[ $xcode_major_version -ge 15 ]]; then + xcode_15_flags="LINKER:-ld_classic" + fi + ++pgo_compile_flags="" ++pgo_linker_flags="" ++if [[ "${EXPENSIFY_PGO_MODE:-off}" == "generate" ]]; then ++ pgo_compile_flags="-fprofile-generate" ++ pgo_linker_flags="-fprofile-generate;LINKER:-alias,___llvm_profile_set_filename,_expensify_llvm_profile_set_filename;LINKER:-alias,___llvm_profile_write_file,_expensify_llvm_profile_write_file;LINKER:-alias,___llvm_profile_reset_counters,_expensify_llvm_profile_reset_counters" ++elif [[ "${EXPENSIFY_PGO_MODE:-off}" == "use" ]]; then ++ if [[ -z "${EXPENSIFY_PGO_PROFILE:-}" ]]; then ++ echo "EXPENSIFY_PGO_PROFILE is required when EXPENSIFY_PGO_MODE=use" >&2 ++ exit 1 ++ fi ++ pgo_compile_flags="-fprofile-use=${EXPENSIFY_PGO_PROFILE}" ++fi ++ + boost_context_flag="" + if [[ $PLATFORM_NAME == "catalyst" ]]; then + boost_context_flag="-DHERMES_ALLOW_BOOST_CONTEXT=0" +@@ -70,7 +84,7 @@ echo "Configure Apple framework" + "$CMAKE_BINARY" \ + -S "${PODS_ROOT}/hermes-engine" \ + -B "${PODS_ROOT}/hermes-engine/build/${PLATFORM_NAME}" \ +- -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags" \ ++ -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags;$pgo_linker_flags" \ + -DHERMES_APPLE_TARGET_PLATFORM:STRING="$PLATFORM_NAME" \ + -DCMAKE_OSX_ARCHITECTURES:STRING="$architectures" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING="$deployment_target" \ +@@ -82,8 +96,8 @@ echo "Configure Apple framework" + -DHERMES_ENABLE_BITCODE:BOOLEAN=false \ + -DHERMES_BUILD_APPLE_FRAMEWORK:BOOLEAN=true \ + -DHERMES_BUILD_SHARED_JSI:BOOLEAN=false \ +- -DCMAKE_CXX_FLAGS:STRING="-gdwarf" \ +- -DCMAKE_C_FLAGS:STRING="-gdwarf" \ ++ -DCMAKE_CXX_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ ++ -DCMAKE_C_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ + -DIMPORT_HOST_COMPILERS:PATH="${hermesc_path}" \ + -DJSI_DIR="$jsi_path" \ + -DHERMES_RELEASE_VERSION="for RN $release_version" \ diff --git a/patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch b/patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch new file mode 100644 index 000000000000..7322b2232e4f --- /dev/null +++ b/patches/react-native/react-native+0.86.0+048+pgo-emulator-abi.patch @@ -0,0 +1,38 @@ +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +--- a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +@@ -4,7 +4,14 @@ + OUTPUT_VARIABLE CLANG_RESOURCE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +- set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(ANDROID_ABI STREQUAL "arm64-v8a") ++ set(PGO_ARCH aarch64) ++ elseif(ANDROID_ABI STREQUAL "x86_64") ++ set(PGO_ARCH x86_64) ++ else() ++ message(FATAL_ERROR "Unsupported PGO ABI: ${ANDROID_ABI}") ++ endif() ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-${PGO_ARCH}-android.a") + if(NOT EXISTS "${PGO_RUNTIME}") + message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") + endif() +diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +--- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt ++++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +@@ -14,7 +14,14 @@ + OUTPUT_VARIABLE CLANG_RESOURCE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +- set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(ANDROID_ABI STREQUAL "arm64-v8a") ++ set(PGO_ARCH aarch64) ++ elseif(ANDROID_ABI STREQUAL "x86_64") ++ set(PGO_ARCH x86_64) ++ else() ++ message(FATAL_ERROR "Unsupported PGO ABI: ${ANDROID_ABI}") ++ endif() ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-${PGO_ARCH}-android.a") + if(NOT EXISTS "${PGO_RUNTIME}") + message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") + endif() diff --git a/scripts/lib/nativeAppBenchmark.ts b/scripts/lib/nativeAppBenchmark.ts index bce54b015663..434fc2c038fe 100644 --- a/scripts/lib/nativeAppBenchmark.ts +++ b/scripts/lib/nativeAppBenchmark.ts @@ -6,6 +6,7 @@ import {assertAndroidAppInstalled, createAndroidAdapter, parseAndroidProcessIden import { createIOSAdapter, iOSBenchmarkMarkerPath, + iOSProcessListArguments, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, @@ -27,6 +28,7 @@ export { createNativeAppBenchmarkAdapter, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseBenchmarkLogEvents, diff --git a/scripts/lib/nativeAppBenchmark/ios.ts b/scripts/lib/nativeAppBenchmark/ios.ts index fa82833d1719..d5a82d60b543 100644 --- a/scripts/lib/nativeAppBenchmark/ios.ts +++ b/scripts/lib/nativeAppBenchmark/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore BEGINSWITH devicectl +// cspell:ignore devicectl /** Implements iOS benchmark setup, launch, process control, and marker collection through CoreDevice. */ @@ -67,20 +67,7 @@ async function createIOSAdapter({rootDirectory, deviceIdentifier, appID}: Omit 0 ? processIdentifier : undefined; } +/** Lists every process so client-side matching works across CoreDevice versions with different filter field names. */ +function iOSProcessListArguments(deviceIdentifier: string, outputPath: string): string[] { + return ['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']; +} + /** Maps a span name to its encoded marker path inside the iOS app data container. */ function iOSBenchmarkMarkerPath(spanName: string): string { return `${IOS_BENCHMARK_DIRECTORY}/${encodeURIComponent(spanName)}.log`; @@ -283,5 +275,13 @@ function isCoreDeviceInstalledApp(value: JsonValue): value is CoreDeviceInstalle return isJSONObject(value) && typeof value.bundleIdentifier === 'string' && typeof value.url === 'string'; } -export {createIOSAdapter, iOSBenchmarkMarkerPath, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, parseIOSRunningAppProcessIdentifier}; +export { + createIOSAdapter, + iOSBenchmarkMarkerPath, + iOSProcessListArguments, + parseIOSInstalledAppURL, + parseIOSInstalledAppsResponse, + parseIOSLaunchProcessIdentifier, + parseIOSRunningAppProcessIdentifier, +}; export type {CoreDeviceInstalledAppsResponse}; diff --git a/scripts/lib/scriptUtils.ts b/scripts/lib/scriptUtils.ts new file mode 100644 index 000000000000..9a665b430b86 --- /dev/null +++ b/scripts/lib/scriptUtils.ts @@ -0,0 +1,104 @@ +/** Reusable helpers for repository tooling scripts. */ + +import {spawnSync} from 'node:child_process'; +import {existsSync, readdirSync} from 'node:fs'; +import {join, resolve} from 'node:path'; +import process from 'node:process'; + +const rootDirectory = resolve(import.meta.dirname, '../..'); + +function fail(message: string): never { + throw new Error(message); +} + +function run(command: string, args: string[], cwd = rootDirectory): void { + const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } +} + +function capture(command: string, args: string[], cwd = rootDirectory): string { + const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const stderr = result.stderr.trim(); + fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { + const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { + const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); + return !result.error && result.status === 0; +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolvePromise) => { + setTimeout(resolvePromise, milliseconds); + }); +} + +function findFiles(directory: string, extension: string): string[] { + if (!existsSync(directory)) { + return []; + } + + return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return findFiles(path, extension); + } + return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; + }); +} + +function requirePositiveInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive integer, received: ${value}`); + } + return value; +} + +function parsePositiveInteger(rawValue: string, label: string): number { + return requirePositiveInteger(Number(rawValue), label); +} + +function parseChoice(rawValue: string, choices: readonly T[], label: string): T { + const choice = choices.find((candidate) => candidate === rawValue); + if (!choice) { + fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); + } + return choice; +} + +function environmentString(name: string): string | undefined { + const value: unknown = process.env[name]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function valueAt(values: readonly T[], index: number): T { + const value = values.at(index); + if (value === undefined) { + fail(`Expected a value at index ${index}.`); + } + return value; +} + +export {capture, captureBinary, environmentString, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run, runAllowFailure, sleep, valueAt}; diff --git a/scripts/pgo/EVALUATION.md b/scripts/pgo/EVALUATION.md new file mode 100644 index 000000000000..2fe1d46fbdab --- /dev/null +++ b/scripts/pgo/EVALUATION.md @@ -0,0 +1,34 @@ +# Verify native PGO in CI + + + +This workflow builds baseline and instrumented HybridApps, runs a bundled synthetic workload three times, merges the native profiles, and rebuilds with PGO. It then runs both baseline and optimized apps on disposable Android and iOS emulators. The workload requires no test account or API credentials. + +## Run the combined PR + +You need permission to push branches to Expensify/App and Expensify/Mobile-Expensify. + +1. In **Expensify/App → Settings → Secrets and variables → Actions**, confirm these existing repository secrets are available: + + | Secret | Purpose | + | --- | --- | + | `OS_BOTIFY_TOKEN` | Read App and the private Mobile-Expensify repository | + | `MAPBOX_SDK_DOWNLOAD_TOKEN` | Download the native Mapbox SDK | + +2. Check out the combined branch and run its copy script. The script copies the pinned native commit first, then pushes the App branch. It preserves both commits and their signatures. + + ```bash + git clone --single-branch --branch @chrispader/perf/pgo-ci-verification git@github.com:margelo/Expensify.git expensify-pgo-ci + cd expensify-pgo-ci + ./scripts/pgo/copy-ci-preview.sh + ``` + +3. Open **Expensify/App → Actions → Verify native PGO on emulators**. The App push starts both jobs automatically, before the workflow is merged into main. Confirm both jobs pass and download `pgo-emulator-android` and `pgo-emulator-ios` for their reports and profiles. Use **Re-run jobs** to repeat the run. + +The workflow installs the toolchains, creates the devices, uses Android's debug signing key, and disables iOS signing. You do not need device IDs, signing certificates, environment files, an AWS account, or a local dependency install. New pushes to the copied branch start another run. The Margelo fork skips the native jobs because it does not own Expensify's private dependency access. + +Each report records the App and native commits, artifact hashes, profile counts, and successful synthetic workload samples. A failed build, incomplete workload, or invalid profile fails the job and leaves the available evidence in its artifact. + +## Interpret the result + +Passing verifies the native profile generation, collection, merge, and optimized rebuild on the emulator target. The bundled workload renders synthetic report and expense lists instead of loading the signed-in Expensify app. Its timings do not establish phone performance, and its profiles must not be used for device releases. The workflow does not change deployments or choose a production binary. diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md new file mode 100644 index 000000000000..c29d117fc601 --- /dev/null +++ b/scripts/pgo/JOURNEY.md @@ -0,0 +1,103 @@ +# Automated native PGO journey + + + +`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full journey, including message sending and native profile export, has been validated on a physical Samsung phone and iPhone 13 Pro. + +## Account and device setup + +Use a dedicated, owner-approved heavy account with a populated Inbox, a long chat history, and enough expenses and expense reports to scroll. Keep the account and its data between runs. Installing over the existing app preserves authentication; uninstalling or clearing app data does not. + +The runner never enters credentials, retrieves a security code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. + +Before every run, it opens Inbox to allow the app's automatic large-account preference update, then opens Account → Preferences and ensures Priority mode is **Most recent**. If it is **#focus**, the runner changes it and verifies the result. It checks the preference again at the end and rejects the run if #focus became enabled. + +Install `agent-device` on the runner and pin its version. Android needs USB debugging and a connected, authorized device. The runner temporarily enables the tool's test keyboard for reliable text replacement and restores the previous keyboard when its session closes. iOS requires macOS, Xcode, a trusted device with Developer Mode enabled, and signing for the tool's XCTest runner. The existing iOS profile collector supports physical devices; simulator profiles are not inputs to a device build. + +Create an ignored fixture, for example `.pgo/journey-fixture.json`: + +```json +{ + "accountEmail": "heavy@example.com", + "accountClass": "heavy", + "description": "Owner-approved high-traffic test account with populated chats, expenses, and reports.", + "report": { + "query": "busy", + "resultLabelPrefix": "#busy, Test workspace", + "title": "#busy" + }, + "personalChat": { + "query": "heavy@example.com", + "resultLabelPrefix": "Test User (you), heavy@example.com", + "title": "Test User (you)" + }, + "allowMessages": false, + "scrolls": 8, + "tabCycles": 3 +} +``` + +Choose the report name and workspace from the actual account. Search matching permits a changing message preview after the configured identity, and rejects ambiguous results. The personal-chat query must be the exact approved account email. That result can show either the email or the latest message beneath its `(you)` title, so both forms are supported for repeated runs. Confirm the personal chat with Chris before setting `allowMessages` to `true`. Messages are restricted to that account's `(you)` chat; each run sends two uniquely marked test messages and leaves them in place. An existing composer draft causes a failure. + +## Validate and record + +First check the account and #focus setting without sending messages: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --preflight +``` + +Use `--navigation-only` instead of `--preflight` to exercise all navigation and scrolling without sending messages. This mode cannot collect a training profile. Omit both flags for the complete journey once message permission is recorded in the fixture. + +The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. Android also edits each draft before sending; iOS enters the final text once because replacing non-empty text in its composer was unreliable during live validation. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. + +After installing and verifying an instrumented build with the existing PGO commands, collect three independent repetitions: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +scripts/pgo/journey.ts ios --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +``` + +Start with three repetitions per platform. Use the recorded duration and resulting native counter coverage to adjust the workload; extra repetitions of the same data do not add new paths. Most activity here exercises Hermes, Fabric mounting, layout, text, native input, and scrolling. This trains the compiled native implementation used by JavaScript; it does not compile the application's JavaScript with LLVM. Avoid spending most of the run waiting on network responses or looping over trivial settings screens. + +On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. + +On the same day, a complete instrumented iOS validation recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. A subsequent three-repetition training batch passed in 6 minutes 41 seconds, 6 minutes 35 seconds, and 6 minutes 37 seconds per journey. The batch exported six raw profiles and produced a 15 MB merged profile containing 77,590 functions. Every repetition sent two approved messages to the account's personal chat, then confirmed the heavy account and #focus off. These timings include automation overhead; they are not app performance measurements. + +The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. + +Each batch lives under `.pgo//journeys//` and includes `result.json`, per-run raw profiles, `journey.profdata`, and a function summary. The result records the fixture, app and device identifiers, both repository revisions, automation version, completed runs, and durations. These artifacts contain account identifiers and must stay in restricted CI storage. + +The journey profile is kept separate from the startup profile. The existing optimized-build command reads `newdot.profdata` from its platform directory: CI must explicitly select or weight the successful scenario profiles, write that final input, and preserve the iOS `.format` marker. Do not silently replace the startup input with an incomplete navigation run. + +## Release pipeline integration + +Use an exclusive device lease on a persistent runner: Linux or macOS with an arm64 Android device, and macOS with an arm64 iPhone. Build the instrumented artifact from the release candidate, install it over the signed-in app, run preflight and the full collection, then build the optimized artifact from the same source and toolchain. Gate publication on a separate performance and correctness check. The instrumented artifact is a CI input, not the store upload. + +Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. + +This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows and measuring held-out interactive performance remain separate integration steps. + +## Held-out interaction benchmark + +The PGO workflow has a shorter, read-only benchmark that is separate from the six-minute profile-collection journey. Each sample relaunches the signed-in app, selects Spend → Expenses, returns to Inbox, then times a warm Inbox → Spend → Inbox switch. The app's existing first-paint, content-load, and Inbox navigation spans measure the UI work. One complete warm-up is discarded for each build, followed by the requested number of fresh-process samples. The benchmark checks the same approved account and #focus preference, but sends no messages. + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Build both the release and optimized artifacts with the current span allowlist first. `build-optimized --profile /absolute/path/to/merged.profdata` selects an immutable archived profile; without `--profile`, it uses `newdot.profdata`. The output uses the existing startup benchmark's raw-sample and summary CSV format in a unique `.pgo//benchmarks/journey//` directory. The two phases run together, so a failed or repeated phase cannot silently pair with stale data. Compare each interactive span separately; adding their durations together does not give a full journey time because there is automation time between taps. Keep this benchmark out of the profile-training batch, and avoid using its result as a release gate until build-order drift and device temperature have been characterized. + +## Heavy report benchmark + +Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes four fast swipes toward older messages and four back toward recent messages. It checks that content changes after each direction; there is no accessibility snapshot between individual flings. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. + +```bash +scripts/pgo/pgo.ts android benchmark-heavy-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-heavy-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Rebuild both app variants after adding the `/r/*` span to the build-time benchmark allowlist. The earlier artifacts cannot emit this metric. Each batch goes to `.pgo//benchmarks/heavy-journey//` with raw and summary CSVs, artifact hashes, and a run manifest. On Android, an additional frame CSV counts rendered and missed-deadline frames during the fast-scroll window. Compare its missed-frame share separately from report-open latency; it is not elapsed scroll time. The current physical-iPhone frame probe could not consistently attach Xcode's Animation Hitches trace to the app process, so the iOS runner validates scroll movement but deliberately reports no scroll-frame score. A reliable iOS scrolling claim needs an XCTest performance test with a signposted scroll interval or another native frame trace that works on the CI device. + +Neither the six-minute collector nor this benchmark's total execution time measures app speed. Android frame counts still include any app rendering triggered during the capture window, and the report-open span can be affected by cached data, network, and device load. Keep the account dataset and device state stable, and warm both variants the same way. Run at least two batches with opposite phase orders by passing `--benchmark-order optimized-first` to the second batch; the default is `release-first`. Report distributions and confidence intervals, and only claim a benefit for the specified device and workload when the improvement exceeds run-to-run variation. No PGO profile can guarantee every launch or interaction will be faster. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md new file mode 100644 index 000000000000..ba8a1538647a --- /dev/null +++ b/scripts/pgo/README.md @@ -0,0 +1,175 @@ +# Native profile-guided optimization + + + +For the hosted emulator workflow and the Expensify engineer handoff, see [Verify native PGO in CI](./EVALUATION.md). + +This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. + +The command enables `ManualAppStartup` and the existing Spend/Inbox navigation spans in every app it builds. Startup profile collection waits for `ManualAppStartup` to finish, then explicitly writes the LLVM counters before the app process is stopped. Both benchmark modes use app-defined span durations rather than device-automation wall time. + +## Prepare local release identifiers + +Bootstrap the release projects once so the apps can be installed beside other Expensify builds: + +```bash +scripts/bootstrapForDevice.ts --build-variants release --identifier-suffix pgo +``` + +For Christoph Pader's local identifiers, this produces: + +| Platform | Identifier | +| --- | --- | +| Android | `com.chrispader.expensify.pgo` | +| iOS | `com.chrispader.expensify.expensifylite.pgo` | + +The PGO command reads the Android release application ID from `Mobile-Expensify/Android/build.gradle` and the iOS bundle identifier from the archived app. Pass `--app-id` to override either value. + +## Preconditions + +- Prefer physical arm64 Android and iOS devices for final profile collection and measurements. An Android emulator running the same ABI and instrumented native build can provide useful counts, but emulator-only behavior can bias the workload. An iOS simulator builds for a different platform, so do not reuse its native profiles for a device build. +- Android requires the NDK version pinned in `Mobile-Expensify/Android/build.gradle`. Set `ANDROID_NDK_HOME` if it is not installed in the default SDK directory. +- iOS requires Xcode command-line tools, an unlocked device, and valid local Apple Development signing for the app and its extensions. Select a team with `IOS_DEVELOPMENT_TEAM` when automatic discovery is insufficient. +- Install dependencies and apply the repository patches before building. +- Seed a test account and data set before recording an interactive profile. + +Every command starts with a platform and a workflow: + +```bash +scripts/pgo/pgo.ts android --help +scripts/pgo/pgo.ts ios --help +``` + +## Collect a startup profile + +Build, verify, and install the instrumented app: + +```bash +scripts/pgo/pgo.ts android build-instrumented +scripts/pgo/pgo.ts android verify-instrumented +scripts/pgo/pgo.ts android install-instrumented +``` + +Record ten cold-process startups, retrieve the raw profiles, and merge them: + +```bash +scripts/pgo/pgo.ts android record-startups +``` + +The optional positional arguments set the run count and span timeout: + +```bash +scripts/pgo/pgo.ts android record-startups 20 45 +``` + +The command removes only old `newdot-*.profraw` files before collection. It requires `ManualAppStartup` to complete on every run and flushes the native counters after each successful run. A missing span fails the collection instead of adding a partial startup. The merged profile is written to `.pgo/android/arm64-v8a/newdot.profdata` or `.pgo/ios/arm64/newdot.profdata`. + +The iOS workflow uses the same commands: + +```bash +scripts/pgo/pgo.ts ios build-instrumented +scripts/pgo/pgo.ts ios verify-instrumented +scripts/pgo/pgo.ts ios install-instrumented +scripts/pgo/pgo.ts ios record-startups +``` + +The app must remain in the foreground until each profile write completes. Force-terminating an iOS app is not a reliable profile flush. + +## Collect an interactive profile + +For the automated heavy-account workload, see [Automated native PGO journey](./JOURNEY.md). It checks the signed-in account, disables #focus, drives the critical user flows, and can archive and merge profiles from each successful run. + +For a broader profile, install the instrumented app, perform the agreed journey, and write the counters while the app is still running: + +```bash +scripts/pgo/pgo.ts android dump +scripts/pgo/pgo.ts android pull +scripts/pgo/pgo.ts android merge +``` + +Use the equivalent `ios` commands on iOS. A representative journey should exercise the common signed-in path: open and scroll chats, send a message, visit a workspace setting, attach and view a file, and run a search. Keep account state, data size, and network conditions stable between collections. + +Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The startup recorder and journey runner each handle that lifecycle; CI must still choose and combine the scenario profiles. + +## Build and benchmark the optimized app + +Build the optimized app only after recording a fresh profile. Build the baseline release from the same source revision and toolchain: + +```bash +scripts/pgo/pgo.ts android build-optimized +scripts/pgo/pgo.ts android build-release +scripts/pgo/pgo.ts android benchmark 20 45 +``` + +Pass `--profile /absolute/path/to/merged.profdata` to `build-optimized` when comparing an archived profile. The path becomes part of the compiler input; keep the file immutable for the build. Without it, the command uses the platform's `newdot.profdata`. Local PGO builds disable Sentry uploads. iOS temporarily skips the FullStory and Sentry upload phases and restores the original Xcode project afterward. Android excludes `fullstoryRelease` and archives the pre-FullStory APK, so its local benchmarks are internally paired but do not represent the fully post-processed production APK. Do not compare their absolute timings directly with earlier or store builds that used FullStory processing. + +The benchmark installs the archived baseline, runs one warm-up and the requested `ManualAppStartup` samples, then repeats the process with the optimized app. Both builds use the same application identifier, so installing the second artifact preserves the seeded account and data. Results are stored under `.pgo//benchmarks/` in the repository benchmark CSV format. + +For a separate, read-only interaction benchmark, use the approved signed-in account fixture described in [JOURNEY.md](./JOURNEY.md): + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id com.chrispader.expensify.pgo --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id com.chrispader.expensify.expensifylite.pgo --fixture .pgo/journey-fixture.json +``` + +Each variant gets one discarded warm-up and 20 measured fresh-process journeys. Before each sample, the runner selects Spend → Expenses and returns to Inbox; it then times a warm Inbox → Spend → Inbox tab switch. Expenses can restore the account's saved query and filters, so keep that state fixed across both phases. The CSVs under a unique `.pgo//benchmarks/journey//` directory contain `ManualNavigateToReportsFirstPaint`, `ManualNavigateToReportsContentLoad`, and `ManualNavigateToInboxTab` samples, alongside a run manifest with artifact hashes. These are app-defined tap-to-paint timings, so automation overhead is outside the measurement. The journey sends no messages and leaves account data unchanged. Both phases run in one command and are compared only within that batch. Rebuild both archived variants with this version of the tooling before benchmarking; older PGO builds only logged startup. + +For a populated-chat, fast-scroll benchmark, use `benchmark-heavy-journey` with the same arguments. It records the app's report-open span on both platforms and Android's rendered/missed scroll frames in a separate CSV. iOS scroll-frame measurement is not currently reliable on the connected phone; the runner verifies that scrolling moves through real content but does not report a scroll performance number. See [JOURNEY.md](./JOURNEY.md) for the exact measured interval, setup, and limitations. Both variants must be rebuilt to include the additional report-open marker. + +The stages can also run independently: + +```bash +scripts/pgo/pgo.ts android benchmark-release 20 45 +scripts/pgo/pgo.ts android benchmark-optimized 20 45 +scripts/pgo/pgo.ts android compare-benchmarks +``` + +Select a device or override an identifier when discovery is ambiguous: + +```bash +scripts/pgo/pgo.ts android benchmark 20 45 --device DEVICE_SERIAL --app-id com.chrispader.expensify.pgo +scripts/pgo/pgo.ts ios benchmark 20 45 --device Chris14Pro --app-id com.chrispader.expensify.expensifylite.pgo +``` + +Release, instrumented, and optimized artifacts are archived under `.pgo/android/arm64-v8a/apk/` and `.pgo/ios/arm64/app/`. Do not apply a profile to another architecture, compiler, source revision, dependency graph, or build configuration. + +## What to collect + +A startup-only profile is useful, but it tends to overfit initialization and can make post-startup code colder. For startup and overall app performance, use a mixed profile with measured weights: + +1. Record a fixed startup suite across the important states, such as a signed-out launch, a signed-in inbox launch, and a launch with a realistically large local data set. +2. Record a small set of common interactive journeys. Prefer bounded flows with deterministic seeded data and an explicit success signal. +3. Keep the raw output for each scenario separate. A longer scenario generates more counter volume than a short one, so equal repetition does not give scenarios equal influence. +4. Keep a separate benchmark suite that does not contribute to the training profile. This catches overfitting. + +The current `record-startups` command accumulates and merges the startup runs automatically. For a mixed CI profile, preserve each scenario's output independently, convert it to its own indexed profile, then combine the scenario profiles with `llvm-profdata merge --weighted-input`. Choose weights from measured production flow frequency and performance impact, not from a default startup-to-interaction ratio. A giant tour of every feature is less useful than a stable suite that reflects real traffic. + +## Collect profiles for device releases + +Run profile generation as a scheduled or release-candidate job, not on every pull request. Use pinned self-hosted runners or a device farm, preferably with physical arm64 devices, the production compiler versions, stable thermal and power conditions, a seeded account, and deterministic local fixtures where possible. Build one instrumented artifact per platform, run the fixed startup and interaction suites against that exact artifact, flush after every successful scenario, merge the separate outputs with documented weights, then build the optimized artifact without changing the checkout or toolchain. + +Store both repository revisions, dependency locks, compiler and SDK versions, ABI, build settings, scenario version, and test-data fixture with each profile. Reject mismatched artifacts before the optimized build. Keep the instrumented build and raw profiles so a failed collection can be investigated without rerunning it. + +Benchmark both variants on the same device after a cooldown, discard post-install warm-ups, and retain raw samples. The `benchmark` workflow runs all baseline samples before all optimized samples, so thermal or time-dependent drift can bias the comparison. The two archived builds intentionally share an identifier to preserve app data, which prevents side-by-side alternation. CI can reduce drift with ABBA install blocks that preserve the shared identifier and discard each post-install warm-up. Another option is to give the builds distinct identifiers, seed matched state, and use the repository's alternating startup benchmark. Evaluate both startup and held-out interactive journeys that were excluded from training. Gate on several releases of data rather than a single noisy run. + +Hosted CI is usually insufficient for the complete workflow because hosted runners rarely expose stable physical devices. Android emulators can validate automation and may contribute useful same-ABI counts when the workload is representative, but final validation should use target devices. iOS device collection also needs signing and physical-device access. A small self-hosted device pool or managed device farm is the realistic route to automation. + +## Limitations + +- LLVM PGO improves compiled native code. It can optimize a source-built Hermes engine, but it does not directly reorder JavaScript or replace JavaScript startup analysis. +- Precompiled vendored frameworks cannot be instrumented after the fact. +- The iOS flush currently exports the app executable's profile runtime and Hermes's separate runtime. Other dynamically linked frameworks need their own export path before their instrumentation can contribute to the merged profile. +- Profiles become stale after native source, dependency, compiler, SDK, ABI, or important build-setting changes. Regenerate them instead of accepting out-of-date warnings. +- Instrumented builds have overhead. Never use them as the performance baseline. +- Local bootstrap settings disable Android minification so synthetic identifiers and debug signing work. A production decision still needs a production-like, minified CI build with the repository's signing and dependency issues resolved. +- Device temperature, battery state, background work, network variability, and test-account drift can overwhelm small gains. Preserve raw samples and use enough repetitions. +- A profile is only as good as its workload. Narrow startup training can regress later interactions, while an unweighted feature tour can dilute hot production paths. + +Android Baseline Profiles are a separate, complementary input to ART compilation of Java and Kotlin. Android Startup Profiles are the related mechanism for DEX layout. Neither replaces this LLVM profile for C, C++, and Swift code. + +## References + +- [Android NDK profile-guided optimization](https://developer.android.com/ndk/guides/pgo) +- [Clang profile-guided optimization](https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization) +- [`llvm-profdata` weighted inputs](https://llvm.org/docs/CommandGuide/llvm-profdata.html#cmdoption-llvm-profdata-merge-weighted-input) +- [Android Baseline Profiles](https://developer.android.com/topic/performance/baselineprofiles/overview) diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts new file mode 100644 index 000000000000..e5d92873fc56 --- /dev/null +++ b/scripts/pgo/android.ts @@ -0,0 +1,329 @@ +// cspell:ignore Dorg jvmargs Metaspace sdcard profdata Ppatched Ppgo profraw Readelf readelf zipinfo libreactnative libhermesvm libjsi cnts + +/** Android-specific release builds, native profile persistence, retrieval, and instrumentation checks. */ + +import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {homedir, platform, tmpdir} from 'node:os'; +import {join} from 'node:path'; +import process from 'node:process'; + +import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PlatformAdapter} from './shared'; + +import {capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from '../lib/scriptUtils'; +import {androidBuildEnvironment} from './proof'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; + +const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; +const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; +const INSTRUMENTED_LIBRARIES = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; +// FullStory's APK transform writes the final APK listing. Without it, AGP's IDE listing redirect fails on a clean build. +const FULLSTORY_EXCLUSIONS = ['-x', 'fullstoryRelease', '-x', 'createReleaseApkListingFileRedirect']; + +function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { + const architecture = process.env.PGO_ANDROID_ARCH ?? 'arm64-v8a'; + if (architecture !== 'arm64-v8a' && architecture !== 'x86_64') { + fail('PGO_ANDROID_ARCH must be arm64-v8a or x86_64.'); + } + const emulatorVerification = process.env.PGO_CI_EMULATOR === 'true'; + const releasePipeline = androidReleasePipeline(process.env.PGO_ANDROID_RELEASE_PIPELINE, process.env.PGO_ANDROID_SHRINK); + const profileDirectory = join(rootDirectory, '.pgo/android', architecture); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const apkDirectory = join(profileDirectory, releasePipeline.artifactDirectory); + const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); + const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); + const buildGradlePath = join(androidDirectory, 'build.gradle'); + const gradleReleaseApkPath = join(androidDirectory, `build/intermediates/apk/release/${releasePipeline.apkTask}/Expensify-release.apk`); + const profileDumpTimeoutSeconds = 5; + + const artifactPaths: BuildArtifactPaths = { + release: join(apkDirectory, 'Expensify-release.apk'), + instrumented: join(apkDirectory, 'Expensify-release-instrumented.apk'), + optimized: join(apkDirectory, 'Expensify-release-optimized.apk'), + }; + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + let cachedAppID: string | undefined; + + function appID(): string { + if (configuredAppID) { + return configuredAppID; + } + cachedAppID ??= androidApplicationID(readFileSync(buildGradlePath, 'utf8')); + return cachedAppID; + } + + function deviceProfileDirectory(): string { + return `/sdcard/Android/data/${appID()}/cache`; + } + + function adb(args: string[]): void { + run('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); + } + + function adbCapture(args: string[]): string { + return capture('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); + } + + function llvmTool(name: string): string { + const toolPath = androidLlvmToolPath(name); + if (!existsSync(toolPath)) { + fail(`Missing ${name} at ${toolPath}. Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build.`); + } + return toolPath; + } + + function archiveReleaseApk(destination: string): void { + if (!existsSync(gradleReleaseApkPath)) { + fail(`Missing built release APK at ${gradleReleaseApkPath}.`); + } + mkdirSync(apkDirectory, {recursive: true}); + copyFileSync(gradleReleaseApkPath, destination); + console.log(`Copied release APK: ${destination}`); + } + + function build(kind: BuildKind, profileOverride?: string): void { + const buildEnvironment = androidBuildEnvironment(); + const profilePath = profileOverride ?? join(profileDirectory, 'newdot.profdata'); + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); + } + + const pgoModes: Record = {release: 'off', instrumented: 'generate', optimized: 'use'}; + const gradleArguments = [ + releasePipeline.task, + '-PpatchedArtifacts.forceBuildFromSource=true', + `-PreactNativeArchitectures=${architecture}`, + `-PpgoMode=${pgoModes[kind]}`, + ...releasePipeline.arguments, + ...(emulatorVerification + ? ['--init-script', join(rootDirectory, 'scripts/pgo/androidEmulator.init.gradle'), '--max-workers=2', '-Dorg.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1024m'] + : []), + ]; + if (releasePipeline.enabled) { + console.log( + 'Android release pipeline rehearsal enables shrinking and FullStory processing, including external asset uploads. The existing local app ID and signing configuration still apply.', + ); + } else if (releasePipeline.shrinking) { + console.log('Android shrinking rehearsal enables release ProGuard rules and disables FullStory tasks. The existing local app ID and signing configuration still apply.'); + } + if (kind === 'optimized') { + gradleArguments.push(`-PpgoProfile=${profilePath}`); + } + + run( + '/usr/bin/env', + [ + BENCHMARK_SPANS_ENVIRONMENT, + 'SENTRY_DISABLE_AUTO_UPLOAD=true', + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...androidRockBuildArguments(releasePipeline.task, gradleArguments.slice(1)), + ], + join(rootDirectory, 'Mobile-Expensify'), + ); + if (JSON.stringify(androidBuildEnvironment()) !== JSON.stringify(buildEnvironment)) { + fail('Build environment changed during the Android build. Restore the selected environment file and rebuild.'); + } + archiveReleaseApk(artifactPaths[kind]); + writeFileSync( + `${artifactPaths[kind]}.build-settings.json`, + `${JSON.stringify({kind, architecture, emulatorVerification, releasePipeline: releasePipeline.enabled, appID: appID(), sourceBuiltRuntime: true, shrinking: releasePipeline.shrinking, fullStory: releasePipeline.enabled, signing: emulatorVerification ? 'debug-keystore' : 'existing-project-configuration', storeEquivalent: false, gradleArguments, buildEnvironment}, null, 2)}\n`, + ); + } + + function install(kind: BuildKind): void { + const apkPath = artifactPaths[kind]; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the APK first.`); + } + adb(['install', '-r', apkPath]); + console.log(`Installed APK: ${apkPath}`); + } + + async function waitForProfileDump(): Promise { + const deadline = Date.now() + profileDumpTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + const profileCount = /Wrote ([0-9]+) LLVM PGO profile/.exec(profileLogs)?.[1]; + if (profileCount !== undefined && Number(profileCount) !== 0) { + if (Number(profileCount) !== INSTRUMENTED_LIBRARIES.length) { + fail( + `The PGO receiver wrote ${profileCount} profiles, but expected ${INSTRUMENTED_LIBRARIES.length}. All instrumented native libraries must write successfully before recording profiles.`, + ); + } + console.log(`Wrote ${profileCount} LLVM PGO profile(s).`); + return; + } + if (profileLogs.includes('Ignoring PGO profile request in a non-instrumented build.')) { + fail('The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files.'); + } + if (profileLogs.includes('Wrote 0 LLVM PGO profile(s).')) { + fail('The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles.'); + } + await sleep(100); + } + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + fail(`The PGO receiver did not confirm a profile write within ${profileDumpTimeoutSeconds}s.\n${profileLogs}`); + } + + async function dumpProfiles(): Promise { + adb(['logcat', '-c']); + adb(['shell', 'am', 'broadcast', '-a', PROFILE_BROADCAST_ACTION, '-n', androidProfileReceiverComponent(appID())]); + await waitForProfileDump(); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(rawProfileDirectory, {recursive: true}); + adb(['pull', `${deviceProfileDirectory()}/.`, rawProfileDirectory]); + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in ${deviceProfileDirectory()}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + function verifyInstrumentation(): void { + const apkPath = artifactPaths.instrumented; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the instrumented release first.`); + } + + const extractedDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-apk-')); + const llvmReadelf = llvmTool('llvm-readelf'); + const llvmNm = llvmTool('llvm-nm'); + const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); + try { + for (const library of INSTRUMENTED_LIBRARIES) { + const apkEntry = `lib/${architecture}/${library}`; + if (!apkEntries.has(apkEntry)) { + fail(`Missing expected ${architecture} library in APK: ${apkEntry}`); + } + const extractedLibrary = join(extractedDirectory, library); + writeFileSync(extractedLibrary, captureBinary('unzip', ['-p', apkPath, apkEntry])); + const sectionHeaders = capture(llvmReadelf, ['-SW', extractedLibrary]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => sectionHeaders.includes(section))) { + fail(`LLVM PGO instrumentation is missing from ${apkEntry}.`); + } + const dynamicSymbols = capture(llvmNm, ['-D', '--defined-only', extractedLibrary]); + if (!['expensify_llvm_profile_set_filename', 'expensify_llvm_profile_write_file'].every((symbol) => dynamicSymbols.includes(symbol))) { + fail(`LLVM PGO profile-writing APIs are not exported from ${apkEntry}.`); + } + if (library === 'libExpensifyNitroUtils.so' && !dynamicSymbols.includes('Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles')) { + fail(`The PGO JNI writer is missing from ${apkEntry}. Rebuild the native module before installing.`); + } + console.log(`Verified LLVM PGO instrumentation: ${apkEntry}`); + } + } finally { + rmSync(extractedDirectory, {recursive: true, force: true}); + } + } + + return { + name: 'android', + profileDirectory, + rawProfileDirectory, + mergedProfilePath: join(profileDirectory, 'newdot.profdata'), + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + appID, + build, + install, + verifyInstrumentation, + clearDeviceProfiles: async () => { + console.log(`Clearing previous device PGO profiles from ${deviceProfileDirectory()}.`); + adb(['shell', `rm -f '${deviceProfileDirectory()}'/newdot-*.profraw`]); + }, + dumpProfiles, + pullProfiles, + llvmTool, + }; +} + +/** Keep release processing separate from bootstrap artifacts without rewriting local signing or identifiers. */ +function androidReleasePipeline(value?: string, shrinkValue?: string): {enabled: boolean; shrinking: boolean; artifactDirectory: string; task: string; apkTask: string; arguments: string[]} { + if (value && value !== 'true' && value !== 'false') { + fail('PGO_ANDROID_RELEASE_PIPELINE must be true or false.'); + } + if (shrinkValue && shrinkValue !== 'true' && shrinkValue !== 'false') { + fail('PGO_ANDROID_SHRINK must be true or false.'); + } + if (value === 'true' && shrinkValue === 'true') { + fail('Choose PGO_ANDROID_RELEASE_PIPELINE or PGO_ANDROID_SHRINK, not both.'); + } + if (shrinkValue === 'true') { + return { + enabled: false, + shrinking: true, + artifactDirectory: 'apk-shrunk', + task: ':packageRelease', + apkTask: 'packageRelease', + arguments: [...FULLSTORY_EXCLUSIONS, '-PpgoSkipFullStory=true', '--init-script', join(rootDirectory, 'scripts/pgo/androidRelease.init.gradle')], + }; + } + if (value !== 'true') { + return {enabled: false, shrinking: false, artifactDirectory: 'apk', task: ':packageRelease', apkTask: 'packageRelease', arguments: [...FULLSTORY_EXCLUSIONS]}; + } + return { + enabled: true, + shrinking: true, + artifactDirectory: 'apk-release-pipeline', + task: ':fullstoryRelease', + apkTask: 'fullstoryRelease', + arguments: ['--init-script', join(rootDirectory, 'scripts/pgo/androidRelease.init.gradle')], + }; +} + +/** Rock prefixes the application module and parses the extra Gradle arguments as one quoted argument list. */ +function androidRockBuildArguments(task: string, gradleArguments: string[]): string[] { + return [ + 'build:android', + '--local', + '--verbose', + '--variant', + 'release', + '--tasks', + // Mobile-Expensify only runs buildGrunt, which bundles OldDot's shared JS into the APK assets, for install, assemble and bundle tasks. + `buildGrunt,${task.replace(/^:/, '')}`, + '--extra-params', + gradleArguments.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + +/** Resolve LLVM tools from the build's explicit NDK or host-specific SDK installation. */ +function androidLlvmToolPath( + name: string, + {hostPlatform = platform(), homeDirectory = homedir(), environment = process.env}: {hostPlatform?: NodeJS.Platform; homeDirectory?: string; environment?: NodeJS.ProcessEnv} = {}, +): string { + const configuredNdkRoot = [environment.ANDROID_NDK_HOME, environment.ANDROID_NDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const configuredSdkRoot = [environment.ANDROID_HOME, environment.ANDROID_SDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const sdkRoot = configuredSdkRoot ?? join(homeDirectory, hostPlatform === 'linux' ? 'Android/Sdk' : 'Library/Android/sdk'); + const ndkRoot = configuredNdkRoot ?? join(sdkRoot, 'ndk/27.1.12297006'); + const hostTag = hostPlatform === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; + return join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); +} + +function androidApplicationID(buildGradle: string, configuredAppID?: string): string { + if (configuredAppID) { + return configuredAppID; + } + const applicationID = /defaultConfig\s*\{[\s\S]*?applicationId\s+["']([^"']+)["']/.exec(buildGradle)?.[1]; + if (!applicationID) { + fail('Unable to read the release application ID from Mobile-Expensify/Android/build.gradle. Pass it with --app-id.'); + } + return applicationID; +} + +function androidProfileReceiverComponent(appID: string): string { + return `${appID}/${ANDROID_JAVA_NAMESPACE}.PgoProfileReceiver`; +} + +export default createAndroidPgoAdapter; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidLlvmToolPath, androidProfileReceiverComponent, androidReleasePipeline, androidRockBuildArguments}; diff --git a/scripts/pgo/androidEmulator.init.gradle b/scripts/pgo/androidEmulator.init.gradle new file mode 100644 index 000000000000..a96c11cb1e8c --- /dev/null +++ b/scripts/pgo/androidEmulator.init.gradle @@ -0,0 +1,28 @@ +// Disposable verification builds use the bundled workload and the Android debug key. +gradle.beforeProject { application -> + application.plugins.withId('com.facebook.react') { + application.extensions.getByName('react').entryFile.set( + application.file('../../src/PgoVerificationApp.tsx') + ) + } + application.plugins.withId('com.android.application') { + application.extensions.getByName('androidComponents').finalizeDsl { android -> + def release = android.buildTypes.getByName('release') + release.signingConfig = android.signingConfigs.getByName('debug') + // R8 needs more heap than a hosted runner can spare beside the emulator, and it only shrinks Java and Kotlin code, which native PGO does not change. + release.minifyEnabled = false + } + } + application.tasks.configureEach { task -> + if (task.name.toLowerCase().startsWith('fullstory')) { + task.enabled = false + } + } + // Without a Gradle cache, the app sends every Maven Central lookup through repo.grails.org, which answers hosted runners with HTTP 429. + // Grails only mirrors Maven Central for this build, so ask Maven Central directly. + application.repositories.withType(MavenArtifactRepository).configureEach { repository -> + if (repository.url.host == 'repo.grails.org') { + repository.url = 'https://repo.maven.apache.org/maven2/' + } + } +} diff --git a/scripts/pgo/androidRelease.init.gradle b/scripts/pgo/androidRelease.init.gradle new file mode 100644 index 000000000000..ed3bdaf1c414 --- /dev/null +++ b/scripts/pgo/androidRelease.init.gradle @@ -0,0 +1,22 @@ +// Apply release processing without changing the owner's local application ID or signing configuration. +gradle.beforeProject { application -> + def skipFullStory = application.providers.gradleProperty('pgoSkipFullStory').getOrElse('false') == 'true' + if (skipFullStory) { + application.tasks.configureEach { task -> + if (task.name.toLowerCase().startsWith('fullstory')) { + task.enabled = false + } + } + } + application.plugins.withId('com.android.application') { + application.extensions.getByName('androidComponents').finalizeDsl { android -> + def release = android.buildTypes.findByName('release') + if (release == null) { + throw new GradleException('The PGO release pipeline requires a release build type.') + } + release.minifyEnabled = true + release.proguardFiles android.getDefaultProguardFile('proguard-android-optimize.txt'), application.file('proguard-rules.pro') + application.logger.lifecycle("PGO shrinking enabled; FullStory tasks ${skipFullStory ? 'disabled' : 'enabled'}; existing signing and application ID preserved.") + } + } +} diff --git a/scripts/pgo/candidateResolve.ts b/scripts/pgo/candidateResolve.ts new file mode 100644 index 000000000000..91f2ed43650f --- /dev/null +++ b/scripts/pgo/candidateResolve.ts @@ -0,0 +1,186 @@ +// cspell:ignore chrispader +/** Resolve immutable trusted candidates; no candidate code runs on the hosted resolver. */ +import type {EvaluationPlan, EvaluationPlatform} from './evaluationSchema'; + +import {isGitID, isRecord, parseCandidate} from './evaluationSchema'; + +type GitHubRead = (path: string) => Promise; +const repository = 'Expensify/App'; +const previewBranch = '@chrispader/perf/pgo-ci-verification'; +const producerName = 'Deploy code to staging or production'; + +async function resolveProducer(metadata: unknown, run: unknown, jobs: unknown, toolingSha: string, read: GitHubRead, platform?: EvaluationPlatform): Promise { + if ( + !isRecord(metadata) || + metadata.schema !== 1 || + metadata.repository !== repository || + !isRecord(run) || + run.name !== producerName || + run.path !== '.github/workflows/deploy.yml' || + run.status !== 'completed' || + !isRecord(run.repository) || + run.repository.full_name !== repository || + !isRecord(jobs) || + !Array.isArray(jobs.jobs) || + !isGitID(toolingSha) + ) { + throw new Error('Untrusted or incomplete producer run.'); + } + if ( + !validID(metadata.runID) || + !validID(metadata.runAttempt) || + !validID(run.id) || + !validID(run.run_attempt) || + !['staging', 'production'].includes(String(run.head_branch)) || + !['push', 'workflow_dispatch'].includes(String(run.event)) || + metadata.runID !== run.id || + metadata.runAttempt !== run.run_attempt || + typeof metadata.nativeEligible !== 'boolean' + ) { + throw new Error('Producer metadata does not match its immutable run attempt.'); + } + const candidate = parseCandidate(metadata.candidate); + if (candidate.environment === 'preview') { + throw new Error('Producer environment must be staging or production.'); + } + await verifyRepositoryCandidate(candidate.appSha, candidate.mobileSha, candidate.version, read); + const comparison = await read(`/repos/${repository}/compare/${candidate.appSha}...${candidate.environment}`); + if (!isRecord(comparison) || !['ahead', 'identical'].includes(String(comparison.status))) { + throw new Error('Producer candidate is not an ancestor of its environment.'); + } + const nativeJobs = jobs.jobs; + const platforms: EvaluationPlatform[] = ['android', 'ios'].filter((value): value is EvaluationPlatform => { + if (!metadata.nativeEligible || (platform && value !== platform)) { + return false; + } + const names = value === 'android' ? ['Build Android HybridApp', 'Build Android HybridApp / Build Android app'] : ['Build iOS HybridApp', 'Build iOS HybridApp / Build iOS HybridApp']; + return nativeJobs.some((job: unknown) => isRecord(job) && names.includes(String(job.name)) && job.conclusion === 'success'); + }); + return { + schema: 1, + repository, + toolingSha, + candidate, + producerRunID: Number(metadata.runID), + producerRunAttempt: Number(metadata.runAttempt), + platforms, + checklistIssueNumber: await matchingChecklist(candidate.version, read), + }; +} + +async function resolvePreview(event: unknown, toolingSha: string, read: GitHubRead, platform?: EvaluationPlatform): Promise { + if ( + !isRecord(event) || + !isRecord(event.pull_request) || + !isRecord(event.pull_request.head) || + !isRecord(event.pull_request.head.repo) || + !isRecord(event.pull_request.base) || + !isRecord(event.pull_request.base.repo) || + event.pull_request.head.repo.full_name !== repository || + event.pull_request.base.repo.full_name !== repository || + event.pull_request.head.ref !== previewBranch || + !isGitID(event.pull_request.head.sha) || + !isGitID(toolingSha) + ) { + throw new Error('Credential devices require the approved same-repository preview branch.'); + } + const sha = event.pull_request.head.sha; + const current = await read(`/repos/${repository}/pulls/${Number(event.number)}`); + if (!isRecord(current) || !isRecord(current.head) || current.head.sha !== sha || current.state !== 'open') { + throw new Error('Preview head changed or pull request is closed.'); + } + const candidate = await repositoryCandidate(sha, read); + return { + schema: 1, + repository, + toolingSha, + candidate: {...candidate, environment: 'preview'}, + producerRunID: null, + producerRunAttempt: null, + platforms: platform ? [platform] : ['android', 'ios'], + checklistIssueNumber: null, + }; +} + +async function repositoryCandidate(sha: string, read: GitHubRead): Promise<{appSha: string; mobileSha: string; version: string}> { + const tree = await read(`/repos/${repository}/git/trees/${sha}`); + const packageFile = await read(`/repos/${repository}/contents/package.json?ref=${sha}`); + if (!isRecord(tree) || !Array.isArray(tree.tree) || !isRecord(packageFile) || typeof packageFile.content !== 'string') { + throw new Error('Missing candidate tree or package metadata.'); + } + const mobile: unknown = tree.tree.find((entry: unknown) => isRecord(entry) && entry.path === 'Mobile-Expensify' && entry.mode === '160000'); + if (!isRecord(mobile) || !isGitID(mobile.sha)) { + throw new Error('Candidate has no immutable Mobile-Expensify gitlink.'); + } + const pkg: unknown = JSON.parse(Buffer.from(packageFile.content, 'base64').toString('utf8')); + if (!isRecord(pkg) || typeof pkg.version !== 'string') { + throw new Error('Missing candidate version.'); + } + return parseCandidate({appSha: sha, mobileSha: mobile.sha, version: pkg.version, environment: 'preview'}); +} + +async function verifyRepositoryCandidate(sha: string, mobile: string, version: string, read: GitHubRead): Promise { + const actual = await repositoryCandidate(sha, read); + if (actual.mobileSha !== mobile || actual.version !== version) { + throw new Error('Producer metadata differs from the actual candidate checkout.'); + } +} + +async function matchingChecklist(version: string, read: GitHubRead): Promise { + const matches: number[] = []; + for (let page = 1; page <= 100; page += 1) { + const issues = await read(`/repos/${repository}/issues?labels=StagingDeployCash&state=all&per_page=100&page=${page}`); + if (!Array.isArray(issues)) { + throw new Error('Invalid deployment checklist response.'); + } + for (const issue of issues) { + if (!checklistMatches(issue, version) || !isRecord(issue) || !validID(issue.number)) { + continue; + } + matches.push(issue.number); + } + if (matches.length > 1) { + throw new Error('Multiple deployment checklists match the exact candidate version.'); + } + if (issues.length < 100) { + return matches.at(0) ?? null; + } + } + throw new Error('Historical checklist search exceeds its bounded pagination limit.'); +} + +function assertProducerEventAttempt(event: unknown, run: unknown): void { + if (isRecord(event) && isRecord(event.workflow_run) && isRecord(run) && event.workflow_run.id === run.id && event.workflow_run.run_attempt === run.run_attempt) { + return; + } + throw new Error('Producer event was superseded by another run or attempt.'); +} + +function selectProducerArtifact(input: unknown, runID: number, runAttempt: number): Record & {id: number} { + if (!isRecord(input) || !Array.isArray(input.artifacts) || typeof input.total_count !== 'number' || input.total_count > 100) { + throw new Error('Invalid producer artifact enumeration.'); + } + const artifacts: unknown[] = input.artifacts.filter((item: unknown) => isRecord(item) && item.name === `pgo-native-candidate-${runAttempt}` && item.expired === false); + const artifact = artifacts.at(0); + if (artifacts.length !== 1 || !isRecord(artifact) || !isRecord(artifact.workflow_run) || artifact.workflow_run.id !== runID || !validID(artifact.id)) { + throw new Error('Missing or ambiguous immutable producer metadata.'); + } + return {...artifact, id: artifact.id}; +} + +function validID(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +function checklistMatches(issue: unknown, version: string): boolean { + if (!isRecord(issue) || typeof issue.body !== 'string' || !Array.isArray(issue.labels) || issue.pull_request) { + return false; + } + return ( + issue.labels.some((label: unknown) => label === 'StagingDeployCash' || (isRecord(label) && label.name === 'StagingDeployCash')) && + issue.body.match(/\*\*Release Version:\*\*\s*`([^`]+)`/)?.at(1) === `${version}-staging` + ); +} + +export {assertProducerEventAttempt, selectProducerArtifact, checklistMatches, matchingChecklist, repositoryCandidate, resolvePreview, resolveProducer}; +export type {GitHubRead}; diff --git a/scripts/pgo/copy-ci-preview.sh b/scripts/pgo/copy-ci-preview.sh new file mode 100755 index 000000000000..e1c10a04495d --- /dev/null +++ b/scripts/pgo/copy-ci-preview.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# cspell:ignore chrispader +set -euo pipefail + +# Push the pinned native revision first so the App push can start CI immediately. +branch='@chrispader/perf/pgo-ci-verification' +repository=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT + +app_sha=$(git -C "$repository" rev-parse HEAD) +mobile_sha=$(git -C "$repository" ls-tree HEAD Mobile-Expensify | awk '{print $3}') +test -n "$mobile_sha" || { echo 'Run this script from the combined PGO PR branch.' >&2; exit 1; } + +git init --quiet --bare "$temporary/native.git" +git -C "$temporary/native.git" fetch --no-tags git@github.com:margelo/Mobile-Expensify.git "$mobile_sha" +test "$(git -C "$temporary/native.git" rev-parse FETCH_HEAD)" = "$mobile_sha" +git -C "$temporary/native.git" push git@github.com:Expensify/Mobile-Expensify.git "FETCH_HEAD:refs/heads/$branch" +git -C "$repository" push git@github.com:Expensify/App.git "$app_sha:refs/heads/$branch" + +echo 'Open Expensify/App > Actions > Verify native PGO on emulators to see the run.' diff --git a/scripts/pgo/emulator.ts b/scripts/pgo/emulator.ts new file mode 100644 index 000000000000..42479101dd44 --- /dev/null +++ b/scripts/pgo/emulator.ts @@ -0,0 +1,166 @@ +// cspell:ignore sdcard profdata profraw simctl +import {appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync} from 'node:fs'; +import {join} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; +import type {VerificationMarker} from './emulatorFixture'; +import type {PlatformAdapter} from './shared'; + +import {capture, fail, findFiles, parseChoice, rootDirectory, run, runAllowFailure, sleep} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import {parseVerificationMarker, VERIFICATION_MARKER} from './emulatorFixture'; +import createIOSPgoAdapter from './ios'; +import {bootIOSVerificationSimulator, createIOSSimulatorDevice} from './iosSimulator'; +import {fileHash, profileCounts} from './proof'; + +/** Verify the real native PGO build cycle with a self-running synthetic workload. */ +async function verifyEmulator(platform: PlatformName): Promise { + const output = join(rootDirectory, '.pgo/ci', platform); + rmSync(output, {recursive: true, force: true}); + mkdirSync(output, {recursive: true}); + process.env.PGO_CI_EMULATOR = 'true'; + process.env.PGO_ANDROID_RELEASE_PIPELINE = 'false'; + process.env.PGO_ANDROID_SHRINK = 'false'; + let simulatorIdentifier: string | undefined; + try { + if (platform === 'android') { + process.env.PGO_ANDROID_ARCH = 'x86_64'; + } else { + simulatorIdentifier = bootIOSVerificationSimulator(); + process.env.PGO_IOS_SIMULATOR_ID = simulatorIdentifier; + } + const identifier = simulatorIdentifier ?? androidEmulatorIdentifier(); + const adapter = platform === 'android' ? createAndroidPgoAdapter(undefined, identifier) : createIOSPgoAdapter(); + + adapter.build('release'); + adapter.build('instrumented'); + adapter.verifyInstrumentation(); + adapter.install('instrumented'); + const workload = createEmulatorWorkload(adapter, identifier); + await workload(); + for (let repetition = 1; repetition <= 3; repetition++) { + await adapter.clearDeviceProfiles(); + await workload(); + await adapter.dumpProfiles(); + adapter.pullProfiles(); + cpSync(adapter.rawProfileDirectory, join(output, 'raw', String(repetition)), {recursive: true}); + } + + const profiles = findFiles(join(output, 'raw'), '.profraw').sort(); + if (profiles.length < 3) { + fail('Each emulator training repetition must produce a native profile.'); + } + const llvm = adapter.llvmTool('llvm-profdata'); + run(llvm, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + if (adapter.profileFormat) { + writeFileSync(`${adapter.mergedProfilePath}.format`, `${adapter.profileFormat}\n`); + } + const profileReport = capture(llvm, ['show', '--all-functions', '--counts', adapter.mergedProfilePath]); + const counts = profileCounts(profileReport); + writeFileSync(join(output, 'profile.txt'), profileReport); + cpSync(adapter.mergedProfilePath, join(output, 'verification.profdata')); + const profileHash = fileHash(adapter.mergedProfilePath); + adapter.build('optimized', adapter.mergedProfilePath); + if (fileHash(adapter.mergedProfilePath) !== profileHash) { + fail('The training profile changed during the optimized build.'); + } + + const samples: Array<{kind: 'release' | 'optimized'; workload: VerificationMarker}> = []; + for (const kind of ['release', 'optimized', 'optimized', 'release'] as const) { + adapter.install(kind); + samples.push({kind, workload: await workload()}); + } + writeFileSync( + join(output, 'result.json'), + `${JSON.stringify( + { + schema: 1, + status: 'passed', + platform, + target: platform === 'android' ? 'android-emulator-x86_64' : `ios-simulator-${process.arch}`, + workload: 'synthetic', + deployable: false, + appSHA: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileSHA: capture('git', ['rev-parse', 'HEAD'], join(rootDirectory, 'Mobile-Expensify')).trim(), + profile: {sha256: profileHash, ...counts}, + artifacts: Object.fromEntries(Object.entries(adapter.artifactPaths).map(([kind, path]) => [kind, fileHash(path)])), + samples, + }, + null, + 2, + )}\n`, + ); + publishSummary( + `Native PGO ${platform} emulator verification passed. Baseline, instrumentation, three training runs, profile merge and optimized rebuild completed. Both builds completed the synthetic workload. These samples do not establish phone performance or select a deployment artifact.\n`, + output, + ); + } catch (error) { + writeFileSync(join(output, 'result.json'), `${JSON.stringify({schema: 1, status: 'failed', platform, workload: 'synthetic', deployable: false}, null, 2)}\n`); + publishSummary(`Native PGO ${platform} emulator verification failed. Open the failed step's log for the build or workload error.\n`, output); + throw error; + } finally { + if (simulatorIdentifier) { + runAllowFailure('xcrun', ['simctl', 'shutdown', simulatorIdentifier]); + runAllowFailure('xcrun', ['simctl', 'delete', simulatorIdentifier]); + } + } +} + +function createEmulatorWorkload(adapter: PlatformAdapter, identifier: string): () => Promise { + const simulator = adapter.name === 'ios' ? createIOSSimulatorDevice(identifier, adapter.appID) : undefined; + const marker = `/sdcard/Android/data/${adapter.appID()}/cache/${VERIFICATION_MARKER}`; + const adb = (args: string[]) => run('adb', ['-s', identifier, ...args]); + return async () => { + if (simulator) { + simulator.removeFile(`Library/Caches/${VERIFICATION_MARKER}`); + simulator.launch(); + } else { + adb(['shell', 'am', 'force-stop', adapter.appID()]); + adb(['shell', 'rm', '-f', marker]); + adb(['shell', 'monkey', '-p', adapter.appID(), '-c', 'android.intent.category.LAUNCHER', '1']); + } + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + let text: string | undefined; + if (simulator) { + text = simulator.readFile(`Library/Caches/${VERIFICATION_MARKER}`); + } else if (runAllowFailure('adb', ['-s', identifier, 'shell', 'test', '-f', marker])) { + text = capture('adb', ['-s', identifier, 'shell', 'cat', marker]); + } + if (text) { + const input: unknown = JSON.parse(text); + return parseVerificationMarker(input); + } + await sleep(500); + } + fail('The bundled emulator workload did not finish within 90 seconds.'); + }; +} + +function androidEmulatorIdentifier(): string { + const devices = capture('adb', ['devices']) + .split('\n') + .flatMap((line) => { + const identifier = /^(emulator-\d+)\s+device$/.exec(line.trim())?.[1]; + return identifier ? [identifier] : []; + }); + const identifier = devices.length === 1 ? devices.at(0) : undefined; + if (!identifier || capture('adb', ['-s', identifier, 'shell', 'getprop', 'ro.product.cpu.abi']).trim() !== 'x86_64') { + fail('PGO CI requires one booted x86_64 Android emulator.'); + } + return identifier; +} + +function publishSummary(text: string, output: string): void { + writeFileSync(join(output, 'summary.md'), text); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); + } +} + +if (import.meta.main) { + const platform = parseChoice(process.argv.at(2) ?? '', ['android', 'ios'], 'Emulator platform'); + await verifyEmulator(platform); +} + +export default verifyEmulator; diff --git a/scripts/pgo/emulatorFixture.ts b/scripts/pgo/emulatorFixture.ts new file mode 100644 index 000000000000..1cd4a2ab4a9d --- /dev/null +++ b/scripts/pgo/emulatorFixture.ts @@ -0,0 +1,34 @@ +import {isRecord} from '@libs/ObjectUtils'; + +const VERIFICATION_SCREENS = ['Inbox', 'Report', 'Expenses', 'Search', 'Inbox'] as const; +const VERIFICATION_ROW_COUNT = 240; +const VERIFICATION_MARKER = 'pgo-ci-workload.json'; +type VerificationMarker = {schema: 1; status: 'passed'; workload: 'synthetic'; screens: number; rows: number; durationMs: number}; + +function parseVerificationMarker(input: unknown): VerificationMarker { + if ( + !isRecord(input) || + input.schema !== 1 || + input.status !== 'passed' || + input.workload !== 'synthetic' || + input.screens !== VERIFICATION_SCREENS.length || + input.rows !== VERIFICATION_ROW_COUNT || + typeof input.durationMs !== 'number' || + !Number.isFinite(input.durationMs) || + input.durationMs <= 0 + ) { + throw new Error('The emulator workload did not complete every synthetic screen.'); + } + return {schema: 1, status: 'passed', workload: 'synthetic', screens: input.screens, rows: input.rows, durationMs: input.durationMs}; +} + +function verificationRows(screen: number) { + return Array.from({length: VERIFICATION_ROW_COUNT}, (_, index) => ({ + id: index + 1, + title: `CI report ${index + 1}`, + amount: ((index * 173 + screen * 31) % 10000) / 100, + })).sort((a, b) => (screen % 2 === 0 ? a.id - b.id : b.amount - a.amount)); +} + +export {parseVerificationMarker, verificationRows, VERIFICATION_MARKER, VERIFICATION_ROW_COUNT, VERIFICATION_SCREENS}; +export type {VerificationMarker}; diff --git a/scripts/pgo/evaluationReport.ts b/scripts/pgo/evaluationReport.ts new file mode 100644 index 000000000000..64eef208ae21 --- /dev/null +++ b/scripts/pgo/evaluationReport.ts @@ -0,0 +1,330 @@ +// cspell:ignore cooldown +/** Validate and summarize public numeric evaluation evidence without selecting a release binary. */ +import {isDeepStrictEqual} from 'node:util'; + +import type {BenchmarkSample} from '../lib/benchmarkStatistics'; +import type {GitHubRead} from './candidateResolve'; +import type {EvaluationPlan, EvaluationPlatform} from './evaluationSchema'; + +import {benchmarkStats, percentile} from '../lib/benchmarkStatistics'; +import {checklistMatches} from './candidateResolve'; +import {candidateKey, isRecord, isSha256} from './evaluationSchema'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; + +// The hyphenated scenario name is the persisted benchmark schema. +// eslint-disable-next-line @typescript-eslint/naming-convention +const scenarios = {startup: ['ManualAppStartup'], journey: [...JOURNEY_SPAN_NAMES], 'heavy-journey': [...HEAVY_JOURNEY_SPAN_NAMES]}; +const publicToolchainKeys = [ + 'compilerVersion', + 'profileToolVersion', + 'sdkVersion', + 'buildToolVersion', + 'identitySha256', + 'reviewedOverlaySha256', + 'fixtureSha256', + 'environmentSha256', + 'releasePodLockSha256', + 'instrumentedPodLockSha256', + 'optimizedPodLockSha256', +]; + +async function evaluationReport(plan: EvaluationPlan, inputs: unknown[], read: GitHubRead) { + const platforms = plan.platforms.map((platform) => { + const candidates = inputs.filter((input) => isRecord(input) && input.platform === platform); + if (candidates.length > 1) { + throw new Error('Duplicate platform evaluation evidence.'); + } + const input = candidates.at(0); + if (!input) { + return {platform, status: 'missing' as const, cohorts: undefined}; + } + return summarizePlatform(plan, platform, input); + }); + const status = platforms.every((entry) => entry.status === 'passed') ? 'complete' : 'incomplete'; + const limitations = [ + 'Evaluation only: canonical baseline artifacts and deployment selection are unchanged.', + 'Measurements retain every sample, including IQR outliers; summaries are descriptive, not a significance or causal benefit claim.', + 'Heavy report first layout can contain a skeleton and does not prove content readiness. Warm navigation and the heavy fixture overlap training.', + 'Device temperature, CI repeatability, store equivalence and isolated scrolling improvement are unverified.', + 'Native artifacts are local evaluation builds with the reviewed tooling overlay; the source fingerprint is not a claim of a pristine candidate checkout.', + 'Compiler configuration and binary/profile hashes alone do not establish actual profile consumption or full coverage.', + ]; + const summary = { + schema: 1, + status, + candidate: plan.candidate, + toolingSha: plan.toolingSha, + producerRunID: plan.producerRunID, + producerRunAttempt: plan.producerRunAttempt, + platforms, + limitations, + }; + const heading = `PGO evaluation for ${plan.candidate.version} (${plan.candidate.environment})`; + const rows = platforms.flatMap((entry) => + entry.cohorts + ? entry.cohorts.flatMap((cohort) => + cohort.statistics.map( + (metric) => + `| ${entry.platform} | ${cohort.scenario} | ${cohort.order} | ${metric.span} | ${metric.release.average.toFixed(2)} | ${metric.optimized.average.toFixed(2)} | ${metric.meanEffectPercent?.toFixed(2) ?? 'N/A'}% |`, + ), + ) + : [`| ${entry.platform} | ${entry.status} | — | — | — | — | — |`], + ); + const markdown = [ + heading, + '', + `Candidate App: ${plan.candidate.appSha}; Mobile: ${plan.candidate.mobileSha}. Tooling: ${plan.toolingSha}.`, + '', + `Evidence status: ${status}. Positive effects mean (baseline − optimized) / baseline × 100.`, + '', + '| Platform | Workload | Order | Span | Baseline mean ms | Optimized mean ms | Effect |', + '| --- | --- | --- | --- | ---: | ---: | ---: |', + ...rows, + '', + 'The attached JSON contains all sanitized raw samples, mean, median, p10/p90, min/max, quartiles, IQR outlier counts and provenance. Both orders are reported separately.', + '', + ...limitations.map((text) => `- ${text}`), + '', + ].join('\n'); + let targetIssueNumber: number | null = null; + if (plan.checklistIssueNumber !== null) { + const issue = await read(`/repos/Expensify/App/issues/${plan.checklistIssueNumber}`); + if (checklistMatches(issue, plan.candidate.version)) { + targetIssueNumber = plan.checklistIssueNumber; + } + } + const marker = ``; + const runLink = process.env.GITHUB_RUN_ID ? `https://github.com/Expensify/App/actions/runs/${process.env.GITHUB_RUN_ID}` : 'Workflow run link unavailable.'; + const body = `Chris's AI co-pilot reporting the CI evaluation. 🤖\n\n${marker}\n${markdown}\nEvaluation run: ${runLink}\n`; + if (body.length > 32000) { + throw new Error('Evaluation comment exceeds the bounded report size.'); + } + return { + summary, + markdown, + comment: { + marker, + body, + targetIssueNumber, + version: plan.candidate.version, + skippedReason: targetIssueNumber === null ? 'No exact historical deployment checklist matches this candidate.' : null, + }, + }; +} + +function summarizePlatform(plan: EvaluationPlan, platform: EvaluationPlatform, input: unknown) { + if (!isRecord(input) || input.schema !== 1 || input.platform !== platform || !isDeepStrictEqual(input.candidate, plan.candidate) || input.toolingSha !== plan.toolingSha) { + throw new Error('Platform result differs from the exact evaluation candidate.'); + } + if (input.status === 'failed') { + const stages = [ + 'candidate-guard', + 'account-preflight', + 'build-release', + 'build-instrumented', + 'verify-instrumented', + 'install-instrumented', + 'training', + 'validate-training', + 'build-optimized', + ...['startup', 'journey', 'heavy-journey'].flatMap((scenario) => ['release-first', 'optimized-first'].map((order) => `${scenario}-${order}`)), + ]; + if (input.failureStage !== undefined && (typeof input.failureStage !== 'string' || !stages.includes(input.failureStage))) { + throw new Error('Invalid categorical failure stage.'); + } + return {platform, status: 'failed' as const, failureStage: input.failureStage}; + } + if ( + input.status !== 'passed' || + !isRecord(input.artifactSha256) || + !['release', 'instrumented', 'optimized'].every((kind) => isSha256(input.artifactSha256 && isRecord(input.artifactSha256) ? input.artifactSha256[kind] : undefined)) || + !isSha256(input.profileSha256) || + !isSha256(input.sourceIdentity) || + !isRecord(input.toolchain) || + !Array.isArray(input.cohorts) || + input.cohorts.length !== 6 || + !['verified', 'unverified'].includes(String(input.compilerConsumption)) + ) { + throw new Error('Incomplete platform provenance.'); + } + const toolchain = Object.fromEntries( + publicToolchainKeys.flatMap((key) => { + const value = isRecord(input.toolchain) ? input.toolchain[key] : undefined; + if (value === undefined) { + return []; + } + if (typeof value !== 'string' || value.length > 2000 || (key.endsWith('Sha256') ? !isSha256(value) : !/^[\w .():,+#-]+$/.test(value))) { + throw new Error('Toolchain field is not a public version or digest.'); + } + return [[key, value]]; + }), + ); + const cohorts = input.cohorts.map(summarizeCohort); + for (const scenario of Object.keys(scenarios)) { + const orders = cohorts + .filter((cohort) => cohort.scenario === scenario) + .map((cohort) => cohort.order) + .sort(); + if (!isDeepStrictEqual(orders, ['optimized-first', 'release-first'])) { + throw new Error('Expected complete opposite orders for each workload.'); + } + } + return { + platform, + status: 'passed' as const, + artifactSha256: {release: String(input.artifactSha256.release), instrumented: String(input.artifactSha256.instrumented), optimized: String(input.artifactSha256.optimized)}, + profileSha256: input.profileSha256, + sourceIdentity: input.sourceIdentity, + toolchain, + compilerConsumption: input.compilerConsumption, + cohorts, + }; +} + +function summarizeCohort(input: unknown) { + if ( + !isRecord(input) || + (input.scenario !== 'startup' && input.scenario !== 'journey' && input.scenario !== 'heavy-journey') || + !['release-first', 'optimized-first'].includes(String(input.order)) || + input.runs !== 20 || + input.warmUpRunsPerBinary !== 1 || + input.cooldownSecondsPerBinary !== 5 || + !isSha256(input.releaseCsvSha256) || + !isSha256(input.optimizedCsvSha256) + ) { + throw new Error('Invalid evaluation cohort metadata.'); + } + const scenario = input.scenario; + const spans = scenarios[scenario]; + if (!isDeepStrictEqual(input.spans, spans)) { + throw new Error('Unexpected evaluation spans.'); + } + const release = numericSamples(input.release, spans); + const optimized = numericSamples(input.optimized, spans); + const statistics = spans.map((span) => { + const baseline = distribution(release.filter((sample) => sample.span === span).map((sample) => sample.durationMs)); + const pgo = distribution(optimized.filter((sample) => sample.span === span).map((sample) => sample.durationMs)); + return {span, release: baseline, optimized: pgo, meanEffectPercent: baseline.average > 0 ? (100 * (baseline.average - pgo.average)) / baseline.average : null}; + }); + return { + scenario, + order: String(input.order), + runs: 20, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: 5, + releaseCsvSha256: input.releaseCsvSha256, + optimizedCsvSha256: input.optimizedCsvSha256, + release, + optimized, + statistics, + frames: frameCoverage(input.frames), + }; +} + +function numericSamples(input: unknown, spans: readonly string[]): BenchmarkSample[] { + if (!Array.isArray(input) || input.length !== 20 * spans.length) { + throw new Error('Incomplete or extra timing samples.'); + } + const samples = input.map((sample: unknown) => { + if ( + !isRecord(sample) || + !Number.isSafeInteger(sample.run) || + typeof sample.span !== 'string' || + !spans.includes(sample.span) || + typeof sample.durationMs !== 'number' || + !Number.isFinite(sample.durationMs) || + sample.durationMs < 0 + ) { + throw new Error('Invalid timing sample.'); + } + return {run: Number(sample.run), span: sample.span, durationMs: sample.durationMs}; + }); + for (const span of spans) { + if ( + !isDeepStrictEqual( + samples.filter((sample) => sample.span === span).map((sample) => sample.run), + Array.from({length: 20}, (_, index) => index + 1), + ) + ) { + throw new Error('Timing samples must be exactly ordered 1 through 20.'); + } + } + return samples; +} + +function frameCoverage(input: unknown) { + if (input === undefined) { + return {status: 'unverified' as const}; + } + if (!isRecord(input)) { + throw new Error('Invalid numeric frame coverage.'); + } + return { + status: 'recorded' as const, + release: summarizeFrames(input.release), + optimized: summarizeFrames(input.optimized), + meaning: 'Native janky frames, not FPS or literal dropped frames; variable automation window and reset isolation are unverified.', + }; +} + +function summarizeFrames(input: unknown) { + if (!Array.isArray(input) || input.length !== 20) { + throw new Error('Frame coverage must record every measured run.'); + } + const samples = input.map((sample: unknown, index) => { + if (!isRecord(sample) || sample.run !== index + 1) { + throw new Error('Frame coverage is not ordered.'); + } + if (sample.status === 'unavailable') { + return {run: index + 1, status: 'unavailable' as const}; + } + const {totalFrames, jankyFrames, originalPercent, sampleWindowMs} = sample; + if ( + sample.status !== 'available' || + typeof totalFrames !== 'number' || + typeof jankyFrames !== 'number' || + typeof originalPercent !== 'number' || + typeof sampleWindowMs !== 'number' || + ![totalFrames, jankyFrames, originalPercent, sampleWindowMs].every(Number.isFinite) || + !Number.isInteger(totalFrames) || + !Number.isInteger(jankyFrames) || + totalFrames <= 0 || + jankyFrames < 0 || + jankyFrames > totalFrames || + originalPercent < 0 || + originalPercent > 100 || + sampleWindowMs <= 0 + ) { + throw new Error('Invalid available native frame sample.'); + } + return {run: index + 1, status: 'available' as const, totalFrames, jankyFrames, originalPercent, sampleWindowMs}; + }); + const available = samples.filter((sample) => sample.status === 'available'); + const sumTotalFrames = available.reduce((total, sample) => total + sample.totalFrames, 0); + const sumJankyFrames = available.reduce((total, sample) => total + sample.jankyFrames, 0); + return { + requestedSamples: 20, + availableSamples: available.length, + unavailableSamples: 20 - available.length, + samples, + summary: available.length + ? { + sumTotalFrames, + sumJankyFrames, + weightedJankyPercent: (100 * sumJankyFrames) / sumTotalFrames, + originalPercent: distribution(available.map((sample) => sample.originalPercent)), + sampleWindowMs: distribution(available.map((sample) => sample.sampleWindowMs)), + } + : null, + }; +} + +function distribution(values: readonly number[]) { + const stats = benchmarkStats(values); + const q1 = percentile(values, 0.25); + const q3 = stats.p75; + const iqr = q3 - q1; + return {...stats, median: stats.p50, p10: percentile(values, 0.1), q1, q3, iqr, iqrOutliers: values.filter((value) => value < q1 - 1.5 * iqr || value > q3 + 1.5 * iqr).length}; +} + +export {frameCoverage, summarizeFrames, distribution, evaluationReport, numericSamples, summarizeCohort, summarizePlatform}; diff --git a/scripts/pgo/evaluationRun.ts b/scripts/pgo/evaluationRun.ts new file mode 100644 index 000000000000..2ba20dbbdbc7 --- /dev/null +++ b/scripts/pgo/evaluationRun.ts @@ -0,0 +1,371 @@ +// cspell:ignore cooldown PodLock profdata + +import {spawnSync} from 'node:child_process'; +/** Executes an isolated evaluation; private device evidence never enters the public result. */ +import {createHash} from 'node:crypto'; +import {appendFileSync, closeSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, writeFileSync} from 'node:fs'; +import {join, resolve} from 'node:path'; +import {pathToFileURL} from 'node:url'; + +import type {BenchmarkSample} from '../lib/benchmarkStatistics'; +import type {EvaluationCohort, EvaluationPlan, EvaluationPlatform, EvaluationResult, FrameObservation} from './evaluationSchema'; +import type {PlatformAdapter} from './shared'; + +import {isRecord} from './evaluationSchema'; + +type ProofModule = {sourceIdentity: () => string; toolchainIdentity: (adapter: PlatformAdapter) => string; fileHash: (path: string) => string}; +type AdapterModule = {default: (appID: string, device: string) => PlatformAdapter}; +type TrainingModule = {validateTraining: (platform: EvaluationPlatform, batch: string) => {profile: string}}; + +const OVERLAY_FILES = ['android.ts', 'ios.ts', 'shared.ts', 'pgo.ts', 'journey.ts', 'journeyBenchmark.ts', 'journeyWorkload.ts', 'proof.ts', 'localProof.ts', 'androidRelease.init.gradle']; +const SCENARIOS = [ + {name: 'heavy-journey', action: 'benchmark-heavy-journey', spans: ['/r/*']}, + {name: 'journey', action: 'benchmark-journey', spans: ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab']}, + {name: 'startup', action: 'benchmark', spans: ['ManualAppStartup']}, +] as const; + +async function executeEvaluation(plan: EvaluationPlan, platform: EvaluationPlatform, publicDirectory: string): Promise { + const candidate = requiredDirectory('PGO_CANDIDATE_DIRECTORY'); + const tooling = requiredDirectory('PGO_TOOLING_DIRECTORY'); + const privateDirectory = join(candidate, '.pgo', 'evaluation', `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`, platform); + mkdirSync(privateDirectory, {recursive: true}); + mkdirSync(publicDirectory, {recursive: true}); + const result: EvaluationResult = { + schema: 1, + platform, + candidate: plan.candidate, + toolingSha: plan.toolingSha, + status: 'failed', + artifactSha256: {release: '', instrumented: '', optimized: ''}, + profileSha256: '', + sourceIdentity: '', + toolchain: {}, + compilerConsumption: 'unverified', + cohorts: [], + }; + let stage = 'candidate-guard'; + const runPrivate = (file: string, args: string[]) => privateCommand(process.execPath, [join(candidate, 'scripts/pgo', file), ...args], candidate, privateDirectory, stage); + try { + verifyCandidate(plan, candidate, tooling); + if (!plan.platforms.includes(platform)) { + throw new Error('Platform not eligible.'); + } + const device = requiredValue(`PGO_${platform.toUpperCase()}_DEVICE_ID`); + const appID = requiredValue(`PGO_${platform.toUpperCase()}_APP_ID`); + const fixture = requiredValue(`PGO_${platform.toUpperCase()}_FIXTURE_PATH`); + // The fixture is privately provisioned on the runner; never print or copy its contents. + const fixtureHash = sha256(readFileSync(fixture)); + process.env.ENVFILE = resolve(candidate, 'Mobile-Expensify', requiredValue(`PGO_${platform.toUpperCase()}_ENVFILE`)); + appendFileSync(join(candidate, '.git/info/exclude'), '\n/evaluation-tooling/\n/pgo-plan/\n/pgo-public/\n'); + const overlay = OVERLAY_FILES.map((file) => { + const source = join(tooling, 'scripts/pgo', file); + const hash = sha256(readFileSync(source)); + cpSync(source, join(candidate, 'scripts/pgo', file)); + return {file, sha256: hash}; + }); + result.toolchain.reviewedOverlaySha256 = sha256(Buffer.from(JSON.stringify(overlay))); + configureSafeEvaluationOptions(candidate); + const proof: unknown = await import(pathToFileURL(join(candidate, 'scripts/pgo/proof.ts')).href); + const adapterModule: unknown = await import(pathToFileURL(join(candidate, `scripts/pgo/${platform}.ts`)).href); + if (!isProofModule(proof) || !isAdapterModule(adapterModule)) { + throw new Error('Reviewed evaluation module contract is missing.'); + } + const adapter = adapterModule.default(appID, device); + result.sourceIdentity = proof.sourceIdentity(); + const toolchain = proof.toolchainIdentity(adapter); + result.toolchain.identitySha256 = sha256(Buffer.from(toolchain)); + result.toolchain.fixtureSha256 = fixtureHash; + const environmentFile = process.env.ENVFILE; + result.toolchain.environmentSha256 = sha256(readFileSync(environmentFile)); + let profilePath: string | undefined; + const guard = () => { + verifyCandidate(plan, candidate, tooling); + if (proof.sourceIdentity() !== result.sourceIdentity || proof.toolchainIdentity(adapter) !== toolchain || sha256(readFileSync(fixture)) !== fixtureHash) { + throw new Error('Frozen evaluation inputs changed.'); + } + if (sha256(readFileSync(environmentFile)) !== result.toolchain.environmentSha256) { + throw new Error('Build environment changed.'); + } + if (profilePath && proof.fileHash(profilePath) !== result.profileSha256) { + throw new Error('Frozen training profile changed.'); + } + for (const kind of ['release', 'instrumented', 'optimized'] as const) { + if (result.artifactSha256[kind] && proof.fileHash(adapter.artifactPaths[kind]) !== result.artifactSha256[kind]) { + throw new Error('Frozen artifact changed.'); + } + } + }; + const deviceArgs = ['--device', device, '--app-id', appID]; + const podLockPath = join(candidate, 'Mobile-Expensify/iOS/Podfile.lock'); + const originalPodLock = platform === 'ios' ? readFileSync(podLockPath) : undefined; + let generatedPodLockHash: string | undefined; + const build = (kind: 'release' | 'instrumented' | 'optimized', profile?: string) => { + try { + runPrivate('pgo.ts', [platform, `build-${kind}`, ...deviceArgs, ...(profile ? ['--profile', profile] : [])]); + if (originalPodLock) { + const generated = readFileSync(podLockPath); + const hash = sha256(generated); + writeFileSync(join(privateDirectory, `${kind}-Podfile.lock`), generated, {mode: 0o600}); + result.toolchain[`${kind}PodLockSha256`] = hash; + if (generatedPodLockHash && hash !== generatedPodLockHash) { + throw new Error('Compiled pod dependency lock changed between build modes.'); + } + generatedPodLockHash = hash; + } + } finally { + if (originalPodLock) { + writeFileSync(podLockPath, originalPodLock); + } + } + }; + stage = 'account-preflight'; + runPrivate('journey.ts', [platform, '--fixture', fixture, ...deviceArgs, '--preflight']); + for (const kind of ['release', 'instrumented'] as const) { + stage = `build-${kind}`; + guard(); + build(kind); + result.artifactSha256[kind] = proof.fileHash(adapter.artifactPaths[kind]); + guard(); + } + stage = 'verify-instrumented'; + runPrivate('pgo.ts', [platform, 'verify-instrumented', ...deviceArgs]); + stage = 'install-instrumented'; + runPrivate('pgo.ts', [platform, 'install-instrumented', ...deviceArgs]); + process.env.PGO_INSTRUMENTED_BINARY_SHA256 = result.artifactSha256.instrumented; + stage = 'training'; + const journeyRoot = join(candidate, '.pgo', platform, 'journeys'); + const previousJourneys = directoryNames(journeyRoot); + runPrivate('journey.ts', [platform, '--fixture', fixture, ...deviceArgs, '--runs', '3', '--collect']); + guard(); + const trainingBatch = newBatch(journeyRoot, previousJourneys); + stage = 'validate-training'; + const validation: unknown = await import(pathToFileURL(join(candidate, 'scripts/pgo/localProof.ts')).href); + if (!isTrainingModule(validation)) { + throw new Error('Reviewed training validator is missing.'); + } + const training = validation.validateTraining(platform, trainingBatch); + profilePath = training.profile; + result.profileSha256 = proof.fileHash(training.profile); + stage = 'build-optimized'; + build('optimized', training.profile); + result.artifactSha256.optimized = proof.fileHash(adapter.artifactPaths.optimized); + guard(); + for (const {order, scenario} of benchmarkStagePlan()) { + stage = `${scenario.name}-${order}`; + const root = join(adapter.benchmarkDirectory, scenario.name); + const before = directoryNames(root); + runGuardedStage(guard, () => runPrivate('pgo.ts', [platform, scenario.action, '20', '60', ...deviceArgs, '--fixture', fixture, '--benchmark-order', order])); + result.cohorts.push(readCohort(newBatch(root, before), scenario.name, order, [...scenario.spans], device, appID)); + } + result.status = 'passed'; + } catch { + result.failureStage = stage; + // The detailed error and raw device output stay private on the runner. + console.error(`PGO evaluation failed at ${stage}. Private runner evidence retained.`); + } finally { + writeFileSync(join(publicDirectory, `evaluation-${platform}.json`), `${JSON.stringify(result, null, 2)}\n`); + } + if (result.status !== 'passed') { + throw new Error('PGO evaluation did not complete.'); + } +} + +function readCohort(batch: string, scenario: EvaluationCohort['scenario'], order: EvaluationCohort['order'], spans: string[], device: string, appID: string): EvaluationCohort { + const metadata: unknown = JSON.parse(readFileSync(join(batch, 'run.json'), 'utf8')); + if ( + !isRecord(metadata) || + metadata.deviceID !== device || + metadata.appID !== appID || + metadata.order !== order || + metadata.runs !== 20 || + metadata.warmUpRunsPerBinary !== 1 || + metadata.cooldownSecondsPerBinary !== 5 + ) { + throw new Error('Benchmark context does not match the verified plan.'); + } + const releasePath = join(batch, 'release.csv'); + const optimizedPath = join(batch, scenario === 'startup' ? 'optimized.csv' : 'pgo-optimized.csv'); + return { + scenario, + order, + spans, + runs: 20, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: 5, + release: readSamples(releasePath, spans), + optimized: readSamples(optimizedPath, spans), + releaseCsvSha256: sha256(readFileSync(releasePath)), + optimizedCsvSha256: sha256(readFileSync(optimizedPath)), + ...(scenario === 'heavy-journey' && metadata.platform === 'android' + ? { + frames: { + release: readPublicFrames(join(batch, 'release-frames.json')), + optimized: readPublicFrames(join(batch, 'pgo-optimized-frames.json')), + }, + } + : {}), + }; +} + +function readPublicFrames(path: string): FrameObservation[] { + if (!existsSync(path)) { + return Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable'})); + } + const input: unknown = JSON.parse(readFileSync(path, 'utf8')); + if (!isRecord(input) || input.schema !== 1 || input.requestedRuns !== 20 || !Array.isArray(input.samples) || input.samples.length !== 20) { + throw new Error('Unexpected optional frame coverage.'); + } + return input.samples.map((sample: unknown, index) => { + if (!isRecord(sample) || sample.run !== index + 1) { + throw new Error('Optional frame run identity changed.'); + } + if (sample.status === 'unavailable') { + return {run: sample.run, status: 'unavailable'}; + } + const metrics = sample.metrics; + if ( + sample.status !== 'available' || + !isRecord(metrics) || + typeof metrics.totalFrames !== 'number' || + !Number.isSafeInteger(metrics.totalFrames) || + metrics.totalFrames <= 0 || + typeof metrics.droppedFrames !== 'number' || + !Number.isSafeInteger(metrics.droppedFrames) || + metrics.droppedFrames < 0 || + metrics.droppedFrames > metrics.totalFrames || + typeof metrics.droppedPercent !== 'number' || + !Number.isFinite(metrics.droppedPercent) || + metrics.droppedPercent < 0 || + metrics.droppedPercent > 100 || + typeof metrics.sampleWindowMs !== 'number' || + !Number.isFinite(metrics.sampleWindowMs) || + metrics.sampleWindowMs <= 0 + ) { + throw new Error('Invalid optional frame metrics.'); + } + return { + run: sample.run, + status: 'available', + totalFrames: metrics.totalFrames, + jankyFrames: metrics.droppedFrames, + originalPercent: metrics.droppedPercent, + sampleWindowMs: metrics.sampleWindowMs, + }; + }); +} + +function readSamples(path: string, spans: string[]): BenchmarkSample[] { + const lines = readFileSync(path, 'utf8').trim().split('\n'); + if (lines.shift() !== 'run,span,duration_ms') { + throw new Error('Unexpected benchmark CSV format.'); + } + const samples = lines.map((line) => { + const [run, span, duration] = line.trim().split(','); + return {run: Number(run), span, durationMs: Number(duration)}; + }); + if ( + samples.length !== 20 * spans.length || + samples.some( + (sample) => !Number.isInteger(sample.run) || sample.run < 1 || sample.run > 20 || !spans.includes(sample.span) || !Number.isFinite(sample.durationMs) || sample.durationMs < 0, + ) || + new Set(samples.map((sample) => `${sample.run}:${sample.span}`)).size !== samples.length + ) { + throw new Error('Expected twenty complete samples for every span.'); + } + return samples; +} + +function verifyCandidate(plan: EvaluationPlan, candidate: string, tooling: string): void { + const git = (cwd: string, args: string[]) => { + const result = spawnSync('git', args, {cwd, encoding: 'utf8'}); + if (result.status !== 0) { + throw new Error('Cannot verify immutable checkout.'); + } + return result.stdout.trim(); + }; + const packageManifest: unknown = JSON.parse(readFileSync(join(candidate, 'package.json'), 'utf8')); + if ( + !isRecord(packageManifest) || + packageManifest.version !== plan.candidate.version || + git(candidate, ['rev-parse', 'HEAD']) !== plan.candidate.appSha || + git(join(candidate, 'Mobile-Expensify'), ['rev-parse', 'HEAD']) !== plan.candidate.mobileSha || + git(tooling, ['rev-parse', 'HEAD']) !== plan.toolingSha || + git(candidate, ['ls-tree', 'HEAD', 'Mobile-Expensify']).split(/\s+/).at(2) !== plan.candidate.mobileSha + ) { + throw new Error('Immutable candidate pins do not match.'); + } +} + +function privateCommand(command: string, args: string[], cwd: string, directory: string, stage: string): void { + const descriptor = openSync(join(directory, `${stage}.log`), 'a', 0o600); + try { + const result = spawnSync(command, args, {cwd, env: process.env, stdio: ['ignore', descriptor, descriptor]}); + if (result.status !== 0) { + throw new Error('Private evaluation stage failed.'); + } + } finally { + closeSync(descriptor); + } +} + +function requiredDirectory(name: string): string { + const value = resolve(requiredValue(name)); + if (!existsSync(value)) { + throw new Error('Missing evaluation checkout.'); + } + return value; +} +function requiredValue(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error('Missing provisioned evaluation configuration.'); + } + return value; +} +function directoryNames(path: string): Set { + return new Set(existsSync(path) ? readdirSync(path) : []); +} +function newBatch(path: string, previous: Set): string { + const added = readdirSync(path).filter((name) => !previous.has(name)); + if (added.length !== 1) { + throw new Error('Expected one isolated evidence batch.'); + } + const name = added.at(0); + if (!name) { + throw new Error('Missing isolated evidence batch.'); + } + return join(path, name); +} +function sha256(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function isProofModule(input: unknown): input is ProofModule { + return isRecord(input) && typeof input.sourceIdentity === 'function' && typeof input.toolchainIdentity === 'function' && typeof input.fileHash === 'function'; +} +function isAdapterModule(input: unknown): input is AdapterModule { + return isRecord(input) && typeof input.default === 'function'; +} +function isTrainingModule(input: unknown): input is TrainingModule { + return isRecord(input) && typeof input.validateTraining === 'function'; +} + +function benchmarkStagePlan() { + return (['release-first', 'optimized-first'] as const).flatMap((order) => SCENARIOS.map((scenario) => ({order, scenario}))); +} + +function configureSafeEvaluationOptions(candidate: string): void { + process.env.PGO_ANDROID_RELEASE_PIPELINE = 'false'; + process.env.PGO_ANDROID_SHRINK = 'true'; + process.env.PGO_BENCHMARK_COOLDOWN_SECONDS = '5'; + process.env.PATH = `${join(candidate, 'node_modules/.bin')}:${process.env.PATH ?? ''}`; +} + +function runGuardedStage(guard: () => void, execute: () => T): T { + guard(); + const result = execute(); + guard(); + return result; +} + +export {benchmarkStagePlan, configureSafeEvaluationOptions, executeEvaluation, readCohort, readPublicFrames, readSamples, runGuardedStage}; diff --git a/scripts/pgo/evaluationSchema.ts b/scripts/pgo/evaluationSchema.ts new file mode 100644 index 000000000000..d0ad7c2e8253 --- /dev/null +++ b/scripts/pgo/evaluationSchema.ts @@ -0,0 +1,110 @@ +// cspell:ignore cooldown +/** Public identities and measurement records for evaluation-only PGO jobs. */ +import {createHash} from 'node:crypto'; + +import type {BenchmarkSample} from '../lib/benchmarkStatistics'; + +type EvaluationPlatform = 'android' | 'ios'; +type CandidateIdentity = {appSha: string; mobileSha: string; version: string; environment: 'staging' | 'production' | 'preview'}; +type CandidateMetadata = {schema: 1; repository: 'Expensify/App'; runID: number; runAttempt: number; nativeEligible: boolean; candidate: CandidateIdentity}; +type EvaluationPlan = { + schema: 1; + repository: 'Expensify/App'; + toolingSha: string; + candidate: CandidateIdentity; + producerRunID: number | null; + producerRunAttempt: number | null; + platforms: EvaluationPlatform[]; + checklistIssueNumber: number | null; +}; +type FrameObservation = {run: number; status: 'unavailable'} | {run: number; status: 'available'; totalFrames: number; jankyFrames: number; originalPercent: number; sampleWindowMs: number}; +type EvaluationCohort = { + scenario: 'startup' | 'journey' | 'heavy-journey'; + order: 'release-first' | 'optimized-first'; + spans: string[]; + runs: number; + warmUpRunsPerBinary: number; + cooldownSecondsPerBinary: number; + release: BenchmarkSample[]; + optimized: BenchmarkSample[]; + releaseCsvSha256: string; + optimizedCsvSha256: string; + frames?: {release: FrameObservation[]; optimized: FrameObservation[]}; +}; +type EvaluationResult = { + schema: 1; + platform: EvaluationPlatform; + candidate: CandidateIdentity; + toolingSha: string; + status: 'passed' | 'failed'; + failureStage?: string; + artifactSha256: {release: string; instrumented: string; optimized: string}; + profileSha256: string; + sourceIdentity: string; + toolchain: Record; + compilerConsumption: 'unverified' | 'verified'; + cohorts: EvaluationCohort[]; +}; + +function candidateKey(candidate: CandidateIdentity): string { + return createHash('sha256') + .update(JSON.stringify([candidate.appSha, candidate.mobileSha, candidate.version, candidate.environment])) + .digest('hex'); +} + +function parsePlan(input: unknown): EvaluationPlan { + if (!isRecord(input) || input.schema !== 1 || input.repository !== 'Expensify/App' || !isGitID(input.toolingSha) || !isRecord(input.candidate) || !Array.isArray(input.platforms)) { + throw new Error('Invalid PGO evaluation plan.'); + } + const candidate = parseCandidate(input.candidate); + const platforms = input.platforms.filter((value): value is EvaluationPlatform => value === 'android' || value === 'ios'); + if ( + platforms.length !== input.platforms.length || + new Set(platforms).size !== platforms.length || + !nullableID(input.producerRunID) || + !nullableID(input.producerRunAttempt) || + !nullableID(input.checklistIssueNumber) + ) { + throw new Error('Invalid PGO evaluation plan identities.'); + } + return { + schema: 1, + repository: 'Expensify/App', + toolingSha: input.toolingSha, + candidate, + platforms, + producerRunID: input.producerRunID, + producerRunAttempt: input.producerRunAttempt, + checklistIssueNumber: input.checklistIssueNumber, + }; +} + +function parseCandidate(input: unknown): CandidateIdentity { + if ( + !isRecord(input) || + !isGitID(input.appSha) || + !isGitID(input.mobileSha) || + typeof input.version !== 'string' || + !/^\d+\.\d+\.\d+(?:-\d+)?$/.test(input.version) || + (input.environment !== 'staging' && input.environment !== 'production' && input.environment !== 'preview') + ) { + throw new Error('Invalid immutable PGO candidate identity.'); + } + return {appSha: input.appSha, mobileSha: input.mobileSha, version: input.version, environment: input.environment}; +} + +function isRecord(input: unknown): input is Record { + return typeof input === 'object' && input !== null && !Array.isArray(input); +} +function isGitID(input: unknown): input is string { + return typeof input === 'string' && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(input); +} +function isSha256(input: unknown): input is string { + return typeof input === 'string' && /^[a-f0-9]{64}$/.test(input); +} +function nullableID(input: unknown): input is number | null { + return input === null || (typeof input === 'number' && Number.isSafeInteger(input) && input > 0); +} + +export {candidateKey, isGitID, isRecord, isSha256, parseCandidate, parsePlan}; +export type {CandidateIdentity, CandidateMetadata, EvaluationCohort, EvaluationPlan, EvaluationPlatform, EvaluationResult, FrameObservation}; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts new file mode 100644 index 000000000000..af4c40e3a43c --- /dev/null +++ b/scripts/pgo/ios.ts @@ -0,0 +1,532 @@ +// cspell:ignore Xfrontend hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled + +/** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ + +import {isRecord} from '@libs/ObjectUtils'; + +import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; + +import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter} from './shared'; + +import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllowFailure, sleep, valueAt} from '../lib/scriptUtils'; +import {createIOSSimulatorDevice} from './iosSimulator'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; + +function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { + const simulatorIdentifier = environmentString('PGO_IOS_SIMULATOR_ID'); + const architecture = simulatorIdentifier ? process.arch : 'arm64'; + const profileFormat = `ios-clang-swift-ir-v3${simulatorIdentifier ? `-simulator-${architecture}` : ''}`; + const profileDirectory = join(rootDirectory, '.pgo/ios', simulatorIdentifier ? `simulator-${architecture}` : architecture); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const appDirectory = join(profileDirectory, 'app'); + const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); + const markerDirectory = join(profileDirectory, 'markers'); + const iosDirectory = join(rootDirectory, 'Mobile-Expensify/iOS'); + const projectPath = join(iosDirectory, 'Expensify.xcodeproj/project.pbxproj'); + const hermesVersionPath = join(rootDirectory, 'node_modules/react-native/sdks/.hermesv1version'); + const remotePgoDirectory = 'Library/Caches/ExpensifyPGO'; + const profileStatusPath = `${remotePgoDirectory}/profile-status.txt`; + const profileWriteNotification = 'com.expensify.pgo.write-profiles'; + const profileClearNotification = 'com.expensify.pgo.clear-profiles'; + const profileOperationTimeoutSeconds = 10; + const developmentTeam = environmentString('IOS_DEVELOPMENT_TEAM'); + const codeSigningAllowed = !simulatorIdentifier && !['0', 'false', 'no'].includes((environmentString('IOS_CODE_SIGNING_ALLOWED') ?? 'yes').toLowerCase()); + const configuredBundleIdentifier = configuredAppID ?? environmentString('IOS_BUNDLE_IDENTIFIER'); + const mergedProfilePath = join(profileDirectory, 'newdot.profdata'); + + const artifactPaths: BuildArtifactPaths = { + release: join(appDirectory, 'Expensify-release.app'), + instrumented: join(appDirectory, 'Expensify-release-instrumented.app'), + optimized: join(appDirectory, 'Expensify-release-optimized.app'), + }; + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + let cachedDeviceIdentifier: string | undefined; + let cachedBundleIdentifier: string | undefined; + const simulator = simulatorIdentifier ? createIOSSimulatorDevice(simulatorIdentifier, bundleIdentifier) : undefined; + + function bundleIdentifier(): string { + if (cachedBundleIdentifier) { + return cachedBundleIdentifier; + } + if (configuredBundleIdentifier) { + cachedBundleIdentifier = configuredBundleIdentifier; + return configuredBundleIdentifier; + } + + const archivedApp = [artifactPaths.instrumented, artifactPaths.optimized, artifactPaths.release].find((path) => existsSync(join(path, 'Info.plist'))); + if (!archivedApp) { + fail('No archived iOS app is available to determine its bundle identifier. Build an app first, or set IOS_BUNDLE_IDENTIFIER.'); + } + cachedBundleIdentifier = capture('/usr/libexec/PlistBuddy', ['-c', 'Print :CFBundleIdentifier', join(archivedApp, 'Info.plist')]).trim(); + if (!cachedBundleIdentifier) { + fail(`The archived app at ${archivedApp} has no CFBundleIdentifier.`); + } + console.log(`Using iOS bundle identifier ${cachedBundleIdentifier}.`); + return cachedBundleIdentifier; + } + + function deviceIdentifier(): string { + if (simulatorIdentifier) { + return simulatorIdentifier; + } + if (cachedDeviceIdentifier) { + return cachedDeviceIdentifier; + } + + const configuredIdentifier = cliDeviceIdentifier ?? environmentString('IOS_DEVICE_ID'); + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-ios-devices-')); + const jsonPath = join(temporaryDirectory, 'devices.json'); + try { + run('xcrun', ['devicectl', 'list', 'devices', '--json-output', jsonPath]); + const parsedDevices: unknown = JSON.parse(readFileSync(jsonPath, 'utf8')); + if (!isRecord(parsedDevices) || !isRecord(parsedDevices.result) || !Array.isArray(parsedDevices.result.devices)) { + fail('CoreDevice returned an unexpected device-list response.'); + } + + const physicalDevices = parsedDevices.result.devices.flatMap((device) => { + if (!isRecord(device) || !isRecord(device.hardwareProperties) || !isRecord(device.deviceProperties)) { + return []; + } + const {hardwareProperties, deviceProperties} = device; + if (hardwareProperties.platform !== 'iOS' || hardwareProperties.reality !== 'physical' || deviceProperties.bootState !== 'booted') { + return []; + } + if (typeof hardwareProperties.udid !== 'string' || typeof deviceProperties.name !== 'string') { + return []; + } + const name = deviceProperties.name.trim(); + const osVersion = typeof deviceProperties.osVersionNumber === 'string' ? deviceProperties.osVersionNumber : undefined; + const aliases = [name, hardwareProperties.udid]; + if (typeof device.identifier === 'string') { + aliases.push(device.identifier); + } + if (typeof hardwareProperties.serialNumber === 'string') { + aliases.push(hardwareProperties.serialNumber); + } + if (osVersion) { + aliases.push(`${name} (${osVersion})`, `${name} (${osVersion}) (${hardwareProperties.udid})`); + } + return [{aliases, name, udid: hardwareProperties.udid}]; + }); + + if (physicalDevices.length === 0) { + fail('No booted physical iOS device is connected. Connect and unlock a device, or set IOS_DEVICE_ID.'); + } + const normalizeSelector = (value: string) => value.trim().replaceAll(/\s+/g, ' ').toLowerCase(); + const matchingDevices = configuredIdentifier + ? physicalDevices.filter(({aliases}) => aliases.some((alias) => normalizeSelector(alias) === normalizeSelector(configuredIdentifier))) + : physicalDevices; + if (matchingDevices.length === 0) { + fail(`No connected physical iOS device matches "${configuredIdentifier}". Available devices: ${physicalDevices.map(({name, udid}) => `${name} (${udid})`).join(', ')}.`); + } + if (matchingDevices.length > 1) { + fail(`Multiple physical iOS devices are connected (${physicalDevices.map(({name}) => name).join(', ')}). Set IOS_DEVICE_ID to the desired UDID.`); + } + + const selectedDevice = valueAt(matchingDevices, 0); + cachedDeviceIdentifier = selectedDevice.udid; + console.log(`Using iOS device ${selectedDevice.name} (${selectedDevice.udid}).`); + return selectedDevice.udid; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } + } + + function llvmTool(name: string): string { + return capture('xcrun', ['--find', name]).trim(); + } + + function sourceBuildEnvironment(mode: PgoMode, profilePath = mergedProfilePath): string[] { + const environment = [ + 'BUILD_RN_FROM_SOURCE=1', + 'RCT_USE_RN_DEP=0', + 'RCT_USE_PREBUILT_RNCORE=0', + 'RCT_BUILD_HERMES_FROM_SOURCE=true', + BENCHMARK_SPANS_ENVIRONMENT, + 'SENTRY_DISABLE_AUTO_UPLOAD=true', + `EXPENSIFY_PGO_MODE=${mode}`, + ]; + if (mode === 'use') { + environment.push(`EXPENSIFY_PGO_PROFILE=${profilePath}`); + } + return environment; + } + + function prepareSourcePods(mode: PgoMode, profilePath: string): void { + if (!existsSync(hermesVersionPath)) { + fail(`Missing React Native's pinned Hermes version file at ${hermesVersionPath}.`); + } + const requiredPodSources = [ + {directory: join(iosDirectory, 'Pods/hermes-engine'), requiredFile: 'CMakeLists.txt'}, + {directory: join(iosDirectory, 'Pods/libdav1d'), requiredFile: 'dav1d/include/common/intops.h'}, + {directory: join(iosDirectory, 'Pods/libwebp'), requiredFile: 'src/webp/types.h'}, + ]; + for (const {directory, requiredFile} of requiredPodSources) { + if (existsSync(directory) && !existsSync(join(directory, requiredFile))) { + console.log(`Removing incomplete CocoaPods checkout: ${directory}`); + rmSync(directory, {recursive: true, force: true}); + } + } + run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); + for (const {directory, requiredFile} of requiredPodSources) { + if (!existsSync(join(directory, requiredFile))) { + fail(`CocoaPods did not install the required source file ${join(directory, requiredFile)}.`); + } + } + rmSync(join(iosDirectory, 'Pods/hermes-engine/build'), {recursive: true, force: true}); + } + + function pgoBuildSettings(kind: BuildKind, profilePath: string): string[] { + const baseSettings = ['ENABLE_CODE_COVERAGE=NO', 'CLANG_COVERAGE_MAPPING=NO', 'CLANG_USE_OPTIMIZATION_PROFILE=NO']; + if (kind === 'instrumented') { + return [ + ...baseSettings, + 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', + 'OTHER_CFLAGS=$(inherited) -fprofile-generate', + 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-generate', + 'OTHER_LDFLAGS=$(inherited) -fprofile-generate', + ]; + } + if (kind === 'optimized') { + const clangProfileFlags = `-fprofile-use=${profilePath} -Wno-error=profile-instr-unprofiled`; + return [ + ...baseSettings, + `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-use=${profilePath}`, + ]; + } + return baseSettings; + } + + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? mergedProfilePath; + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); + } + if (kind === 'optimized') { + const profileFormatPath = `${profilePath}.format`; + const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; + if (recordedFormat !== profileFormat) { + fail('The merged iOS profile does not match the current Clang and Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); + } + } + + let mode: PgoMode = 'off'; + if (kind === 'instrumented') { + mode = 'generate'; + } else if (kind === 'optimized') { + mode = 'use'; + } + prepareSourcePods(mode, profilePath); + const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); + const builtAppPath = join(derivedDataDirectory, `Build/Products/Release-${simulator ? 'iphonesimulator' : 'iphoneos'}/Expensify.app`); + const signingSettings = codeSigningAllowed + ? [ + 'CODE_SIGN_STYLE=Automatic', + 'CODE_SIGN_IDENTITY=Apple Development', + 'PROVISIONING_PROFILE=', + 'PROVISIONING_PROFILE_SPECIFIER=', + ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), + ] + : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; + const rockArguments = iosRockBuildArguments( + derivedDataDirectory, + [ + 'build', + `ARCHS=${architecture}`, + 'ONLY_ACTIVE_ARCH=YES', + 'COMPILER_INDEX_STORE_ENABLE=NO', + ...(simulator ? ['-sdk', 'iphonesimulator', '-jobs', '2', `ENTRY_FILE=${join(rootDirectory, 'src/PgoVerificationApp.tsx')}`] : []), + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), + ...signingSettings, + ...pgoBuildSettings(kind, profilePath), + ], + simulator ? 'generic/platform=iOS Simulator' : 'generic/platform=iOS', + ); + withLocalUploadPhasesDisabled(() => + run( + '/usr/bin/env', + [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], + join(rootDirectory, 'Mobile-Expensify'), + ), + ); + if (!existsSync(builtAppPath)) { + fail(`Missing built iOS app at ${builtAppPath}.`); + } + mkdirSync(appDirectory, {recursive: true}); + rmSync(artifactPaths[kind], {recursive: true, force: true}); + cpSync(builtAppPath, artifactPaths[kind], {recursive: true, preserveTimestamps: true}); + if (simulator) { + // Hosted runners keep only the installable between the three source builds. + rmSync(derivedDataDirectory, {recursive: true, force: true}); + } + console.log(`Copied release app: ${artifactPaths[kind]}`); + } + + /** Prevent local release builds from publishing source maps or FullStory assets; restore the existing project text even if Xcode fails. */ + function withLocalUploadPhasesDisabled(buildApp: () => void): void { + const replacements = [ + // Xcode expands these literal variables inside the project build phase. + { + // eslint-disable-next-line no-template-curly-in-string + command: '\\"${PODS_ROOT}/FullStory/tools/FullStoryCommandLine\\" \\"${CONFIGURATION_BUILD_DIR}/${WRAPPER_NAME}\\"', + disabled: 'echo Skipping FullStory upload for local PGO build', + }, + {command: 'source \\"$PROJECT_ROOT/tools/upload-sentry-source-maps.sh\\"', disabled: 'echo Skipping Sentry source-map upload for local PGO build'}, + ]; + const originalProject = readFileSync(projectPath, 'utf8'); + const guardedProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, command, disabled), originalProject); + let restored = false; + let guarded = false; + const restoreProject = () => { + if (restored) { + return; + } + if (!guarded) { + writeFileSync(projectPath, originalProject); + restored = true; + return; + } + const currentProject = readFileSync(projectPath, 'utf8'); + const restoredProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, disabled, command), currentProject); + writeFileSync(projectPath, restoredProject); + restored = true; + }; + const onInterrupt = () => { + restoreProject(); + process.exit(130); + }; + const onTermination = () => { + restoreProject(); + process.exit(143); + }; + process.once('SIGINT', onInterrupt); + process.once('SIGTERM', onTermination); + try { + writeFileSync(projectPath, guardedProject); + guarded = true; + buildApp(); + } finally { + try { + restoreProject(); + } finally { + process.off('SIGINT', onInterrupt); + process.off('SIGTERM', onTermination); + } + } + } + + function install(kind: BuildKind): void { + const appPath = artifactPaths[kind]; + if (!existsSync(appPath)) { + fail(`Missing app at ${appPath}. Build it first.`); + } + if (simulator) { + simulator.install(appPath); + return; + } + if (!existsSync(join(appPath, '_CodeSignature/CodeResources'))) { + fail(`The app at ${appPath} is unsigned. Rebuild without IOS_CODE_SIGNING_ALLOWED=NO before installing it on a device.`); + } + run('xcrun', ['devicectl', 'device', 'install', 'app', '--device', deviceIdentifier(), appPath]); + console.log(`Installed app: ${appPath}`); + } + + function tryReadDeviceFile(remotePath: string, localName: string): string | undefined { + if (simulator) { + return simulator.readFile(remotePath); + } + mkdirSync(markerDirectory, {recursive: true}); + const localPath = join(markerDirectory, localName); + rmSync(localPath, {recursive: true, force: true}); + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePath, + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8').trim() : undefined; + } + + function postNotification(name: string): void { + if (simulator) { + simulator.notify(name); + return; + } + run('xcrun', ['devicectl', 'device', 'notification', 'post', '--device', deviceIdentifier(), '--name', name]); + } + + async function waitForProfileStatus(expectedStatus: string, previousStatus: string | undefined): Promise { + const deadline = Date.now() + profileOperationTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const statusContents = tryReadDeviceFile(profileStatusPath, 'profile-status.txt'); + if (statusContents && statusContents !== previousStatus) { + const [, status, rawResult] = statusContents.split(','); + if (status === 'not-instrumented') { + fail('The installed iOS app is not instrumented. Build and install build-instrumented before recording profiles.'); + } + if (status === expectedStatus && Number(rawResult) === 0) { + console.log(`iOS PGO profiles ${expectedStatus}.`); + return; + } + fail(`The iOS PGO profile operation failed: ${statusContents}`); + } + await sleep(250); + } + fail(`The iOS app did not confirm that PGO profiles were ${expectedStatus} within ${profileOperationTimeoutSeconds}s.`); + } + + async function clearDeviceProfiles(): Promise { + console.log(`Clearing previous iOS PGO profiles from ${remotePgoDirectory}.`); + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-clear.txt'); + postNotification(profileClearNotification); + await waitForProfileStatus('cleared', previousStatus); + } + + async function dumpProfiles(): Promise { + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-write.txt'); + postNotification(profileWriteNotification); + await waitForProfileStatus('written', previousStatus); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(dirname(rawProfileDirectory), {recursive: true}); + if (simulator) { + simulator.copyDirectory(remotePgoDirectory, rawProfileDirectory); + if (findFiles(rawProfileDirectory, '.profraw').length === 0) { + fail('The simulator did not produce any native PGO profiles.'); + } + return; + } + run('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePgoDirectory, + '--destination', + rawProfileDirectory, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + ]); + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in the iOS app container at ${remotePgoDirectory}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + function verifyInstrumentation(): void { + const appPath = artifactPaths.instrumented; + const executablePath = join(appPath, 'Expensify'); + if (!existsSync(executablePath)) { + fail(`Missing instrumented iOS executable at ${executablePath}.`); + } + const requiredSections = ['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names']; + const sectionHeaders = capture('xcrun', ['size', '-m', executablePath]); + if (!requiredSections.every((section) => sectionHeaders.includes(section))) { + fail('LLVM PGO instrumentation is missing from the iOS app executable.'); + } + if (!capture('strings', ['-a', executablePath]).includes('ExpensifyPGO: wrote LLVM profile')) { + fail('The iOS app executable is missing the PGO flush marker.'); + } + console.log(`Verified LLVM PGO instrumentation: ${executablePath}`); + + const hermesExecutable = join(appPath, 'Frameworks/hermesvm.framework/hermesvm'); + if (!existsSync(hermesExecutable)) { + fail(`Missing embedded source-built Hermes framework at ${hermesExecutable}.`); + } + const hermesSections = capture('xcrun', ['size', '-m', hermesExecutable]); + if (!requiredSections.every((section) => hermesSections.includes(section))) { + fail('LLVM PGO instrumentation is missing from the source-built iOS Hermes framework.'); + } + const hermesSymbols = capture('xcrun', ['nm', '-gU', hermesExecutable]); + if (!['expensify_llvm_profile_set_filename', 'expensify_llvm_profile_write_file', 'expensify_llvm_profile_reset_counters'].every((symbol) => hermesSymbols.includes(symbol))) { + fail('The source-built iOS Hermes framework is missing the exported PGO profile runtime APIs.'); + } + console.log(`Verified LLVM PGO instrumentation: ${hermesExecutable}`); + } + + return { + name: 'ios', + profileDirectory, + rawProfileDirectory, + mergedProfilePath, + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + profileFormat, + appID: bundleIdentifier, + build, + install, + verifyInstrumentation, + clearDeviceProfiles, + dumpProfiles, + pullProfiles, + llvmTool, + }; +} + +function replaceOnce(content: string, previous: string, next: string): string { + const offset = content.indexOf(previous); + if (offset < 0 || content.indexOf(previous, offset + previous.length) >= 0) { + fail('The iOS project upload phases changed. Refusing to build until the local PGO upload guard is updated.'); + } + return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; +} + +/** Preserve each Xcode setting as one argument through Rock's string argument parser. */ +function iosRockBuildArguments(derivedDataDirectory: string, settings: string[], destination = 'generic/platform=iOS'): string[] { + return [ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + destination, + '--build-folder', + derivedDataDirectory, + '--no-install-pods', + '--extra-params', + settings.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + +export {iosRockBuildArguments}; +export default createIOSPgoAdapter; diff --git a/scripts/pgo/iosSimulator.ts b/scripts/pgo/iosSimulator.ts new file mode 100644 index 000000000000..4d5eb6ed7cbe --- /dev/null +++ b/scripts/pgo/iosSimulator.ts @@ -0,0 +1,60 @@ +import {isRecord} from '@libs/ObjectUtils'; + +// cspell:ignore bootstatus simctl notifyutil UDID +import {cpSync, existsSync, readFileSync, rmSync} from 'node:fs'; +import {join} from 'node:path'; + +import {capture, fail, run} from '../lib/scriptUtils'; + +/** Create a disposable simulator from the selected Xcode's installed iOS runtime. */ +function bootIOSVerificationSimulator(): string { + const input: unknown = JSON.parse(capture('xcrun', ['simctl', 'list', 'runtimes', '--json'])); + const runtime = selectIOSVerificationRuntime(input); + const identifier = capture('xcrun', ['simctl', 'create', 'PGO verification', 'com.apple.CoreSimulator.SimDeviceType.iPhone-16', runtime]).trim(); + if (!/^[a-f0-9-]{36}$/i.test(identifier)) { + fail('Xcode did not return a simulator identifier.'); + } + run('xcrun', ['simctl', 'boot', identifier]); + run('xcrun', ['simctl', 'bootstatus', identifier, '-b']); + return identifier; +} + +function selectIOSVerificationRuntime(input: unknown): string { + if (!isRecord(input) || !Array.isArray(input.runtimes)) { + fail('Xcode returned an invalid simulator runtime list.'); + } + const runtimes = input.runtimes.flatMap((item: unknown) => { + if ( + !isRecord(item) || + item.isAvailable !== true || + typeof item.identifier !== 'string' || + typeof item.version !== 'string' || + !item.identifier.startsWith('com.apple.CoreSimulator.SimRuntime.iOS-') + ) { + return []; + } + return [{identifier: item.identifier, version: item.version}]; + }); + const runtime = runtimes.sort((a, b) => b.version.localeCompare(a.version, undefined, {numeric: true})).at(0); + if (!runtime) { + fail('The selected Xcode has no available iOS simulator runtime.'); + } + return runtime.identifier; +} + +function createIOSSimulatorDevice(identifier: string, appID: () => string) { + const containerPath = () => capture('xcrun', ['simctl', 'get_app_container', identifier, appID(), 'data']).trim(); + return { + install: (path: string) => run('xcrun', ['simctl', 'install', identifier, path]), + launch: () => run('xcrun', ['simctl', 'launch', '--terminate-running-process', identifier, appID()]), + notify: (name: string) => run('xcrun', ['simctl', 'spawn', identifier, 'notifyutil', '-p', name]), + removeFile: (path: string) => rmSync(join(containerPath(), path), {force: true}), + readFile: (path: string): string | undefined => { + const source = join(containerPath(), path); + return existsSync(source) ? readFileSync(source, 'utf8').trim() : undefined; + }, + copyDirectory: (path: string, destination: string) => cpSync(join(containerPath(), path), destination, {recursive: true}), + }; +} + +export {bootIOSVerificationSimulator, createIOSSimulatorDevice, selectIOSVerificationRuntime}; diff --git a/scripts/pgo/journey.ts b/scripts/pgo/journey.ts new file mode 100755 index 000000000000..48a52380ace3 --- /dev/null +++ b/scripts/pgo/journey.ts @@ -0,0 +1,158 @@ +#!/usr/bin/env bun + +// cspell:ignore profraw profdata + +import CLI from 'expensify-common/CLI'; +import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {dirname, join, relative, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {PLATFORM_NAMES} from '../lib/nativeAppBenchmark'; +import {capture, findFiles, parseChoice, parsePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {parseJourneyFixture} from './journeyConfig'; +import {createJourneyDevice} from './journeyDevice'; +import {prepareJourney, runJourneyWorkload, verifyJourneyAccount} from './journeyWorkload'; +import {sourceIdentity, toolchainIdentity} from './proof'; + +/** Run the same UI workload on a selected Android or iOS device; never automate authentication. */ +async function main(): Promise { + /* eslint-disable @typescript-eslint/naming-convention */ + const cli = new CLI({ + positionalArgs: [{name: 'platform', description: 'android or ios', parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform')}], + namedArgs: { + fixture: {description: 'JSON fixture identifying the approved heavy account and reports'}, + device: {description: 'Android serial or iOS UDID'}, + 'app-id': {description: 'Installed application or bundle identifier'}, + runs: {description: 'Complete journeys to run', default: 1, parse: (value) => parsePositiveInteger(value, 'Runs')}, + }, + flags: { + collect: {description: 'Flush, archive, and merge LLVM profiles from an already-installed instrumented app'}, + preflight: {description: 'Verify account and disable #focus, then stop without running the workload'}, + 'navigation-only': {description: 'Validate navigation without sending messages; cannot collect training profiles'}, + }, + }); + /* eslint-enable @typescript-eslint/naming-convention */ + const platform = parseChoice(String(cli.positionalArgs.platform), PLATFORM_NAMES, 'Platform'); + const fixtureInput: unknown = JSON.parse(readFileSync(resolve(cli.namedArgs.fixture), 'utf8')); + const fixture = parseJourneyFixture(fixtureInput); + if (cli.flags['navigation-only'] && cli.flags.collect) { + throw new Error('Navigation-only validation cannot produce a complete training profile.'); + } + if (!cli.flags.preflight && !cli.flags['navigation-only'] && !fixture.allowMessages) { + throw new Error('Confirm the personal chat with Chris before setting allowMessages=true in the fixture. Use --navigation-only to validate without sending.'); + } + const runs = parsePositiveInteger(String(cli.namedArgs.runs), 'Runs'); + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const batchDirectory = join(rootDirectory, '.pgo', platform, 'journeys', batchID); + mkdirSync(batchDirectory, {recursive: true}); + const device = createJourneyDevice({platform, device: cli.namedArgs.device, appID: cli.namedArgs['app-id'], session: `pgo-${platform}-${batchID}`}); + const adapter = platform === 'android' ? createAndroidPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device) : createIOSPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device); + const completedProfiles: string[] = []; + const evidence = { + platform, + device: cli.namedArgs.device, + appID: cli.namedArgs['app-id'], + accountEmail: fixture.accountEmail, + accountClass: fixture.accountClass, + fixture, + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + sourceIdentity: sourceIdentity(), + toolchain: toolchainIdentity(adapter), + instrumentedBinarySha256: process.env.PGO_INSTRUMENTED_BINARY_SHA256, + automationVersion: capture('agent-device', ['--version']).trim(), + requestedRuns: runs, + collectProfiles: cli.flags.collect, + navigationOnly: cli.flags['navigation-only'], + completedRuns: 0, + runDurationsSeconds: [] as number[], + status: 'running', + failure: '', + }; + const saveEvidence = () => writeFileSync(join(batchDirectory, 'result.json'), `${JSON.stringify(evidence, null, 2)}\n`); + saveEvidence(); + try { + device.open(false); + await prepareJourney(device, fixture); + if (cli.flags.preflight) { + evidence.status = 'preflight-passed'; + return; + } + if (cli.flags.collect) { + // Verify the installed build's writer before spending time on the workload; discard these setup counters. + await adapter.dumpProfiles(); + await adapter.clearDeviceProfiles(); + } + for (let index = 1; index <= runs; index += 1) { + const runID = `${batchID}-${index}`; + const runDirectory = join(batchDirectory, `run-${index}`); + mkdirSync(runDirectory); + if (index > 1) { + await prepareJourney(device, fixture); + if (cli.flags.collect) { + await adapter.clearDeviceProfiles(); + } + } + console.log(`Starting journey ${index}/${runs}.`); + const startedAt = Date.now(); + device.open(true); + await runJourneyWorkload(device, fixture, runID, cli.flags['navigation-only']); + await verifyJourneyAccount(device, fixture); + if (cli.flags.collect) { + // Flush once per fresh process. Never merge partial or failed journeys. + await adapter.dumpProfiles(); + adapter.pullProfiles(); + const rawDirectory = join(runDirectory, 'raw'); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + throw new Error('The completed journey produced no LLVM profiles.'); + } + for (const profile of profiles) { + // Archive counters only, not unrelated files pulled from the app's cache directory. + const archivedProfile = join(rawDirectory, relative(adapter.rawProfileDirectory, profile)); + mkdirSync(dirname(archivedProfile), {recursive: true}); + copyFileSync(profile, archivedProfile); + completedProfiles.push(archivedProfile); + } + } + evidence.completedRuns = index; + evidence.runDurationsSeconds.push(Math.round((Date.now() - startedAt) / 1000)); + saveEvidence(); + } + if (cli.flags.collect) { + const mergedProfile = join(batchDirectory, 'journey.profdata'); + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${mergedProfile}`, ...completedProfiles]); + if (adapter.profileFormat) { + writeFileSync(`${mergedProfile}.format`, `${adapter.profileFormat}\n`); + } + writeFileSync(`${mergedProfile}.txt`, capture(adapter.llvmTool('llvm-profdata'), ['show', '--all-functions', mergedProfile])); + console.log(`Journey profile: ${mergedProfile}`); + } + evidence.status = cli.flags['navigation-only'] ? 'navigation-passed' : 'passed'; + } catch (error) { + evidence.status = 'failed'; + evidence.failure = error instanceof Error ? error.message : String(error); + throw error; + } finally { + saveEvidence(); + // Release the device and restore Android's original keyboard, including on failed attempts. + // Closing a session leaves persisted authentication intact. + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + console.log(`Journey result: ${join(batchDirectory, 'result.json')}`); + } +} + +if (import.meta.main) { + main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = message.startsWith('SIGN_IN_REQUIRED:') ? 2 : 1; + }); +} diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts new file mode 100644 index 000000000000..593c35c1011f --- /dev/null +++ b/scripts/pgo/journeyBenchmark.ts @@ -0,0 +1,392 @@ +// cspell:ignore cooldown Cooldown + +/** Runs separate read-only signed-in navigation and heavy-report benchmarks. */ + +// cspell:ignore gfxinfo + +import {isRecord} from '@libs/ObjectUtils'; + +import {createHash} from 'node:crypto'; +import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, writeFileSync} from 'node:fs'; +import {join, relative} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; +import type {JourneyFixture} from './journeyConfig'; +import type {PlatformAdapter, BenchmarkKind} from './shared'; + +import {createBenchmarkRecorder} from '../lib/benchmarkAppStartup'; +import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; +import {capture, fail} from '../lib/scriptUtils'; +import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; +import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; +import {createJourneyDevice} from './journeyDevice'; +import {openReport, scrollFastBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {benchmarkCooldownSeconds, cooldownBeforeBenchmark, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; + +type JourneyBenchmarkReader = { + begin: (spanNames: readonly string[]) => void; + collect: (spanNames: readonly string[], timeoutSeconds: number) => Promise; +}; + +type FrameHealth = {totalFrames: number; droppedFrames: number; droppedPercent: number; sampleWindowMs: number}; +type FrameHealthResult = {status: 'available'; metrics: FrameHealth; raw: unknown} | {status: 'unavailable'; reason: string; raw: unknown}; +type JourneyMeasurement = {events: BenchmarkLogEvent[]; frameTelemetry?: FrameHealthResult}; +type BenchmarkOrder = 'release-first' | 'optimized-first'; + +type JourneyScenario = { + directory: 'journey' | 'heavy-journey'; + spanNames: readonly string[]; + measure: (device: ReturnType, reader: JourneyBenchmarkReader, fixture: JourneyFixture, timeoutSeconds: number) => Promise; +}; + +const SHORT_JOURNEY: JourneyScenario = {directory: 'journey', spanNames: JOURNEY_SPAN_NAMES, measure: measureJourney}; +const HEAVY_JOURNEY: JourneyScenario = {directory: 'heavy-journey', spanNames: HEAVY_JOURNEY_SPAN_NAMES, measure: measureHeavyJourney}; + +function journeyBenchmarkPaths( + adapter: Pick, + batchID: string, + scenarioDirectory: JourneyScenario['directory'] = 'journey', +): Record { + const directory = join(adapter.benchmarkDirectory, scenarioDirectory, batchID); + return {release: join(directory, 'release.csv'), optimized: join(directory, 'pgo-optimized.csv')}; +} + +/** Installs the selected build, discards one complete warm-up, then records every requested run. */ +async function benchmarkJourneyBuild( + adapter: PlatformAdapter, + kind: BenchmarkKind, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + outputPath: string, + scenario: JourneyScenario, +): Promise { + console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); + adapter.install(kind); + await cooldownBeforeBenchmark(); + + const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-${scenario.directory}-${adapter.name}-${kind}-${Date.now()}`}); + const reader: JourneyBenchmarkReader = + adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); + try { + device.open(false); + await showTab(device, 'Inbox'); + await verifyJourneyAccount(device, fixture); + console.log(`Running one unmeasured ${kind} ${scenario.directory} warm-up.`); + await scenario.measure(device, reader, fixture, timeoutSeconds); + + const recorder = createBenchmarkRecorder({spanNames: [...scenario.spanNames], runs, outputPath}); + const frameSamples: Array = []; + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + const measurement = await scenario.measure(device, reader, fixture, timeoutSeconds); + console.log(`${kind} ${scenario.directory} ${runNumber}/${runs}: ${recorder.record(measurement.events, runNumber).join(', ')}`); + if (measurement.frameTelemetry) { + frameSamples.push({run: runNumber, ...measurement.frameTelemetry}); + if (measurement.frameTelemetry.status === 'available') { + const {totalFrames, droppedFrames, droppedPercent} = measurement.frameTelemetry.metrics; + console.log(`Android frame window: ${droppedFrames}/${totalFrames} native janky frames (${droppedPercent.toFixed(1)}%).`); + } else { + console.log(`Android frame telemetry unavailable for run ${runNumber}: ${measurement.frameTelemetry.reason}`); + } + } + } + await verifyJourneyAccount(device, fixture); + await recorder.complete(`${kind} ${scenario.directory} metrics`); + if (frameSamples.length > 0) { + writeFileSync( + outputPath.replace(/\.csv$/, '-frames.json'), + `${JSON.stringify({schema: 1, requestedRuns: runs, samples: frameSamples, resetVerified: false, isolatedScrollWindowVerified: false}, null, 2)}\n`, + ); + const rows = frameSamples.flatMap((sample) => { + if (sample.status !== 'available') { + return []; + } + const {totalFrames, droppedFrames, droppedPercent, sampleWindowMs} = sample.metrics; + return [`${sample.run},${totalFrames},${droppedFrames},${droppedPercent},${sampleWindowMs}`]; + }); + writeFileSync(outputPath.replace(/\.csv$/, '-frames.csv'), `run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs\n${rows.join('\n')}\n`); + } + } finally { + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +/** Uses a new app process for each sample; automation latency is outside the timed spans. */ +async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, _fixture: JourneyFixture, timeoutSeconds: number): Promise { + device.open(true); + await waitForTab(device, 'Account'); + // Normalize the saved Spend view before timing either tab transition. This makes the measured taps warm navigations from a known Expenses state. + await showTab(device, 'Spend'); + selectSpendSection(device, 'Expenses'); + device.waitLabel('Filters'); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + + const spendSpans = JOURNEY_SPAN_NAMES.slice(0, 2); + const inboxSpans = [JOURNEY_SPAN_NAMES[2]]; + reader.begin(spendSpans); + await showTab(device, 'Spend'); + device.waitLabel('Filters'); + const spendEvents = await reader.collect(spendSpans, timeoutSeconds); + + reader.begin(inboxSpans); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + const inboxEvents = await reader.collect(inboxSpans, timeoutSeconds); + return {events: [...spendEvents, ...inboxEvents]}; +} + +/** Opens a populated chat and fast-scrolls its history; device-command time is never a benchmark sample. */ +async function measureHeavyJourney( + device: ReturnType, + reader: JourneyBenchmarkReader, + fixture: JourneyFixture, + timeoutSeconds: number, +): Promise { + device.open(true); + await showTab(device, 'Inbox'); + await waitForTab(device, 'Account'); + await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); + const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); + // The sampler requests a reset after reading; actual reset success is not verified. + const beforeFrames = device.platform === 'android' ? readAndroidFrameHealth(device, false) : undefined; + scrollFastBothWays(device, Math.min(fixture.scrolls, 4)); + const afterFrames = device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined; + const frameTelemetry = beforeFrames && afterFrames ? frameWindowResult(beforeFrames, afterFrames) : undefined; + await showTab(device, 'Inbox'); + return {events, frameTelemetry}; +} + +/** Ancillary frame telemetry cannot erase otherwise complete native timing samples. */ +function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealthResult { + try { + return parseAndroidFrameHealth(device.command('perf', 'frames'), requireFrames); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + return {status: 'unavailable', reason, raw: {commandError: reason}}; + } +} + +function frameWindowResult(before: FrameHealthResult, after: FrameHealthResult): FrameHealthResult { + const raw = {before: before.raw, after: after.raw}; + if (before.status === 'unavailable') { + return {status: 'unavailable', reason: `Pre-scroll sampler unavailable; window boundary unverified: ${before.reason}`, raw}; + } + return {...after, raw}; +} + +function parseAndroidFrameHealth(result: unknown, requireFrames: boolean): FrameHealthResult { + const unavailable = (reason: string): FrameHealthResult => ({status: 'unavailable', reason, raw: result}); + if (!isRecord(result) || !isRecord(result.metrics) || !isRecord(result.metrics.fps) || !isRecord(result.sampling) || !isRecord(result.sampling.fps)) { + return unavailable('Android frame sampler returned an unexpected result.'); + } + if (result.sampling.fps.resetsAfterRead !== true) { + return unavailable('Android frame sampler does not declare reset-after-read support; reset success is not verified.'); + } + if (result.metrics.fps.available !== true) { + const reason = typeof result.metrics.fps.reason === 'string' ? result.metrics.fps.reason : 'No sampler reason provided.'; + return unavailable(`Android frame sampler unavailable: ${reason}`); + } + const {totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs} = result.metrics.fps; + if ( + typeof totalFrameCount !== 'number' || + typeof droppedFrameCount !== 'number' || + typeof droppedFramePercent !== 'number' || + typeof sampleWindowMs !== 'number' || + ![totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs].every(Number.isFinite) || + !Number.isInteger(totalFrameCount) || + !Number.isInteger(droppedFrameCount) || + totalFrameCount < 0 || + droppedFrameCount < 0 || + droppedFrameCount > totalFrameCount || + droppedFramePercent < 0 || + droppedFramePercent > 100 || + sampleWindowMs < 0 + ) { + return unavailable('Android frame sampler returned invalid frame counts or window.'); + } + if (requireFrames && (totalFrameCount === 0 || sampleWindowMs === 0)) { + return unavailable('No frames or positive frame window were available for this scroll sample.'); + } + return {status: 'available', metrics: {totalFrames: totalFrameCount, droppedFrames: droppedFrameCount, droppedPercent: droppedFramePercent, sampleWindowMs}, raw: result}; +} + +async function benchmarkJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, SHORT_JOURNEY, order); +} + +async function benchmarkHeavyJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, HEAVY_JOURNEY, order); +} + +async function benchmarkJourneyScenario( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + scenario: JourneyScenario, + order: BenchmarkOrder, +): Promise { + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const paths = journeyBenchmarkPaths(adapter, batchID, scenario.directory); + const directory = join(adapter.benchmarkDirectory, scenario.directory, batchID); + if (existsSync(directory)) { + fail(`Journey benchmark batch already exists: ${directory}`); + } + mkdirSync(directory, {recursive: true}); + writeFileSync( + join(directory, 'run.json'), + `${JSON.stringify( + { + platform: adapter.name, + deviceID, + appID: adapter.appID(), + accountEmail: fixture.accountEmail, + runs, + timeoutSeconds, + order, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: benchmarkCooldownSeconds(), + thermalStateVerified: false, + scenario: scenario.directory, + spans: scenario.spanNames, + ...(scenario === HEAVY_JOURNEY + ? { + scrollsPerDirection: Math.min(fixture.scrolls, 4), + scrollDurationMs: 120, + scrollTimeMeasured: false, + androidScrollFrameHealthMeasured: false, + androidFrameTelemetry: 'optional; per-sample availability recorded; reset and isolated window unverified', + iosScrollFrameHealthMeasured: false, + } + : {}), + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + artifacts: adapter.artifactPaths, + artifactSHA256: { + release: await sha256Artifact(adapter.artifactPaths.release), + optimized: await sha256Artifact(adapter.artifactPaths.optimized), + }, + }, + null, + 2, + )}\n`, + ); + console.log(`${scenario.directory} benchmark batch: ${directory}`); + const phases: BenchmarkKind[] = order === 'release-first' ? ['release', 'optimized'] : ['optimized', 'release']; + for (const [index, kind] of phases.entries()) { + console.log(`=== ${scenario.directory} benchmark phase ${index + 1}/2: ${kind} ===`); + await benchmarkJourneyBuild(adapter, kind, deviceID, fixture, runs, timeoutSeconds, paths[kind], scenario); + } + await compareJourneyBenchmarks(paths, runs, scenario, adapter.name); +} + +async function compareJourneyBenchmarks(paths: Record, runs: number, scenario: JourneyScenario, platform: PlatformAdapter['name']): Promise { + if (!existsSync(paths.release) || !existsSync(paths.optimized)) { + fail('Missing paired journey samples in this benchmark batch.'); + } + const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(paths.release), readBenchmarkSamples(paths.optimized)]); + const release = benchmarkMetrics(releaseSamples, scenario.spanNames); + const optimized = benchmarkMetrics(optimizedSamples, scenario.spanNames); + + console.log(`=== ${scenario.directory} benchmark comparison (positive percentages are faster) ===`); + for (const span of scenario.spanNames) { + const baseline = release[span]?.stats; + const pgo = optimized[span]?.stats; + if (!baseline || !pgo || baseline.runs !== runs || pgo.runs !== runs) { + fail(`Expected ${runs} paired ${span} samples in this journey benchmark batch.`); + } + const improvement = ((baseline.average - pgo.average) / baseline.average) * 100; + console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); + } + if (scenario === HEAVY_JOURNEY && platform === 'android') { + const releaseFrames = readAndroidFrameSamples(paths.release.replace(/\.csv$/, '-frames.json'), runs); + const optimizedFrames = readAndroidFrameSamples(paths.optimized.replace(/\.csv$/, '-frames.json'), runs); + console.log( + `Android frame telemetry availability: release ${releaseFrames.available}/${runs}, PGO ${optimizedFrames.available}/${runs}. Reset and isolated scroll window are unverified; no scrolling gain is inferred.`, + ); + } + console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); +} + +function readAndroidFrameSamples(path: string, runs: number): {available: number} { + const evidence: unknown = JSON.parse(readFileSync(path, 'utf8')); + if (!isRecord(evidence) || !Array.isArray(evidence.samples) || evidence.samples.length !== runs) { + fail(`Expected ${runs} Android frame telemetry statuses in ${path}.`); + } + let available = 0; + for (const [index, sample] of evidence.samples.entries()) { + if (!isRecord(sample) || sample.run !== index + 1 || !['available', 'unavailable'].includes(String(sample.status))) { + fail(`Invalid Android frame telemetry status ${index + 1} in ${path}.`); + } + if (sample.status === 'available') { + available += 1; + } + } + return {available}; +} + +async function sha256Artifact(path: string): Promise { + if (lstatSync(path).isDirectory()) { + return sha256Directory(path); + } + const hash = capture('shasum', ['-a', '256', path]).trim().split(/\s+/).at(0); + if (!hash || !/^[\da-f]{64}$/.test(hash)) { + fail(`Could not record the benchmark artifact SHA-256 for ${path}.`); + } + return hash; +} + +/** Include every bundle file, relative path, directory and symlink in a deterministic iOS artifact digest. */ +async function sha256Directory(directory: string): Promise { + const hash = createHash('sha256'); + async function visit(path: string): Promise { + for (const name of readdirSync(path).toSorted()) { + const child = join(path, name); + const details = lstatSync(child); + hash.update(relative(directory, child)).update('\0'); + if (details.isDirectory()) { + hash.update('directory\0'); + await visit(child); + } else if (details.isSymbolicLink()) { + hash.update('link\0').update(readlinkSync(child)).update('\0'); + } else if (details.isFile()) { + hash.update('file\0').update(String(details.size)).update('\0'); + for await (const chunk of createReadStream(child)) { + const bytes: unknown = chunk; + if (!Buffer.isBuffer(bytes)) { + fail(`Artifact file did not stream bytes: ${child}`); + } + hash.update(bytes); + } + } else { + fail(`Unsupported artifact entry in ${directory}: ${child}`); + } + } + } + await visit(directory); + return hash.digest('hex'); +} + +export {benchmarkHeavyJourneyAll, benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact, parseAndroidFrameHealth, frameWindowResult}; +export type {BenchmarkOrder}; diff --git a/scripts/pgo/journeyBenchmarkAndroid.ts b/scripts/pgo/journeyBenchmarkAndroid.ts new file mode 100644 index 000000000000..9aa305e40acf --- /dev/null +++ b/scripts/pgo/journeyBenchmarkAndroid.ts @@ -0,0 +1,52 @@ +/** Collects completed interactive benchmark spans from the current Android app process. */ + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {POLL_INTERVAL_MS, parseBenchmarkLogEvents, sleep} from '../lib/nativeAppBenchmark/shared'; +import {capture, run} from '../lib/scriptUtils'; + +function createAndroidJourneyBenchmarkReader(deviceID: string, appID: string) { + const adbArgs = (args: string[]) => ['-s', deviceID, ...args]; + let consumedEventCount = 0; + + function begin(): void { + run('adb', adbArgs(['logcat', '-c'])); + consumedEventCount = 0; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + if (spanNames.length === 0) { + throw new Error('Select at least one Android journey benchmark span.'); + } + const processID = capture('adb', adbArgs(['shell', 'pidof', appID])) + .trim() + .split(/\s+/) + .find((candidate) => /^\d+$/.test(candidate)); + if (!processID) { + throw new Error(`Unable to find the running Android process for ${appID}.`); + } + + const deadline = Date.now() + timeoutSeconds * 1000; + let observedSpanNames: string[] = []; + while (Date.now() < deadline) { + const logs = capture('adb', adbArgs(['logcat', `--pid=${processID}`, '-d', '-v', 'raw'])); + const events = parseBenchmarkLogEvents(logs); + if (events.length < consumedEventCount) { + throw new Error('Android benchmark logcat events were discarded during collection. Restart the journey benchmark.'); + } + const freshEvents = events.slice(consumedEventCount); + const collected = spanNames.map((spanName) => freshEvents.findLast((event) => event.span === spanName)); + observedSpanNames = [...new Set(freshEvents.map((event) => event.span))]; + if (collected.every((event) => event !== undefined)) { + consumedEventCount = events.length; + return collected.filter((event): event is BenchmarkLogEvent => event !== undefined); + } + await sleep(POLL_INTERVAL_MS); + } + throw new Error(`Timed out after ${timeoutSeconds}s waiting for Android journey benchmark spans: ${spanNames.join(', ')}. Observed: ${observedSpanNames.join(', ') || 'none'}.`); + } + + return {begin, collect}; +} + +export default createAndroidJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyBenchmarkIOS.ts b/scripts/pgo/journeyBenchmarkIOS.ts new file mode 100644 index 000000000000..f6287014ed46 --- /dev/null +++ b/scripts/pgo/journeyBenchmarkIOS.ts @@ -0,0 +1,102 @@ +/** Collects completed interactive benchmark spans from an iOS app's cache markers. */ + +// cspell:ignore devicectl + +import {existsSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {iOSBenchmarkMarkerPath, parseBenchmarkLogEvents} from '../lib/nativeAppBenchmark'; +import {runAllowFailure, sleep} from '../lib/scriptUtils'; + +const POLL_INTERVAL_MS = 250; + +/** Snapshots markers before an action, then waits for a fresh event for every requested span. */ +function createIOSJourneyBenchmarkReader(deviceID: string, appID: string) { + let previousMarkers: Map | undefined; + + function begin(spanNames: readonly string[]): void { + if (spanNames.length === 0) { + throw new Error('At least one iOS benchmark span is required.'); + } + const markers = new Map(); + for (const spanName of new Set(spanNames)) { + markers.set(spanName, readMarker(deviceID, appID, spanName)); + } + previousMarkers = markers; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + const snapshot = previousMarkers; + if (spanNames.length === 0 || !snapshot || spanNames.some((spanName) => !snapshot.has(spanName))) { + throw new Error('Snapshot every requested iOS benchmark span with begin() before the UI action.'); + } + if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) { + throw new Error('The iOS benchmark span timeout must be positive.'); + } + + const uniqueSpanNames = [...new Set(spanNames)]; + const deadline = Date.now() + timeoutSeconds * 1000; + const events = new Map(); + while (Date.now() < deadline) { + for (const spanName of uniqueSpanNames) { + if (events.has(spanName)) { + continue; + } + const marker = readMarker(deviceID, appID, spanName); + if (marker === undefined || marker === snapshot.get(spanName)) { + continue; + } + const event = parseBenchmarkLogEvents(marker).findLast((candidate) => candidate.span === spanName); + if (event) { + events.set(spanName, event); + } + } + if (events.size === uniqueSpanNames.length) { + return spanNames.map((spanName) => { + const event = events.get(spanName); + if (!event) { + throw new Error(`Missing collected iOS benchmark span ${spanName}.`); + } + return event; + }); + } + await sleep(POLL_INTERVAL_MS); + } + const missing = uniqueSpanNames.filter((spanName) => !events.has(spanName)); + throw new Error(`Timed out after ${timeoutSeconds}s waiting for iOS benchmark span(s): ${missing.join(', ')}.`); + } + + return {begin, collect}; +} + +function readMarker(deviceID: string, appID: string, spanName: string): string | undefined { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-journey-benchmark-ios-')); + const localPath = join(temporaryDirectory, 'benchmark.log'); + try { + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceID, + '--source', + iOSBenchmarkMarkerPath(spanName), + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + appID, + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8') : undefined; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } +} + +export default createIOSJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyConfig.ts b/scripts/pgo/journeyConfig.ts new file mode 100644 index 000000000000..8a6f0cb0fe6a --- /dev/null +++ b/scripts/pgo/journeyConfig.ts @@ -0,0 +1,71 @@ +import {isRecord} from '@libs/ObjectUtils'; + +type JourneyReport = { + query: string; + resultLabelPrefix: string; + title: string; +}; + +type JourneyFixture = { + accountEmail: string; + accountClass: 'heavy'; + description: string; + report: JourneyReport; + personalChat: JourneyReport; + allowMessages: boolean; + scrolls: number; + tabCycles: number; +}; + +/** Require an explicitly selected heavy account instead of silently training on whoever is signed in. */ +function parseJourneyFixture(input: unknown): JourneyFixture { + if (!isRecord(input) || input.accountClass !== 'heavy') { + throw new Error('The journey requires a fixture with accountClass="heavy", approved by the account owner.'); + } + const accountEmail = requiredString(input.accountEmail, 'accountEmail'); + if (!accountEmail.includes('@')) { + throw new Error('accountEmail must identify the approved heavy account.'); + } + const personalChat = parseReport(input.personalChat, 'personalChat'); + if (personalChat.query !== accountEmail || !personalChat.title.endsWith('(you)') || personalChat.resultLabelPrefix !== `${personalChat.title}, ${accountEmail}`) { + throw new Error('personalChat must be the approved account\'s own "(you)" chat, with its email as query and its title and email in resultLabelPrefix.'); + } + return { + accountEmail, + accountClass: 'heavy', + description: requiredString(input.description, 'description'), + report: parseReport(input.report, 'report'), + personalChat, + allowMessages: input.allowMessages === true, + scrolls: boundedInteger(input.scrolls ?? 8, 'scrolls', 4, 30), + tabCycles: boundedInteger(input.tabCycles ?? 3, 'tabCycles', 2, 10), + }; +} + +function parseReport(input: unknown, name: string): JourneyReport { + if (!isRecord(input)) { + throw new Error(`${name} must specify query, resultLabelPrefix, and title.`); + } + return { + query: requiredString(input.query, `${name}.query`), + resultLabelPrefix: requiredString(input.resultLabelPrefix, `${name}.resultLabelPrefix`), + title: requiredString(input.title, `${name}.title`), + }; +} + +function requiredString(input: unknown, name: string): string { + if (typeof input !== 'string' || input.trim().length === 0) { + throw new Error(`${name} must be a nonempty string.`); + } + return input.trim(); +} + +function boundedInteger(input: unknown, name: string, minimum: number, maximum: number): number { + if (typeof input !== 'number' || !Number.isSafeInteger(input) || input < minimum || input > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`); + } + return input; +} + +export {parseJourneyFixture}; +export type {JourneyFixture, JourneyReport}; diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts new file mode 100644 index 000000000000..89dc251fa1e2 --- /dev/null +++ b/scripts/pgo/journeyDevice.ts @@ -0,0 +1,148 @@ +import {isRecord} from '@libs/ObjectUtils'; + +import {spawnSync} from 'node:child_process'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +type JourneyNode = { + label: string; + value: string; + identifier: string; + type: string; + enabled: boolean; + width: number; + height: number; + x: number; + y: number; +}; + +type JourneyDevice = ReturnType; + +/** Execute deterministic device commands. Authentication is always handled by the user. */ +function createJourneyDevice(options: {platform: PlatformName; device: string; appID: string; session: string}) { + const targetArguments = ['--platform', options.platform, options.platform === 'android' ? '--serial' : '--udid', options.device, '--session', options.session, '--json']; + + function command(...args: string[]): unknown { + const result = spawnSync('agent-device', [...args, ...targetArguments], {encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024}); + if (result.error) { + throw new Error(`Device command ${args.at(0)} failed: ${result.error.message}`); + } + let response: unknown; + try { + response = JSON.parse(result.stdout); + } catch { + throw new Error(`Device command ${args.at(0)} did not return JSON: ${result.stderr.trim()}`); + } + if (!isRecord(response) || response.success !== true || result.status !== 0) { + const error = isRecord(response) && isRecord(response.error) ? response.error : undefined; + throw new Error(`Device command ${args.at(0)} ${args.at(1) ?? ''} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); + } + return response.data; + } + + function snapshot(): JourneyNode[] { + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const nodes = parseJourneySnapshot(command('snapshot', '-i', '--force-full'), options.appID); + assertSignedIn(nodes); + return nodes; + } catch (error) { + // The Android accessibility service can briefly return only system UI while a heavy list mounts. + // Retry this read alone; repeating a tap or send could perform the action twice. + if (attempt === 2 || !(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + command('wait', '500'); + } + } + throw new Error('Unable to read the app accessibility tree.'); + } + + function wait(selector: string): void { + try { + command('wait', selector, '30000'); + } catch (error) { + // Give sign-in a specific, actionable failure instead of continuing through an expired session. + snapshot(); + throw error; + } + } + + function press(selector: string): void { + wait(selector); + command('press', selector, '--settle'); + } + + function labels(): string[] { + return snapshot().map((node) => normalizeLabel(node.label)); + } + + function hasLabel(label: string): boolean { + return labels().includes(normalizeLabel(label)); + } + + return { + platform: options.platform, + command, + snapshot, + wait, + press, + hasLabel, + pressLabel: (label: string) => press(labelSelector(label)), + waitLabel: (label: string) => wait(labelSelector(label)), + fill: (identifier: string, text: string) => { + wait(idSelector(identifier)); + command('fill', idSelector(identifier), text, '--settle'); + }, + back: () => command('back', '--settle'), + open: (relaunch: boolean) => command('open', options.appID, ...(relaunch ? ['--relaunch'] : []), ...(options.platform === 'android' ? ['--test-ime'] : [])), + }; +} + +function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { + if (!isRecord(data) || !Array.isArray(data.nodes)) { + throw new Error('The device did not return an accessibility tree.'); + } + if (data.appBundleId !== appID) { + throw new Error('The automation is no longer attached to the expected app.'); + } + return data.nodes + .filter(isRecord) + .filter((node) => !node.bundleId || node.bundleId === appID) + .map((node) => ({ + label: typeof node.label === 'string' ? node.label : '', + value: typeof node.value === 'string' ? node.value : '', + identifier: typeof node.identifier === 'string' ? node.identifier : '', + type: typeof node.type === 'string' ? node.type : '', + enabled: node.enabled !== false, + width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, + height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + x: isRecord(node.rect) && typeof node.rect.x === 'number' ? node.rect.x : 0, + y: isRecord(node.rect) && typeof node.rect.y === 'number' ? node.rect.y : 0, + })); +} + +function assertSignedIn(nodes: JourneyNode[]): void { + const signInLabels = new Set(['Sign in', 'Email or phone number', 'Enter your email or phone number', 'Enter your magic code']); + if (nodes.some((node) => ['username', 'validateCode'].includes(node.identifier) || signInLabels.has(normalizeLabel(node.label)))) { + throw new Error('SIGN_IN_REQUIRED: Ask Chris to sign into the approved heavy account, then restart the journey. Automated sign-in is disabled.'); + } +} + +function normalizeLabel(label: string): string { + return label + .replaceAll(/[\u2066-\u2069]/g, '') + .replaceAll(/\s+/g, ' ') + .trim(); +} + +function labelSelector(label: string): string { + return `label=${JSON.stringify(label)}`; +} + +function idSelector(identifier: string): string { + return `id=${JSON.stringify(identifier)}`; +} + +export {assertSignedIn, createJourneyDevice, idSelector, labelSelector, normalizeLabel, parseJourneySnapshot}; +export type {JourneyDevice, JourneyNode}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts new file mode 100644 index 000000000000..c6a61e948860 --- /dev/null +++ b/scripts/pgo/journeyWorkload.ts @@ -0,0 +1,409 @@ +import type {JourneyFixture, JourneyReport} from './journeyConfig'; +import type {JourneyDevice, JourneyNode} from './journeyDevice'; + +import {sleep} from '../lib/scriptUtils'; +import {idSelector, normalizeLabel} from './journeyDevice'; + +/** Run only after the user has signed into and approved the fixture's heavy account. */ +async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture, runID: string, navigationOnly = false): Promise { + console.log('Scrolling the full inbox.'); + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + selectAllInboxFilter(device); + await scrollBothWays(device, fixture.scrolls, true); + + console.log('Opening and scrolling a populated report.'); + await openReport(device, fixture.report); + await scrollBothWays(device, fixture.scrolls, true, 'up'); + + console.log(navigationOnly ? 'Opening the personal chat without sending.' : 'Composing and sending messages in the personal chat.'); + await openReport(device, fixture.personalChat, true); + for (let messageNumber = 1; messageNumber <= (navigationOnly ? 0 : 2); messageNumber += 1) { + // Check the destination immediately before every write. Search-result matching alone is insufficient. + if (!device.hasLabel(fixture.personalChat.title)) { + throw new Error('The personal chat title is not visible before sending.'); + } + const composer = device.snapshot().find((node) => node.identifier === 'composer'); + if (!composer) { + throw new Error('The personal chat composer is missing.'); + } + if (composer.value && composer.value !== 'Write something...') { + throw new Error('The personal chat contains an existing draft. Clear or send it manually before running the journey.'); + } + const token = `PGO ${runID} message ${messageNumber}`; + if (device.platform === 'android') { + device.fill('composer', `${token} draft`); + } + device.fill('composer', `${token}. Answered by Codex, instructed by Chris.`); + device.pressLabel('Send'); + await waitForMessage(device, token); + } + + console.log('Scrolling Spend expenses and reports.'); + await showTab(device, 'Spend'); + selectSpendSection(device, 'Expenses'); + device.waitLabel('Filters'); + await scrollBothWays(device, fixture.scrolls, true); + selectSpendSection(device, 'Reports'); + device.waitLabel('Filters'); + await scrollBothWays(device, Math.max(4, Math.floor(fixture.scrolls / 2)), true); + selectSpendSection(device, 'Expenses'); + + console.log('Switching tabs and reopening the report.'); + for (let cycle = 0; cycle < fixture.tabCycles; cycle += 1) { + for (const tab of ['Inbox', 'Spend', 'Workspaces', 'Inbox']) { + await showTab(device, tab); + } + } + await openReport(device, fixture.report); + await showTab(device, 'Inbox'); +} + +/** Check identity before changing preferences; never sign in or switch accounts automatically. */ +async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): Promise { + // Entering Inbox first allows the app's automatic large-account focus prompt to appear before we check preferences. + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + if (!device.hasLabel(fixture.accountEmail)) { + throw new Error(`SIGN_IN_REQUIRED: Ask Chris to sign into ${fixture.accountEmail}. The journey cannot run on another account.`); + } + pressPreferences(device); + await ensureFocusDisabled(device, true); + pressInAppBack(device); + await showTab(device, 'Inbox'); + console.log('Verified the approved heavy account and disabled #focus.'); +} + +/** End-of-run verification rejects a profile if focus was enabled during the workload. */ +async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixture): Promise { + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + if (!device.hasLabel(fixture.accountEmail)) { + throw new Error(`The account changed during the journey. Expected ${fixture.accountEmail}; discard this run.`); + } + pressPreferences(device); + await ensureFocusDisabled(device, false); + pressInAppBack(device); + await showTab(device, 'Inbox'); +} + +async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): Promise { + if (device.hasLabel('Priority mode, Most recent')) { + return; + } + if (!device.hasLabel('Priority mode, #focus')) { + throw new Error('Cannot verify Priority mode. The journey requires English UI labels and #focus disabled.'); + } + if (!allowChange) { + throw new Error('#focus became enabled during the journey. Discard this run and restart after disabling it.'); + } + device.pressLabel('Priority mode, #focus'); + device.pressLabel('Most recent'); + device.pressLabel('Save'); + device.waitLabel('Priority mode, Most recent'); +} + +async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false, beforePress?: () => void): Promise { + await showTab(device, 'Inbox'); + device.press(idSelector('searchButton')); + device.fill('search-autocomplete-text-input', report.query); + const deadline = Date.now() + 30_000; + let resultLabel: string | undefined; + while (!resultLabel && Date.now() < deadline) { + resultLabel = findReportResult(device.snapshot(), report.resultLabelPrefix, isPersonalChat ? report.title : undefined); + if (!resultLabel) { + await sleep(500); + } + } + if (!resultLabel) { + throw new Error(`No search result matched ${report.resultLabelPrefix}.`); + } + beforePress?.(); + device.pressLabel(resultLabel); + device.wait(idSelector('composer')); + if (!device.hasLabel(report.title)) { + throw new Error(`The opened report does not show the expected title ${report.title}.`); + } +} + +async function showTab(device: JourneyDevice, tab: string, now = Date.now): Promise { + const deadline = now() + 30_000; + let backPresses = 0; + for (;;) { + const nodes = device.snapshot(); + const tabNode = findTabNode(nodes, tab); + if (tabNode) { + device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); + return; + } + // A completed Back may use the remaining budget. Inspect its result before timing out. + if (now() >= deadline) { + throw new Error(`Cannot return to the ${tab} tab.`); + } + // A relaunch can expose a loading view before tabs mount. Only navigate back from a real nested page. + if (backPresses < 5 && inAppBackTapPoint(nodes)) { + pressInAppBack(device, nodes); + backPresses += 1; + } + await sleep(500); + } +} + +/** Wait for a mounted bottom tab without navigating away from a partially loaded screen. */ +async function waitForTab(device: Pick, tab: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + if (findTabNode(device.snapshot(), tab)) { + return; + } + } catch (error) { + if (!(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + } + await sleep(300); + } + throw new Error(`Timed out waiting for the ${tab} bottom tab to mount.`); +} + +function selectAllInboxFilter(device: JourneyDevice): void { + const tapPoint = allFilterTapPoint(device.snapshot()); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel('All'); +} + +function pressPreferences(device: JourneyDevice): void { + const cell = device.snapshot().filter((node) => normalizeLabel(node.label) === 'Preferences' && node.type === 'Cell' && node.width > 0 && node.height > 0); + const preferenceCell = cell.length === 1 ? cell.at(0) : undefined; + if (preferenceCell) { + device.command('press', String(Math.round(preferenceCell.x + preferenceCell.width / 2)), String(Math.round(preferenceCell.y + preferenceCell.height / 2)), '--settle'); + return; + } + device.pressLabel('Preferences'); +} + +function pressInAppBack(device: JourneyDevice, nodes = device.snapshot()): void { + const tapPoint = inAppBackTapPoint(nodes); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.back(); +} + +/** Use the app's top-left Back control; it can be non-hittable on iOS while the keyboard is open. */ +function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); + if (!screen) { + return undefined; + } + const candidates = nodes.filter( + (node) => + (node.type === 'Button' || node.type === 'android.widget.Button') && + normalizeLabel(node.label) === 'Back' && + node.enabled && + node.width > 0 && + node.height > 0 && + node.x >= 0 && + node.y >= 0 && + node.x < screen.width * 0.25 && + node.y < screen.height * 0.2, + ); + if (candidates.length > 1) { + throw new Error('Ambiguous in-app Back control.'); + } + const back = candidates.at(0); + return back ? {x: Math.round(back.x + back.width / 2), y: Math.round(back.y + back.height / 2)} : undefined; +} + +function selectSpendSection(device: JourneyDevice, section: 'Expenses' | 'Reports'): void { + const tapPoint = spendSectionTapPoint(device.snapshot(), section); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel(section); +} + +/** On iOS, Expenses labels the whole horizontal strip while Reports has its own bounds. */ +function spendSectionTapPoint(nodes: JourneyNode[], section: 'Expenses' | 'Reports'): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'Expenses' && node.type === 'Cell' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const reports = nodes.find((node) => normalizeLabel(node.label) === 'Reports' && node.x > 0 && node.height > 0); + if (!reports) { + throw new Error('Cannot locate the Expenses and Reports segments in Spend.'); + } + return {x: Math.round(section === 'Reports' ? reports.x + reports.width / 2 : reports.x / 2), y: Math.round(reports.y + reports.height / 2)}; +} + +/** iOS exposes the whole filter strip as the All cell; its center lands on Unread. */ +function allFilterTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'All' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const unread = nodes.find((node) => normalizeLabel(node.label) === 'Unread' && node.x > 0 && node.height > 0); + if (!unread) { + throw new Error('Cannot locate the All filter inside the iOS Inbox strip.'); + } + return {x: Math.round(unread.x / 2), y: Math.round(unread.y + unread.height / 2)}; +} + +/** iOS appends unread/review status to tab accessibility labels. */ +function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined { + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); + const candidates = nodes.filter((node) => { + const label = normalizeLabel(node.label); + const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); + return matches && (!screen || (node.y >= screen.height * 0.7 && node.width >= screen.width * 0.1 && node.height >= screen.height * 0.05)); + }); + const first = candidates.at(0); + const duplicatedNode = + first && + candidates.every( + (node) => + node.type === first.type && + normalizeLabel(node.label) === normalizeLabel(first.label) && + node.x === first.x && + node.y === first.y && + node.width === first.width && + node.height === first.height, + ); + if (candidates.length > 1 && !duplicatedNode) { + throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); + } + return first; +} + +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { + const initialNodes = device.snapshot(); + const pixels = scrollDistance(initialNodes); + let previous = contentSignature(initialNodes); + let changed = 0; + for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { + let stationary = 0; + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', String(pixels), '--settle'); + const current = contentSignature(device.snapshot()); + if (current !== previous) { + changed += 1; + stationary = 0; + } else { + stationary += 1; + } + previous = current; + if (stationary === 2) { + break; + } + } + } + if (requireMovement && changed < 2) { + throw new Error('The list did not expose enough changing content. Use a populated heavy-account fixture; do not collect this run.'); + } +} + +/** Fixed rapid swipes with one movement check per direction, keeping accessibility polling out of each fling. */ +function scrollFastBothWays(device: Pick, count: number): void { + const initialNodes = device.snapshot(); + const pixels = String(scrollDistance(initialNodes, true)); + let previous = contentSignature(initialNodes); + for (const direction of ['up', 'down']) { + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', pixels, '--duration-ms', '120', ...(scroll === count - 1 ? ['--settle'] : [])); + } + const current = contentSignature(device.snapshot()); + if (current === previous) { + throw new Error(`The populated report did not move during fast ${direction} scrolling. Discard this benchmark run.`); + } + previous = current; + } +} + +async function waitForMessage(device: JourneyDevice, token: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const composer = nodes.find((node) => node.identifier === 'composer'); + const visibleMessage = nodes.some((node) => node.identifier !== 'composer' && !/EditText|TextField|TextViewInput/i.test(node.type) && node.label.includes(token)); + if (composer && !composer.value.includes(token) && visibleMessage) { + return; + } + await sleep(500); + } + throw new Error('The sent message did not appear with an empty composer. Do not retry the send automatically.'); +} + +function contentSignature(nodes: JourneyNode[]): string { + const screenHeight = nodes.find((node) => node.type === 'Application')?.height; + return nodes + .filter((node) => !screenHeight || (node.y + node.height > screenHeight * 0.13 && node.y < screenHeight * 0.88)) + .filter((node) => !!normalizeLabel(node.label)) + .map((node) => `${normalizeLabel(node.label)}:${Math.round(node.y / 8)}`) + .join('\n'); +} + +/** Keep both ends of the gesture inside the center of portrait lists, below Spend's fixed header. */ +function scrollDistance(nodes: JourneyNode[], fast = false): number { + const scrollAreas = nodes.filter((node) => /ScrollView|RecyclerView|ScrollArea/i.test(node.type) && node.width > 0 && node.height > 0); + const viewport = scrollAreas.toSorted((left, right) => right.width * right.height - left.width * left.height).at(0); + if (!viewport) { + throw new Error('Cannot determine scroll bounds on this device.'); + } + return Math.round(Math.min(viewport.height * (fast ? 0.65 : 0.25), viewport.width * (fast ? 1.5 : 0.6))); +} + +/** Match stable report identity while allowing the last-message preview to change. */ +function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitle?: string): string | undefined { + const identity = normalizeLabel(prefix); + // The fixture binds this title to the signed-in account and an exact email search. A populated self-chat replaces the email subtitle with its last message. + const personalPrefix = personalChatTitle ? `${normalizeLabel(personalChatTitle)}, ` : undefined; + const matches = nodes.filter((node) => { + const label = normalizeLabel(node.label); + return label === identity || label.startsWith(`${identity} •`) || (personalPrefix !== undefined && label.startsWith(personalPrefix)); + }); + if (matches.length > 1) { + throw new Error(`Ambiguous report search: ${prefix}. Use a more specific fixture.`); + } + return matches.at(0)?.label; +} + +export { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + openReport, + prepareJourney, + runJourneyWorkload, + scrollBothWays, + scrollFastBothWays, + scrollDistance, + selectSpendSection, + showTab, + spendSectionTapPoint, + verifyJourneyAccount, + waitForTab, +}; diff --git a/scripts/pgo/localProof.ts b/scripts/pgo/localProof.ts new file mode 100644 index 000000000000..3b09f2112735 --- /dev/null +++ b/scripts/pgo/localProof.ts @@ -0,0 +1,119 @@ +#!/usr/bin/env bun + +// cspell:ignore profdata profraw fprofile unbuilt + +import {existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {capture, fail, findFiles, run, valueAt} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {fileHash, hasExecutedSwiftFunctions, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney} from './proof'; + +/** Validate the immutable collected profile before optimization. */ +function validateTraining(platform: PlatformName, batch: string) { + const adapter = platform === 'android' ? createAndroidPgoAdapter() : createIOSPgoAdapter(); + const toolchain = toolchainIdentity(adapter); + const identity = sourceIdentity(); + const result: unknown = JSON.parse(readFileSync(join(batch, 'result.json'), 'utf8')); + validateJourney(result, platform, identity, toolchain); + const instrumentedBinarySha256 = fileHash(adapter.artifactPaths.instrumented); + if (!isRecord(result) || result.instrumentedBinarySha256 !== instrumentedBinarySha256) { + fail('The collected journey is not bound to the current instrumented artifact.'); + } + const profile = join(batch, 'journey.profdata'); + const counts = profileCounts(capture(adapter.llvmTool('llvm-profdata'), ['show', profile])); + if (platform === 'ios' && !hasExecutedSwiftFunctions(capture(adapter.llvmTool('llvm-profdata'), ['show', '--all-functions', '--counts', profile]))) { + fail('The merged iOS profile has no executed Swift functions. Rebuild with Swift IR instrumentation and collect fresh profiles.'); + } + const runs = [1, 2, 3].map((runNumber) => { + const paths = findFiles(join(batch, `run-${runNumber}`, 'raw'), '.profraw').sort(); + if (paths.length === 0) { + fail(`Journey ${runNumber} has no archived raw profiles.`); + } + return paths.map((path) => ({path, sha256: fileHash(path)})); + }); + const temporary = mkdtempSync(join(tmpdir(), 'pgo-proof-')); + try { + const merged = join(temporary, 'merged.profdata'); + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${merged}`, ...runs.flatMap((profiles) => profiles.map(({path}) => path))]); + if (fileHash(merged) !== fileHash(profile)) { + fail('The selected profile does not match the three archived journeys.'); + } + for (const profiles of runs) { + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${merged}`, ...profiles.map(({path}) => path)]); + profileCounts(capture(adapter.llvmTool('llvm-profdata'), ['show', merged])); + } + } finally { + rmSync(temporary, {recursive: true, force: true}); + } + if (adapter.profileFormat && readFileSync(`${profile}.format`, 'utf8').trim() !== adapter.profileFormat) { + fail('The profile format does not match the current instrumentation.'); + } + return {toolchain, identity, counts, runs, profile, instrumentedBinarySha256}; +} + +function createProof(platform: PlatformName, batch: string, binary: string, commands: string, output: string): void { + const {toolchain, identity, counts, runs, profile, instrumentedBinarySha256} = validateTraining(platform, batch); + const compilation: unknown = JSON.parse(readFileSync(commands, 'utf8')); + const {objects, configuredUnbuilt} = compilerEvidence(compilation, profile); + writeFileSync( + output, + `${JSON.stringify({schema: 1, platform, sourceIdentity: identity, toolchain, instrumentedBinarySha256, counts, runs, profile: {path: profile, sha256: fileHash(profile)}, binary: {path: binary, sha256: fileHash(binary)}, compilation: {status: 'requested-configuration', consumption: 'unverified', path: commands, sha256: fileHash(commands), objects, configuredUnbuilt}}, null, 2)}\n`, + ); +} + +/** Commands are evidence of requested flags; existing object hashes bind their outputs separately. */ +function compilerObjects(input: unknown, profile: string): Array<{path: string; sha256: string}> { + return compilerEvidence(input, profile).objects; +} + +/** Separate unbuilt configured targets from existing output hashes without claiming execution. */ +function compilerEvidence(input: unknown, profile: string): {objects: Array<{path: string; sha256: string}>; configuredUnbuilt: string[]} { + if (!Array.isArray(input)) { + fail('Expected a compilation database.'); + } + const configuredUnbuilt: string[] = []; + const objects = input.flatMap((entry: unknown) => { + if (!isRecord(entry) || typeof entry.directory !== 'string') { + fail('Invalid compilation database entry.'); + } + const args = Array.isArray(entry.arguments) && entry.arguments.every((arg: unknown) => typeof arg === 'string') ? entry.arguments.join(' ') : entry.command; + if (typeof args !== 'string' || !/-fprofile-(?:instr-)?use=/.test(args)) { + return []; + } + const selected = /-fprofile-(?:instr-)?use=(?:"([^"]+)"|'([^']+)'|([^\s]+))/.exec(args); + const selectedPath = selected?.[1] ?? selected?.[2] ?? selected?.[3]; + if (!selectedPath || resolve(entry.directory, selectedPath) !== resolve(profile)) { + fail('Compiler command consumes a different PGO profile.'); + } + const outputMatch = /(?:^|\s)-o\s+(?:"([^"]+)"|'([^']+)'|([^\s]+))/.exec(args); + const objectPath = typeof entry.output === 'string' ? entry.output : (outputMatch?.[1] ?? outputMatch?.[2] ?? outputMatch?.[3]); + if (!objectPath || !args.includes(' -c ')) { + fail('PGO evidence must include a compilation command and its output object.'); + } + const path = resolve(entry.directory, objectPath); + if (!existsSync(path)) { + configuredUnbuilt.push(path); + return []; + } + return [{path, sha256: fileHash(path)}]; + }); + if (objects.length === 0) { + fail('No matching compiler commands have existing object outputs for the selected PGO profile.'); + } + return {objects, configuredUnbuilt}; +} + +if (import.meta.main) { + const [platform, ...args] = process.argv.slice(2); + if ((platform !== 'android' && platform !== 'ios') || args.length !== 4) { + fail('Expected .'); + } + createProof(platform, resolve(valueAt(args, 0)), resolve(valueAt(args, 1)), resolve(valueAt(args, 2)), resolve(valueAt(args, 3))); +} + +export {compilerEvidence, compilerObjects, createProof, validateTraining}; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts new file mode 100755 index 000000000000..cc58511ab68e --- /dev/null +++ b/scripts/pgo/pgo.ts @@ -0,0 +1,365 @@ +#!/usr/bin/env bun + +// cspell:ignore cooldown Cooldown + +// cspell:ignore profraw Profdata profdata + +import type {TupleToUnion} from 'type-fest'; + +import CLI from 'expensify-common/CLI'; +import {copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {join, resolve} from 'node:path'; + +import type {BenchmarkStats} from '../lib/benchmarkStatistics'; +import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; +import type {BenchmarkOrder} from './journeyBenchmark'; +import type {BenchmarkKind, PlatformAdapter} from './shared'; + +import {benchmarkStartups as runBenchmarkStartups} from '../lib/benchmarkAppStartup'; +import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; +import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppBenchmark'; +import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {benchmarkHeavyJourneyAll, benchmarkJourneyAll} from './journeyBenchmark'; +import {parseJourneyFixture} from './journeyConfig'; +import {androidBuildEnvironment, fileHash, sourceIdentity, toolchainIdentity} from './proof'; +import {benchmarkCooldownSeconds, cooldownBeforeBenchmark, STARTUP_SPAN_NAME} from './shared'; + +const DEFAULT_STARTUP_RUNS = 10; +const DEFAULT_STARTUP_WAIT_SECONDS = 30; +const BENCHMARK_ORDERS = ['release-first', 'optimized-first'] as const; +const WORKFLOW_COMMANDS = [ + 'build-release', + 'build-instrumented', + 'build-optimized', + 'verify-instrumented', + 'install-release', + 'install-instrumented', + 'install-optimized', + 'record-startups', + 'benchmark-release', + 'benchmark-optimized', + 'benchmark', + 'compare-benchmarks', + 'benchmark-journey', + 'benchmark-heavy-journey', + 'dump', + 'pull', + 'merge', +] as const; + +type WorkflowCommand = TupleToUnion; + +async function main(): Promise { + // The CLI framework requires kebab-case named argument keys, which the naming-convention rule cannot express. + /* eslint-disable @typescript-eslint/naming-convention */ + const cli = new CLI({ + positionalArgs: [ + { + name: 'platform', + description: `Native platform to target (${PLATFORM_NAMES.join(', ')})`, + parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform'), + }, + { + name: 'workflow', + description: `PGO workflow to run (${WORKFLOW_COMMANDS.join(', ')})`, + parse: (value): WorkflowCommand => parseChoice(value, WORKFLOW_COMMANDS, 'Workflow'), + }, + { + name: 'runs', + description: 'Number of measured startup or journey runs', + default: DEFAULT_STARTUP_RUNS, + parse: (value) => parsePositiveInteger(value, 'Run count'), + }, + { + name: 'timeout', + description: `Seconds to wait for the ${STARTUP_SPAN_NAME} or journey benchmark spans`, + default: DEFAULT_STARTUP_WAIT_SECONDS, + parse: (value) => parsePositiveInteger(value, 'Span timeout'), + }, + ], + namedArgs: { + 'app-id': { + description: 'Application ID or bundle identifier; defaults to the bootstrapped release identifier', + required: false, + }, + device: { + description: 'Device identifier to use (adb serial on Android; CoreDevice identifier, UDID, serial number, or device name on iOS)', + required: false, + }, + fixture: { + description: 'Approved account fixture for journey benchmarks', + required: false, + }, + 'benchmark-order': { + description: 'Journey phase order; alternate it across paired batches to check build-order drift', + required: false, + }, + profile: { + description: 'Immutable merged profile to use when building the optimized app', + required: false, + }, + }, + }); + /* eslint-enable @typescript-eslint/naming-convention */ + + const {platform: platformName, workflow, runs, timeout} = cli.positionalArgs; + await runWorkflow( + parseChoice(String(platformName), PLATFORM_NAMES, 'Platform'), + parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), + requirePositiveInteger(Number(runs), 'Run count'), + requirePositiveInteger(Number(timeout), 'Span timeout'), + cli.namedArgs['app-id'], + cli.namedArgs.device, + cli.namedArgs.fixture, + cli.namedArgs.profile, + cli.namedArgs['benchmark-order'], + ); +} + +async function runWorkflow( + platformName: PlatformName, + workflow: WorkflowCommand, + runs: number, + timeoutSeconds: number, + appID?: string, + deviceIdentifier?: string, + fixturePath?: string, + profilePath?: string, + benchmarkOrder?: string, +): Promise { + const adapter = getAdapter(platformName, appID, deviceIdentifier); + + switch (workflow) { + case 'build-release': + adapter.build('release'); + return; + case 'build-instrumented': + adapter.build('instrumented'); + return; + case 'build-optimized': + adapter.build('optimized', profilePath ? resolve(profilePath) : undefined); + return; + case 'verify-instrumented': + adapter.verifyInstrumentation(); + return; + case 'install-release': + adapter.install('release'); + return; + case 'install-instrumented': + adapter.install('instrumented'); + return; + case 'install-optimized': + adapter.install('optimized'); + return; + case 'record-startups': + await recordStartups(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), runs, timeoutSeconds); + return; + case 'benchmark-release': + adapter.install('release'); + await benchmarkBuild(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), 'release', runs, timeoutSeconds); + return; + case 'benchmark-optimized': + adapter.install('optimized'); + await benchmarkBuild(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), 'optimized', runs, timeoutSeconds); + return; + case 'benchmark': + await benchmarkAll(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + case 'compare-benchmarks': + await compareBenchmarks(adapter); + return; + case 'benchmark-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + } + case 'benchmark-heavy-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkHeavyJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + } + case 'dump': + await adapter.dumpProfiles(); + return; + case 'pull': + adapter.pullProfiles(); + return; + case 'merge': + mergeProfiles(adapter); + return; + default: + fail('Unsupported workflow.'); + } +} + +function readBenchmarkOrder(value?: string): BenchmarkOrder { + return value ? parseChoice(value, BENCHMARK_ORDERS, 'Journey benchmark order') : 'release-first'; +} + +function readJourneyFixture(fixturePath?: string): ReturnType { + if (!fixturePath) { + fail('Journey benchmarks require --fixture with the approved signed-in account.'); + } + const input: unknown = JSON.parse(readFileSync(resolve(fixturePath), 'utf8')); + return parseJourneyFixture(input); +} + +function mergeProfiles(adapter: PlatformAdapter): void { + mkdirSync(adapter.profileDirectory, {recursive: true}); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail('No .profraw files found. Run dump and pull first.'); + } + + const llvmProfdata = adapter.llvmTool('llvm-profdata'); + run(llvmProfdata, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + if (adapter.profileFormat) { + writeFileSync(`${adapter.mergedProfilePath}.format`, `${adapter.profileFormat}\n`); + } + const profileReport = capture(llvmProfdata, ['show', '--all-functions', adapter.mergedProfilePath]); + writeFileSync(`${adapter.mergedProfilePath}.txt`, profileReport); + console.log(`Merged PGO profile: ${adapter.mergedProfilePath}`); +} + +async function measureStartup(adapter: NativeAppBenchmarkAdapter, waitTimeSeconds: number): Promise { + await adapter.prepareStartup('process'); + const events = await adapter.launchAndCollect({spanNames: [STARTUP_SPAN_NAME], waitTimeSeconds, waitUntilSpan: STARTUP_SPAN_NAME}); + const duration = events.find((event) => event.span === STARTUP_SPAN_NAME)?.durationMs; + if (duration === undefined) { + fail(`The ${STARTUP_SPAN_NAME} benchmark span did not complete. Rebuild the app through this PGO tool before collecting a profile.`); + } + return duration; +} + +async function recordStartups(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, runs: number, waitTimeSeconds: number): Promise { + if (adapter.name === 'ios') { + console.log('Launching the instrumented iOS app once so its PGO notification handlers are active.'); + await measureStartup(benchmarkAdapter, waitTimeSeconds); + } else { + await benchmarkAdapter.prepareStartup('process'); + } + await adapter.clearDeviceProfiles(); + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + console.log(`Recording cold-process startup ${runNumber}/${runs}.`); + const duration = await measureStartup(benchmarkAdapter, waitTimeSeconds); + console.log(`${STARTUP_SPAN_NAME}=${duration}ms`); + await adapter.dumpProfiles(); + } + + adapter.pullProfiles(); + mergeProfiles(adapter); +} + +async function benchmarkBuild(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, kind: BenchmarkKind, runs: number, waitTimeSeconds: number): Promise { + mkdirSync(adapter.benchmarkDirectory, {recursive: true}); + await cooldownBeforeBenchmark(); + await runBenchmarkStartups(benchmarkAdapter, { + mode: 'process', + spanNames: [STARTUP_SPAN_NAME], + runs, + waitTimeSeconds, + waitUntilSpan: STARTUP_SPAN_NAME, + outputPath: adapter.benchmarkPaths[kind], + }); +} + +async function benchmarkAll(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, runs: number, waitTimeSeconds: number, order: BenchmarkOrder): Promise { + const batch = join(adapter.benchmarkDirectory, 'startup', new Date().toISOString().replaceAll(/[:.]/g, '-')); + mkdirSync(batch, {recursive: true}); + writeFileSync( + join(batch, 'run.json'), + `${JSON.stringify( + { + platform: adapter.name, + deviceID: benchmarkAdapter.deviceIdentifier, + appID: adapter.appID(), + order, + runs, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: benchmarkCooldownSeconds(), + thermalStateVerified: false, + measurementDriverIdentity: sourceIdentity(), + toolchain: toolchainIdentity(adapter), + buildEnvironment: adapter.name === 'android' ? androidBuildEnvironment() : null, + artifacts: {release: adapter.artifactPaths.release, optimized: adapter.artifactPaths.optimized}, + artifactSHA256: {release: fileHash(adapter.artifactPaths.release), optimized: fileHash(adapter.artifactPaths.optimized)}, + }, + null, + 2, + )}\n`, + ); + const phases: BenchmarkKind[] = order === 'release-first' ? ['release', 'optimized'] : ['optimized', 'release']; + for (const kind of phases) { + await benchmarkAdapter.prepareStartup('process'); + console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); + adapter.install(kind); + await benchmarkBuild(adapter, benchmarkAdapter, kind, runs, waitTimeSeconds); + copyFileSync(adapter.benchmarkPaths[kind], join(batch, `${kind}.csv`)); + } + + console.log('=== Benchmark comparison ==='); + await compareBenchmarks(adapter); +} + +function percentageImprovement(releaseValue: number, optimizedValue: number): number { + return ((releaseValue - optimizedValue) / releaseValue) * 100; +} + +async function compareBenchmarks(adapter: PlatformAdapter): Promise { + const releasePath = adapter.benchmarkPaths.release; + const optimizedPath = adapter.benchmarkPaths.optimized; + if (!existsSync(releasePath) || !existsSync(optimizedPath)) { + fail('Missing benchmark data. Run benchmark-release and benchmark-optimized first.'); + } + + const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(releasePath), readBenchmarkSamples(optimizedPath)]); + const release = benchmarkMetrics(releaseSamples, [STARTUP_SPAN_NAME])[STARTUP_SPAN_NAME]?.stats; + const optimized = benchmarkMetrics(optimizedSamples, [STARTUP_SPAN_NAME])[STARTUP_SPAN_NAME]?.stats; + if (!release || !optimized) { + fail(`No ${STARTUP_SPAN_NAME} benchmark samples were found.`); + } + const numericColumns: Array> = ['average', 'p50', 'p75', 'p90', 'p95', 'p99', 'min', 'max']; + const improvements = numericColumns.map((key) => percentageImprovement(release[key], optimized[key])); + + const formatLabel = (value: string) => value.padEnd(18); + const formatCount = (value: string) => value.padStart(5); + const formatMetric = (value: string) => value.padStart(10); + const row = (label: string, count: string, values: string[]) => [formatLabel(label), formatCount(count), ...values.map(formatMetric)].join(' '); + const statsRow = (label: string, stats: BenchmarkStats) => + row( + label, + String(stats.runs), + [stats.average, stats.p50, stats.p75, stats.p90, stats.p95, stats.p99, stats.min, stats.max].map((value) => value.toFixed(2)), + ); + + console.log('Positive percentages are faster; negative percentages are regressions.'); + console.log(row('Build', 'Runs', ['Average', 'P50', 'P75', 'P90', 'P95', 'P99', 'Min', 'Max'])); + console.log(statsRow('Release', release)); + console.log(statsRow('PGO optimized', optimized)); + console.log( + row( + 'PGO improvement', + '-', + improvements.map((improvement) => `${improvement.toFixed(2)}%`), + ), + ); +} + +function getAdapter(platformName: PlatformName, appID?: string, deviceIdentifier?: string): PlatformAdapter { + return platformName === 'android' ? createAndroidPgoAdapter(appID, deviceIdentifier) : createIOSPgoAdapter(appID, deviceIdentifier); +} + +async function createBenchmarkAdapter(adapter: PlatformAdapter, deviceIdentifier?: string): Promise { + return createNativeAppBenchmarkAdapter({platform: adapter.name, rootDirectory, appID: adapter.appID(), deviceIdentifier}); +} + +if (import.meta.main) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} + +export default percentageImprovement; diff --git a/scripts/pgo/pgoCandidate.ts b/scripts/pgo/pgoCandidate.ts new file mode 100755 index 000000000000..a85cb8ff9a15 --- /dev/null +++ b/scripts/pgo/pgoCandidate.ts @@ -0,0 +1,194 @@ +#!/usr/bin/env bun +/* eslint-disable @typescript-eslint/naming-convention -- CLI and GitHub output keys use their external hyphenated schema. */ +/** Capture deployment identity or resolve a trusted evaluation plan without device access. */ +import {createHash} from 'node:crypto'; +import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join, resolve} from 'node:path'; +import {parseArgs} from 'node:util'; + +import type {GitHubRead} from './candidateResolve'; +import type {CandidateMetadata} from './evaluationSchema'; + +import {capture} from '../lib/scriptUtils'; +import {assertProducerEventAttempt, resolvePreview, resolveProducer, selectProducerArtifact} from './candidateResolve'; +import {isRecord, parseCandidate} from './evaluationSchema'; + +async function main(): Promise { + const {values, positionals} = parseArgs({ + allowPositionals: true, + options: { + output: {type: 'string'}, + environment: {type: 'string'}, + 'native-eligible': {type: 'string'}, + 'source-run-id': {type: 'string'}, + platform: {type: 'string', default: 'both'}, + }, + }); + const output = values.output; + if (!output) { + throw new Error('An explicit output path is required.'); + } + if (positionals.at(0) === 'capture') { + if (!['true', 'false'].includes(values['native-eligible'] ?? '')) { + throw new Error('Explicit native eligibility is required.'); + } + const candidate = captureCandidate(values.environment ?? '', values['native-eligible'] === 'true'); + writeJSON(output, candidate); + return; + } + if (positionals.at(0) !== 'prepare' || process.env.GITHUB_REPOSITORY !== 'Expensify/App') { + throw new Error('Expected a trusted prepare invocation in Expensify/App.'); + } + const read = createGitHubReader(); + const event: unknown = JSON.parse(readFileSync(requiredEnvironment('GITHUB_EVENT_PATH'), 'utf8')); + const toolingSha = capture('git', ['rev-parse', 'HEAD']).trim(); + const platform = values.platform === 'both' ? undefined : values.platform; + if (platform !== undefined && platform !== 'android' && platform !== 'ios') { + throw new Error('Invalid evaluation platform.'); + } + let plan; + if (process.env.GITHUB_EVENT_NAME === 'pull_request') { + plan = await resolvePreview(event, toolingSha, read, platform); + } else { + let runID = 0; + if (values['source-run-id']) { + runID = positiveID(values['source-run-id']); + } else if (isRecord(event) && isRecord(event.workflow_run)) { + runID = positiveID(String(event.workflow_run.id)); + } + if (runID === 0) { + throw new Error('An immutable producer run ID is required.'); + } + const run = await read(`/repos/Expensify/App/actions/runs/${runID}`); + if (!isRecord(run)) { + throw new Error('Invalid producer run.'); + } + const jobs = await read(`/repos/Expensify/App/actions/runs/${runID}/attempts/${positiveID(String(run.run_attempt))}/jobs?per_page=100`); + if (!isRecord(jobs) || Number(jobs.total_count) > 100) { + throw new Error('Producer jobs require complete bounded enumeration.'); + } + if (process.env.GITHUB_EVENT_NAME === 'workflow_run') { + assertProducerEventAttempt(event, run); + } + plan = await resolveProducer(await producerMetadata(runID, positiveID(String(run.run_attempt)), read), run, jobs, toolingSha, read, platform); + } + writeJSON(output, plan); + appendOutputs({ + eligible: String(plan.platforms.length > 0), + 'app-sha': plan.candidate.appSha, + 'mobile-sha': plan.candidate.mobileSha, + version: plan.candidate.version, + platforms: plan.platforms.join(','), + 'platform-matrix': JSON.stringify(plan.platforms), + }); +} + +function captureCandidate(environment: string, nativeEligible: boolean, readSource = capture): CandidateMetadata { + if (process.env.GITHUB_REPOSITORY !== 'Expensify/App' || !['staging', 'production'].includes(environment)) { + throw new Error('Invalid producer repository or environment.'); + } + const pkg: unknown = JSON.parse(readFileSync('package.json', 'utf8')); + if (!isRecord(pkg)) { + throw new Error('Missing deployment package metadata.'); + } + const mobileSha = readSource('git', ['rev-parse', 'HEAD:Mobile-Expensify']).trim(); + if (readSource('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim() !== mobileSha) { + throw new Error('Native checkout differs from the committed deployment gitlink.'); + } + const candidate = parseCandidate({appSha: readSource('git', ['rev-parse', 'HEAD']).trim(), mobileSha, version: pkg.version, environment}); + return { + schema: 1, + repository: 'Expensify/App', + runID: positiveID(requiredEnvironment('GITHUB_RUN_ID')), + runAttempt: positiveID(requiredEnvironment('GITHUB_RUN_ATTEMPT')), + nativeEligible, + candidate, + }; +} + +async function producerMetadata(runID: number, runAttempt: number, read: GitHubRead): Promise { + const response = await read(`/repos/Expensify/App/actions/runs/${runID}/artifacts?per_page=100`); + const artifact = selectProducerArtifact(response, runID, runAttempt); + const responseZip = await fetch(`https://api.github.com/repos/Expensify/App/actions/artifacts/${artifact.id}/zip`, {headers: githubHeaders()}); + if (!responseZip.ok) { + throw new Error('Producer metadata download failed.'); + } + const bytes = Buffer.from(await responseZip.arrayBuffer()); + if (bytes.length > 1024 * 1024) { + throw new Error('Producer metadata archive exceeds its size bound.'); + } + if (typeof artifact.digest === 'string' && artifact.digest !== `sha256:${createHash('sha256').update(bytes).digest('hex')}`) { + throw new Error('Producer metadata digest differs.'); + } + const directory = mkdtempSync(join(tmpdir(), 'pgo-candidate-')); + try { + const archive = join(directory, 'candidate.zip'); + writeFileSync(archive, bytes); + const entries = capture('unzip', ['-Z1', archive]).trim().split('\n'); + if (entries.length !== 1 || entries.at(0) !== 'candidate.json') { + throw new Error('Unexpected producer metadata archive contents.'); + } + return JSON.parse(capture('unzip', ['-p', archive, 'candidate.json'])) as unknown; + } finally { + rmSync(directory, {recursive: true, force: true}); + } +} + +function createGitHubReader(): GitHubRead { + return async (path) => { + if (!path.startsWith('/repos/Expensify/App/')) { + throw new Error('GitHub read escaped the approved repository.'); + } + const response = await fetch(`https://api.github.com${path}`, {headers: githubHeaders()}); + if (!response.ok) { + throw new Error(`GitHub metadata request failed (${response.status}).`); + } + return response.json(); + }; +} + +function githubHeaders(): Record { + return {Authorization: `Bearer ${requiredEnvironment('GITHUB_TOKEN')}`, Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'}; +} +function positiveID(value: string): number { + const id = Number(value); + if (!Number.isSafeInteger(id) || id <= 0) { + throw new Error('Invalid immutable run identity.'); + } + return id; +} +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing ${name}.`); + } + return value; +} +function writeJSON(path: string, value: unknown): void { + mkdirSync(dirname(resolve(path)), {recursive: true}); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} +function appendOutputs(values: Record): void { + if (!process.env.GITHUB_OUTPUT) { + return; + } + writeFileSync( + process.env.GITHUB_OUTPUT, + Object.entries(values) + .map(([key, value]) => `${key}=${value}\n`) + .join(''), + {flag: 'a'}, + ); +} + +if (import.meta.main) { + try { + await main(); + } catch { + appendOutputs({eligible: 'false', reason: 'Candidate metadata or trust validation failed.'}); + console.error('PGO candidate metadata or trust validation failed; no device evaluation is eligible.'); + process.exitCode = 1; + } +} +export {captureCandidate, createGitHubReader, main}; diff --git a/scripts/pgo/pgoEvaluation.ts b/scripts/pgo/pgoEvaluation.ts new file mode 100755 index 000000000000..f4721d68c18a --- /dev/null +++ b/scripts/pgo/pgoEvaluation.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env bun +/** Evaluation-only CLI: native execution and hosted public reporting are separate operations. */ +import {mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {basename, join} from 'node:path'; +import {parseArgs} from 'node:util'; + +import {findFiles} from '../lib/scriptUtils'; +import {evaluationReport} from './evaluationReport'; +import {parsePlan} from './evaluationSchema'; +import {createGitHubReader} from './pgoCandidate'; + +async function main(): Promise { + const {values, positionals} = parseArgs({allowPositionals: true, options: {plan: {type: 'string'}, output: {type: 'string'}, results: {type: 'string'}}}); + if (!values.plan || !values.output) { + throw new Error('Explicit plan and output paths are required.'); + } + const plan = parsePlan(JSON.parse(readFileSync(values.plan, 'utf8'))); + mkdirSync(values.output, {recursive: true}); + if (positionals.at(0) === 'execute') { + const platform = positionals.at(1); + if ((platform !== 'android' && platform !== 'ios') || !plan.platforms.includes(platform)) { + throw new Error('Platform is not eligible for this immutable plan.'); + } + const {executeEvaluation} = await import('./evaluationRun'); + await executeEvaluation(plan, platform, values.output); + return; + } + if (positionals.at(0) !== 'report' || !values.results) { + throw new Error('Expected execute or report with explicit results.'); + } + const files = findFiles(values.results, '.json').filter((path) => /^evaluation-(android|ios)\.json$/.test(basename(path))); + const report = await evaluationReport( + plan, + files.map((path) => JSON.parse(readFileSync(path, 'utf8')) as unknown), + createGitHubReader(), + ); + writeFileSync(join(values.output, 'summary.json'), `${JSON.stringify(report.summary, null, 2)}\n`); + writeFileSync(join(values.output, 'summary.md'), report.markdown); + writeFileSync(join(values.output, 'comment.json'), `${JSON.stringify(report.comment, null, 2)}\n`); +} + +if (import.meta.main) { + await main(); +} +export default main; diff --git a/scripts/pgo/proof.ts b/scripts/pgo/proof.ts new file mode 100644 index 000000000000..59678c53c269 --- /dev/null +++ b/scripts/pgo/proof.ts @@ -0,0 +1,105 @@ +// cspell:ignore profdata profraw fprofile + +import {createHash} from 'node:crypto'; +import {readFileSync, readdirSync, realpathSync, statSync} from 'node:fs'; +import {join, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; +import type {PlatformAdapter} from './shared'; + +import {capture, captureBinary, fail, rootDirectory} from '../lib/scriptUtils'; + +function toolchainIdentity(adapter: PlatformAdapter): string { + return JSON.stringify({ + profileTool: capture(adapter.llvmTool('llvm-profdata'), ['--version']).trim(), + compiler: capture(adapter.llvmTool('clang'), ['--version']).trim(), + platform: adapter.name, + buildTool: adapter.name === 'ios' ? capture('xcodebuild', ['-version']).trim() : capture('java', ['--version']).trim(), + sdk: adapter.name === 'ios' ? capture('xcrun', ['--sdk', 'iphoneos', '--show-sdk-version']).trim() : adapter.llvmTool('clang'), + androidReleasePipeline: process.env.PGO_ANDROID_RELEASE_PIPELINE ?? 'false', + androidShrink: process.env.PGO_ANDROID_SHRINK ?? 'false', + buildEnvironment: adapter.name === 'android' ? androidBuildEnvironment() : null, + }); +} + +/** Resolve ENVFILE exactly relative to the hybrid Android project's parent directory. */ +function androidBuildEnvironment(envFile = process.env.ENVFILE): {path: string; sha256: string} | null { + if (!envFile) { + return null; + } + const path = realpathSync(resolve(rootDirectory, 'Mobile-Expensify', envFile)); + return {path, sha256: createHash('sha256').update(readFileSync(path)).digest('hex')}; +} + +/** Bind tracked changes and untracked source files without changing either checkout. */ +function sourceIdentity(): string { + const hash = createHash('sha256'); + for (const directory of [rootDirectory, join(rootDirectory, 'Mobile-Expensify')]) { + hash.update(capture('git', ['rev-parse', 'HEAD'], directory)); + hash.update(captureBinary('git', ['diff', 'HEAD', '--binary'], directory)); + const paths = capture('git', ['ls-files', '--others', '--exclude-standard', '-z'], directory).split('\0').filter(Boolean).sort(); + for (const path of paths) { + hash.update(path); + hash.update('\0'); + hash.update(readFileSync(join(directory, path))); + } + } + return hash.digest('hex'); +} + +/** Reject partial batches and profiles collected from different source contents. */ +function validateJourney(input: unknown, platform: PlatformName, identity: string, toolchain: string): void { + if ( + isRecord(input) && + input.platform === platform && + input.status === 'passed' && + input.accountClass === 'heavy' && + input.requestedRuns === 3 && + input.completedRuns === 3 && + input.collectProfiles === true && + input.navigationOnly === false && + input.sourceIdentity === identity && + input.toolchain === toolchain + ) { + return; + } + fail('PGO proof requires three complete collected heavy-account journeys from the current source and toolchain.'); +} + +function profileCounts(summary: string): {functions: number; count: string} { + const functions = Number(/Total functions:\s*([0-9]+)/.exec(summary)?.[1]); + const count = /Total count:\s*([0-9]+)/.exec(summary)?.[1] ?? /Maximum function count:\s*([0-9]+)/.exec(summary)?.[1]; + if (!Number.isSafeInteger(functions) || functions <= 0 || !count || BigInt(count) <= 0n) { + fail('The LLVM profile has no executed functions.'); + } + return {functions, count}; +} + +/** Require Swift execution without borrowing counters from a neighboring Clang function. */ +function hasExecutedSwiftFunctions(report: string): boolean { + const functions = report.matchAll(/^ {2}(\S[^\n]*):\r?\n((?: {4}[^\n]*\r?\n)*)/gm); + return Array.from(functions).some((entry) => /(?:^|:)\$s/.test(entry[1] ?? '') && /(?:Function count:\s*|Block counts:\s*\[[^\]\n]*)[1-9][0-9]*/.test(entry[2] ?? '')); +} + +function fileHash(path: string, allowEmpty = false): string { + const stat = statSync(path); + if (stat.isDirectory()) { + const hash = createHash('sha256'); + for (const name of readdirSync(path).sort()) { + hash.update(name); + hash.update('\0'); + hash.update(fileHash(join(path, name), true)); + } + return hash.digest('hex'); + } + if (stat.size === 0 && !allowEmpty) { + fail(`Empty PGO evidence file: ${path}`); + } + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export {androidBuildEnvironment, fileHash, hasExecutedSwiftFunctions, isRecord, profileCounts, sourceIdentity, toolchainIdentity, validateJourney}; diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts new file mode 100644 index 000000000000..c02c63c7f802 --- /dev/null +++ b/scripts/pgo/shared.ts @@ -0,0 +1,53 @@ +// cspell:ignore cooldown Cooldown + +/** Shared PGO workflow types and constants. */ + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {fail, sleep} from '../lib/scriptUtils'; + +const STARTUP_SPAN_NAME = 'ManualAppStartup'; +const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; +const HEAVY_JOURNEY_SPAN_NAMES = ['/r/*'] as const; +const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES].join(',')}`; + +type BuildKind = 'release' | 'instrumented' | 'optimized'; +type BenchmarkKind = Extract; +type PgoMode = 'off' | 'generate' | 'use'; +type BuildArtifactPaths = Record; + +type PlatformAdapter = { + readonly name: PlatformName; + readonly profileDirectory: string; + readonly rawProfileDirectory: string; + readonly mergedProfilePath: string; + readonly benchmarkDirectory: string; + readonly benchmarkPaths: Record; + readonly artifactPaths: BuildArtifactPaths; + readonly profileFormat?: string; + appID: () => string; + build: (kind: BuildKind, profilePath?: string) => void; + install: (kind: BuildKind) => void; + verifyInstrumentation: () => void; + clearDeviceProfiles: () => Promise; + dumpProfiles: () => Promise; + pullProfiles: () => void; + llvmTool: (name: string) => string; +}; + +function benchmarkCooldownSeconds(): number { + const seconds = Number(process.env.PGO_BENCHMARK_COOLDOWN_SECONDS ?? 0); + if (!Number.isSafeInteger(seconds) || seconds < 0 || seconds > 60) { + fail('Benchmark cooldown seconds must be an integer between 0 and 60.'); + } + return seconds; +} + +async function cooldownBeforeBenchmark(): Promise { + const seconds = benchmarkCooldownSeconds(); + console.log(`Waiting ${seconds}s before the discarded warm-up. This delay does not verify thermal state.`); + await sleep(seconds * 1000); +} + +export {benchmarkCooldownSeconds, cooldownBeforeBenchmark, BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; +export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/src/PgoVerificationApp.tsx b/src/PgoVerificationApp.tsx new file mode 100644 index 000000000000..be6b2c891e76 --- /dev/null +++ b/src/PgoVerificationApp.tsx @@ -0,0 +1,78 @@ +import type {TurboModule} from 'react-native'; + +// This entry is selected only by the emulator verification build, never by index.js. +import React, {useEffect, useRef, useState} from 'react'; +// eslint-disable-next-line no-restricted-imports -- This isolated CI entry runs without application providers. +import {AppRegistry, Platform, ScrollView, StyleSheet, Text, TurboModuleRegistry, View} from 'react-native'; +import RNFS from 'react-native-fs'; + +import {verificationRows, VERIFICATION_MARKER, VERIFICATION_ROW_COUNT, VERIFICATION_SCREENS} from '../scripts/pgo/emulatorFixture'; + +type BootSplashModule = TurboModule & {hide: () => Promise}; +const BootSplash = TurboModuleRegistry.getEnforcing('BootSplash'); +const startedAt = performance.now(); + +const styles = StyleSheet.create({ + container: {flex: 1, padding: 24, paddingTop: 64, backgroundColor: '#ffffff'}, + heading: {marginBottom: 12}, + row: {paddingVertical: 12, flexDirection: 'row', justifyContent: 'space-between'}, +}); + +/** Exercise Hermes, native layout and file I/O using public synthetic data. */ +function VerificationApp() { + const [screen, setScreen] = useState(0); + const scheduledScreen = useRef(-1); + const rows = verificationRows(screen); + + useEffect(() => { + BootSplash.hide(); + }, []); + + const advance = () => { + if (screen <= scheduledScreen.current) { + return; + } + scheduledScreen.current = screen; + if (screen < VERIFICATION_SCREENS.length - 1) { + setTimeout(() => setScreen((previous) => previous + 1), 500); + return; + } + const directory = Platform.OS === 'android' ? RNFS.ExternalCachesDirectoryPath : RNFS.CachesDirectoryPath; + const temporaryPath = `${directory}/${VERIFICATION_MARKER}.tmp`; + RNFS.writeFile( + temporaryPath, + JSON.stringify({ + schema: 1, + status: 'passed', + workload: 'synthetic', + screens: VERIFICATION_SCREENS.length, + rows: VERIFICATION_ROW_COUNT, + durationMs: performance.now() - startedAt, + }), + 'utf8', + ).then(() => RNFS.moveFile(temporaryPath, `${directory}/${VERIFICATION_MARKER}`)); + }; + + return ( + + PGO CI verification + {VERIFICATION_SCREENS.at(screen)} + + {rows.map((row) => ( + + {row.title} + {row.amount.toFixed(2)} + + ))} + + + ); +} + +AppRegistry.registerComponent('NewExpensify', () => VerificationApp); diff --git a/tests/tooling/benchmarkAppStartup.test.ts b/tests/tooling/benchmarkAppStartup.test.ts index 0385db56a12a..4c213eb7ef12 100644 --- a/tests/tooling/benchmarkAppStartup.test.ts +++ b/tests/tooling/benchmarkAppStartup.test.ts @@ -1,4 +1,4 @@ -// cspell:ignore appex +// cspell:ignore appex devicectl import {describe, expect, it, jest} from 'bun:test'; @@ -7,6 +7,7 @@ import { assertAndroidAppInstalled, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseIOSInstalledAppURL, @@ -78,6 +79,35 @@ describe('benchmarkAppStartup', () => { expect(processIdentifier).toBe(456); }); + it('lists iOS processes without a version-specific CoreDevice filter', () => { + // Given + const deviceIdentifier = '00008120-EXAMPLE'; + const outputPath = '/tmp/processes.json'; + + // When + const args = iOSProcessListArguments(deviceIdentifier, outputPath); + + // Then + expect(args).toEqual(['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']); + expect(args).not.toContain('--filter'); + }); + + it('ignores an extension process when the main iOS app is not running', () => { + // Given + const appURL = 'file:///private/var/containers/Bundle/Application/EXAMPLE/Expensify.app/'; + const response = { + result: { + runningProcesses: [{executable: `${appURL}PlugIns/LiveActivityExtension.appex/LiveActivityExtension`, processIdentifier: 20702}], + }, + }; + + // When + const processIdentifier = parseIOSRunningAppProcessIdentifier(response, appURL); + + // Then + expect(processIdentifier).toBeUndefined(); + }); + it('rejects malformed CoreDevice installed-app responses before using them', () => { expect(() => parseIOSInstalledAppsResponse({result: {apps: [{bundleIdentifier: 'com.example.app'}]}})).toThrow('unexpected installed-app response'); }); diff --git a/tests/tooling/pgo.test.ts b/tests/tooling/pgo.test.ts new file mode 100644 index 000000000000..ac3ff4ba66e0 --- /dev/null +++ b/tests/tooling/pgo.test.ts @@ -0,0 +1,41 @@ +import {describe, expect, it} from 'bun:test'; +// cspell:ignore chrispader + +import createAndroidPgoAdapter, {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; +import percentageImprovement from '@scripts/pgo/pgo'; + +describe('PGO tooling', () => { + it('exports the Android adapter as the default export', () => { + expect(createAndroidPgoAdapter).toBeFunction(); + }); + + it('reads the bootstrapped release application ID', () => { + // Given + const buildGradle = ` + android { + defaultConfig { + applicationId "com.chrispader.expensify.pgo" + } + } + `; + + // When / Then + expect(androidApplicationID(buildGradle)).toBe('com.chrispader.expensify.pgo'); + }); + + it('uses an explicit application ID instead of the Gradle value', () => { + // Given / When / Then + expect(androidApplicationID('invalid build file', 'com.example.expensify.pgo')).toBe('com.example.expensify.pgo'); + }); + + it('targets the bootstrapped package with the original Java receiver namespace', () => { + // Given / When / Then + expect(androidProfileReceiverComponent('com.chrispader.expensify.pgo')).toBe('com.chrispader.expensify.pgo/org.me.mobiexpensifyg.PgoProfileReceiver'); + expect(PROFILE_BROADCAST_ACTION).toBe('com.expensify.chat.action.WRITE_PGO_PROFILES'); + }); + + it('reports positive values when the optimized build is faster', () => { + // Given / When / Then + expect(percentageImprovement(500, 400)).toBe(20); + }); +}); diff --git a/tests/tooling/pgoAndroid.test.ts b/tests/tooling/pgoAndroid.test.ts new file mode 100644 index 000000000000..ab82b96c3f1b --- /dev/null +++ b/tests/tooling/pgoAndroid.test.ts @@ -0,0 +1,130 @@ +// cspell:ignore profdata + +import {afterAll, beforeEach, describe, expect, it, mock} from 'bun:test'; + +import * as scriptUtils from '@scripts/lib/scriptUtils'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; +import process from 'node:process'; + +let profileLogs = ''; +// Reproduce the CI checkout, which does not initialize the native submodule. +const rootDirectory = mkdtempSync(join(tmpdir(), 'pgo-android-test-')); +const runCommand = mock(() => undefined); +const captureCommand = mock(() => profileLogs); +// Stub the device boundary while keeping the adapter's broadcast, polling and validation real. +mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, rootDirectory, run: runCommand, capture: captureCommand})); +const {default: createAndroidPgoAdapter, androidLlvmToolPath} = await import('@scripts/pgo/android'); + +afterAll(() => rmSync(rootDirectory, {recursive: true, force: true})); + +describe('PGO Android profile dumps', () => { + beforeEach(() => { + profileLogs = ''; + runCommand.mockClear(); + captureCommand.mockClear(); + }); + + it('accepts a dump only when all four native libraries wrote profiles', async () => { + // Given a complete writer response from the installed instrumented APK. + profileLogs = 'I/PgoProfileReceiver: Wrote 4 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then profile collection can continue. + await expect(dump).resolves.toBeUndefined(); + expect(runCommand.mock.calls).toEqual([ + ['adb', ['logcat', '-c']], + ['adb', ['shell', 'am', 'broadcast', '-a', 'com.expensify.chat.action.WRITE_PGO_PROFILES', '-n', 'com.example.pgo/org.me.mobiexpensifyg.PgoProfileReceiver']], + ]); + }); + + it.each([1, 2, 3, 5])('rejects a dump reporting %i profiles', async (count) => { + // Given a partial or unexpected writer response despite successful broadcast delivery. + profileLogs = `I/PgoProfileReceiver: Wrote ${count} LLVM PGO profile(s).`; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then training cannot silently accept an incomplete set of native counters. + await expect(dump).rejects.toThrow(`wrote ${count} profiles, but expected 4`); + }); + + it('keeps the diagnostic for a dump with no instrumented libraries', async () => { + // Given a receiver that could not find any profile writers. + profileLogs = 'I/PgoProfileReceiver: Wrote 0 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it explains that the instrumented APK must be rebuilt. + await expect(dump).rejects.toThrow('found no instrumented native libraries'); + }); + + it('rejects a non-instrumented APK', async () => { + // Given a receiver explicitly rejecting profiling in a normal release build. + profileLogs = 'Ignoring PGO profile request in a non-instrumented build.'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it directs the caller to install the instrumented build. + await expect(dump).rejects.toThrow('installed APK is not instrumented'); + }); +}); + +describe('PGO Android LLVM tools', () => { + it.each([ + ['linux', '/home/test/Android/Sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'], + ['darwin', '/home/test/Library/Android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-profdata'], + ] satisfies Array<[NodeJS.Platform, string]>)('uses the default SDK installation on %s', (hostPlatform, expected) => { + // Given an unconfigured host with its normal Android Studio SDK location. + // When LLVM tools are resolved without explicit environment settings. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform, homeDirectory: '/home/test', environment: {}}); + // Then the selected path uses that host's SDK directory and toolchain. + expect(path).toBe(expected); + }); + + it.each(['ANDROID_HOME', 'ANDROID_SDK_ROOT'])('honors %s on Linux', (variable) => { + // Given a Linux runner with its SDK installed outside the default home directory. + // When the SDK environment setting is used to resolve LLVM. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/android/sdk'}}); + // Then the pinned NDK is resolved within the configured SDK. + expect(path).toBe('/opt/android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it.each(['ANDROID_NDK_HOME', 'ANDROID_NDK_ROOT'])('prefers %s over SDK settings', (variable) => { + // Given an explicit NDK override and a different SDK location. + // When the tool path is resolved. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/custom-ndk', ANDROID_HOME: '/opt/android/sdk'}}); + // Then an explicitly selected build toolchain wins over the pinned SDK fallback. + expect(path).toBe('/opt/custom-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('prefers ANDROID_HOME over the deprecated SDK root setting and ignores empty NDK overrides', () => { + // Given both SDK settings and empty NDK settings. + const environment = {ANDROID_HOME: '/opt/current-sdk', ANDROID_SDK_ROOT: '/opt/old-sdk', ANDROID_NDK_HOME: '', ANDROID_NDK_ROOT: ''}; + // When LLVM tools are resolved on Linux. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment}); + // Then empty settings do not mask the SDK selected by ANDROID_HOME. + expect(path).toBe('/opt/current-sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('uses the resolved tool path in the adapter', () => { + // Given an explicit temporary NDK with a real tool file. + const directory = mkdtempSync(join(tmpdir(), 'pgo-ndk-test-')); + const previousNdk = process.env.ANDROID_NDK_HOME; + process.env.ANDROID_NDK_HOME = directory; + try { + const expected = androidLlvmToolPath('llvm-profdata'); + mkdirSync(dirname(expected), {recursive: true}); + writeFileSync(expected, ''); + // When the adapter checks tool availability. + const path = createAndroidPgoAdapter().llvmTool('llvm-profdata'); + // Then it returns the existing file in the selected NDK. + expect(path).toBe(expected); + } finally { + if (previousNdk === undefined) { + delete process.env.ANDROID_NDK_HOME; + } else { + process.env.ANDROID_NDK_HOME = previousNdk; + } + rmSync(directory, {recursive: true, force: true}); + } + }); +}); diff --git a/tests/tooling/pgoAndroidRelease.test.ts b/tests/tooling/pgoAndroidRelease.test.ts new file mode 100644 index 000000000000..3be6ef3a94d5 --- /dev/null +++ b/tests/tooling/pgoAndroidRelease.test.ts @@ -0,0 +1,78 @@ +// cspell:ignore Ppgo profdata + +import {describe, expect, it} from 'bun:test'; + +import {androidReleasePipeline, androidRockBuildArguments} from '@scripts/pgo/android'; + +import {parseArgs} from '@rock-js/tools'; + +describe('PGO Android release processing', () => { + it('builds locally through Rock with the exact optimized profile and FullStory exclusions', () => { + // Given a shrinking build and an immutable profile in a directory containing spaces. + const pipeline = androidReleasePipeline(undefined, 'true'); + const parameters = ['-PpgoMode=use', '-PpgoProfile=/tmp/profile batch/journey.profdata', ...pipeline.arguments]; + // When the adapter forwards this build to Rock. + const args = androidRockBuildArguments(pipeline.task, parameters); + const forwarded = parseArgs(args.at(-1) ?? ''); + // Then Rock cannot reuse a cached build or lose profile and upload-prevention arguments, and Gradle bundles OldDot's shared JS first because packaging alone skips it. + expect(args.slice(0, 7)).toEqual(['build:android', '--local', '--verbose', '--variant', 'release', '--tasks', 'buildGrunt,packageRelease']); + expect(forwarded).toEqual(parameters); + expect(forwarded).toContain('-PpgoSkipFullStory=true'); + expect(forwarded).toContain('fullstoryRelease'); + }); + it('keeps the bootstrap build separate from FullStory output', () => { + // Given no explicit release pipeline request. + // When the build selects its task and artifact. + const options = androidReleasePipeline(); + // Then bootstrap processing does not masquerade as the final release pipeline, and a clean build skips the IDE listing redirect that needs FullStory's output. + expect(options).toEqual({ + enabled: false, + shrinking: false, + artifactDirectory: 'apk', + task: ':packageRelease', + apkTask: 'packageRelease', + arguments: ['-x', 'fullstoryRelease', '-x', 'createReleaseApkListingFileRedirect'], + }); + }); + + it('selects the processed APK without overriding local credentials', () => { + // Given an explicit request to exercise shrinking and FullStory. + // When the build selects its task and artifact. + const options = androidReleasePipeline('true'); + // Then it targets the final processed APK and supplies no signing or identity overrides. + expect(options.task).toBe(':fullstoryRelease'); + expect(options.apkTask).toBe('fullstoryRelease'); + expect(options.arguments.at(0)).toBe('--init-script'); + expect(options.arguments.join(' ')).not.toMatch(/signing|password|applicationId/i); + }); + + it('rejects ambiguous opt-in settings', () => { + // Given a mistyped environment setting. + // When the build decides whether release processing is required. + const select = () => androidReleasePipeline('yes'); + // Then it fails instead of silently substituting the bootstrap build. + expect(select).toThrow('must be true or false'); + }); + + it('shrinks locally while excluding external FullStory processing', () => { + // Given a request for release shrinking without asset uploads. + // When the build selects its tasks and output directory. + const options = androidReleasePipeline(undefined, 'true'); + // Then the APK remains distinct, FullStory is both excluded and disabled, and the IDE listing redirect that needs its output is skipped. + expect(options.shrinking).toBe(true); + expect(options.enabled).toBe(false); + expect(options.artifactDirectory).toBe('apk-shrunk'); + expect(options.task).toBe(':packageRelease'); + expect(options.arguments).toContain('-PpgoSkipFullStory=true'); + expect(options.arguments).toContain('-x'); + expect(options.arguments).toContain('createReleaseApkListingFileRedirect'); + }); + + it('rejects simultaneous shrinking and upload pipeline requests', () => { + // Given conflicting local-only and external processing choices. + // When the build selects a pipeline. + const select = () => androidReleasePipeline('true', 'true'); + // Then it cannot silently choose the upload-capable task. + expect(select).toThrow('not both'); + }); +}); diff --git a/tests/tooling/pgoCandidate.test.ts b/tests/tooling/pgoCandidate.test.ts new file mode 100644 index 000000000000..eab28e50462b --- /dev/null +++ b/tests/tooling/pgoCandidate.test.ts @@ -0,0 +1,169 @@ +// cspell:ignore chrispader +/* eslint-disable @typescript-eslint/naming-convention -- Fixtures reproduce the GitHub API snake_case schema. */ +import {describe, expect, it} from 'bun:test'; + +import {execFileSync} from 'node:child_process'; + +import type {GitHubRead} from '../../scripts/pgo/candidateResolve'; + +import {assertProducerEventAttempt, checklistMatches, matchingChecklist, resolvePreview, resolveProducer, selectProducerArtifact} from '../../scripts/pgo/candidateResolve'; +import {captureCandidate} from '../../scripts/pgo/pgoCandidate'; + +const appSha = 'a'.repeat(40); +const mobileSha = 'b'.repeat(40); +const toolingSha = 'c'.repeat(40); +const candidate = {appSha, mobileSha, version: '9.4.94-0', environment: 'staging'}; +const metadata = {schema: 1, repository: 'Expensify/App', runID: 10, runAttempt: 2, nativeEligible: true, candidate}; +const run = { + id: 10, + run_attempt: 2, + repository: {full_name: 'Expensify/App'}, + name: 'Deploy code to staging or production', + path: '.github/workflows/deploy.yml', + status: 'completed', + conclusion: 'failure', + event: 'workflow_dispatch', + head_branch: 'staging', + head_sha: toolingSha, +}; +const jobs = { + jobs: [ + {name: 'Build Android HybridApp / Build Android app', conclusion: 'success'}, + {name: 'Build iOS HybridApp / Build iOS HybridApp', conclusion: 'failure'}, + ], +}; +const read: GitHubRead = async (path) => { + if (path.includes('/git/trees/')) { + return {tree: [{path: 'Mobile-Expensify', mode: '160000', sha: mobileSha}]}; + } + if (path.includes('/contents/package.json')) { + return {content: Buffer.from(JSON.stringify({version: candidate.version})).toString('base64')}; + } + if (path.includes('/compare/')) { + return {status: 'ahead'}; + } + if (path.includes('/pulls/')) { + return {head: {sha: appSha}, state: 'open'}; + } + if (path.includes('/issues?')) { + return [{number: 42, body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]}]; + } + throw new Error('Unexpected metadata request.'); +}; + +function preview(repo = 'Expensify/App', sha = appSha) { + return {number: 3, pull_request: {head: {sha, ref: '@chrispader/perf/pgo-ci-verification', repo: {full_name: repo}}, base: {repo: {full_name: 'Expensify/App'}}}}; +} + +describe('PGO immutable candidate resolution', () => { + it('uses actual producer metadata rather than dispatch head and preserves independent platform success', async () => { + // Given a completed deployment with web/iOS failure and a different dispatch head. + // When the exact producer metadata and nested native jobs are resolved. + const plan = await resolveProducer(metadata, run, jobs, toolingSha, read); + // Then the successful Android build alone is eligible at the actual immutable source. + expect(plan.platforms).toEqual(['android']); + expect(plan.candidate.appSha).toBe(appSha); + expect(plan.candidate.appSha).not.toBe(run.head_sha); + expect(plan.checklistIssueNumber).toBe(42); + }); + + it('rejects missing metadata, retry drift, wrong environment and candidate gitlink drift', async () => { + // Given metadata tied to one repository, run attempt and native checkout. + const changedRead: GitHubRead = async (path) => (path.includes('/git/trees/') ? {tree: [{path: 'Mobile-Expensify', mode: '160000', sha: 'd'.repeat(40)}]} : read(path)); + // When a required identity differs or metadata is absent. + // Then no credential-device plan may be created. + await expect(resolveProducer(undefined, run, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer({...metadata, runAttempt: 1}, run, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer(metadata, {...run, head_branch: 'feature'}, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer(metadata, run, jobs, toolingSha, changedRead)).rejects.toThrow(); + await expect(resolveProducer({...metadata, runID: '10'}, run, jobs, toolingSha, read)).rejects.toThrow(); + await expect(resolveProducer({...metadata, repository: 'fork/App'}, run, jobs, toolingSha, read)).rejects.toThrow(); + }); + + it('does not confuse unrelated successful jobs or non-native production promotions with eligible builds', async () => { + // Given a spoofed job suffix and a promotion that reuses prior native binaries. + // When platform eligibility is resolved. + const unrelated = await resolveProducer(metadata, run, {jobs: [{name: 'Unrelated / Build Android app', conclusion: 'success'}]}, toolingSha, read); + const reused = await resolveProducer({...metadata, nativeEligible: false}, run, jobs, toolingSha, read); + // Then neither can initiate evaluation on credential devices. + expect(unrelated.platforms).toEqual([]); + expect(reused.platforms).toEqual([]); + }); + + it('requires an open exact same-repository preview head and rejects forks or stale heads', async () => { + // Given the one approved preview branch in Expensify/App. + // When a fork, stale revision or closed pull request is supplied. + const plan = await resolvePreview(preview(), toolingSha, read); + // Then only the current trusted head can evaluate; normal forks receive no device authorization. + expect(plan.platforms).toEqual(['android', 'ios']); + await expect(resolvePreview(preview('fork/App'), toolingSha, read)).rejects.toThrow(); + const wrongBranch = preview(); + wrongBranch.pull_request.head.ref = 'other-preview'; + await expect(resolvePreview(wrongBranch, toolingSha, read)).rejects.toThrow(); + await expect(resolvePreview(preview('Expensify/App', toolingSha), toolingSha, read)).rejects.toThrow(); + await expect(resolvePreview(preview(), toolingSha, async (path) => (path.includes('/pulls/') ? {head: {sha: appSha}, state: 'closed'} : read(path)))).rejects.toThrow(); + }); + + it('matches historical release headers exactly instead of a latest issue or version substring', () => { + // Given one labelled historical checklist and unrelated version text. + const issue = {number: 42, body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]}; + // When the immutable candidate version is compared. + // Then only its exact labelled release header qualifies. + expect(checklistMatches(issue, '9.4.94-0')).toBe(true); + expect(checklistMatches(issue, '9.4.94')).toBe(false); + expect(checklistMatches({...issue, body: 'A link mentions 9.4.94-0'}, '9.4.94-0')).toBe(false); + expect(checklistMatches({...issue, labels: []}, '9.4.94-0')).toBe(false); + }); + + it('paginates historical checklists and rejects ambiguous exact versions', async () => { + // Given more than one page of historical deployment checklists. + const otherIssues = Array.from({length: 100}, () => ({body: '**Release Version:** `1.0.0-staging`', labels: [{name: 'StagingDeployCash'}], number: 1})); + const historical = {number: 42, body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]}; + // When the matching older version is on the second page. + const found = await matchingChecklist('9.4.94-0', async (path) => (path.endsWith('page=1') ? otherIssues : [historical])); + // Then the exact historical issue is used, and duplicate matches fail closed. + expect(found).toBe(42); + await expect(matchingChecklist('9.4.94-0', async () => [historical, {...historical, number: 43}])).rejects.toThrow(); + }); + + it('rejects a queued producer event superseded by a newer attempt', () => { + // Given a workflow event emitted before its producer was rerun. + const oldEvent = {workflow_run: {id: 10, run_attempt: 1}}; + // When current metadata refers to completed attempt two. + // Then the queued event cannot silently evaluate the newer producer. + expect(() => assertProducerEventAttempt(oldEvent, run)).toThrow(); + expect(() => assertProducerEventAttempt({workflow_run: {id: 10, run_attempt: 2}}, run)).not.toThrow(); + }); + + it('selects only metadata from the current immutable producer attempt', () => { + // Given retained artifacts from two attempts of the same workflow run. + const old = {id: 1, name: 'pgo-native-candidate-1', expired: false, workflow_run: {id: 10}}; + const current = {id: 2, name: 'pgo-native-candidate-2', expired: false, workflow_run: {id: 10}}; + // When manual evaluation resolves the latest completed attempt. + const selected = selectProducerArtifact({total_count: 2, artifacts: [old, current]}, 10, 2); + // Then stale or cross-run metadata cannot substitute for its exact artifact. + expect(selected.id).toBe(2); + expect(() => selectProducerArtifact({total_count: 1, artifacts: [old]}, 10, 2)).toThrow(); + expect(() => selectProducerArtifact({total_count: 1, artifacts: [{...current, workflow_run: {id: 11}}]}, 10, 2)).toThrow(); + }); + + it('captures the actual local deployment HEAD and gitlink even when the workflow context SHA differs', () => { + // Given a checked-out deployment and an unrelated workflow-context revision. + const previous = {...process.env}; + Object.assign(process.env, {GITHUB_REPOSITORY: 'Expensify/App', GITHUB_RUN_ID: '10', GITHUB_RUN_ATTEMPT: '2', GITHUB_SHA: appSha}); + try { + // When the producer captures its actual repository state. + const actualHead = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim(); + const actualLink = execFileSync('git', ['rev-parse', 'HEAD:Mobile-Expensify'], {encoding: 'utf8'}).trim(); + const readSource = (_command: string, args: string[]) => (args.includes('-C') || args.includes('HEAD:Mobile-Expensify') ? actualLink : actualHead); + const captured = captureCandidate('staging', true, readSource); + expect(() => captureCandidate('staging', true, (_command, args) => (args.includes('-C') ? appSha : readSource(_command, args)))).toThrow(); + // Then HEAD and the committed native gitlink are bound independently of GITHUB_SHA. + expect(captured.candidate.appSha).toBe(execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8'}).trim()); + expect(captured.candidate.mobileSha).toBe(execFileSync('git', ['rev-parse', 'HEAD:Mobile-Expensify'], {encoding: 'utf8'}).trim()); + expect(captured.runAttempt).toBe(2); + } finally { + process.env = previous; + } + }); +}); diff --git a/tests/tooling/pgoEmulator.test.ts b/tests/tooling/pgoEmulator.test.ts new file mode 100644 index 000000000000..e40dad5b64c5 --- /dev/null +++ b/tests/tooling/pgoEmulator.test.ts @@ -0,0 +1,93 @@ +// cspell:ignore esac simctl profdata +import {afterAll, describe, expect, it} from 'bun:test'; + +import createAndroidPgoAdapter from '@scripts/pgo/android'; +import {parseVerificationMarker, verificationRows, VERIFICATION_ROW_COUNT, VERIFICATION_SCREENS} from '@scripts/pgo/emulatorFixture'; +import {selectIOSVerificationRuntime} from '@scripts/pgo/iosSimulator'; + +import {spawnSync} from 'node:child_process'; +import {mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; + +const directory = mkdtempSync(join(tmpdir(), 'pgo-emulator-tests-')); +afterAll(() => rmSync(directory, {recursive: true, force: true})); + +describe('PGO emulator verification', () => { + it('accepts only a completed synthetic workload with finite timing', () => { + // Given the synthetic workload's explicit completion marker. + const marker = {schema: 1, status: 'passed', workload: 'synthetic', screens: VERIFICATION_SCREENS.length, rows: VERIFICATION_ROW_COUNT, durationMs: 1200}; + // When complete, partial and malformed results arrive from the app. + expect(parseVerificationMarker(marker).durationMs).toBe(1200); + // Then an incomplete render, an unapproved workload or invalid timing fails verification. + for (const change of [{screens: 1}, {rows: 0}, {workload: 'live'}, {durationMs: Number.NaN}, {durationMs: Number.POSITIVE_INFINITY}, {durationMs: 0}]) { + expect(() => parseVerificationMarker({...marker, ...change})).toThrow('did not complete'); + } + expect(new Set(verificationRows(1).map((row) => row.id)).size).toBe(VERIFICATION_ROW_COUNT); + }); + + it('selects an installed iOS runtime without choosing an unavailable or non-iOS runtime', () => { + // Given runtime versions that require numeric ordering and an unavailable newer runtime. + const runtimes = [ + {identifier: 'com.apple.CoreSimulator.SimRuntime.iOS-26-9', version: '26.9', isAvailable: true}, + {identifier: 'com.apple.CoreSimulator.SimRuntime.iOS-26-10', version: '26.10', isAvailable: true}, + {identifier: 'com.apple.CoreSimulator.SimRuntime.iOS-27', version: '27', isAvailable: false}, + {identifier: 'com.apple.CoreSimulator.SimRuntime.watchOS-27', version: '27', isAvailable: true}, + ]; + // When a disposable simulator's runtime is selected. + const selected = selectIOSVerificationRuntime({runtimes}); + // Then the latest installed iOS runtime is selected and missing runtimes fail clearly. + expect(selected).toBe('com.apple.CoreSimulator.SimRuntime.iOS-26-10'); + expect(() => selectIOSVerificationRuntime({runtimes: []})).toThrow('no available iOS'); + }); + + it('isolates emulator profiles from physical Android profiles', () => { + // Given the hosted emulator architecture instead of the physical-device default. + const previous = process.env.PGO_ANDROID_ARCH; + try { + process.env.PGO_ANDROID_ARCH = 'x86_64'; + // When its platform adapter chooses profile paths. + const adapter = createAndroidPgoAdapter('com.example.pgo'); + // Then it cannot overwrite or consume the physical ARM64 profile by default. + expect(adapter.mergedProfilePath).toEndWith('/android/x86_64/newdot.profdata'); + process.env.PGO_ANDROID_ARCH = '../arm64-v8a'; + expect(() => createAndroidPgoAdapter()).toThrow('must be arm64-v8a or x86_64'); + } finally { + if (previous === undefined) { + delete process.env.PGO_ANDROID_ARCH; + } else { + process.env.PGO_ANDROID_ARCH = previous; + } + } + }); + + it.each([0, 1])('publishes App only after the pinned native push succeeds with status %i', (nativeStatus) => { + // Given a fake Git transport that records pushes without contacting GitHub. + const log = join(directory, `copy-${nativeStatus}.log`); + writeFileSync( + join(directory, 'git'), + `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$PGO_COPY_LOG" +case "$*" in + *'ls-tree HEAD Mobile-Expensify'*) echo '160000 commit bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Mobile-Expensify';; + *'rev-parse FETCH_HEAD'*) echo 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb';; + *'rev-parse HEAD'*) echo 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';; + *'push git@github.com:Expensify/Mobile-Expensify.git'*) exit "$PGO_COPY_NATIVE_STATUS";; +esac +`, + {mode: 0o755}, + ); + // When the engineer runs the actual copy helper. + const result = spawnSync('bash', [resolve('scripts/pgo/copy-ci-preview.sh')], { + encoding: 'utf8', + env: {...process.env, PATH: `${directory}:${process.env.PATH}`, PGO_COPY_LOG: log, PGO_COPY_NATIVE_STATUS: String(nativeStatus)}, + }); + const commands = readFileSync(log, 'utf8'); + // Then a failed native upload never starts an App workflow against a missing gitlink. + expect(result.status).toBe(nativeStatus); + expect(commands.includes('push git@github.com:Expensify/App.git')).toBe(nativeStatus === 0); + if (nativeStatus === 0) { + expect(commands.indexOf('push git@github.com:Expensify/Mobile-Expensify.git')).toBeLessThan(commands.indexOf('push git@github.com:Expensify/App.git')); + } + }); +}); diff --git a/tests/tooling/pgoEvaluationReport.test.ts b/tests/tooling/pgoEvaluationReport.test.ts new file mode 100644 index 000000000000..951ea217faeb --- /dev/null +++ b/tests/tooling/pgoEvaluationReport.test.ts @@ -0,0 +1,173 @@ +// cspell:ignore cooldown +import {describe, expect, it} from 'bun:test'; + +import type {EvaluationCohort, EvaluationPlan, EvaluationResult} from '../../scripts/pgo/evaluationSchema'; + +import {distribution, evaluationReport, summarizePlatform, summarizeFrames} from '../../scripts/pgo/evaluationReport'; +import {candidateKey, parsePlan} from '../../scripts/pgo/evaluationSchema'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from '../../scripts/pgo/shared'; + +const hash = 'a'.repeat(64); +const plan: EvaluationPlan = { + schema: 1, + repository: 'Expensify/App', + toolingSha: 'c'.repeat(40), + candidate: {appSha: 'a'.repeat(40), mobileSha: 'b'.repeat(40), version: '9.4.94-0', environment: 'staging'}, + producerRunID: 10, + producerRunAttempt: 2, + platforms: ['android'], + checklistIssueNumber: 42, +}; +function result(): EvaluationResult { + const definitions = [ + ['startup', ['ManualAppStartup']], + ['journey', [...JOURNEY_SPAN_NAMES]], + ['heavy-journey', [...HEAVY_JOURNEY_SPAN_NAMES]], + ] as const; + const cohorts: EvaluationCohort[] = definitions.flatMap(([scenario, spans]) => + (['release-first', 'optimized-first'] as const).map((order) => ({ + scenario, + order, + spans: [...spans], + runs: 20, + warmUpRunsPerBinary: 1, + cooldownSecondsPerBinary: 5, + releaseCsvSha256: hash, + optimizedCsvSha256: hash, + release: Array.from({length: 20}, (_, index) => spans.map((span) => ({run: index + 1, span, durationMs: 100}))).flat(), + optimized: Array.from({length: 20}, (_, index) => spans.map((span) => ({run: index + 1, span, durationMs: 90}))).flat(), + })), + ); + return { + schema: 1, + platform: 'android', + candidate: plan.candidate, + toolingSha: plan.toolingSha, + status: 'passed', + artifactSha256: {release: hash, instrumented: hash, optimized: hash}, + profileSha256: hash, + sourceIdentity: hash, + toolchain: {compilerVersion: 'Clang 17'}, + compilerConsumption: 'unverified', + cohorts, + }; +} + +describe('PGO public evaluation reports', () => { + it('retains all observations and computes distributions and positive-faster effects without selection', async () => { + // Given complete opposite-order cohorts for every workload. + const input = result(); + // When the report computes numeric summaries. + const report = await evaluationReport(plan, [input], async () => ({body: '**Release Version:** `9.4.94-0-staging`', labels: [{name: 'StagingDeployCash'}]})); + const summary = report.summary; + // Then all samples remain and the mean effect is ten percent without choosing a deployment artifact. + expect(summary.status).toBe('complete'); + expect(summary.platforms.at(0)?.cohorts?.at(0)?.statistics.at(0)?.meanEffectPercent).toBe(10); + expect(report.markdown).toContain('canonical baseline artifacts and deployment selection are unchanged'); + expect(report.comment.targetIssueNumber).toBe(42); + expect(report.comment.body).toStartWith("Chris's AI"); + }); + + it('rejects incomplete, extra, reordered and cross-candidate measurements rather than salvaging subsets', () => { + // Given a complete result bound to the candidate. + const input = result(); + const cohort = input.cohorts.at(0); + if (!cohort) { + throw new Error('Missing synthetic cohort.'); + } + // When any mandatory measurement or identity is changed. + // Then the entire claimed complete platform evidence is rejected. + expect(() => summarizePlatform(plan, 'android', {...input, candidate: {...plan.candidate, appSha: 'd'.repeat(40)}})).toThrow(); + expect(() => summarizePlatform(plan, 'android', {...input, cohorts: input.cohorts.slice(1)})).toThrow(); + for (const release of [cohort.release.slice(1), [...cohort.release, {...cohort.release.at(0)}], [...cohort.release].reverse()]) { + expect(() => summarizePlatform(plan, 'android', {...input, cohorts: [{...cohort, release}, ...input.cohorts.slice(1)]})).toThrow(); + } + expect(() => summarizePlatform(plan, 'android', {...input, cohorts: [{...cohort, warmUpRunsPerBinary: 0}, ...input.cohorts.slice(1)]})).toThrow(); + }); + + it('whitelists public numeric provenance and excludes private fields and raw errors', async () => { + // Given private extra fields that must never enter uploaded or commented report data. + const input = {...result(), accountEmail: 'private@example.com', fixture: '/private/fixture.json', rawError: 'private error', deviceID: 'secret-device'}; + // When explicit output fields are selected and an unavailable platform is represented. + const report = await evaluationReport(plan, [input], async () => ({body: 'wrong version', labels: [{name: 'StagingDeployCash'}]})); + const serialized = JSON.stringify(report); + // Then secrets are omitted and an obsolete checklist cannot receive a report for another version. + expect(serialized).not.toContain('private@example.com'); + expect(serialized).not.toContain('secret-device'); + expect(serialized).not.toContain('/private/fixture.json'); + expect(report.comment.targetIssueNumber).toBeNull(); + expect(report.comment.skippedReason).toContain('No exact historical'); + }); + + it('keeps missing and failed platforms explicit while candidate-specific markers remain stable across reruns', async () => { + // Given a missing result or a failed cohort for the same immutable candidate. + const first = await evaluationReport(plan, [], async () => ({})); + const second = await evaluationReport( + plan, + [{schema: 1, candidate: plan.candidate, toolingSha: plan.toolingSha, platform: 'android', status: 'failed', rawError: 'secret'}], + async () => ({}), + ); + // When the version or App commit changes. + const other = {...plan.candidate, appSha: 'd'.repeat(40)}; + // Then the old failed result is not converted to numeric success, and other candidates have separate markers. + expect(JSON.stringify(first.summary)).toContain('missing'); + expect(JSON.stringify(second.summary)).toContain('failed'); + expect(first.comment.marker).toBe(second.comment.marker); + expect(candidateKey(other)).not.toBe(candidateKey(plan.candidate)); + expect(JSON.stringify(second)).not.toContain('secret'); + }); + + it('retains hashed overlay/config provenance and reports missing frames without zero filling', () => { + // Given approved digest fields and twenty unavailable ancillary samples. + const input = {...result(), toolchain: {identitySha256: hash, reviewedOverlaySha256: hash, environmentSha256: hash, fixtureSha256: hash}}; + const missing = Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable', raw: 'private'})); + // When numeric coverage and the public hash whitelist are summarized. + const publicResult = summarizePlatform(plan, 'android', input); + const coverage = summarizeFrames(missing); + // Then provenance survives while unavailable frames have no fabricated summary or raw data. + expect(publicResult.toolchain?.reviewedOverlaySha256).toBe(hash); + expect(coverage.availableSamples).toBe(0); + expect(coverage.unavailableSamples).toBe(20); + expect(coverage.summary).toBeNull(); + expect(JSON.stringify(coverage)).not.toContain('private'); + expect(() => summarizeFrames(missing.slice(1))).toThrow(); + }); + + it('weights native jank by total frames while preserving original percentages and windows', () => { + // Given two available windows with unequal total frame counts. + const samples = Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable'})); + const available = [ + {run: 1, status: 'available', totalFrames: 10, jankyFrames: 5, originalPercent: 50, sampleWindowMs: 100}, + {run: 2, status: 'available', totalFrames: 90, jankyFrames: 0, originalPercent: 0, sampleWindowMs: 200}, + ]; + // When total-frame weighted jank and original per-window values are calculated. + const coverage = summarizeFrames([...available, ...samples.slice(2)]); + // Then the weighted percentage is five rather than the sample-mean twenty-five. + expect(coverage.summary?.weightedJankyPercent).toBe(5); + expect(coverage.summary?.originalPercent.average).toBe(25); + expect(coverage.unavailableSamples).toBe(18); + expect(coverage.summary?.sampleWindowMs.average).toBe(150); + }); + + it('uses known interpolated statistics while retaining IQR outliers', () => { + // Given a small known distribution containing an outlier. + // When all values are summarized without trimming. + const stats = distribution([1, 2, 3, 100]); + // Then the arithmetic and retained outlier count are explicit. + expect(stats.runs).toBe(4); + expect(stats.average).toBe(26.5); + expect(stats.median).toBe(2.5); + expect(stats.p10).toBe(1.3); + expect(stats.iqrOutliers).toBe(1); + }); + + it('rejects malformed plan identities before execution or reporting', () => { + // Given a valid immutable plan. + // When a platform or artifact identity is malformed. + // Then no permissive fallback can interpret it as another candidate. + expect(parsePlan(plan)).toEqual(plan); + expect(() => parsePlan({...plan, toolingSha: 'short'})).toThrow(); + expect(() => parsePlan({...plan, platforms: ['android', 'android']})).toThrow(); + expect(() => summarizePlatform(plan, 'android', {...result(), artifactSha256: {release: 'a'.repeat(40), optimized: hash, instrumented: hash}})).toThrow(); + }); +}); diff --git a/tests/tooling/pgoEvaluationRun.test.ts b/tests/tooling/pgoEvaluationRun.test.ts new file mode 100644 index 000000000000..c2b2157a1f6f --- /dev/null +++ b/tests/tooling/pgoEvaluationRun.test.ts @@ -0,0 +1,229 @@ +// cspell:ignore cooldown probey + +import {describe, expect, it} from 'bun:test'; + +import createAndroidPgoAdapter from '@scripts/pgo/android'; +import {benchmarkStagePlan, configureSafeEvaluationOptions, readCohort, readPublicFrames, readSamples, runGuardedStage} from '@scripts/pgo/evaluationRun'; +import {hasExecutedSwiftFunctions, profileCounts, validateJourney} from '@scripts/pgo/proof'; + +import {mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +describe('PGO evaluation cohort acceptance', () => { + it('selects the shrunk artifact directory before constructing the Android adapter', () => { + // Given inherited settings that would select the ordinary bootstrap directory. + const names = ['PGO_ANDROID_RELEASE_PIPELINE', 'PGO_ANDROID_SHRINK', 'PGO_BENCHMARK_COOLDOWN_SECONDS', 'PATH']; + const previous = names.map((name) => [name, process.env[name]] as const); + process.env.PGO_ANDROID_SHRINK = 'false'; + try { + // When safe evaluation options are applied before adapter construction. + configureSafeEvaluationOptions(process.cwd()); + const adapter = createAndroidPgoAdapter('evaluation.app', 'physical'); + // Then the parent and child build both refer to the shrunk artifact path. + expect(adapter.artifactPaths.instrumented).toContain('/apk-shrunk/'); + expect(process.env.PGO_ANDROID_RELEASE_PIPELINE).toBe('false'); + } finally { + for (const [name, value] of previous) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + } + }); + it('publishes frame availability without private responses or invented zeros', () => { + // Given twenty unavailable frame windows containing private raw responses. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + const path = join(directory, 'frames.json'); + writeFileSync( + path, + JSON.stringify({ + schema: 1, + requestedRuns: 20, + samples: Array.from({length: 20}, (_, index) => ({run: index + 1, status: 'unavailable', raw: 'private UI response', reason: 'private path'})), + }), + ); + try { + // When frame observations pass into the public result. + const frames = readPublicFrames(path); + // Then unavailable observations retain their run identity and no private fields or numeric replacements. + expect(frames).toHaveLength(20); + expect(frames.at(0)).toEqual({run: 1, status: 'unavailable'}); + expect(JSON.stringify(frames)).not.toContain('private'); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + it('rejects partial and foreign training before optimizing', () => { + // Given a complete training result and its expected source and toolchain. + const result = { + platform: 'android', + status: 'passed', + accountClass: 'heavy', + requestedRuns: 3, + completedRuns: 3, + collectProfiles: true, + navigationOnly: false, + sourceIdentity: 'source', + toolchain: 'tools', + }; + // When a completed run is missing or the source differs. + // Then neither result can be accepted for optimization. + expect(() => validateJourney({...result, completedRuns: 2}, 'android', 'source', 'tools')).toThrow(); + expect(() => validateJourney(result, 'android', 'foreign', 'tools')).toThrow(); + expect(() => validateJourney(result, 'android', 'source', 'tools')).not.toThrow(); + }); + + it('retains large LLVM counters and rejects a zero execution maximum', () => { + // Given LLVM summaries using Maximum function count rather than Total count. + const positive = 'Total functions: 2\nMaximum function count: 4294967297\n'; + // When counts exceed the unsigned 32-bit range. + const counts = profileCounts(positive); + // Then the exact count survives and a zero maximum remains invalid. + expect(counts.count).toBe('4294967297'); + expect(() => profileCounts('Total functions: 2\nMaximum function count: 0\n')).toThrow(); + }); + it('prefers an explicit aggregate LLVM count over a positive maximum', () => { + // Given frontend and IR summaries with different supported execution-count fields. + const frontend = 'Total functions: 2\nMaximum function count: 42\nTotal count: 100'; + const ir = 'Total functions: 2\nMaximum function count: 42'; + // When both summaries are parsed, or an explicit aggregate count is zero. + expect(profileCounts(frontend).count).toBe('100'); + expect(profileCounts(ir).count).toBe('42'); + // Then an executed maximum cannot override a reported zero aggregate count. + expect(() => profileCounts(frontend.replace('Total count: 100', 'Total count: 0'))).toThrow('no executed functions'); + }); + + it('requires executed Swift counters without borrowing a neighboring function count', () => { + // Given a zero-count Swift function followed by an executed Clang function. + const report = 'Counters:\n $s5Probe5probeyS2iF:\n Hash: 0x0\n Counters: 1\n Function count: 0\n Block counts: []\n clangFunction:\n Function count: 42\n'; + // When the iOS execution guard reads frontend or IR function counter reports. + expect(hasExecutedSwiftFunctions(report)).toBe(false); + expect(hasExecutedSwiftFunctions(report.replace('Function count: 0', 'Function count: 3'))).toBe(true); + expect(hasExecutedSwiftFunctions(report.replace('$s5Probe5probeyS2iF', 'anotherClangFunction'))).toBe(false); + expect(hasExecutedSwiftFunctions('Total count: 42')).toBe(false); + // Then only Swift functions with their own positive execution counters pass. + expect(hasExecutedSwiftFunctions('Counters:\n $s5Probe5probeyS2iF:\n Hash: 0x12\n Counters: 3\n Block counts: [0, 102, 1]\n')).toBe(true); + expect(hasExecutedSwiftFunctions('Counters:\n $s5Probe5probeyS2iF:\n Block counts: [0, 0, 0]\n')).toBe(false); + expect(hasExecutedSwiftFunctions(report.replace('$s5Probe5probeyS2iF', '/tmp/Probe.swift:$s5Probe5probeyS2iF').replace('Function count: 0', 'Function count: 3'))).toBe(true); + }); + + it('executes fresh paired workloads in both orders using supported CLI actions', () => { + // Given the evaluation stage plan used by the physical runner. + const stages = benchmarkStagePlan(); + // When its actions and orders are inspected without touching a device. + const actions = stages.map(({order, scenario}) => `${scenario.action}:${order}`); + // Then all three workload types have distinct cohorts in each order. + expect(actions).toEqual([ + 'benchmark-heavy-journey:release-first', + 'benchmark-journey:release-first', + 'benchmark:release-first', + 'benchmark-heavy-journey:optimized-first', + 'benchmark-journey:optimized-first', + 'benchmark:optimized-first', + ]); + }); + + it('stops before launching a workload when immutable inputs changed', () => { + // Given a failed source or profile guard and a workload that has not started. + let launched = false; + // When the guarded execution boundary is entered. + const execute = () => + runGuardedStage( + () => { + throw new Error('changed'); + }, + () => { + launched = true; + }, + ); + // Then no device operation is launched. + expect(execute).toThrow('changed'); + expect(launched).toBe(false); + }); + + it('rejects completed work if its inputs changed during execution', () => { + // Given a guard that succeeds before work and fails afterward. + let checks = 0; + // When the workload completes against changing inputs. + const execute = () => + runGuardedStage( + () => { + checks += 1; + if (checks === 2) { + throw new Error('changed'); + } + }, + () => 'completed', + ); + // Then the stage cannot be accepted as a successful cohort. + expect(execute).toThrow('changed'); + expect(checks).toBe(2); + }); + + it('rejects a cohort from the wrong physical device before reading samples', () => { + // Given metadata from a different device despite a successful command exit. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + writeFileSync(join(directory, 'run.json'), JSON.stringify({deviceID: 'other', appID: 'app', order: 'release-first', runs: 20, warmUpRunsPerBinary: 1, cooldownSecondsPerBinary: 5})); + try { + // When the expected cohort is bound to the approved device. + // Then metadata mismatch prevents accepting unrelated CSV files. + expect(() => readCohort(directory, 'startup', 'release-first', ['ManualAppStartup'], 'approved', 'app')).toThrow('context'); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + + it('accepts actual startup batch filenames and preserves every native timing', () => { + // Given a completed startup batch using the startup producer's optimized.csv name. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + writeFileSync( + join(directory, 'run.json'), + JSON.stringify({platform: 'android', deviceID: 'approved', appID: 'app', order: 'optimized-first', runs: 20, warmUpRunsPerBinary: 1, cooldownSecondsPerBinary: 5}), + ); + const csv = `run,span,duration_ms\n${Array.from({length: 20}, (_, index) => `${index + 1},ManualAppStartup,${100 + index}`).join('\n')}\n`; + writeFileSync(join(directory, 'release.csv'), csv); + writeFileSync(join(directory, 'optimized.csv'), csv); + try { + // When the startup cohort reader binds the physical device and run order. + const cohort = readCohort(directory, 'startup', 'optimized-first', ['ManualAppStartup'], 'approved', 'app'); + // Then both complete observations and their actual file hashes are retained. + expect(cohort.release).toHaveLength(20); + expect(cohort.optimized).toHaveLength(20); + expect(cohort.optimizedCsvSha256).toBe(cohort.releaseCsvSha256); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + it('retains zero duration samples and requires twenty distinct runs', () => { + // Given a complete native span cohort including a zero duration observation. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + const path = join(directory, 'samples.csv'); + writeFileSync(path, `run,span,duration_ms\n${Array.from({length: 20}, (_, index) => `${index + 1},startup,${index}`).join('\n')}\n`); + try { + // When the exact expected span is read. + const samples = readSamples(path, ['startup']); + // Then all observations are retained without inventing an exclusion threshold. + expect(samples).toHaveLength(20); + expect(samples.at(0)?.durationMs).toBe(0); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); + + it('rejects duplicate runs even when the total row count is twenty', () => { + // Given a CSV whose final row repeats run nineteen instead of run twenty. + const directory = mkdtempSync(join(tmpdir(), 'pgo-evaluation-')); + const path = join(directory, 'samples.csv'); + writeFileSync(path, `run,span,duration_ms\n${Array.from({length: 20}, (_, index) => `${Math.min(index + 1, 19)},startup,1`).join('\n')}\n`); + try { + // When cohort validation checks each run and span pair. + // Then the incomplete cohort cannot become a successful public evaluation. + expect(() => readSamples(path, ['startup'])).toThrow('twenty complete samples'); + } finally { + rmSync(directory, {recursive: true, force: true}); + } + }); +}); diff --git a/tests/tooling/pgoEvaluationWorkflow.test.ts b/tests/tooling/pgoEvaluationWorkflow.test.ts new file mode 100644 index 000000000000..3cee4b862a38 --- /dev/null +++ b/tests/tooling/pgoEvaluationWorkflow.test.ts @@ -0,0 +1,78 @@ +// cspell:ignore chrispader +import {describe, expect, it} from 'bun:test'; + +import {isRecord} from '@scripts/pgo/evaluationSchema'; + +import {readFileSync} from 'node:fs'; +import {runInNewContext} from 'node:vm'; + +const input: unknown = Bun.YAML.parse(readFileSync('.github/workflows/pgoEvaluation.yml', 'utf8')); +if (!isRecord(input) || !isRecord(input.jobs) || !isRecord(input.jobs.verify)) { + throw new Error('Missing emulator verification workflow.'); +} +const workflow = input; +const job = input.jobs.verify; + +describe('PGO emulator workflow', () => { + it.each([ + ['Expensify/App', 'refs/heads/@chrispader/perf/pgo-ci-verification', true], + ['margelo/Expensify', 'refs/heads/@chrispader/perf/pgo-ci-verification', false], + ['Expensify/App', 'refs/heads/main', false], + ['Expensify/App', 'refs/pull/34/merge', false], + ])('limits private dependency access for %s at %s', (repository, ref, expected) => { + // Given the workflow's actual authorization expression. + if (typeof job.if !== 'string') { + throw new Error('Missing authorization condition.'); + } + // When Actions evaluates a repository and branch. + const result: unknown = runInNewContext(job.if, {github: {repository, ref}}); + // Then only the reviewed upstream preview branch receives credentials. + expect(result).toBe(expected); + }); + + it('starts on branch copy without requiring a deployment or a default-branch merge', () => { + // Given the verification workflow's event configuration. + const events = workflow.on; + // When its push trigger is inspected. + if (!isRecord(events) || !isRecord(events.push)) { + throw new Error('Missing push trigger.'); + } + // Then copying the branch starts verification and no deployment starts it. + expect(events.push.branches).toEqual(['@chrispader/perf/pgo-ci-verification']); + expect(events.workflow_run).toBeUndefined(); + expect(events.pull_request_target).toBeUndefined(); + expect(events.workflow_dispatch).toBeDefined(); + }); + + it.each(['both', 'android', 'ios'])('runs the requested hosted platform selection %s', (platform) => { + // Given the actual platform matrix expression and hosted runner expression. + if (!isRecord(job.strategy) || !isRecord(job.strategy.matrix) || typeof job.strategy.matrix.platform !== 'string' || typeof job['runs-on'] !== 'string') { + throw new Error('Missing emulator matrix.'); + } + const expression = (value: string) => value.slice(3, -2).trim(); + // When the selected platforms are expanded by Actions. + const platforms: unknown = runInNewContext(expression(job.strategy.matrix.platform), {inputs: {platform}, fromJSON: JSON.parse}); + // Then neither platform requests a self-hosted physical runner. + expect(platforms).toEqual(platform === 'both' ? ['android', 'ios'] : [platform]); + expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'android'}})).toBe('ubuntu-24.04'); + expect(runInNewContext(expression(job['runs-on']), {matrix: {platform: 'ios'}})).toBe('blacksmith-12vcpu-macos-latest'); + }); + + it('uses only the existing checkout and SDK secrets and preserves failure artifacts', () => { + // Given the complete published workflow. + const text = readFileSync('.github/workflows/pgoEvaluation.yml', 'utf8'); + // When private configuration references are collected. + const secrets = [...new Set(Array.from(text.matchAll(/secrets[.]([A-Z_]+)/g), (match) => match[1]))].sort(); + // Then account login, device IDs and signing secrets are unnecessary. + expect(secrets).toEqual(['MAPBOX_SDK_DOWNLOAD_TOKEN', 'OS_BOTIFY_TOKEN']); + expect(text).not.toContain('PGO_ANDROID_DEVICE_ID'); + expect(text).not.toContain('PGO_IOS_DEVICE_ID'); + expect(text).not.toContain('issues: write'); + if (!Array.isArray(job.steps)) { + throw new Error('Missing verification steps.'); + } + const upload: unknown = job.steps.find((step: unknown) => isRecord(step) && step.name === 'Upload verification profiles and report'); + expect(isRecord(upload) && upload.if).toBe('always()'); + expect(readFileSync('.github/workflows/deploy.yml', 'utf8')).not.toContain('pgoCandidate'); + }); +}); diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts new file mode 100644 index 000000000000..85bc67f6d068 --- /dev/null +++ b/tests/tooling/pgoIOSRock.test.ts @@ -0,0 +1,41 @@ +// cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata + +import {describe, expect, it} from 'bun:test'; + +import {iosRockBuildArguments} from '@scripts/pgo/ios'; + +import {parseArgs} from '@rock-js/tools'; + +describe('PGO iOS Rock build arguments', () => { + it('selects a simulator without changing the default device destination', () => { + // Given an explicit verification destination and unsigned simulator settings. + const settings = ['build', '-sdk', 'iphonesimulator', 'CODE_SIGNING_ALLOWED=NO']; + // When the verification build is routed through Rock. + const args = iosRockBuildArguments('/tmp/pgo-simulator', settings, 'generic/platform=iOS Simulator'); + // Then the simulator SDK stays separate from device builds and settings remain intact. + expect(args.at(8)).toBe('generic/platform=iOS Simulator'); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); + it('selects a local device Release build and preserves exact compiler settings', () => { + // Given compiler settings containing spaces and inherited Xcode variables. + const settings = ['build', 'ARCHS=arm64', 'CODE_SIGN_IDENTITY=Apple Development', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-use=/tmp/profile directory/journey.profdata']; + // When the adapter creates a Rock invocation. + const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); + // Then the local Release destination and existing pod preparation are explicit. + expect(args.slice(0, 12)).toEqual([ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + '/tmp/derived data/optimized', + '--no-install-pods', + ]); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); +}); diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts new file mode 100644 index 000000000000..6adee4af2183 --- /dev/null +++ b/tests/tooling/pgoJourney.test.ts @@ -0,0 +1,437 @@ +import {describe, expect, it} from 'bun:test'; + +import {frameWindowResult, journeyBenchmarkPaths, parseAndroidFrameHealth, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; +import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; +import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; +import { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + scrollDistance, + scrollFastBothWays, + showTab, + spendSectionTapPoint, + waitForTab, +} from '@scripts/pgo/journeyWorkload'; +import {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +const fixture = { + accountEmail: 'heavy@example.com', + accountClass: 'heavy', + description: 'Owner-approved account with a populated inbox, message history, expenses, and reports.', + report: {query: 'busy', resultLabelPrefix: '#busy, Test workspace', title: '#busy'}, + personalChat: {query: 'heavy@example.com', resultLabelPrefix: 'Test User (you), heavy@example.com', title: 'Test User (you)'}, +}; + +describe('PGO journey safeguards', () => { + it('retains unavailable ancillary telemetry without inventing a zero frame sample', () => { + // Given native telemetry is unavailable while workload timing remains independently recorded. + const raw = {sampling: {fps: {resetsAfterRead: true}}, metrics: {fps: {available: false, reason: 'No graphics counters'}}}; + // When the response is classified. + const result = parseAndroidFrameHealth(raw, true); + // Then the raw reason survives and no numeric frame sample is fabricated. + expect(result).toEqual({status: 'unavailable', reason: 'Android frame sampler unavailable: No graphics counters', raw}); + expect('metrics' in result).toBe(false); + }); + + it('requires declared reset support and valid positive post-scroll counters', () => { + // Given a sampler response with valid counters and a declared reset capability. + const raw = {sampling: {fps: {resetsAfterRead: true}}, metrics: {fps: {available: true, totalFrameCount: 10, droppedFrameCount: 2, droppedFramePercent: 20, sampleWindowMs: 500}}}; + // When valid, malformed, unsupported and empty windows are classified. + expect(parseAndroidFrameHealth(raw, true).status).toBe('available'); + expect(parseAndroidFrameHealth({...raw, sampling: {fps: {resetsAfterRead: false}}}, true).status).toBe('unavailable'); + expect(parseAndroidFrameHealth({metrics: null}, true).status).toBe('unavailable'); + for (const invalid of [{totalFrameCount: -1}, {droppedFrameCount: 11}, {sampleWindowMs: Number.NaN}, {totalFrameCount: 0, droppedFrameCount: 0, sampleWindowMs: 0}]) { + // Then invalid telemetry is unavailable rather than a zero-valued observation. + expect(parseAndroidFrameHealth({...raw, metrics: {fps: {...raw.metrics.fps, ...invalid}}}, true).status).toBe('unavailable'); + } + expect(parseAndroidFrameHealth({...raw, metrics: {fps: {...raw.metrics.fps, totalFrameCount: 0, droppedFrameCount: 0, sampleWindowMs: 0}}}, false).status).toBe('available'); + }); + + it('preserves both responses when the pre-scroll boundary is unavailable', () => { + // Given an unavailable boundary and otherwise valid post-scroll counters. + const before = parseAndroidFrameHealth({unavailable: true}, false); + const after = parseAndroidFrameHealth( + {sampling: {fps: {resetsAfterRead: true}}, metrics: {fps: {available: true, totalFrameCount: 10, droppedFrameCount: 1, droppedFramePercent: 10, sampleWindowMs: 500}}}, + true, + ); + // When the window is assembled without another destructive sampler read. + const result = frameWindowResult(before, after); + // Then the window remains unavailable with its complete evidence. + expect(result.status).toBe('unavailable'); + expect(result.raw).toEqual({before: before.raw, after: after.raw}); + expect('metrics' in result).toBe(false); + }); + + it('logs the separate navigation spans in both PGO builds without replacing startup samples', () => { + // Given release and optimized artifacts share one build-time benchmark allowlist. + const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); + // When / Then the startup span and every short-journey span remain available. + expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES]); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1')).toEqual({ + release: '/pgo/ios/benchmarks/journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/journey/batch-1/pgo-optimized.csv', + }); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1', 'heavy-journey')).toEqual({ + release: '/pgo/ios/benchmarks/heavy-journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/heavy-journey/batch-1/pgo-optimized.csv', + }); + }); + + it('hashes iOS bundle resources as well as the app executable', async () => { + const app = mkdtempSync(join(tmpdir(), 'pgo-test-app-')); + try { + mkdirSync(join(app, 'Frameworks')); + writeFileSync(join(app, 'Expensify'), 'same executable'); + writeFileSync(join(app, 'main.jsbundle'), 'first bundle'); + const before = await sha256Artifact(app); + writeFileSync(join(app, 'main.jsbundle'), 'second bundle'); + expect(await sha256Artifact(app)).not.toBe(before); + } finally { + rmSync(app, {recursive: true, force: true}); + } + }); + + it('requires explicit message approval in the fixture', () => { + // Given account approval alone does not authorize a particular message destination. + for (const allowMessages of [undefined, false, 'true']) { + // When / Then full journeys remain unavailable until approval is recorded explicitly. + expect(parseJourneyFixture({...fixture, allowMessages}).allowMessages).toBe(false); + } + expect(parseJourneyFixture({...fixture, allowMessages: true}).allowMessages).toBe(true); + }); + + it('requires the account owner to select a heavy account explicitly', () => { + // Given an unclassified or ordinary account, a populated-looking screen is insufficient evidence. + for (const accountClass of [undefined, 'ordinary', '']) { + // When / Then the fixture cannot start a training run. + expect(() => parseJourneyFixture({...fixture, accountClass})).toThrow('accountClass="heavy"'); + } + }); + + it("rejects message destinations that are not the approved account's personal chat", () => { + // Given a fixture that would send messages to another person or to a different account's personal chat. + for (const personalChat of [ + {...fixture.personalChat, query: 'another@example.com'}, + {...fixture.personalChat, title: 'Another person'}, + {...fixture.personalChat, resultLabelPrefix: 'Other User (you), other@example.com'}, + {...fixture.personalChat, resultLabelPrefix: 'Test User (you), heavy@example.com.invalid'}, + ]) { + // When / Then no device action is permitted for that fixture. + expect(() => parseJourneyFixture({...fixture, personalChat})).toThrow('own "(you)" chat'); + } + }); + + it('rejects an empty workload or an unbounded scrolling loop', () => { + // Given invalid workload sizes that would create an unrepresentative or runaway recording. + for (const scrolls of [0, 1, 31, NaN, '8']) { + // When / Then parsing fails before the app is touched. + expect(() => parseJourneyFixture({...fixture, scrolls})).toThrow('scrolls'); + } + }); + + it('filters system keyboard content out of app assertions', () => { + // Given Android's keyboard can display labels unrelated to the foreground app. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {bundleId: 'keyboard.app', label: 'Sign in'}, + {bundleId: 'test.app', label: 'Inbox'}, + ], + }, + 'test.app', + ); + // When / Then keyboard content cannot falsely trigger a sign-in failure. + expect(nodes).toHaveLength(1); + expect(() => assertSignedIn(nodes)).not.toThrow(); + }); + + it('stops for manual sign-in on both platforms', () => { + // Given either platform has reached an authentication or code-entry screen. + for (const identifier of ['username', 'validateCode']) { + for (const type of ['android.widget.EditText', 'TextField']) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{identifier, type}]}, 'test.app'); + // When / Then the runner requests user input instead of filling credentials. + expect(() => assertSignedIn(nodes)).toThrow('SIGN_IN_REQUIRED'); + } + } + }); + + it('rejects a snapshot from the wrong app', () => { + // Given device focus has moved to an unexpected application. + const snapshot = {appBundleId: 'other.app', nodes: []}; + // When / Then its labels cannot satisfy the journey's assertions. + expect(() => parseJourneySnapshot(snapshot, 'test.app')).toThrow('expected app'); + }); + + it('normalizes accessibility text direction markers without changing its words', () => { + // Given translated UI labels include bidirectional isolation and non-breaking spaces. + const label = '\u2066Most\u00a0recent\u2069'; + // When / Then the same label can be compared across native platforms. + expect(normalizeLabel(label)).toBe('Most recent'); + }); + + it('matches report identity independently of the changing last-message preview', () => { + // Given the workspace has similarly named reports and a new message changes the preview. + const label = '#busy, Test workspace • New message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: '#busy, Test workspace copy • Other message'}]}, 'test.app'); + // When / Then only the specified workspace can be selected. + expect(findReportResult(nodes, '#busy, Test workspace')).toBe(label); + }); + + it('refuses ambiguous report matches', () => { + // Given two results share the fixture identity. + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label: '#busy, Test workspace'}, {label: '#busy, Test workspace • Message'}]}, 'test.app'); + // When / Then choosing the first result would risk opening an unintended destination. + expect(() => findReportResult(nodes, '#busy, Test workspace')).toThrow('Ambiguous'); + }); + + it('finds the approved personal chat after its email subtitle becomes a message preview', () => { + // Given a previous run sent a message, while another account has a similarly named chat. + const label = 'Test User (you), PGO previous run message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: 'Test User (you)'}, {label: 'Test User, Message from another account'}]}, 'test.app'); + // When / Then the validated personal-chat title matches only the full result row. + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix, fixture.personalChat.title)).toBe(label); + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix)).toBeUndefined(); + }); + + it('sizes scrolls for different screens without starting inside the fixed header', () => { + // Given Android uses physical pixels while iOS accessibility exposes logical screen dimensions. + for (const [width, height] of [ + [1080, 2280], + [390, 844], + ]) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width, height}}]}, 'test.app'); + // When the gesture is centered, both ends stay within the middle half of the portrait viewport. + const distance = scrollDistance(nodes); + expect(distance).toBeGreaterThan(0); + expect(distance).toBeLessThanOrEqual((height ?? 0) / 2); + expect(scrollDistance(nodes, true)).toBeGreaterThan(distance); + } + // Then missing viewport information must fail instead of falling back to a screen-edge gesture. + expect(() => scrollDistance([])).toThrow('scroll bounds'); + }); + + it('uses fixed rapid swipes and validates movement in both directions', () => { + // Given a report with distinct content at the recent and older scroll positions. + const snapshots = ['recent', 'older', 'recent'].map((label) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width: 390, height: 844}}, {label}]}, 'test.app'), + ); + const commands: string[][] = []; + let read = 0; + // When the heavy benchmark scrolls in both directions. + scrollFastBothWays( + { + snapshot: () => snapshots.at(read++) ?? [], + command: (...args) => commands.push(args), + }, + 4, + ); + // Then it sends fixed-duration gestures and checks content after each direction. + expect(commands).toHaveLength(8); + expect(commands.every((args) => args.includes('120'))).toBe(true); + expect(commands.filter((args) => args.includes('--settle'))).toHaveLength(2); + expect(commands.at(0)?.at(1)).toBe('up'); + expect(commands.at(-1)?.at(1)).toBe('down'); + }); + + it('selects iOS status-bearing bottom tabs without confusing them with page headings', () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Inbox', rect: {width: 390, height: 50, y: 80}}, + {label: 'Inbox. Your review is required', rect: {width: 78, height: 71, y: 749}}, + {label: 'Inbox', type: 'StaticText', rect: {width: 26, height: 14, y: 793}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + expect(findTabNode(nodes, 'Inbox')?.label).toBe('Inbox. Your review is required'); + expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); + }); + + it('waits for a status-bearing iOS Account tab without requiring an exact label', async () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + const device = {snapshot: () => nodes}; + await expect(waitForTab(device, 'Account')).resolves.toBeUndefined(); + }); + + it('waits through Android relaunch loading before navigating from a restored page', async () => { + // Given a loading view, then a restored nested page, then the Inbox tab after app Back. + const screen = {type: 'android.widget.ScrollView', rect: {x: 0, y: 0, width: 1080, height: 2280}}; + const snapshots = [ + [screen], + [screen, {type: 'android.widget.Button', label: 'Back', rect: {x: 21, y: 144, width: 67, height: 105}}], + [screen, {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}], + ].map((nodes) => parseJourneySnapshot({appBundleId: 'test.app', nodes}, 'test.app')); + const commands: string[][] = []; + let read = 0; + const device: Parameters[0] = { + platform: 'android', + snapshot: () => snapshots[Math.min(read++, snapshots.length - 1)], + command: (...args: string[]) => commands.push(args), + wait: () => {}, + press: () => {}, + hasLabel: () => false, + pressLabel: () => {}, + waitLabel: () => {}, + fill: () => {}, + back: () => commands.push(['hardware back']), + open: () => {}, + }; + + // When the benchmark navigates to Inbox after relaunch. + await showTab(device, 'Inbox'); + + // Then loading does not trigger Back, and the app's Back control is used once. + expect(commands).toEqual([ + ['press', '55', '197', '--settle'], + ['press', '324', '2061', '--settle'], + ]); + }); + + it.each([true, false])('checks a slow Back result before enforcing the tab deadline (tab appears: %j)', async (tabAppears) => { + // Given a successful Back command that completes after the navigation deadline. + const screen = {type: 'Application', rect: {x: 0, y: 0, width: 1080, height: 2280}}; + const back = {type: 'android.widget.Button', label: 'Back', rect: {x: 21, y: 144, width: 67, height: 105}}; + const inbox = {type: 'android.view.View', label: 'Inbox', rect: {x: 216, y: 1968, width: 216, height: 186}}; + let elapsed = 0; + let reads = 0; + const commands: string[][] = []; + const device: Parameters[0] = { + platform: 'android', + snapshot: () => parseJourneySnapshot({appBundleId: 'test.app', nodes: [screen, reads++ > 0 && tabAppears ? inbox : back]}, 'test.app'), + command: (...args: string[]) => { + commands.push(args); + elapsed = 31_000; + }, + wait: () => {}, + press: () => {}, + hasLabel: () => false, + pressLabel: () => {}, + waitLabel: () => {}, + fill: () => {}, + back: () => {}, + open: () => {}, + }; + // When the loop checks the completed Back's result. + const navigation = showTab(device, 'Inbox', () => elapsed); + // Then a mounted tab succeeds, while an unchanged screen fails without another Back. + if (tabAppears) { + await expect(navigation).resolves.toBeUndefined(); + expect(commands).toHaveLength(2); + } else { + await expect(navigation).rejects.toThrow('Cannot return to the Inbox tab.'); + expect(commands).toHaveLength(1); + } + expect(reads).toBe(2); + }); + + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { + const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; + const snapshot = (tabs: Array) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'Application', rect: {x: 0, y: 0, width: 1080, height: 2280}}, ...tabs]}, 'test.app'); + + expect(findTabNode(snapshot([tab, tab]), 'Inbox')?.x).toBe(216); + expect(() => findTabNode(snapshot([tab, {...tab, rect: {...tab.rect, x: 432}}]), 'Inbox')).toThrow('Ambiguous Inbox tab'); + }); + + it('targets the visible All filter rather than the iOS strip cell center over Unread', () => { + // Given an iOS accessibility cell whose bounds cover the whole Inbox filter strip. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'All', type: 'Cell', rect: {width: 390, height: 52, x: 0, y: 115}}, + {label: 'Unread', type: 'Other', rect: {width: 136, height: 40, x: 102, y: 115}}, + ], + }, + 'test.app', + ); + // When the journey chooses the All filter's tap point. + const tapPoint = allFilterTapPoint(nodes); + // Then it stays left of Unread. + expect(tapPoint).toEqual({x: 51, y: 135}); + }); + + it('taps the iOS search Back button even when the keyboard marks it non-hittable', () => { + // Given the search screen's bounded Back button beneath an open keyboard. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Back', type: 'Button', hittable: false, rect: {width: 40, height: 40, x: 8, y: 59}}, + {label: 'Back', type: 'Button', rect: {width: 40, height: 40, x: 300, y: 700}}, + ], + }, + 'test.app', + ); + // When the journey looks for the in-app Back control. + const tapPoint = inAppBackTapPoint(nodes); + // Then it chooses the top-left button rather than the keyboard area. + expect(tapPoint).toEqual({x: 28, y: 79}); + }); + + it('detects iOS report scrolling when offscreen accessibility labels stay loaded', () => { + // Given the same report labels at different vertical positions after a scroll. + const reportNodes = (firstMessageY: number) => + parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Old message', rect: {width: 390, height: 40, y: firstMessageY - 80}}, + {label: 'Recent message', rect: {width: 390, height: 40, y: firstMessageY}}, + ], + }, + 'test.app', + ); + // When content signatures are compared. + const before = contentSignature(reportNodes(500)); + const after = contentSignature(reportNodes(266)); + // Then movement is visible despite identical labels. + expect(before).not.toBe(after); + }); + + it('targets the visible iOS Spend segments despite a wide Expenses strip cell', () => { + // Given an Expenses cell that covers the full segment strip and a bounded Reports control. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Expenses', type: 'Cell', rect: {width: 1426, height: 52, x: 0, y: 115}}, + {label: 'Reports', type: 'Other', rect: {width: 94, height: 40, x: 125, y: 115}}, + ], + }, + 'test.app', + ); + // When each section is selected. + const expenses = spendSectionTapPoint(nodes, 'Expenses'); + const reports = spendSectionTapPoint(nodes, 'Reports'); + // Then both tap points land inside their visible segments. + expect(expenses).toEqual({x: 63, y: 135}); + expect(reports).toEqual({x: 172, y: 135}); + }); +});