diff --git a/.gitignore b/.gitignore index 8c6d0691fdb9..dda47ce6a069 100644 --- a/.gitignore +++ b/.gitignore @@ -136,6 +136,9 @@ config/certificatePinning/roots/ # Rspack build profiling output (RSPACK_PROFILE=OVERVIEW|ALL) .rspack-profile-*/ +# Local native PGO profiles +.pgo/ + # Expo .expo dist/ diff --git a/android/app/build.gradle b/android/app/build.gradle index 7c663046e0ab..3ab4a8309087 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -84,6 +84,16 @@ project.ext.envConfigFiles = [ * Set this to true to Run Proguard on Release builds to minify the Java bytecode. */ def enableProguardInReleaseBuilds = true +def pgoMode = providers.gradleProperty('pgoMode').getOrElse('off') +def pgoProfile = providers.gradleProperty('pgoProfile').orNull + +if (!(pgoMode in ['off', 'generate', 'use'])) { + throw new GradleException("Unsupported pgoMode '${pgoMode}'. Use off, generate, or use.") +} + +if (pgoMode == 'use' && !pgoProfile) { + throw new GradleException('pgoProfile is required when pgoMode=use.') +} /** * The preferred build flavor of JavaScriptCore (JSC) @@ -119,6 +129,7 @@ android { // Supported language variants must be declared here to avoid from being removed during the compilation. // This also helps us to not include unnecessary language variants in the APK. resConfigs "en", "es" + buildConfigField 'boolean', 'PGO_PROFILE_GENERATION', (pgoMode == 'generate').toString() } flavorDimensions "default" diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index c0d4ff50a0c2..4ea4287658b7 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -28,6 +28,14 @@ android:theme="@style/AppTheme" tools:replace="android:supportsRtl"> + + + + + + #import +#ifndef EXPENSIFY_PGO_GENERATE +#define EXPENSIFY_PGO_GENERATE 0 +#endif + +#if EXPENSIFY_PGO_GENERATE +#include + +extern "C" void __llvm_profile_set_filename(const char *FilenamePat); +extern "C" int __llvm_profile_write_file(void); +extern "C" void __llvm_profile_reset_counters(void); + +using PGOSetFilename = void (*)(const char *); +using PGOWriteFile = int (*)(); +using PGOResetCounters = void (*)(); + +template +static Function HermesPGOFunction(const char *symbol) { + return reinterpret_cast(dlsym(RTLD_DEFAULT, symbol)); +} +#endif + +static NSString *const PGO_DIRECTORY_NAME = @"ExpensifyPGO"; +static NSString *const PGO_STATUS_MARKER = @"profile-status.txt"; +static CFStringRef const PGO_WRITE_NOTIFICATION = CFSTR("com.expensify.pgo.write-profiles"); +static CFStringRef const PGO_CLEAR_NOTIFICATION = CFSTR("com.expensify.pgo.clear-profiles"); + static RCTSurfaceHostingProxyRootView *_rootView = nil; static UIView *_loadingView = nil; @@ -20,6 +46,29 @@ static bool _fade = false; static bool _nativeHidden = false; +static long long CurrentTimeMilliseconds(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000); +} + +@interface RCTBootSplash () ++ (NSURL *)pgoDirectoryURL; ++ (void)writePGOStatus:(NSString *)status result:(int)result; ++ (void)writePGOProfiles; ++ (void)clearPGOProfiles; +@end + +static void HandlePGONotification(__unused CFNotificationCenterRef center, + __unused void *observer, + CFStringRef name, + __unused const void *object, + __unused CFDictionaryRef userInfo) { + if (CFEqual(name, PGO_WRITE_NOTIFICATION)) { + [RCTBootSplash writePGOProfiles]; + } else if (CFEqual(name, PGO_CLEAR_NOTIFICATION)) { + [RCTBootSplash clearPGOProfiles]; + } +} + @implementation RCTBootSplash RCT_EXPORT_MODULE(); @@ -32,6 +81,81 @@ + (BOOL)requiresMainQueueSetup { return NO; } ++ (NSURL *)pgoDirectoryURL { + NSURL *cachesURL = [[[NSFileManager defaultManager] URLsForDirectory:NSCachesDirectory + inDomains:NSUserDomainMask] firstObject]; + NSURL *directoryURL = [cachesURL URLByAppendingPathComponent:PGO_DIRECTORY_NAME isDirectory:YES]; + [[NSFileManager defaultManager] createDirectoryAtURL:directoryURL + withIntermediateDirectories:YES + attributes:nil + error:nil]; + return directoryURL; +} + ++ (void)writePGOStatus:(NSString *)status result:(int)result { + NSString *contents = [NSString stringWithFormat:@"%lld,%@,%d", CurrentTimeMilliseconds(), status, result]; + NSURL *statusURL = [[self pgoDirectoryURL] URLByAppendingPathComponent:PGO_STATUS_MARKER]; + [contents writeToURL:statusURL atomically:YES encoding:NSUTF8StringEncoding error:nil]; +} + ++ (void)initializePGOProfileCollection { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + CFNotificationCenterRef notificationCenter = CFNotificationCenterGetDarwinNotifyCenter(); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_WRITE_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + CFNotificationCenterAddObserver(notificationCenter, NULL, HandlePGONotification, PGO_CLEAR_NOTIFICATION, NULL, CFNotificationSuspensionBehaviorDeliverImmediately); + +#if EXPENSIFY_PGO_GENERATE + NSString *profilePattern = [[[self pgoDirectoryURL] URLByAppendingPathComponent:@"newdot-%m.profraw"] path]; + __llvm_profile_set_filename(profilePattern.fileSystemRepresentation); + PGOSetFilename setHermesFilename = HermesPGOFunction("expensify_llvm_profile_set_filename"); + if (setHermesFilename != nullptr) { + setHermesFilename(profilePattern.fileSystemRepresentation); + } +#endif + }); +} + ++ (void)writePGOProfiles { +#if EXPENSIFY_PGO_GENERATE + int appResult = __llvm_profile_write_file(); + PGOWriteFile writeHermesFile = HermesPGOFunction("expensify_llvm_profile_write_file"); + int hermesResult = writeHermesFile != nullptr ? writeHermesFile() : 1; + int result = appResult == 0 && hermesResult == 0 ? 0 : 1; + [self writePGOStatus:@"written" result:result]; + NSLog(@"ExpensifyPGO: wrote LLVM profile with result=%d appResult=%d hermesResult=%d", result, appResult, hermesResult); +#else + [self writePGOStatus:@"not-instrumented" result:1]; + NSLog(@"ExpensifyPGO: ignored profile write in a non-instrumented build"); +#endif +} + ++ (void)clearPGOProfiles { + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSURL *directoryURL = [self pgoDirectoryURL]; + NSArray *files = [fileManager contentsOfDirectoryAtURL:directoryURL + includingPropertiesForKeys:nil + options:0 + error:nil]; + for (NSURL *fileURL in files) { + if ([fileURL.lastPathComponent hasSuffix:@".profraw"] || [fileURL.lastPathComponent isEqualToString:PGO_STATUS_MARKER]) { + [fileManager removeItemAtURL:fileURL error:nil]; + } + } + +#if EXPENSIFY_PGO_GENERATE + __llvm_profile_reset_counters(); + PGOResetCounters resetHermesCounters = HermesPGOFunction("expensify_llvm_profile_reset_counters"); + int result = resetHermesCounters != nullptr ? 0 : 1; + if (resetHermesCounters != nullptr) { + resetHermesCounters(); + } + [self writePGOStatus:@"cleared" result:result]; +#else + [self writePGOStatus:@"not-instrumented" result:1]; +#endif +} + + (bool)isLoadingViewVisible { return _loadingView != nil && ![_loadingView isHidden]; } diff --git a/knip.json b/knip.json index 0f5564cf69bf..911d792e104d 100644 --- a/knip.json +++ b/knip.json @@ -90,5 +90,5 @@ "diff-so-fancy", "@babel/plugin-proposal-class-properties" ], - "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl"] + "ignoreBinaries": ["metro-symbolicate", "mkcert", "openssl", "agent-device"] } diff --git a/modules/ExpensifyNitroUtils/android/CMakeLists.txt b/modules/ExpensifyNitroUtils/android/CMakeLists.txt index 0f384e8dd874..ab05b09f3c0c 100644 --- a/modules/ExpensifyNitroUtils/android/CMakeLists.txt +++ b/modules/ExpensifyNitroUtils/android/CMakeLists.txt @@ -1,13 +1,42 @@ project(ExpensifyNitroUtils) +# cspell:ignore STREQUAL libclang fprofile defsym + cmake_minimum_required(VERSION 3.9.0) set(PACKAGE_NAME ExpensifyNitroUtils) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_CXX_STANDARD 20) +if(EXPENSIFY_PGO_MODE STREQUAL "generate") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir + OUTPUT_VARIABLE CLANG_RESOURCE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") + if(NOT EXISTS "${PGO_RUNTIME}") + message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") + endif() + add_compile_options(-fprofile-generate) + add_link_options( + -fprofile-generate + "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" + -Wl,--undefined=__llvm_profile_set_filename + -Wl,--undefined=__llvm_profile_write_file + -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename + -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file + -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename + -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file + ) +elseif(EXPENSIFY_PGO_MODE STREQUAL "use") + add_compile_options("-fprofile-use=${EXPENSIFY_PGO_PROFILE}") + add_link_options("-fprofile-use=${EXPENSIFY_PGO_PROFILE}") +endif() + # Define C++ library and add all sources add_library(${PACKAGE_NAME} SHARED src/main/cpp/cpp-adapter.cpp + src/main/cpp/PgoProfileWriter.cpp ) # Add Nitrogen specs @@ -29,4 +58,4 @@ target_link_libraries( ${PACKAGE_NAME} ${LOG_LIB} android -) \ No newline at end of file +) diff --git a/modules/ExpensifyNitroUtils/android/build.gradle b/modules/ExpensifyNitroUtils/android/build.gradle index cef4d21f1e8f..692d58148fa6 100644 --- a/modules/ExpensifyNitroUtils/android/build.gradle +++ b/modules/ExpensifyNitroUtils/android/build.gradle @@ -34,6 +34,17 @@ def getExtOrIntegerDefault(name) { return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["ExpensifyNitroUtils_" + name]).toInteger() } +def pgoMode = rootProject.findProperty('pgoMode') ?: 'off' +def pgoProfile = rootProject.findProperty('pgoProfile') + +if (!(pgoMode in ['off', 'generate', 'use'])) { + throw new GradleException("Unsupported pgoMode '${pgoMode}'. Use off, generate, or use.") +} + +if (pgoMode == 'use' && !pgoProfile) { + throw new GradleException('pgoProfile is required when pgoMode=use.') +} + android { namespace "com.margelo.nitro.utils" ndkVersion getExtOrDefault("ndkVersion") @@ -47,7 +58,10 @@ android { externalNativeBuild { cmake { cppFlags "-O2 -frtti -fexceptions -Wall -fstack-protector-all" - arguments "-DANDROID_STL=c++_shared" + arguments "-DANDROID_STL=c++_shared", "-DEXPENSIFY_PGO_MODE=${pgoMode}" + if (pgoMode == 'use') { + arguments "-DEXPENSIFY_PGO_PROFILE=${file(pgoProfile).absolutePath}" + } abiFilters (*reactNativeArchitectures()) } } @@ -126,4 +140,3 @@ dependencies { // Add a dependency on NitroModules implementation project(":react-native-nitro-modules") } - diff --git a/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp new file mode 100644 index 000000000000..492d68e1f09e --- /dev/null +++ b/modules/ExpensifyNitroUtils/android/src/main/cpp/PgoProfileWriter.cpp @@ -0,0 +1,63 @@ +#include +#include + +#include +#include + +namespace { +using SetFilename = void (*)(const char *); +using WriteFile = int (*)(); + +constexpr std::array kInstrumentedLibraries = { + "libhermesvm.so", + "libreactnative.so", + "libjsi.so", + "libExpensifyNitroUtils.so", +}; + +bool writeProfile(void *handle, const std::string &profilePath) { + const auto setFilename = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_set_filename")); + const auto writeFile = reinterpret_cast(dlsym(handle, "expensify_llvm_profile_write_file")); + if (setFilename == nullptr || writeFile == nullptr) { + return false; + } + + setFilename(profilePath.c_str()); + return writeFile() == 0; +} + +jint writeProfiles(JNIEnv *env, jstring directory) { + const char *directoryChars = env->GetStringUTFChars(directory, nullptr); + if (directoryChars == nullptr) { + return 0; + } + + const std::string profilePath = std::string(directoryChars) + "/newdot-%m.profraw"; + env->ReleaseStringUTFChars(directory, directoryChars); + + jint writtenProfiles = 0; + for (const char *library : kInstrumentedLibraries) { + void *handle = dlopen(library, RTLD_NOW | RTLD_NOLOAD); + if (handle == nullptr) { + continue; + } + + if (writeProfile(handle, profilePath)) { + ++writtenProfiles; + } + dlclose(handle); + } + + return writtenProfiles; +} +} // namespace + +extern "C" JNIEXPORT jint JNICALL +Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { + return writeProfiles(env, directory); +} + +extern "C" JNIEXPORT jint JNICALL +Java_com_expensify_chat_PgoProfileWriter_writeProfiles(JNIEnv *env, jclass, jstring directory) { + return writeProfiles(env, directory); +} diff --git a/package-lock.json b/package-lock.json index dbbc5af3fb6b..2a5b8526b109 100644 --- a/package-lock.json +++ b/package-lock.json @@ -193,6 +193,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/package.json b/package.json index b2d70deac898..ce674e72c060 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build-staging": "ENV=staging bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "build-adhoc": "ENV=adhoc bun ./node_modules/.bin/rsbuild build --config config/rsbuild/rsbuild.config.ts && bun ./scripts/combine-web-sourcemaps.ts", "benchmark-app-startup": "bun scripts/benchmarkAppStartup.ts", + "pgo:journey": "bun scripts/pgo/journey.ts", "bootstrap-device": "bun scripts/bootstrapForDevice.ts", "createDocsRoutes": "bun .github/scripts/createDocsRoutes.ts", "generateAllowedUrls": "bun .github/scripts/generateAllowedUrls.ts", @@ -269,6 +270,7 @@ "@rock-js/plugin-metro": "0.13.4", "@rock-js/plugin-repack": "0.13.4", "@rock-js/provider-s3": "0.13.4", + "@rock-js/tools": "0.13.4", "@rsbuild/core": "^2.1.4", "@rsbuild/plugin-svgr": "^2.0.5", "@rsdoctor/rspack-plugin": "^1.5.17", diff --git a/patches/react-native/details.md b/patches/react-native/details.md index 0d83081b6d09..2b18be31ad06 100644 --- a/patches/react-native/details.md +++ b/patches/react-native/details.md @@ -1,5 +1,12 @@ # `react-native` patches +### [react-native+0.86.0+044+add-local-pgo-build-switch.patch](react-native+0.86.0+044+add-local-pgo-build-switch.patch) + +- Reason: Adds an opt-in local LLVM PGO mode to the React Native Android runtime and Hermes source builds. The mode is disabled by default and is used only by the local PGO tooling. +- Upstream PR/issue: 🛑 +- E/App issue: 🛑 +- PR Introducing Patch: 🛑 + ### [react-native+0.86.0+001+initial.patch](react-native+0.86.0+001+initial.patch) - Reason: Fixes keyboard flickering issue when opening/closing modals. When an input is blurred and a modal is opened, the `rootView` becomes the `firstResponder`, causing the system to retain an incorrect keyboard state. This leads to keyboard flickering when the modal is closed. The patch adds code to resign the `rootView`'s `firstResponder` status before presenting the modal to prevent this issue. diff --git a/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch new file mode 100644 index 000000000000..d7e9802f6fd1 --- /dev/null +++ b/patches/react-native/react-native+0.86.0+044+add-local-pgo-build-switch.patch @@ -0,0 +1,184 @@ +diff --git a/node_modules/react-native/ReactAndroid/build.gradle.kts b/node_modules/react-native/ReactAndroid/build.gradle.kts +index bae034d..94d11e1 100644 +--- a/node_modules/react-native/ReactAndroid/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/build.gradle.kts +@@ -42,6 +42,16 @@ val reactNativeRootDir = projectDir.parent + val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() + ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} ++ + // We put the publishing version from gradle.properties inside ext. so other + // subprojects can access it as well. + extra["publishing_version"] = project.findProperty("VERSION_NAME")?.toString()!! +@@ -579,8 +589,13 @@ android { + "-DANDROID_TOOLCHAIN=clang", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW", ++ "-DPGO_MODE=$pgoMode", + ) + ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } ++ + if (hermesV1Enabled) { + arguments("-DHERMES_V1_ENABLED=1") + } +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +index 6610782..853b937 100644 +--- a/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/build.gradle.kts +@@ -52,6 +52,18 @@ fun getSDKManagerPath(): String { + + val hermesV1Enabled = + rootProject.extensions.getByType(PrivateReactExtension::class.java).hermesV1Enabled.get() ++ ++val pgoMode = project.findProperty("pgoMode")?.toString() ?: "off" ++val pgoProfile = project.findProperty("pgoProfile")?.toString() ++val pgoCmakePath = File(projectDir, "pgo.cmake") ++ ++check(pgoMode in setOf("off", "generate", "use")) { ++ "Unsupported pgoMode '$pgoMode'. Use off, generate, or use." ++} ++if (pgoMode == "use") { ++ check(!pgoProfile.isNullOrBlank()) { "pgoProfile is required when pgoMode=use." } ++} ++ + val reactNativeRootDir = project(":packages:react-native:ReactAndroid").projectDir.parent + val customDownloadDir = System.getenv("REACT_NATIVE_DOWNLOADS_DIR") + val downloadsDir = +@@ -415,8 +427,14 @@ android { + // We intentionally build Hermes with Intl support only. This is to simplify + // the build setup and to avoid overcomplicating the build-type matrix. + "-DHERMES_ENABLE_INTL=True", ++ "-DPGO_MODE=$pgoMode", ++ "-DCMAKE_PROJECT_Hermes_INCLUDE=${pgoCmakePath.absolutePath}", + ) + ++ if (pgoMode == "use") { ++ arguments("-DPGO_PROFILE=$pgoProfile") ++ } ++ + if (hermesV1Enabled) { + arguments("-DHERMESVM_HEAP_HV_MODE=HEAP_HV_PREFER32") + } +diff --git a/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +new file mode 100644 +index 0000000..bcc335d +--- /dev/null ++++ b/node_modules/react-native/ReactAndroid/hermes-engine/pgo.cmake +@@ -0,0 +1,25 @@ ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() +diff --git a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +index 8125e02..d466798 100644 +--- a/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt ++++ b/node_modules/react-native/ReactAndroid/src/main/jni/CMakeLists.txt +@@ -8,6 +8,32 @@ set(CMAKE_VERBOSE_MAKEFILE on) + + project(ReactAndroid) + ++if(PGO_MODE STREQUAL "generate") ++ execute_process( ++ COMMAND ${CMAKE_CXX_COMPILER} --print-resource-dir ++ OUTPUT_VARIABLE CLANG_RESOURCE_DIR ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ set(PGO_RUNTIME "${CLANG_RESOURCE_DIR}/lib/linux/libclang_rt.profile-aarch64-android.a") ++ if(NOT EXISTS "${PGO_RUNTIME}") ++ message(FATAL_ERROR "LLVM profiling runtime was not found at ${PGO_RUNTIME}") ++ endif() ++ add_compile_options(-fprofile-generate) ++ add_link_options( ++ -fprofile-generate ++ "-Wl,--whole-archive,${PGO_RUNTIME},--no-whole-archive" ++ -Wl,--undefined=__llvm_profile_set_filename ++ -Wl,--undefined=__llvm_profile_write_file ++ -Wl,--defsym=expensify_llvm_profile_set_filename=__llvm_profile_set_filename ++ -Wl,--defsym=expensify_llvm_profile_write_file=__llvm_profile_write_file ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_set_filename ++ -Wl,--export-dynamic-symbol=expensify_llvm_profile_write_file ++ ) ++elseif(PGO_MODE STREQUAL "use") ++ add_compile_options("-fprofile-use=${PGO_PROFILE}") ++ add_link_options("-fprofile-use=${PGO_PROFILE}") ++endif() ++ + include(CheckIPOSupported) + check_ipo_supported(RESULT IPO_SUPPORT) + if (IPO_SUPPORT) +diff --git a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +--- a/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh ++++ b/node_modules/react-native/sdks/hermes-engine/utils/build-hermes-xcode.sh +@@ -59,6 +59,19 @@ if [[ $xcode_major_version -ge 15 ]]; then + xcode_15_flags="LINKER:-ld_classic" + fi + ++pgo_compile_flags="" ++pgo_linker_flags="" ++if [[ "${EXPENSIFY_PGO_MODE:-off}" == "generate" ]]; then ++ pgo_compile_flags="-fprofile-generate" ++ pgo_linker_flags="-fprofile-generate;LINKER:-alias,___llvm_profile_set_filename,_expensify_llvm_profile_set_filename;LINKER:-alias,___llvm_profile_write_file,_expensify_llvm_profile_write_file;LINKER:-alias,___llvm_profile_reset_counters,_expensify_llvm_profile_reset_counters" ++elif [[ "${EXPENSIFY_PGO_MODE:-off}" == "use" ]]; then ++ if [[ -z "${EXPENSIFY_PGO_PROFILE:-}" ]]; then ++ echo "EXPENSIFY_PGO_PROFILE is required when EXPENSIFY_PGO_MODE=use" >&2 ++ exit 1 ++ fi ++ pgo_compile_flags="-fprofile-use=${EXPENSIFY_PGO_PROFILE}" ++fi ++ + boost_context_flag="" + if [[ $PLATFORM_NAME == "catalyst" ]]; then + boost_context_flag="-DHERMES_ALLOW_BOOST_CONTEXT=0" +@@ -70,7 +84,7 @@ echo "Configure Apple framework" + "$CMAKE_BINARY" \ + -S "${PODS_ROOT}/hermes-engine" \ + -B "${PODS_ROOT}/hermes-engine/build/${PLATFORM_NAME}" \ +- -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags" \ ++ -DHERMES_EXTRA_LINKER_FLAGS="$xcode_15_flags;$pgo_linker_flags" \ + -DHERMES_APPLE_TARGET_PLATFORM:STRING="$PLATFORM_NAME" \ + -DCMAKE_OSX_ARCHITECTURES:STRING="$architectures" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING="$deployment_target" \ +@@ -82,8 +96,8 @@ echo "Configure Apple framework" + -DHERMES_ENABLE_BITCODE:BOOLEAN=false \ + -DHERMES_BUILD_APPLE_FRAMEWORK:BOOLEAN=true \ + -DHERMES_BUILD_SHARED_JSI:BOOLEAN=false \ +- -DCMAKE_CXX_FLAGS:STRING="-gdwarf" \ +- -DCMAKE_C_FLAGS:STRING="-gdwarf" \ ++ -DCMAKE_CXX_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ ++ -DCMAKE_C_FLAGS:STRING="-gdwarf $pgo_compile_flags" \ + -DIMPORT_HOST_COMPILERS:PATH="${hermesc_path}" \ + -DJSI_DIR="$jsi_path" \ + -DHERMES_RELEASE_VERSION="for RN $release_version" \ diff --git a/scripts/lib/nativeAppBenchmark.ts b/scripts/lib/nativeAppBenchmark.ts index bce54b015663..434fc2c038fe 100644 --- a/scripts/lib/nativeAppBenchmark.ts +++ b/scripts/lib/nativeAppBenchmark.ts @@ -6,6 +6,7 @@ import {assertAndroidAppInstalled, createAndroidAdapter, parseAndroidProcessIden import { createIOSAdapter, iOSBenchmarkMarkerPath, + iOSProcessListArguments, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, @@ -27,6 +28,7 @@ export { createNativeAppBenchmarkAdapter, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseBenchmarkLogEvents, diff --git a/scripts/lib/nativeAppBenchmark/ios.ts b/scripts/lib/nativeAppBenchmark/ios.ts index fa82833d1719..d5a82d60b543 100644 --- a/scripts/lib/nativeAppBenchmark/ios.ts +++ b/scripts/lib/nativeAppBenchmark/ios.ts @@ -1,4 +1,4 @@ -// cspell:ignore BEGINSWITH devicectl +// cspell:ignore devicectl /** Implements iOS benchmark setup, launch, process control, and marker collection through CoreDevice. */ @@ -67,20 +67,7 @@ async function createIOSAdapter({rootDirectory, deviceIdentifier, appID}: Omit 0 ? processIdentifier : undefined; } +/** Lists every process so client-side matching works across CoreDevice versions with different filter field names. */ +function iOSProcessListArguments(deviceIdentifier: string, outputPath: string): string[] { + return ['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']; +} + /** Maps a span name to its encoded marker path inside the iOS app data container. */ function iOSBenchmarkMarkerPath(spanName: string): string { return `${IOS_BENCHMARK_DIRECTORY}/${encodeURIComponent(spanName)}.log`; @@ -283,5 +275,13 @@ function isCoreDeviceInstalledApp(value: JsonValue): value is CoreDeviceInstalle return isJSONObject(value) && typeof value.bundleIdentifier === 'string' && typeof value.url === 'string'; } -export {createIOSAdapter, iOSBenchmarkMarkerPath, parseIOSInstalledAppURL, parseIOSInstalledAppsResponse, parseIOSLaunchProcessIdentifier, parseIOSRunningAppProcessIdentifier}; +export { + createIOSAdapter, + iOSBenchmarkMarkerPath, + iOSProcessListArguments, + parseIOSInstalledAppURL, + parseIOSInstalledAppsResponse, + parseIOSLaunchProcessIdentifier, + parseIOSRunningAppProcessIdentifier, +}; export type {CoreDeviceInstalledAppsResponse}; diff --git a/scripts/lib/scriptUtils.ts b/scripts/lib/scriptUtils.ts new file mode 100644 index 000000000000..9a665b430b86 --- /dev/null +++ b/scripts/lib/scriptUtils.ts @@ -0,0 +1,104 @@ +/** Reusable helpers for repository tooling scripts. */ + +import {spawnSync} from 'node:child_process'; +import {existsSync, readdirSync} from 'node:fs'; +import {join, resolve} from 'node:path'; +import process from 'node:process'; + +const rootDirectory = resolve(import.meta.dirname, '../..'); + +function fail(message: string): never { + throw new Error(message); +} + +function run(command: string, args: string[], cwd = rootDirectory): void { + const result = spawnSync(command, args, {cwd, stdio: 'inherit'}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } +} + +function capture(command: string, args: string[], cwd = rootDirectory): string { + const result = spawnSync(command, args, {cwd, encoding: 'utf8', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + const stderr = result.stderr.trim(); + fail(stderr.length > 0 ? stderr : `${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function captureBinary(command: string, args: string[], cwd = rootDirectory): Buffer { + const result = spawnSync(command, args, {cwd, encoding: 'buffer', maxBuffer: 100 * 1024 * 1024}); + if (result.error) { + fail(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + fail(`${command} exited with status ${result.status ?? 'unknown'}.`); + } + return result.stdout; +} + +function runAllowFailure(command: string, args: string[], cwd = rootDirectory): boolean { + const result = spawnSync(command, args, {cwd, stdio: 'ignore'}); + return !result.error && result.status === 0; +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolvePromise) => { + setTimeout(resolvePromise, milliseconds); + }); +} + +function findFiles(directory: string, extension: string): string[] { + if (!existsSync(directory)) { + return []; + } + + return readdirSync(directory, {withFileTypes: true}).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return findFiles(path, extension); + } + return entry.isFile() && entry.name.endsWith(extension) ? [path] : []; + }); +} + +function requirePositiveInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive integer, received: ${value}`); + } + return value; +} + +function parsePositiveInteger(rawValue: string, label: string): number { + return requirePositiveInteger(Number(rawValue), label); +} + +function parseChoice(rawValue: string, choices: readonly T[], label: string): T { + const choice = choices.find((candidate) => candidate === rawValue); + if (!choice) { + fail(`${label} must be one of: ${choices.join(', ')}. Received: ${rawValue}`); + } + return choice; +} + +function environmentString(name: string): string | undefined { + const value: unknown = process.env[name]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function valueAt(values: readonly T[], index: number): T { + const value = values.at(index); + if (value === undefined) { + fail(`Expected a value at index ${index}.`); + } + return value; +} + +export {capture, captureBinary, environmentString, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run, runAllowFailure, sleep, valueAt}; diff --git a/scripts/pgo/JOURNEY.md b/scripts/pgo/JOURNEY.md new file mode 100644 index 000000000000..c29d117fc601 --- /dev/null +++ b/scripts/pgo/JOURNEY.md @@ -0,0 +1,103 @@ +# Automated native PGO journey + + + +`journey.ts` drives the installed app through its accessibility tree using `agent-device` (developed with version 0.20.6). It runs without an LLM. The same TypeScript workload accepts an Android serial or iOS UDID. It uses labels and test IDs, and sizes scroll gestures from the device's reported bounds. The app must use English and portrait orientation. The full journey, including message sending and native profile export, has been validated on a physical Samsung phone and iPhone 13 Pro. + +## Account and device setup + +Use a dedicated, owner-approved heavy account with a populated Inbox, a long chat history, and enough expenses and expense reports to scroll. Keep the account and its data between runs. Installing over the existing app preserves authentication; uninstalling or clearing app data does not. + +The runner never enters credentials, retrieves a security code, or switches accounts. It checks the account email in Account and exits with `SIGN_IN_REQUIRED` (exit code 2) if sign-in is needed. Stop and ask Chris to sign in manually before continuing. A CI job must surface this failure for manual intervention instead of substituting another account. + +Before every run, it opens Inbox to allow the app's automatic large-account preference update, then opens Account → Preferences and ensures Priority mode is **Most recent**. If it is **#focus**, the runner changes it and verifies the result. It checks the preference again at the end and rejects the run if #focus became enabled. + +Install `agent-device` on the runner and pin its version. Android needs USB debugging and a connected, authorized device. The runner temporarily enables the tool's test keyboard for reliable text replacement and restores the previous keyboard when its session closes. iOS requires macOS, Xcode, a trusted device with Developer Mode enabled, and signing for the tool's XCTest runner. The existing iOS profile collector supports physical devices; simulator profiles are not inputs to a device build. + +Create an ignored fixture, for example `.pgo/journey-fixture.json`: + +```json +{ + "accountEmail": "heavy@example.com", + "accountClass": "heavy", + "description": "Owner-approved high-traffic test account with populated chats, expenses, and reports.", + "report": { + "query": "busy", + "resultLabelPrefix": "#busy, Test workspace", + "title": "#busy" + }, + "personalChat": { + "query": "heavy@example.com", + "resultLabelPrefix": "Test User (you), heavy@example.com", + "title": "Test User (you)" + }, + "allowMessages": false, + "scrolls": 8, + "tabCycles": 3 +} +``` + +Choose the report name and workspace from the actual account. Search matching permits a changing message preview after the configured identity, and rejects ambiguous results. The personal-chat query must be the exact approved account email. That result can show either the email or the latest message beneath its `(you)` title, so both forms are supported for repeated runs. Confirm the personal chat with Chris before setting `allowMessages` to `true`. Messages are restricted to that account's `(you)` chat; each run sends two uniquely marked test messages and leaves them in place. An existing composer draft causes a failure. + +## Validate and record + +First check the account and #focus setting without sending messages: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --preflight +``` + +Use `--navigation-only` instead of `--preflight` to exercise all navigation and scrolling without sending messages. This mode cannot collect a training profile. Omit both flags for the complete journey once message permission is recorded in the fixture. + +The complete journey relaunches the app, scrolls Inbox in both directions, opens and scrolls the selected report, composes two messages in the personal chat, scrolls Spend expenses and reports, switches between Inbox, Spend, and Workspaces three times, and reopens the report. Android also edits each draft before sending; iOS enters the final text once because replacing non-empty text in its composer was unreliable during live validation. The scroll count is a maximum per direction: two unchanged snapshots stop that direction at the list boundary. Scrolling must expose changing content; an empty or stationary list fails the run. + +After installing and verifying an instrumented build with the existing PGO commands, collect three independent repetitions: + +```bash +scripts/pgo/journey.ts android --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +scripts/pgo/journey.ts ios --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json --runs 3 --collect +``` + +Start with three repetitions per platform. Use the recorded duration and resulting native counter coverage to adjust the workload; extra repetitions of the same data do not add new paths. Most activity here exercises Hermes, Fabric mounting, layout, text, native input, and scrolling. This trains the compiled native implementation used by JavaScript; it does not compile the application's JavaScript with LLVM. Avoid spending most of the run waiting on network responses or looping over trivial settings screens. + +On September 23, 2026, the Android navigation-only run completed in 6 minutes 7 seconds. Two complete instrumented recordings passed in 7 minutes 1 second and 6 minutes 55 seconds, excluding preflight. Each sent two messages and exported four raw profiles; both recordings merged successfully, separately and together. These timings include automation overhead and the final account/preference check; they are not app performance measurements. The first recording's four profiles were also checked individually for nonzero native counters. + +On the same day, a complete instrumented iOS validation recording passed on a connected iPhone 13 Pro in 6 minutes 35 seconds, excluding preflight. A subsequent three-repetition training batch passed in 6 minutes 41 seconds, 6 minutes 35 seconds, and 6 minutes 37 seconds per journey. The batch exported six raw profiles and produced a 15 MB merged profile containing 77,590 functions. Every repetition sent two approved messages to the account's personal chat, then confirmed the heavy account and #focus off. These timings include automation overhead; they are not app performance measurements. + +The collector checks that the installed app can flush profiles, discards setup output, clears previous device profiles, and starts a fresh process for each repetition. It flushes once after a successful journey and verification, then archives that run's raw files. It merges only after every requested repetition succeeds. Failed batches produce no final merged profile. A failed send is never retried automatically. + +Each batch lives under `.pgo//journeys//` and includes `result.json`, per-run raw profiles, `journey.profdata`, and a function summary. The result records the fixture, app and device identifiers, both repository revisions, automation version, completed runs, and durations. These artifacts contain account identifiers and must stay in restricted CI storage. + +The journey profile is kept separate from the startup profile. The existing optimized-build command reads `newdot.profdata` from its platform directory: CI must explicitly select or weight the successful scenario profiles, write that final input, and preserve the iOS `.format` marker. Do not silently replace the startup input with an incomplete navigation run. + +## Release pipeline integration + +Use an exclusive device lease on a persistent runner: Linux or macOS with an arm64 Android device, and macOS with an arm64 iPhone. Build the instrumented artifact from the release candidate, install it over the signed-in app, run preflight and the full collection, then build the optimized artifact from the same source and toolchain. Gate publication on a separate performance and correctness check. The instrumented artifact is a CI input, not the store upload. + +Run these steps for each staging or production release candidate before publication. Pin the compiler, SDK, dependencies, architecture, build settings, and automation version; retain them alongside the profile. The current runner records source revisions but does not validate binary provenance. The release pipeline must enforce that the training artifact and optimized build match, and keep simultaneous jobs from sharing a device or profile output directory. Store signing credentials separately from the fixture. + +This change supplies the journey and profile collection hook for local and CI execution. Connecting it to the store release workflows and measuring held-out interactive performance remain separate integration steps. + +## Held-out interaction benchmark + +The PGO workflow has a shorter, read-only benchmark that is separate from the six-minute profile-collection journey. Each sample relaunches the signed-in app, selects Spend → Expenses, returns to Inbox, then times a warm Inbox → Spend → Inbox switch. The app's existing first-paint, content-load, and Inbox navigation spans measure the UI work. One complete warm-up is discarded for each build, followed by the requested number of fresh-process samples. The benchmark checks the same approved account and #focus preference, but sends no messages. + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Build both the release and optimized artifacts with the current span allowlist first. `build-optimized --profile /absolute/path/to/merged.profdata` selects an immutable archived profile; without `--profile`, it uses `newdot.profdata`. The output uses the existing startup benchmark's raw-sample and summary CSV format in a unique `.pgo//benchmarks/journey//` directory. The two phases run together, so a failed or repeated phase cannot silently pair with stale data. Compare each interactive span separately; adding their durations together does not give a full journey time because there is automation time between taps. Keep this benchmark out of the profile-training batch, and avoid using its result as a release gate until build-order drift and device temperature have been characterized. + +## Heavy report benchmark + +Use a separate read-only benchmark for a populated report. Each sample starts a fresh signed-in process, finds the fixture's report through Search, measures the app-defined `/r/*` span from report navigation to the report list's first layout, then makes four fast swipes toward older messages and four back toward recent messages. It checks that content changes after each direction; there is no accessibility snapshot between individual flings. The search, device commands, settling, and swipes are **not** included in the report-open latency sample. The span can end when a skeleton is laid out, so treat it as first report render, not complete message or network loading. + +```bash +scripts/pgo/pgo.ts android benchmark-heavy-journey 20 30 --device DEVICE_SERIAL --app-id APP_ID --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-heavy-journey 20 30 --device DEVICE_UDID --app-id BUNDLE_ID --fixture .pgo/journey-fixture.json +``` + +Rebuild both app variants after adding the `/r/*` span to the build-time benchmark allowlist. The earlier artifacts cannot emit this metric. Each batch goes to `.pgo//benchmarks/heavy-journey//` with raw and summary CSVs, artifact hashes, and a run manifest. On Android, an additional frame CSV counts rendered and missed-deadline frames during the fast-scroll window. Compare its missed-frame share separately from report-open latency; it is not elapsed scroll time. The current physical-iPhone frame probe could not consistently attach Xcode's Animation Hitches trace to the app process, so the iOS runner validates scroll movement but deliberately reports no scroll-frame score. A reliable iOS scrolling claim needs an XCTest performance test with a signposted scroll interval or another native frame trace that works on the CI device. + +Neither the six-minute collector nor this benchmark's total execution time measures app speed. Android frame counts still include any app rendering triggered during the capture window, and the report-open span can be affected by cached data, network, and device load. Keep the account dataset and device state stable, and warm both variants the same way. Run at least two batches with opposite phase orders by passing `--benchmark-order optimized-first` to the second batch; the default is `release-first`. Report distributions and confidence intervals, and only claim a benefit for the specified device and workload when the improvement exceeds run-to-run variation. No PGO profile can guarantee every launch or interaction will be faster. diff --git a/scripts/pgo/README.md b/scripts/pgo/README.md new file mode 100644 index 000000000000..868d86a0aa0e --- /dev/null +++ b/scripts/pgo/README.md @@ -0,0 +1,173 @@ +# Native profile-guided optimization + + + +This directory contains the local LLVM PGO workflow for Android and iOS. The `pgo.ts` command builds release, instrumented, and profile-optimized apps, retrieves native profiles, merges them, and compares startup performance. On Android, the current flags cover source-built React Native, Hermes, and ExpensifyNitroUtils libraries. On iOS, the workflow forces React Native and Hermes source builds and instruments the app and source-based CocoaPods targets. Precompiled frameworks do not participate. + +The command enables `ManualAppStartup` and the existing Spend/Inbox navigation spans in every app it builds. Startup profile collection waits for `ManualAppStartup` to finish, then explicitly writes the LLVM counters before the app process is stopped. Both benchmark modes use app-defined span durations rather than device-automation wall time. + +## Prepare local release identifiers + +Bootstrap the release projects once so the apps can be installed beside other Expensify builds: + +```bash +scripts/bootstrapForDevice.ts --build-variants release --identifier-suffix pgo +``` + +For Christoph Pader's local identifiers, this produces: + +| Platform | Identifier | +| --- | --- | +| Android | `com.chrispader.expensify.pgo` | +| iOS | `com.chrispader.expensify.expensifylite.pgo` | + +The PGO command reads the Android release application ID from `Mobile-Expensify/Android/build.gradle` and the iOS bundle identifier from the archived app. Pass `--app-id` to override either value. + +## Preconditions + +- Prefer physical arm64 Android and iOS devices for final profile collection and measurements. An Android emulator running the same ABI and instrumented native build can provide useful counts, but emulator-only behavior can bias the workload. An iOS simulator builds for a different platform, so do not reuse its native profiles for a device build. +- Android requires the NDK version pinned in `Mobile-Expensify/Android/build.gradle`. Set `ANDROID_NDK_HOME` if it is not installed in the default SDK directory. +- iOS requires Xcode command-line tools, an unlocked device, and valid local Apple Development signing for the app and its extensions. Select a team with `IOS_DEVELOPMENT_TEAM` when automatic discovery is insufficient. +- Install dependencies and apply the repository patches before building. +- Seed a test account and data set before recording an interactive profile. + +Every command starts with a platform and a workflow: + +```bash +scripts/pgo/pgo.ts android --help +scripts/pgo/pgo.ts ios --help +``` + +## Collect a startup profile + +Build, verify, and install the instrumented app: + +```bash +scripts/pgo/pgo.ts android build-instrumented +scripts/pgo/pgo.ts android verify-instrumented +scripts/pgo/pgo.ts android install-instrumented +``` + +Record ten cold-process startups, retrieve the raw profiles, and merge them: + +```bash +scripts/pgo/pgo.ts android record-startups +``` + +The optional positional arguments set the run count and span timeout: + +```bash +scripts/pgo/pgo.ts android record-startups 20 45 +``` + +The command removes only old `newdot-*.profraw` files before collection. It requires `ManualAppStartup` to complete on every run and flushes the native counters after each successful run. A missing span fails the collection instead of adding a partial startup. The merged profile is written to `.pgo/android/arm64-v8a/newdot.profdata` or `.pgo/ios/arm64/newdot.profdata`. + +The iOS workflow uses the same commands: + +```bash +scripts/pgo/pgo.ts ios build-instrumented +scripts/pgo/pgo.ts ios verify-instrumented +scripts/pgo/pgo.ts ios install-instrumented +scripts/pgo/pgo.ts ios record-startups +``` + +The app must remain in the foreground until each profile write completes. Force-terminating an iOS app is not a reliable profile flush. + +## Collect an interactive profile + +For the automated heavy-account workload, see [Automated native PGO journey](./JOURNEY.md). It checks the signed-in account, disables #focus, drives the critical user flows, and can archive and merge profiles from each successful run. + +For a broader profile, install the instrumented app, perform the agreed journey, and write the counters while the app is still running: + +```bash +scripts/pgo/pgo.ts android dump +scripts/pgo/pgo.ts android pull +scripts/pgo/pgo.ts android merge +``` + +Use the equivalent `ios` commands on iOS. A representative journey should exercise the common signed-in path: open and scroll chats, send a message, visit a workspace setting, attach and view a file, and run a search. Keep account state, data size, and network conditions stable between collections. + +Write the counters once per app process. Repeated dumps without resetting counters can merge the same accumulated work again. Independent scenarios need a fresh process, cleared device profile files, and separately archived output. The startup recorder and journey runner each handle that lifecycle; CI must still choose and combine the scenario profiles. + +## Build and benchmark the optimized app + +Build the optimized app only after recording a fresh profile. Build the baseline release from the same source revision and toolchain: + +```bash +scripts/pgo/pgo.ts android build-optimized +scripts/pgo/pgo.ts android build-release +scripts/pgo/pgo.ts android benchmark 20 45 +``` + +Pass `--profile /absolute/path/to/merged.profdata` to `build-optimized` when comparing an archived profile. The path becomes part of the compiler input; keep the file immutable for the build. Without it, the command uses the platform's `newdot.profdata`. Local PGO builds disable Sentry uploads. iOS temporarily skips the FullStory and Sentry upload phases and restores the original Xcode project afterward. Android excludes `fullstoryRelease` and archives the pre-FullStory APK, so its local benchmarks are internally paired but do not represent the fully post-processed production APK. Do not compare their absolute timings directly with earlier or store builds that used FullStory processing. + +The benchmark installs the archived baseline, runs one warm-up and the requested `ManualAppStartup` samples, then repeats the process with the optimized app. Both builds use the same application identifier, so installing the second artifact preserves the seeded account and data. Results are stored under `.pgo//benchmarks/` in the repository benchmark CSV format. + +For a separate, read-only interaction benchmark, use the approved signed-in account fixture described in [JOURNEY.md](./JOURNEY.md): + +```bash +scripts/pgo/pgo.ts android benchmark-journey 20 30 --device DEVICE_SERIAL --app-id com.chrispader.expensify.pgo --fixture .pgo/journey-fixture.json +scripts/pgo/pgo.ts ios benchmark-journey 20 30 --device DEVICE_UDID --app-id com.chrispader.expensify.expensifylite.pgo --fixture .pgo/journey-fixture.json +``` + +Each variant gets one discarded warm-up and 20 measured fresh-process journeys. Before each sample, the runner selects Spend → Expenses and returns to Inbox; it then times a warm Inbox → Spend → Inbox tab switch. Expenses can restore the account's saved query and filters, so keep that state fixed across both phases. The CSVs under a unique `.pgo//benchmarks/journey//` directory contain `ManualNavigateToReportsFirstPaint`, `ManualNavigateToReportsContentLoad`, and `ManualNavigateToInboxTab` samples, alongside a run manifest with artifact hashes. These are app-defined tap-to-paint timings, so automation overhead is outside the measurement. The journey sends no messages and leaves account data unchanged. Both phases run in one command and are compared only within that batch. Rebuild both archived variants with this version of the tooling before benchmarking; older PGO builds only logged startup. + +For a populated-chat, fast-scroll benchmark, use `benchmark-heavy-journey` with the same arguments. It records the app's report-open span on both platforms and Android's rendered/missed scroll frames in a separate CSV. iOS scroll-frame measurement is not currently reliable on the connected phone; the runner verifies that scrolling moves through real content but does not report a scroll performance number. See [JOURNEY.md](./JOURNEY.md) for the exact measured interval, setup, and limitations. Both variants must be rebuilt to include the additional report-open marker. + +The stages can also run independently: + +```bash +scripts/pgo/pgo.ts android benchmark-release 20 45 +scripts/pgo/pgo.ts android benchmark-optimized 20 45 +scripts/pgo/pgo.ts android compare-benchmarks +``` + +Select a device or override an identifier when discovery is ambiguous: + +```bash +scripts/pgo/pgo.ts android benchmark 20 45 --device DEVICE_SERIAL --app-id com.chrispader.expensify.pgo +scripts/pgo/pgo.ts ios benchmark 20 45 --device Chris14Pro --app-id com.chrispader.expensify.expensifylite.pgo +``` + +Release, instrumented, and optimized artifacts are archived under `.pgo/android/arm64-v8a/apk/` and `.pgo/ios/arm64/app/`. Do not apply a profile to another architecture, compiler, source revision, dependency graph, or build configuration. + +## What to collect + +A startup-only profile is useful, but it tends to overfit initialization and can make post-startup code colder. For startup and overall app performance, use a mixed profile with measured weights: + +1. Record a fixed startup suite across the important states, such as a signed-out launch, a signed-in inbox launch, and a launch with a realistically large local data set. +2. Record a small set of common interactive journeys. Prefer bounded flows with deterministic seeded data and an explicit success signal. +3. Keep the raw output for each scenario separate. A longer scenario generates more counter volume than a short one, so equal repetition does not give scenarios equal influence. +4. Keep a separate benchmark suite that does not contribute to the training profile. This catches overfitting. + +The current `record-startups` command accumulates and merges the startup runs automatically. For a mixed CI profile, preserve each scenario's output independently, convert it to its own indexed profile, then combine the scenario profiles with `llvm-profdata merge --weighted-input`. Choose weights from measured production flow frequency and performance impact, not from a default startup-to-interaction ratio. A giant tour of every feature is less useful than a stable suite that reflects real traffic. + +## A realistic CI design + +Run profile generation as a scheduled or release-candidate job, not on every pull request. Use pinned self-hosted runners or a device farm, preferably with physical arm64 devices, the production compiler versions, stable thermal and power conditions, a seeded account, and deterministic local fixtures where possible. Build one instrumented artifact per platform, run the fixed startup and interaction suites against that exact artifact, flush after every successful scenario, merge the separate outputs with documented weights, then build the optimized artifact without changing the checkout or toolchain. + +Store both repository revisions, dependency locks, compiler and SDK versions, ABI, build settings, scenario version, and test-data fixture with each profile. Reject mismatched artifacts before the optimized build. Keep the instrumented build and raw profiles so a failed collection can be investigated without rerunning it. + +Benchmark both variants on the same device after a cooldown, discard post-install warm-ups, and retain raw samples. The `benchmark` workflow runs all baseline samples before all optimized samples, so thermal or time-dependent drift can bias the comparison. The two archived builds intentionally share an identifier to preserve app data, which prevents side-by-side alternation. CI can reduce drift with ABBA install blocks that preserve the shared identifier and discard each post-install warm-up. Another option is to give the builds distinct identifiers, seed matched state, and use the repository's alternating startup benchmark. Evaluate both startup and held-out interactive journeys that were excluded from training. Gate on several releases of data rather than a single noisy run. + +Hosted CI is usually insufficient for the complete workflow because hosted runners rarely expose stable physical devices. Android emulators can validate automation and may contribute useful same-ABI counts when the workload is representative, but final validation should use target devices. iOS device collection also needs signing and physical-device access. A small self-hosted device pool or managed device farm is the realistic route to automation. + +## Limitations + +- LLVM PGO improves compiled native code. It can optimize a source-built Hermes engine, but it does not directly reorder JavaScript or replace JavaScript startup analysis. +- Precompiled vendored frameworks cannot be instrumented after the fact. +- The iOS flush currently exports the app executable's profile runtime and Hermes's separate runtime. Other dynamically linked frameworks need their own export path before their instrumentation can contribute to the merged profile. +- Profiles become stale after native source, dependency, compiler, SDK, ABI, or important build-setting changes. Regenerate them instead of accepting out-of-date warnings. +- Instrumented builds have overhead. Never use them as the performance baseline. +- Local bootstrap settings disable Android minification so synthetic identifiers and debug signing work. A production decision still needs a production-like, minified CI build with the repository's signing and dependency issues resolved. +- Device temperature, battery state, background work, network variability, and test-account drift can overwhelm small gains. Preserve raw samples and use enough repetitions. +- A profile is only as good as its workload. Narrow startup training can regress later interactions, while an unweighted feature tour can dilute hot production paths. + +Android Baseline Profiles are a separate, complementary input to ART compilation of Java and Kotlin. Android Startup Profiles are the related mechanism for DEX layout. Neither replaces this LLVM profile for C, C++, and Swift code. + +## References + +- [Android NDK profile-guided optimization](https://developer.android.com/ndk/guides/pgo) +- [Clang profile-guided optimization](https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization) +- [`llvm-profdata` weighted inputs](https://llvm.org/docs/CommandGuide/llvm-profdata.html#cmdoption-llvm-profdata-merge-weighted-input) +- [Android Baseline Profiles](https://developer.android.com/topic/performance/baselineprofiles/overview) diff --git a/scripts/pgo/android.ts b/scripts/pgo/android.ts new file mode 100644 index 000000000000..c296c680791d --- /dev/null +++ b/scripts/pgo/android.ts @@ -0,0 +1,240 @@ +// cspell:ignore sdcard profdata Ppatched Ppgo profraw Readelf readelf zipinfo libreactnative libhermesvm libjsi cnts + +/** Android-specific release builds, native profile persistence, retrieval, and instrumentation checks. */ + +import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {homedir, platform, tmpdir} from 'node:os'; +import {join} from 'node:path'; +import process from 'node:process'; + +import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PlatformAdapter} from './shared'; + +import {capture, captureBinary, fail, findFiles, rootDirectory, run, sleep} from '../lib/scriptUtils'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; + +const ANDROID_JAVA_NAMESPACE = 'org.me.mobiexpensifyg'; +const PROFILE_BROADCAST_ACTION = 'com.expensify.chat.action.WRITE_PGO_PROFILES'; +const INSTRUMENTED_LIBRARIES = ['libreactnative.so', 'libhermesvm.so', 'libjsi.so', 'libExpensifyNitroUtils.so']; + +function createAndroidPgoAdapter(configuredAppID?: string, configuredDeviceIdentifier?: string): PlatformAdapter { + const profileDirectory = join(rootDirectory, '.pgo/android/arm64-v8a'); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const apkDirectory = join(profileDirectory, 'apk'); + const benchmarkDirectory = join(rootDirectory, '.pgo/android/benchmarks'); + const androidDirectory = join(rootDirectory, 'Mobile-Expensify/Android'); + const buildGradlePath = join(androidDirectory, 'build.gradle'); + const gradleReleaseApkPath = join(androidDirectory, 'build/intermediates/apk/release/packageRelease/Expensify-release.apk'); + const profileDumpTimeoutSeconds = 5; + + const artifactPaths: BuildArtifactPaths = { + release: join(apkDirectory, 'Expensify-release.apk'), + instrumented: join(apkDirectory, 'Expensify-release-instrumented.apk'), + optimized: join(apkDirectory, 'Expensify-release-optimized.apk'), + }; + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + let cachedAppID: string | undefined; + + function appID(): string { + if (configuredAppID) { + return configuredAppID; + } + cachedAppID ??= androidApplicationID(readFileSync(buildGradlePath, 'utf8')); + return cachedAppID; + } + + function deviceProfileDirectory(): string { + return `/sdcard/Android/data/${appID()}/cache`; + } + + function adb(args: string[]): void { + run('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); + } + + function adbCapture(args: string[]): string { + return capture('adb', configuredDeviceIdentifier ? ['-s', configuredDeviceIdentifier, ...args] : args); + } + + function llvmTool(name: string): string { + const toolPath = androidLlvmToolPath(name); + if (!existsSync(toolPath)) { + fail(`Missing ${name} at ${toolPath}. Set ANDROID_NDK_HOME (or ANDROID_NDK_ROOT) to the NDK used by the build.`); + } + return toolPath; + } + + function archiveReleaseApk(destination: string): void { + if (!existsSync(gradleReleaseApkPath)) { + fail(`Missing built release APK at ${gradleReleaseApkPath}.`); + } + mkdirSync(apkDirectory, {recursive: true}); + copyFileSync(gradleReleaseApkPath, destination); + console.log(`Copied release APK: ${destination}`); + } + + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? join(profileDirectory, 'newdot.profdata'); + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); + } + + const pgoModes: Record = {release: 'off', instrumented: 'generate', optimized: 'use'}; + const gradleArguments = [ + ':packageRelease', + '-PpatchedArtifacts.forceBuildFromSource=true', + '-PreactNativeArchitectures=arm64-v8a', + `-PpgoMode=${pgoModes[kind]}`, + '-x', + 'fullstoryRelease', + ]; + if (kind === 'optimized') { + gradleArguments.push(`-PpgoProfile=${profilePath}`); + } + + run('/usr/bin/env', [BENCHMARK_SPANS_ENVIRONMENT, 'SENTRY_DISABLE_AUTO_UPLOAD=true', './gradlew', ...gradleArguments], androidDirectory); + archiveReleaseApk(artifactPaths[kind]); + } + + function install(kind: BuildKind): void { + const apkPath = artifactPaths[kind]; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the APK first.`); + } + adb(['install', '-r', apkPath]); + console.log(`Installed APK: ${apkPath}`); + } + + async function waitForProfileDump(): Promise { + const deadline = Date.now() + profileDumpTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + const profileCount = /Wrote ([0-9]+) LLVM PGO profile/.exec(profileLogs)?.[1]; + if (profileCount !== undefined && Number(profileCount) !== 0) { + if (Number(profileCount) !== INSTRUMENTED_LIBRARIES.length) { + fail( + `The PGO receiver wrote ${profileCount} profiles, but expected ${INSTRUMENTED_LIBRARIES.length}. All instrumented native libraries must write successfully before recording profiles.`, + ); + } + console.log(`Wrote ${profileCount} LLVM PGO profile(s).`); + return; + } + if (profileLogs.includes('Ignoring PGO profile request in a non-instrumented build.')) { + fail('The installed APK is not instrumented. Build and install build-instrumented before recording profiles; build-optimized cannot generate .profraw files.'); + } + if (profileLogs.includes('Wrote 0 LLVM PGO profile(s).')) { + fail('The receiver found no instrumented native libraries in the installed APK. Rebuild and install build-instrumented before recording profiles.'); + } + await sleep(100); + } + const profileLogs = adbCapture(['logcat', '-d', '-s', 'PgoProfileReceiver:I', '*:S']); + fail(`The PGO receiver did not confirm a profile write within ${profileDumpTimeoutSeconds}s.\n${profileLogs}`); + } + + async function dumpProfiles(): Promise { + adb(['logcat', '-c']); + adb(['shell', 'am', 'broadcast', '-a', PROFILE_BROADCAST_ACTION, '-n', androidProfileReceiverComponent(appID())]); + await waitForProfileDump(); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(rawProfileDirectory, {recursive: true}); + adb(['pull', `${deviceProfileDirectory()}/.`, rawProfileDirectory]); + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in ${deviceProfileDirectory()}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + function verifyInstrumentation(): void { + const apkPath = artifactPaths.instrumented; + if (!existsSync(apkPath)) { + fail(`Missing APK at ${apkPath}. Build the instrumented release first.`); + } + + const extractedDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-apk-')); + const llvmReadelf = llvmTool('llvm-readelf'); + const llvmNm = llvmTool('llvm-nm'); + const apkEntries = new Set(capture('zipinfo', ['-1', apkPath]).split('\n')); + try { + for (const library of INSTRUMENTED_LIBRARIES) { + const apkEntry = `lib/arm64-v8a/${library}`; + if (!apkEntries.has(apkEntry)) { + fail(`Missing expected arm64 library in APK: ${apkEntry}`); + } + const extractedLibrary = join(extractedDirectory, library); + writeFileSync(extractedLibrary, captureBinary('unzip', ['-p', apkPath, apkEntry])); + const sectionHeaders = capture(llvmReadelf, ['-SW', extractedLibrary]); + if (!['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names'].every((section) => sectionHeaders.includes(section))) { + fail(`LLVM PGO instrumentation is missing from ${apkEntry}.`); + } + const dynamicSymbols = capture(llvmNm, ['-D', '--defined-only', extractedLibrary]); + if (!['expensify_llvm_profile_set_filename', 'expensify_llvm_profile_write_file'].every((symbol) => dynamicSymbols.includes(symbol))) { + fail(`LLVM PGO profile-writing APIs are not exported from ${apkEntry}.`); + } + if (library === 'libExpensifyNitroUtils.so' && !dynamicSymbols.includes('Java_org_me_mobiexpensifyg_PgoProfileWriter_writeProfiles')) { + fail(`The PGO JNI writer is missing from ${apkEntry}. Rebuild the native module before installing.`); + } + console.log(`Verified LLVM PGO instrumentation: ${apkEntry}`); + } + } finally { + rmSync(extractedDirectory, {recursive: true, force: true}); + } + } + + return { + name: 'android', + profileDirectory, + rawProfileDirectory, + mergedProfilePath: join(profileDirectory, 'newdot.profdata'), + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + appID, + build, + install, + verifyInstrumentation, + clearDeviceProfiles: async () => { + console.log(`Clearing previous device PGO profiles from ${deviceProfileDirectory()}.`); + adb(['shell', `rm -f '${deviceProfileDirectory()}'/newdot-*.profraw`]); + }, + dumpProfiles, + pullProfiles, + llvmTool, + }; +} + +/** Resolve LLVM tools from the build's explicit NDK or host-specific SDK installation. */ +function androidLlvmToolPath( + name: string, + {hostPlatform = platform(), homeDirectory = homedir(), environment = process.env}: {hostPlatform?: NodeJS.Platform; homeDirectory?: string; environment?: NodeJS.ProcessEnv} = {}, +): string { + const configuredNdkRoot = [environment.ANDROID_NDK_HOME, environment.ANDROID_NDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const configuredSdkRoot = [environment.ANDROID_HOME, environment.ANDROID_SDK_ROOT].find((value): value is string => typeof value === 'string' && value.length > 0); + const sdkRoot = configuredSdkRoot ?? join(homeDirectory, hostPlatform === 'linux' ? 'Android/Sdk' : 'Library/Android/sdk'); + const ndkRoot = configuredNdkRoot ?? join(sdkRoot, 'ndk/27.1.12297006'); + const hostTag = hostPlatform === 'linux' ? 'linux-x86_64' : 'darwin-x86_64'; + return join(ndkRoot, `toolchains/llvm/prebuilt/${hostTag}/bin/${name}`); +} + +function androidApplicationID(buildGradle: string, configuredAppID?: string): string { + if (configuredAppID) { + return configuredAppID; + } + const applicationID = /defaultConfig\s*\{[\s\S]*?applicationId\s+["']([^"']+)["']/.exec(buildGradle)?.[1]; + if (!applicationID) { + fail('Unable to read the release application ID from Mobile-Expensify/Android/build.gradle. Pass it with --app-id.'); + } + return applicationID; +} + +function androidProfileReceiverComponent(appID: string): string { + return `${appID}/${ANDROID_JAVA_NAMESPACE}.PgoProfileReceiver`; +} + +export default createAndroidPgoAdapter; +export {PROFILE_BROADCAST_ACTION, androidApplicationID, androidLlvmToolPath, androidProfileReceiverComponent}; diff --git a/scripts/pgo/ios.ts b/scripts/pgo/ios.ts new file mode 100644 index 000000000000..204855eaca7c --- /dev/null +++ b/scripts/pgo/ios.ts @@ -0,0 +1,498 @@ +// cspell:ignore Xfrontend hermesv profdata devicectl libdav intops libwebp fprofile CPLUSPLUSFLAGS ARCHS profraw cnts hermesvm unprofiled + +/** iOS-specific release builds, signing, native profile persistence, retrieval, and instrumentation checks. */ + +import {isRecord} from '@libs/ObjectUtils'; + +import {cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; + +import type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter} from './shared'; + +import {capture, environmentString, fail, findFiles, rootDirectory, run, runAllowFailure, sleep, valueAt} from '../lib/scriptUtils'; +import {BENCHMARK_SPANS_ENVIRONMENT} from './shared'; + +function createIOSPgoAdapter(configuredAppID?: string, cliDeviceIdentifier?: string): PlatformAdapter { + const profileFormat = 'ios-clang-swift-ir-v3'; + const profileDirectory = join(rootDirectory, '.pgo/ios/arm64'); + const rawProfileDirectory = join(profileDirectory, 'raw'); + const appDirectory = join(profileDirectory, 'app'); + const benchmarkDirectory = join(rootDirectory, '.pgo/ios/benchmarks'); + const markerDirectory = join(profileDirectory, 'markers'); + const iosDirectory = join(rootDirectory, 'Mobile-Expensify/iOS'); + const projectPath = join(iosDirectory, 'Expensify.xcodeproj/project.pbxproj'); + const hermesVersionPath = join(rootDirectory, 'node_modules/react-native/sdks/.hermesv1version'); + const remotePgoDirectory = 'Library/Caches/ExpensifyPGO'; + const profileStatusPath = `${remotePgoDirectory}/profile-status.txt`; + const profileWriteNotification = 'com.expensify.pgo.write-profiles'; + const profileClearNotification = 'com.expensify.pgo.clear-profiles'; + const profileOperationTimeoutSeconds = 10; + const developmentTeam = environmentString('IOS_DEVELOPMENT_TEAM'); + const codeSigningAllowed = !['0', 'false', 'no'].includes((environmentString('IOS_CODE_SIGNING_ALLOWED') ?? 'yes').toLowerCase()); + const configuredBundleIdentifier = configuredAppID ?? environmentString('IOS_BUNDLE_IDENTIFIER'); + const mergedProfilePath = join(profileDirectory, 'newdot.profdata'); + + const artifactPaths: BuildArtifactPaths = { + release: join(appDirectory, 'Expensify-release.app'), + instrumented: join(appDirectory, 'Expensify-release-instrumented.app'), + optimized: join(appDirectory, 'Expensify-release-optimized.app'), + }; + const benchmarkPaths: Record = { + release: join(benchmarkDirectory, 'release.csv'), + optimized: join(benchmarkDirectory, 'pgo-optimized.csv'), + }; + + let cachedDeviceIdentifier: string | undefined; + let cachedBundleIdentifier: string | undefined; + + function bundleIdentifier(): string { + if (cachedBundleIdentifier) { + return cachedBundleIdentifier; + } + if (configuredBundleIdentifier) { + cachedBundleIdentifier = configuredBundleIdentifier; + return configuredBundleIdentifier; + } + + const archivedApp = [artifactPaths.instrumented, artifactPaths.optimized, artifactPaths.release].find((path) => existsSync(join(path, 'Info.plist'))); + if (!archivedApp) { + fail('No archived iOS app is available to determine its bundle identifier. Build an app first, or set IOS_BUNDLE_IDENTIFIER.'); + } + cachedBundleIdentifier = capture('/usr/libexec/PlistBuddy', ['-c', 'Print :CFBundleIdentifier', join(archivedApp, 'Info.plist')]).trim(); + if (!cachedBundleIdentifier) { + fail(`The archived app at ${archivedApp} has no CFBundleIdentifier.`); + } + console.log(`Using iOS bundle identifier ${cachedBundleIdentifier}.`); + return cachedBundleIdentifier; + } + + function deviceIdentifier(): string { + if (cachedDeviceIdentifier) { + return cachedDeviceIdentifier; + } + + const configuredIdentifier = cliDeviceIdentifier ?? environmentString('IOS_DEVICE_ID'); + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-pgo-ios-devices-')); + const jsonPath = join(temporaryDirectory, 'devices.json'); + try { + run('xcrun', ['devicectl', 'list', 'devices', '--json-output', jsonPath]); + const parsedDevices: unknown = JSON.parse(readFileSync(jsonPath, 'utf8')); + if (!isRecord(parsedDevices) || !isRecord(parsedDevices.result) || !Array.isArray(parsedDevices.result.devices)) { + fail('CoreDevice returned an unexpected device-list response.'); + } + + const physicalDevices = parsedDevices.result.devices.flatMap((device) => { + if (!isRecord(device) || !isRecord(device.hardwareProperties) || !isRecord(device.deviceProperties)) { + return []; + } + const {hardwareProperties, deviceProperties} = device; + if (hardwareProperties.platform !== 'iOS' || hardwareProperties.reality !== 'physical' || deviceProperties.bootState !== 'booted') { + return []; + } + if (typeof hardwareProperties.udid !== 'string' || typeof deviceProperties.name !== 'string') { + return []; + } + const name = deviceProperties.name.trim(); + const osVersion = typeof deviceProperties.osVersionNumber === 'string' ? deviceProperties.osVersionNumber : undefined; + const aliases = [name, hardwareProperties.udid]; + if (typeof device.identifier === 'string') { + aliases.push(device.identifier); + } + if (typeof hardwareProperties.serialNumber === 'string') { + aliases.push(hardwareProperties.serialNumber); + } + if (osVersion) { + aliases.push(`${name} (${osVersion})`, `${name} (${osVersion}) (${hardwareProperties.udid})`); + } + return [{aliases, name, udid: hardwareProperties.udid}]; + }); + + if (physicalDevices.length === 0) { + fail('No booted physical iOS device is connected. Connect and unlock a device, or set IOS_DEVICE_ID.'); + } + const normalizeSelector = (value: string) => value.trim().replaceAll(/\s+/g, ' ').toLowerCase(); + const matchingDevices = configuredIdentifier + ? physicalDevices.filter(({aliases}) => aliases.some((alias) => normalizeSelector(alias) === normalizeSelector(configuredIdentifier))) + : physicalDevices; + if (matchingDevices.length === 0) { + fail(`No connected physical iOS device matches "${configuredIdentifier}". Available devices: ${physicalDevices.map(({name, udid}) => `${name} (${udid})`).join(', ')}.`); + } + if (matchingDevices.length > 1) { + fail(`Multiple physical iOS devices are connected (${physicalDevices.map(({name}) => name).join(', ')}). Set IOS_DEVICE_ID to the desired UDID.`); + } + + const selectedDevice = valueAt(matchingDevices, 0); + cachedDeviceIdentifier = selectedDevice.udid; + console.log(`Using iOS device ${selectedDevice.name} (${selectedDevice.udid}).`); + return selectedDevice.udid; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } + } + + function llvmTool(name: string): string { + return capture('xcrun', ['--find', name]).trim(); + } + + function sourceBuildEnvironment(mode: PgoMode, profilePath = mergedProfilePath): string[] { + const environment = [ + 'BUILD_RN_FROM_SOURCE=1', + 'RCT_USE_RN_DEP=0', + 'RCT_USE_PREBUILT_RNCORE=0', + 'RCT_BUILD_HERMES_FROM_SOURCE=true', + BENCHMARK_SPANS_ENVIRONMENT, + 'SENTRY_DISABLE_AUTO_UPLOAD=true', + `EXPENSIFY_PGO_MODE=${mode}`, + ]; + if (mode === 'use') { + environment.push(`EXPENSIFY_PGO_PROFILE=${profilePath}`); + } + return environment; + } + + function prepareSourcePods(mode: PgoMode, profilePath: string): void { + if (!existsSync(hermesVersionPath)) { + fail(`Missing React Native's pinned Hermes version file at ${hermesVersionPath}.`); + } + const requiredPodSources = [ + {directory: join(iosDirectory, 'Pods/hermes-engine'), requiredFile: 'CMakeLists.txt'}, + {directory: join(iosDirectory, 'Pods/libdav1d'), requiredFile: 'dav1d/include/common/intops.h'}, + {directory: join(iosDirectory, 'Pods/libwebp'), requiredFile: 'src/webp/types.h'}, + ]; + for (const {directory, requiredFile} of requiredPodSources) { + if (existsSync(directory) && !existsSync(join(directory, requiredFile))) { + console.log(`Removing incomplete CocoaPods checkout: ${directory}`); + rmSync(directory, {recursive: true, force: true}); + } + } + run('/usr/bin/env', [...sourceBuildEnvironment(mode, profilePath), 'bundle', 'exec', 'pod', 'install', '--silent'], iosDirectory); + for (const {directory, requiredFile} of requiredPodSources) { + if (!existsSync(join(directory, requiredFile))) { + fail(`CocoaPods did not install the required source file ${join(directory, requiredFile)}.`); + } + } + rmSync(join(iosDirectory, 'Pods/hermes-engine/build'), {recursive: true, force: true}); + } + + function pgoBuildSettings(kind: BuildKind, profilePath: string): string[] { + const baseSettings = ['ENABLE_CODE_COVERAGE=NO', 'CLANG_COVERAGE_MAPPING=NO', 'CLANG_USE_OPTIMIZATION_PROFILE=NO']; + if (kind === 'instrumented') { + return [ + ...baseSettings, + 'GCC_PREPROCESSOR_DEFINITIONS=$(inherited) EXPENSIFY_PGO_GENERATE=1', + 'OTHER_CFLAGS=$(inherited) -fprofile-generate', + 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-generate', + 'OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-generate', + 'OTHER_LDFLAGS=$(inherited) -fprofile-generate', + ]; + } + if (kind === 'optimized') { + const clangProfileFlags = `-fprofile-use=${profilePath} -Wno-error=profile-instr-unprofiled`; + return [ + ...baseSettings, + `OTHER_CFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_CPLUSPLUSFLAGS=$(inherited) ${clangProfileFlags}`, + `OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -ir-profile-use=${profilePath}`, + ]; + } + return baseSettings; + } + + function build(kind: BuildKind, profileOverride?: string): void { + const profilePath = profileOverride ?? mergedProfilePath; + if (kind === 'optimized' && !existsSync(profilePath)) { + fail(`Missing ${profilePath}. Run merge first or pass --profile.`); + } + if (kind === 'optimized') { + const profileFormatPath = `${profilePath}.format`; + const recordedFormat = existsSync(profileFormatPath) ? readFileSync(profileFormatPath, 'utf8').trim() : undefined; + if (recordedFormat !== profileFormat) { + fail('The merged iOS profile does not match the current Clang and Swift IR instrumentation. Install the latest instrumented app and run record-startups again.'); + } + } + + let mode: PgoMode = 'off'; + if (kind === 'instrumented') { + mode = 'generate'; + } else if (kind === 'optimized') { + mode = 'use'; + } + prepareSourcePods(mode, profilePath); + const derivedDataDirectory = join(profileDirectory, `derived-data/${kind}`); + const builtAppPath = join(derivedDataDirectory, 'Build/Products/Release-iphoneos/Expensify.app'); + const signingSettings = codeSigningAllowed + ? [ + 'CODE_SIGN_STYLE=Automatic', + 'CODE_SIGN_IDENTITY=Apple Development', + 'PROVISIONING_PROFILE=', + 'PROVISIONING_PROFILE_SPECIFIER=', + ...(developmentTeam ? [`DEVELOPMENT_TEAM=${developmentTeam}`] : []), + ] + : ['CODE_SIGNING_ALLOWED=NO', 'CODE_SIGNING_REQUIRED=NO']; + const rockArguments = iosRockBuildArguments(derivedDataDirectory, [ + 'build', + 'ARCHS=arm64', + 'ONLY_ACTIVE_ARCH=YES', + 'COMPILER_INDEX_STORE_ENABLE=NO', + ...(codeSigningAllowed ? ['-allowProvisioningUpdates'] : []), + ...signingSettings, + ...pgoBuildSettings(kind, profilePath), + ]); + withLocalUploadPhasesDisabled(() => + run( + '/usr/bin/env', + [ + ...sourceBuildEnvironment(mode, profilePath), + 'IS_HYBRID_APP=true', + 'BUNDLER=metro', + `PATH=${join(rootDirectory, 'node_modules/.bin')}:${process.env.PATH ?? ''}`, + 'node', + join(rootDirectory, 'node_modules/rock/dist/src/bin.js'), + ...rockArguments, + ], + join(rootDirectory, 'Mobile-Expensify'), + ), + ); + if (!existsSync(builtAppPath)) { + fail(`Missing built iOS app at ${builtAppPath}.`); + } + mkdirSync(appDirectory, {recursive: true}); + rmSync(artifactPaths[kind], {recursive: true, force: true}); + cpSync(builtAppPath, artifactPaths[kind], {recursive: true, preserveTimestamps: true}); + console.log(`Copied release app: ${artifactPaths[kind]}`); + } + + /** Prevent local release builds from publishing source maps or FullStory assets; restore the existing project text even if Xcode fails. */ + function withLocalUploadPhasesDisabled(buildApp: () => void): void { + const replacements = [ + // Xcode expands these literal variables inside the project build phase. + { + // eslint-disable-next-line no-template-curly-in-string + command: '\\"${PODS_ROOT}/FullStory/tools/FullStoryCommandLine\\" \\"${CONFIGURATION_BUILD_DIR}/${WRAPPER_NAME}\\"', + disabled: 'echo Skipping FullStory upload for local PGO build', + }, + {command: 'source \\"$PROJECT_ROOT/tools/upload-sentry-source-maps.sh\\"', disabled: 'echo Skipping Sentry source-map upload for local PGO build'}, + ]; + const originalProject = readFileSync(projectPath, 'utf8'); + const guardedProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, command, disabled), originalProject); + let restored = false; + let guarded = false; + const restoreProject = () => { + if (restored) { + return; + } + if (!guarded) { + writeFileSync(projectPath, originalProject); + restored = true; + return; + } + const currentProject = readFileSync(projectPath, 'utf8'); + const restoredProject = replacements.reduce((content, {command, disabled}) => replaceOnce(content, disabled, command), currentProject); + writeFileSync(projectPath, restoredProject); + restored = true; + }; + const onInterrupt = () => { + restoreProject(); + process.exit(130); + }; + const onTermination = () => { + restoreProject(); + process.exit(143); + }; + process.once('SIGINT', onInterrupt); + process.once('SIGTERM', onTermination); + try { + writeFileSync(projectPath, guardedProject); + guarded = true; + buildApp(); + } finally { + try { + restoreProject(); + } finally { + process.off('SIGINT', onInterrupt); + process.off('SIGTERM', onTermination); + } + } + } + + function install(kind: BuildKind): void { + const appPath = artifactPaths[kind]; + if (!existsSync(appPath)) { + fail(`Missing app at ${appPath}. Build it first.`); + } + if (!existsSync(join(appPath, '_CodeSignature/CodeResources'))) { + fail(`The app at ${appPath} is unsigned. Rebuild without IOS_CODE_SIGNING_ALLOWED=NO before installing it on a device.`); + } + run('xcrun', ['devicectl', 'device', 'install', 'app', '--device', deviceIdentifier(), appPath]); + console.log(`Installed app: ${appPath}`); + } + + function tryReadDeviceFile(remotePath: string, localName: string): string | undefined { + mkdirSync(markerDirectory, {recursive: true}); + const localPath = join(markerDirectory, localName); + rmSync(localPath, {recursive: true, force: true}); + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePath, + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8').trim() : undefined; + } + + function postNotification(name: string): void { + run('xcrun', ['devicectl', 'device', 'notification', 'post', '--device', deviceIdentifier(), '--name', name]); + } + + async function waitForProfileStatus(expectedStatus: string, previousStatus: string | undefined): Promise { + const deadline = Date.now() + profileOperationTimeoutSeconds * 1000; + while (Date.now() < deadline) { + const statusContents = tryReadDeviceFile(profileStatusPath, 'profile-status.txt'); + if (statusContents && statusContents !== previousStatus) { + const [, status, rawResult] = statusContents.split(','); + if (status === 'not-instrumented') { + fail('The installed iOS app is not instrumented. Build and install build-instrumented before recording profiles.'); + } + if (status === expectedStatus && Number(rawResult) === 0) { + console.log(`iOS PGO profiles ${expectedStatus}.`); + return; + } + fail(`The iOS PGO profile operation failed: ${statusContents}`); + } + await sleep(250); + } + fail(`The iOS app did not confirm that PGO profiles were ${expectedStatus} within ${profileOperationTimeoutSeconds}s.`); + } + + async function clearDeviceProfiles(): Promise { + console.log(`Clearing previous iOS PGO profiles from ${remotePgoDirectory}.`); + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-clear.txt'); + postNotification(profileClearNotification); + await waitForProfileStatus('cleared', previousStatus); + } + + async function dumpProfiles(): Promise { + const previousStatus = tryReadDeviceFile(profileStatusPath, 'profile-status-before-write.txt'); + postNotification(profileWriteNotification); + await waitForProfileStatus('written', previousStatus); + } + + function pullProfiles(): void { + rmSync(rawProfileDirectory, {recursive: true, force: true}); + mkdirSync(dirname(rawProfileDirectory), {recursive: true}); + run('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceIdentifier(), + '--source', + remotePgoDirectory, + '--destination', + rawProfileDirectory, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + bundleIdentifier(), + ]); + const profiles = findFiles(rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail(`No .profraw files found in the iOS app container at ${remotePgoDirectory}. Run dump first.`); + } + console.log(profiles.join('\n')); + } + + function verifyInstrumentation(): void { + const appPath = artifactPaths.instrumented; + const executablePath = join(appPath, 'Expensify'); + if (!existsSync(executablePath)) { + fail(`Missing instrumented iOS executable at ${executablePath}.`); + } + const requiredSections = ['__llvm_prf_data', '__llvm_prf_cnts', '__llvm_prf_names']; + const sectionHeaders = capture('xcrun', ['size', '-m', executablePath]); + if (!requiredSections.every((section) => sectionHeaders.includes(section))) { + fail('LLVM PGO instrumentation is missing from the iOS app executable.'); + } + if (!capture('strings', ['-a', executablePath]).includes('ExpensifyPGO: wrote LLVM profile')) { + fail('The iOS app executable is missing the PGO flush marker.'); + } + console.log(`Verified LLVM PGO instrumentation: ${executablePath}`); + + const hermesExecutable = join(appPath, 'Frameworks/hermesvm.framework/hermesvm'); + if (!existsSync(hermesExecutable)) { + fail(`Missing embedded source-built Hermes framework at ${hermesExecutable}.`); + } + const hermesSections = capture('xcrun', ['size', '-m', hermesExecutable]); + if (!requiredSections.every((section) => hermesSections.includes(section))) { + fail('LLVM PGO instrumentation is missing from the source-built iOS Hermes framework.'); + } + const hermesSymbols = capture('xcrun', ['nm', '-gU', hermesExecutable]); + if (!['expensify_llvm_profile_set_filename', 'expensify_llvm_profile_write_file', 'expensify_llvm_profile_reset_counters'].every((symbol) => hermesSymbols.includes(symbol))) { + fail('The source-built iOS Hermes framework is missing the exported PGO profile runtime APIs.'); + } + console.log(`Verified LLVM PGO instrumentation: ${hermesExecutable}`); + } + + return { + name: 'ios', + profileDirectory, + rawProfileDirectory, + mergedProfilePath, + benchmarkDirectory, + benchmarkPaths, + artifactPaths, + profileFormat, + appID: bundleIdentifier, + build, + install, + verifyInstrumentation, + clearDeviceProfiles, + dumpProfiles, + pullProfiles, + llvmTool, + }; +} + +function replaceOnce(content: string, previous: string, next: string): string { + const offset = content.indexOf(previous); + if (offset < 0 || content.indexOf(previous, offset + previous.length) >= 0) { + fail('The iOS project upload phases changed. Refusing to build until the local PGO upload guard is updated.'); + } + return `${content.slice(0, offset)}${next}${content.slice(offset + previous.length)}`; +} + +/** Preserve each Xcode setting as one argument through Rock's string argument parser. */ +function iosRockBuildArguments(derivedDataDirectory: string, settings: string[]): string[] { + return [ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + derivedDataDirectory, + '--no-install-pods', + '--extra-params', + settings.map((argument) => JSON.stringify(argument)).join(' '), + ]; +} + +export {iosRockBuildArguments}; +export default createIOSPgoAdapter; diff --git a/scripts/pgo/journey.ts b/scripts/pgo/journey.ts new file mode 100755 index 000000000000..624f67f2c56e --- /dev/null +++ b/scripts/pgo/journey.ts @@ -0,0 +1,154 @@ +#!/usr/bin/env bun + +// cspell:ignore profraw profdata + +import CLI from 'expensify-common/CLI'; +import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {dirname, join, relative, resolve} from 'node:path'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +import {PLATFORM_NAMES} from '../lib/nativeAppBenchmark'; +import {capture, findFiles, parseChoice, parsePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {parseJourneyFixture} from './journeyConfig'; +import {createJourneyDevice} from './journeyDevice'; +import {prepareJourney, runJourneyWorkload, verifyJourneyAccount} from './journeyWorkload'; + +/** Run the same UI workload on a selected Android or iOS device; never automate authentication. */ +async function main(): Promise { + /* eslint-disable @typescript-eslint/naming-convention */ + const cli = new CLI({ + positionalArgs: [{name: 'platform', description: 'android or ios', parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform')}], + namedArgs: { + fixture: {description: 'JSON fixture identifying the approved heavy account and reports'}, + device: {description: 'Android serial or iOS UDID'}, + 'app-id': {description: 'Installed application or bundle identifier'}, + runs: {description: 'Complete journeys to run', default: 1, parse: (value) => parsePositiveInteger(value, 'Runs')}, + }, + flags: { + collect: {description: 'Flush, archive, and merge LLVM profiles from an already-installed instrumented app'}, + preflight: {description: 'Verify account and disable #focus, then stop without running the workload'}, + 'navigation-only': {description: 'Validate navigation without sending messages; cannot collect training profiles'}, + }, + }); + /* eslint-enable @typescript-eslint/naming-convention */ + const platform = parseChoice(String(cli.positionalArgs.platform), PLATFORM_NAMES, 'Platform'); + const fixtureInput: unknown = JSON.parse(readFileSync(resolve(cli.namedArgs.fixture), 'utf8')); + const fixture = parseJourneyFixture(fixtureInput); + if (cli.flags['navigation-only'] && cli.flags.collect) { + throw new Error('Navigation-only validation cannot produce a complete training profile.'); + } + if (!cli.flags.preflight && !cli.flags['navigation-only'] && !fixture.allowMessages) { + throw new Error('Confirm the personal chat with Chris before setting allowMessages=true in the fixture. Use --navigation-only to validate without sending.'); + } + const runs = parsePositiveInteger(String(cli.namedArgs.runs), 'Runs'); + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const batchDirectory = join(rootDirectory, '.pgo', platform, 'journeys', batchID); + mkdirSync(batchDirectory, {recursive: true}); + const device = createJourneyDevice({platform, device: cli.namedArgs.device, appID: cli.namedArgs['app-id'], session: `pgo-${platform}-${batchID}`}); + const adapter = platform === 'android' ? createAndroidPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device) : createIOSPgoAdapter(cli.namedArgs['app-id'], cli.namedArgs.device); + const completedProfiles: string[] = []; + const evidence = { + platform, + device: cli.namedArgs.device, + appID: cli.namedArgs['app-id'], + accountEmail: fixture.accountEmail, + accountClass: fixture.accountClass, + fixture, + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + automationVersion: capture('agent-device', ['--version']).trim(), + requestedRuns: runs, + collectProfiles: cli.flags.collect, + navigationOnly: cli.flags['navigation-only'], + completedRuns: 0, + runDurationsSeconds: [] as number[], + status: 'running', + failure: '', + }; + const saveEvidence = () => writeFileSync(join(batchDirectory, 'result.json'), `${JSON.stringify(evidence, null, 2)}\n`); + saveEvidence(); + try { + device.open(false); + await prepareJourney(device, fixture); + if (cli.flags.preflight) { + evidence.status = 'preflight-passed'; + return; + } + if (cli.flags.collect) { + // Verify the installed build's writer before spending time on the workload; discard these setup counters. + await adapter.dumpProfiles(); + await adapter.clearDeviceProfiles(); + } + for (let index = 1; index <= runs; index += 1) { + const runID = `${batchID}-${index}`; + const runDirectory = join(batchDirectory, `run-${index}`); + mkdirSync(runDirectory); + if (index > 1) { + await prepareJourney(device, fixture); + if (cli.flags.collect) { + await adapter.clearDeviceProfiles(); + } + } + console.log(`Starting journey ${index}/${runs}.`); + const startedAt = Date.now(); + device.open(true); + await runJourneyWorkload(device, fixture, runID, cli.flags['navigation-only']); + await verifyJourneyAccount(device, fixture); + if (cli.flags.collect) { + // Flush once per fresh process. Never merge partial or failed journeys. + await adapter.dumpProfiles(); + adapter.pullProfiles(); + const rawDirectory = join(runDirectory, 'raw'); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + throw new Error('The completed journey produced no LLVM profiles.'); + } + for (const profile of profiles) { + // Archive counters only, not unrelated files pulled from the app's cache directory. + const archivedProfile = join(rawDirectory, relative(adapter.rawProfileDirectory, profile)); + mkdirSync(dirname(archivedProfile), {recursive: true}); + copyFileSync(profile, archivedProfile); + completedProfiles.push(archivedProfile); + } + } + evidence.completedRuns = index; + evidence.runDurationsSeconds.push(Math.round((Date.now() - startedAt) / 1000)); + saveEvidence(); + } + if (cli.flags.collect) { + const mergedProfile = join(batchDirectory, 'journey.profdata'); + run(adapter.llvmTool('llvm-profdata'), ['merge', `--output=${mergedProfile}`, ...completedProfiles]); + if (adapter.profileFormat) { + writeFileSync(`${mergedProfile}.format`, `${adapter.profileFormat}\n`); + } + writeFileSync(`${mergedProfile}.txt`, capture(adapter.llvmTool('llvm-profdata'), ['show', '--all-functions', mergedProfile])); + console.log(`Journey profile: ${mergedProfile}`); + } + evidence.status = cli.flags['navigation-only'] ? 'navigation-passed' : 'passed'; + } catch (error) { + evidence.status = 'failed'; + evidence.failure = error instanceof Error ? error.message : String(error); + throw error; + } finally { + saveEvidence(); + // Release the device and restore Android's original keyboard, including on failed attempts. + // Closing a session leaves persisted authentication intact. + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + console.log(`Journey result: ${join(batchDirectory, 'result.json')}`); + } +} + +if (import.meta.main) { + main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = message.startsWith('SIGN_IN_REQUIRED:') ? 2 : 1; + }); +} diff --git a/scripts/pgo/journeyBenchmark.ts b/scripts/pgo/journeyBenchmark.ts new file mode 100644 index 000000000000..40294812c581 --- /dev/null +++ b/scripts/pgo/journeyBenchmark.ts @@ -0,0 +1,348 @@ +/** Runs separate read-only signed-in navigation and heavy-report benchmarks. */ + +// cspell:ignore gfxinfo + +import {isRecord} from '@libs/ObjectUtils'; + +import {createHash} from 'node:crypto'; +import {createReadStream, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, writeFileSync} from 'node:fs'; +import {join, relative} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; +import type {JourneyFixture} from './journeyConfig'; +import type {PlatformAdapter, BenchmarkKind} from './shared'; + +import {createBenchmarkRecorder} from '../lib/benchmarkAppStartup'; +import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; +import {capture, fail} from '../lib/scriptUtils'; +import createAndroidJourneyBenchmarkReader from './journeyBenchmarkAndroid'; +import createIOSJourneyBenchmarkReader from './journeyBenchmarkIOS'; +import {createJourneyDevice} from './journeyDevice'; +import {openReport, scrollFastBothWays, selectSpendSection, showTab, verifyJourneyAccount, waitForTab} from './journeyWorkload'; +import {HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES} from './shared'; + +type JourneyBenchmarkReader = { + begin: (spanNames: readonly string[]) => void; + collect: (spanNames: readonly string[], timeoutSeconds: number) => Promise; +}; + +type FrameHealth = {totalFrames: number; droppedFrames: number; droppedPercent: number; sampleWindowMs: number}; +type JourneyMeasurement = {events: BenchmarkLogEvent[]; frameHealth?: FrameHealth}; +type BenchmarkOrder = 'release-first' | 'optimized-first'; + +type JourneyScenario = { + directory: 'journey' | 'heavy-journey'; + spanNames: readonly string[]; + measure: (device: ReturnType, reader: JourneyBenchmarkReader, fixture: JourneyFixture, timeoutSeconds: number) => Promise; +}; + +const SHORT_JOURNEY: JourneyScenario = {directory: 'journey', spanNames: JOURNEY_SPAN_NAMES, measure: measureJourney}; +const HEAVY_JOURNEY: JourneyScenario = {directory: 'heavy-journey', spanNames: HEAVY_JOURNEY_SPAN_NAMES, measure: measureHeavyJourney}; + +function journeyBenchmarkPaths( + adapter: Pick, + batchID: string, + scenarioDirectory: JourneyScenario['directory'] = 'journey', +): Record { + const directory = join(adapter.benchmarkDirectory, scenarioDirectory, batchID); + return {release: join(directory, 'release.csv'), optimized: join(directory, 'pgo-optimized.csv')}; +} + +/** Installs the selected build, discards one complete warm-up, then records every requested run. */ +async function benchmarkJourneyBuild( + adapter: PlatformAdapter, + kind: BenchmarkKind, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + outputPath: string, + scenario: JourneyScenario, +): Promise { + console.log(`Installing ${kind} artifact: ${adapter.artifactPaths[kind]}`); + adapter.install(kind); + + const device = createJourneyDevice({platform: adapter.name, device: deviceID, appID: adapter.appID(), session: `pgo-${scenario.directory}-${adapter.name}-${kind}-${Date.now()}`}); + const reader: JourneyBenchmarkReader = + adapter.name === 'android' ? createAndroidJourneyBenchmarkReader(deviceID, adapter.appID()) : createIOSJourneyBenchmarkReader(deviceID, adapter.appID()); + try { + device.open(false); + await showTab(device, 'Inbox'); + await verifyJourneyAccount(device, fixture); + console.log(`Running one unmeasured ${kind} ${scenario.directory} warm-up.`); + await scenario.measure(device, reader, fixture, timeoutSeconds); + + const recorder = createBenchmarkRecorder({spanNames: [...scenario.spanNames], runs, outputPath}); + const frameRows: string[] = []; + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + const measurement = await scenario.measure(device, reader, fixture, timeoutSeconds); + console.log(`${kind} ${scenario.directory} ${runNumber}/${runs}: ${recorder.record(measurement.events, runNumber).join(', ')}`); + if (measurement.frameHealth) { + const {totalFrames, droppedFrames, droppedPercent, sampleWindowMs} = measurement.frameHealth; + frameRows.push(`${runNumber},${totalFrames},${droppedFrames},${droppedPercent},${sampleWindowMs}`); + console.log(`Android scroll frames: ${droppedFrames}/${totalFrames} missed deadlines (${droppedPercent.toFixed(1)}%).`); + } + } + await verifyJourneyAccount(device, fixture); + await recorder.complete(`${kind} ${scenario.directory} metrics`); + if (frameRows.length > 0) { + if (frameRows.length !== runs) { + fail(`Only ${frameRows.length}/${runs} Android scroll frame samples were recorded.`); + } + writeFileSync(outputPath.replace(/\.csv$/, '-frames.csv'), `run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs\n${frameRows.join('\n')}\n`); + } + } finally { + try { + device.command('close'); + } catch (error) { + console.error(`Device cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +/** Uses a new app process for each sample; automation latency is outside the timed spans. */ +async function measureJourney(device: ReturnType, reader: JourneyBenchmarkReader, _fixture: JourneyFixture, timeoutSeconds: number): Promise { + device.open(true); + await waitForTab(device, 'Account'); + // Normalize the saved Spend view before timing either tab transition. This makes the measured taps warm navigations from a known Expenses state. + await showTab(device, 'Spend'); + selectSpendSection(device, 'Expenses'); + device.waitLabel('Filters'); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + + const spendSpans = JOURNEY_SPAN_NAMES.slice(0, 2); + const inboxSpans = [JOURNEY_SPAN_NAMES[2]]; + reader.begin(spendSpans); + await showTab(device, 'Spend'); + device.waitLabel('Filters'); + const spendEvents = await reader.collect(spendSpans, timeoutSeconds); + + reader.begin(inboxSpans); + await showTab(device, 'Inbox'); + device.waitLabel('All'); + const inboxEvents = await reader.collect(inboxSpans, timeoutSeconds); + return {events: [...spendEvents, ...inboxEvents]}; +} + +/** Opens a populated chat and fast-scrolls its history; device-command time is never a benchmark sample. */ +async function measureHeavyJourney( + device: ReturnType, + reader: JourneyBenchmarkReader, + fixture: JourneyFixture, + timeoutSeconds: number, +): Promise { + device.open(true); + await showTab(device, 'Inbox'); + await waitForTab(device, 'Account'); + await openReport(device, fixture.report, false, () => reader.begin(HEAVY_JOURNEY_SPAN_NAMES)); + const events = await reader.collect(HEAVY_JOURNEY_SPAN_NAMES, timeoutSeconds); + if (device.platform === 'android') { + // Android gfxinfo resets after each read. Exclude search and report opening from this frame window. + readAndroidFrameHealth(device, false); + } + scrollFastBothWays(device, Math.min(fixture.scrolls, 4)); + const frameHealth = device.platform === 'android' ? readAndroidFrameHealth(device, true) : undefined; + await showTab(device, 'Inbox'); + return {events, frameHealth}; +} + +function readAndroidFrameHealth(device: ReturnType, requireFrames: boolean): FrameHealth | undefined { + const result = device.command('perf', 'frames'); + if (!isRecord(result) || !isRecord(result.metrics) || !isRecord(result.metrics.fps) || !isRecord(result.sampling) || !isRecord(result.sampling.fps)) { + throw new Error('Android frame sampler returned an unexpected result.'); + } + if (result.sampling.fps.resetsAfterRead !== true || result.metrics.fps.available !== true) { + throw new Error('Android frame sampler is unavailable or did not reset its counter.'); + } + const {totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs} = result.metrics.fps; + if ( + typeof totalFrameCount !== 'number' || + typeof droppedFrameCount !== 'number' || + typeof droppedFramePercent !== 'number' || + typeof sampleWindowMs !== 'number' || + ![totalFrameCount, droppedFrameCount, droppedFramePercent, sampleWindowMs].every(Number.isFinite) + ) { + throw new Error('Android frame sampler did not return finite frame counts.'); + } + if (requireFrames && totalFrameCount === 0) { + throw new Error('The Android scroll produced no measured frames. Discard this run.'); + } + return {totalFrames: totalFrameCount, droppedFrames: droppedFrameCount, droppedPercent: droppedFramePercent, sampleWindowMs}; +} + +async function benchmarkJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, SHORT_JOURNEY, order); +} + +async function benchmarkHeavyJourneyAll( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + order: BenchmarkOrder = 'release-first', +): Promise { + await benchmarkJourneyScenario(adapter, deviceID, fixture, runs, timeoutSeconds, HEAVY_JOURNEY, order); +} + +async function benchmarkJourneyScenario( + adapter: PlatformAdapter, + deviceID: string, + fixture: JourneyFixture, + runs: number, + timeoutSeconds: number, + scenario: JourneyScenario, + order: BenchmarkOrder, +): Promise { + const batchID = new Date().toISOString().replaceAll(/[:.]/g, '-'); + const paths = journeyBenchmarkPaths(adapter, batchID, scenario.directory); + const directory = join(adapter.benchmarkDirectory, scenario.directory, batchID); + if (existsSync(directory)) { + fail(`Journey benchmark batch already exists: ${directory}`); + } + mkdirSync(directory, {recursive: true}); + writeFileSync( + join(directory, 'run.json'), + `${JSON.stringify( + { + platform: adapter.name, + deviceID, + appID: adapter.appID(), + accountEmail: fixture.accountEmail, + runs, + timeoutSeconds, + order, + scenario: scenario.directory, + spans: scenario.spanNames, + ...(scenario === HEAVY_JOURNEY + ? { + scrollsPerDirection: Math.min(fixture.scrolls, 4), + scrollDurationMs: 120, + scrollTimeMeasured: false, + androidScrollFrameHealthMeasured: true, + iosScrollFrameHealthMeasured: false, + } + : {}), + sourceRevision: capture('git', ['rev-parse', 'HEAD']).trim(), + mobileRevision: capture('git', ['-C', 'Mobile-Expensify', 'rev-parse', 'HEAD']).trim(), + artifacts: adapter.artifactPaths, + artifactSHA256: { + release: await sha256Artifact(adapter.artifactPaths.release), + optimized: await sha256Artifact(adapter.artifactPaths.optimized), + }, + }, + null, + 2, + )}\n`, + ); + console.log(`${scenario.directory} benchmark batch: ${directory}`); + const phases: BenchmarkKind[] = order === 'release-first' ? ['release', 'optimized'] : ['optimized', 'release']; + for (const [index, kind] of phases.entries()) { + console.log(`=== ${scenario.directory} benchmark phase ${index + 1}/2: ${kind} ===`); + await benchmarkJourneyBuild(adapter, kind, deviceID, fixture, runs, timeoutSeconds, paths[kind], scenario); + } + await compareJourneyBenchmarks(paths, runs, scenario, adapter.name); +} + +async function compareJourneyBenchmarks(paths: Record, runs: number, scenario: JourneyScenario, platform: PlatformAdapter['name']): Promise { + if (!existsSync(paths.release) || !existsSync(paths.optimized)) { + fail('Missing paired journey samples in this benchmark batch.'); + } + const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(paths.release), readBenchmarkSamples(paths.optimized)]); + const release = benchmarkMetrics(releaseSamples, scenario.spanNames); + const optimized = benchmarkMetrics(optimizedSamples, scenario.spanNames); + + console.log(`=== ${scenario.directory} benchmark comparison (positive percentages are faster) ===`); + for (const span of scenario.spanNames) { + const baseline = release[span]?.stats; + const pgo = optimized[span]?.stats; + if (!baseline || !pgo || baseline.runs !== runs || pgo.runs !== runs) { + fail(`Expected ${runs} paired ${span} samples in this journey benchmark batch.`); + } + const improvement = ((baseline.average - pgo.average) / baseline.average) * 100; + console.log(`${span}: release ${baseline.average.toFixed(2)} ms, PGO ${pgo.average.toFixed(2)} ms, ${improvement.toFixed(2)}% (${baseline.runs}/${pgo.runs} runs)`); + } + if (scenario === HEAVY_JOURNEY && platform === 'android') { + const releaseFrames = readAndroidFrameSamples(paths.release.replace(/\.csv$/, '-frames.csv'), runs); + const optimizedFrames = readAndroidFrameSamples(paths.optimized.replace(/\.csv$/, '-frames.csv'), runs); + console.log(`Android scroll missed-frame share: release ${releaseFrames.toFixed(2)}%, PGO ${optimizedFrames.toFixed(2)}% (lower is better; separate from report-open latency).`); + } + console.log(`Raw journey samples: ${paths.release} and ${paths.optimized}`); +} + +function readAndroidFrameSamples(path: string, runs: number): number { + if (!existsSync(path)) { + fail(`Missing Android scroll frame samples: ${path}`); + } + const rows = readFileSync(path, 'utf8').trim().split('\n'); + if (rows.shift() !== 'run,totalFrames,droppedFrames,droppedPercent,sampleWindowMs' || rows.length !== runs) { + fail(`Expected ${runs} Android scroll frame samples in ${path}.`); + } + let totalFrames = 0; + let droppedFrames = 0; + for (const [index, row] of rows.entries()) { + const values = row.split(',').map(Number); + const runNumber = values.at(0) ?? NaN; + const frames = values.at(1) ?? NaN; + const dropped = values.at(2) ?? NaN; + if (values.length !== 5 || values.some((value) => !Number.isFinite(value)) || runNumber !== index + 1 || frames <= 0 || dropped < 0 || dropped > frames) { + fail(`Invalid Android scroll frame sample ${index + 1} in ${path}.`); + } + totalFrames += frames; + droppedFrames += dropped; + } + return (droppedFrames / totalFrames) * 100; +} + +async function sha256Artifact(path: string): Promise { + if (lstatSync(path).isDirectory()) { + return sha256Directory(path); + } + const hash = capture('shasum', ['-a', '256', path]).trim().split(/\s+/).at(0); + if (!hash || !/^[\da-f]{64}$/.test(hash)) { + fail(`Could not record the benchmark artifact SHA-256 for ${path}.`); + } + return hash; +} + +/** Include every bundle file, relative path, directory and symlink in a deterministic iOS artifact digest. */ +async function sha256Directory(directory: string): Promise { + const hash = createHash('sha256'); + async function visit(path: string): Promise { + for (const name of readdirSync(path).toSorted()) { + const child = join(path, name); + const details = lstatSync(child); + hash.update(relative(directory, child)).update('\0'); + if (details.isDirectory()) { + hash.update('directory\0'); + await visit(child); + } else if (details.isSymbolicLink()) { + hash.update('link\0').update(readlinkSync(child)).update('\0'); + } else if (details.isFile()) { + hash.update('file\0').update(String(details.size)).update('\0'); + for await (const chunk of createReadStream(child)) { + const bytes: unknown = chunk; + if (!Buffer.isBuffer(bytes)) { + fail(`Artifact file did not stream bytes: ${child}`); + } + hash.update(bytes); + } + } else { + fail(`Unsupported artifact entry in ${directory}: ${child}`); + } + } + } + await visit(directory); + return hash.digest('hex'); +} + +export {benchmarkHeavyJourneyAll, benchmarkJourneyAll, journeyBenchmarkPaths, sha256Artifact}; +export type {BenchmarkOrder}; diff --git a/scripts/pgo/journeyBenchmarkAndroid.ts b/scripts/pgo/journeyBenchmarkAndroid.ts new file mode 100644 index 000000000000..9aa305e40acf --- /dev/null +++ b/scripts/pgo/journeyBenchmarkAndroid.ts @@ -0,0 +1,52 @@ +/** Collects completed interactive benchmark spans from the current Android app process. */ + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {POLL_INTERVAL_MS, parseBenchmarkLogEvents, sleep} from '../lib/nativeAppBenchmark/shared'; +import {capture, run} from '../lib/scriptUtils'; + +function createAndroidJourneyBenchmarkReader(deviceID: string, appID: string) { + const adbArgs = (args: string[]) => ['-s', deviceID, ...args]; + let consumedEventCount = 0; + + function begin(): void { + run('adb', adbArgs(['logcat', '-c'])); + consumedEventCount = 0; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + if (spanNames.length === 0) { + throw new Error('Select at least one Android journey benchmark span.'); + } + const processID = capture('adb', adbArgs(['shell', 'pidof', appID])) + .trim() + .split(/\s+/) + .find((candidate) => /^\d+$/.test(candidate)); + if (!processID) { + throw new Error(`Unable to find the running Android process for ${appID}.`); + } + + const deadline = Date.now() + timeoutSeconds * 1000; + let observedSpanNames: string[] = []; + while (Date.now() < deadline) { + const logs = capture('adb', adbArgs(['logcat', `--pid=${processID}`, '-d', '-v', 'raw'])); + const events = parseBenchmarkLogEvents(logs); + if (events.length < consumedEventCount) { + throw new Error('Android benchmark logcat events were discarded during collection. Restart the journey benchmark.'); + } + const freshEvents = events.slice(consumedEventCount); + const collected = spanNames.map((spanName) => freshEvents.findLast((event) => event.span === spanName)); + observedSpanNames = [...new Set(freshEvents.map((event) => event.span))]; + if (collected.every((event) => event !== undefined)) { + consumedEventCount = events.length; + return collected.filter((event): event is BenchmarkLogEvent => event !== undefined); + } + await sleep(POLL_INTERVAL_MS); + } + throw new Error(`Timed out after ${timeoutSeconds}s waiting for Android journey benchmark spans: ${spanNames.join(', ')}. Observed: ${observedSpanNames.join(', ') || 'none'}.`); + } + + return {begin, collect}; +} + +export default createAndroidJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyBenchmarkIOS.ts b/scripts/pgo/journeyBenchmarkIOS.ts new file mode 100644 index 000000000000..f6287014ed46 --- /dev/null +++ b/scripts/pgo/journeyBenchmarkIOS.ts @@ -0,0 +1,102 @@ +/** Collects completed interactive benchmark spans from an iOS app's cache markers. */ + +// cspell:ignore devicectl + +import {existsSync, mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +import type {BenchmarkLogEvent} from '../lib/nativeAppBenchmark'; + +import {iOSBenchmarkMarkerPath, parseBenchmarkLogEvents} from '../lib/nativeAppBenchmark'; +import {runAllowFailure, sleep} from '../lib/scriptUtils'; + +const POLL_INTERVAL_MS = 250; + +/** Snapshots markers before an action, then waits for a fresh event for every requested span. */ +function createIOSJourneyBenchmarkReader(deviceID: string, appID: string) { + let previousMarkers: Map | undefined; + + function begin(spanNames: readonly string[]): void { + if (spanNames.length === 0) { + throw new Error('At least one iOS benchmark span is required.'); + } + const markers = new Map(); + for (const spanName of new Set(spanNames)) { + markers.set(spanName, readMarker(deviceID, appID, spanName)); + } + previousMarkers = markers; + } + + async function collect(spanNames: readonly string[], timeoutSeconds: number): Promise { + const snapshot = previousMarkers; + if (spanNames.length === 0 || !snapshot || spanNames.some((spanName) => !snapshot.has(spanName))) { + throw new Error('Snapshot every requested iOS benchmark span with begin() before the UI action.'); + } + if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) { + throw new Error('The iOS benchmark span timeout must be positive.'); + } + + const uniqueSpanNames = [...new Set(spanNames)]; + const deadline = Date.now() + timeoutSeconds * 1000; + const events = new Map(); + while (Date.now() < deadline) { + for (const spanName of uniqueSpanNames) { + if (events.has(spanName)) { + continue; + } + const marker = readMarker(deviceID, appID, spanName); + if (marker === undefined || marker === snapshot.get(spanName)) { + continue; + } + const event = parseBenchmarkLogEvents(marker).findLast((candidate) => candidate.span === spanName); + if (event) { + events.set(spanName, event); + } + } + if (events.size === uniqueSpanNames.length) { + return spanNames.map((spanName) => { + const event = events.get(spanName); + if (!event) { + throw new Error(`Missing collected iOS benchmark span ${spanName}.`); + } + return event; + }); + } + await sleep(POLL_INTERVAL_MS); + } + const missing = uniqueSpanNames.filter((spanName) => !events.has(spanName)); + throw new Error(`Timed out after ${timeoutSeconds}s waiting for iOS benchmark span(s): ${missing.join(', ')}.`); + } + + return {begin, collect}; +} + +function readMarker(deviceID: string, appID: string, spanName: string): string | undefined { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'expensify-journey-benchmark-ios-')); + const localPath = join(temporaryDirectory, 'benchmark.log'); + try { + const copied = runAllowFailure('xcrun', [ + 'devicectl', + 'device', + 'copy', + 'from', + '--device', + deviceID, + '--source', + iOSBenchmarkMarkerPath(spanName), + '--destination', + localPath, + '--domain-type', + 'appDataContainer', + '--domain-identifier', + appID, + '--quiet', + ]); + return copied && existsSync(localPath) ? readFileSync(localPath, 'utf8') : undefined; + } finally { + rmSync(temporaryDirectory, {recursive: true, force: true}); + } +} + +export default createIOSJourneyBenchmarkReader; diff --git a/scripts/pgo/journeyConfig.ts b/scripts/pgo/journeyConfig.ts new file mode 100644 index 000000000000..8a6f0cb0fe6a --- /dev/null +++ b/scripts/pgo/journeyConfig.ts @@ -0,0 +1,71 @@ +import {isRecord} from '@libs/ObjectUtils'; + +type JourneyReport = { + query: string; + resultLabelPrefix: string; + title: string; +}; + +type JourneyFixture = { + accountEmail: string; + accountClass: 'heavy'; + description: string; + report: JourneyReport; + personalChat: JourneyReport; + allowMessages: boolean; + scrolls: number; + tabCycles: number; +}; + +/** Require an explicitly selected heavy account instead of silently training on whoever is signed in. */ +function parseJourneyFixture(input: unknown): JourneyFixture { + if (!isRecord(input) || input.accountClass !== 'heavy') { + throw new Error('The journey requires a fixture with accountClass="heavy", approved by the account owner.'); + } + const accountEmail = requiredString(input.accountEmail, 'accountEmail'); + if (!accountEmail.includes('@')) { + throw new Error('accountEmail must identify the approved heavy account.'); + } + const personalChat = parseReport(input.personalChat, 'personalChat'); + if (personalChat.query !== accountEmail || !personalChat.title.endsWith('(you)') || personalChat.resultLabelPrefix !== `${personalChat.title}, ${accountEmail}`) { + throw new Error('personalChat must be the approved account\'s own "(you)" chat, with its email as query and its title and email in resultLabelPrefix.'); + } + return { + accountEmail, + accountClass: 'heavy', + description: requiredString(input.description, 'description'), + report: parseReport(input.report, 'report'), + personalChat, + allowMessages: input.allowMessages === true, + scrolls: boundedInteger(input.scrolls ?? 8, 'scrolls', 4, 30), + tabCycles: boundedInteger(input.tabCycles ?? 3, 'tabCycles', 2, 10), + }; +} + +function parseReport(input: unknown, name: string): JourneyReport { + if (!isRecord(input)) { + throw new Error(`${name} must specify query, resultLabelPrefix, and title.`); + } + return { + query: requiredString(input.query, `${name}.query`), + resultLabelPrefix: requiredString(input.resultLabelPrefix, `${name}.resultLabelPrefix`), + title: requiredString(input.title, `${name}.title`), + }; +} + +function requiredString(input: unknown, name: string): string { + if (typeof input !== 'string' || input.trim().length === 0) { + throw new Error(`${name} must be a nonempty string.`); + } + return input.trim(); +} + +function boundedInteger(input: unknown, name: string, minimum: number, maximum: number): number { + if (typeof input !== 'number' || !Number.isSafeInteger(input) || input < minimum || input > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`); + } + return input; +} + +export {parseJourneyFixture}; +export type {JourneyFixture, JourneyReport}; diff --git a/scripts/pgo/journeyDevice.ts b/scripts/pgo/journeyDevice.ts new file mode 100644 index 000000000000..89dc251fa1e2 --- /dev/null +++ b/scripts/pgo/journeyDevice.ts @@ -0,0 +1,148 @@ +import {isRecord} from '@libs/ObjectUtils'; + +import {spawnSync} from 'node:child_process'; + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +type JourneyNode = { + label: string; + value: string; + identifier: string; + type: string; + enabled: boolean; + width: number; + height: number; + x: number; + y: number; +}; + +type JourneyDevice = ReturnType; + +/** Execute deterministic device commands. Authentication is always handled by the user. */ +function createJourneyDevice(options: {platform: PlatformName; device: string; appID: string; session: string}) { + const targetArguments = ['--platform', options.platform, options.platform === 'android' ? '--serial' : '--udid', options.device, '--session', options.session, '--json']; + + function command(...args: string[]): unknown { + const result = spawnSync('agent-device', [...args, ...targetArguments], {encoding: 'utf8', timeout: 120_000, maxBuffer: 16 * 1024 * 1024}); + if (result.error) { + throw new Error(`Device command ${args.at(0)} failed: ${result.error.message}`); + } + let response: unknown; + try { + response = JSON.parse(result.stdout); + } catch { + throw new Error(`Device command ${args.at(0)} did not return JSON: ${result.stderr.trim()}`); + } + if (!isRecord(response) || response.success !== true || result.status !== 0) { + const error = isRecord(response) && isRecord(response.error) ? response.error : undefined; + throw new Error(`Device command ${args.at(0)} ${args.at(1) ?? ''} failed: ${typeof error?.message === 'string' ? error.message : result.stderr.trim()}`); + } + return response.data; + } + + function snapshot(): JourneyNode[] { + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const nodes = parseJourneySnapshot(command('snapshot', '-i', '--force-full'), options.appID); + assertSignedIn(nodes); + return nodes; + } catch (error) { + // The Android accessibility service can briefly return only system UI while a heavy list mounts. + // Retry this read alone; repeating a tap or send could perform the action twice. + if (attempt === 2 || !(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + command('wait', '500'); + } + } + throw new Error('Unable to read the app accessibility tree.'); + } + + function wait(selector: string): void { + try { + command('wait', selector, '30000'); + } catch (error) { + // Give sign-in a specific, actionable failure instead of continuing through an expired session. + snapshot(); + throw error; + } + } + + function press(selector: string): void { + wait(selector); + command('press', selector, '--settle'); + } + + function labels(): string[] { + return snapshot().map((node) => normalizeLabel(node.label)); + } + + function hasLabel(label: string): boolean { + return labels().includes(normalizeLabel(label)); + } + + return { + platform: options.platform, + command, + snapshot, + wait, + press, + hasLabel, + pressLabel: (label: string) => press(labelSelector(label)), + waitLabel: (label: string) => wait(labelSelector(label)), + fill: (identifier: string, text: string) => { + wait(idSelector(identifier)); + command('fill', idSelector(identifier), text, '--settle'); + }, + back: () => command('back', '--settle'), + open: (relaunch: boolean) => command('open', options.appID, ...(relaunch ? ['--relaunch'] : []), ...(options.platform === 'android' ? ['--test-ime'] : [])), + }; +} + +function parseJourneySnapshot(data: unknown, appID: string): JourneyNode[] { + if (!isRecord(data) || !Array.isArray(data.nodes)) { + throw new Error('The device did not return an accessibility tree.'); + } + if (data.appBundleId !== appID) { + throw new Error('The automation is no longer attached to the expected app.'); + } + return data.nodes + .filter(isRecord) + .filter((node) => !node.bundleId || node.bundleId === appID) + .map((node) => ({ + label: typeof node.label === 'string' ? node.label : '', + value: typeof node.value === 'string' ? node.value : '', + identifier: typeof node.identifier === 'string' ? node.identifier : '', + type: typeof node.type === 'string' ? node.type : '', + enabled: node.enabled !== false, + width: isRecord(node.rect) && typeof node.rect.width === 'number' ? node.rect.width : 0, + height: isRecord(node.rect) && typeof node.rect.height === 'number' ? node.rect.height : 0, + x: isRecord(node.rect) && typeof node.rect.x === 'number' ? node.rect.x : 0, + y: isRecord(node.rect) && typeof node.rect.y === 'number' ? node.rect.y : 0, + })); +} + +function assertSignedIn(nodes: JourneyNode[]): void { + const signInLabels = new Set(['Sign in', 'Email or phone number', 'Enter your email or phone number', 'Enter your magic code']); + if (nodes.some((node) => ['username', 'validateCode'].includes(node.identifier) || signInLabels.has(normalizeLabel(node.label)))) { + throw new Error('SIGN_IN_REQUIRED: Ask Chris to sign into the approved heavy account, then restart the journey. Automated sign-in is disabled.'); + } +} + +function normalizeLabel(label: string): string { + return label + .replaceAll(/[\u2066-\u2069]/g, '') + .replaceAll(/\s+/g, ' ') + .trim(); +} + +function labelSelector(label: string): string { + return `label=${JSON.stringify(label)}`; +} + +function idSelector(identifier: string): string { + return `id=${JSON.stringify(identifier)}`; +} + +export {assertSignedIn, createJourneyDevice, idSelector, labelSelector, normalizeLabel, parseJourneySnapshot}; +export type {JourneyDevice, JourneyNode}; diff --git a/scripts/pgo/journeyWorkload.ts b/scripts/pgo/journeyWorkload.ts new file mode 100644 index 000000000000..c126050d20d2 --- /dev/null +++ b/scripts/pgo/journeyWorkload.ts @@ -0,0 +1,406 @@ +import type {JourneyFixture, JourneyReport} from './journeyConfig'; +import type {JourneyDevice, JourneyNode} from './journeyDevice'; + +import {sleep} from '../lib/scriptUtils'; +import {idSelector, normalizeLabel} from './journeyDevice'; + +/** Run only after the user has signed into and approved the fixture's heavy account. */ +async function runJourneyWorkload(device: JourneyDevice, fixture: JourneyFixture, runID: string, navigationOnly = false): Promise { + console.log('Scrolling the full inbox.'); + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + selectAllInboxFilter(device); + await scrollBothWays(device, fixture.scrolls, true); + + console.log('Opening and scrolling a populated report.'); + await openReport(device, fixture.report); + await scrollBothWays(device, fixture.scrolls, true, 'up'); + + console.log(navigationOnly ? 'Opening the personal chat without sending.' : 'Composing and sending messages in the personal chat.'); + await openReport(device, fixture.personalChat, true); + for (let messageNumber = 1; messageNumber <= (navigationOnly ? 0 : 2); messageNumber += 1) { + // Check the destination immediately before every write. Search-result matching alone is insufficient. + if (!device.hasLabel(fixture.personalChat.title)) { + throw new Error('The personal chat title is not visible before sending.'); + } + const composer = device.snapshot().find((node) => node.identifier === 'composer'); + if (!composer) { + throw new Error('The personal chat composer is missing.'); + } + if (composer.value && composer.value !== 'Write something...') { + throw new Error('The personal chat contains an existing draft. Clear or send it manually before running the journey.'); + } + const token = `PGO ${runID} message ${messageNumber}`; + if (device.platform === 'android') { + device.fill('composer', `${token} draft`); + } + device.fill('composer', `${token}. Answered by Codex, instructed by Chris.`); + device.pressLabel('Send'); + await waitForMessage(device, token); + } + + console.log('Scrolling Spend expenses and reports.'); + await showTab(device, 'Spend'); + selectSpendSection(device, 'Expenses'); + device.waitLabel('Filters'); + await scrollBothWays(device, fixture.scrolls, true); + selectSpendSection(device, 'Reports'); + device.waitLabel('Filters'); + await scrollBothWays(device, Math.max(4, Math.floor(fixture.scrolls / 2)), true); + selectSpendSection(device, 'Expenses'); + + console.log('Switching tabs and reopening the report.'); + for (let cycle = 0; cycle < fixture.tabCycles; cycle += 1) { + for (const tab of ['Inbox', 'Spend', 'Workspaces', 'Inbox']) { + await showTab(device, tab); + } + } + await openReport(device, fixture.report); + await showTab(device, 'Inbox'); +} + +/** Check identity before changing preferences; never sign in or switch accounts automatically. */ +async function prepareJourney(device: JourneyDevice, fixture: JourneyFixture): Promise { + // Entering Inbox first allows the app's automatic large-account focus prompt to appear before we check preferences. + await showTab(device, 'Inbox'); + if (device.hasLabel('No thanks')) { + device.pressLabel('No thanks'); + } + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + if (!device.hasLabel(fixture.accountEmail)) { + throw new Error(`SIGN_IN_REQUIRED: Ask Chris to sign into ${fixture.accountEmail}. The journey cannot run on another account.`); + } + pressPreferences(device); + await ensureFocusDisabled(device, true); + pressInAppBack(device); + await showTab(device, 'Inbox'); + console.log('Verified the approved heavy account and disabled #focus.'); +} + +/** End-of-run verification rejects a profile if focus was enabled during the workload. */ +async function verifyJourneyAccount(device: JourneyDevice, fixture: JourneyFixture): Promise { + await showTab(device, 'Account'); + device.command('scroll', 'top', '--settle'); + if (!device.hasLabel(fixture.accountEmail)) { + throw new Error(`The account changed during the journey. Expected ${fixture.accountEmail}; discard this run.`); + } + pressPreferences(device); + await ensureFocusDisabled(device, false); + pressInAppBack(device); + await showTab(device, 'Inbox'); +} + +async function ensureFocusDisabled(device: JourneyDevice, allowChange: boolean): Promise { + if (device.hasLabel('Priority mode, Most recent')) { + return; + } + if (!device.hasLabel('Priority mode, #focus')) { + throw new Error('Cannot verify Priority mode. The journey requires English UI labels and #focus disabled.'); + } + if (!allowChange) { + throw new Error('#focus became enabled during the journey. Discard this run and restart after disabling it.'); + } + device.pressLabel('Priority mode, #focus'); + device.pressLabel('Most recent'); + device.pressLabel('Save'); + device.waitLabel('Priority mode, Most recent'); +} + +async function openReport(device: JourneyDevice, report: JourneyReport, isPersonalChat = false, beforePress?: () => void): Promise { + await showTab(device, 'Inbox'); + device.press(idSelector('searchButton')); + device.fill('search-autocomplete-text-input', report.query); + const deadline = Date.now() + 30_000; + let resultLabel: string | undefined; + while (!resultLabel && Date.now() < deadline) { + resultLabel = findReportResult(device.snapshot(), report.resultLabelPrefix, isPersonalChat ? report.title : undefined); + if (!resultLabel) { + await sleep(500); + } + } + if (!resultLabel) { + throw new Error(`No search result matched ${report.resultLabelPrefix}.`); + } + beforePress?.(); + device.pressLabel(resultLabel); + device.wait(idSelector('composer')); + if (!device.hasLabel(report.title)) { + throw new Error(`The opened report does not show the expected title ${report.title}.`); + } +} + +async function showTab(device: JourneyDevice, tab: string): Promise { + const deadline = Date.now() + 30_000; + let backPresses = 0; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const tabNode = findTabNode(nodes, tab); + if (tabNode) { + device.command('press', String(Math.round(tabNode.x + tabNode.width / 2)), String(Math.round(tabNode.y + tabNode.height / 2)), '--settle'); + return; + } + // A relaunch can expose a loading view before tabs mount. Only navigate back from a real nested page. + if (backPresses < 5 && inAppBackTapPoint(nodes)) { + pressInAppBack(device, nodes); + backPresses += 1; + } + await sleep(500); + } + throw new Error(`Cannot return to the ${tab} tab.`); +} + +/** Wait for a mounted bottom tab without navigating away from a partially loaded screen. */ +async function waitForTab(device: Pick, tab: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + if (findTabNode(device.snapshot(), tab)) { + return; + } + } catch (error) { + if (!(error instanceof Error) || !error.message.includes('insufficient foreground app content')) { + throw error; + } + } + await sleep(300); + } + throw new Error(`Timed out waiting for the ${tab} bottom tab to mount.`); +} + +function selectAllInboxFilter(device: JourneyDevice): void { + const tapPoint = allFilterTapPoint(device.snapshot()); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel('All'); +} + +function pressPreferences(device: JourneyDevice): void { + const cell = device.snapshot().filter((node) => normalizeLabel(node.label) === 'Preferences' && node.type === 'Cell' && node.width > 0 && node.height > 0); + const preferenceCell = cell.length === 1 ? cell.at(0) : undefined; + if (preferenceCell) { + device.command('press', String(Math.round(preferenceCell.x + preferenceCell.width / 2)), String(Math.round(preferenceCell.y + preferenceCell.height / 2)), '--settle'); + return; + } + device.pressLabel('Preferences'); +} + +function pressInAppBack(device: JourneyDevice, nodes = device.snapshot()): void { + const tapPoint = inAppBackTapPoint(nodes); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.back(); +} + +/** Use the app's top-left Back control; it can be non-hittable on iOS while the keyboard is open. */ +function inAppBackTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); + if (!screen) { + return undefined; + } + const candidates = nodes.filter( + (node) => + (node.type === 'Button' || node.type === 'android.widget.Button') && + normalizeLabel(node.label) === 'Back' && + node.enabled && + node.width > 0 && + node.height > 0 && + node.x >= 0 && + node.y >= 0 && + node.x < screen.width * 0.25 && + node.y < screen.height * 0.2, + ); + if (candidates.length > 1) { + throw new Error('Ambiguous in-app Back control.'); + } + const back = candidates.at(0); + return back ? {x: Math.round(back.x + back.width / 2), y: Math.round(back.y + back.height / 2)} : undefined; +} + +function selectSpendSection(device: JourneyDevice, section: 'Expenses' | 'Reports'): void { + const tapPoint = spendSectionTapPoint(device.snapshot(), section); + if (tapPoint) { + device.command('press', String(tapPoint.x), String(tapPoint.y), '--settle'); + return; + } + device.pressLabel(section); +} + +/** On iOS, Expenses labels the whole horizontal strip while Reports has its own bounds. */ +function spendSectionTapPoint(nodes: JourneyNode[], section: 'Expenses' | 'Reports'): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'Expenses' && node.type === 'Cell' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const reports = nodes.find((node) => normalizeLabel(node.label) === 'Reports' && node.x > 0 && node.height > 0); + if (!reports) { + throw new Error('Cannot locate the Expenses and Reports segments in Spend.'); + } + return {x: Math.round(section === 'Reports' ? reports.x + reports.width / 2 : reports.x / 2), y: Math.round(reports.y + reports.height / 2)}; +} + +/** iOS exposes the whole filter strip as the All cell; its center lands on Unread. */ +function allFilterTapPoint(nodes: JourneyNode[]): {x: number; y: number} | undefined { + const screenWidth = nodes.find((node) => node.type === 'Application')?.width; + if (!screenWidth || !nodes.some((node) => normalizeLabel(node.label) === 'All' && node.width >= screenWidth * 0.8)) { + return undefined; + } + const unread = nodes.find((node) => normalizeLabel(node.label) === 'Unread' && node.x > 0 && node.height > 0); + if (!unread) { + throw new Error('Cannot locate the All filter inside the iOS Inbox strip.'); + } + return {x: Math.round(unread.x / 2), y: Math.round(unread.y + unread.height / 2)}; +} + +/** iOS appends unread/review status to tab accessibility labels. */ +function findTabNode(nodes: JourneyNode[], tab: string): JourneyNode | undefined { + const screen = + nodes.find((node) => node.type === 'Application') ?? + nodes + .filter((node) => node.x === 0 && node.y === 0 && node.width > 0 && node.height > 0) + .toSorted((left, right) => right.width * right.height - left.width * left.height) + .at(0); + const candidates = nodes.filter((node) => { + const label = normalizeLabel(node.label); + const matches = label === tab || label.startsWith(`${tab}.`) || label.startsWith(`${tab},`); + return matches && (!screen || (node.y >= screen.height * 0.7 && node.width >= screen.width * 0.1 && node.height >= screen.height * 0.05)); + }); + const first = candidates.at(0); + const duplicatedNode = + first && + candidates.every( + (node) => + node.type === first.type && + normalizeLabel(node.label) === normalizeLabel(first.label) && + node.x === first.x && + node.y === first.y && + node.width === first.width && + node.height === first.height, + ); + if (candidates.length > 1 && !duplicatedNode) { + throw new Error(`Ambiguous ${tab} tab in the accessibility tree.`); + } + return first; +} + +async function scrollBothWays(device: JourneyDevice, count: number, requireMovement: boolean, firstDirection: 'up' | 'down' = 'down'): Promise { + const initialNodes = device.snapshot(); + const pixels = scrollDistance(initialNodes); + let previous = contentSignature(initialNodes); + let changed = 0; + for (const direction of [firstDirection, firstDirection === 'up' ? 'down' : 'up']) { + let stationary = 0; + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', String(pixels), '--settle'); + const current = contentSignature(device.snapshot()); + if (current !== previous) { + changed += 1; + stationary = 0; + } else { + stationary += 1; + } + previous = current; + if (stationary === 2) { + break; + } + } + } + if (requireMovement && changed < 2) { + throw new Error('The list did not expose enough changing content. Use a populated heavy-account fixture; do not collect this run.'); + } +} + +/** Fixed rapid swipes with one movement check per direction, keeping accessibility polling out of each fling. */ +function scrollFastBothWays(device: Pick, count: number): void { + const initialNodes = device.snapshot(); + const pixels = String(scrollDistance(initialNodes, true)); + let previous = contentSignature(initialNodes); + for (const direction of ['up', 'down']) { + for (let scroll = 0; scroll < count; scroll += 1) { + device.command('scroll', direction, '--pixels', pixels, '--duration-ms', '120', ...(scroll === count - 1 ? ['--settle'] : [])); + } + const current = contentSignature(device.snapshot()); + if (current === previous) { + throw new Error(`The populated report did not move during fast ${direction} scrolling. Discard this benchmark run.`); + } + previous = current; + } +} + +async function waitForMessage(device: JourneyDevice, token: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const nodes = device.snapshot(); + const composer = nodes.find((node) => node.identifier === 'composer'); + const visibleMessage = nodes.some((node) => node.identifier !== 'composer' && !/EditText|TextField|TextViewInput/i.test(node.type) && node.label.includes(token)); + if (composer && !composer.value.includes(token) && visibleMessage) { + return; + } + await sleep(500); + } + throw new Error('The sent message did not appear with an empty composer. Do not retry the send automatically.'); +} + +function contentSignature(nodes: JourneyNode[]): string { + const screenHeight = nodes.find((node) => node.type === 'Application')?.height; + return nodes + .filter((node) => !screenHeight || (node.y + node.height > screenHeight * 0.13 && node.y < screenHeight * 0.88)) + .filter((node) => !!normalizeLabel(node.label)) + .map((node) => `${normalizeLabel(node.label)}:${Math.round(node.y / 8)}`) + .join('\n'); +} + +/** Keep both ends of the gesture inside the center of portrait lists, below Spend's fixed header. */ +function scrollDistance(nodes: JourneyNode[], fast = false): number { + const scrollAreas = nodes.filter((node) => /ScrollView|RecyclerView|ScrollArea/i.test(node.type) && node.width > 0 && node.height > 0); + const viewport = scrollAreas.toSorted((left, right) => right.width * right.height - left.width * left.height).at(0); + if (!viewport) { + throw new Error('Cannot determine scroll bounds on this device.'); + } + return Math.round(Math.min(viewport.height * (fast ? 0.65 : 0.25), viewport.width * (fast ? 1.5 : 0.6))); +} + +/** Match stable report identity while allowing the last-message preview to change. */ +function findReportResult(nodes: JourneyNode[], prefix: string, personalChatTitle?: string): string | undefined { + const identity = normalizeLabel(prefix); + // The fixture binds this title to the signed-in account and an exact email search. A populated self-chat replaces the email subtitle with its last message. + const personalPrefix = personalChatTitle ? `${normalizeLabel(personalChatTitle)}, ` : undefined; + const matches = nodes.filter((node) => { + const label = normalizeLabel(node.label); + return label === identity || label.startsWith(`${identity} •`) || (personalPrefix !== undefined && label.startsWith(personalPrefix)); + }); + if (matches.length > 1) { + throw new Error(`Ambiguous report search: ${prefix}. Use a more specific fixture.`); + } + return matches.at(0)?.label; +} + +export { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + openReport, + prepareJourney, + runJourneyWorkload, + scrollBothWays, + scrollFastBothWays, + scrollDistance, + selectSpendSection, + showTab, + spendSectionTapPoint, + verifyJourneyAccount, + waitForTab, +}; diff --git a/scripts/pgo/pgo.ts b/scripts/pgo/pgo.ts new file mode 100755 index 000000000000..88af0b143175 --- /dev/null +++ b/scripts/pgo/pgo.ts @@ -0,0 +1,339 @@ +#!/usr/bin/env bun + +// cspell:ignore profraw Profdata profdata + +import type {TupleToUnion} from 'type-fest'; + +import CLI from 'expensify-common/CLI'; +import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; +import {resolve} from 'node:path'; + +import type {BenchmarkStats} from '../lib/benchmarkStatistics'; +import type {NativeAppBenchmarkAdapter, PlatformName} from '../lib/nativeAppBenchmark'; +import type {BenchmarkOrder} from './journeyBenchmark'; +import type {BenchmarkKind, PlatformAdapter} from './shared'; + +import {benchmarkStartups as runBenchmarkStartups} from '../lib/benchmarkAppStartup'; +import {benchmarkMetrics, readBenchmarkSamples} from '../lib/benchmarkStatistics'; +import {PLATFORM_NAMES, createNativeAppBenchmarkAdapter} from '../lib/nativeAppBenchmark'; +import {capture, fail, findFiles, parseChoice, parsePositiveInteger, requirePositiveInteger, rootDirectory, run} from '../lib/scriptUtils'; +import createAndroidPgoAdapter from './android'; +import createIOSPgoAdapter from './ios'; +import {benchmarkHeavyJourneyAll, benchmarkJourneyAll} from './journeyBenchmark'; +import {parseJourneyFixture} from './journeyConfig'; +import {STARTUP_SPAN_NAME} from './shared'; + +const DEFAULT_STARTUP_RUNS = 10; +const DEFAULT_STARTUP_WAIT_SECONDS = 30; +const BENCHMARK_ORDERS = ['release-first', 'optimized-first'] as const; +const WORKFLOW_COMMANDS = [ + 'build-release', + 'build-instrumented', + 'build-optimized', + 'verify-instrumented', + 'install-release', + 'install-instrumented', + 'install-optimized', + 'record-startups', + 'benchmark-release', + 'benchmark-optimized', + 'benchmark', + 'compare-benchmarks', + 'benchmark-journey', + 'benchmark-heavy-journey', + 'dump', + 'pull', + 'merge', +] as const; + +type WorkflowCommand = TupleToUnion; + +async function main(): Promise { + // The CLI framework requires kebab-case named argument keys, which the naming-convention rule cannot express. + /* eslint-disable @typescript-eslint/naming-convention */ + const cli = new CLI({ + positionalArgs: [ + { + name: 'platform', + description: `Native platform to target (${PLATFORM_NAMES.join(', ')})`, + parse: (value): PlatformName => parseChoice(value, PLATFORM_NAMES, 'Platform'), + }, + { + name: 'workflow', + description: `PGO workflow to run (${WORKFLOW_COMMANDS.join(', ')})`, + parse: (value): WorkflowCommand => parseChoice(value, WORKFLOW_COMMANDS, 'Workflow'), + }, + { + name: 'runs', + description: 'Number of measured startup or journey runs', + default: DEFAULT_STARTUP_RUNS, + parse: (value) => parsePositiveInteger(value, 'Run count'), + }, + { + name: 'timeout', + description: `Seconds to wait for the ${STARTUP_SPAN_NAME} or journey benchmark spans`, + default: DEFAULT_STARTUP_WAIT_SECONDS, + parse: (value) => parsePositiveInteger(value, 'Span timeout'), + }, + ], + namedArgs: { + 'app-id': { + description: 'Application ID or bundle identifier; defaults to the bootstrapped release identifier', + required: false, + }, + device: { + description: 'Device identifier to use (adb serial on Android; CoreDevice identifier, UDID, serial number, or device name on iOS)', + required: false, + }, + fixture: { + description: 'Approved account fixture for journey benchmarks', + required: false, + }, + 'benchmark-order': { + description: 'Journey phase order; alternate it across paired batches to check build-order drift', + required: false, + }, + profile: { + description: 'Immutable merged profile to use when building the optimized app', + required: false, + }, + }, + }); + /* eslint-enable @typescript-eslint/naming-convention */ + + const {platform: platformName, workflow, runs, timeout} = cli.positionalArgs; + await runWorkflow( + parseChoice(String(platformName), PLATFORM_NAMES, 'Platform'), + parseChoice(String(workflow), WORKFLOW_COMMANDS, 'Workflow'), + requirePositiveInteger(Number(runs), 'Run count'), + requirePositiveInteger(Number(timeout), 'Span timeout'), + cli.namedArgs['app-id'], + cli.namedArgs.device, + cli.namedArgs.fixture, + cli.namedArgs.profile, + cli.namedArgs['benchmark-order'], + ); +} + +async function runWorkflow( + platformName: PlatformName, + workflow: WorkflowCommand, + runs: number, + timeoutSeconds: number, + appID?: string, + deviceIdentifier?: string, + fixturePath?: string, + profilePath?: string, + benchmarkOrder?: string, +): Promise { + const adapter = getAdapter(platformName, appID, deviceIdentifier); + + switch (workflow) { + case 'build-release': + adapter.build('release'); + return; + case 'build-instrumented': + adapter.build('instrumented'); + return; + case 'build-optimized': + adapter.build('optimized', profilePath ? resolve(profilePath) : undefined); + return; + case 'verify-instrumented': + adapter.verifyInstrumentation(); + return; + case 'install-release': + adapter.install('release'); + return; + case 'install-instrumented': + adapter.install('instrumented'); + return; + case 'install-optimized': + adapter.install('optimized'); + return; + case 'record-startups': + await recordStartups(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), runs, timeoutSeconds); + return; + case 'benchmark-release': + adapter.install('release'); + await benchmarkBuild(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), 'release', runs, timeoutSeconds); + return; + case 'benchmark-optimized': + adapter.install('optimized'); + await benchmarkBuild(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), 'optimized', runs, timeoutSeconds); + return; + case 'benchmark': + await benchmarkAll(adapter, await createBenchmarkAdapter(adapter, deviceIdentifier), runs, timeoutSeconds); + return; + case 'compare-benchmarks': + await compareBenchmarks(adapter); + return; + case 'benchmark-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + } + case 'benchmark-heavy-journey': { + const benchmarkAdapter = await createBenchmarkAdapter(adapter, deviceIdentifier); + await benchmarkHeavyJourneyAll(adapter, benchmarkAdapter.deviceIdentifier, readJourneyFixture(fixturePath), runs, timeoutSeconds, readBenchmarkOrder(benchmarkOrder)); + return; + } + case 'dump': + await adapter.dumpProfiles(); + return; + case 'pull': + adapter.pullProfiles(); + return; + case 'merge': + mergeProfiles(adapter); + return; + default: + fail('Unsupported workflow.'); + } +} + +function readBenchmarkOrder(value?: string): BenchmarkOrder { + return value ? parseChoice(value, BENCHMARK_ORDERS, 'Journey benchmark order') : 'release-first'; +} + +function readJourneyFixture(fixturePath?: string): ReturnType { + if (!fixturePath) { + fail('Journey benchmarks require --fixture with the approved signed-in account.'); + } + const input: unknown = JSON.parse(readFileSync(resolve(fixturePath), 'utf8')); + return parseJourneyFixture(input); +} + +function mergeProfiles(adapter: PlatformAdapter): void { + mkdirSync(adapter.profileDirectory, {recursive: true}); + const profiles = findFiles(adapter.rawProfileDirectory, '.profraw'); + if (profiles.length === 0) { + fail('No .profraw files found. Run dump and pull first.'); + } + + const llvmProfdata = adapter.llvmTool('llvm-profdata'); + run(llvmProfdata, ['merge', `--output=${adapter.mergedProfilePath}`, ...profiles]); + if (adapter.profileFormat) { + writeFileSync(`${adapter.mergedProfilePath}.format`, `${adapter.profileFormat}\n`); + } + const profileReport = capture(llvmProfdata, ['show', '--all-functions', adapter.mergedProfilePath]); + writeFileSync(`${adapter.mergedProfilePath}.txt`, profileReport); + console.log(`Merged PGO profile: ${adapter.mergedProfilePath}`); +} + +async function measureStartup(adapter: NativeAppBenchmarkAdapter, waitTimeSeconds: number): Promise { + await adapter.prepareStartup('process'); + const events = await adapter.launchAndCollect({spanNames: [STARTUP_SPAN_NAME], waitTimeSeconds, waitUntilSpan: STARTUP_SPAN_NAME}); + const duration = events.find((event) => event.span === STARTUP_SPAN_NAME)?.durationMs; + if (duration === undefined) { + fail(`The ${STARTUP_SPAN_NAME} benchmark span did not complete. Rebuild the app through this PGO tool before collecting a profile.`); + } + return duration; +} + +async function recordStartups(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, runs: number, waitTimeSeconds: number): Promise { + if (adapter.name === 'ios') { + console.log('Launching the instrumented iOS app once so its PGO notification handlers are active.'); + await measureStartup(benchmarkAdapter, waitTimeSeconds); + } else { + await benchmarkAdapter.prepareStartup('process'); + } + await adapter.clearDeviceProfiles(); + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + console.log(`Recording cold-process startup ${runNumber}/${runs}.`); + const duration = await measureStartup(benchmarkAdapter, waitTimeSeconds); + console.log(`${STARTUP_SPAN_NAME}=${duration}ms`); + await adapter.dumpProfiles(); + } + + adapter.pullProfiles(); + mergeProfiles(adapter); +} + +async function benchmarkBuild(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, kind: BenchmarkKind, runs: number, waitTimeSeconds: number): Promise { + mkdirSync(adapter.benchmarkDirectory, {recursive: true}); + await runBenchmarkStartups(benchmarkAdapter, { + mode: 'process', + spanNames: [STARTUP_SPAN_NAME], + runs, + waitTimeSeconds, + waitUntilSpan: STARTUP_SPAN_NAME, + outputPath: adapter.benchmarkPaths[kind], + }); +} + +async function benchmarkAll(adapter: PlatformAdapter, benchmarkAdapter: NativeAppBenchmarkAdapter, runs: number, waitTimeSeconds: number): Promise { + console.log('=== Benchmark phase 1/2: Release ==='); + console.log(`Installing release artifact: ${adapter.artifactPaths.release}`); + adapter.install('release'); + await benchmarkBuild(adapter, benchmarkAdapter, 'release', runs, waitTimeSeconds); + + console.log('=== Benchmark phase 2/2: PGO optimized ==='); + await benchmarkAdapter.prepareStartup('process'); + console.log(`Installing PGO optimized artifact: ${adapter.artifactPaths.optimized}`); + adapter.install('optimized'); + await benchmarkBuild(adapter, benchmarkAdapter, 'optimized', runs, waitTimeSeconds); + + console.log('=== Benchmark comparison ==='); + await compareBenchmarks(adapter); +} + +function percentageImprovement(releaseValue: number, optimizedValue: number): number { + return ((releaseValue - optimizedValue) / releaseValue) * 100; +} + +async function compareBenchmarks(adapter: PlatformAdapter): Promise { + const releasePath = adapter.benchmarkPaths.release; + const optimizedPath = adapter.benchmarkPaths.optimized; + if (!existsSync(releasePath) || !existsSync(optimizedPath)) { + fail('Missing benchmark data. Run benchmark-release and benchmark-optimized first.'); + } + + const [releaseSamples, optimizedSamples] = await Promise.all([readBenchmarkSamples(releasePath), readBenchmarkSamples(optimizedPath)]); + const release = benchmarkMetrics(releaseSamples, [STARTUP_SPAN_NAME])[STARTUP_SPAN_NAME]?.stats; + const optimized = benchmarkMetrics(optimizedSamples, [STARTUP_SPAN_NAME])[STARTUP_SPAN_NAME]?.stats; + if (!release || !optimized) { + fail(`No ${STARTUP_SPAN_NAME} benchmark samples were found.`); + } + const numericColumns: Array> = ['average', 'p50', 'p75', 'p90', 'p95', 'p99', 'min', 'max']; + const improvements = numericColumns.map((key) => percentageImprovement(release[key], optimized[key])); + + const formatLabel = (value: string) => value.padEnd(18); + const formatCount = (value: string) => value.padStart(5); + const formatMetric = (value: string) => value.padStart(10); + const row = (label: string, count: string, values: string[]) => [formatLabel(label), formatCount(count), ...values.map(formatMetric)].join(' '); + const statsRow = (label: string, stats: BenchmarkStats) => + row( + label, + String(stats.runs), + [stats.average, stats.p50, stats.p75, stats.p90, stats.p95, stats.p99, stats.min, stats.max].map((value) => value.toFixed(2)), + ); + + console.log('Positive percentages are faster; negative percentages are regressions.'); + console.log(row('Build', 'Runs', ['Average', 'P50', 'P75', 'P90', 'P95', 'P99', 'Min', 'Max'])); + console.log(statsRow('Release', release)); + console.log(statsRow('PGO optimized', optimized)); + console.log( + row( + 'PGO improvement', + '-', + improvements.map((improvement) => `${improvement.toFixed(2)}%`), + ), + ); +} + +function getAdapter(platformName: PlatformName, appID?: string, deviceIdentifier?: string): PlatformAdapter { + return platformName === 'android' ? createAndroidPgoAdapter(appID, deviceIdentifier) : createIOSPgoAdapter(appID, deviceIdentifier); +} + +async function createBenchmarkAdapter(adapter: PlatformAdapter, deviceIdentifier?: string): Promise { + return createNativeAppBenchmarkAdapter({platform: adapter.name, rootDirectory, appID: adapter.appID(), deviceIdentifier}); +} + +if (import.meta.main) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} + +export default percentageImprovement; diff --git a/scripts/pgo/shared.ts b/scripts/pgo/shared.ts new file mode 100644 index 000000000000..bc50f922d7fe --- /dev/null +++ b/scripts/pgo/shared.ts @@ -0,0 +1,35 @@ +/** Shared PGO workflow types and constants. */ + +import type {PlatformName} from '../lib/nativeAppBenchmark'; + +const STARTUP_SPAN_NAME = 'ManualAppStartup'; +const JOURNEY_SPAN_NAMES = ['ManualNavigateToReportsFirstPaint', 'ManualNavigateToReportsContentLoad', 'ManualNavigateToInboxTab'] as const; +const HEAVY_JOURNEY_SPAN_NAMES = ['/r/*'] as const; +const BENCHMARK_SPANS_ENVIRONMENT = `EXPO_PUBLIC_BENCHMARK_SENTRY_SPANS=${[STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES].join(',')}`; + +type BuildKind = 'release' | 'instrumented' | 'optimized'; +type BenchmarkKind = Extract; +type PgoMode = 'off' | 'generate' | 'use'; +type BuildArtifactPaths = Record; + +type PlatformAdapter = { + readonly name: PlatformName; + readonly profileDirectory: string; + readonly rawProfileDirectory: string; + readonly mergedProfilePath: string; + readonly benchmarkDirectory: string; + readonly benchmarkPaths: Record; + readonly artifactPaths: BuildArtifactPaths; + readonly profileFormat?: string; + appID: () => string; + build: (kind: BuildKind, profilePath?: string) => void; + install: (kind: BuildKind) => void; + verifyInstrumentation: () => void; + clearDeviceProfiles: () => Promise; + dumpProfiles: () => Promise; + pullProfiles: () => void; + llvmTool: (name: string) => string; +}; + +export {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME}; +export type {BenchmarkKind, BuildArtifactPaths, BuildKind, PgoMode, PlatformAdapter}; diff --git a/tests/tooling/benchmarkAppStartup.test.ts b/tests/tooling/benchmarkAppStartup.test.ts index 0385db56a12a..4c213eb7ef12 100644 --- a/tests/tooling/benchmarkAppStartup.test.ts +++ b/tests/tooling/benchmarkAppStartup.test.ts @@ -1,4 +1,4 @@ -// cspell:ignore appex +// cspell:ignore appex devicectl import {describe, expect, it, jest} from 'bun:test'; @@ -7,6 +7,7 @@ import { assertAndroidAppInstalled, findBenchmarkDuration, iOSBenchmarkMarkerPath, + iOSProcessListArguments, latestBenchmarkEvents, parseAndroidProcessIdentifier, parseIOSInstalledAppURL, @@ -78,6 +79,35 @@ describe('benchmarkAppStartup', () => { expect(processIdentifier).toBe(456); }); + it('lists iOS processes without a version-specific CoreDevice filter', () => { + // Given + const deviceIdentifier = '00008120-EXAMPLE'; + const outputPath = '/tmp/processes.json'; + + // When + const args = iOSProcessListArguments(deviceIdentifier, outputPath); + + // Then + expect(args).toEqual(['devicectl', 'device', 'info', 'processes', '--device', deviceIdentifier, '--json-output', outputPath, '--quiet']); + expect(args).not.toContain('--filter'); + }); + + it('ignores an extension process when the main iOS app is not running', () => { + // Given + const appURL = 'file:///private/var/containers/Bundle/Application/EXAMPLE/Expensify.app/'; + const response = { + result: { + runningProcesses: [{executable: `${appURL}PlugIns/LiveActivityExtension.appex/LiveActivityExtension`, processIdentifier: 20702}], + }, + }; + + // When + const processIdentifier = parseIOSRunningAppProcessIdentifier(response, appURL); + + // Then + expect(processIdentifier).toBeUndefined(); + }); + it('rejects malformed CoreDevice installed-app responses before using them', () => { expect(() => parseIOSInstalledAppsResponse({result: {apps: [{bundleIdentifier: 'com.example.app'}]}})).toThrow('unexpected installed-app response'); }); diff --git a/tests/tooling/pgo.test.ts b/tests/tooling/pgo.test.ts new file mode 100644 index 000000000000..ac3ff4ba66e0 --- /dev/null +++ b/tests/tooling/pgo.test.ts @@ -0,0 +1,41 @@ +import {describe, expect, it} from 'bun:test'; +// cspell:ignore chrispader + +import createAndroidPgoAdapter, {PROFILE_BROADCAST_ACTION, androidApplicationID, androidProfileReceiverComponent} from '@scripts/pgo/android'; +import percentageImprovement from '@scripts/pgo/pgo'; + +describe('PGO tooling', () => { + it('exports the Android adapter as the default export', () => { + expect(createAndroidPgoAdapter).toBeFunction(); + }); + + it('reads the bootstrapped release application ID', () => { + // Given + const buildGradle = ` + android { + defaultConfig { + applicationId "com.chrispader.expensify.pgo" + } + } + `; + + // When / Then + expect(androidApplicationID(buildGradle)).toBe('com.chrispader.expensify.pgo'); + }); + + it('uses an explicit application ID instead of the Gradle value', () => { + // Given / When / Then + expect(androidApplicationID('invalid build file', 'com.example.expensify.pgo')).toBe('com.example.expensify.pgo'); + }); + + it('targets the bootstrapped package with the original Java receiver namespace', () => { + // Given / When / Then + expect(androidProfileReceiverComponent('com.chrispader.expensify.pgo')).toBe('com.chrispader.expensify.pgo/org.me.mobiexpensifyg.PgoProfileReceiver'); + expect(PROFILE_BROADCAST_ACTION).toBe('com.expensify.chat.action.WRITE_PGO_PROFILES'); + }); + + it('reports positive values when the optimized build is faster', () => { + // Given / When / Then + expect(percentageImprovement(500, 400)).toBe(20); + }); +}); diff --git a/tests/tooling/pgoAndroid.test.ts b/tests/tooling/pgoAndroid.test.ts new file mode 100644 index 000000000000..ab82b96c3f1b --- /dev/null +++ b/tests/tooling/pgoAndroid.test.ts @@ -0,0 +1,130 @@ +// cspell:ignore profdata + +import {afterAll, beforeEach, describe, expect, it, mock} from 'bun:test'; + +import * as scriptUtils from '@scripts/lib/scriptUtils'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; +import process from 'node:process'; + +let profileLogs = ''; +// Reproduce the CI checkout, which does not initialize the native submodule. +const rootDirectory = mkdtempSync(join(tmpdir(), 'pgo-android-test-')); +const runCommand = mock(() => undefined); +const captureCommand = mock(() => profileLogs); +// Stub the device boundary while keeping the adapter's broadcast, polling and validation real. +mock.module('@scripts/lib/scriptUtils', () => ({...scriptUtils, rootDirectory, run: runCommand, capture: captureCommand})); +const {default: createAndroidPgoAdapter, androidLlvmToolPath} = await import('@scripts/pgo/android'); + +afterAll(() => rmSync(rootDirectory, {recursive: true, force: true})); + +describe('PGO Android profile dumps', () => { + beforeEach(() => { + profileLogs = ''; + runCommand.mockClear(); + captureCommand.mockClear(); + }); + + it('accepts a dump only when all four native libraries wrote profiles', async () => { + // Given a complete writer response from the installed instrumented APK. + profileLogs = 'I/PgoProfileReceiver: Wrote 4 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then profile collection can continue. + await expect(dump).resolves.toBeUndefined(); + expect(runCommand.mock.calls).toEqual([ + ['adb', ['logcat', '-c']], + ['adb', ['shell', 'am', 'broadcast', '-a', 'com.expensify.chat.action.WRITE_PGO_PROFILES', '-n', 'com.example.pgo/org.me.mobiexpensifyg.PgoProfileReceiver']], + ]); + }); + + it.each([1, 2, 3, 5])('rejects a dump reporting %i profiles', async (count) => { + // Given a partial or unexpected writer response despite successful broadcast delivery. + profileLogs = `I/PgoProfileReceiver: Wrote ${count} LLVM PGO profile(s).`; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then training cannot silently accept an incomplete set of native counters. + await expect(dump).rejects.toThrow(`wrote ${count} profiles, but expected 4`); + }); + + it('keeps the diagnostic for a dump with no instrumented libraries', async () => { + // Given a receiver that could not find any profile writers. + profileLogs = 'I/PgoProfileReceiver: Wrote 0 LLVM PGO profile(s).'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it explains that the instrumented APK must be rebuilt. + await expect(dump).rejects.toThrow('found no instrumented native libraries'); + }); + + it('rejects a non-instrumented APK', async () => { + // Given a receiver explicitly rejecting profiling in a normal release build. + profileLogs = 'Ignoring PGO profile request in a non-instrumented build.'; + // When the adapter requests a device dump. + const dump = createAndroidPgoAdapter('com.example.pgo').dumpProfiles(); + // Then it directs the caller to install the instrumented build. + await expect(dump).rejects.toThrow('installed APK is not instrumented'); + }); +}); + +describe('PGO Android LLVM tools', () => { + it.each([ + ['linux', '/home/test/Android/Sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'], + ['darwin', '/home/test/Library/Android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-profdata'], + ] satisfies Array<[NodeJS.Platform, string]>)('uses the default SDK installation on %s', (hostPlatform, expected) => { + // Given an unconfigured host with its normal Android Studio SDK location. + // When LLVM tools are resolved without explicit environment settings. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform, homeDirectory: '/home/test', environment: {}}); + // Then the selected path uses that host's SDK directory and toolchain. + expect(path).toBe(expected); + }); + + it.each(['ANDROID_HOME', 'ANDROID_SDK_ROOT'])('honors %s on Linux', (variable) => { + // Given a Linux runner with its SDK installed outside the default home directory. + // When the SDK environment setting is used to resolve LLVM. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/android/sdk'}}); + // Then the pinned NDK is resolved within the configured SDK. + expect(path).toBe('/opt/android/sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it.each(['ANDROID_NDK_HOME', 'ANDROID_NDK_ROOT'])('prefers %s over SDK settings', (variable) => { + // Given an explicit NDK override and a different SDK location. + // When the tool path is resolved. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment: {[variable]: '/opt/custom-ndk', ANDROID_HOME: '/opt/android/sdk'}}); + // Then an explicitly selected build toolchain wins over the pinned SDK fallback. + expect(path).toBe('/opt/custom-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('prefers ANDROID_HOME over the deprecated SDK root setting and ignores empty NDK overrides', () => { + // Given both SDK settings and empty NDK settings. + const environment = {ANDROID_HOME: '/opt/current-sdk', ANDROID_SDK_ROOT: '/opt/old-sdk', ANDROID_NDK_HOME: '', ANDROID_NDK_ROOT: ''}; + // When LLVM tools are resolved on Linux. + const path = androidLlvmToolPath('llvm-profdata', {hostPlatform: 'linux', environment}); + // Then empty settings do not mask the SDK selected by ANDROID_HOME. + expect(path).toBe('/opt/current-sdk/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-profdata'); + }); + + it('uses the resolved tool path in the adapter', () => { + // Given an explicit temporary NDK with a real tool file. + const directory = mkdtempSync(join(tmpdir(), 'pgo-ndk-test-')); + const previousNdk = process.env.ANDROID_NDK_HOME; + process.env.ANDROID_NDK_HOME = directory; + try { + const expected = androidLlvmToolPath('llvm-profdata'); + mkdirSync(dirname(expected), {recursive: true}); + writeFileSync(expected, ''); + // When the adapter checks tool availability. + const path = createAndroidPgoAdapter().llvmTool('llvm-profdata'); + // Then it returns the existing file in the selected NDK. + expect(path).toBe(expected); + } finally { + if (previousNdk === undefined) { + delete process.env.ANDROID_NDK_HOME; + } else { + process.env.ANDROID_NDK_HOME = previousNdk; + } + rmSync(directory, {recursive: true, force: true}); + } + }); +}); diff --git a/tests/tooling/pgoIOSRock.test.ts b/tests/tooling/pgoIOSRock.test.ts new file mode 100644 index 000000000000..9809f0dc38ed --- /dev/null +++ b/tests/tooling/pgoIOSRock.test.ts @@ -0,0 +1,32 @@ +// cspell:ignore ARCHS CPLUSPLUSFLAGS fprofile profdata + +import {describe, expect, it} from 'bun:test'; + +import {iosRockBuildArguments} from '@scripts/pgo/ios'; + +import {parseArgs} from '@rock-js/tools'; + +describe('PGO iOS Rock build arguments', () => { + it('selects a local device Release build and preserves exact compiler settings', () => { + // Given compiler settings containing spaces and inherited Xcode variables. + const settings = ['build', 'ARCHS=arm64', 'CODE_SIGN_IDENTITY=Apple Development', 'OTHER_CPLUSPLUSFLAGS=$(inherited) -fprofile-use=/tmp/profile directory/journey.profdata']; + // When the adapter creates a Rock invocation. + const args = iosRockBuildArguments('/tmp/derived data/optimized', settings); + // Then the local Release destination and existing pod preparation are explicit. + expect(args.slice(0, 12)).toEqual([ + 'build:ios', + '--local', + '--verbose', + '--configuration', + 'Release', + '--scheme', + 'Expensify', + '--destination', + 'generic/platform=iOS', + '--build-folder', + '/tmp/derived data/optimized', + '--no-install-pods', + ]); + expect(parseArgs(args.at(13) ?? '')).toEqual(settings); + }); +}); diff --git a/tests/tooling/pgoJourney.test.ts b/tests/tooling/pgoJourney.test.ts new file mode 100644 index 000000000000..0c3c6044c56b --- /dev/null +++ b/tests/tooling/pgoJourney.test.ts @@ -0,0 +1,361 @@ +import {describe, expect, it} from 'bun:test'; + +import {journeyBenchmarkPaths, sha256Artifact} from '@scripts/pgo/journeyBenchmark'; +import {parseJourneyFixture} from '@scripts/pgo/journeyConfig'; +import {assertSignedIn, normalizeLabel, parseJourneySnapshot} from '@scripts/pgo/journeyDevice'; +import { + allFilterTapPoint, + contentSignature, + findReportResult, + findTabNode, + inAppBackTapPoint, + scrollDistance, + scrollFastBothWays, + showTab, + spendSectionTapPoint, + waitForTab, +} from '@scripts/pgo/journeyWorkload'; +import {BENCHMARK_SPANS_ENVIRONMENT, HEAVY_JOURNEY_SPAN_NAMES, JOURNEY_SPAN_NAMES, STARTUP_SPAN_NAME} from '@scripts/pgo/shared'; + +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; + +const fixture = { + accountEmail: 'heavy@example.com', + accountClass: 'heavy', + description: 'Owner-approved account with a populated inbox, message history, expenses, and reports.', + report: {query: 'busy', resultLabelPrefix: '#busy, Test workspace', title: '#busy'}, + personalChat: {query: 'heavy@example.com', resultLabelPrefix: 'Test User (you), heavy@example.com', title: 'Test User (you)'}, +}; + +describe('PGO journey safeguards', () => { + it('logs the held-out navigation spans in both PGO builds without replacing startup samples', () => { + // Given release and optimized artifacts share one build-time benchmark allowlist. + const enabledSpans = BENCHMARK_SPANS_ENVIRONMENT.split('=').at(1)?.split(','); + // When / Then the startup span and every short-journey span remain available. + expect(enabledSpans).toEqual([STARTUP_SPAN_NAME, ...JOURNEY_SPAN_NAMES, ...HEAVY_JOURNEY_SPAN_NAMES]); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1')).toEqual({ + release: '/pgo/ios/benchmarks/journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/journey/batch-1/pgo-optimized.csv', + }); + expect(journeyBenchmarkPaths({benchmarkDirectory: '/pgo/ios/benchmarks'}, 'batch-1', 'heavy-journey')).toEqual({ + release: '/pgo/ios/benchmarks/heavy-journey/batch-1/release.csv', + optimized: '/pgo/ios/benchmarks/heavy-journey/batch-1/pgo-optimized.csv', + }); + }); + + it('hashes iOS bundle resources as well as the app executable', async () => { + const app = mkdtempSync(join(tmpdir(), 'pgo-test-app-')); + try { + mkdirSync(join(app, 'Frameworks')); + writeFileSync(join(app, 'Expensify'), 'same executable'); + writeFileSync(join(app, 'main.jsbundle'), 'first bundle'); + const before = await sha256Artifact(app); + writeFileSync(join(app, 'main.jsbundle'), 'second bundle'); + expect(await sha256Artifact(app)).not.toBe(before); + } finally { + rmSync(app, {recursive: true, force: true}); + } + }); + + it('requires explicit message approval in the fixture', () => { + // Given account approval alone does not authorize a particular message destination. + for (const allowMessages of [undefined, false, 'true']) { + // When / Then full journeys remain unavailable until approval is recorded explicitly. + expect(parseJourneyFixture({...fixture, allowMessages}).allowMessages).toBe(false); + } + expect(parseJourneyFixture({...fixture, allowMessages: true}).allowMessages).toBe(true); + }); + + it('requires the account owner to select a heavy account explicitly', () => { + // Given an unclassified or ordinary account, a populated-looking screen is insufficient evidence. + for (const accountClass of [undefined, 'ordinary', '']) { + // When / Then the fixture cannot start a training run. + expect(() => parseJourneyFixture({...fixture, accountClass})).toThrow('accountClass="heavy"'); + } + }); + + it("rejects message destinations that are not the approved account's personal chat", () => { + // Given a fixture that would send messages to another person or to a different account's personal chat. + for (const personalChat of [ + {...fixture.personalChat, query: 'another@example.com'}, + {...fixture.personalChat, title: 'Another person'}, + {...fixture.personalChat, resultLabelPrefix: 'Other User (you), other@example.com'}, + {...fixture.personalChat, resultLabelPrefix: 'Test User (you), heavy@example.com.invalid'}, + ]) { + // When / Then no device action is permitted for that fixture. + expect(() => parseJourneyFixture({...fixture, personalChat})).toThrow('own "(you)" chat'); + } + }); + + it('rejects an empty workload or an unbounded scrolling loop', () => { + // Given invalid workload sizes that would create an unrepresentative or runaway recording. + for (const scrolls of [0, 1, 31, NaN, '8']) { + // When / Then parsing fails before the app is touched. + expect(() => parseJourneyFixture({...fixture, scrolls})).toThrow('scrolls'); + } + }); + + it('filters system keyboard content out of app assertions', () => { + // Given Android's keyboard can display labels unrelated to the foreground app. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {bundleId: 'keyboard.app', label: 'Sign in'}, + {bundleId: 'test.app', label: 'Inbox'}, + ], + }, + 'test.app', + ); + // When / Then keyboard content cannot falsely trigger a sign-in failure. + expect(nodes).toHaveLength(1); + expect(() => assertSignedIn(nodes)).not.toThrow(); + }); + + it('stops for manual sign-in on both platforms', () => { + // Given either platform has reached an authentication or code-entry screen. + for (const identifier of ['username', 'validateCode']) { + for (const type of ['android.widget.EditText', 'TextField']) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{identifier, type}]}, 'test.app'); + // When / Then the runner requests user input instead of filling credentials. + expect(() => assertSignedIn(nodes)).toThrow('SIGN_IN_REQUIRED'); + } + } + }); + + it('rejects a snapshot from the wrong app', () => { + // Given device focus has moved to an unexpected application. + const snapshot = {appBundleId: 'other.app', nodes: []}; + // When / Then its labels cannot satisfy the journey's assertions. + expect(() => parseJourneySnapshot(snapshot, 'test.app')).toThrow('expected app'); + }); + + it('normalizes accessibility text direction markers without changing its words', () => { + // Given translated UI labels include bidirectional isolation and non-breaking spaces. + const label = '\u2066Most\u00a0recent\u2069'; + // When / Then the same label can be compared across native platforms. + expect(normalizeLabel(label)).toBe('Most recent'); + }); + + it('matches report identity independently of the changing last-message preview', () => { + // Given the workspace has similarly named reports and a new message changes the preview. + const label = '#busy, Test workspace • New message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: '#busy, Test workspace copy • Other message'}]}, 'test.app'); + // When / Then only the specified workspace can be selected. + expect(findReportResult(nodes, '#busy, Test workspace')).toBe(label); + }); + + it('refuses ambiguous report matches', () => { + // Given two results share the fixture identity. + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label: '#busy, Test workspace'}, {label: '#busy, Test workspace • Message'}]}, 'test.app'); + // When / Then choosing the first result would risk opening an unintended destination. + expect(() => findReportResult(nodes, '#busy, Test workspace')).toThrow('Ambiguous'); + }); + + it('finds the approved personal chat after its email subtitle becomes a message preview', () => { + // Given a previous run sent a message, while another account has a similarly named chat. + const label = 'Test User (you), PGO previous run message'; + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{label}, {label: 'Test User (you)'}, {label: 'Test User, Message from another account'}]}, 'test.app'); + // When / Then the validated personal-chat title matches only the full result row. + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix, fixture.personalChat.title)).toBe(label); + expect(findReportResult(nodes, fixture.personalChat.resultLabelPrefix)).toBeUndefined(); + }); + + it('sizes scrolls for different screens without starting inside the fixed header', () => { + // Given Android uses physical pixels while iOS accessibility exposes logical screen dimensions. + for (const [width, height] of [ + [1080, 2280], + [390, 844], + ]) { + const nodes = parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width, height}}]}, 'test.app'); + // When the gesture is centered, both ends stay within the middle half of the portrait viewport. + const distance = scrollDistance(nodes); + expect(distance).toBeGreaterThan(0); + expect(distance).toBeLessThanOrEqual((height ?? 0) / 2); + expect(scrollDistance(nodes, true)).toBeGreaterThan(distance); + } + // Then missing viewport information must fail instead of falling back to a screen-edge gesture. + expect(() => scrollDistance([])).toThrow('scroll bounds'); + }); + + it('uses fixed rapid swipes and validates movement in both directions', () => { + // Given a report with distinct content at the recent and older scroll positions. + const snapshots = ['recent', 'older', 'recent'].map((label) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'ScrollView', rect: {width: 390, height: 844}}, {label}]}, 'test.app'), + ); + const commands: string[][] = []; + let read = 0; + // When the heavy benchmark scrolls in both directions. + scrollFastBothWays( + { + snapshot: () => snapshots.at(read++) ?? [], + command: (...args) => commands.push(args), + }, + 4, + ); + // Then it sends fixed-duration gestures and checks content after each direction. + expect(commands).toHaveLength(8); + expect(commands.every((args) => args.includes('120'))).toBe(true); + expect(commands.filter((args) => args.includes('--settle'))).toHaveLength(2); + expect(commands.at(0)?.at(1)).toBe('up'); + expect(commands.at(-1)?.at(1)).toBe('down'); + }); + + it('selects iOS status-bearing bottom tabs without confusing them with page headings', () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Inbox', rect: {width: 390, height: 50, y: 80}}, + {label: 'Inbox. Your review is required', rect: {width: 78, height: 71, y: 749}}, + {label: 'Inbox', type: 'StaticText', rect: {width: 26, height: 14, y: 793}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + expect(findTabNode(nodes, 'Inbox')?.label).toBe('Inbox. Your review is required'); + expect(findTabNode(nodes, 'Account')?.label).toBe('Account, My settings. Your review is required.'); + }); + + it('waits for a status-bearing iOS Account tab without requiring an exact label', async () => { + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Account, My settings. Your review is required.', rect: {width: 78, height: 71, y: 749}}, + ], + }, + 'test.app', + ); + const device = {snapshot: () => nodes}; + await expect(waitForTab(device, 'Account')).resolves.toBeUndefined(); + }); + + it('waits through Android relaunch loading before navigating from a restored page', async () => { + // Given a loading view, then a restored nested page, then the Inbox tab after app Back. + const screen = {type: 'android.widget.ScrollView', rect: {x: 0, y: 0, width: 1080, height: 2280}}; + const snapshots = [ + [screen], + [screen, {type: 'android.widget.Button', label: 'Back', rect: {x: 21, y: 144, width: 67, height: 105}}], + [screen, {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}], + ].map((nodes) => parseJourneySnapshot({appBundleId: 'test.app', nodes}, 'test.app')); + const commands: string[][] = []; + let read = 0; + const device: Parameters[0] = { + platform: 'android', + snapshot: () => snapshots[Math.min(read++, snapshots.length - 1)], + command: (...args: string[]) => commands.push(args), + wait: () => {}, + press: () => {}, + hasLabel: () => false, + pressLabel: () => {}, + waitLabel: () => {}, + fill: () => {}, + back: () => commands.push(['hardware back']), + open: () => {}, + }; + + // When the benchmark navigates to Inbox after relaunch. + await showTab(device, 'Inbox'); + + // Then loading does not trigger Back, and the app's Back control is used once. + expect(commands).toEqual([ + ['press', '55', '197', '--settle'], + ['press', '324', '2061', '--settle'], + ]); + }); + + it('accepts a duplicated Android navigation subtree but rejects distinct tab targets', () => { + const tab = {type: 'android.view.View', label: 'Inbox. Your review is required', rect: {x: 216, y: 1968, width: 216, height: 186}}; + const snapshot = (tabs: Array) => + parseJourneySnapshot({appBundleId: 'test.app', nodes: [{type: 'Application', rect: {x: 0, y: 0, width: 1080, height: 2280}}, ...tabs]}, 'test.app'); + + expect(findTabNode(snapshot([tab, tab]), 'Inbox')?.x).toBe(216); + expect(() => findTabNode(snapshot([tab, {...tab, rect: {...tab.rect, x: 432}}]), 'Inbox')).toThrow('Ambiguous Inbox tab'); + }); + + it('targets the visible All filter rather than the iOS strip cell center over Unread', () => { + // Given an iOS accessibility cell whose bounds cover the whole Inbox filter strip. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'All', type: 'Cell', rect: {width: 390, height: 52, x: 0, y: 115}}, + {label: 'Unread', type: 'Other', rect: {width: 136, height: 40, x: 102, y: 115}}, + ], + }, + 'test.app', + ); + // When the journey chooses the All filter's tap point. + const tapPoint = allFilterTapPoint(nodes); + // Then it stays left of Unread. + expect(tapPoint).toEqual({x: 51, y: 135}); + }); + + it('taps the iOS search Back button even when the keyboard marks it non-hittable', () => { + // Given the search screen's bounded Back button beneath an open keyboard. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Back', type: 'Button', hittable: false, rect: {width: 40, height: 40, x: 8, y: 59}}, + {label: 'Back', type: 'Button', rect: {width: 40, height: 40, x: 300, y: 700}}, + ], + }, + 'test.app', + ); + // When the journey looks for the in-app Back control. + const tapPoint = inAppBackTapPoint(nodes); + // Then it chooses the top-left button rather than the keyboard area. + expect(tapPoint).toEqual({x: 28, y: 79}); + }); + + it('detects iOS report scrolling when offscreen accessibility labels stay loaded', () => { + // Given the same report labels at different vertical positions after a scroll. + const reportNodes = (firstMessageY: number) => + parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, y: 0}}, + {label: 'Old message', rect: {width: 390, height: 40, y: firstMessageY - 80}}, + {label: 'Recent message', rect: {width: 390, height: 40, y: firstMessageY}}, + ], + }, + 'test.app', + ); + // When content signatures are compared. + const before = contentSignature(reportNodes(500)); + const after = contentSignature(reportNodes(266)); + // Then movement is visible despite identical labels. + expect(before).not.toBe(after); + }); + + it('targets the visible iOS Spend segments despite a wide Expenses strip cell', () => { + // Given an Expenses cell that covers the full segment strip and a bounded Reports control. + const nodes = parseJourneySnapshot( + { + appBundleId: 'test.app', + nodes: [ + {type: 'Application', rect: {width: 390, height: 844, x: 0, y: 0}}, + {label: 'Expenses', type: 'Cell', rect: {width: 1426, height: 52, x: 0, y: 115}}, + {label: 'Reports', type: 'Other', rect: {width: 94, height: 40, x: 125, y: 115}}, + ], + }, + 'test.app', + ); + // When each section is selected. + const expenses = spendSectionTapPoint(nodes, 'Expenses'); + const reports = spendSectionTapPoint(nodes, 'Reports'); + // Then both tap points land inside their visible segments. + expect(expenses).toEqual({x: 63, y: 135}); + expect(reports).toEqual({x: 172, y: 135}); + }); +});