From 13afbaf0263972faa3fbcaabc880e4b8f4371a55 Mon Sep 17 00:00:00 2001 From: pdx15 <301492+pdx15@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:02:35 +0000 Subject: [PATCH 01/18] Apply upstream 73fabcc..5c522b6 with fork adaptations Brings the first 11 upstream commits (30b2909..5c522b6 of majd/ipatool) into the fork in one reconstructed commit after a sandbox history reset: - 73fabcc, abdb590, e6acb9c, b8bac29, 843f5c6, a814a71: download/resume, zip framing/headers, state directory, ranged responses - 24f93d3: .github/FUNDING.yml (fork README kept) - d1845ba: authentication transport isolation (DisableKeepAlives for auth client), 429/Retry-After handling, auth retry with backoff in login flow, stage-aware error messages; fork MZFinance pod redirect flow preserved (parseLoginResponse stays 2-arg; no endpoint validation) - ba180d7: visionos/macos platform support, owned-apps platform filtering, multi-storefront (34/13) fetch + mergeOwnedApps, --platform CLI flag (fork All flag and SignAction-based signing preserved) - 5c522b6: macOS external version lookup via Mac product page (storefrontClient added to appstore), wired into Download() for PlatformMacOS; fork download flow (fetchDownloadItem/redownload) kept Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/FUNDING.yml | 1 + cmd/common.go | 44 +- cmd/purchases.go | 20 +- cmd/purchases_test.go | 88 +++ cmd/state_directory.go | 61 ++ cmd/state_directory_test.go | 182 ++++++ pkg/appstore/app.go | 12 +- pkg/appstore/app_test.go | 16 + pkg/appstore/appstore.go | 62 +- pkg/appstore/appstore_download.go | 96 ++- pkg/appstore/appstore_download_artwork.go | 46 ++ .../appstore_download_artwork_test.go | 112 ++++ pkg/appstore/appstore_download_range.go | 47 ++ pkg/appstore/appstore_download_resume_test.go | 130 ++++ pkg/appstore/appstore_download_test.go | 116 +++- pkg/appstore/appstore_login.go | 125 +++- pkg/appstore/appstore_login_test.go | 2 +- pkg/appstore/appstore_macos_version_lookup.go | 131 ++++ .../appstore_macos_version_lookup_test.go | 54 ++ pkg/appstore/appstore_owned_apps.go | 137 +++- pkg/appstore/appstore_owned_apps_test.go | 584 ++++++++++++++---- pkg/appstore/appstore_replicate_sinf.go | 69 ++- pkg/appstore/appstore_zip_framing_test.go | 124 ++++ pkg/appstore/appstore_zip_headers.go | 289 +++++++++ pkg/appstore/appstore_zip_headers_test.go | 209 +++++++ pkg/appstore/authentication_retry_test.go | 175 ++++++ pkg/appstore/constants.go | 1 + pkg/appstore/platform.go | 37 +- pkg/http/authentication_test.go | 164 +++++ pkg/http/client.go | 62 +- 30 files changed, 2931 insertions(+), 265 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 cmd/purchases_test.go create mode 100644 cmd/state_directory.go create mode 100644 cmd/state_directory_test.go create mode 100644 pkg/appstore/appstore_download_artwork.go create mode 100644 pkg/appstore/appstore_download_artwork_test.go create mode 100644 pkg/appstore/appstore_download_range.go create mode 100644 pkg/appstore/appstore_download_resume_test.go create mode 100644 pkg/appstore/appstore_macos_version_lookup.go create mode 100644 pkg/appstore/appstore_macos_version_lookup_test.go create mode 100644 pkg/appstore/appstore_zip_framing_test.go create mode 100644 pkg/appstore/appstore_zip_headers.go create mode 100644 pkg/appstore/appstore_zip_headers_test.go create mode 100644 pkg/appstore/authentication_retry_test.go create mode 100644 pkg/http/authentication_test.go diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..4785480 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: majd diff --git a/cmd/common.go b/cmd/common.go index 62ef605..490139d 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -57,8 +57,8 @@ func newLogger(format OutputFormat, verbose bool) log.Logger { // file is corrupt (e.g. left over from an interrupted write or an // incompatible cookie format), the broken file is moved aside and a fresh // jar is created instead of crashing the whole program on startup. -func newCookieJar(machine machine.Machine) http.CookieJar { - filename := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName) +func newCookieJar(stateDirectory string) http.CookieJar { + filename := filepath.Join(stateDirectory, CookieJarFileName) jar, err := cookiejar.New(&cookiejar.Options{Filename: filename}) if err == nil { @@ -120,22 +120,22 @@ func (envSessionKeychain) Remove(_ string) error { } // keychainPassphraseFile is the name of the file that stores the auto-generated -// keychain passphrase. Keeping it in the ipatool config directory means the +// keychain passphrase. Keeping it in the ipatool state directory means the // session token stays decryptable across runs without prompting the user. const keychainPassphraseFile = "keychain-passphrase" // resolveKeychainPassphrase returns the passphrase used to encrypt the local // keychain file. An explicitly provided --keychain-passphrase wins; otherwise a // random passphrase is generated on first use and persisted in the ipatool -// config directory, so the user is never prompted for a separate local +// state directory, so the user is never prompted for a separate local // password. (The OS keyring backends, when present, are used in preference to // the file backend anyway.) -func resolveKeychainPassphrase(machine machine.Machine) (string, error) { +func resolveKeychainPassphrase(stateDirectory string) (string, error) { if keychainPassphrase != "" { return keychainPassphrase, nil } - dir := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName) + dir := stateDirectory path := filepath.Join(dir, keychainPassphraseFile) if data, err := os.ReadFile(path); err == nil { @@ -150,7 +150,7 @@ func resolveKeychainPassphrase(machine machine.Machine) (string, error) { passphrase := hex.EncodeToString(random) if err := os.MkdirAll(dir, 0o700); err != nil { - return "", fmt.Errorf("failed to create config directory: %w", err) + return "", fmt.Errorf("failed to create state directory: %w", err) } if err := os.WriteFile(path, []byte(passphrase+"\n"), 0o600); err != nil { @@ -161,12 +161,12 @@ func resolveKeychainPassphrase(machine machine.Machine) (string, error) { } // newKeychain returns a new keychain instance. -func newKeychain(machine machine.Machine) keychain.Keychain { +func newKeychain(stateDirectory string) keychain.Keychain { if session := os.Getenv("IPATOOL_SESSION"); session != "" { return envSessionKeychain{data: []byte(session)} } - passphrase, err := resolveKeychainPassphrase(machine) + passphrase, err := resolveKeychainPassphrase(stateDirectory) if err != nil { util.Must("", err) } @@ -178,7 +178,7 @@ func newKeychain(machine machine.Machine) keychain.Keychain { keyring.FileBackend, }, ServiceName: KeychainServiceName, - FileDir: filepath.Join(machine.HomeDirectory(), ConfigDirectoryName), + FileDir: stateDirectory, FilePasswordFunc: func(s string) (string, error) { return passphrase, nil }, @@ -195,31 +195,13 @@ func initWithCommand(cmd *cobra.Command) { dependencies.Logger = newLogger(format, verbose) dependencies.OS = operatingsystem.New() dependencies.Machine = machine.New(machine.Args{OS: dependencies.OS}) - dependencies.CookieJar = newCookieJar(dependencies.Machine) - dependencies.Keychain = newKeychain(dependencies.Machine) + stateDirectory := util.Must(prepareStateDirectory(dependencies.OS, dependencies.Machine.HomeDirectory())) + dependencies.CookieJar = newCookieJar(stateDirectory) + dependencies.Keychain = newKeychain(stateDirectory) dependencies.AppStore = appstore.NewAppStore(appstore.Args{ CookieJar: dependencies.CookieJar, OperatingSystem: dependencies.OS, Keychain: dependencies.Keychain, Machine: dependencies.Machine, }) - - util.Must("", createConfigDirectory(dependencies.OS, dependencies.Machine)) -} - -// createConfigDirectory creates the configuration directory for the CLI tool, if needed. -func createConfigDirectory(os operatingsystem.OperatingSystem, machine machine.Machine) error { - configDirectoryPath := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName) - _, err := os.Stat(configDirectoryPath) - - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(configDirectoryPath, 0700) - if err != nil { - return fmt.Errorf("failed to create config directory: %w", err) - } - } else if err != nil { - return fmt.Errorf("could not read metadata: %w", err) - } - - return nil } diff --git a/cmd/purchases.go b/cmd/purchases.go index e07104e..c19d871 100644 --- a/cmd/purchases.go +++ b/cmd/purchases.go @@ -18,11 +18,21 @@ func listPurchasesCmd() *cobra.Command { all bool ) + var platformValue string + + var platform appstore.Platform + cmd := &cobra.Command{ Use: "list-purchases", Short: "List apps owned by the authenticated App Store account", Args: cobra.NoArgs, PreRunE: func(cmd *cobra.Command, args []string) error { + var err error + platform, err = appstore.ParsePlatform(platformValue) + if err != nil { + return err + } + if all { return nil } @@ -56,10 +66,11 @@ func listPurchasesCmd() *cobra.Command { } output, err := dependencies.AppStore.OwnedApps(appstore.OwnedAppsInput{ - Account: acc, - Page: page, - Limit: maxResults, - All: all, + Account: acc, + Page: page, + Limit: maxResults, + Platform: platform, + All: all, }) if err != nil { return err @@ -90,6 +101,7 @@ func listPurchasesCmd() *cobra.Command { cmd.Flags().IntVarP(&maxResults, "max-results", "l", appstore.DefaultOwnedAppsLimit, "maximum number of apps to return per page") cmd.Flags().IntVarP(&page, "page", "p", 1, "page of owned apps to return") cmd.Flags().BoolVar(&all, "all", false, "return every owned app in one response (ignores --page and --max-results)") + cmd.Flags().StringVar(&platformValue, "platform", "", "Filter by platform: iphone (iOS), ipad (iPadOS), appletv (tvOS), visionos, or macos") return cmd } diff --git a/cmd/purchases_test.go b/cmd/purchases_test.go new file mode 100644 index 0000000..9c94d8d --- /dev/null +++ b/cmd/purchases_test.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "github.com/majd/ipatool/v2/pkg/appstore" + "github.com/majd/ipatool/v2/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("List Purchases command", func() { + DescribeTable("passes the platform filter to owned apps", + func(value string, expected appstore.Platform) { + store := &fakePurchasesAppStore{} + previousDependencies := dependencies + DeferCleanup(func() { dependencies = previousDependencies }) + dependencies.AppStore = store + dependencies.Logger = log.NewLogger(log.Args{}) + cmd := listPurchasesCmd() + cmd.SetArgs([]string{"--platform", value}) + + Expect(cmd.Execute()).To(Succeed()) + Expect(store.input.Platform).To(Equal(expected)) + }, + Entry("all platforms by default", "", appstore.Platform("")), + Entry("iOS alias", "ios", appstore.PlatformIPhone), + Entry("iPadOS alias", "ipados", appstore.PlatformIPad), + Entry("tvOS alias", "tvos", appstore.PlatformAppleTV), + Entry("visionOS", "visionos", appstore.PlatformVisionOS), + Entry("macOS", "macos", appstore.PlatformMacOS), + ) + + DescribeTable("rejects an invalid platform before accessing the account", func(value string) { + cmd := listPurchasesCmd() + Expect(cmd.Flags().Set("platform", value)).To(Succeed()) + Expect(cmd.PreRunE(cmd, nil)).To(MatchError(ContainSubstring("invalid platform"))) + }, + Entry("invalid value", "invalid"), + Entry("unknown is output only", "unknown"), + Entry("uppercase unknown", "UNKNOWN"), + ) + + It("uses the pagination defaults", func() { + cmd := listPurchasesCmd() + + page, err := cmd.Flags().GetInt("page") + Expect(err).ToNot(HaveOccurred()) + Expect(page).To(Equal(1)) + + maxResults, err := cmd.Flags().GetInt("max-results") + Expect(err).ToNot(HaveOccurred()) + Expect(maxResults).To(Equal(appstore.DefaultOwnedAppsLimit)) + }) + + DescribeTable("rejects invalid pagination", + func(flag, value, errorText string) { + cmd := listPurchasesCmd() + Expect(cmd.Flags().Set(flag, value)).To(Succeed()) + + err := cmd.PreRunE(cmd, nil) + Expect(err).To(MatchError(ContainSubstring(errorText))) + }, + Entry("page below one", "page", "0", "page"), + Entry("max results below one", "max-results", "0", "max results"), + Entry("max results over limit", "max-results", "101", "100"), + ) + + It("is registered on the root command", func() { + cmd, _, err := rootCmd().Find([]string{"list-purchases"}) + + Expect(err).ToNot(HaveOccurred()) + Expect(cmd.Name()).To(Equal("list-purchases")) + }) +}) + +type fakePurchasesAppStore struct { + appstore.AppStore + input appstore.OwnedAppsInput +} + +func (*fakePurchasesAppStore) AccountInfo() (appstore.AccountInfoOutput, error) { + return appstore.AccountInfoOutput{}, nil +} + +func (s *fakePurchasesAppStore) OwnedApps(input appstore.OwnedAppsInput) (appstore.OwnedAppsOutput, error) { + s.input = input + + return appstore.OwnedAppsOutput{}, nil +} diff --git a/cmd/state_directory.go b/cmd/state_directory.go new file mode 100644 index 0000000..c23d3bc --- /dev/null +++ b/cmd/state_directory.go @@ -0,0 +1,61 @@ +package cmd + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" +) + +// prepareStateDirectory resolves and migrates session storage before it is opened. +func prepareStateDirectory(os operatingsystem.OperatingSystem, homeDirectory string) (string, error) { + legacyDirectory := filepath.Join(homeDirectory, ConfigDirectoryName) + stateDirectory := legacyDirectory + + for _, variable := range []string{"XDG_STATE_HOME", "XDG_DATA_HOME"} { + if base := os.Getenv(variable); filepath.IsAbs(base) { + stateDirectory = filepath.Join(base, "ipatool") + + break + } + } + + info, err := os.Stat(legacyDirectory) + if err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("could not read legacy state directory metadata: %w", err) + } + + if err == nil { + if !info.IsDir() { + return "", fmt.Errorf("legacy state path is not a directory: %s", legacyDirectory) + } + // Never merge sessions or move a directory into itself. Keeping the legacy + // directory also preserves authentication when migration is unavailable. + relative, err := filepath.Rel(legacyDirectory, stateDirectory) + if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return legacyDirectory, nil + } + + if _, err := os.Stat(stateDirectory); err == nil || !os.IsNotExist(err) { + return legacyDirectory, nil + } + + if err := os.MkdirAll(filepath.Dir(stateDirectory), 0700); err != nil { + return legacyDirectory, nil + } + // Rename keeps cookies and encrypted credentials together, with their + // existing permissions. Cross-filesystem moves fall back to legacy storage. + if err := os.Rename(legacyDirectory, stateDirectory); err != nil { + return legacyDirectory, nil + } + + return stateDirectory, nil + } + + if err := os.MkdirAll(stateDirectory, 0700); err != nil { + return "", fmt.Errorf("failed to create state directory: %w", err) + } + + return stateDirectory, nil +} diff --git a/cmd/state_directory_test.go b/cmd/state_directory_test.go new file mode 100644 index 0000000..4e3fca9 --- /dev/null +++ b/cmd/state_directory_test.go @@ -0,0 +1,182 @@ +package cmd + +import ( + "errors" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/byteness/keyring" + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("State directory", func() { + var home, legacy, target string + + BeforeEach(func() { + home = GinkgoT().TempDir() + legacy = filepath.Join(home, ConfigDirectoryName) + target = filepath.Join(home, "state", "ipatool") + GinkgoT().Setenv("XDG_STATE_HOME", filepath.Dir(target)) + GinkgoT().Setenv("XDG_DATA_HOME", "") + }) + + DescribeTable("selects the session directory", func(state, data, want string) { + for variable, value := range map[string]string{"XDG_STATE_HOME": state, "XDG_DATA_HOME": data} { + if value != "" && value != "relative" { + value = filepath.Join(home, value) + } + + GinkgoT().Setenv(variable, value) + } + + got, err := prepareStateDirectory(operatingsystem.New(), home) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(filepath.Join(home, want))) + Expect(got).To(BeADirectory()) + + if got != legacy { + Expect(legacy).ToNot(BeAnExistingFile()) + } + }, + Entry("defaults to legacy storage", "", "", ConfigDirectoryName), + Entry("prefers state storage", "state", "data", "state/ipatool"), + Entry("falls back to data storage", "", "data", "data/ipatool"), + Entry("ignores a relative state path", "relative", "data", "data/ipatool"), + Entry("ignores relative paths", "relative", "relative", ConfigDirectoryName), + ) + + It("preserves authentication after migration and subsequent startup", func() { + Expect(os.MkdirAll(legacy, 0700)).To(Succeed()) + + openKeyring := func(directory string) keyring.Keyring { + ring, err := keyring.Open(keyring.Config{ + AllowedBackends: []keyring.BackendType{keyring.FileBackend}, + ServiceName: KeychainServiceName, + FileDir: directory, + FilePasswordFunc: keyring.FixedStringPrompt("test-passphrase"), + }) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + + return ring + } + item := keyring.Item{Key: "account", Data: []byte("existing-session"), Label: KeychainServiceName} + Expect(openKeyring(legacy).Set(item)).To(Succeed()) + + cookieURL, err := url.Parse("https://apps.apple.com/") + Expect(err).ToNot(HaveOccurred()) + + jar := newCookieJar(legacy) + jar.SetCookies(cookieURL, []*http.Cookie{{Name: "session", Value: "existing-cookie", Path: "/", Expires: time.Now().Add(time.Hour)}}) + Expect(jar.Save()).To(Succeed()) + + before, err := os.ReadFile(filepath.Join(legacy, item.Key)) + Expect(err).ToNot(HaveOccurred()) + + for i := 0; i < 2; i++ { + directory, err := prepareStateDirectory(operatingsystem.New(), home) + Expect(err).ToNot(HaveOccurred()) + Expect(directory).To(Equal(target)) + + got, err := openKeyring(directory).Get(item.Key) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Data).To(Equal(item.Data)) + + after, err := os.ReadFile(filepath.Join(directory, item.Key)) + Expect(err).ToNot(HaveOccurred()) + Expect(after).To(Equal(before), "encrypted credentials must remain unchanged") + + cookies := newCookieJar(directory).Cookies(cookieURL) + Expect(cookies).To(HaveLen(1)) + Expect(cookies[0].Name).To(Equal("session")) + Expect(cookies[0].Value).To(Equal("existing-cookie")) + } + + Expect(legacy).ToNot(BeAnExistingFile()) + }) + + DescribeTable("preserves legacy storage when migration fails", func(mkdirErr, renameErr error) { + Expect(os.MkdirAll(legacy, 0700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(legacy, CookieJarFileName), []byte("session"), 0600)).To(Succeed()) + + got, err := prepareStateDirectory(migrationFailureOS{operatingsystem.New(), mkdirErr, renameErr}, home) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(legacy)) + + data, err := os.ReadFile(filepath.Join(got, CookieJarFileName)) + Expect(err).ToNot(HaveOccurred()) + Expect(string(data)).To(Equal("session")) + }, + Entry("parent creation denied", os.ErrPermission, nil), + Entry("rename denied", nil, os.ErrPermission), + Entry("cross filesystem", nil, errors.New("cross-device link")), + ) + + It("preserves conflicting sessions and uses legacy storage", func() { + for _, directory := range []string{legacy, target} { + Expect(os.MkdirAll(directory, 0700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(directory, "account"), []byte(directory), 0600)).To(Succeed()) + } + + got, err := prepareStateDirectory(operatingsystem.New(), home) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(legacy)) + + for _, directory := range []string{legacy, target} { + data, err := os.ReadFile(filepath.Join(directory, "account")) + Expect(err).ToNot(HaveOccurred()) + Expect(string(data)).To(Equal(directory)) + } + }) + + DescribeTable("rejects files at directory paths", func(useLegacy bool) { + path := target + if useLegacy { + path = legacy + } + + Expect(os.MkdirAll(filepath.Dir(path), 0700)).To(Succeed()) + Expect(os.WriteFile(path, []byte("existing file"), 0600)).To(Succeed()) + + _, err := prepareStateDirectory(operatingsystem.New(), home) + Expect(err).To(HaveOccurred()) + }, + Entry("legacy", true), + Entry("destination", false), + ) + + It("preserves legacy storage for a nested destination", func() { + GinkgoT().Setenv("XDG_STATE_HOME", filepath.Join(legacy, "state")) + Expect(os.MkdirAll(legacy, 0700)).To(Succeed()) + + got, err := prepareStateDirectory(operatingsystem.New(), home) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(legacy)) + + entries, err := os.ReadDir(legacy) + Expect(err).ToNot(HaveOccurred()) + Expect(entries).To(BeEmpty(), "migration must not create directories inside legacy storage") + }) +}) + +type migrationFailureOS struct { + operatingsystem.OperatingSystem + mkdirErr, renameErr error +} + +// nolint:wrapcheck +func (o migrationFailureOS) MkdirAll(path string, mode os.FileMode) error { + if o.mkdirErr != nil { + return o.mkdirErr + } + + return o.OperatingSystem.MkdirAll(path, mode) +} + +func (o migrationFailureOS) Rename(from, to string) error { + return o.renameErr +} diff --git a/pkg/appstore/app.go b/pkg/appstore/app.go index e190623..58fd2fe 100644 --- a/pkg/appstore/app.go +++ b/pkg/appstore/app.go @@ -57,7 +57,8 @@ type App struct { RatingCount int64 `json:"userRatingCount,omitempty"` Genres []string `json:"genres,omitempty"` // PurchaseDate is only populated for apps returned by OwnedApps. - PurchaseDate time.Time `json:"purchaseDate,omitzero"` + PurchaseDate time.Time `json:"purchaseDate,omitzero"` + Platforms []Platform `json:"platforms,omitzero"` } type VersionHistoryInfo struct { @@ -92,4 +93,13 @@ func (a App) MarshalZerologObject(event *zerolog.Event) { if !a.PurchaseDate.IsZero() { event.Time("purchaseDate", a.PurchaseDate) } + + if a.Platforms != nil { + platforms := make([]string, len(a.Platforms)) + for index, platform := range a.Platforms { + platforms[index] = string(platform) + } + + event.Strs("platforms", platforms) + } } diff --git a/pkg/appstore/app_test.go b/pkg/appstore/app_test.go index 1a190c5..a52e7ba 100644 --- a/pkg/appstore/app_test.go +++ b/pkg/appstore/app_test.go @@ -65,6 +65,22 @@ var _ = Describe("App", func() { Expect(out["price"]).To(Equal(float64(0))) }) + DescribeTable("serializes purchase platforms as an array", + func(platforms []Platform, expected string) { + app := App{ID: 42, Platforms: platforms} + data, err := json.Marshal(app) + Expect(err).ToNot(HaveOccurred()) + Expect(string(data)).To(ContainSubstring(`"platforms":` + expected)) + + buffer := bytes.NewBuffer(nil) + logger := zerolog.New(buffer) + logger.Log().Object("app", app).Send() + Expect(buffer.String()).To(ContainSubstring(`"platforms":` + expected)) + }, + Entry("multiple platforms", []Platform{PlatformIPhone, PlatformIPad, PlatformMacOS}, `["iphone","ipad","macos"]`), + Entry("unknown platforms", []Platform{PlatformUnknown}, `["unknown"]`), + ) + It("formats ipa name correctly", func() { app := App{ ID: 42, diff --git a/pkg/appstore/appstore.go b/pkg/appstore/appstore.go index bc1d8f6..fcdb4cd 100644 --- a/pkg/appstore/appstore.go +++ b/pkg/appstore/appstore.go @@ -1,6 +1,8 @@ package appstore import ( + "time" + "github.com/majd/ipatool/v2/pkg/anisette" "github.com/majd/ipatool/v2/pkg/gsa" "github.com/majd/ipatool/v2/pkg/http" @@ -62,20 +64,25 @@ type gsaClient interface { } type appstore struct { - keychain keychain.Keychain - cookieJar http.CookieJar - loginClient http.Client[loginResult] - searchClient http.Client[searchResult] - purchaseClient http.Client[purchaseResult] - downloadClient http.Client[downloadResult] - platformClient http.Client[platformVersionLookupResult] - bagClient http.Client[bagResult] - ownedAppsClient http.Client[[]byte] - httpClient http.Client[interface{}] - machine machine.Machine - os operatingsystem.OperatingSystem - gsa gsaClient + keychain keychain.Keychain + cookieJar http.CookieJar + storefrontClient http.Client[[]byte] + loginClient http.Client[loginResult] + searchClient http.Client[searchResult] + purchaseClient http.Client[purchaseResult] + downloadClient http.Client[downloadResult] + platformClient http.Client[platformVersionLookupResult] + bagClient http.Client[bagResult] + ownedAppsClient http.Client[[]byte] + httpClient http.Client[interface{}] + machine machine.Machine + os operatingsystem.OperatingSystem + gsa gsaClient anisette anisette.Provider + + // authRetrySleep waits between authentication request retries. Tests + // replace it to avoid real delays. + authRetrySleep func(delay time.Duration) } type Args struct { @@ -92,19 +99,20 @@ func NewAppStore(args Args) AppStore { } return &appstore{ - keychain: args.Keychain, - cookieJar: args.CookieJar, - loginClient: http.NewClient[loginResult](clientArgs), - searchClient: http.NewClient[searchResult](clientArgs), - purchaseClient: http.NewClient[purchaseResult](clientArgs), - downloadClient: http.NewClient[downloadResult](clientArgs), - platformClient: http.NewClient[platformVersionLookupResult](clientArgs), - bagClient: http.NewClient[bagResult](clientArgs), - ownedAppsClient: http.NewClient[[]byte](clientArgs), - httpClient: http.NewClient[interface{}](clientArgs), - machine: args.Machine, - os: args.OperatingSystem, - gsa: gsa.NewClient(args.CookieJar), - anisette: anisette.NewProvider(nil), + keychain: args.Keychain, + cookieJar: args.CookieJar, + storefrontClient: http.NewClient[[]byte](clientArgs), + loginClient: http.NewClient[loginResult](http.Args{CookieJar: args.CookieJar, ActionSigner: mescal.Sign, Authentication: true}), + searchClient: http.NewClient[searchResult](clientArgs), + purchaseClient: http.NewClient[purchaseResult](clientArgs), + downloadClient: http.NewClient[downloadResult](clientArgs), + platformClient: http.NewClient[platformVersionLookupResult](clientArgs), + bagClient: http.NewClient[bagResult](clientArgs), + ownedAppsClient: http.NewClient[[]byte](clientArgs), + httpClient: http.NewClient[interface{}](clientArgs), + machine: args.Machine, + os: args.OperatingSystem, + gsa: gsa.NewClient(args.CookieJar), + anisette: anisette.NewProvider(nil), } } diff --git a/pkg/appstore/appstore_download.go b/pkg/appstore/appstore_download.go index d13643d..a1e5fc9 100644 --- a/pkg/appstore/appstore_download.go +++ b/pkg/appstore/appstore_download.go @@ -2,9 +2,11 @@ package appstore import ( "archive/zip" + "context" "errors" "fmt" "io" + gohttp "net/http" "os" "path/filepath" "strconv" @@ -271,6 +273,15 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { } } + if externalVersionID == "" && input.Platform == PlatformMacOS { + // The legacy MDM lookup can return an iOS offer even for Mac + // downloads, so the Mac product page is consulted instead. + externalVersionID, err = t.lookupLatestMacOSExternalVersionID(input.Account, input.App) + if err != nil { + return DownloadOutput{}, fmt.Errorf("failed to resolve latest macOS version for download: %w", err) + } + } + item, err := t.fetchDownloadItem(input.Account, input.App, guid, externalVersionID) if err != nil { return DownloadOutput{}, err @@ -298,7 +309,12 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { return DownloadOutput{}, fmt.Errorf("failed to download file: %w", err) } - err = t.applyPatches(item, input.Account, tmpPath, destination) + artwork, err := t.downloadArtwork(context.Background(), item.ArtworkURL) + if err != nil { + return DownloadOutput{}, fmt.Errorf("failed to download artwork: %w", err) + } + + err = t.applyPatches(item, input.Account, tmpPath, destination, artwork) if err != nil { return DownloadOutput{}, fmt.Errorf("failed to apply patches: %w", err) } @@ -399,10 +415,11 @@ func (*appstore) validatePackagePlatform(path string, platform Platform) error { } type downloadItemResult struct { - HashMD5 string `plist:"md5,omitempty"` - URL string `plist:"URL,omitempty"` - Sinfs []Sinf `plist:"sinfs,omitempty"` - Metadata map[string]interface{} `plist:"metadata,omitempty"` + ArtworkURL string `plist:"artworkURL,omitempty"` + HashMD5 string `plist:"md5,omitempty"` + URL string `plist:"URL,omitempty"` + Sinfs []Sinf `plist:"sinfs,omitempty"` + Metadata map[string]interface{} `plist:"metadata,omitempty"` } type downloadResult struct { @@ -411,7 +428,8 @@ type downloadResult struct { Items []downloadItemResult `plist:"songList,omitempty"` } -func (t *appstore) downloadFile(src, dst string, progress *progressbar.ProgressBar, progressWriter io.Writer, onTotal func(int64)) error { +//nolint:nonamedreturns // Deferred close errors must propagate to callers. +func (t *appstore) downloadFile(src, dst string, progress *progressbar.ProgressBar, progressWriter io.Writer, onTotal func(int64)) (err error) { req, err := t.httpClient.NewRequest("GET", src, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) @@ -422,7 +440,11 @@ func (t *appstore) downloadFile(src, dst string, progress *progressbar.ProgressB return fmt.Errorf("failed to open file: %w", err) } - defer file.Close() + defer func() { + if closeErr := file.Close(); closeErr != nil { + err = joinCleanupError(err, "failed to close downloaded file", closeErr) + } + }() stat, err := t.os.Stat(dst) if err != nil { @@ -439,41 +461,60 @@ func (t *appstore) downloadFile(src, dst string, progress *progressbar.ProgressB } defer res.Body.Close() - total := res.ContentLength + stat.Size() - if onTotal != nil { - onTotal(total) + offset, remaining, total, complete, err := downloadResponseRange(res, stat.Size()) + if err != nil { + return err } - if progress != nil { - progress.ChangeMax64(total) - err = progress.Set64(stat.Size()) - if err != nil { - return fmt.Errorf("can not set bar progress: %w", err) + if complete { + return nil + } + + if res.StatusCode == gohttp.StatusOK { + if err := file.Truncate(0); err != nil { + return fmt.Errorf("failed to restart download: %w", err) } } - // Seek to the end so a resumed download appends after the already-downloaded - // range (for a fresh file this is a no-op at offset 0). - _, err = file.Seek(0, io.SeekEnd) - if err != nil { + if _, err := file.Seek(offset, io.SeekStart); err != nil { return fmt.Errorf("can not seek file: %w", err) } + if onTotal != nil { + onTotal(total) + } + // The CLI progress bar (progress) and any raw-bytes tracker (progressWriter, // e.g. the GUI progress tracker) each receive the raw downloaded bytes. writers := []io.Writer{file} + if progress != nil { + progress.ChangeMax64(total) + + if err := progress.Set64(offset); err != nil { + return fmt.Errorf("can not set bar progress: %w", err) + } + writers = append(writers, progress) } if progressWriter != nil { writers = append(writers, progressWriter) } - _, err = io.Copy(io.MultiWriter(writers...), res.Body) + var body io.Reader = res.Body + if remaining >= 0 { + body = io.LimitReader(body, remaining) + } + + written, err := io.Copy(io.MultiWriter(writers...), body) if err != nil { return fmt.Errorf("failed to write file: %w", err) } + if remaining >= 0 && written != remaining || total >= 0 && offset+written != total { + return fmt.Errorf("download is incomplete: %w", io.ErrUnexpectedEOF) + } + return nil } @@ -622,7 +663,7 @@ func (t *appstore) isDirectory(path string) (bool, error) { return info.IsDir(), nil } -func (t *appstore) applyPatches(item downloadItemResult, acc Account, src, dst string) error { +func (t *appstore) applyPatches(item downloadItemResult, acc Account, src, dst string, artwork []byte) error { srcZip, err := zip.OpenReader(src) if err != nil { return fmt.Errorf("failed to open zip reader: %w", err) @@ -638,11 +679,22 @@ func (t *appstore) applyPatches(item downloadItemResult, acc Account, src, dst s dstZip := zip.NewWriter(dstFile) defer dstZip.Close() - err = t.replicateZip(srcZip, dstZip) + err = t.replicateZip(srcZip, dstZip, src) if err != nil { return fmt.Errorf("failed to replicate zip: %w", err) } + if len(artwork) != 0 { + file, err := dstZip.Create("iTunesArtwork") + if err != nil { + return fmt.Errorf("failed to create artwork: %w", err) + } + + if _, err := file.Write(artwork); err != nil { + return fmt.Errorf("failed to write artwork: %w", err) + } + } + err = t.writeMetadata(item.Metadata, acc, dstZip) if err != nil { return fmt.Errorf("failed to write metadata: %w", err) diff --git a/pkg/appstore/appstore_download_artwork.go b/pkg/appstore/appstore_download_artwork.go new file mode 100644 index 0000000..7accf0d --- /dev/null +++ b/pkg/appstore/appstore_download_artwork.go @@ -0,0 +1,46 @@ +package appstore + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" +) + +func (t *appstore) downloadArtwork(ctx context.Context, url string) ([]byte, error) { + if url == "" { + return nil, nil + } + + if ctx == nil { + ctx = context.Background() + } + + req, err := t.httpClient.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + res, err := t.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected artwork response status: %d", res.StatusCode) + } + + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read artwork: %w", err) + } + + if len(data) == 0 { + return nil, errors.New("artwork response is empty") + } + + return data, nil +} diff --git a/pkg/appstore/appstore_download_artwork_test.go b/pkg/appstore/appstore_download_artwork_test.go new file mode 100644 index 0000000..12c4faf --- /dev/null +++ b/pkg/appstore/appstore_download_artwork_test.go @@ -0,0 +1,112 @@ +package appstore + +import ( + "archive/zip" + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + apphttp "github.com/majd/ipatool/v2/pkg/http" + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" +) + +func TestDownloadArtworkAndIncludeInIPA(t *testing.T) { + requests := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + _, _ = io.WriteString(w, "artwork bytes") + })) + defer server.Close() + + store := &appstore{httpClient: apphttp.NewClient[interface{}](apphttp.Args{}), os: operatingsystem.New()} + for _, url := range []string{"", server.URL} { + data, err := store.downloadArtwork(context.Background(), url) + if err != nil { + t.Fatal(err) + } + + if url == "" && (len(data) != 0 || requests != 0) { + t.Fatal("artwork was downloaded without a URL") + } + + if url != "" && string(data) != "artwork bytes" { + t.Fatal("artwork missing") + } + + path := filepath.Join(t.TempDir(), "source.zip") + + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + + writer := zip.NewWriter(file) + if _, err := writer.Create("Payload/App.app/"); err != nil { + t.Fatal(err) + } + + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + if err := file.Close(); err != nil { + t.Fatal(err) + } + + if err := store.applyPatches(downloadItemResult{Metadata: map[string]interface{}{}}, Account{}, path, path+".ipa", data); err != nil { + t.Fatal(err) + } + + reader, err := zip.OpenReader(path + ".ipa") + if err != nil { + t.Fatal(err) + } + + found := false + + for _, entry := range reader.File { + if entry.Name != "iTunesArtwork" { + continue + } + + found = true + + r, err := entry.Open() + if err != nil { + t.Fatal(err) + } + + actual, err := io.ReadAll(r) + r.Close() + + if err != nil || string(actual) != "artwork bytes" { + t.Fatalf("artwork = %q; %v", actual, err) + } + } + + reader.Close() + + if found != (url != "") { + t.Fatal("incorrect artwork presence") + } + } +} + +func TestDownloadArtworkRejectsFailedResponse(t *testing.T) { + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + + store := &appstore{httpClient: apphttp.NewClient[interface{}](apphttp.Args{})} + if _, err := store.downloadArtwork(context.Background(), server.URL); err == nil { + t.Fatal("accepted error page as artwork") + } + + if data, err := store.downloadArtwork(context.Background(), ""); err != nil || len(data) != 0 { + t.Fatal("missing artwork URL should be skipped") + } +} diff --git a/pkg/appstore/appstore_download_range.go b/pkg/appstore/appstore_download_range.go new file mode 100644 index 0000000..a537843 --- /dev/null +++ b/pkg/appstore/appstore_download_range.go @@ -0,0 +1,47 @@ +package appstore + +import ( + "fmt" + "net/http" + "strconv" + "strings" +) + +// downloadResponseRange validates the response before any partial bytes change. +// It returns the write offset, remaining bytes, total size, and completion state. +func downloadResponseRange(res *http.Response, localSize int64) (int64, int64, int64, bool, error) { + switch res.StatusCode { + case http.StatusOK: + return 0, res.ContentLength, res.ContentLength, false, nil + case http.StatusRequestedRangeNotSatisfiable: + value, found := strings.CutPrefix(res.Header.Get("Content-Range"), "bytes */") + size, parseErr := strconv.ParseInt(value, 10, 64) + + if found && parseErr == nil && size >= 0 && size == localSize { + return localSize, 0, size, true, nil + } + + return 0, 0, 0, false, fmt.Errorf("download range rejected: local size %d does not match server range %q", localSize, res.Header.Get("Content-Range")) + case http.StatusPartialContent: + header := res.Header.Get("Content-Range") + value, found := strings.CutPrefix(header, "bytes ") + bounds, totalText, hasTotal := strings.Cut(value, "/") + startText, endText, hasEnd := strings.Cut(bounds, "-") + start, startErr := strconv.ParseInt(startText, 10, 64) + end, endErr := strconv.ParseInt(endText, 10, 64) + size, sizeErr := strconv.ParseInt(totalText, 10, 64) + + if !found || !hasTotal || !hasEnd || startErr != nil || endErr != nil || sizeErr != nil || start != localSize || start < 0 || end < start || size <= end { + return 0, 0, 0, false, fmt.Errorf("invalid download content range %q for local size %d", header, localSize) + } + + length := end - start + 1 + if res.ContentLength >= 0 && res.ContentLength != length { + return 0, 0, 0, false, fmt.Errorf("download content length %d does not match range length %d", res.ContentLength, length) + } + + return start, length, size, false, nil + default: + return 0, 0, 0, false, fmt.Errorf("unexpected download response status: %d", res.StatusCode) + } +} diff --git a/pkg/appstore/appstore_download_resume_test.go b/pkg/appstore/appstore_download_resume_test.go new file mode 100644 index 0000000..d626829 --- /dev/null +++ b/pkg/appstore/appstore_download_resume_test.go @@ -0,0 +1,130 @@ +package appstore + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + apphttp "github.com/majd/ipatool/v2/pkg/http" + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" + "github.com/schollz/progressbar/v3" +) + +func TestDownloadFileResumesWithAndWithoutProgress(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Range"); got != "bytes=7-" { + t.Errorf("range = %q", got) + } + + w.Header().Set("Content-Range", "bytes 7-14/15") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "download") + })) + defer server.Close() + + for _, interactive := range []bool{false, true} { + t.Run(map[bool]string{false: "non-interactive", true: "interactive"}[interactive], func(t *testing.T) { + path := filepath.Join(t.TempDir(), "app.tmp") + if err := os.WriteFile(path, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + + store := &appstore{httpClient: apphttp.NewClient[interface{}](apphttp.Args{}), os: operatingsystem.New()} + + var progress *progressbar.ProgressBar + if interactive { + progress = progressbar.NewOptions64(15, progressbar.OptionSetWriter(io.Discard)) + } + + if err := store.downloadFile(server.URL, path, progress, nil, nil); err != nil { + t.Fatal(err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + if string(data) != "partialdownload" { + t.Fatalf("download = %q", data) + } + }) + } +} + +func TestDownloadFileValidatesResumeResponses(t *testing.T) { + for _, test := range []struct { + name string + status int + rangeHeader string + body string + want string + wantError bool + chunked bool + }{ + {name: "range ignored", status: http.StatusOK, body: "new", want: "new"}, + {name: "already complete", status: http.StatusRequestedRangeNotSatisfiable, rangeHeader: "bytes */7", body: "error page", want: "partial"}, + {name: "stale oversized partial", status: http.StatusRequestedRangeNotSatisfiable, rangeHeader: "bytes */3", body: "error page", want: "partial", wantError: true}, + {name: "incomplete rejected partial", status: http.StatusRequestedRangeNotSatisfiable, rangeHeader: "bytes */15", want: "partial", wantError: true}, + {name: "malformed unsatisfied range", status: http.StatusRequestedRangeNotSatisfiable, rangeHeader: "7", want: "partial", wantError: true}, + {name: "http error", status: http.StatusForbidden, body: "error page", want: "partial", wantError: true}, + {name: "wrong offset", status: http.StatusPartialContent, rangeHeader: "bytes 0-7/15", body: "download", want: "partial", wantError: true}, + {name: "missing range", status: http.StatusPartialContent, body: "download", want: "partial", wantError: true}, + {name: "malformed range", status: http.StatusPartialContent, rangeHeader: "bytes 7-x/15", body: "download", want: "partial", wantError: true}, + {name: "invalid total", status: http.StatusPartialContent, rangeHeader: "bytes 7-14/14", body: "download", want: "partial", wantError: true}, + {name: "length mismatch", status: http.StatusPartialContent, rangeHeader: "bytes 7-14/15", body: "bad", want: "partial", wantError: true}, + {name: "truncated chunked response", status: http.StatusPartialContent, rangeHeader: "bytes 7-14/15", body: "dow", want: "partialdow", wantError: true, chunked: true}, + } { + for _, interactive := range []bool{false, true} { + t.Run(test.name+map[bool]string{false: "/non-interactive", true: "/interactive"}[interactive], func(t *testing.T) { + path := filepath.Join(t.TempDir(), "app.tmp") + if err := os.WriteFile(path, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Range") != "bytes=7-" { + t.Errorf("range = %q", r.Header.Get("Range")) + } + + if test.rangeHeader != "" { + w.Header().Set("Content-Range", test.rangeHeader) + } + + w.WriteHeader(test.status) + + if test.chunked { + w.(http.Flusher).Flush() + } + + _, _ = io.WriteString(w, test.body) + })) + defer server.Close() + + store := &appstore{httpClient: apphttp.NewClient[interface{}](apphttp.Args{}), os: operatingsystem.New()} + + var progress *progressbar.ProgressBar + if interactive { + progress = progressbar.NewOptions64(1, progressbar.OptionSetWriter(io.Discard)) + } + + err := store.downloadFile(server.URL, path, progress, nil, nil) + if (err != nil) != test.wantError { + t.Fatalf("download error = %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + if string(data) != test.want { + t.Fatalf("file = %q, want %q", data, test.want) + } + }) + } + } +} diff --git a/pkg/appstore/appstore_download_test.go b/pkg/appstore/appstore_download_test.go index 98904a8..5285f78 100644 --- a/pkg/appstore/appstore_download_test.go +++ b/pkg/appstore/appstore_download_test.go @@ -39,6 +39,7 @@ var _ = Describe("AppStore (Download)", func() { mockPlatformClient *http.MockClient[platformVersionLookupResult] mockPurchaseClient *http.MockClient[purchaseResult] mockLoginClient *http.MockClient[loginResult] + mockStorefront *http.MockClient[[]byte] mockHTTPClient *http.MockClient[interface{}] mockOS *operatingsystem.MockOperatingSystem mockMachine *machine.MockMachine @@ -53,17 +54,19 @@ var _ = Describe("AppStore (Download)", func() { mockLoginClient = http.NewMockClient[loginResult](ctrl) mockPurchaseClient = http.NewMockClient[purchaseResult](ctrl) mockHTTPClient = http.NewMockClient[interface{}](ctrl) + mockStorefront = http.NewMockClient[[]byte](ctrl) mockOS = operatingsystem.NewMockOperatingSystem(ctrl) mockMachine = machine.NewMockMachine(ctrl) as = &appstore{ - keychain: mockKeychain, - loginClient: mockLoginClient, - purchaseClient: mockPurchaseClient, - downloadClient: mockDownloadClient, - platformClient: mockPlatformClient, - httpClient: mockHTTPClient, - machine: mockMachine, - os: mockOS, + keychain: mockKeychain, + loginClient: mockLoginClient, + purchaseClient: mockPurchaseClient, + downloadClient: mockDownloadClient, + platformClient: mockPlatformClient, + storefrontClient: mockStorefront, + httpClient: mockHTTPClient, + machine: mockMachine, + os: mockOS, } }) @@ -187,6 +190,47 @@ var _ = Describe("AppStore (Download)", func() { }) }) + When("the platform is macOS and no external version id is provided", func() { + BeforeEach(func() { + mockMachine.EXPECT(). + MacAddress(). + Return("00:11:22:33:44:55", nil) + + mockStorefront.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://apps.apple.com/de/app/id6472431552?platform=mac")) + }). + Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: macVersionPage(karingMacConfiguration), + }, nil) + + mockDownloadClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + payload, ok := req.Payload.(*http.XMLPayload) + Expect(ok).To(BeTrue()) + Expect(payload.Content["externalVersionId"]).To(Equal("876660716")) + }). + Return(http.Result[downloadResult]{}, errors.New("request error")) + }) + + It("resolves and sends the macOS external version id", func() { + _, err := as.Download(DownloadInput{ + Account: Account{ + StoreFront: "143443", + }, + App: App{ + ID: 6472431552, + BundleID: "com.nebula.karing", + }, + Platform: PlatformMacOS, + }) + Expect(err).To(HaveOccurred()) + }) + }) + DescribeTable("platform uses the standard download request", func(platform Platform) { mockMachine.EXPECT(). @@ -487,7 +531,9 @@ var _ = Describe("AppStore (Download)", func() { mockHTTPClient.EXPECT(). Do(gomock.Any()). Return(&gohttp.Response{ - Body: io.NopCloser(strings.NewReader("ping")), + StatusCode: gohttp.StatusOK, + Body: io.NopCloser(strings.NewReader("ping")), + ContentLength: 4, }, nil) }) @@ -547,7 +593,9 @@ var _ = Describe("AppStore (Download)", func() { mockHTTPClient.EXPECT(). Do(gomock.Any()). Return(&gohttp.Response{ - Body: io.NopCloser(strings.NewReader("ping")), + StatusCode: gohttp.StatusOK, + Body: io.NopCloser(strings.NewReader("ping")), + ContentLength: 4, }, nil) }) @@ -589,7 +637,7 @@ var _ = Describe("AppStore (Download)", func() { mockOS.EXPECT(). Stat(gomock.Any()). - Return(nil, nil) + Return(&dummyFileInfo{}, nil) mockOS.EXPECT(). Remove(tmpFile.Name()). @@ -665,4 +713,50 @@ var _ = Describe("AppStore (Download)", func() { Expect(err.Error()).To(ContainSubstring("AppleTVOS")) }) }) + + Describe("resuming a download", func() { + It("appends the ranged response to the partial file", func() { + testFile, err := os.CreateTemp("", "ipatool-download-*") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(testFile.Name()) + + _, err = testFile.WriteString("partial-") + Expect(err).ToNot(HaveOccurred()) + _, err = testFile.Seek(0, io.SeekStart) + Expect(err).ToNot(HaveOccurred()) + + request := &gohttp.Request{Header: make(gohttp.Header)} + info, err := testFile.Stat() + Expect(err).ToNot(HaveOccurred()) + + mockHTTPClient.EXPECT(). + NewRequest("GET", "https://example.com/app.ipa", nil). + Return(request, nil) + mockOS.EXPECT(). + OpenFile(testFile.Name(), os.O_CREATE|os.O_RDWR, os.FileMode(0644)). + Return(testFile, nil) + mockOS.EXPECT(). + Stat(testFile.Name()). + Return(info, nil) + mockHTTPClient.EXPECT(). + Do(request). + DoAndReturn(func(request *gohttp.Request) (*gohttp.Response, error) { + Expect(request.Header.Get("range")).To(Equal("bytes=8-")) + + return &gohttp.Response{ + StatusCode: gohttp.StatusPartialContent, + Header: gohttp.Header{"Content-Range": []string{"bytes 8-16/17"}}, + ContentLength: 9, + Body: io.NopCloser(strings.NewReader("remainder")), + }, nil + }) + + err = as.(*appstore).downloadFile("https://example.com/app.ipa", testFile.Name(), nil, nil, nil) + Expect(err).ToNot(HaveOccurred()) + + data, err := os.ReadFile(testFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(data)).To(Equal("partial-remainder")) + }) + }) }) diff --git a/pkg/appstore/appstore_login.go b/pkg/appstore/appstore_login.go index 3acb9c0..ff561a8 100644 --- a/pkg/appstore/appstore_login.go +++ b/pkg/appstore/appstore_login.go @@ -9,6 +9,7 @@ import ( "runtime" "strconv" "strings" + "time" "github.com/majd/ipatool/v2/pkg/gsa" "github.com/majd/ipatool/v2/pkg/http" @@ -21,6 +22,12 @@ var ( const legacyAuthenticateEndpoint = "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" +const ( + maxAuthenticationRequestAttempts = 3 + authenticationRetryDelay = 10 * time.Second + maxAuthenticationRetryDelay = 30 * time.Second +) + type LoginInput struct { Email string Password string @@ -233,14 +240,23 @@ func (t *appstore) login(email, password, authCode, guid, endpoint string) (Acco request := t.loginRequest(email, password, authCode, guid, endpoint, requestAttempt) request.URL, _ = util.IfEmpty(redirect, request.URL), "" - res, err = t.loginClient.Send(request) + res, err = t.sendAuthenticationRequest(request) if err != nil { if shouldRetryWithLegacyAuthenticate(endpoint, err) { return t.login(email, password, authCode, guid, legacyAuthenticateEndpoint) } - return Account{}, fmt.Errorf("request failed: %w", err) + stage := "sign-in" + if authCode != "" { + stage = "2FA verification" + } + + if redirect != "" { + stage += " at Store pod" + } + + return Account{}, fmt.Errorf("%s request failed: %w", stage, err) } if retry, redirect, err = t.parseLoginResponse(&res, authCode); err != nil { @@ -286,6 +302,109 @@ func (t *appstore) login(email, password, authCode, guid, endpoint string) (Acco return acc, nil } +// sendAuthenticationRequest repeats an authentication request while Apple +// answers with a transient failure (empty 204, 404, 429 or a 5xx gateway +// error), honoring a Retry-After header when Apple sends one. +func (t *appstore) sendAuthenticationRequest(request http.Request) (http.Result[loginResult], error) { + statuses := make([]string, 0, maxAuthenticationRequestAttempts) + + sleep := t.authRetrySleep + if sleep == nil { + sleep = time.Sleep + } + + for attempt := 1; ; attempt++ { + result, err := t.loginClient.Send(request) + + status, retry := retryableAuthenticationError(err) + if !retry { + if err != nil { + return result, authenticationRequestError(err) + } + + return result, nil + } + + statuses = append(statuses, strconv.Itoa(status)) + + if attempt == maxAuthenticationRequestAttempts { + return result, fmt.Errorf( + "authentication request failed after %d attempts (HTTP %s): %w", + maxAuthenticationRequestAttempts, strings.Join(statuses, ", "), authenticationRequestError(err), + ) + } + + delay := min(authenticationRetryDelay<<(attempt-1), maxAuthenticationRetryDelay) + + var responseErr *http.UnexpectedResponseError + if errors.As(err, &responseErr) { + if requested, ok := authenticationRetryAfter(responseErr.RetryAfter, time.Now()); ok { + if requested > maxAuthenticationRetryDelay { + return result, fmt.Errorf("apple requested a wait longer than %s; try again later: %w", maxAuthenticationRetryDelay, err) + } + + // Retry-After takes precedence over the fallback backoff. + delay = max(requested, time.Second) + } + } + + sleep(delay) + } +} + +// retryableAuthenticationError reports whether the authentication failure is +// transient enough to be worth another attempt. +func retryableAuthenticationError(err error) (int, bool) { + var responseErr *http.UnexpectedResponseError + if !errors.As(err, &responseErr) { + return 0, false + } + + status := responseErr.StatusCode + retry := status == gohttp.StatusNoContent || + status == gohttp.StatusNotFound || + status == gohttp.StatusTooManyRequests || + status/100 == 5 + + return status, retry +} + +// authenticationRequestError turns a raw unexpected-response failure into a +// message that tells the user what to do instead of leaking Apple's HTML. +func authenticationRequestError(err error) error { + var responseErr *http.UnexpectedResponseError + if !errors.As(err, &responseErr) { + return err + } + + if responseErr.StatusCode == gohttp.StatusTooManyRequests { + return fmt.Errorf("apple rate limited authentication; try again later: %w", err) + } + + return fmt.Errorf("apple returned no usable authentication response; try again later or from another network: %w", err) +} + +// authenticationRetryAfter parses a Retry-After header value (delta-seconds +// or HTTP-date) into a wait duration. +func authenticationRetryAfter(value string, now time.Time) (time.Duration, bool) { + value = strings.TrimSpace(value) + if seconds, err := strconv.ParseUint(value, 10, 64); err == nil { + // Saturate before converting to Duration to avoid overflow. A wait over + // the budget ends this login rather than retrying before Apple's deadline. + if seconds > uint64(maxAuthenticationRetryDelay/time.Second) { + return maxAuthenticationRetryDelay + time.Second, true + } + + return time.Duration(seconds) * time.Second, true + } + + if date, err := gohttp.ParseTime(value); err == nil { + return max(time.Duration(0), date.Sub(now)), true + } + + return 0, false +} + func shouldRetryWithLegacyAuthenticate(endpoint string, err error) bool { if !strings.Contains(endpoint, "/native/") { return false @@ -325,7 +444,7 @@ func (t *appstore) parseLoginResponse(res *http.Result[loginResult], authCode st if res.Data.CustomerMessage != "" { err = NewErrorWithMetadata(errors.New(res.Data.CustomerMessage), res) } else { - err = NewErrorWithMetadata(fmt.Errorf("something went wrong (failure type %s)", res.Data.FailureType), res) + err = NewErrorWithMetadata(fmt.Errorf("apple returned no usable authentication response (HTTP %d): missing account credentials or unexpected status; try again later or from another network", res.StatusCode), res) } } else if res.StatusCode != gohttp.StatusOK || res.Data.PasswordToken == "" || res.Data.DirectoryServicesID == "" { err = NewErrorWithMetadata(errors.New("something went wrong"), res) diff --git a/pkg/appstore/appstore_login_test.go b/pkg/appstore/appstore_login_test.go index 6cfb32c..af9857f 100644 --- a/pkg/appstore/appstore_login_test.go +++ b/pkg/appstore/appstore_login_test.go @@ -257,7 +257,7 @@ var _ = Describe("AppStore (Login)", func() { _, err := as.Login(LoginInput{ Password: testPassword, }) - Expect(err).To(MatchError("request failed: test complete")) + Expect(err).To(MatchError("sign-in at Store pod request failed: test complete")) }) }) diff --git a/pkg/appstore/appstore_macos_version_lookup.go b/pkg/appstore/appstore_macos_version_lookup.go new file mode 100644 index 0000000..9616ce9 --- /dev/null +++ b/pkg/appstore/appstore_macos_version_lookup.go @@ -0,0 +1,131 @@ +package appstore + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + gohttp "net/http" + "net/url" + "slices" + "strconv" + "strings" + + "github.com/majd/ipatool/v2/pkg/http" +) + +const serializedServerDataID = "serialized-server-data" + +// serializedServerData extracts the JSON blob Apple embeds in its storefront +// pages inside the serialized-server-data script tag. +func serializedServerData(body []byte) ([]byte, error) { + markerIndex := bytes.Index(body, []byte(`id="`+serializedServerDataID+`"`)) + if markerIndex == -1 { + markerIndex = bytes.Index(body, []byte(`id='`+serializedServerDataID+`'`)) + } + + if markerIndex == -1 { + return nil, errors.New("serialized server data was not found") + } + + scriptStart := bytes.LastIndex(body[:markerIndex], []byte("') + if contentOffset == -1 { + return nil, errors.New("serialized server data script is malformed") + } + + contentStart := markerIndex + contentOffset + 1 + + contentEndOffset := bytes.Index(body[contentStart:], []byte("")) + if contentEndOffset == -1 { + return nil, errors.New("serialized server data script is not closed") + } + + return bytes.TrimSpace(body[contentStart : contentStart+contentEndOffset]), nil +} + +func (t *appstore) lookupLatestMacOSExternalVersionID(acc Account, app App) (string, error) { + countryCode, err := countryCodeFromStoreFront(acc.StoreFront) + if err != nil { + return "", fmt.Errorf("failed to resolve the country code: %w", err) + } + + // The legacy MDM lookup can return an iOS offer even with platform=osx. + // Use the Mac product page to select the native Mac offer instead. + res, err := t.storefrontClient.Send(http.Request{ + URL: fmt.Sprintf("https://apps.apple.com/%s/app/id%d?platform=mac", strings.ToLower(countryCode), app.ID), + Method: http.MethodGET, + ResponseFormat: http.ResponseFormatRaw, + }) + if err != nil { + return "", fmt.Errorf("macOS version lookup request failed: %w", err) + } + + if res.StatusCode != gohttp.StatusOK { + return "", fmt.Errorf("macOS version lookup returned HTTP %d", res.StatusCode) + } + + return macOSExternalVersionID(res.Data, app) +} + +func macOSExternalVersionID(body []byte, app App) (string, error) { + data, err := serializedServerData(body) + if err != nil { + return "", err + } + + var value interface{} + if err := json.Unmarshal(data, &value); err != nil { + return "", fmt.Errorf("failed to decode Mac product page: %w", err) + } + + versions := make(map[string]struct{}) + collectMacOSExternalVersions(value, app, versions) + + if len(versions) == 0 { + return "", errors.New("macOS purchase configuration has no external version id for the requested app") + } + + if len(versions) != 1 { + return "", errors.New("macOS purchase configurations contain conflicting external version ids") + } + + for version := range versions { + return version, nil + } + + return "", errors.New("macOS external version id was not found") +} + +func collectMacOSExternalVersions(value interface{}, app App, versions map[string]struct{}) { + switch value := value.(type) { + case []interface{}: + for _, child := range value { + collectMacOSExternalVersions(child, app, versions) + } + case map[string]interface{}: + if configuration, ok := value["purchaseConfiguration"].(map[string]interface{}); ok { + platforms, _ := configuration["appPlatforms"].([]interface{}) + bundleID, _ := configuration["bundleId"].(string) + buyParams, _ := configuration["buyParams"].(string) + + params, err := url.ParseQuery(buyParams) + if err == nil && slices.Contains(platforms, interface{}("mac")) && + params.Get("salableAdamId") == strconv.FormatInt(app.ID, 10) && + (app.BundleID == "" || app.BundleID == bundleID) { + version := params.Get("appExtVrsId") + if id, err := strconv.ParseUint(version, 10, 64); err == nil && id != 0 { + versions[version] = struct{}{} + } + } + } + + for _, child := range value { + collectMacOSExternalVersions(child, app, versions) + } + } +} diff --git a/pkg/appstore/appstore_macos_version_lookup_test.go b/pkg/appstore/appstore_macos_version_lookup_test.go new file mode 100644 index 0000000..08ddedd --- /dev/null +++ b/pkg/appstore/appstore_macos_version_lookup_test.go @@ -0,0 +1,54 @@ +package appstore + +import ( + "errors" + + "github.com/majd/ipatool/v2/pkg/http" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" +) + +const karingMacConfiguration = `{"purchaseConfiguration":{"bundleId":"com.nebula.karing","appPlatforms":["mac","tv","phone","pad"],"metricsPlatformDisplayStyle":"ios","buyParams":"salableAdamId=6472431552&appExtVrsId=876660716"}}` + +func macVersionPage(configuration string) []byte { + return []byte(``) +} + +var _ = Describe("Mac purchase version selection", func() { + app := App{ID: 6472431552, BundleID: "com.nebula.karing"} + + It("selects Karing's Mac offer and ignores unrelated and iOS offers", func() { + body := macVersionPage(`[{"purchaseConfiguration":{"bundleId":"com.nebula.karing","appPlatforms":["phone","pad"],"buyParams":"salableAdamId=6472431552&appExtVrsId=891116578"}}, {"purchaseConfiguration":{"appPlatforms":["mac"],"buyParams":"salableAdamId=100&appExtVrsId=123"}}, ` + karingMacConfiguration + `]`) + version, err := macOSExternalVersionID(body, app) + Expect(err).ToNot(HaveOccurred()) + Expect(version).To(Equal("876660716")) + }) + + DescribeTable("rejects invalid catalog metadata", func(configuration string) { + _, err := macOSExternalVersionID(macVersionPage(configuration), app) + Expect(err).To(HaveOccurred()) + }, + Entry("invalid JSON", `{`), + Entry("no offer", `{}`), + Entry("iOS only", `{"purchaseConfiguration":{"bundleId":"com.nebula.karing","appPlatforms":["phone","pad"],"buyParams":"salableAdamId=6472431552&appExtVrsId=891116578"}}`), + Entry("wrong bundle", `{"purchaseConfiguration":{"bundleId":"other","appPlatforms":["mac"],"buyParams":"salableAdamId=6472431552&appExtVrsId=123"}}`), + Entry("invalid version", `{"purchaseConfiguration":{"bundleId":"com.nebula.karing","appPlatforms":["mac"],"buyParams":"salableAdamId=6472431552&appExtVrsId=invalid"}}`), + Entry("conflicting offers", `[`+karingMacConfiguration+`,{"purchaseConfiguration":{"bundleId":"com.nebula.karing","appPlatforms":["mac"],"buyParams":"salableAdamId=6472431552&appExtVrsId=123"}}]`), + ) + + + DescribeTable("propagates lookup failures", func(status int, body []byte, requestErr error) { + ctrl := gomock.NewController(GinkgoT()) + defer ctrl.Finish() + pages := http.NewMockClient[[]byte](ctrl) + store := &appstore{storefrontClient: pages} + pages.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: status, Data: body}, requestErr) + _, err := store.lookupLatestMacOSExternalVersionID(Account{StoreFront: "143443-2,34"}, app) + Expect(err).To(HaveOccurred()) + }, + Entry("HTTP failure", 500, nil, nil), + Entry("network failure", 0, nil, errors.New("connection reset")), + Entry("missing Mac offer", 200, macVersionPage(`{}`), nil), + ) +}) diff --git a/pkg/appstore/appstore_owned_apps.go b/pkg/appstore/appstore_owned_apps.go index 9b876cc..6699062 100644 --- a/pkg/appstore/appstore_owned_apps.go +++ b/pkg/appstore/appstore_owned_apps.go @@ -6,6 +6,7 @@ import ( "fmt" "math" gohttp "net/http" + "slices" "sort" "strconv" "strings" @@ -32,7 +33,9 @@ const ( MaxOwnedAppsLimit = 100 // ownedAppsMediaKind is the extended media kind Apple uses for iOS apps. - ownedAppsMediaKind = 131072 + ownedAppsMediaKind = 131072 + ownedArcadeAppsMediaKind = 262144 + ownedMacAppsMediaKind = 67108864 ) type OwnedAppsInput struct { @@ -41,6 +44,9 @@ type OwnedAppsInput struct { Page int // Limit is the page size. Zero means DefaultOwnedAppsLimit. Limit int + // Platform filters the results by the device family an app is available + // on. Empty returns every platform. + Platform Platform // All returns every owned app in a single page, ignoring Page and Limit. All bool } @@ -72,6 +78,12 @@ func (t *appstore) OwnedApps(input OwnedAppsInput) (OwnedAppsOutput, error) { return OwnedAppsOutput{}, err } + if input.Platform != "" { + apps = slices.DeleteFunc(apps, func(app App) bool { + return !slices.Contains(app.Platforms, input.Platform) + }) + } + apps = ownedAppsSortedByPurchaseDate(apps) pageApps := apps @@ -88,6 +100,12 @@ func (t *appstore) OwnedApps(input OwnedAppsInput) (OwnedAppsOutput, error) { } func normalizeOwnedAppsInput(input OwnedAppsInput) (OwnedAppsInput, error) { + switch input.Platform { + case "", PlatformIPhone, PlatformIPad, PlatformAppleTV, PlatformVisionOS, PlatformMacOS: + default: + return OwnedAppsInput{}, fmt.Errorf("invalid platform %q", input.Platform) + } + if input.All { input.Page = 1 input.Limit = MaxOwnedAppsLimit @@ -117,6 +135,26 @@ func normalizeOwnedAppsInput(input OwnedAppsInput) (OwnedAppsInput, error) { } func (t *appstore) fetchOwnedApps(acc Account, guid string) ([]App, error) { + var apps []App + + storeFront, _, _ := strings.Cut(acc.StoreFront, ",") + // Configurator (34) lists mobile apps, while the Mac App Store (13) + // also lists Mac software and mobile apps available on Apple silicon. + for _, store := range []string{"34", "13"} { + acc.StoreFront = storeFront + "," + store + + results, err := t.fetchOwnedAppsForStorefront(acc, guid) + if err != nil { + return nil, err + } + + apps = append(apps, results...) + } + + return mergeOwnedApps(apps), nil +} + +func (t *appstore) fetchOwnedAppsForStorefront(acc Account, guid string) ([]App, error) { loginResult, err := t.ownedAppsClient.Send(t.ownedAppsLoginRequest(acc, guid)) if err != nil { return nil, fmt.Errorf("failed to open purchase history session: %w", err) @@ -139,7 +177,7 @@ func (t *appstore) fetchOwnedApps(acc Account, guid string) ([]App, error) { return nil, errors.New("purchase history login response did not contain a valid session ID") } - query := fmt.Sprintf("('com.apple.itunes.extended\\-media\\-kind:%d')", ownedAppsMediaKind) + query := fmt.Sprintf("('com.apple.itunes.extended\\-media\\-kind:%d','com.apple.itunes.extended\\-media\\-kind:%d','com.apple.itunes.extended\\-media\\-kind:%d')", ownedAppsMediaKind, ownedArcadeAppsMediaKind, ownedMacAppsMediaKind) updateResult, err := t.ownedAppsClient.Send(t.ownedAppsUpdateRequest(acc, guid, uint32(sessionID), query)) if err != nil { @@ -353,7 +391,6 @@ func firstDMAPUint(data []byte, target string) (uint64, bool, error) { func parseOwnedApps(data []byte) ([]App, error) { apps := make([]App, 0) - seen := make(map[int64]struct{}) err := walkDMAP(data, 0, func(tag string, payload []byte) error { if tag != "mlit" { @@ -369,12 +406,6 @@ func parseOwnedApps(data []byte) ([]App, error) { return nil } - if _, ok := seen[app.ID]; ok { - return nil - } - - seen[app.ID] = struct{}{} - apps = append(apps, app) return nil @@ -383,14 +414,71 @@ func parseOwnedApps(data []byte) ([]App, error) { return nil, err } - return apps, nil + return mergeOwnedApps(apps), nil +} + +func mergeOwnedApps(apps []App) []App { + merged := make([]App, 0, len(apps)) + + seen := make(map[int64]int) + for _, app := range apps { + if index, ok := seen[app.ID]; ok { + for _, platform := range app.Platforms { + if !slices.Contains(merged[index].Platforms, platform) { + merged[index].Platforms = append(merged[index].Platforms, platform) + } + } + + if app.PurchaseDate.After(merged[index].PurchaseDate) { + merged[index].PurchaseDate = app.PurchaseDate + } + + continue + } + + seen[app.ID] = len(merged) + merged = append(merged, app) + } + + for index := range merged { + if len(merged[index].Platforms) > 1 { + merged[index].Platforms = slices.DeleteFunc(merged[index].Platforms, func(platform Platform) bool { + return platform == PlatformUnknown + }) + } + } + + return merged } func parseOwnedApp(data []byte) (App, error) { - var app App + app := App{Platforms: []Platform{}} + + var mediaKind, supportedProducts uint64 err := walkDMAP(data, 0, func(tag string, payload []byte) error { switch tag { + case "aeMk", "aeSS": + var value uint64 + + switch len(payload) { + case 1: + value = uint64(payload[0]) + case 2: + value = uint64(binary.BigEndian.Uint16(payload)) + case 4: + value = uint64(binary.BigEndian.Uint32(payload)) + case 8: + value = binary.BigEndian.Uint64(payload) + default: + return fmt.Errorf("owned app %s has invalid integer length %d", tag, len(payload)) + } + + if tag == "aeMk" { + mediaKind = value + } else { + supportedProducts = value + } case "aeSI": id, err := dmapInt64(payload) if err != nil { @@ -419,6 +507,33 @@ func parseOwnedApp(data []byte) (App, error) { return nil }) + // Mac software uses a separate media kind; its supported-products bits + // do not describe mobile device support. + switch mediaKind { + case ownedMacAppsMediaKind: + app.Platforms = append(app.Platforms, PlatformMacOS) + case ownedAppsMediaKind, ownedArcadeAppsMediaKind: + // The purchase-history mask uses 8 for Mac and 16 for visionOS. + // It does not encode Apple TV support. + for _, product := range []struct { + mask uint64 + platform Platform + }{ + {1, PlatformIPhone}, + {2, PlatformIPad}, + {16, PlatformVisionOS}, + {8, PlatformMacOS}, + } { + if supportedProducts&product.mask != 0 { + app.Platforms = append(app.Platforms, product.platform) + } + } + } + + if len(app.Platforms) == 0 { + app.Platforms = []Platform{PlatformUnknown} + } + return app, err } diff --git a/pkg/appstore/appstore_owned_apps_test.go b/pkg/appstore/appstore_owned_apps_test.go index 4718447..93d9616 100644 --- a/pkg/appstore/appstore_owned_apps_test.go +++ b/pkg/appstore/appstore_owned_apps_test.go @@ -3,6 +3,7 @@ package appstore import ( "encoding/binary" "errors" + "math" gohttp "net/http" "time" @@ -17,7 +18,7 @@ var _ = Describe("AppStore (OwnedApps)", func() { const ( testDSID = "123456789" testGUID = "AABBCCDDEEFF" - testStoreFront = "143441" + testStoreFront = "143441-1,34" testToken = "password-token" testSessionID = uint32(42) testRevision = uint32(99) @@ -56,101 +57,114 @@ var _ = Describe("AppStore (OwnedApps)", func() { Return("aa:bb:cc:dd:ee:ff", nil) } - expectLogin := func() *gomock.Call { - return mockOwnedClient.EXPECT(). - Send(gomock.Any()). - Do(func(req http.Request) { - Expect(req.Method).To(Equal(http.MethodPOST)) - Expect(req.URL).To(Equal(PrivatePurchaseDAAPBaseURL + "/login")) - Expect(req.SignAction).To(BeFalse()) - Expect(req.Payload).To(BeNil()) - Expect(req.ResponseFormat).To(Equal(http.ResponseFormatRaw)) - expectOwnedAppsHeaders(req.Headers, account, testGUID) - }). - Return(http.Result[[]byte]{ - StatusCode: gohttp.StatusOK, - Data: dmapTag("mlog", dmapUint32("mlid", testSessionID)), - }, nil) - } - - expectUpdate := func() *gomock.Call { - return mockOwnedClient.EXPECT(). - Send(gomock.Any()). - Do(func(req http.Request) { - Expect(req.Method).To(Equal(http.MethodPOST)) - Expect(req.URL).To(Equal(PrivatePurchaseDAAPBaseURL + "/update")) - Expect(req.SignAction).To(BeTrue()) - Expect(req.Headers).To(HaveKeyWithValue("Content-Type", "application/x-www-form-urlencoded")) - expectOwnedAppsHeaders(req.Headers, account, testGUID) - - payload, ok := req.Payload.(*http.RawPayload) - Expect(ok).To(BeTrue()) - Expect(string(payload.Content)).To(Equal("session-id=42&revision-number=(null)&query=('com.apple.itunes.extended\\-media\\-kind:131072')")) - }). - Return(http.Result[[]byte]{ - StatusCode: gohttp.StatusOK, - Data: dmapTag("mupd", dmapUint32("musr", testRevision)), - }, nil) - } - - expectItems := func(apps []App) *gomock.Call { - return mockOwnedClient.EXPECT(). - Send(gomock.Any()). - Do(func(req http.Request) { - Expect(req.Method).To(Equal(http.MethodPOST)) - Expect(req.URL).To(Equal(PrivatePurchaseDAAPBaseURL + "/databases/99/items")) - Expect(req.SignAction).To(BeTrue()) - Expect(req.Headers).To(HaveKeyWithValue("Content-Type", "application/x-dmap-tagged")) - expectOwnedAppsHeaders(req.Headers, account, testGUID) - - payload, ok := req.Payload.(*http.RawPayload) - Expect(ok).To(BeTrue()) - sessionID, found, err := firstDMAPUint(payload.Content, "mlid") - Expect(err).ToNot(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(sessionID).To(Equal(uint64(testSessionID))) - revision, found, err := firstDMAPUint(payload.Content, "musr") - Expect(err).ToNot(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(revision).To(Equal(uint64(testRevision))) - }). - Return(http.Result[[]byte]{ - StatusCode: gohttp.StatusOK, - Data: ownedAppsDMAPResponse(apps), - }, nil) + expectStorefront := func(storefront string, data []byte) { + responses := []http.Result[[]byte]{ + {StatusCode: gohttp.StatusOK, Data: dmapTag("mlog", dmapUint32("mlid", testSessionID))}, + {StatusCode: gohttp.StatusOK, Data: dmapTag("mupd", dmapUint32("musr", testRevision))}, + {StatusCode: gohttp.StatusOK, Data: data}, + } + calls := make([]*gomock.Call, 0, len(responses)) + for _, response := range responses { + calls = append(calls, mockOwnedClient.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.Headers).To(HaveKeyWithValue("X-Apple-Store-Front", storefront)) + }).Return(response, nil)) + } + gomock.InOrder(calls[0], calls[1], calls[2]) } - When("the account has more than one page of apps", func() { - It("sorts all apps by descending purchase date before returning the requested page", func() { + DescribeTable("filters merged purchases before sorting and pagination", + func(platform Platform, page, total int, ids []int64) { expectMac() + item := func(id, kind, products, date uint32) []byte { + data := append(dmapUint32("aeSI", id), dmapUint32("aeMk", kind)...) + data = append(data, dmapUint32("aeSS", products)...) + data = append(data, dmapUint32("asdp", date)...) - responseOrder := []int{5, 12, 1, 9, 3, 11, 2, 8, 4, 10, 6, 7} - apps := make([]App, 0, 12) - for _, index := range responseOrder { - apps = append(apps, App{ - ID: int64(1000 + index), - BundleID: "com.example.app" + string(rune('a'+index-1)), - Name: "DAAP app", - Version: "1.0", - PurchaseDate: time.Unix(1_700_000_000+int64(index*60), 0).UTC(), - }) + return dmapTag("mlit", data) } + mobile := append(item(1, 131072, 1, 300), item(2, 131072, 19, 200)...) + mobile = append(mobile, item(3, 131072, 0, 100)...) + mac := append(item(2, 67108864, 0, 200), item(4, 67108864, 0, 400)...) + expectStorefront("143441-1,34", dmapTag("adbs", dmapTag("mlcl", mobile))) + expectStorefront("143441-1,13", dmapTag("adbs", dmapTag("mlcl", mac))) - loginCall := expectLogin() - updateCall := expectUpdate() - itemsCall := expectItems(apps) - gomock.InOrder(loginCall, updateCall, itemsCall) + output, err := as.OwnedApps(OwnedAppsInput{Account: account, Platform: platform, Page: page, Limit: 1}) - out, err := as.OwnedApps(OwnedAppsInput{Account: account, Page: 2, Limit: 10}) Expect(err).ToNot(HaveOccurred()) - Expect(out.TotalCount).To(Equal(12)) - Expect(out.Page).To(Equal(2)) - Expect(out.Count).To(Equal(2)) - Expect(out.Results).To(HaveLen(2)) - Expect(out.Results[0].ID).To(Equal(int64(1002))) - Expect(out.Results[1].ID).To(Equal(int64(1001))) - }) + Expect(output.TotalCount).To(Equal(total)) + Expect(output.Count).To(Equal(len(ids))) + Expect(output.Page).To(Equal(page)) + actualIDs := make([]int64, 0, len(output.Results)) + for _, app := range output.Results { + actualIDs = append(actualIDs, app.ID) + } + Expect(actualIDs).To(Equal(ids)) + if len(ids) == 1 && ids[0] == 2 { + Expect(output.Results[0].Platforms).To(Equal([]Platform{PlatformIPhone, PlatformIPad, PlatformVisionOS, PlatformMacOS})) + } + }, + Entry("no filter", Platform(""), 1, 4, []int64{4}), + Entry("iPhone", PlatformIPhone, 1, 2, []int64{1}), + Entry("iPad", PlatformIPad, 1, 1, []int64{2}), + Entry("visionOS", PlatformVisionOS, 1, 1, []int64{2}), + Entry("Mac second page includes merged platform", PlatformMacOS, 2, 2, []int64{2}), + Entry("no matching Apple TV apps", PlatformAppleTV, 1, 0, []int64{}), + Entry("page past filtered results", PlatformVisionOS, 2, 1, []int64{}), + ) + + It("puts a new Mac purchase first and merges apps from both storefronts", func() { + expectMac() + mobile := append(dmapUint32("aeSI", 123), dmapUint32("aeSS", 3)...) + mobile = append(mobile, dmapUint32("aeMk", 131072)...) + mobile = append(mobile, dmapUint32("asdp", 100)...) + arcade := append(dmapUint32("aeSI", 456), dmapUint32("aeMk", 262144)...) + arcade = append(arcade, dmapUint32("aeSS", 3)...) + mobileListing := append(dmapTag("mlit", mobile), dmapTag("mlit", arcade)...) + expectStorefront("143441-1,34", dmapTag("adbs", dmapTag("mlcl", mobileListing))) + + // SnippetsLab's Mac media kind and empty supported-products field + // reproduce the fields that were previously missing from the results. + mac := append(dmapUint32("aeSI", 1006087419), dmapUint32("aeMk", 67108864)...) + mac = append(mac, dmapTag("aeSS", []byte{0, 0})...) + mac = append(mac, dmapString("aeLN", "SnippetsLab")...) + mac = append(mac, dmapUint32("asdp", 300)...) + macListing := append(dmapTag("mlit", mobile), dmapTag("mlit", mac)...) + expectStorefront("143441-1,13", dmapTag("adbs", dmapTag("mlcl", macListing))) + + output, err := as.OwnedApps(OwnedAppsInput{Account: account, Limit: 2}) + Expect(err).ToNot(HaveOccurred()) + Expect(output.TotalCount).To(Equal(3)) + Expect(output.Count).To(Equal(2)) + Expect(output.Results[0].Name).To(Equal("SnippetsLab")) + Expect(output.Results[0].Platforms).To(Equal([]Platform{PlatformMacOS})) + Expect(output.Results[1].ID).To(Equal(int64(123))) + Expect(output.Results[1].Platforms).To(Equal([]Platform{PlatformIPhone, PlatformIPad})) + }) + + It("does not infer a platform from the storefront or app name", func() { + expectMac() + expectStorefront("143441-1,34", dmapTag("adbs", nil)) + item := append(dmapUint32("aeSI", 123), dmapString("aeLN", "Example for Mac")...) + expectStorefront("143441-1,13", dmapTag("adbs", dmapTag("mlcl", dmapTag("mlit", item)))) + + output, err := as.OwnedApps(OwnedAppsInput{Account: account}) + + Expect(err).ToNot(HaveOccurred()) + Expect(output.Results).To(HaveLen(1)) + Expect(output.Results[0].Platforms).To(Equal([]Platform{PlatformUnknown})) + }) + It("does not return an incomplete list if the Mac storefront fails", func() { + expectMac() + expectStorefront("143441-1,34", ownedAppsDMAPResponse([]App{{ID: 123}})) + mockOwnedClient.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: gohttp.StatusUnauthorized}, nil) + + output, err := as.OwnedApps(OwnedAppsInput{Account: account}) + Expect(errors.Is(err, ErrPasswordTokenExpired)).To(BeTrue()) + Expect(output.Results).To(BeEmpty()) + }) + + When("the account has more than one page of apps", func() { It("returns every app when All is set", func() { expectMac() @@ -161,7 +175,8 @@ var _ = Describe("AppStore (OwnedApps)", func() { {ID: 4, BundleID: "d", Name: "D", PurchaseDate: time.Unix(200, 0).UTC()}, } - gomock.InOrder(expectLogin(), expectUpdate(), expectItems(apps)) + expectStorefront("143441-1,34", ownedAppsDMAPResponse(apps)) + expectStorefront("143441-1,13", ownedAppsDMAPResponse(nil)) out, err := as.OwnedApps(OwnedAppsInput{Account: account, All: true}) Expect(err).ToNot(HaveOccurred()) @@ -171,6 +186,106 @@ var _ = Describe("AppStore (OwnedApps)", func() { // newest first; apps without a purchase date go last Expect(ids).To(Equal([]int64{2, 4, 1, 3})) }) + + It("sorts all apps by descending purchase date before returning the requested partial page", func() { + expectMac() + + responseOrder := []int{5, 12, 1, 9, 3, 11, 2, 8, 4, 10, 6, 7} + apps := make([]App, 0, 12) + for _, index := range responseOrder { + apps = append(apps, App{ + ID: int64(1000 + index), + BundleID: "com.example.app" + string(rune('a'+index-1)), + Name: "DAAP app", + Version: "1.0", + PurchaseDate: time.Unix(1_700_000_000+int64(index*60), 0).UTC(), + }) + } + + loginCall := mockOwnedClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.Method).To(Equal(http.MethodPOST)) + Expect(req.URL).To(Equal(PrivatePurchaseDAAPBaseURL + "/login")) + Expect(req.SignAction).To(BeFalse()) + Expect(req.Payload).To(BeNil()) + expectOwnedAppsHeaders(req.Headers, account, testGUID) + }). + Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: dmapTag("mlog", dmapUint32("mlid", testSessionID)), + }, nil) + + updateCall := mockOwnedClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.Method).To(Equal(http.MethodPOST)) + Expect(req.URL).To(Equal(PrivatePurchaseDAAPBaseURL + "/update")) + Expect(req.SignAction).To(BeTrue()) + Expect(req.Headers).To(HaveKeyWithValue("Content-Type", "application/x-www-form-urlencoded")) + expectOwnedAppsHeaders(req.Headers, account, testGUID) + + payload, ok := req.Payload.(*http.RawPayload) + Expect(ok).To(BeTrue()) + Expect(string(payload.Content)).To(Equal("session-id=42&revision-number=(null)&query=('com.apple.itunes.extended\\-media\\-kind:131072','com.apple.itunes.extended\\-media\\-kind:262144','com.apple.itunes.extended\\-media\\-kind:67108864')")) + }). + Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: dmapTag("mupd", dmapUint32("musr", testRevision)), + }, nil) + + itemsCall := mockOwnedClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.Method).To(Equal(http.MethodPOST)) + Expect(req.URL).To(Equal(PrivatePurchaseDAAPBaseURL + "/databases/99/items")) + Expect(req.SignAction).To(BeTrue()) + Expect(req.Headers).To(HaveKeyWithValue("Content-Type", "application/x-dmap-tagged")) + expectOwnedAppsHeaders(req.Headers, account, testGUID) + + payload, ok := req.Payload.(*http.RawPayload) + Expect(ok).To(BeTrue()) + sessionID, found, err := firstDMAPUint(payload.Content, "mlid") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(sessionID).To(Equal(uint64(testSessionID))) + revision, found, err := firstDMAPUint(payload.Content, "musr") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(revision).To(Equal(uint64(testRevision))) + err = walkDMAP(payload.Content, 0, func(tag string, data []byte) error { + if tag == "mque" { + Expect(string(data)).To(Equal("('com.apple.itunes.extended\\-media\\-kind:131072','com.apple.itunes.extended\\-media\\-kind:262144','com.apple.itunes.extended\\-media\\-kind:67108864')")) + } + + return nil + }) + Expect(err).ToNot(HaveOccurred()) + }). + Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: ownedAppsDMAPResponse(apps), + }, nil) + + gomock.InOrder(loginCall, updateCall, itemsCall) + expectStorefront("143441-1,13", ownedAppsDMAPResponse(nil)) + + output, err := as.OwnedApps(OwnedAppsInput{ + Account: account, + Page: 2, + Limit: 10, + }) + + Expect(err).ToNot(HaveOccurred()) + Expect(output.Count).To(Equal(2)) + Expect(output.TotalCount).To(Equal(12)) + Expect(output.Page).To(Equal(2)) + Expect(output.Results).To(HaveLen(2)) + Expect(output.Results[0].ID).To(Equal(int64(1002))) + Expect(output.Results[0].PurchaseDate).To(Equal(time.Unix(1_700_000_120, 0).UTC())) + Expect(output.Results[1].ID).To(Equal(int64(1001))) + Expect(output.Results[1].PurchaseDate).To(Equal(time.Unix(1_700_000_060, 0).UTC())) + }) }) When("the response repeats an app", func() { @@ -182,7 +297,8 @@ var _ = Describe("AppStore (OwnedApps)", func() { {ID: 7, BundleID: "x", Name: "X", PurchaseDate: time.Unix(100, 0).UTC()}, } - gomock.InOrder(expectLogin(), expectUpdate(), expectItems(apps)) + expectStorefront("143441-1,34", ownedAppsDMAPResponse(apps)) + expectStorefront("143441-1,13", ownedAppsDMAPResponse(nil)) out, err := as.OwnedApps(OwnedAppsInput{Account: account, All: true}) Expect(err).ToNot(HaveOccurred()) @@ -190,18 +306,56 @@ var _ = Describe("AppStore (OwnedApps)", func() { }) }) - When("the DAAP login answers with an unauthorized status", func() { - It("returns ErrPasswordTokenExpired", func() { + When("the requested page is past the end", func() { + It("returns an empty page without a metadata lookup", func() { expectMac() + gomock.InOrder( + mockOwnedClient.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: dmapTag("mlog", dmapUint32("mlid", testSessionID)), + }, nil), + mockOwnedClient.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: dmapTag("mupd", dmapUint32("musr", testRevision)), + }, nil), + mockOwnedClient.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{ + StatusCode: gohttp.StatusOK, + Data: ownedAppsDMAPResponse([]App{{ID: 1}}), + }, nil), + ) + + expectStorefront("143441-1,13", ownedAppsDMAPResponse(nil)) + + output, err := as.OwnedApps(OwnedAppsInput{Account: account, Page: 2, Limit: 10}) + + Expect(err).ToNot(HaveOccurred()) + Expect(output.Count).To(Equal(0)) + Expect(output.TotalCount).To(Equal(1)) + Expect(output.Results).To(BeEmpty()) + }) + }) + + When("the authenticated session is rejected", func() { + It("returns the password-token-expired error", func() { + expectMac() mockOwnedClient.EXPECT(). Send(gomock.Any()). - Return(http.Result[[]byte]{ - StatusCode: gohttp.StatusUnauthorized, - }, nil) + Return(http.Result[[]byte]{StatusCode: gohttp.StatusUnauthorized}, nil) _, err := as.OwnedApps(OwnedAppsInput{Account: account}) - Expect(err).To(MatchError(ErrPasswordTokenExpired)) + + Expect(errors.Is(err, ErrPasswordTokenExpired)).To(BeTrue()) + }) + }) + + When("the machine address cannot be read", func() { + It("returns an error before fetching purchase history", func() { + mockMachine.EXPECT().MacAddress().Return("", errors.New("machine error")) + + _, err := as.OwnedApps(OwnedAppsInput{Account: account}) + + Expect(err).To(MatchError(ContainSubstring("failed to get mac address"))) }) }) @@ -234,29 +388,6 @@ var _ = Describe("AppStore (OwnedApps)", func() { }) }) - When("the MAC address cannot be resolved", func() { - It("returns error", func() { - mockMachine.EXPECT(). - MacAddress(). - Return("", errors.New("")) - - _, err := as.OwnedApps(OwnedAppsInput{Account: account}) - Expect(err).To(HaveOccurred()) - }) - }) - - When("input is invalid", func() { - It("rejects a negative page", func() { - _, err := as.OwnedApps(OwnedAppsInput{Account: account, Page: -1}) - Expect(err).To(HaveOccurred()) - }) - - It("rejects a limit above the maximum", func() { - _, err := as.OwnedApps(OwnedAppsInput{Account: account, Limit: MaxOwnedAppsLimit + 1}) - Expect(err).To(HaveOccurred()) - }) - }) - Describe("walkDMAP", func() { It("rejects truncated payloads", func() { data := dmapTag("mlog", dmapUint32("mlid", 1)) @@ -273,6 +404,217 @@ var _ = Describe("AppStore (OwnedApps)", func() { Expect(err).To(HaveOccurred()) }) }) + + DescribeTable("validates and defaults pagination", + func(input OwnedAppsInput, expected OwnedAppsInput, errorText string) { + actual, err := normalizeOwnedAppsInput(input) + if errorText != "" { + Expect(err).To(MatchError(ContainSubstring(errorText))) + + return + } + + Expect(err).ToNot(HaveOccurred()) + Expect(actual).To(Equal(expected)) + }, + Entry("defaults page and limit", OwnedAppsInput{}, OwnedAppsInput{Page: 1, Limit: 10}, ""), + Entry("accepts the maximum limit", OwnedAppsInput{Page: 3, Limit: 100}, OwnedAppsInput{Page: 3, Limit: 100}, ""), + Entry("rejects a negative page", OwnedAppsInput{Page: -1, Limit: 10}, OwnedAppsInput{}, "page"), + Entry("rejects a negative limit", OwnedAppsInput{Page: 1, Limit: -1}, OwnedAppsInput{}, "limit"), + Entry("rejects a limit over 100", OwnedAppsInput{Page: 1, Limit: 101}, OwnedAppsInput{}, "100"), + Entry("rejects an invalid platform", OwnedAppsInput{Platform: "invalid"}, OwnedAppsInput{}, "invalid platform"), + Entry("rejects unknown as a filter", OwnedAppsInput{Platform: PlatformUnknown}, OwnedAppsInput{}, "invalid platform"), + Entry("all overrides page and limit", OwnedAppsInput{All: true, Page: 3, Limit: 5}, OwnedAppsInput{All: true, Page: 1, Limit: MaxOwnedAppsLimit}, ""), + ) +}) + +var _ = Describe("owned-app DMAP parsing", func() { + It("parses app fields, ignores unknown fields, and removes duplicate IDs", func() { + item := make([]byte, 0, 128) + item = append(item, dmapString("zzzz", "unknown")...) + item = append(item, dmapAppIDForTest(123)...) + item = append(item, dmapString("aeBI", "com.example.app")...) + item = append(item, dmapString("aeLN", "Example")...) + item = append(item, dmapString("aePd", "1.2.3")...) + item = append(item, dmapUint32("asdp", 1_700_000_000)...) + listing := append(dmapTag("mlit", item), dmapTag("mlit", item)...) + response := dmapTag("adbs", dmapTag("mlcl", listing)) + + apps, err := parseOwnedApps(response) + + Expect(err).ToNot(HaveOccurred()) + Expect(apps).To(Equal([]App{{ + ID: 123, + BundleID: "com.example.app", + Name: "Example", + Version: "1.2.3", + PurchaseDate: time.Unix(1_700_000_000, 0).UTC(), + Platforms: []Platform{PlatformUnknown}, + }})) + }) + + DescribeTable("decodes supported platforms", + func(kind uint32, products []byte, expected []Platform) { + item := append(dmapUint32("aeMk", kind), dmapTag("aeSS", products)...) + app, err := parseOwnedApp(item) + Expect(err).ToNot(HaveOccurred()) + Expect(app.Platforms).To(Equal(expected)) + }, + Entry("iPhone", uint32(131072), []byte{0, 1}, []Platform{PlatformIPhone}), + Entry("iPad", uint32(131072), []byte{0, 2}, []Platform{PlatformIPad}), + Entry("Mac universal purchase", uint32(131072), []byte{0, 8}, []Platform{PlatformMacOS}), + Entry("visionOS", uint32(131072), []byte{0, 16}, []Platform{PlatformVisionOS}), + Entry("Arcade does not imply Mac support", uint32(262144), []byte{0, 3}, []Platform{PlatformIPhone, PlatformIPad}), + Entry("universal app", uint32(131072), []byte{0, 27}, []Platform{PlatformIPhone, PlatformIPad, PlatformVisionOS, PlatformMacOS}), + Entry("Mac ignores mobile product bits", uint32(67108864), []byte{0, 7}, []Platform{PlatformMacOS}), + Entry("iPod touch only", uint32(131072), []byte{0, 4}, []Platform{PlatformUnknown}), + Entry("unknown product bit does not imply visionOS", uint32(131072), []byte{0, 32}, []Platform{PlatformUnknown}), + Entry("empty products", uint32(131072), []byte{0, 0}, []Platform{PlatformUnknown}), + Entry("unknown media kind", uint32(999), []byte{0, 31}, []Platform{PlatformUnknown}), + Entry("unknown products", uint32(131072), []byte{128, 0}, []Platform{PlatformUnknown}), + Entry("one-byte products", uint32(131072), []byte{3}, []Platform{PlatformIPhone, PlatformIPad}), + Entry("four-byte products", uint32(131072), []byte{0, 0, 0, 3}, []Platform{PlatformIPhone, PlatformIPad}), + Entry("eight-byte products", uint32(131072), []byte{0, 0, 0, 0, 0, 0, 0, 3}, []Platform{PlatformIPhone, PlatformIPad}), + ) + + DescribeTable("returns unknown for missing platform metadata", + func(item []byte) { + app, err := parseOwnedApp(item) + Expect(err).ToNot(HaveOccurred()) + Expect(app.Platforms).To(Equal([]Platform{PlatformUnknown})) + }, + Entry("missing products", dmapUint32("aeMk", 131072)), + Entry("missing media kind", dmapUint32("aeSS", 31)), + ) + + DescribeTable("replaces unknown when another record identifies the platform", + func(platforms ...[]Platform) { + records := make([]App, 0, len(platforms)) + for _, value := range platforms { + records = append(records, App{ID: 123, Platforms: value}) + } + apps := mergeOwnedApps(records) + Expect(apps).To(HaveLen(1)) + Expect(apps[0].Platforms).To(Equal([]Platform{PlatformMacOS})) + }, + Entry("unknown first", []Platform{PlatformUnknown}, []Platform{PlatformMacOS}), + Entry("unknown last", []Platform{PlatformMacOS}, []Platform{PlatformUnknown}), + ) + + It("identifies YouTube for visionOS as visionOS only", func() { + item := dmapAppIDForTest(6745572359) + item = append(item, dmapString("aeBI", "com.google.visionosyoutube")...) + item = append(item, dmapString("aeLN", "YouTube for visionOS")...) + item = append(item, dmapUint32("aeMk", 131072)...) + item = append(item, dmapTag("aeSS", []byte{0, 16})...) + + apps, err := parseOwnedApps(dmapTag("adbs", dmapTag("mlcl", dmapTag("mlit", item)))) + + Expect(err).ToNot(HaveOccurred()) + Expect(apps).To(HaveLen(1)) + Expect(apps[0].ID).To(Equal(int64(6745572359))) + Expect(apps[0].Platforms).To(Equal([]Platform{PlatformVisionOS})) + }) + + It("merges platforms and keeps the latest purchase date before pagination", func() { + mobile := append(dmapUint32("aeSI", 123), dmapUint32("aeSS", 3)...) + mobile = append(mobile, dmapUint32("aeMk", 131072)...) + mobile = append(mobile, dmapUint32("asdp", 100)...) + mac := append(dmapUint32("aeSI", 123), dmapUint32("aeMk", 67108864)...) + mac = append(mac, dmapUint32("asdp", 300)...) + other := append(dmapUint32("aeSI", 456), dmapUint32("asdp", 200)...) + listing := append(dmapTag("mlit", mobile), dmapTag("mlit", other)...) + listing = append(listing, dmapTag("mlit", mac)...) + listing = append(listing, dmapTag("mlit", mobile)...) + + apps, err := parseOwnedApps(dmapTag("adbs", dmapTag("mlcl", listing))) + Expect(err).ToNot(HaveOccurred()) + Expect(apps).To(HaveLen(2)) + page := ownedAppsPage(ownedAppsSortedByPurchaseDate(apps), 1, 1) + Expect(page[0].ID).To(Equal(int64(123))) + Expect(page[0].Platforms).To(Equal([]Platform{PlatformIPhone, PlatformIPad, PlatformMacOS})) + Expect(page[0].PurchaseDate).To(Equal(time.Unix(300, 0).UTC())) + }) + + DescribeTable("rejects malformed platform metadata", + func(tag string) { + _, err := parseOwnedApp(dmapTag(tag, []byte{1, 2, 3})) + Expect(err).To(MatchError(ContainSubstring("invalid integer length 3"))) + }, + Entry("media kind", "aeMk"), + Entry("supported products", "aeSS"), + ) + + It("rejects a malformed purchase date", func() { + item := append(dmapAppIDForTest(123), dmapTag("asdp", []byte{1, 2, 3})...) + response := dmapTag("adbs", dmapTag("mlcl", dmapTag("mlit", item))) + + _, err := parseOwnedApps(response) + + Expect(err).To(MatchError(ContainSubstring("purchase date has invalid length 3"))) + }) + + It("rejects a truncated tag", func() { + _, err := parseOwnedApps([]byte("adbs\x00\x00\x00\x10short")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exceeds remaining response length")) + }) + + It("rejects a protocol-level error status", func() { + response := dmapTag("mlog", dmapUint32("mstt", gohttp.StatusForbidden)) + + err := ownedAppsDMAPStatusError("purchase history login", response) + + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrPasswordTokenExpired)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("DAAP status 403")) + }) + + It("preserves a partial last page", func() { + apps := []App{{ID: 1}, {ID: 2}, {ID: 3}} + + Expect(ownedAppsPage(apps, 2, 2)).To(Equal([]App{{ID: 3}})) + Expect(ownedAppsPage(apps, 3, 2)).To(BeEmpty()) + Expect(ownedAppsPage(apps, math.MaxInt, 100)).To(BeEmpty()) + }) + + It("sorts missing dates last and preserves response order for ties", func() { + date := time.Unix(1_700_000_000, 0).UTC() + apps := []App{ + {ID: 1}, + {ID: 2, PurchaseDate: date}, + {ID: 3, PurchaseDate: date.Add(time.Hour)}, + {ID: 4, PurchaseDate: date}, + {ID: 5}, + } + + sorted := ownedAppsSortedByPurchaseDate(apps) + + Expect(sorted).To(Equal([]App{ + {ID: 3, PurchaseDate: date.Add(time.Hour)}, + {ID: 2, PurchaseDate: date}, + {ID: 4, PurchaseDate: date}, + {ID: 1}, + {ID: 5}, + })) + Expect(apps[0].ID).To(Equal(int64(1))) + }) + + It("builds the dynamic DMAP item request body", func() { + now := time.Unix(1_700_000_000, 0) + query := "('com.apple.itunes.extended\\-media\\-kind:131072')" + body := ownedAppsItemsBody(42, 99, query, now) + + timestamp, found, err := firstDMAPUint(body, "mstc") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(timestamp).To(Equal(uint64(1_700_000_000))) + sessionID, found, err := firstDMAPUint(body, "mlid") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(sessionID).To(Equal(uint64(42))) + }) }) func expectOwnedAppsHeaders(headers map[string]string, account Account, guid string) { @@ -290,7 +632,7 @@ func ownedAppsDMAPResponse(apps []App) []byte { for _, app := range apps { item := make([]byte, 0, 128) - item = append(item, dmapUint64ForTest("aeSI", uint64(app.ID))...) + item = append(item, dmapAppIDForTest(uint64(app.ID))...) item = append(item, dmapString("aeBI", app.BundleID)...) item = append(item, dmapString("aeLN", app.Name)...) item = append(item, dmapString("aePd", app.Version)...) @@ -305,9 +647,9 @@ func ownedAppsDMAPResponse(apps []App) []byte { return dmapTag("adbs", dmapTag("mlcl", listing)) } -func dmapUint64ForTest(name string, value uint64) []byte { +func dmapAppIDForTest(value uint64) []byte { payload := make([]byte, 8) binary.BigEndian.PutUint64(payload, value) - return dmapTag(name, payload) + return dmapTag("aeSI", payload) } diff --git a/pkg/appstore/appstore_replicate_sinf.go b/pkg/appstore/appstore_replicate_sinf.go index e4ee2f0..21f919c 100644 --- a/pkg/appstore/appstore_replicate_sinf.go +++ b/pkg/appstore/appstore_replicate_sinf.go @@ -48,7 +48,7 @@ func (t *appstore) ReplicateSinf(input ReplicateSinfInput) error { zipWriter := zip.NewWriter(tmpFile) - err = t.replicateZip(zipReader, zipWriter) + err = t.replicateZip(zipReader, zipWriter, input.PackagePath) if err != nil { return fmt.Errorf("failed to replicate zip: %w", err) } @@ -239,31 +239,60 @@ func (t *appstore) replicateSinfFromInfo(info packageInfo, zip *zip.Writer, sinf return nil } -func (t *appstore) replicateZip(src *zip.ReadCloser, dst *zip.Writer) error { +func (t *appstore) replicateZip(src *zip.ReadCloser, dst *zip.Writer, sourcePath string) error { + source, err := os.Open(sourcePath) + if err != nil { + return fmt.Errorf("failed to open zip headers: %w", err) + } + + defer source.Close() + + info, err := source.Stat() + if err != nil { + return fmt.Errorf("failed to stat zip: %w", err) + } + + headers, err := newZIPLocalHeaders(source, info.Size()) + if err != nil { + return fmt.Errorf("failed to read zip directory: %w", err) + } + for _, file := range src.File { - err := func() error { - srcFile, err := file.Open() - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer srcFile.Close() - header := file.FileHeader - dstFile, err := dst.CreateHeader(&header) + srcFile, err := file.OpenRaw() + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } - if err != nil { - return fmt.Errorf("failed to create file: %w", err) - } + header := file.FileHeader - _, err = io.Copy(dstFile, srcFile) - if err != nil { - return fmt.Errorf("failed to copy file: %w", err) - } + localExtra, err := headers.extra(file) + if err != nil { + return fmt.Errorf("failed to read local metadata for %q: %w", file.Name, err) + } + + header.Extra = zipStoredExtra(&header, localExtra) + // Frame deflated files with a trailing descriptor for streaming + // installers, without changing their compressed data or timestamps. + // Stored files need inline sizes because they have no stream terminator. + if strings.HasSuffix(header.Name, "/") || header.Method == zip.Store { + header.Flags &^= 0x8 + } else if header.Method == zip.Deflate { + header.Flags |= 0x8 + } + + dstFile, err := dst.CreateRaw(&header) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + + // CreateRaw writes the local header immediately and retains this pointer + // for the central directory written by Close. Keep the two extras distinct. + header.Extra = zipExtraWithoutZIP64(file.Extra) - return nil - }() + _, err = io.Copy(dstFile, srcFile) if err != nil { - return err + return fmt.Errorf("failed to copy file: %w", err) } } diff --git a/pkg/appstore/appstore_zip_framing_test.go b/pkg/appstore/appstore_zip_framing_test.go new file mode 100644 index 0000000..c49f3b9 --- /dev/null +++ b/pkg/appstore/appstore_zip_framing_test.go @@ -0,0 +1,124 @@ +package appstore + +import ( + "archive/zip" + "bytes" + "compress/flate" + "encoding/binary" + "hash/crc32" + "io" + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("AppStore (ZIP streaming framing)", func() { + DescribeTable("preserves entry data and metadata with streaming-compatible framing", + func(name string, method, sourceFlags uint16) { + payload := []byte("streaming archive test payload") + if strings.HasSuffix(name, "/") { + payload = []byte{} + } + raw := payload + if method == zip.Deflate { + var compressed bytes.Buffer + compressor, err := flate.NewWriter(&compressed, flate.HuffmanOnly) + Expect(err).ToNot(HaveOccurred()) + _, err = compressor.Write(payload) + Expect(err).ToNot(HaveOccurred()) + Expect(compressor.Close()).To(Succeed()) + raw = compressed.Bytes() + } + + By("creating an archive with the requested source framing") + var source bytes.Buffer + sourceWriter := zip.NewWriter(&source) + writer, err := sourceWriter.CreateRaw(&zip.FileHeader{ + Name: name, + Method: method, + Flags: sourceFlags, + CRC32: crc32.ChecksumIEEE(payload), + CompressedSize64: uint64(len(raw)), + UncompressedSize64: uint64(len(payload)), + ModifiedDate: 0x5821, + ModifiedTime: 0x1234, + ExternalAttrs: 0x81a40000, + }) + Expect(err).ToNot(HaveOccurred()) + _, err = writer.Write(raw) + Expect(err).ToNot(HaveOccurred()) + Expect(sourceWriter.Close()).To(Succeed()) + + directory, err := os.MkdirTemp("", "ipatool-zip-framing-*") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(os.RemoveAll, directory) + path := filepath.Join(directory, "source.zip") + Expect(os.WriteFile(path, source.Bytes(), 0600)).To(Succeed()) + src, err := zip.OpenReader(path) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(src.Close) + + var output bytes.Buffer + dst := zip.NewWriter(&output) + Expect((&appstore{}).replicateZip(src, dst, path)).To(Succeed()) + Expect(dst.Close()).To(Succeed()) + result, err := zip.NewReader(bytes.NewReader(output.Bytes()), int64(output.Len())) + Expect(err).ToNot(HaveOccurred()) + Expect(result.File).To(HaveLen(1)) + file := result.File[0] + original := src.File[0] + + By("checking local framing independently of the central directory") + offset, err := file.DataOffset() + Expect(err).ToNot(HaveOccurred()) + local := output.Bytes()[:offset] + flags := binary.LittleEndian.Uint16(local[6:8]) + Expect(flags).To(Equal(file.Flags)) + end := offset + int64(file.CompressedSize64) + if method == zip.Deflate { + Expect(flags & 8).To(Equal(uint16(8))) + Expect(local[14:26]).To(Equal(make([]byte, 12))) + descriptor := output.Bytes()[end : end+16] + Expect(binary.LittleEndian.Uint32(descriptor)).To(Equal(uint32(0x08074b50))) + Expect(binary.LittleEndian.Uint32(descriptor[4:])).To(Equal(file.CRC32)) + Expect(binary.LittleEndian.Uint32(descriptor[8:])).To(Equal(uint32(file.CompressedSize64))) + Expect(binary.LittleEndian.Uint32(descriptor[12:])).To(Equal(uint32(file.UncompressedSize64))) + } else { + Expect(flags & 8).To(BeZero()) + Expect(binary.LittleEndian.Uint32(local[14:])).To(Equal(file.CRC32)) + Expect(binary.LittleEndian.Uint32(local[18:])).To(Equal(uint32(file.CompressedSize64))) + Expect(binary.LittleEndian.Uint32(local[22:])).To(Equal(uint32(file.UncompressedSize64))) + Expect(binary.LittleEndian.Uint32(output.Bytes()[end:])).To(Equal(uint32(0x02014b50))) + } + + By("preserving the compressed bytes and metadata") + reader, err := file.OpenRaw() + Expect(err).ToNot(HaveOccurred()) + copied, err := io.ReadAll(reader) + Expect(err).ToNot(HaveOccurred()) + Expect(copied).To(Equal(raw)) + Expect(file.Name).To(Equal(original.Name)) + Expect(file.Method).To(Equal(original.Method)) + Expect(file.ModifiedDate).To(Equal(original.ModifiedDate)) + Expect(file.ModifiedTime).To(Equal(original.ModifiedTime)) + Expect(file.ExternalAttrs).To(Equal(original.ExternalAttrs)) + Expect(file.Extra).To(Equal(original.Extra)) + + By("reading the content back with CRC verification") + contentReader, err := file.Open() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(contentReader.Close) + content, err := io.ReadAll(contentReader) + Expect(err).ToNot(HaveOccurred()) + Expect(content).To(Equal(payload)) + }, + Entry("directory", "Payload/", zip.Store, uint16(0)), + Entry("stored file without a descriptor", "stored", zip.Store, uint16(0)), + Entry("stored file with an existing descriptor", "stored-descriptor", zip.Store, uint16(8)), + Entry("deflated file without a descriptor", "deflated", zip.Deflate, uint16(0)), + Entry("deflated file with an existing descriptor", "deflated-descriptor", zip.Deflate, uint16(8)), + ) +}) diff --git a/pkg/appstore/appstore_zip_headers.go b/pkg/appstore/appstore_zip_headers.go new file mode 100644 index 0000000..0935760 --- /dev/null +++ b/pkg/appstore/appstore_zip_headers.go @@ -0,0 +1,289 @@ +package appstore + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "fmt" + "io" + "math" +) + +const ( + zipLocalHeaderSignature = 0x04034b50 + zipCentralHeaderSignature = 0x02014b50 + zipDirectoryEndSignature = 0x06054b50 + zip64DirectoryEndSignature = 0x06064b50 + zip64LocatorSignature = 0x07064b50 + zip64ExtraID = 0x0001 + + zipLocalHeaderLen = 30 + zipCentralHeaderLen = 46 + zipDirectoryEndLen = 22 + zip64DirectoryEndLen = 56 + zip64LocatorLen = 20 +) + +// zipLocalHeaders reads the local metadata that archive/zip's FileHeader omits. +// The central directory provides offsets rather than relying on entry order or +// searching compressed data for local-header signatures. +type zipLocalHeaders struct { + source io.ReaderAt + directory *io.SectionReader + base int64 +} + +func newZIPLocalHeaders(source io.ReaderAt, size int64) (*zipLocalHeaders, error) { + directory, err := readZIPDirectoryEnd(source, size) + if err != nil { + return nil, err + } + + if directory.size > uint64(directory.endOffset) || directory.offset > math.MaxInt64 { + return nil, zip.ErrFormat + } + + base := directory.endOffset - int64(directory.size) - int64(directory.offset) + // Match archive/zip's handling of prefixed archives and inaccurate directory + // sizes: prefer the declared offset if it already points to a central header. + var signature [4]byte + if base > 0 { + if _, err := source.ReadAt(signature[:], int64(directory.offset)); err == nil && binary.LittleEndian.Uint32(signature[:]) == zipCentralHeaderSignature { + base = 0 + } + } + + if base < 0 || base > size-int64(directory.offset) { + return nil, zip.ErrFormat + } + + start := base + int64(directory.offset) + + return &zipLocalHeaders{ + source: source, + directory: io.NewSectionReader(source, start, size-start), + base: base, + }, nil +} + +type zipDirectoryEnd struct { + offset uint64 + size uint64 + endOffset int64 +} + +func readZIPDirectoryEnd(source io.ReaderAt, size int64) (zipDirectoryEnd, error) { + // The end record can be followed by a comment of up to 65535 bytes. + tail := make([]byte, min(size, zipDirectoryEndLen+math.MaxUint16)) + if _, err := source.ReadAt(tail, size-int64(len(tail))); err != nil { + return zipDirectoryEnd{}, fmt.Errorf("failed to read zip trailer: %w", err) + } + + for i := len(tail) - zipDirectoryEndLen; i >= 0; i-- { + if binary.LittleEndian.Uint32(tail[i:]) != zipDirectoryEndSignature { + continue + } + + commentLen := int(binary.LittleEndian.Uint16(tail[i+20:])) + if i+zipDirectoryEndLen+commentLen > len(tail) { + continue + } + + directory := zipDirectoryEnd{ + size: uint64(binary.LittleEndian.Uint32(tail[i+12:])), + offset: uint64(binary.LittleEndian.Uint32(tail[i+16:])), + endOffset: size - int64(len(tail)) + int64(i), + } + + return readZIP64DirectoryEnd(source, size, directory) + } + + return zipDirectoryEnd{}, zip.ErrFormat +} + +func readZIP64DirectoryEnd(source io.ReaderAt, size int64, directory zipDirectoryEnd) (zipDirectoryEnd, error) { + // A ZIP64 locator, when present, immediately precedes the ordinary end record. + if directory.endOffset < zip64LocatorLen { + return directory, nil + } + + var locator [zip64LocatorLen]byte + if _, err := source.ReadAt(locator[:], directory.endOffset-zip64LocatorLen); err != nil { + return zipDirectoryEnd{}, fmt.Errorf("failed to read zip64 locator: %w", err) + } + + if binary.LittleEndian.Uint32(locator[:]) != zip64LocatorSignature { + return directory, nil + } + + offset := binary.LittleEndian.Uint64(locator[8:]) + disk := binary.LittleEndian.Uint32(locator[4:]) + diskCount := binary.LittleEndian.Uint32(locator[16:]) + + if offset > uint64(size) || disk != 0 || diskCount != 1 { + return zipDirectoryEnd{}, zip.ErrFormat + } + + var end [zip64DirectoryEndLen]byte + if _, err := source.ReadAt(end[:], int64(offset)); err != nil { + return zipDirectoryEnd{}, fmt.Errorf("failed to read zip64 trailer: %w", err) + } + + if binary.LittleEndian.Uint32(end[:]) != zip64DirectoryEndSignature { + return zipDirectoryEnd{}, zip.ErrFormat + } + + return zipDirectoryEnd{ + size: binary.LittleEndian.Uint64(end[40:]), + offset: binary.LittleEndian.Uint64(end[48:]), + endOffset: int64(offset), + }, nil +} + +func (r *zipLocalHeaders) extra(file *zip.File) ([]byte, error) { + var central [zipCentralHeaderLen]byte + if _, err := io.ReadFull(r.directory, central[:]); err != nil { + return nil, fmt.Errorf("failed to read central header: %w", err) + } + + if binary.LittleEndian.Uint32(central[:]) != zipCentralHeaderSignature { + return nil, zip.ErrFormat + } + + nameLen := int(binary.LittleEndian.Uint16(central[28:])) + extraLen := int(binary.LittleEndian.Uint16(central[30:])) + commentLen := int(binary.LittleEndian.Uint16(central[32:])) + + metadata := make([]byte, nameLen+extraLen+commentLen) + if _, err := io.ReadFull(r.directory, metadata); err != nil { + return nil, fmt.Errorf("failed to read central metadata: %w", err) + } + + if string(metadata[:nameLen]) != file.Name { + return nil, zip.ErrFormat + } + + offset, err := zipLocalHeaderOffset(central[:], metadata[nameLen:nameLen+extraLen]) + if err != nil { + return nil, err + } + + if offset > uint64(math.MaxInt64-r.base) { + return nil, zip.ErrFormat + } + + localOffset := r.base + int64(offset) + + var local [zipLocalHeaderLen]byte + + if _, err := r.source.ReadAt(local[:], localOffset); err != nil { + return nil, fmt.Errorf("failed to read local header: %w", err) + } + + if binary.LittleEndian.Uint32(local[:]) != zipLocalHeaderSignature { + return nil, zip.ErrFormat + } + + localNameLen := int(binary.LittleEndian.Uint16(local[26:])) + localExtraLen := int(binary.LittleEndian.Uint16(local[28:])) + + dataOffset, err := file.DataOffset() + if err != nil { + return nil, fmt.Errorf("failed to locate entry data: %w", err) + } + + if dataOffset-localOffset != int64(zipLocalHeaderLen+localNameLen+localExtraLen) { + return nil, zip.ErrFormat + } + + localMetadata := make([]byte, localNameLen+localExtraLen) + if _, err := r.source.ReadAt(localMetadata, localOffset+zipLocalHeaderLen); err != nil { + return nil, fmt.Errorf("failed to read local metadata: %w", err) + } + + if !bytes.Equal(localMetadata[:localNameLen], metadata[:nameLen]) { + return nil, zip.ErrFormat + } + + return localMetadata[localNameLen:], nil +} + +// ZIP64 central-directory offsets belong to the source archive. Let zip.Writer +// regenerate structural ZIP64 values for the output instead of leaving an old +// block ahead of the one it appends. Preserve all other extra bytes verbatim. +func zipExtraWithoutZIP64(extra []byte) []byte { + result := make([]byte, 0, len(extra)) + + for len(extra) >= 4 { + length := int(binary.LittleEndian.Uint16(extra[2:])) + 4 + if length > len(extra) { + break + } + + if binary.LittleEndian.Uint16(extra) != zip64ExtraID { + result = append(result, extra[:length]...) + } + + extra = extra[length:] + } + + return append(result, extra...) +} + +// Stored ZIP64 entries have no descriptor after framing normalization, so their +// local ZIP64 sizes must be authoritative even if the source used placeholders. +func zipStoredExtra(header *zip.FileHeader, extra []byte) []byte { + if header.Method != zip.Store || (header.CompressedSize64 < math.MaxUint32 && header.UncompressedSize64 < math.MaxUint32) { + return extra + } + + result := zipExtraWithoutZIP64(extra) + + var sizes [20]byte + + binary.LittleEndian.PutUint16(sizes[:], zip64ExtraID) + binary.LittleEndian.PutUint16(sizes[2:], 16) + binary.LittleEndian.PutUint64(sizes[4:], header.UncompressedSize64) + binary.LittleEndian.PutUint64(sizes[12:], header.CompressedSize64) + + return append(result, sizes[:]...) +} + +// ZIP64 values appear only for saturated fields, in this order: +// uncompressed size, compressed size, local-header offset, disk number. +func zipLocalHeaderOffset(central, extra []byte) (uint64, error) { + offset := binary.LittleEndian.Uint32(central[42:]) + if offset != math.MaxUint32 { + return uint64(offset), nil + } + + skip := 0 + if binary.LittleEndian.Uint32(central[24:]) == math.MaxUint32 { + skip += 8 + } + + if binary.LittleEndian.Uint32(central[20:]) == math.MaxUint32 { + skip += 8 + } + + for len(extra) >= 4 { + tag := binary.LittleEndian.Uint16(extra) + + length := int(binary.LittleEndian.Uint16(extra[2:])) + if length > len(extra)-4 { + return 0, zip.ErrFormat + } + + if tag == zip64ExtraID { + if length < skip+8 { + return 0, zip.ErrFormat + } + + return binary.LittleEndian.Uint64(extra[4+skip:]), nil + } + + extra = extra[4+length:] + } + + return 0, zip.ErrFormat +} diff --git a/pkg/appstore/appstore_zip_headers_test.go b/pkg/appstore/appstore_zip_headers_test.go new file mode 100644 index 0000000..a42d1d0 --- /dev/null +++ b/pkg/appstore/appstore_zip_headers_test.go @@ -0,0 +1,209 @@ +package appstore + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "hash/crc32" + "io" + "math" + "os" + "path/filepath" + + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("AppStore (ZIP local metadata)", func() { + // Local Unix timestamp field captured from the affected United Airlines IPA. + unixTimestamp := []byte{0x55, 0x58, 12, 0, 0xb1, 0x24, 0x51, 0x6a, 0xb1, 0x24, 0x51, 0x6a, 0xcb, 0, 0xc8, 0} + localTimestamp := []byte{0x55, 0x54, 9, 0, 3, 0xb2, 0x27, 0x51, 0x6a, 0xb2, 0x27, 0x51, 0x6a} + centralTimestamp := []byte{0x55, 0x54, 5, 0, 1, 0xb2, 0x27, 0x51, 0x6a} + DescribeTable("preserves independent extra fields through metadata and SINF patches", + func(local, central []byte) { + source := timestampZIP(local, central) + dir, err := os.MkdirTemp("", "ipatool-zip-timestamps-*") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + original, patched := filepath.Join(dir, "original.ipa"), filepath.Join(dir, "patched.ipa") + Expect(os.WriteFile(original, source, 0600)).To(Succeed()) + store := &appstore{os: operatingsystem.New()} + Expect(store.applyPatches(downloadItemResult{Metadata: map[string]interface{}{}}, Account{}, original, patched, nil)).To(Succeed()) + checkTimestampZIP(patched, local, central) + Expect(store.ReplicateSinf(ReplicateSinfInput{PackagePath: patched, Sinfs: []Sinf{{Data: []byte("sinf")}}})).To(Succeed()) + checkTimestampZIP(patched, local, central) + }, + Entry("United Airlines local-only Unix timestamp", unixTimestamp, []byte{}), + Entry("identical local and central timestamps", centralTimestamp, centralTimestamp), + Entry("local-only extended timestamp", localTimestamp, []byte{}), + Entry("different local and central timestamps", localTimestamp, centralTimestamp), + Entry("central-only extended timestamp", []byte{}, centralTimestamp), + Entry("no extended timestamps", []byte{}, []byte{}), + Entry("unknown local field", []byte{0xfe, 0xca, 3, 0, 1, 2, 3}, []byte{}), + ) + + It("reads local extras in central-directory order, including prefixed archives", func() { + data := timestampZIP(localTimestamp, nil) + end := len(data) - 22 + start := int(binary.LittleEndian.Uint32(data[end+16:])) + firstLen := 46 + int(binary.LittleEndian.Uint16(data[start+28:])) + int(binary.LittleEndian.Uint16(data[start+30:])) + reordered := append([]byte{}, data[:start]...) + reordered = append(reordered, data[start+firstLen:end]...) + reordered = append(reordered, data[start:start+firstLen]...) + reordered = append(reordered, data[end:]...) + reordered = append([]byte("archive prefix"), reordered...) + reader, err := zip.NewReader(bytes.NewReader(reordered), int64(len(reordered))) + Expect(err).ToNot(HaveOccurred()) + headers, err := newZIPLocalHeaders(bytes.NewReader(reordered), int64(len(reordered))) + Expect(err).ToNot(HaveOccurred()) + for _, file := range reader.File { + extra, err := headers.extra(file) + Expect(err).ToNot(HaveOccurred()) + if file.Name == "Payload/Test.app/_CodeSignature/CodeResources" { + Expect(extra).To(Equal(localTimestamp)) + } + } + }) + + It("reads ZIP64 entry offsets and a ZIP64 end record", func() { + out := zip64TimestampZIP(localTimestamp) + reader, err := zip.NewReader(bytes.NewReader(out), int64(len(out))) + Expect(err).ToNot(HaveOccurred()) + headers, err := newZIPLocalHeaders(bytes.NewReader(out), int64(len(out))) + Expect(err).ToNot(HaveOccurred()) + extra, err := headers.extra(reader.File[0]) + Expect(err).ToNot(HaveOccurred()) + Expect(extra).To(Equal(localTimestamp)) + + By("rewriting the ZIP64 archive without retaining stale central offsets") + dir, err := os.MkdirTemp("", "ipatool-zip64-*") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + path := filepath.Join(dir, "source.zip") + Expect(os.WriteFile(path, out, 0600)).To(Succeed()) + src, err := zip.OpenReader(path) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(src.Close) + var output bytes.Buffer + dst := zip.NewWriter(&output) + Expect((&appstore{}).replicateZip(src, dst, path)).To(Succeed()) + Expect(dst.Close()).To(Succeed()) + resultPath := filepath.Join(dir, "result.zip") + Expect(os.WriteFile(resultPath, output.Bytes(), 0600)).To(Succeed()) + checkTimestampZIP(resultPath, localTimestamp, []byte{}) + }) + + It("replaces local ZIP64 placeholder sizes when removing a stored entry's descriptor", func() { + extra := append([]byte{}, unixTimestamp...) + placeholder := make([]byte, 20) + binary.LittleEndian.PutUint16(placeholder, 1) + binary.LittleEndian.PutUint16(placeholder[2:], 16) + extra = append(extra, placeholder...) + size := uint64(math.MaxUint32) + 10 + result := zipStoredExtra(&zip.FileHeader{Method: zip.Store, CompressedSize64: size, UncompressedSize64: size}, extra) + Expect(result[:len(unixTimestamp)]).To(Equal(unixTimestamp)) + Expect(result).To(HaveLen(len(unixTimestamp) + 20)) + Expect(binary.LittleEndian.Uint64(result[len(unixTimestamp)+4:])).To(Equal(size)) + Expect(binary.LittleEndian.Uint64(result[len(unixTimestamp)+12:])).To(Equal(size)) + }) + + It("rejects a truncated local extra field", func() { + data := timestampZIP(localTimestamp, nil) + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + Expect(err).ToNot(HaveOccurred()) + binary.LittleEndian.PutUint16(data[28:], 0xffff) + headers, err := newZIPLocalHeaders(bytes.NewReader(data), int64(len(data))) + Expect(err).ToNot(HaveOccurred()) + _, err = headers.extra(reader.File[0]) + Expect(err).To(HaveOccurred()) + }) +}) + +// Build distinct local/central metadata without using the production helper. +func timestampZIP(local, central []byte) []byte { + var output bytes.Buffer + writer := zip.NewWriter(&output) + payload := []byte("signature") + header := &zip.FileHeader{ + Name: "Payload/Test.app/_CodeSignature/CodeResources", + Method: zip.Store, + ModifiedDate: 0x5cea, + ModifiedTime: 0x874d, + Extra: local, + CRC32: crc32.ChecksumIEEE(payload), + CompressedSize64: uint64(len(payload)), + UncompressedSize64: uint64(len(payload)), + } + entry, err := writer.CreateRaw(header) + Expect(err).ToNot(HaveOccurred()) + _, err = entry.Write(payload) + Expect(err).ToNot(HaveOccurred()) + + header.Extra = central + info, err := writer.Create("Payload/Test.app/Info.plist") + Expect(err).ToNot(HaveOccurred()) + _, err = io.WriteString(info, `CFBundleExecutableTest`) + Expect(err).ToNot(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + return output.Bytes() +} + +func checkTimestampZIP(path string, local, central []byte) { + data, err := os.ReadFile(path) + Expect(err).ToNot(HaveOccurred()) + // This fixture's CodeResources is the first entry. Inspect its local bytes + // independently of the production reader and Go's central-only FileHeader. + nameLen := int(binary.LittleEndian.Uint16(data[26:])) + extraLen := int(binary.LittleEndian.Uint16(data[28:])) + Expect(data[30+nameLen : 30+nameLen+extraLen]).To(Equal(local)) + Expect(binary.LittleEndian.Uint16(data[10:])).To(Equal(uint16(0x874d))) + Expect(binary.LittleEndian.Uint16(data[12:])).To(Equal(uint16(0x5cea))) + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + Expect(err).ToNot(HaveOccurred()) + Expect(reader.File[0].Extra).To(BeEquivalentTo(central)) + entry, err := reader.File[0].Open() + Expect(err).ToNot(HaveOccurred()) + payload, err := io.ReadAll(entry) + Expect(err).ToNot(HaveOccurred()) + Expect(entry.Close()).To(Succeed()) + Expect(string(payload)).To(Equal("signature")) +} + +// Force ZIP64 offsets and end records without allocating a multi-gigabyte fixture. +func zip64TimestampZIP(local []byte) []byte { + data := timestampZIP(local, nil) + end := len(data) - 22 + start := int(binary.LittleEndian.Uint32(data[end+16:])) + nameLen := int(binary.LittleEndian.Uint16(data[start+28:])) + offsetExtra := make([]byte, 12) + binary.LittleEndian.PutUint16(offsetExtra, 1) + binary.LittleEndian.PutUint16(offsetExtra[2:], 8) + binary.LittleEndian.PutUint32(data[start+42:], 0xffffffff) + binary.LittleEndian.PutUint16(data[start+30:], 12) + out := append([]byte{}, data[:start+46+nameLen]...) + out = append(out, offsetExtra...) + out = append(out, data[start+46+nameLen:end]...) + zip64Offset := len(out) + end64 := make([]byte, 56) + binary.LittleEndian.PutUint32(end64, 0x06064b50) + binary.LittleEndian.PutUint64(end64[4:], 44) + binary.LittleEndian.PutUint64(end64[24:], 2) + binary.LittleEndian.PutUint64(end64[32:], 2) + binary.LittleEndian.PutUint64(end64[40:], uint64(zip64Offset-start)) + binary.LittleEndian.PutUint64(end64[48:], uint64(start)) + out = append(out, end64...) + locator := make([]byte, 20) + binary.LittleEndian.PutUint32(locator, 0x07064b50) + binary.LittleEndian.PutUint64(locator[8:], uint64(zip64Offset)) + binary.LittleEndian.PutUint32(locator[16:], 1) + out = append(out, locator...) + eocd := append([]byte{}, data[end:]...) + binary.LittleEndian.PutUint16(eocd[10:], 0xffff) + binary.LittleEndian.PutUint32(eocd[12:], 0xffffffff) + binary.LittleEndian.PutUint32(eocd[16:], 0xffffffff) + out = append(out, eocd...) + + return out +} diff --git a/pkg/appstore/authentication_retry_test.go b/pkg/appstore/authentication_retry_test.go new file mode 100644 index 0000000..f401fe9 --- /dev/null +++ b/pkg/appstore/authentication_retry_test.go @@ -0,0 +1,175 @@ +package appstore + +import ( + "encoding/base64" + "errors" + "io" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + "time" + + "github.com/majd/ipatool/v2/pkg/keychain" + "howett.net/plist" + + apphttp "github.com/majd/ipatool/v2/pkg/http" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" +) + +const testAuthEndpoint = "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" + +var _ = Describe("Authentication recovery", func() { + DescribeTable("honors Retry-After without exceeding the wait budget", + func(status int, header string, calls int, expectedWaits []time.Duration) { + ctrl := gomock.NewController(GinkgoT()) + client := apphttp.NewMockClient[loginResult](ctrl) + responseErr := &apphttp.UnexpectedResponseError{StatusCode: status, RetryAfter: header} + client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{}, responseErr).Times(calls) + var waits []time.Duration + sut := &appstore{loginClient: client, authRetrySleep: func(delay time.Duration) { waits = append(waits, delay) }} + _, err := sut.sendAuthenticationRequest(apphttp.Request{}) + Expect(errors.Is(err, responseErr)).To(BeTrue()) + if calls == 1 { + Expect(waits).To(BeEmpty()) + } else { + Expect(waits).To(Equal(expectedWaits)) + } + }, + Entry("503 with a requested delay", 503, "5", 3, []time.Duration{5 * time.Second, 5 * time.Second}), + Entry("429 with a requested delay", 429, "3", 3, []time.Duration{3 * time.Second, 3 * time.Second}), + Entry("429 without a deadline", 429, "", 3, []time.Duration{10 * time.Second, 20 * time.Second}), + Entry("429 with an invalid deadline", 429, "invalid", 3, []time.Duration{10 * time.Second, 20 * time.Second}), + Entry("503 without a deadline", 503, "", 3, []time.Duration{10 * time.Second, 20 * time.Second}), + Entry("204 without a deadline", 204, "", 3, []time.Duration{10 * time.Second, 20 * time.Second}), + Entry("404 without a deadline", 404, "", 3, []time.Duration{10 * time.Second, 20 * time.Second}), + Entry("Retry-After zero avoids a tight retry loop", 429, "0", 3, []time.Duration{time.Second, time.Second}), + Entry("503 with a long deadline", 503, "3600", 1, []time.Duration(nil)), + Entry("403 is not retried", 403, "1", 1, []time.Duration(nil)), + Entry("malformed redirect is not retried", 302, "", 1, []time.Duration(nil)), + ) + + It("parses dates and seconds without overflow or negative waits", func() { + now := time.Date(2026, 9, 13, 12, 0, 0, 0, time.UTC) + for value, want := range map[string]time.Duration{ + now.Add(5 * time.Second).Format(http.TimeFormat): 5 * time.Second, + now.Add(-time.Second).Format(http.TimeFormat): 0, + "0": 0, " 5 ": 5 * time.Second, + "18446744073709551615": maxAuthenticationRetryDelay + time.Second, + } { + delay, ok := authenticationRetryAfter(value, now) + Expect(ok).To(BeTrue()) + Expect(delay).To(Equal(want)) + } + for _, value := range []string{"", "-1", "invalid"} { + _, ok := authenticationRetryAfter(value, now) + Expect(ok).To(BeFalse()) + } + }) + + It("does not retry populated Apple credential errors or 2FA challenges", func() { + ctrl := gomock.NewController(GinkgoT()) + client := apphttp.NewMockClient[loginResult](ctrl) + sut := &appstore{loginClient: client, authRetrySleep: func(time.Duration) { Fail("must not sleep") }} + for _, data := range []loginResult{ + {FailureType: FailureTypeInvalidCredentials, CustomerMessage: "invalid credentials"}, + {CustomerMessage: CustomerMessageBadLogin}, + } { + client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{StatusCode: 200, Data: data}, nil).Times(1) + result, err := sut.sendAuthenticationRequest(apphttp.Request{}) + Expect(err).NotTo(HaveOccurred()) + _, _, err = sut.parseLoginResponse(&result, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("another network")) + } + }) +}) + +// The adapter only redirects network traffic to the local server; login still +// validates Apple's advertised pod URL before this adapter receives a request. +type loginSessionJar struct{ *cookiejar.Jar } + +func (loginSessionJar) Save() error { return nil } + +var _ = Describe("Login session continuity", func() { + It("retains cookies and the signed payload through retries, a pod redirect and 2FA", func() { + ctrl := gomock.NewController(GinkgoT()) + jar, err := cookiejar.New(nil) + Expect(err).NotTo(HaveOccurred()) + client := apphttp.NewClient[loginResult](apphttp.Args{ + CookieJar: loginSessionJar{jar}, + Authentication: true, + ActionSigner: func(data []byte) ([]byte, error) { return data, nil }, + }) + adapter := apphttp.NewMockClient[loginResult](ctrl) + keychain := keychain.NewMockKeychain(ctrl) + keychain.EXPECT().Set("account", gomock.Any()).Return(nil).Times(1) + sut := &appstore{loginClient: adapter, keychain: keychain, authRetrySleep: func(time.Duration) {}} + connections := make(chan string, 4) + calls := 0 + podURL := "https://p7-buy.itunes.apple.com" + PrivateAppStoreAPIPathAuth + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + calls++ + connections <- r.RemoteAddr + Expect(r.Method).To(Equal(http.MethodPost)) + body, err := io.ReadAll(r.Body) + Expect(err).NotTo(HaveOccurred()) + signature, err := base64.StdEncoding.DecodeString(r.Header.Get(apphttp.HeaderAppleActionSignature)) + Expect(err).NotTo(HaveOccurred()) + Expect(signature).To(Equal(body)) + var payload map[string]string + _, err = plist.Unmarshal(body, &payload) + Expect(err).NotTo(HaveOccurred()) + Expect(payload["attempt"]).To(Equal("1")) + if calls > 1 { + cookie, err := r.Cookie("session") + Expect(err).NotTo(HaveOccurred()) + Expect(cookie.Value).To(Equal("retained")) + } + switch calls { + case 1: + http.SetCookie(w, &http.Cookie{Name: "session", Value: "retained", Path: "/"}) + w.WriteHeader(http.StatusNoContent) + case 2: + w.Header().Set("Location", podURL) + w.WriteHeader(http.StatusFound) + case 3: + Expect(payload["password"]).To(Equal("password")) + data, err := plist.Marshal(map[string]string{"customerMessage": CustomerMessageBadLogin}, plist.XMLFormat) + Expect(err).NotTo(HaveOccurred()) + _, err = w.Write(data) + Expect(err).NotTo(HaveOccurred()) + case 4: + Expect(payload["password"]).To(Equal("password123456")) + w.Header().Set(HTTPHeaderStoreFront, "143441-1,29") + _, err = w.Write([]byte("passwordTokentokendsPersonId123")) + Expect(err).NotTo(HaveOccurred()) + default: + Fail("unexpected request") + } + })) + DeferCleanup(srv.Close) + adapted := 0 + adapter.EXPECT().Send(gomock.Any()).DoAndReturn(func(request apphttp.Request) (apphttp.Result[loginResult], error) { + adapted++ + if adapted == 3 { + Expect(request.URL).To(Equal(podURL)) + } + request.URL = srv.URL + PrivateAppStoreAPIPathAuth + + return client.Send(request) + }).Times(4) + _, err = sut.login("email", "password", "", "guid", testAuthEndpoint) + Expect(errors.Is(err, ErrAuthCodeRequired)).To(BeTrue()) + account, err := sut.login("email", "password", "123456", "guid", testAuthEndpoint) + Expect(err).NotTo(HaveOccurred()) + Expect(account.PasswordToken).To(Equal("token")) + seen := map[string]bool{} + for range 4 { + seen[<-connections] = true + } + Expect(seen).To(HaveLen(4)) + }) +}) diff --git a/pkg/appstore/constants.go b/pkg/appstore/constants.go index 354f916..52216c3 100644 --- a/pkg/appstore/constants.go +++ b/pkg/appstore/constants.go @@ -22,6 +22,7 @@ const ( PrivateInitPath = "/bag.xml" PrivateAppStoreAPIDomain = "buy." + iTunesAPIDomain + PrivateAppStoreAPIPathAuth = "/WebObjects/MZFinance.woa/wa/authenticate" PrivateAppStoreAPIPathPurchase = "/WebObjects/MZFinance.woa/wa/buyProduct" PrivateAppStoreAPIPathDownload = "/WebObjects/MZFinance.woa/wa/volumeStoreDownloadProduct" diff --git a/pkg/appstore/platform.go b/pkg/appstore/platform.go index e76ee42..f446293 100644 --- a/pkg/appstore/platform.go +++ b/pkg/appstore/platform.go @@ -1,25 +1,36 @@ package appstore -import "fmt" +import ( + "fmt" + "strings" +) type Platform string const ( - PlatformIPhone Platform = "iphone" - PlatformIPad Platform = "ipad" - PlatformAppleTV Platform = "appletv" + // PlatformUnknown is an output value, not a selectable store platform. + PlatformUnknown Platform = "unknown" + PlatformIPhone Platform = "iphone" + PlatformIPad Platform = "ipad" + PlatformAppleTV Platform = "appletv" + PlatformVisionOS Platform = "visionos" + PlatformMacOS Platform = "macos" ) func ParsePlatform(value string) (Platform, error) { - switch value { + switch strings.ToLower(value) { case "": return "", nil - case "iphone", "iPhone", "ios", "iOS": + case "iphone", "ios": return PlatformIPhone, nil - case "ipad", "iPad": + case "ipad", "ipados": return PlatformIPad, nil - case "appletv", "AppleTV", "apple-tv", "tvos", "tvOS": + case "appletv", "apple-tv", "tvos": return PlatformAppleTV, nil + case "vision", "visionos", "visionpro", "xros", "realitydevice": + return PlatformVisionOS, nil + case "mac", "macos", "osx": + return PlatformMacOS, nil default: return "", fmt.Errorf("invalid platform %q", value) } @@ -35,6 +46,10 @@ func (p Platform) lookupEntity() (string, error) { return "iPadSoftware", nil case PlatformAppleTV: return "tvSoftware", nil + case PlatformVisionOS: + return "xrosSoftware", nil + case PlatformMacOS: + return "macSoftware", nil default: return "", fmt.Errorf("invalid platform %q", p) } @@ -50,6 +65,10 @@ func (p Platform) searchEntity() (string, error) { return "iPadSoftware", nil case PlatformAppleTV: return "software,tvSoftware", nil + case PlatformVisionOS: + return "xrosSoftware", nil + case PlatformMacOS: + return "macSoftware", nil default: return "", fmt.Errorf("invalid platform %q", p) } @@ -61,6 +80,8 @@ func (p Platform) metadataPlatform() (string, error) { return "enterprisestore", nil case PlatformAppleTV: return "atv9", nil + case PlatformVisionOS: + return "realityDevice", nil default: return "", fmt.Errorf("invalid platform %q", p) } diff --git a/pkg/http/authentication_test.go b/pkg/http/authentication_test.go new file mode 100644 index 0000000..567983d --- /dev/null +++ b/pkg/http/authentication_test.go @@ -0,0 +1,164 @@ +package http + +import ( + "encoding/base64" + "io" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type sessionCookieJar struct{ *cookiejar.Jar } + +func (sessionCookieJar) Save() error { return nil } + +var _ = Describe("Authentication transport", func() { + DescribeTable("preserves cookies and signed POSTs across fresh connections", + func(http2 bool) { + jar, err := cookiejar.New(nil) + Expect(err).NotTo(HaveOccurred()) + signer := &recordingActionSigner{signature: []byte("signature")} + addresses := make(chan string, 4) + calls := 0 + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + calls++ + addresses <- r.RemoteAddr + Expect(r.ProtoMajor).To(Equal(map[bool]int{false: 1, true: 2}[http2])) + Expect(r.Method).To(Equal(http.MethodPost)) + data, err := io.ReadAll(r.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(data).To(Equal(signer.data)) + Expect(r.Header.Get(HeaderAppleActionSignature)).To(Equal(base64.StdEncoding.EncodeToString(signer.signature))) + if calls > 1 { + cookie, err := r.Cookie("session") + Expect(err).NotTo(HaveOccurred()) + Expect(cookie.Value).To(Equal("retained")) + } + switch calls { + case 1: + http.SetCookie(w, &http.Cookie{Name: "session", Value: "retained", Path: "/"}) + w.WriteHeader(http.StatusNoContent) + case 2: + w.Header().Set("Location", "https://p7-buy.itunes.apple.com"+appStoreAuthPath) + w.WriteHeader(http.StatusFound) + default: + _, err = w.Write([]byte("foook")) + Expect(err).NotTo(HaveOccurred()) + } + })) + srv.EnableHTTP2 = http2 + srv.StartTLS() + DeferCleanup(srv.Close) + + original := http.DefaultTransport.(*http.Transport).DisableKeepAlives + sut := NewClient[struct { + Foo string `plist:"foo"` + }](Args{CookieJar: sessionCookieJar{jar}, Authentication: true, ActionSigner: func(data []byte) ([]byte, error) { + signer.data = data + signer.calls++ + + return signer.signature, nil + }}).(*client[struct { + Foo string `plist:"foo"` + }]) + transport := sut.internalClient.Transport.(*AddHeaderTransport).T.(*http.Transport) + // Trust only the test server's certificate; retain the production transport settings. + transport.TLSClientConfig = srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + DeferCleanup(transport.CloseIdleConnections) + request := Request{URL: srv.URL + appStoreAuthPath, Method: MethodPOST, ResponseFormat: ResponseFormatXML, SignAction: true, Payload: &XMLPayload{Content: map[string]interface{}{"password": "password", "attempt": "1"}}} + _, err = sut.Send(request) + Expect(err).To(HaveOccurred()) + redirect, err := sut.Send(request) + Expect(err).NotTo(HaveOccurred()) + Expect(redirect.StatusCode).To(Equal(http.StatusFound)) + Expect(redirect.Headers).To(HaveKey("Location")) + // The appstore layer validates the pod URL and explicitly repeats the POST. + // Route that POST to this server to inspect it without contacting Apple. + _, err = sut.Send(request) + Expect(err).NotTo(HaveOccurred()) + request.Payload = &XMLPayload{Content: map[string]interface{}{"password": "password123456", "attempt": "1"}} + _, err = sut.Send(request) + Expect(err).NotTo(HaveOccurred()) + Expect(signer.calls).To(Equal(4)) + seen := map[string]bool{} + for range 4 { + seen[<-addresses] = true + } + Expect(seen).To(HaveLen(4)) + Expect(http.DefaultTransport.(*http.Transport).DisableKeepAlives).To(Equal(original)) + }, Entry("HTTP/1.1", false), Entry("HTTP/2", true), + ) + + It("preserves a populated Apple error even with HTTP 429", func() { + recorder := httptest.NewRecorder() + recorder.Header().Set("Retry-After", "5") + recorder.WriteHeader(http.StatusTooManyRequests) + _, err := recorder.WriteString("failureTypebadCredentials") + Expect(err).NotTo(HaveOccurred()) + sut := &client[map[string]string]{authentication: true} + result, err := sut.handleXMLResponse(recorder.Result()) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Data).To(HaveKeyWithValue("failureType", "badCredentials")) + Expect(result.StatusCode).To(Equal(http.StatusTooManyRequests)) + }) + + It("retains pooling for ordinary clients", func() { + addresses := make(chan string, 2) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + addresses <- r.RemoteAddr + w.WriteHeader(http.StatusNoContent) + })) + DeferCleanup(srv.Close) + sut := NewClient[struct{}](Args{}) + for range 2 { + request, err := http.NewRequest(http.MethodGet, srv.URL, nil) + Expect(err).NotTo(HaveOccurred()) + response, err := sut.Do(request) + Expect(err).NotTo(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + } + Expect(<-addresses).To(Equal(<-addresses)) + }) + + DescribeTable("reports malformed responses without retaining credentials", + func(status int, body, reason string) { + recorder := httptest.NewRecorder() + recorder.Header().Set("Content-Type", "text/html") + recorder.Header().Set("X-Apple-Jingle-Correlation-Key", "correlation-123") + recorder.Header().Set("Set-Cookie", "secret-cookie") + recorder.Header().Set("Retry-After", "5") + recorder.WriteHeader(status) + _, err := recorder.WriteString(body) + Expect(err).NotTo(HaveOccurred()) + sut := &client[struct{}]{authentication: true} + _, err = sut.handleXMLResponse(recorder.Result()) + Expect(err).To(HaveOccurred()) + responseErr, ok := err.(*UnexpectedResponseError) + Expect(ok).To(BeTrue()) + Expect(responseErr.StatusCode).To(Equal(status)) + Expect(responseErr.BodyLength).To(Equal(len(body))) + Expect(responseErr.ContentType).To(Equal("text/html")) + Expect(responseErr.CorrelationID).To(Equal("correlation-123")) + Expect(responseErr.RetryAfter).To(Equal("5")) + Expect(responseErr.Reason).To(Equal(reason)) + Expect(responseErr.Snippet).To(BeEmpty()) + Expect(err.Error()).NotTo(ContainSubstring("secret")) + }, + Entry("empty forbidden response", 403, "", "empty or non-plist authentication response"), + Entry("HTML failure", 500, "secret-password", "empty or non-plist authentication response"), + Entry("redirect without Location", 302, "secret-token", "authentication redirect is missing Location"), + Entry("rate limit", 429, "secret-token", "rate limited by Apple"), + Entry("malformed plist", 200, "secret-token", "malformed authentication plist"), + ) +}) + +// recordingActionSigner captures what the client signed. +type recordingActionSigner struct { + data []byte + signature []byte + calls int +} diff --git a/pkg/http/client.go b/pkg/http/client.go index 72a2e22..8a2b5bc 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -18,7 +18,8 @@ import ( ) const ( - appStoreAuthURL = "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" + appStoreAuthURL = "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" + appStoreAuthPath = "/WebObjects/MZFinance.woa/wa/authenticate" ) // Connection setup budgets. @@ -56,23 +57,36 @@ type client[R interface{}] struct { internalClient http.Client cookieJar CookieJar actionSigner ActionSigner + authentication bool } type ActionSigner func(data []byte) ([]byte, error) type Args struct { - CookieJar CookieJar + CookieJar CookieJar + // ActionSigner signs Apple actions (SAP) for endpoints that require it. ActionSigner ActionSigner + // Authentication isolates login connections and reports only allowlisted response diagnostics. + Authentication bool } // UnexpectedResponseError preserves the HTTP status when Apple returns an // HTML or empty response where an XML plist was expected. type UnexpectedResponseError struct { - StatusCode int - Snippet string + StatusCode int + Snippet string + Reason string + BodyLength int + ContentType string + CorrelationID string + RetryAfter string } func (e *UnexpectedResponseError) Error() string { + if e.Reason != "" { + return fmt.Sprintf("unexpected response from Apple (HTTP %d): %s (body length=%d, content type=%q, correlation ID=%q)", e.StatusCode, e.Reason, e.BodyLength, e.ContentType, e.CorrelationID) + } + if e.Snippet == "" { return fmt.Sprintf("unexpected response from Apple (HTTP %d): empty or non-plist body", e.StatusCode) } @@ -106,6 +120,14 @@ func NewClient[R interface{}](args Args) Client[R] { transport = http.DefaultTransport.(*http.Transport).Clone() } + if args.Authentication { + // Keep the cookie jar, but do not share pooled connections with other + // Store operations or reuse a connection for the next login attempt. + isolated := transport.Clone() + isolated.DisableKeepAlives = true + transport = isolated + } + // Bound the setup phases rather than the whole exchange; streaming bodies // stay unlimited so large packages are unaffected. transport.DialContext = (&net.Dialer{ @@ -118,6 +140,7 @@ func NewClient[R interface{}](args Args) Client[R] { transport.IdleConnTimeout = idleConnTimeout return &client[R]{ + authentication: args.Authentication, internalClient: http.Client{ Timeout: 0, Jar: args.CookieJar, @@ -259,7 +282,11 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { return Result[R]{}, fmt.Errorf("failed to read response body: %w", err) } - if res.StatusCode == http.StatusTooManyRequests { + if c.authentication && res.StatusCode == http.StatusFound && strings.TrimSpace(res.Header.Get("Location")) == "" { + return Result[R]{}, authenticationResponseError(res, body, "authentication redirect is missing Location") + } + + if !c.authentication && res.StatusCode == http.StatusTooManyRequests { return Result[R]{}, fmt.Errorf("rate limited by Apple (HTTP %d): %s", res.StatusCode, strings.TrimSpace(string(body))) } @@ -278,6 +305,14 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { normalizedBody := normalizeXMLPlistBody(body) if !looksLikePropertyList(normalizedBody) { + if c.authentication { + if res.StatusCode == http.StatusTooManyRequests { + return Result[R]{}, authenticationResponseError(res, body, "rate limited by Apple") + } + + return Result[R]{}, authenticationResponseError(res, body, "empty or non-plist authentication response") + } + snippet := bodySnippet(body) return Result[R]{}, &UnexpectedResponseError{ @@ -288,6 +323,10 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { _, err = plist.Unmarshal(normalizedBody, &data) if err != nil { + if c.authentication { + return Result[R]{}, authenticationResponseError(res, body, "malformed authentication plist") + } + return Result[R]{}, fmt.Errorf("failed to unmarshal xml: %w", err) } @@ -408,3 +447,16 @@ func extractDocumentInnerBody(body []byte) []byte { return documentBody } + +// Do not retain bodies or arbitrary response headers: authentication responses +// can contain credentials, tokens and cookies, including on failure. +func authenticationResponseError(res *http.Response, body []byte, reason string) *UnexpectedResponseError { + return &UnexpectedResponseError{ + StatusCode: res.StatusCode, + Reason: reason, + BodyLength: len(body), + ContentType: bodySnippet([]byte(res.Header.Get("Content-Type"))), + CorrelationID: bodySnippet([]byte(res.Header.Get("X-Apple-Jingle-Correlation-Key"))), + RetryAfter: res.Header.Get("Retry-After"), + } +} From 90893e426d631768e47c4cb908dde5f4bc8ea9e4 Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 17:00:54 +0200 Subject: [PATCH 02/18] feat: support macos in list-versions Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- cmd/list_versions.go | 22 +++++--- cmd/list_versions_test.go | 60 +++++++++++++++++++++ pkg/appstore/appstore_list_versions.go | 40 +++++++++++--- pkg/appstore/appstore_list_versions_test.go | 37 +++++++++++++ 4 files changed, 146 insertions(+), 13 deletions(-) create mode 100644 cmd/list_versions_test.go diff --git a/cmd/list_versions.go b/cmd/list_versions.go index a583703..987e0b2 100644 --- a/cmd/list_versions.go +++ b/cmd/list_versions.go @@ -13,18 +13,24 @@ import ( // nolint:wrapcheck func ListVersionsCmd() *cobra.Command { var ( - appID int64 - bundleID string + appID int64 + bundleID string + platformValue string ) cmd := &cobra.Command{ Use: "list-versions", - Short: "List the available versions of an iOS app", + Short: "List the available versions of an App Store app", RunE: func(cmd *cobra.Command, args []string) error { if appID == 0 && bundleID == "" { return errors.New("either the app ID or the bundle identifier must be specified") } + platform, err := appstore.ParsePlatform(platformValue) + if err != nil { + return err + } + var lastErr error var acc appstore.Account @@ -56,7 +62,7 @@ func ListVersionsCmd() *cobra.Command { app := appstore.App{ID: appID} if bundleID != "" { - lookupResult, err := dependencies.AppStore.Lookup(appstore.LookupInput{Account: acc, BundleID: bundleID}) + lookupResult, err := dependencies.AppStore.Lookup(appstore.LookupInput{Account: acc, BundleID: bundleID, Platform: platform}) if err != nil { return err } @@ -64,7 +70,7 @@ func ListVersionsCmd() *cobra.Command { app = lookupResult.App } - out, err := dependencies.AppStore.ListVersions(appstore.ListVersionsInput{Account: acc, App: app}) + out, err := dependencies.AppStore.ListVersions(appstore.ListVersionsInput{Account: acc, App: app, Platform: platform}) if err != nil { return err } @@ -90,8 +96,10 @@ func ListVersionsCmd() *cobra.Command { }, } - cmd.Flags().Int64VarP(&appID, "app-id", "i", 0, "ID of the target iOS app (required)") - cmd.Flags().StringVarP(&bundleID, "bundle-identifier", "b", "", "The bundle identifier of the target iOS app (overrides the app ID)") + cmd.Flags().Int64VarP(&appID, "app-id", "i", 0, "ID of the target app (required)") + cmd.Flags().StringVarP(&bundleID, "bundle-identifier", "b", "", "The bundle identifier of the target app (overrides the app ID)") + + cmd.Flags().StringVar(&platformValue, "platform", "", "Platform to list versions for: iphone (iOS), ipad (iPadOS), appletv (tvOS), visionos, or macos") return cmd } diff --git a/cmd/list_versions_test.go b/cmd/list_versions_test.go new file mode 100644 index 0000000..719cc08 --- /dev/null +++ b/cmd/list_versions_test.go @@ -0,0 +1,60 @@ +package cmd + +import ( + "github.com/majd/ipatool/v2/pkg/appstore" + "github.com/majd/ipatool/v2/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("List Versions command", func() { + DescribeTable("passes the selected platform to lookup and version listing", func(args []string, expected appstore.Platform, lookup bool) { + store := &fakeListVersionsAppStore{} + previous := dependencies + DeferCleanup(func() { dependencies = previous }) + dependencies.AppStore = store + dependencies.Logger = log.NewLogger(log.Args{}) + cmd := ListVersionsCmd() + cmd.SetArgs(args) + Expect(cmd.Execute()).To(Succeed()) + Expect(store.input.Platform).To(Equal(expected)) + Expect(store.input.App.ID).To(Equal(int64(42))) + Expect(store.lookupCalled).To(Equal(lookup)) + if lookup { + Expect(store.lookupInput.Platform).To(Equal(expected)) + } + }, + Entry("Mac bundle ID", []string{"-b", "com.example.mac", "--platform", "macos"}, appstore.PlatformMacOS, true), + Entry("Mac app ID alias", []string{"-i", "42", "--platform", "mac"}, appstore.PlatformMacOS, false), + Entry("default platform", []string{"-i", "42"}, appstore.Platform(""), false), + Entry("iOS alias", []string{"-i", "42", "--platform", "ios"}, appstore.PlatformIPhone, false), + ) + + It("rejects unknown as a platform before accessing the account", func() { + cmd := ListVersionsCmd() + cmd.SetArgs([]string{"-i", "42", "--platform", "unknown"}) + Expect(cmd.Execute()).To(MatchError(`invalid platform "unknown"`)) + }) +}) + +type fakeListVersionsAppStore struct { + appstore.AppStore + input appstore.ListVersionsInput + lookupInput appstore.LookupInput + lookupCalled bool +} + +func (*fakeListVersionsAppStore) AccountInfo() (appstore.AccountInfoOutput, error) { + return appstore.AccountInfoOutput{}, nil +} +func (s *fakeListVersionsAppStore) Lookup(input appstore.LookupInput) (appstore.LookupOutput, error) { + s.lookupCalled = true + s.lookupInput = input + + return appstore.LookupOutput{App: appstore.App{ID: 42, BundleID: input.BundleID}}, nil +} +func (s *fakeListVersionsAppStore) ListVersions(input appstore.ListVersionsInput) (appstore.ListVersionsOutput, error) { + s.input = input + + return appstore.ListVersionsOutput{ExternalVersionIdentifiers: []string{"123"}}, nil +} diff --git a/pkg/appstore/appstore_list_versions.go b/pkg/appstore/appstore_list_versions.go index 9cc7d1f..8382aa4 100644 --- a/pkg/appstore/appstore_list_versions.go +++ b/pkg/appstore/appstore_list_versions.go @@ -9,8 +9,9 @@ import ( ) type ListVersionsInput struct { - Account Account - App App + Account Account + App App + Platform Platform } type ListVersionsOutput struct { @@ -19,6 +20,17 @@ type ListVersionsOutput struct { } func (t *appstore) ListVersions(input ListVersionsInput) (ListVersionsOutput, error) { + platform := input.Platform + if platform == "" { + platform = PlatformIPhone + } + + switch platform { + case PlatformIPhone, PlatformIPad, PlatformAppleTV, PlatformVisionOS, PlatformMacOS: + default: + return ListVersionsOutput{}, fmt.Errorf("invalid platform %q", platform) + } + macAddr, err := t.machine.MacAddress() if err != nil { return ListVersionsOutput{}, fmt.Errorf("failed to get mac address: %w", err) @@ -26,9 +38,21 @@ func (t *appstore) ListVersions(input ListVersionsInput) (ListVersionsOutput, er guid := strings.ReplaceAll(strings.ToUpper(macAddr), ":", "") - req := t.listVersionsRequest(input.Account, input.App, guid) - res, err := t.downloadClient.Send(req) + var externalVersionID string + + switch platform { + case PlatformMacOS: + externalVersionID, err = t.lookupLatestMacOSExternalVersionID(input.Account, input.App) + case PlatformAppleTV, PlatformVisionOS: + externalVersionID, err = t.lookupLatestExternalVersionID(input.Account, input.App, platform) + } + if err != nil { + return ListVersionsOutput{}, fmt.Errorf("failed to resolve platform version: %w", err) + } + + req := t.listVersionsRequest(input.Account, input.App, guid, externalVersionID) + res, err := t.downloadClient.Send(req) if err != nil { return ListVersionsOutput{}, fmt.Errorf("failed to send http request: %w", err) } @@ -71,7 +95,7 @@ func (t *appstore) ListVersions(input ListVersionsInput) (ListVersionsOutput, er if len(res.Data.Items) == 0 { // Try redownload endpoint as fallback - redownloadReq := t.redownloadRequest(input.Account, input.App, guid, "") + redownloadReq := t.redownloadRequest(input.Account, input.App, guid, externalVersionID) redownloadRes, redownloadErr := t.downloadClient.Send(redownloadReq) if redownloadErr != nil { return ListVersionsOutput{}, fmt.Errorf("both endpoints failed: primary=invalid response, redownload=%w", redownloadErr) @@ -109,7 +133,7 @@ func (t *appstore) ListVersions(input ListVersionsInput) (ListVersionsOutput, er }, nil } -func (t *appstore) listVersionsRequest(acc Account, app App, guid string) http.Request { +func (t *appstore) listVersionsRequest(acc Account, app App, guid, externalVersionID string) http.Request { payload := map[string]interface{}{ "creditDisplay": "", "guid": guid, @@ -117,6 +141,10 @@ func (t *appstore) listVersionsRequest(acc Account, app App, guid string) http.R "serialNumber": "0", } + if externalVersionID != "" { + payload["externalVersionId"] = externalVersionID + } + podPrefix := "" if acc.Pod != "" { podPrefix = "p" + acc.Pod + "-" diff --git a/pkg/appstore/appstore_list_versions_test.go b/pkg/appstore/appstore_list_versions_test.go index 7b4fb67..43809a5 100644 --- a/pkg/appstore/appstore_list_versions_test.go +++ b/pkg/appstore/appstore_list_versions_test.go @@ -2,6 +2,7 @@ package appstore import ( "errors" + gohttp "net/http" "github.com/majd/ipatool/v2/pkg/http" "github.com/majd/ipatool/v2/pkg/util/machine" @@ -32,6 +33,42 @@ var _ = Describe("AppStore (ListVersions)", func() { ctrl.Finish() }) + It("pins the Mac offer before requesting a universal app's version history", func() { + pages := http.NewMockClient[[]byte](ctrl) + as.(*appstore).storefrontClient = pages + mockMachine.EXPECT().MacAddress().Return("00:11:22:33:44:55", nil) + gomock.InOrder( + pages.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://apps.apple.com/de/app/id6472431552?platform=mac")) + }).Return(http.Result[[]byte]{StatusCode: gohttp.StatusOK, Data: macVersionPage(karingMacConfiguration)}, nil), + mockDownloadClient.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("externalVersionId", "876660716")) + }).Return(http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{Items: []downloadItemResult{{Metadata: map[string]interface{}{ + "softwareVersionExternalIdentifiers": []interface{}{uint64(876660700), uint64(876660716)}, + "softwareVersionExternalIdentifier": uint64(876660716), + }}}}}, nil), + ) + out, err := as.ListVersions(ListVersionsInput{Account: Account{StoreFront: "143443-2,34"}, App: App{ID: 6472431552, BundleID: "com.nebula.karing"}, Platform: PlatformMacOS}) + Expect(err).ToNot(HaveOccurred()) + Expect(out.ExternalVersionIdentifiers).To(Equal([]string{"876660700", "876660716"})) + Expect(out.LatestExternalVersionID).To(Equal("876660716")) + }) + + It("does not fall back to iOS when the Mac version cannot be resolved", func() { + pages := http.NewMockClient[[]byte](ctrl) + as.(*appstore).storefrontClient = pages + mockMachine.EXPECT().MacAddress().Return("00:11:22:33:44:55", nil) + pages.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: gohttp.StatusOK, Data: macVersionPage(`{}`)}, nil) + _, err := as.ListVersions(ListVersionsInput{Account: Account{StoreFront: "143443-2,34"}, App: App{ID: 42}, Platform: PlatformMacOS}) + Expect(err).To(MatchError(ContainSubstring("failed to resolve platform version"))) + }) + + It("rejects unsupported platforms before making requests", func() { + _, err := as.ListVersions(ListVersionsInput{Platform: PlatformUnknown}) + Expect(err).To(MatchError(`invalid platform "unknown"`)) + }) + When("fails to get MAC address", func() { BeforeEach(func() { mockMachine.EXPECT(). From 4e98a8e2a9833db0ce2b99788972ebbbe750bf9e Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 17:00:54 +0200 Subject: [PATCH 03/18] feat: support macos version metadata Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- cmd/get_version_metadata.go | 18 +++++-- cmd/get_version_metadata_test.go | 48 +++++++++++++++++++ pkg/appstore/appstore_get_version_metadata.go | 15 +++++- 3 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 cmd/get_version_metadata_test.go diff --git a/cmd/get_version_metadata.go b/cmd/get_version_metadata.go index d2eaff8..98b9071 100644 --- a/cmd/get_version_metadata.go +++ b/cmd/get_version_metadata.go @@ -16,6 +16,7 @@ func getVersionMetadataCmd() *cobra.Command { appID int64 bundleID string externalVersionID string + platformValue string ) cmd := &cobra.Command{ @@ -26,6 +27,11 @@ func getVersionMetadataCmd() *cobra.Command { return errors.New("either the app ID or the bundle identifier must be specified") } + platform, err := appstore.ParsePlatform(platformValue) + if err != nil { + return err + } + var lastErr error var acc appstore.Account @@ -57,7 +63,7 @@ func getVersionMetadataCmd() *cobra.Command { app := appstore.App{ID: appID} if bundleID != "" { - lookupResult, err := dependencies.AppStore.Lookup(appstore.LookupInput{Account: acc, BundleID: bundleID}) + lookupResult, err := dependencies.AppStore.Lookup(appstore.LookupInput{Account: acc, BundleID: bundleID, Platform: platform}) if err != nil { return err } @@ -66,9 +72,11 @@ func getVersionMetadataCmd() *cobra.Command { } out, err := dependencies.AppStore.GetVersionMetadata(appstore.GetVersionMetadataInput{ + Context: cmd.Context(), Account: acc, App: app, VersionID: externalVersionID, + Platform: platform, }) if err != nil { return err @@ -97,9 +105,11 @@ func getVersionMetadataCmd() *cobra.Command { }, } - cmd.Flags().Int64VarP(&appID, "app-id", "i", 0, "ID of the target iOS app (required)") - cmd.Flags().StringVarP(&bundleID, "bundle-identifier", "b", "", "The bundle identifier of the target iOS app (overrides the app ID)") - cmd.Flags().StringVar(&externalVersionID, "external-version-id", "", "External version identifier of the target iOS app (required)") + cmd.Flags().Int64VarP(&appID, "app-id", "i", 0, "ID of the target app (required)") + cmd.Flags().StringVarP(&bundleID, "bundle-identifier", "b", "", "The bundle identifier of the target app (overrides the app ID)") + cmd.Flags().StringVar(&externalVersionID, "external-version-id", "", "External version identifier of the target app (required)") + + cmd.Flags().StringVar(&platformValue, "platform", "", "Platform to retrieve metadata for: iphone (iOS), ipad (iPadOS), appletv (tvOS), visionos, or macos") _ = cmd.MarkFlagRequired("external-version-id") diff --git a/cmd/get_version_metadata_test.go b/cmd/get_version_metadata_test.go new file mode 100644 index 0000000..7d915cc --- /dev/null +++ b/cmd/get_version_metadata_test.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "github.com/majd/ipatool/v2/pkg/appstore" + "github.com/majd/ipatool/v2/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Get Version Metadata command", func() { + DescribeTable("passes the platform and exact version to metadata lookup", func(args []string, platform appstore.Platform, lookup bool) { + store := &fakeVersionMetadataAppStore{} + previous := dependencies + DeferCleanup(func() { dependencies = previous }) + dependencies.AppStore = store + dependencies.Logger = log.NewLogger(log.Args{}) + cmd := getVersionMetadataCmd() + cmd.SetArgs(append(args, "--external-version-id", "876660716")) + Expect(cmd.Execute()).To(Succeed()) + Expect(store.input.Platform).To(Equal(platform)) + Expect(store.input.VersionID).To(Equal("876660716")) + Expect(store.input.App.ID).To(Equal(int64(42))) + Expect(store.lookupCalled).To(Equal(lookup)) + if lookup { + Expect(store.lookupInput.Platform).To(Equal(platform)) + } + }, + Entry("Mac bundle", []string{"-b", "com.example.mac", "--platform", "macos"}, appstore.PlatformMacOS, true), + Entry("Mac ID alias", []string{"-i", "42", "--platform", "mac"}, appstore.PlatformMacOS, false), + Entry("default", []string{"-i", "42"}, appstore.Platform(""), false), + ) + It("rejects unknown as a filter", func() { + cmd := getVersionMetadataCmd() + cmd.SetArgs([]string{"-i", "42", "--external-version-id", "123", "--platform", "unknown"}) + Expect(cmd.Execute()).To(MatchError(`invalid platform "unknown"`)) + }) +}) + +type fakeVersionMetadataAppStore struct { + fakeListVersionsAppStore + input appstore.GetVersionMetadataInput +} + +func (s *fakeVersionMetadataAppStore) GetVersionMetadata(input appstore.GetVersionMetadataInput) (appstore.GetVersionMetadataOutput, error) { + s.input = input + + return appstore.GetVersionMetadataOutput{}, nil +} diff --git a/pkg/appstore/appstore_get_version_metadata.go b/pkg/appstore/appstore_get_version_metadata.go index baf78ba..db23c37 100644 --- a/pkg/appstore/appstore_get_version_metadata.go +++ b/pkg/appstore/appstore_get_version_metadata.go @@ -1,6 +1,7 @@ package appstore import ( + "context" "errors" "fmt" "strings" @@ -10,9 +11,11 @@ import ( ) type GetVersionMetadataInput struct { + Context context.Context Account Account App App VersionID string + Platform Platform } type GetVersionMetadataOutput struct { @@ -22,6 +25,17 @@ type GetVersionMetadataOutput struct { } func (t *appstore) GetVersionMetadata(input GetVersionMetadataInput) (GetVersionMetadataOutput, error) { + platform := input.Platform + if platform == "" { + platform = PlatformIPhone + } + + switch platform { + case PlatformIPhone, PlatformIPad, PlatformAppleTV, PlatformVisionOS, PlatformMacOS: + default: + return GetVersionMetadataOutput{}, fmt.Errorf("invalid platform %q", platform) + } + macAddr, err := t.machine.MacAddress() if err != nil { return GetVersionMetadataOutput{}, fmt.Errorf("failed to get mac address: %w", err) @@ -31,7 +45,6 @@ func (t *appstore) GetVersionMetadata(input GetVersionMetadataInput) (GetVersion req := t.getVersionMetadataRequest(input.Account, input.App, guid, input.VersionID) res, err := t.downloadClient.Send(req) - if err != nil { return GetVersionMetadataOutput{}, fmt.Errorf("failed to send http request: %w", err) } From a71f14fe2eb3a803c8b579632f064c461292e9fd Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 21:45:28 +0200 Subject: [PATCH 04/18] fix: handle downloads without sinf data --- pkg/appstore/appstore_replicate_sinf.go | 6 +++ pkg/appstore/appstore_replicate_sinf_test.go | 54 ++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/pkg/appstore/appstore_replicate_sinf.go b/pkg/appstore/appstore_replicate_sinf.go index 21f919c..afa0376 100644 --- a/pkg/appstore/appstore_replicate_sinf.go +++ b/pkg/appstore/appstore_replicate_sinf.go @@ -53,6 +53,12 @@ func (t *appstore) ReplicateSinf(input ReplicateSinfInput) error { return fmt.Errorf("failed to replicate zip: %w", err) } + // Device-based downloads can omit sinfs even when the package has a + // manifest. Preserve the archive rewrite, but only inject supplied data. + if len(input.Sinfs) == 0 { + return nil + } + bundleName, err := t.readBundleName(zipReader) if err != nil { return fmt.Errorf("failed to read bundle name: %w", err) diff --git a/pkg/appstore/appstore_replicate_sinf_test.go b/pkg/appstore/appstore_replicate_sinf_test.go index daeeaa1..08c8876 100644 --- a/pkg/appstore/appstore_replicate_sinf_test.go +++ b/pkg/appstore/appstore_replicate_sinf_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "github.com/majd/ipatool/v2/pkg/http" "github.com/majd/ipatool/v2/pkg/keychain" @@ -335,3 +336,56 @@ var _ = Describe("AppStore (ReplicateSinf)", func() { }) }) }) + +var _ = Describe("SINF replication with optional license data", func() { + DescribeTable("preserves package contents when Apple supplies no sinfs", func(withManifest bool, sinfs []Sinf) { + packagePath := filepath.Join(GinkgoT().TempDir(), "app.ipa") + file, err := os.Create(packagePath) + Expect(err).ToNot(HaveOccurred()) + writer := zip.NewWriter(file) + info, err := plist.Marshal(packageInfo{BundleExecutable: "Test"}, plist.BinaryFormat) + Expect(err).ToNot(HaveOccurred()) + contents := map[string][]byte{ + "Payload/Test.app/Info.plist": info, + "Payload/Test.app/Test": []byte("executable"), + "Payload/Test.app/SC_Info/Test.supf": []byte("existing protection data"), + } + if withManifest { + manifest, err := plist.Marshal(packageManifest{SinfPaths: []string{"SC_Info/Test.sinf"}}, plist.BinaryFormat) + Expect(err).ToNot(HaveOccurred()) + contents["Payload/Test.app/SC_Info/Manifest.plist"] = manifest + } + for name, data := range contents { + entry, err := writer.Create(name) + Expect(err).ToNot(HaveOccurred()) + _, err = entry.Write(data) + Expect(err).ToNot(HaveOccurred()) + } + Expect(writer.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + + store := &appstore{os: operatingsystem.New()} + Expect(store.ReplicateSinf(ReplicateSinfInput{PackagePath: packagePath, Sinfs: sinfs})).To(Succeed()) + + reader, err := zip.OpenReader(packagePath) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(reader.Close) + Expect(reader.File).To(HaveLen(len(contents))) + for _, entry := range reader.File { + Expect(contents).To(HaveKey(entry.Name)) + src, err := entry.Open() + Expect(err).ToNot(HaveOccurred()) + data, err := io.ReadAll(src) + Expect(err).ToNot(HaveOccurred()) + Expect(src.Close()).To(Succeed()) + Expect(data).To(Equal(contents[entry.Name])) + } + _, err = os.Stat(packagePath + ".tmp") + Expect(os.IsNotExist(err)).To(BeTrue()) + }, + Entry("with a manifest and omitted sinfs", true, []Sinf(nil)), + Entry("with a manifest and empty sinfs", true, []Sinf{}), + Entry("without a manifest and omitted sinfs", false, []Sinf(nil)), + Entry("without a manifest and empty sinfs", false, []Sinf{}), + ) +}) From 911ee4d914814bd87ffa51489f99ae36a86d15f7 Mon Sep 17 00:00:00 2001 From: leoca Date: Sun, 13 Sep 2026 17:47:42 +0200 Subject: [PATCH 05/18] fix: report zip finalization errors when applying patches --- pkg/appstore/appstore_download.go | 16 ++++- .../appstore_download_patches_test.go | 71 +++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 pkg/appstore/appstore_download_patches_test.go diff --git a/pkg/appstore/appstore_download.go b/pkg/appstore/appstore_download.go index a1e5fc9..6df81dd 100644 --- a/pkg/appstore/appstore_download.go +++ b/pkg/appstore/appstore_download.go @@ -663,7 +663,8 @@ func (t *appstore) isDirectory(path string) (bool, error) { return info.IsDir(), nil } -func (t *appstore) applyPatches(item downloadItemResult, acc Account, src, dst string, artwork []byte) error { +//nolint:nonamedreturns // Deferred close errors must propagate to callers. +func (t *appstore) applyPatches(item downloadItemResult, acc Account, src, dst string, artwork []byte) (err error) { srcZip, err := zip.OpenReader(src) if err != nil { return fmt.Errorf("failed to open zip reader: %w", err) @@ -674,10 +675,19 @@ func (t *appstore) applyPatches(item downloadItemResult, acc Account, src, dst s if err != nil { return fmt.Errorf("failed to open file: %w", err) } - defer dstFile.Close() + + defer func() { + if closeErr := dstFile.Close(); closeErr != nil { + err = joinCleanupError(err, "failed to close patched file", closeErr) + } + }() dstZip := zip.NewWriter(dstFile) - defer dstZip.Close() + defer func() { + if closeErr := dstZip.Close(); closeErr != nil { + err = joinCleanupError(err, "failed to close zip writer", closeErr) + } + }() err = t.replicateZip(srcZip, dstZip, src) if err != nil { diff --git a/pkg/appstore/appstore_download_patches_test.go b/pkg/appstore/appstore_download_patches_test.go new file mode 100644 index 0000000..ed8a419 --- /dev/null +++ b/pkg/appstore/appstore_download_patches_test.go @@ -0,0 +1,71 @@ +package appstore + +import ( + "archive/zip" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" +) + +type readOnlyDestinationOS struct { + operatingsystem.OperatingSystem + destination string +} + +func (o readOnlyDestinationOS) OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + if name != o.destination { + file, err := o.OperatingSystem.OpenFile(name, flag, perm) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + + return file, nil + } + + if err := os.WriteFile(name, nil, perm); err != nil { + return nil, fmt.Errorf("failed to create destination: %w", err) + } + + // Hand back a read-only descriptor so every write to the archive fails. + file, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("failed to open destination: %w", err) + } + + return file, nil +} + +func TestApplyPatchesReportsZipWriteFailure(t *testing.T) { + dir := t.TempDir() + src, dst := filepath.Join(dir, "source.zip"), filepath.Join(dir, "patched.ipa") + + file, err := os.Create(src) + if err != nil { + t.Fatal(err) + } + + writer := zip.NewWriter(file) + if _, err := writer.Create("Payload/App.app/"); err != nil { + t.Fatal(err) + } + + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + if err := file.Close(); err != nil { + t.Fatal(err) + } + + // The patched archive is small enough to stay buffered until the zip writer + // is closed, so the write failure only surfaces when the archive is finalized. + store := &appstore{os: readOnlyDestinationOS{OperatingSystem: operatingsystem.New(), destination: dst}} + + err = store.applyPatches(downloadItemResult{Metadata: map[string]interface{}{}}, Account{}, src, dst, nil) + if err == nil { + t.Fatal("expected applyPatches to report that the patched archive could not be written") + } +} From 3826230b98d597844d783ef4231446dbee0b03aa Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 21:51:18 +0200 Subject: [PATCH 06/18] test: strengthen patch finalization regression coverage --- .../appstore_download_patches_test.go | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/pkg/appstore/appstore_download_patches_test.go b/pkg/appstore/appstore_download_patches_test.go index ed8a419..5d2e93b 100644 --- a/pkg/appstore/appstore_download_patches_test.go +++ b/pkg/appstore/appstore_download_patches_test.go @@ -2,12 +2,15 @@ package appstore import ( "archive/zip" + "bytes" + "errors" "fmt" "os" "path/filepath" - "testing" "github.com/majd/ipatool/v2/pkg/util/operatingsystem" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) type readOnlyDestinationOS struct { @@ -38,34 +41,26 @@ func (o readOnlyDestinationOS) OpenFile(name string, flag int, perm os.FileMode) return file, nil } -func TestApplyPatchesReportsZipWriteFailure(t *testing.T) { - dir := t.TempDir() - src, dst := filepath.Join(dir, "source.zip"), filepath.Join(dir, "patched.ipa") +var _ = Describe("AppStore (patch finalization)", func() { + It("reports buffered writes that fail when closing the ZIP writer", func() { + dir := GinkgoT().TempDir() + src, dst := filepath.Join(dir, "source.zip"), filepath.Join(dir, "patched.ipa") - file, err := os.Create(src) - if err != nil { - t.Fatal(err) - } - - writer := zip.NewWriter(file) - if _, err := writer.Create("Payload/App.app/"); err != nil { - t.Fatal(err) - } - - if err := writer.Close(); err != nil { - t.Fatal(err) - } + var source bytes.Buffer + writer := zip.NewWriter(&source) + _, err := writer.Create("Payload/App.app/") + Expect(err).ToNot(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + Expect(os.WriteFile(src, source.Bytes(), 0600)).To(Succeed()) - if err := file.Close(); err != nil { - t.Fatal(err) - } - - // The patched archive is small enough to stay buffered until the zip writer - // is closed, so the write failure only surfaces when the archive is finalized. - store := &appstore{os: readOnlyDestinationOS{OperatingSystem: operatingsystem.New(), destination: dst}} - - err = store.applyPatches(downloadItemResult{Metadata: map[string]interface{}{}}, Account{}, src, dst, nil) - if err == nil { - t.Fatal("expected applyPatches to report that the patched archive could not be written") - } -} + // The patched archive stays buffered until Close, where the read-only + // destination descriptor rejects the write. + store := &appstore{os: readOnlyDestinationOS{OperatingSystem: operatingsystem.New(), destination: dst}} + err = store.applyPatches(downloadItemResult{Metadata: map[string]interface{}{}}, Account{}, src, dst, nil) + Expect(err).To(MatchError(ContainSubstring("failed to close zip writer"))) + var pathErr *os.PathError + Expect(errors.As(err, &pathErr)).To(BeTrue()) + Expect(pathErr.Op).To(Equal("write")) + Expect(pathErr.Path).To(Equal(dst)) + }) +}) From 197dd72542de151bb156b92913c8755037ea84ac Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 22:13:11 +0200 Subject: [PATCH 07/18] feat: include platforms in search results Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- pkg/appstore/appstore_search.go | 61 ++++++++++++++++++++++++++++ pkg/appstore/appstore_search_test.go | 22 ++++++++++ 2 files changed, 83 insertions(+) diff --git a/pkg/appstore/appstore_search.go b/pkg/appstore/appstore_search.go index 015bfd1..3114536 100644 --- a/pkg/appstore/appstore_search.go +++ b/pkg/appstore/appstore_search.go @@ -1,10 +1,12 @@ package appstore import ( + "encoding/json" "errors" "fmt" gohttp "net/http" "net/url" + "slices" "strconv" "github.com/majd/ipatool/v2/pkg/http" @@ -53,6 +55,65 @@ type searchResult struct { Results []App `json:"results,omitempty"` } +// UnmarshalJSON derives platforms from catalog metadata without exposing the +// catalog's device list in app output. +func (r *searchResult) UnmarshalJSON(data []byte) error { + var result struct { + Count int `json:"resultCount"` + Results []struct { + App + Kind string `json:"kind"` + SupportedDevices []string `json:"supportedDevices"` + } `json:"results"` + } + + if err := json.Unmarshal(data, &result); err != nil { + return fmt.Errorf("failed to decode search results: %w", err) + } + + r.Count = result.Count + r.Results = nil + + if result.Results != nil { + r.Results = make([]App, 0, len(result.Results)) + } + + for _, item := range result.Results { + platforms := item.Platforms + + for _, family := range []struct { + prefix string + platform Platform + }{ + {"iPhone", PlatformIPhone}, + {"iPod", PlatformIPhone}, + {"iPad", PlatformIPad}, + {"AppleTV", PlatformAppleTV}, + {"RealityDevice", PlatformVisionOS}, + {"Mac", PlatformMacOS}, + } { + for _, device := range item.SupportedDevices { + if strings.HasPrefix(device, family.prefix) && !slices.Contains(platforms, family.platform) { + platforms = append(platforms, family.platform) + } + } + } + + if item.Kind == "mac-software" && !slices.Contains(platforms, PlatformMacOS) { + platforms = append(platforms, PlatformMacOS) + } + + if len(platforms) == 0 { + platforms = []Platform{PlatformUnknown} + } + + item.Platforms = platforms + r.Results = append(r.Results, item.App) + } + + return nil +} + func (t *appstore) searchRequest(term, countryCode string, limit int64, platform Platform) (http.Request, error) { url, err := t.searchURL(term, countryCode, limit, platform) if err != nil { diff --git a/pkg/appstore/appstore_search_test.go b/pkg/appstore/appstore_search_test.go index 635d567..151b5e7 100644 --- a/pkg/appstore/appstore_search_test.go +++ b/pkg/appstore/appstore_search_test.go @@ -1,6 +1,7 @@ package appstore import ( + "encoding/json" "errors" "net/url" @@ -29,6 +30,27 @@ var _ = Describe("AppStore (Search)", func() { ctrl.Finish() }) + DescribeTable("decodes catalog platforms", + func(metadata string, expected []Platform) { + var result searchResult + Expect(json.Unmarshal([]byte(`{"resultCount":1,"results":[{"trackId":42,`+metadata+`}]}`), &result)).To(Succeed()) + mockClient.EXPECT().Send(gomock.Any()).Return(http.Result[searchResult]{StatusCode: 200, Data: result}, nil) + out, err := as.Search(SearchInput{Account: Account{StoreFront: "143441"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(out.Count).To(Equal(1)) + Expect(out.Results).To(Equal([]App{{ID: 42, Platforms: expected}})) + }, + Entry("universal app with duplicate devices", `"supportedDevices":["iPadAir-iPadAir","iPhone5s-iPhone5s","iPhone6-iPhone6","iPodTouchSixthGen-iPodTouchSixthGen"]`, []Platform{PlatformIPhone, PlatformIPad}), + Entry("iPad only", `"supportedDevices":["iPadAir-iPadAir"]`, []Platform{PlatformIPad}), + Entry("iPod", `"supportedDevices":["iPodTouchSixthGen-iPodTouchSixthGen"]`, []Platform{PlatformIPhone}), + Entry("TV", `"supportedDevices":["AppleTV4-AppleTV4"]`, []Platform{PlatformAppleTV}), + Entry("vision", `"supportedDevices":["RealityDevice-RealityDevice"]`, []Platform{PlatformVisionOS}), + Entry("Mac", `"kind":"mac-software"`, []Platform{PlatformMacOS}), + Entry("multiple families", `"supportedDevices":["MacDesktop-MacDesktop","RealityDevice-RealityDevice","iPadAir-iPadAir"]`, []Platform{PlatformIPad, PlatformVisionOS, PlatformMacOS}), + Entry("missing metadata", `"kind":"software"`, []Platform{PlatformUnknown}), + Entry("unrecognized devices", `"supportedDevices":["FutureDevice"]`, []Platform{PlatformUnknown}), + ) + When("request is successful", func() { const ( testID = 0 From 807242ba0a96ae2217937dbd29f4ff47d9f07bdd Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 22:25:12 +0200 Subject: [PATCH 08/18] fix: select macos packages for universal apps Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- pkg/appstore/appstore_download_test.go | 210 ++++++++++++++++++ .../appstore_macos_version_lookup_test.go | 4 +- 2 files changed, 212 insertions(+), 2 deletions(-) diff --git a/pkg/appstore/appstore_download_test.go b/pkg/appstore/appstore_download_test.go index 5285f78..9f42b42 100644 --- a/pkg/appstore/appstore_download_test.go +++ b/pkg/appstore/appstore_download_test.go @@ -696,9 +696,122 @@ var _ = Describe("AppStore (Download)", func() { return file.Name() } +<<<<<<< HEAD It("accepts AppleTVOS packages", func() { path := writePackage([]string{"AppleTVOS"}) defer os.Remove(path) +======= + store := &appstore{ + downloadClient: mockDownloadClient, + bagClient: mockBagClient, + platformClient: mockPlatformClient, + httpClient: mockHTTPClient, + machine: mockMachine, + os: operatingsystem.New(), + } + out, err := store.Download(DownloadInput{ + Account: Account{StoreFront: "143441", Email: "test@example.com"}, + App: App{ID: 42}, + OutputPath: outputPath, + Platform: platform, + ExternalVersionID: versionID, + }) + if succeeds { + Expect(err).ToNot(HaveOccurred()) + Expect(out.DestinationPath).To(Equal(outputPath)) + actual, readErr := os.ReadFile(outputPath) + Expect(readErr).ToNot(HaveOccurred()) + Expect(actual).ToNot(Equal(previousOutput)) + } else { + Expect(err).To(MatchError(ContainSubstring("failed to validate package platform"))) + Expect(out.DestinationPath).To(BeEmpty()) + Expect(os.ReadFile(outputPath)).To(Equal(previousOutput)) + } + Expect(outputPath + ".tmp").ToNot(BeAnExistingFile()) + }, + Entry("iPhone accepts iOS", PlatformIPhone, "", "", "iPhoneOS", true), + Entry("iPad accepts iOS", PlatformIPad, "", "", "iPhoneOS", true), + Entry("iPhone rejects tvOS", PlatformIPhone, "", "", "AppleTVOS", false), + Entry("iPad rejects tvOS", PlatformIPad, "", "", "AppleTVOS", false), + Entry("iPhone rejects visionOS", PlatformIPhone, "", "", "XROS", false), + Entry("a conflicting explicit version is rejected, not replaced", PlatformIPhone, "818970197", "", "AppleTVOS", false), + Entry("default iOS fallback accepts iOS", Platform(""), "", "redownload", "iPhoneOS", true), + Entry("default iOS fallback rejects tvOS", Platform(""), "", "redownload", "AppleTVOS", false), + Entry("latest update accepts iOS", Platform(""), "", "update", "iPhoneOS", true), + Entry("pinned update accepts iOS", Platform(""), "818970197", "update", "iPhoneOS", true), + Entry("pinned update rejects tvOS", Platform(""), "818970197", "update", "AppleTVOS", false), + Entry("pinned redownload preserves an unspecified platform", Platform(""), "818970197", "redownload", "AppleTVOS", true), + Entry("normal unspecified-platform behavior is preserved", Platform(""), "", "", "AppleTVOS", true), + Entry("an explicit tvOS version without a platform is preserved", Platform(""), "818970197", "", "AppleTVOS", true), + Entry("explicit tvOS downloads still work", PlatformAppleTV, "818970197", "", "AppleTVOS", true), + ) + + Describe("macOS packages", func() { + It("decrypts the downloaded package with in-memory machine identity and dpInfo", func() { + tempDir := GinkgoT().TempDir() + requestedPath := filepath.Join(tempDir, "custom-output.pkg") + packageData := []byte("encrypted package") + decryptedData := makeTestXAR([]byte("decrypted payload"), false) + dpInfo := bytes.Repeat([]byte{0x42}, 88) + decrypter := &fakeMacPackageDecrypter{output: decryptedData} + + mockMachine.EXPECT(). + MacAddress(). + Return("00:11:22:aa:bb:cc", nil) + + mockDownloadClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + payload := req.Payload.(*http.XMLPayload) + Expect(payload.Content["guid"]).To(Equal("001122AABBCC")) + }). + Return(http.Result[downloadResult]{ + StatusCode: 200, + Data: downloadResult{ + Items: []downloadItemResult{{ + URL: "https://example.test/app.pkg", + Sinfs: []Sinf{{DPInfo: dpInfo}}, + Metadata: map[string]interface{}{ + "bundleShortVersionString": "1.2.3", + "software-platform": "macos", + "product-type": "ios-app", + }, + }}, + }, + }, nil) + + mockHTTPClient.EXPECT(). + NewRequest("GET", "https://example.test/app.pkg", nil). + Return(&gohttp.Request{Header: gohttp.Header{}}, nil) + mockHTTPClient.EXPECT(). + Do(gomock.Any()). + Return(&gohttp.Response{ + StatusCode: gohttp.StatusOK, + Body: io.NopCloser(bytes.NewReader(packageData)), + ContentLength: int64(len(packageData)), + }, nil) + + store := &appstore{ + downloadClient: mockDownloadClient, + httpClient: mockHTTPClient, + machine: mockMachine, + os: operatingsystem.New(), + macDecrypterFactory: func(ctx context.Context, hardwareID, gotDPInfo []byte) (macPackageDecrypter, error) { + Expect(ctx).ToNot(BeNil()) + Expect(hardwareID).To(Equal([]byte{0x00, 0x11, 0x22, 0xaa, 0xbb, 0xcc})) + Expect(gotDPInfo).To(Equal(dpInfo)) + + return decrypter, nil + }, + } + out, err := store.Download(DownloadInput{ + Context: context.Background(), + App: App{ID: 42, BundleID: "com.example.mac"}, + OutputPath: requestedPath, + Platform: PlatformMacOS, + ExternalVersionID: "123456789", + }) +>>>>>>> c66d653 (fix: select macos packages for universal apps) err := (&appstore{}).validatePackagePlatform(path, PlatformAppleTV) Expect(err).ToNot(HaveOccurred()) @@ -708,9 +821,106 @@ var _ = Describe("AppStore (Download)", func() { path := writePackage([]string{"iPhoneOS"}) defer os.Remove(path) +<<<<<<< HEAD err := (&appstore{}).validatePackagePlatform(path, PlatformAppleTV) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("AppleTVOS")) +======= + mockMachine.EXPECT(). + MacAddress(). + Return("00:11:22:aa:bb:cc", nil) + mockDownloadClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[downloadResult]{ + StatusCode: 200, + Data: downloadResult{ + Items: []downloadItemResult{{ + URL: "https://example.test/developer.ipa", + Sinfs: []Sinf{{ + ID: 0, + Data: []byte("mobile sinf"), + }}, + Metadata: map[string]interface{}{ + "bundleShortVersionString": "11.0.2", + "software-platform": "ios", + "product-type": "ios-app", + }, + }}, + }, + }, nil) + mockHTTPClient.EXPECT(). + NewRequest("GET", "https://example.test/developer.ipa", nil). + Return(&gohttp.Request{Header: gohttp.Header{}}, nil) + mockHTTPClient.EXPECT(). + Do(gomock.Any()). + Return(&gohttp.Response{ + StatusCode: gohttp.StatusOK, + Body: io.NopCloser(bytes.NewReader(packageBuffer.Bytes())), + ContentLength: int64(packageBuffer.Len()), + }, nil) + + store := &appstore{ + downloadClient: mockDownloadClient, + httpClient: mockHTTPClient, + machine: mockMachine, + os: operatingsystem.New(), + macDecrypterFactory: func(context.Context, []byte, []byte) (macPackageDecrypter, error) { + Fail("mobile packages must not initialize the macOS package decrypter") + + return nil, nil + }, + } + + out, err := store.Download(DownloadInput{ + Context: context.Background(), + Account: Account{Email: "test@example.com"}, + App: App{ID: 640199958, BundleID: "developer.apple.wwdc-Release"}, + OutputPath: tempDir, + Platform: PlatformMacOS, + ExternalVersionID: "123456789", + }) + + Expect(err).ToNot(HaveOccurred()) + Expect(out.DestinationPath).To(Equal(requestedPath)) + Expect(out.Sinfs).To(Equal([]Sinf{{ID: 0, Data: []byte("mobile sinf")}})) + Expect(requestedPath).To(BeAnExistingFile()) + Expect(requestedPath + macEncryptedStageSuffix).ToNot(BeAnExistingFile()) + Expect(requestedPath + macDecryptedStageSuffix).ToNot(BeAnExistingFile()) + }) + + It("uses a pkg name derived from the generated package name", func() { + store := &appstore{os: operatingsystem.New()} + tempDir := GinkgoT().TempDir() + + packagePath, err := store.resolveDestinationPath( + App{ID: 42, BundleID: "com.example.mac"}, + "1.2.3", + tempDir, + PlatformMacOS, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(packagePath).To(Equal(filepath.Join(tempDir, "com.example.mac_42_1.2.3.pkg"))) + }) + + It("preserves an explicit output path", func() { + store := &appstore{os: operatingsystem.New()} + tempDir := GinkgoT().TempDir() + requestedPath := filepath.Join(tempDir, "custom-output") + + packagePath, err := store.resolveDestinationPath(App{}, "1.2.3", requestedPath, PlatformMacOS) + Expect(err).ToNot(HaveOccurred()) + Expect(packagePath).To(Equal(requestedPath)) + }) + + It("rejects a download response without dpInfo", func() { + _, err := macDPInfo(nil) + Expect(err).To(MatchError(ContainSubstring("dpInfo"))) + }) + + It("rejects conflicting dpInfo values", func() { + _, err := macDPInfo([]Sinf{{DPInfo: []byte("one")}, {DPInfo: []byte("two")}}) + Expect(err).To(MatchError(ContainSubstring("conflicting"))) +>>>>>>> c66d653 (fix: select macos packages for universal apps) }) }) diff --git a/pkg/appstore/appstore_macos_version_lookup_test.go b/pkg/appstore/appstore_macos_version_lookup_test.go index 08ddedd..3d8a2ec 100644 --- a/pkg/appstore/appstore_macos_version_lookup_test.go +++ b/pkg/appstore/appstore_macos_version_lookup_test.go @@ -42,9 +42,9 @@ var _ = Describe("Mac purchase version selection", func() { ctrl := gomock.NewController(GinkgoT()) defer ctrl.Finish() pages := http.NewMockClient[[]byte](ctrl) - store := &appstore{storefrontClient: pages} + store := &appstore{storefrontClient: pages, downloadClient: http.NewMockClient[downloadResult](ctrl)} pages.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: status, Data: body}, requestErr) - _, err := store.lookupLatestMacOSExternalVersionID(Account{StoreFront: "143443-2,34"}, app) + _, _, err := store.sendDownloadProduct(Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) Expect(err).To(HaveOccurred()) }, Entry("HTTP failure", 500, nil, nil), From bb0d2350b3909da3ce8c75e56249a05713d4917a Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 23:04:41 +0200 Subject: [PATCH 09/18] feat: add ios builds for jailbroken devices Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/dry-build.yml | 11 ++ .github/workflows/release.yml | 42 +++++ cmd/common.go | 2 +- cmd/keyring_default.go | 12 ++ cmd/keyring_ios.go | 48 ++++++ cmd/keyring_ios_test.go | 27 ++++ go.mod | 3 +- pkg/appstore/appstore_download_test.go | 210 ------------------------- resources/ios-entitlements.plist | 14 ++ tools/build-ios.sh | 42 +++++ tools/patches/unicorn-ios.patch | 76 +++++++++ 11 files changed, 275 insertions(+), 212 deletions(-) create mode 100644 cmd/keyring_default.go create mode 100644 cmd/keyring_ios.go create mode 100644 cmd/keyring_ios_test.go create mode 100644 resources/ios-entitlements.plist create mode 100755 tools/build-ios.sh create mode 100644 tools/patches/unicorn-ios.patch diff --git a/.github/workflows/dry-build.yml b/.github/workflows/dry-build.yml index dadfb22..c20ef39 100644 --- a/.github/workflows/dry-build.yml +++ b/.github/workflows/dry-build.yml @@ -66,3 +66,14 @@ jobs: CGO_ENABLED: 1 CGO_CFLAGS: -mmacosx-version-min=10.15 CGO_LDFLAGS: -mmacosx-version-min=10.15 + build_ios: + name: Build for iOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.0" + cache: true + - run: brew install ldid cmake + - run: ./tools/build-ios.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ce0a73..2474870 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -130,6 +130,48 @@ jobs: name: ipatool-${{ needs.get_version.outputs.version }}-linux-${{ matrix.arch }} path: ipatool-${{ needs.get_version.outputs.version }}-linux-${{ matrix.arch }} if-no-files-found: error + build_ios: + name: Build for iOS + runs-on: macos-latest + needs: [get_version, test] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.0" + cache: true + - run: brew install ldid cmake + - run: ./tools/build-ios.sh "ipatool-$VERSION-ios-arm64" + env: + VERSION: ${{ needs.get_version.outputs.version }} + - uses: actions/upload-artifact@v4 + with: + name: ipatool-${{ needs.get_version.outputs.version }}-ios-arm64 + path: ipatool-${{ needs.get_version.outputs.version }}-ios-arm64 + if-no-files-found: error + release_ios: + name: Release for iOS + runs-on: ubuntu-latest + needs: [get_version, build_ios, release_windows] + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: ipatool-${{ needs.get_version.outputs.version }}-ios-arm64 + path: bin + - run: chmod +x "bin/$FILE" && tar -czvf "$FILE.tar.gz" "bin/$FILE" + env: + FILE: ipatool-${{ needs.get_version.outputs.version }}-ios-arm64 + - run: ./tools/sha256sum.sh "$TARBALL" > "$TARBALL.sha256sum" + env: + TARBALL: ipatool-${{ needs.get_version.outputs.version }}-ios-arm64.tar.gz + - uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ipatool-${{ needs.get_version.outputs.version }}-ios-arm64.* + tag: ${{ github.ref }} + overwrite: false + file_glob: true release_windows: name: Release for Windows runs-on: ubuntu-latest diff --git a/cmd/common.go b/cmd/common.go index 490139d..890723a 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -171,7 +171,7 @@ func newKeychain(stateDirectory string) keychain.Keychain { util.Must("", err) } - ring := util.Must(keyring.Open(keyring.Config{ + ring := util.Must(openKeyring(keyring.Config{ AllowedBackends: []keyring.BackendType{ keyring.KeychainBackend, keyring.SecretServiceBackend, diff --git a/cmd/keyring_default.go b/cmd/keyring_default.go new file mode 100644 index 0000000..a8ac6ae --- /dev/null +++ b/cmd/keyring_default.go @@ -0,0 +1,12 @@ +//go:build !ios + +package cmd + +import ( + "github.com/byteness/keyring" + "github.com/majd/ipatool/v2/pkg/keychain" +) + +func openKeyring(config keyring.Config) (keychain.Keyring, error) { + return keyring.Open(config) //nolint:wrapcheck +} diff --git a/cmd/keyring_ios.go b/cmd/keyring_ios.go new file mode 100644 index 0000000..85650cc --- /dev/null +++ b/cmd/keyring_ios.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "fmt" + + gokeychain "github.com/byteness/go-keychain" + "github.com/byteness/keyring" + "github.com/majd/ipatool/v2/pkg/keychain" +) + +// iosKeyring fixes the update query in keyring v1.9.0, which includes attributes +// that SecItemUpdate rejects on iOS (kSecMatchLimit and kSecReturnAttributes). +type iosKeyring struct { + keyring.Keyring + service string +} + +func openKeyring(config keyring.Config) (keychain.Keyring, error) { + ring, err := keyring.Open(config) + if err != nil { + return nil, fmt.Errorf("open iOS keyring: %w", err) + } + + return &iosKeyring{Keyring: ring, service: config.ServiceName}, nil +} + +func (k *iosKeyring) Set(item keyring.Item) error { + query := gokeychain.NewItem() + query.SetSecClass(gokeychain.SecClassGenericPassword) + query.SetService(k.service) + query.SetAccount(item.Key) + + attributes := gokeychain.NewItem() + attributes.SetData(item.Data) + attributes.SetLabel(item.Label) + attributes.SetDescription(item.Description) + + err := gokeychain.UpdateItem(query, attributes) + if err == gokeychain.ErrorItemNotFound { + return k.Keyring.Set(item) //nolint:wrapcheck + } + + if err != nil { + return fmt.Errorf("update iOS keyring item: %w", err) + } + + return nil +} diff --git a/cmd/keyring_ios_test.go b/cmd/keyring_ios_test.go new file mode 100644 index 0000000..8dde886 --- /dev/null +++ b/cmd/keyring_ios_test.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "fmt" + "time" + + "github.com/byteness/keyring" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("iOS keyring", func() { + It("persists and updates credentials without invalid SecItemUpdate parameters", func() { + ring, err := openKeyring(keyring.Config{ + AllowedBackends: []keyring.BackendType{keyring.KeychainBackend}, + ServiceName: fmt.Sprintf("ipatool-test-%d", time.Now().UnixNano()), + }) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(ring.Remove("account")).To(Succeed()) }) + Expect(ring.Set(keyring.Item{Key: "account", Data: []byte("first"), Label: "ipatool"})).To(Succeed()) + Expect(ring.Set(keyring.Item{Key: "account", Data: []byte("updated"), Label: "ipatool"})).To(Succeed()) + item, err := ring.Get("account") + Expect(err).NotTo(HaveOccurred()) + Expect(item.Data).To(Equal([]byte("updated"))) + Expect(item.Label).To(Equal("ipatool")) + }) +}) diff --git a/go.mod b/go.mod index edbfe88..db2a6c9 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/avast/retry-go v3.0.0+incompatible + github.com/byteness/go-keychain v0.0.0-20191008050251-8e49817e8af4 github.com/byteness/keyring v1.9.0 github.com/juju/persistent-cookiejar v1.0.0 github.com/onsi/ginkgo/v2 v2.5.0 @@ -22,7 +23,7 @@ require ( require ( github.com/1Password/connect-sdk-go v1.5.4-0.20250417152128-c154b387248b // indirect github.com/1password/onepassword-sdk-go v0.4.1-beta.1 // indirect - github.com/byteness/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect + github.com/byteness/go-libsecret v0.0.0-20260108215642-107379d3dee0 // indirect github.com/byteness/percent v0.2.2 // indirect github.com/danieljoos/wincred v1.2.3 // indirect diff --git a/pkg/appstore/appstore_download_test.go b/pkg/appstore/appstore_download_test.go index 9f42b42..5285f78 100644 --- a/pkg/appstore/appstore_download_test.go +++ b/pkg/appstore/appstore_download_test.go @@ -696,122 +696,9 @@ var _ = Describe("AppStore (Download)", func() { return file.Name() } -<<<<<<< HEAD It("accepts AppleTVOS packages", func() { path := writePackage([]string{"AppleTVOS"}) defer os.Remove(path) -======= - store := &appstore{ - downloadClient: mockDownloadClient, - bagClient: mockBagClient, - platformClient: mockPlatformClient, - httpClient: mockHTTPClient, - machine: mockMachine, - os: operatingsystem.New(), - } - out, err := store.Download(DownloadInput{ - Account: Account{StoreFront: "143441", Email: "test@example.com"}, - App: App{ID: 42}, - OutputPath: outputPath, - Platform: platform, - ExternalVersionID: versionID, - }) - if succeeds { - Expect(err).ToNot(HaveOccurred()) - Expect(out.DestinationPath).To(Equal(outputPath)) - actual, readErr := os.ReadFile(outputPath) - Expect(readErr).ToNot(HaveOccurred()) - Expect(actual).ToNot(Equal(previousOutput)) - } else { - Expect(err).To(MatchError(ContainSubstring("failed to validate package platform"))) - Expect(out.DestinationPath).To(BeEmpty()) - Expect(os.ReadFile(outputPath)).To(Equal(previousOutput)) - } - Expect(outputPath + ".tmp").ToNot(BeAnExistingFile()) - }, - Entry("iPhone accepts iOS", PlatformIPhone, "", "", "iPhoneOS", true), - Entry("iPad accepts iOS", PlatformIPad, "", "", "iPhoneOS", true), - Entry("iPhone rejects tvOS", PlatformIPhone, "", "", "AppleTVOS", false), - Entry("iPad rejects tvOS", PlatformIPad, "", "", "AppleTVOS", false), - Entry("iPhone rejects visionOS", PlatformIPhone, "", "", "XROS", false), - Entry("a conflicting explicit version is rejected, not replaced", PlatformIPhone, "818970197", "", "AppleTVOS", false), - Entry("default iOS fallback accepts iOS", Platform(""), "", "redownload", "iPhoneOS", true), - Entry("default iOS fallback rejects tvOS", Platform(""), "", "redownload", "AppleTVOS", false), - Entry("latest update accepts iOS", Platform(""), "", "update", "iPhoneOS", true), - Entry("pinned update accepts iOS", Platform(""), "818970197", "update", "iPhoneOS", true), - Entry("pinned update rejects tvOS", Platform(""), "818970197", "update", "AppleTVOS", false), - Entry("pinned redownload preserves an unspecified platform", Platform(""), "818970197", "redownload", "AppleTVOS", true), - Entry("normal unspecified-platform behavior is preserved", Platform(""), "", "", "AppleTVOS", true), - Entry("an explicit tvOS version without a platform is preserved", Platform(""), "818970197", "", "AppleTVOS", true), - Entry("explicit tvOS downloads still work", PlatformAppleTV, "818970197", "", "AppleTVOS", true), - ) - - Describe("macOS packages", func() { - It("decrypts the downloaded package with in-memory machine identity and dpInfo", func() { - tempDir := GinkgoT().TempDir() - requestedPath := filepath.Join(tempDir, "custom-output.pkg") - packageData := []byte("encrypted package") - decryptedData := makeTestXAR([]byte("decrypted payload"), false) - dpInfo := bytes.Repeat([]byte{0x42}, 88) - decrypter := &fakeMacPackageDecrypter{output: decryptedData} - - mockMachine.EXPECT(). - MacAddress(). - Return("00:11:22:aa:bb:cc", nil) - - mockDownloadClient.EXPECT(). - Send(gomock.Any()). - Do(func(req http.Request) { - payload := req.Payload.(*http.XMLPayload) - Expect(payload.Content["guid"]).To(Equal("001122AABBCC")) - }). - Return(http.Result[downloadResult]{ - StatusCode: 200, - Data: downloadResult{ - Items: []downloadItemResult{{ - URL: "https://example.test/app.pkg", - Sinfs: []Sinf{{DPInfo: dpInfo}}, - Metadata: map[string]interface{}{ - "bundleShortVersionString": "1.2.3", - "software-platform": "macos", - "product-type": "ios-app", - }, - }}, - }, - }, nil) - - mockHTTPClient.EXPECT(). - NewRequest("GET", "https://example.test/app.pkg", nil). - Return(&gohttp.Request{Header: gohttp.Header{}}, nil) - mockHTTPClient.EXPECT(). - Do(gomock.Any()). - Return(&gohttp.Response{ - StatusCode: gohttp.StatusOK, - Body: io.NopCloser(bytes.NewReader(packageData)), - ContentLength: int64(len(packageData)), - }, nil) - - store := &appstore{ - downloadClient: mockDownloadClient, - httpClient: mockHTTPClient, - machine: mockMachine, - os: operatingsystem.New(), - macDecrypterFactory: func(ctx context.Context, hardwareID, gotDPInfo []byte) (macPackageDecrypter, error) { - Expect(ctx).ToNot(BeNil()) - Expect(hardwareID).To(Equal([]byte{0x00, 0x11, 0x22, 0xaa, 0xbb, 0xcc})) - Expect(gotDPInfo).To(Equal(dpInfo)) - - return decrypter, nil - }, - } - out, err := store.Download(DownloadInput{ - Context: context.Background(), - App: App{ID: 42, BundleID: "com.example.mac"}, - OutputPath: requestedPath, - Platform: PlatformMacOS, - ExternalVersionID: "123456789", - }) ->>>>>>> c66d653 (fix: select macos packages for universal apps) err := (&appstore{}).validatePackagePlatform(path, PlatformAppleTV) Expect(err).ToNot(HaveOccurred()) @@ -821,106 +708,9 @@ var _ = Describe("AppStore (Download)", func() { path := writePackage([]string{"iPhoneOS"}) defer os.Remove(path) -<<<<<<< HEAD err := (&appstore{}).validatePackagePlatform(path, PlatformAppleTV) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("AppleTVOS")) -======= - mockMachine.EXPECT(). - MacAddress(). - Return("00:11:22:aa:bb:cc", nil) - mockDownloadClient.EXPECT(). - Send(gomock.Any()). - Return(http.Result[downloadResult]{ - StatusCode: 200, - Data: downloadResult{ - Items: []downloadItemResult{{ - URL: "https://example.test/developer.ipa", - Sinfs: []Sinf{{ - ID: 0, - Data: []byte("mobile sinf"), - }}, - Metadata: map[string]interface{}{ - "bundleShortVersionString": "11.0.2", - "software-platform": "ios", - "product-type": "ios-app", - }, - }}, - }, - }, nil) - mockHTTPClient.EXPECT(). - NewRequest("GET", "https://example.test/developer.ipa", nil). - Return(&gohttp.Request{Header: gohttp.Header{}}, nil) - mockHTTPClient.EXPECT(). - Do(gomock.Any()). - Return(&gohttp.Response{ - StatusCode: gohttp.StatusOK, - Body: io.NopCloser(bytes.NewReader(packageBuffer.Bytes())), - ContentLength: int64(packageBuffer.Len()), - }, nil) - - store := &appstore{ - downloadClient: mockDownloadClient, - httpClient: mockHTTPClient, - machine: mockMachine, - os: operatingsystem.New(), - macDecrypterFactory: func(context.Context, []byte, []byte) (macPackageDecrypter, error) { - Fail("mobile packages must not initialize the macOS package decrypter") - - return nil, nil - }, - } - - out, err := store.Download(DownloadInput{ - Context: context.Background(), - Account: Account{Email: "test@example.com"}, - App: App{ID: 640199958, BundleID: "developer.apple.wwdc-Release"}, - OutputPath: tempDir, - Platform: PlatformMacOS, - ExternalVersionID: "123456789", - }) - - Expect(err).ToNot(HaveOccurred()) - Expect(out.DestinationPath).To(Equal(requestedPath)) - Expect(out.Sinfs).To(Equal([]Sinf{{ID: 0, Data: []byte("mobile sinf")}})) - Expect(requestedPath).To(BeAnExistingFile()) - Expect(requestedPath + macEncryptedStageSuffix).ToNot(BeAnExistingFile()) - Expect(requestedPath + macDecryptedStageSuffix).ToNot(BeAnExistingFile()) - }) - - It("uses a pkg name derived from the generated package name", func() { - store := &appstore{os: operatingsystem.New()} - tempDir := GinkgoT().TempDir() - - packagePath, err := store.resolveDestinationPath( - App{ID: 42, BundleID: "com.example.mac"}, - "1.2.3", - tempDir, - PlatformMacOS, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(packagePath).To(Equal(filepath.Join(tempDir, "com.example.mac_42_1.2.3.pkg"))) - }) - - It("preserves an explicit output path", func() { - store := &appstore{os: operatingsystem.New()} - tempDir := GinkgoT().TempDir() - requestedPath := filepath.Join(tempDir, "custom-output") - - packagePath, err := store.resolveDestinationPath(App{}, "1.2.3", requestedPath, PlatformMacOS) - Expect(err).ToNot(HaveOccurred()) - Expect(packagePath).To(Equal(requestedPath)) - }) - - It("rejects a download response without dpInfo", func() { - _, err := macDPInfo(nil) - Expect(err).To(MatchError(ContainSubstring("dpInfo"))) - }) - - It("rejects conflicting dpInfo values", func() { - _, err := macDPInfo([]Sinf{{DPInfo: []byte("one")}, {DPInfo: []byte("two")}}) - Expect(err).To(MatchError(ContainSubstring("conflicting"))) ->>>>>>> c66d653 (fix: select macos packages for universal apps) }) }) diff --git a/resources/ios-entitlements.plist b/resources/ios-entitlements.plist new file mode 100644 index 0000000..441964d --- /dev/null +++ b/resources/ios-entitlements.plist @@ -0,0 +1,14 @@ + + + + + application-identifier + dev.majd.ipatool + com.apple.private.security.no-container + + keychain-access-groups + + dev.majd.ipatool + + + diff --git a/tools/build-ios.sh b/tools/build-ios.sh new file mode 100755 index 0000000..cd833bf --- /dev/null +++ b/tools/build-ios.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu + +# Build a standalone executable for jailbroken arm64 devices running iOS 15+. +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +OUTPUT=${1:-ipatool-ios-arm64} +VERSION=${VERSION:-dev} + +for TOOL in ldid cmake; do + command -v "$TOOL" >/dev/null 2>&1 || { + echo "$TOOL is required; install it with 'brew install ldid cmake'." >&2 + exit 1 + } +done + +SDKROOT=$(xcrun --sdk iphoneos --show-sdk-path) +CC=$(xcrun --sdk iphoneos --find clang) +export SDKROOT CC +export IPHONEOS_DEPLOYMENT_TARGET=15.0 +export GOOS=ios GOARCH=arm64 CGO_ENABLED=1 + +BUILD_DIR=$(mktemp -d /tmp/ipatool-ios.XXXXXX) +trap 'rm -rf "$BUILD_DIR"' EXIT HUP INT TERM + +# Keep this version in sync with internal/sap/unicorn/artifact.go. +curl -fL --retry 3 https://github.com/unicorn-engine/unicorn/archive/refs/tags/2.1.4.tar.gz -o "$BUILD_DIR/unicorn.tar.gz" +echo "ea8863f095a0136388694e5a6063afd9bb7650e30243dd6251af59c5ce5601f4 $BUILD_DIR/unicorn.tar.gz" | shasum -a 256 -c - +tar -xzf "$BUILD_DIR/unicorn.tar.gz" -C "$BUILD_DIR" +# Use iOS W^X memory protection for Unicorn, including nested Go callbacks. +patch -d "$BUILD_DIR/unicorn-2.1.4" -p1 < "$ROOT/tools/patches/unicorn-ios.patch" +ARCHFLAGS='-arch arm64' cmake -S "$BUILD_DIR/unicorn-2.1.4" -B "$BUILD_DIR/build" \ + -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_SYSROOT="$SDKROOT" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ + -DCMAKE_C_FLAGS='-target arm64-apple-ios15.0' -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF -DUNICORN_ARCH=x86 -DUNICORN_BUILD_TESTS=OFF -DUNICORN_INSTALL=OFF +cmake --build "$BUILD_DIR/build" --parallel "$(sysctl -n hw.ncpu)" + +# purego resolves the statically linked Unicorn API through dlsym. +export CGO_LDFLAGS="${CGO_LDFLAGS:-} -Wl,-force_load,$BUILD_DIR/build/libunicorn.a -Wl,-export_dynamic" +cd "$ROOT" +go build -ldflags="-X github.com/majd/ipatool/v2/cmd.version=$VERSION" -o "$OUTPUT" . +ldid -S"$ROOT/resources/ios-entitlements.plist" "$OUTPUT" diff --git a/tools/patches/unicorn-ios.patch b/tools/patches/unicorn-ios.patch new file mode 100644 index 0000000..5f83473 --- /dev/null +++ b/tools/patches/unicorn-ios.patch @@ -0,0 +1,76 @@ +--- a/qemu/configure ++++ b/qemu/configure +@@ -2152,7 +2152,7 @@ + if [ "$darwin" = "yes" ] ; then + cat > $TMPC << EOF + #include +-int main() { pthread_jit_write_protect_supported_np(); return 0;} ++int main() { pthread_jit_write_protect_supported_np(); pthread_jit_write_protect_np(0); return 0;} + EOF + if ! compile_prog ""; then + have_pthread_jit_protect='no' +--- a/qemu/include/tcg/tcg-apple-jit.h ++++ b/qemu/include/tcg/tcg-apple-jit.h +@@ -104,13 +104,11 @@ + #endif + + +-#if defined(__APPLE__) && defined(HAVE_PTHREAD_JIT_PROTECT) && (defined(__arm__) || defined(__aarch64__)) ++#if defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) + +-/* write protect enable = write disable */ +-static inline void jit_write_protect(int enabled) +-{ +- return pthread_jit_write_protect_np(enabled); +-} ++struct uc_struct; ++void ipatool_jit_write_protect(struct uc_struct *uc, int enabled); ++#define jit_write_protect(enabled) ipatool_jit_write_protect(uc, enabled) + + #define JIT_CALLBACK_GUARD(x) \ + { \ +--- a/qemu/accel/tcg/translate-all.c ++++ b/qemu/accel/tcg/translate-all.c +@@ -1019,7 +1019,7 @@ + static inline void *alloc_code_gen_buffer(struct uc_struct *uc) + { + TCGContext *tcg_ctx = uc->tcg_ctx; +- int prot = PROT_WRITE | PROT_READ | PROT_EXEC; ++ int prot = PROT_WRITE | PROT_READ; + int flags = MAP_PRIVATE | MAP_ANONYMOUS; + size_t size = tcg_ctx->code_gen_buffer_size; + void *buf; +@@ -2172,7 +2172,20 @@ + } + + +-#if defined(__APPLE__) && defined(HAVE_PTHREAD_JIT_PROTECT) && (defined(__arm__) || defined(__aarch64__)) ++/* iOS CLI processes cannot use MAP_JIT outside an app sandbox. Toggle the ++ * translation buffer between writable and executable instead of using RWX. */ ++void ipatool_jit_write_protect(struct uc_struct *uc, int enabled) ++{ ++ TCGContext *ctx = uc->tcg_ctx; ++ if (ctx && ctx->initial_buffer && ++ mprotect(ctx->initial_buffer, ctx->initial_buffer_size, ++ PROT_READ | (enabled ? PROT_EXEC : PROT_WRITE)) != 0) { ++ perror("protect Unicorn translation buffer"); ++ abort(); ++ } ++} ++ ++#if defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) + static bool tb_exec_is_locked(struct uc_struct *uc) + { + return uc->current_executable; +--- a/uc.c ++++ b/uc.c +@@ -35,8 +35,7 @@ + static uc_err uc_snapshot(uc_engine *uc); + static uc_err uc_restore_latest_snapshot(uc_engine *uc); + +-#if defined(__APPLE__) && defined(HAVE_PTHREAD_JIT_PROTECT) && \ +- (defined(__arm__) || defined(__aarch64__)) ++#if defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) + static void save_jit_state(uc_engine *uc) + { + if (!uc->nested) { From 40d7a5c40147d1022d4ddd521e7c93ca29a1db05 Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 23:20:40 +0200 Subject: [PATCH 10/18] fix: keep authentication input inline and mask passwords Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- cmd/auth.go | 25 ++---------- cmd/prompt.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++ cmd/prompt_test.go | 48 +++++++++++++++++++++++ 3 files changed, 149 insertions(+), 22 deletions(-) create mode 100644 cmd/prompt.go create mode 100644 cmd/prompt_test.go diff --git a/cmd/auth.go b/cmd/auth.go index 7f3582d..f2bfdb2 100644 --- a/cmd/auth.go +++ b/cmd/auth.go @@ -1,19 +1,16 @@ package cmd import ( - "bufio" "encoding/json" "errors" "fmt" "io" "os" - "strings" "time" "github.com/avast/retry-go" "github.com/majd/ipatool/v2/pkg/appstore" "github.com/spf13/cobra" - "golang.org/x/term" ) func authCmd() *cobra.Command { @@ -32,18 +29,6 @@ func authCmd() *cobra.Command { } func loginCmd() *cobra.Command { - promptForAuthCode := func() (string, error) { - authCode, err := bufio.NewReader(os.Stdin).ReadString('\n') - if err != nil { - return "", fmt.Errorf("failed to read string: %w", err) - } - - authCode = strings.Trim(authCode, "\n") - authCode = strings.Trim(authCode, "\r") - - return authCode, nil - } - var email, password, authCode, sessionOutput string var mzfinance bool @@ -58,13 +43,11 @@ func loginCmd() *cobra.Command { } if password == "" && interactive { - dependencies.Logger.Log().Msg("enter password:") - - bytes, err := term.ReadPassword(int(os.Stdin.Fd())) + value, err := readPrompt("enter password: ", true) if err != nil { return fmt.Errorf("failed to read password: %w", err) } - password = string(bytes) + password = value } var lastErr error @@ -72,10 +55,8 @@ func loginCmd() *cobra.Command { // nolint:wrapcheck return retry.Do(func() error { if errors.Is(lastErr, appstore.ErrAuthCodeRequired) && interactive { - dependencies.Logger.Log().Msg("enter 2FA code:") - var err error - authCode, err = promptForAuthCode() + authCode, err = readPrompt("enter 2FA code: ", false) if err != nil { return fmt.Errorf("failed to read auth code: %w", err) } diff --git a/cmd/prompt.go b/cmd/prompt.go new file mode 100644 index 0000000..322d8c8 --- /dev/null +++ b/cmd/prompt.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strings" + "unicode" + + "golang.org/x/term" +) + +func readPrompt(prompt string, masked bool) (string, error) { + if masked { + state, err := term.MakeRaw(int(os.Stdin.Fd())) + if err != nil { + return "", fmt.Errorf("failed to configure terminal: %w", err) + } + defer term.Restore(int(os.Stdin.Fd()), state) //nolint:errcheck + } + + // Prompts belong on stderr so structured output on stdout stays valid. + if _, err := io.WriteString(os.Stderr, prompt); err != nil { + return "", fmt.Errorf("failed to write prompt: %w", err) + } + + if masked { + defer fmt.Fprint(os.Stderr, "\r\n") + + return readMaskedInput(os.Stdin, os.Stderr) + } + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return "", fmt.Errorf("failed to read input: %w", err) + } + + return strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r"), nil +} + +func readMaskedInput(input io.Reader, output io.Writer) (string, error) { + reader := bufio.NewReader(input) + + var ( + password []rune + escape int + ) + + for { + key, _, err := reader.ReadRune() + if err != nil { + return "", fmt.Errorf("failed to read input: %w", err) + } + + // Ignore terminal escape sequences (for example, arrow keys). + if escape != 0 { + if escape == 1 && (key == '[' || key == 'O') { + escape = 2 + } else if escape == 1 || (key >= 0x40 && key <= 0x7e) { + escape = 0 + } + + continue + } + + var echo string + + switch key { + case '\r', '\n': + return string(password), nil + case 3: + return "", errors.New("input interrupted") + case 4: + return "", io.EOF + case 27: + escape = 1 + case '\b', 127: + if len(password) > 0 { + password = password[:len(password)-1] + echo = "\b \b" + } + case 21: + echo = strings.Repeat("\b \b", len(password)) + password = password[:0] + default: + if unicode.IsPrint(key) { + password = append(password, key) + echo = "*" + } + } + + if _, err := io.WriteString(output, echo); err != nil { + return "", fmt.Errorf("failed to echo input: %w", err) + } + } +} diff --git a/cmd/prompt_test.go b/cmd/prompt_test.go new file mode 100644 index 0000000..bf2a208 --- /dev/null +++ b/cmd/prompt_test.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "bytes" + "io" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Masked input", func() { + DescribeTable("echoes masks while preserving the entered password", func(input, password, echo string) { + var output bytes.Buffer + value, err := readMaskedInput(strings.NewReader(input), &output) + + Expect(err).NotTo(HaveOccurred()) + Expect(value).To(Equal(password)) + Expect(output.String()).To(Equal(echo)) + }, + Entry("enter", "secret\r", "secret", "******"), + Entry("newline", "secret\n", "secret", "******"), + Entry("unicode", "pä密🔑\r", "pä密🔑", "****"), + Entry("delete", "ab\x7fc\r", "ac", "**\b \b*"), + Entry("unicode backspace", "a密\bc\r", "ac", "**\b \b*"), + Entry("backspace on empty input", "\b\x7fa\r", "a", "*"), + Entry("clear line", "ab\x15c\r", "c", "**\b \b\b \b*"), + Entry("arrow keys", "ab\x1b[D\x1bOCc\r", "abc", "***"), + Entry("empty input", "\r", "", ""), + ) + + It("cancels without returning the password on Ctrl-C", func() { + value, err := readMaskedInput(strings.NewReader("secret\x03"), io.Discard) + + Expect(err).To(MatchError("input interrupted")) + Expect(value).To(BeEmpty()) + }) + + DescribeTable("discards incomplete input", func(input string) { + value, err := readMaskedInput(strings.NewReader(input), io.Discard) + + Expect(err).To(MatchError(ContainSubstring("EOF"))) + Expect(value).To(BeEmpty()) + }, + Entry("end of stream", "secret"), + Entry("Ctrl-D", "secret\x04"), + ) +}) From 2c1955fbb069d99c277037a0d0179d420896f330 Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sun, 13 Sep 2026 23:28:20 +0200 Subject: [PATCH 11/18] feat: prompt for email during interactive login --- cmd/auth.go | 14 ++++++- cmd/auth_test.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 cmd/auth_test.go diff --git a/cmd/auth.go b/cmd/auth.go index f2bfdb2..4c022ba 100644 --- a/cmd/auth.go +++ b/cmd/auth.go @@ -38,6 +38,18 @@ func loginCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { interactive := cmd.Context().Value(interactiveKey).(bool) + if email == "" && !interactive { + return errors.New("email is required when not running in interactive mode; use the \"--email\" flag") + } + + if email == "" && interactive { + value, err := readPrompt("enter email: ", false) + if err != nil { + return fmt.Errorf("failed to read email: %w", err) + } + email = value + } + if password == "" && !interactive { return errors.New("password is required when not running in interactive mode; use the \"--password\" flag") } @@ -138,8 +150,6 @@ func loginCmd() *cobra.Command { cmd.Flags().StringVar(&sessionOutput, "session-output", "", "path to save the account session to after a successful login") cmd.Flags().BoolVar(&mzfinance, "mzfinance", false, "use the stable legacy MZFinance login flow (GSA -> MZFinance) instead of the default native/fast path") - _ = cmd.MarkFlagRequired("email") - return cmd } diff --git a/cmd/auth_test.go b/cmd/auth_test.go new file mode 100644 index 0000000..b3fbbf0 --- /dev/null +++ b/cmd/auth_test.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "context" + "os" + "path/filepath" + + "github.com/majd/ipatool/v2/pkg/appstore" + "github.com/majd/ipatool/v2/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Login command", func() { + var store *fakeLoginAppStore + + BeforeEach(func() { + previousDependencies := dependencies + DeferCleanup(func() { dependencies = previousDependencies }) + store = &fakeLoginAppStore{} + dependencies.AppStore = store + dependencies.Logger = log.NewLogger(log.Args{}) + }) + + DescribeTable("requires credentials in non-interactive mode", func(args []string, message string) { + cmd := loginCmd() + cmd.SetContext(context.WithValue(context.Background(), interactiveKey, false)) + cmd.SetArgs(args) + + Expect(cmd.Execute()).To(MatchError(message)) + Expect(store.loginCalls).To(BeZero()) + }, + Entry("missing email", []string{"--password", "secret"}, "email is required when not running in interactive mode; use the \"--email\" flag"), + Entry("missing password", []string{"--email", "user@example.com"}, "password is required when not running in interactive mode; use the \"--password\" flag"), + ) + + DescribeTable("uses supplied credentials", func(interactive bool) { + cmd := loginCmd() + cmd.SetContext(context.WithValue(context.Background(), interactiveKey, interactive)) + cmd.SetArgs([]string{"--email", "user@example.com", "--password", "secret"}) + + Expect(cmd.Execute()).To(Succeed()) + Expect(store.loginCalls).To(Equal(1)) + Expect(store.input).To(Equal(appstore.LoginInput{Email: "user@example.com", Password: "secret"})) + }, + Entry("interactive", true), + Entry("non-interactive", false), + ) + + DescribeTable("prompts for email", func(input, message string) { + dir := GinkgoT().TempDir() + inputPath := filepath.Join(dir, "stdin") + Expect(os.WriteFile(inputPath, []byte(input), 0o600)).To(Succeed()) + stdin, err := os.Open(inputPath) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(stdin.Close) + stderr, err := os.Create(filepath.Join(dir, "stderr")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(stderr.Close) + + previousStdin, previousStderr := os.Stdin, os.Stderr + os.Stdin, os.Stderr = stdin, stderr + DeferCleanup(func() { os.Stdin, os.Stderr = previousStdin, previousStderr }) + + cmd := loginCmd() + cmd.SetContext(context.WithValue(context.Background(), interactiveKey, true)) + cmd.SetArgs([]string{"--password", "secret"}) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + + err = cmd.Execute() + if message == "" { + Expect(err).NotTo(HaveOccurred()) + Expect(store.loginCalls).To(Equal(1)) + Expect(store.input).To(Equal(appstore.LoginInput{Email: "user@example.com", Password: "secret"})) + } else { + Expect(err).To(MatchError(message)) + Expect(store.loginCalls).To(BeZero()) + } + + prompt, err := os.ReadFile(stderr.Name()) + Expect(err).NotTo(HaveOccurred()) + Expect(string(prompt)).To(Equal("enter email: ")) + }, + Entry("reads unmasked input without requiring a terminal", "user@example.com\n", ""), + Entry("reports input errors", "", "failed to read email: failed to read input: EOF"), + ) +}) + +type fakeLoginAppStore struct { + appstore.AppStore + input appstore.LoginInput + loginCalls int +} + +func (f *fakeLoginAppStore) Login(input appstore.LoginInput) (appstore.LoginOutput, error) { + f.input = input + f.loginCalls++ + + return appstore.LoginOutput{}, nil +} From 13bf043521d611fbcd8a5e22f241963d4fd39260 Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Fri, 18 Sep 2026 23:10:34 +0200 Subject: [PATCH 12/18] fix: fall back to consumer catalogs for ios version lookup Cherry-pick of upstream e5211d6 (adapted): - lookupLatestExternalVersionID now tries the enterprise catalog first and falls back to the iphone/ipad consumer catalogs for iOS platforms - platformVersionLookupRequest takes the catalog string directly - visionOS lookup split out to the storefront product page; ported visionProductURL/visionExternalVersionID helpers into storefront.go - skipped appstore_download_product_test.go changes (file does not exist in this fork) - added upstream appstore_platform_version_lookup_test.go Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .../appstore_platform_version_lookup.go | 98 +++++++++++----- .../appstore_platform_version_lookup_test.go | 99 ++++++++++++++++ pkg/appstore/storefront.go | 110 ++++++++++++++++++ 3 files changed, 279 insertions(+), 28 deletions(-) create mode 100644 pkg/appstore/appstore_platform_version_lookup_test.go diff --git a/pkg/appstore/appstore_platform_version_lookup.go b/pkg/appstore/appstore_platform_version_lookup.go index 3334fe9..3631bd3 100644 --- a/pkg/appstore/appstore_platform_version_lookup.go +++ b/pkg/appstore/appstore_platform_version_lookup.go @@ -66,52 +66,94 @@ func (t *appstore) lookupLatestExternalVersionID(acc Account, app App, platform return "", fmt.Errorf("failed to resolve the country code: %w", err) } - request, err := t.platformVersionLookupRequest(app.ID, countryCode, platform) - if err != nil { - return "", fmt.Errorf("failed to create platform version lookup request: %w", err) + if platform == PlatformVisionOS { + return t.lookupLatestVisionOSExternalVersionID(app.ID, countryCode) } - res, err := t.platformClient.Send(request) + metadataPlatform, err := platform.metadataPlatform() if err != nil { - return "", fmt.Errorf("platform version lookup request failed: %w", err) - } - - if res.StatusCode != gohttp.StatusOK { - return "", NewErrorWithMetadata(errors.New("platform version lookup request failed"), res) + return "", fmt.Errorf("failed to create platform version lookup request: %w", err) } - item, ok := res.Data.Results[strconv.FormatInt(app.ID, 10)] - if !ok { - return "", NewErrorWithMetadata(errors.New("platform version lookup returned no app"), res) + catalogs := []string{metadataPlatform} + if platform == PlatformIPhone || platform == PlatformIPad { + // Some storefronts have no enterprise listing even when the consumer + // catalogs contain the app. Keep the account's country for each lookup. + catalogs = append(catalogs, "iphone", "ipad") } - if len(item.Offers) == 0 { - return "", NewErrorWithMetadata(errors.New("platform version lookup returned no offers"), res) - } + var lastErr error - offer := item.Offers[0] - externalVersionID := string(offer.Version.ExternalID) + for _, catalog := range catalogs { + request := t.platformVersionLookupRequest(app.ID, countryCode, catalog) - if externalVersionID == "" { - externalVersionID, err = externalVersionIDFromBuyParams(offer.BuyParams) + res, err := t.platformClient.Send(request) if err != nil { - return "", fmt.Errorf("failed to parse buy params: %w", err) + return "", fmt.Errorf("platform version lookup request failed: %w", err) } - } - if externalVersionID == "" { - return "", NewErrorWithMetadata(errors.New("platform version lookup returned no external version id"), res) + if res.StatusCode != gohttp.StatusOK { + return "", NewErrorWithMetadata(errors.New("platform version lookup request failed"), res) + } + + item, ok := res.Data.Results[strconv.FormatInt(app.ID, 10)] + if !ok { + lastErr = NewErrorWithMetadata(errors.New("platform version lookup returned no app"), res) + + continue + } + + if len(item.Offers) == 0 { + lastErr = NewErrorWithMetadata(errors.New("platform version lookup returned no offers"), res) + + continue + } + + offer := item.Offers[0] + externalVersionID := string(offer.Version.ExternalID) + + if externalVersionID == "" { + externalVersionID, err = externalVersionIDFromBuyParams(offer.BuyParams) + if err != nil { + return "", fmt.Errorf("failed to parse buy params: %w", err) + } + } + + if externalVersionID == "" { + return "", NewErrorWithMetadata(errors.New("platform version lookup returned no external version id"), res) + } + + return externalVersionID, nil } - return externalVersionID, nil + return "", fmt.Errorf("app %d in storefront %s (catalogs: %s): %w", app.ID, countryCode, strings.Join(catalogs, ", "), lastErr) } -func (*appstore) platformVersionLookupRequest(appID int64, countryCode string, platform Platform) (http.Request, error) { - metadataPlatform, err := platform.metadataPlatform() +func (t *appstore) lookupLatestVisionOSExternalVersionID(appID int64, countryCode string) (string, error) { + request := http.Request{ + URL: visionProductURL(appID, countryCode), + Method: http.MethodGET, + ResponseFormat: http.ResponseFormatRaw, + } + + res, err := t.storefrontClient.Send(request) + if err != nil { + return "", fmt.Errorf("visionOS version lookup request failed: %w", err) + } + + if res.StatusCode != gohttp.StatusOK { + return "", NewErrorWithMetadata(errors.New("visionOS version lookup request failed"), res) + } + + externalVersionID, err := visionExternalVersionID(res.Data, appID) if err != nil { - return http.Request{}, err + return "", fmt.Errorf("failed to parse visionOS version lookup response: %w", err) } + return externalVersionID, nil +} + +func (*appstore) platformVersionLookupRequest(appID int64, countryCode, metadataPlatform string) http.Request { params := url.Values{} params.Add("version", "2") params.Add("id", strconv.FormatInt(appID, 10)) @@ -125,7 +167,7 @@ func (*appstore) platformVersionLookupRequest(appID int64, countryCode string, p URL: fmt.Sprintf("https://uclient-api.itunes.apple.com/WebObjects/MZStorePlatform.woa/wa/lookup?%s", params.Encode()), Method: http.MethodGET, ResponseFormat: http.ResponseFormatJSON, - }, nil + } } func externalVersionIDFromBuyParams(buyParams string) (string, error) { diff --git a/pkg/appstore/appstore_platform_version_lookup_test.go b/pkg/appstore/appstore_platform_version_lookup_test.go new file mode 100644 index 0000000..9ad18e3 --- /dev/null +++ b/pkg/appstore/appstore_platform_version_lookup_test.go @@ -0,0 +1,99 @@ +package appstore + +import ( + "errors" + "net/url" + + "github.com/majd/ipatool/v2/pkg/http" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" +) + +var _ = Describe("Platform version catalog fallback", func() { + var client *http.MockClient[platformVersionLookupResult] + var store *appstore + account := Account{StoreFront: "143469-1,34"} + app := App{ID: 6472431552} + missing := platformVersionLookupResult{} + noOffers := platformVersionLookupResult{Results: map[string]platformVersionLookupItem{"6472431552": {}}} + withOffer := func(offer platformVersionLookupOffer) platformVersionLookupResult { + return platformVersionLookupResult{Results: map[string]platformVersionLookupItem{"6472431552": {Offers: []platformVersionLookupOffer{offer}}}} + } + valid := withOffer(platformVersionLookupOffer{Version: platformVersionLookupVersion{ExternalID: "891116578"}}) + + BeforeEach(func() { + client = http.NewMockClient[platformVersionLookupResult](gomock.NewController(GinkgoT())) + store = &appstore{platformClient: client} + }) + + expectLookup := func(catalog string, status int, data platformVersionLookupResult, sendErr error) *gomock.Call { + return client.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + u, err := url.Parse(req.URL) + Expect(err).ToNot(HaveOccurred()) + Expect(u.Query().Get("platform")).To(Equal(catalog)) + Expect(u.Query().Get("cc")).To(Equal("ru")) + Expect(u.Query().Get("id")).To(Equal("6472431552")) + }).Return(http.Result[platformVersionLookupResult]{StatusCode: status, Data: data}, sendErr) + } + + DescribeTable("stops at the first available iOS catalog", func(platform Platform, empty platformVersionLookupResult, successIndex int, offer platformVersionLookupResult) { + var calls []any + for i, catalog := range []string{"enterprisestore", "iphone", "ipad"} { + data := empty + if i == successIndex { + data = offer + } + calls = append(calls, expectLookup(catalog, 200, data, nil)) + if i == successIndex { + break + } + } + gomock.InOrder(calls...) + version, err := store.lookupLatestExternalVersionID(account, app, platform) + Expect(err).ToNot(HaveOccurred()) + Expect(version).To(Equal("891116578")) + }, + Entry("enterprise succeeds", PlatformIPhone, missing, 0, valid), + Entry("missing enterprise app", PlatformIPhone, missing, 1, valid), + Entry("empty enterprise offers", PlatformIPhone, noOffers, 1, valid), + Entry("only iPad catalog succeeds", PlatformIPhone, missing, 2, valid), + Entry("empty offers until iPad", PlatformIPad, noOffers, 2, valid), + Entry("iPad request uses iPhone fallback first", PlatformIPad, missing, 1, valid), + Entry("consumer buy params version", PlatformIPhone, missing, 1, withOffer(platformVersionLookupOffer{BuyParams: "appExtVrsId=891116578"})), + ) + + DescribeTable("reports exhausted catalogs", func(empty platformVersionLookupResult, message string) { + gomock.InOrder(expectLookup("enterprisestore", 200, empty, nil), expectLookup("iphone", 200, empty, nil), expectLookup("ipad", 200, empty, nil)) + version, err := store.lookupLatestExternalVersionID(account, app, PlatformIPhone) + Expect(version).To(BeEmpty()) + Expect(err).To(MatchError(And(ContainSubstring(message), ContainSubstring("6472431552"), ContainSubstring("RU"), ContainSubstring("enterprisestore, iphone, ipad")))) + }, Entry("missing apps", missing, "returned no app"), Entry("empty offers", noOffers, "returned no offers")) + + DescribeTable("does not fall back after a request or offer error", func(status int, data platformVersionLookupResult, sendErr error, message string) { + expectLookup("enterprisestore", status, data, sendErr) + _, err := store.lookupLatestExternalVersionID(account, app, PlatformIPhone) + Expect(err).To(MatchError(ContainSubstring(message))) + if sendErr != nil { + Expect(errors.Is(err, sendErr)).To(BeTrue()) + } + }, + Entry("transport failure", 0, missing, errors.New("connection reset"), "request failed"), + Entry("decoding failure", 200, missing, errors.New("invalid JSON"), "request failed"), + Entry("HTTP failure", 503, missing, nil, "request failed"), + Entry("missing version", 200, withOffer(platformVersionLookupOffer{}), nil, "no external version id"), + Entry("malformed buy params", 200, withOffer(platformVersionLookupOffer{BuyParams: "appExtVrsId=%zz"}), nil, "failed to parse buy params"), + ) + + It("stops on an HTTP error in a consumer catalog", func() { + gomock.InOrder(expectLookup("enterprisestore", 200, missing, nil), expectLookup("iphone", 503, missing, nil)) + _, err := store.lookupLatestExternalVersionID(account, app, PlatformIPhone) + Expect(err).To(MatchError(ContainSubstring("request failed"))) + }) + + It("does not use iOS catalogs for Apple TV", func() { + expectLookup("atv9", 200, missing, nil) + _, err := store.lookupLatestExternalVersionID(account, app, PlatformAppleTV) + Expect(err).To(MatchError(ContainSubstring("returned no app"))) + }) +}) diff --git a/pkg/appstore/storefront.go b/pkg/appstore/storefront.go index 8f0f7a9..7b8b553 100644 --- a/pkg/appstore/storefront.go +++ b/pkg/appstore/storefront.go @@ -1,7 +1,11 @@ package appstore import ( + "encoding/json" + "errors" "fmt" + "net/url" + "strconv" "strings" ) @@ -153,3 +157,109 @@ var storeFronts = map[string]string{ "YE": "143571", "ZA": "143472", } + +func visionProductURL(appID int64, countryCode string) string { + params := url.Values{} + params.Set("platform", "vision") + + return fmt.Sprintf("https://apps.apple.com/%s/app/id%d?%s", strings.ToLower(countryCode), appID, params.Encode()) +} + +func visionExternalVersionID(body []byte, appID int64) (string, error) { + data, err := serializedServerData(body) + if err != nil { + return "", err + } + + var value interface{} + if err := json.Unmarshal(data, &value); err != nil { + return "", fmt.Errorf("failed to decode serialized server data: %w", err) + } + + externalVersionID, found := findVisionExternalVersionID(value, appID) + if !found { + return "", errors.New("visionOS purchase configuration was not found") + } + + if externalVersionID == "" { + return "", errors.New("visionOS purchase configuration has no external version id") + } + + return externalVersionID, nil +} + +func findVisionExternalVersionID(value interface{}, appID int64) (string, bool) { + matched := false + + switch typedValue := value.(type) { + case []interface{}: + for _, item := range typedValue { + if externalVersionID, found := findVisionExternalVersionID(item, appID); found { + if externalVersionID != "" { + return externalVersionID, true + } + + matched = true + } + } + case map[string]interface{}: + if configuration, ok := typedValue["purchaseConfiguration"].(map[string]interface{}); ok { + if externalVersionID, found := externalVersionIDFromVisionConfiguration(configuration, appID); found { + if externalVersionID != "" { + return externalVersionID, true + } + + matched = true + } + } + + for _, child := range typedValue { + if externalVersionID, found := findVisionExternalVersionID(child, appID); found { + if externalVersionID != "" { + return externalVersionID, true + } + + matched = true + } + } + } + + return "", matched +} + +func externalVersionIDFromVisionConfiguration(configuration map[string]interface{}, appID int64) (string, bool) { + if configuration["metricsPlatformDisplayStyle"] != "vision" { + return "", false + } + + platforms, ok := configuration["appPlatforms"].([]interface{}) + if !ok { + return "", false + } + + isVisionApp := false + + for _, platform := range platforms { + if platform == "vision" { + isVisionApp = true + + break + } + } + + if !isVisionApp { + return "", false + } + + buyParams, ok := configuration["buyParams"].(string) + if !ok || buyParams == "" { + return "", false + } + + values, err := url.ParseQuery(buyParams) + if err != nil || values.Get("salableAdamId") != strconv.FormatInt(appID, 10) { + return "", false + } + + return values.Get("appExtVrsId"), true +} From 7b2a34dadf9ca5326f4278e49b80bfb31dd14774 Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Sat, 19 Sep 2026 16:43:49 +0200 Subject: [PATCH 13/18] fix: normalize 2fa input and clarify verification errors Cherry-pick of upstream 735b689 (adapted to the fork layout): - normalizeAuthCode strips whitespace, bracketed-paste markers and validates the six-digit format; applied in both Login entry points (Login and LoginMZFinance) so every downstream path (legacy, GSA) receives the normalized code - parseLoginResponse now distinguishes a missing 2FA code from a failed verification with a dedicated error message - dropped the incoming parseLoginResponse/loginRequest duplicates (upstream's SAP-signer variants) in favor of the fork's versions - adopted all four test additions (malformed-code table, password normalization table, incomplete-verification case) Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- pkg/appstore/appstore_login.go | 47 +++++++++++++++++++++++++++-- pkg/appstore/appstore_login_test.go | 43 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/pkg/appstore/appstore_login.go b/pkg/appstore/appstore_login.go index ff561a8..9a4d388 100644 --- a/pkg/appstore/appstore_login.go +++ b/pkg/appstore/appstore_login.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "time" + "unicode" "github.com/majd/ipatool/v2/pkg/gsa" "github.com/majd/ipatool/v2/pkg/http" @@ -40,6 +41,12 @@ type LoginOutput struct { } func (t *appstore) Login(input LoginInput) (LoginOutput, error) { + authCode, err := normalizeAuthCode(input.AuthCode) + if err != nil { + return LoginOutput{}, err + } + input.AuthCode = authCode + macAddr, err := t.machine.MacAddress() if err != nil { return LoginOutput{}, fmt.Errorf("failed to get mac address: %w", err) @@ -112,6 +119,12 @@ func (t *appstore) Login(input LoginInput) (LoginOutput, error) { // endpoint (the same stable path used on Windows), bypassing the glitchy // native/fast endpoint that Login may fall back to on macOS. func (t *appstore) LoginMZFinance(input LoginInput) (LoginOutput, error) { + authCode, err := normalizeAuthCode(input.AuthCode) + if err != nil { + return LoginOutput{}, err + } + input.AuthCode = authCode + macAddr, err := t.machine.MacAddress() if err != nil { return LoginOutput{}, fmt.Errorf("failed to get mac address: %w", err) @@ -202,6 +215,32 @@ func (t *appstore) loginWithGSA(input LoginInput, guid string) (Account, error) return out, nil } +func normalizeAuthCode(code string) (string, error) { + if code == "" { + return "", nil + } + + // Terminals may wrap pasted input in bracketed-paste markers. Strip only + // a matched outer pair; other escape sequences are invalid input. + code = strings.TrimSpace(code) + if strings.HasPrefix(code, "\x1b[200~") && strings.HasSuffix(code, "\x1b[201~") { + code = strings.TrimSuffix(strings.TrimPrefix(code, "\x1b[200~"), "\x1b[201~") + } + + code = strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + + return r + }, code) + if len(code) != 6 || strings.IndexFunc(code, func(r rune) bool { return r < '0' || r > '9' }) != -1 { + return "", errors.New("2FA code must contain exactly six digits") + } + + return code, nil +} + type loginAddressResult struct { FirstName string `plist:"firstName,omitempty"` LastName string `plist:"lastName,omitempty"` @@ -436,8 +475,12 @@ func (t *appstore) parseLoginResponse(res *http.Result[loginResult], authCode st } else { retry = true } - } else if res.Data.FailureType == "" && authCode == "" && res.Data.CustomerMessage == CustomerMessageBadLogin { - err = ErrAuthCodeRequired + } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageBadLogin { + if authCode == "" { + err = ErrAuthCodeRequired + } else { + err = errors.New("apple did not complete verification; try a fresh 2FA code") + } } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageAccountDisabled { err = NewErrorWithMetadata(errors.New("account is disabled"), res) } else if res.Data.FailureType != "" { diff --git a/pkg/appstore/appstore_login_test.go b/pkg/appstore/appstore_login_test.go index af9857f..0c5a1bd 100644 --- a/pkg/appstore/appstore_login_test.go +++ b/pkg/appstore/appstore_login_test.go @@ -47,6 +47,21 @@ var _ = Describe("AppStore (Login)", func() { ctrl.Finish() }) + DescribeTable("rejects malformed 2FA codes before preparing authentication", func(code string) { + _, err := as.Login(LoginInput{AuthCode: code}) + + Expect(err).To(MatchError("2FA code must contain exactly six digits")) + }, + Entry("whitespace only", " \t\r\n"), + Entry("too short", "12345"), + Entry("too long", "1234567"), + Entry("letters", "12345a"), + Entry("non-ASCII digits", "１２３４５６"), + Entry("other escape sequences", "\x1b[31m123456"), + Entry("unmatched paste marker", "\x1b[200~123456"), + Entry("empty paste", "\x1b[200~\x1b[201~"), + Entry("embedded paste markers", "123\x1b[200~456\x1b[201~"), + ) When("fails to read Machine's MAC address", func() { BeforeEach(func() { mockMachine.EXPECT(). @@ -69,6 +84,26 @@ var _ = Describe("AppStore (Login)", func() { Return("00:00:00:00:00:00", nil) }) + DescribeTable("normalizes 2FA codes without changing the password", func(code, suffix string) { + const password = " \tpäss word\n" + mockClient.EXPECT().Send(gomock.Any()).DoAndReturn(func(req http.Request) (http.Result[loginResult], error) { + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("password", password+suffix)) + + return http.Result[loginResult]{}, errors.New("test complete") + }) + + _, err := as.Login(LoginInput{Password: password, AuthCode: code}) + + Expect(err).To(MatchError(ContainSubstring("test complete"))) + }, + Entry("no code on initial login", "", ""), + Entry("plain code with leading zero", "012345", "012345"), + Entry("spaces", "123 456", "123456"), + Entry("Unicode whitespace", "\t123\u00a0456\r\n", "123456"), + Entry("bracketed paste", "\x1b[200~123456\x1b[201~", "123456"), + Entry("bracketed paste with whitespace", " \x1b[200~123 456\n\x1b[201~\r\n", "123456"), + ) + When("client returns error", func() { BeforeEach(func() { mockClient.EXPECT(). @@ -209,6 +244,7 @@ var _ = Describe("AppStore (Login)", func() { mockClient.EXPECT(). Send(gomock.Any()). Return(http.Result[loginResult]{ + StatusCode: 200, Data: loginResult{ FailureType: "", CustomerMessage: CustomerMessageBadLogin, @@ -222,6 +258,13 @@ var _ = Describe("AppStore (Login)", func() { }) Expect(err).To(Equal(ErrAuthCodeRequired)) }) + + It("reports an incomplete verification when a code was already supplied", func() { + _, err := as.Login(LoginInput{Password: testPassword, AuthCode: "123456"}) + + Expect(err).To(MatchError("apple did not complete verification; try a fresh 2FA code")) + Expect(errors.Is(err, ErrAuthCodeRequired)).To(BeFalse()) + }) }) When("store API redirects", func() { From 7b4739745cd36922a2b16a636b583f830c3cc74d Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Tue, 29 Sep 2026 23:27:52 +0200 Subject: [PATCH 14/18] fix: recover redacted mac addresses on macos 27 Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- go.mod | 1 + go.sum | 2 + pkg/util/machine/mac_address_darwin.go | 127 +++++++++++ pkg/util/machine/mac_address_darwin_test.go | 237 ++++++++++++++++++++ pkg/util/machine/mac_address_default.go | 9 + pkg/util/machine/machine.go | 2 +- pkg/util/machine/machine_test.go | 5 + 7 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 pkg/util/machine/mac_address_darwin.go create mode 100644 pkg/util/machine/mac_address_darwin_test.go create mode 100644 pkg/util/machine/mac_address_default.go diff --git a/go.mod b/go.mod index db2a6c9..c92eb34 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/byteness/go-keychain v0.0.0-20191008050251-8e49817e8af4 github.com/byteness/keyring v1.9.0 + github.com/ebitengine/purego v0.10.2 github.com/juju/persistent-cookiejar v1.0.0 github.com/onsi/ginkgo/v2 v2.5.0 github.com/onsi/gomega v1.24.0 diff --git a/go.sum b/go.sum index 987d513..e498ea6 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/byteness/percent v0.2.2 h1:vnIFh8WBR1xoC+U2etz0EMB1cgp+vsK6vynqTCeDzi github.com/byteness/percent v0.2.2/go.mod h1:nwavge92FhIyfnldz4YWZD8uxPVvdh8NlzLRd1VYRDs= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= +github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/util/machine/mac_address_darwin.go b/pkg/util/machine/mac_address_darwin.go new file mode 100644 index 0000000..e802add --- /dev/null +++ b/pkg/util/machine/mac_address_darwin.go @@ -0,0 +1,127 @@ +//go:build darwin && !ios + +package machine + +import ( + "bytes" + "errors" + "fmt" + "net" + "unsafe" + + "github.com/ebitengine/purego" +) + +func interfaceMacAddress(networkInterface net.Interface) (string, error) { + return macAddressWithHardwareLookup(networkInterface, hardwareMacAddress) +} + +func macAddressWithHardwareLookup(networkInterface net.Interface, lookup func(string) (net.HardwareAddr, error)) (string, error) { + // macOS 27 can redact network interface addresses with a shared placeholder. + // Resolve the same interface through IOKit to preserve its existing identity. + if !bytes.Equal(networkInterface.HardwareAddr, []byte{0x02, 0, 0, 0, 0, 0}) { + return networkInterface.HardwareAddr.String(), nil + } + + address, err := lookup(networkInterface.Name) + if err != nil { + return "", fmt.Errorf("macOS redacted the mac address for %q; failed to read its hardware address: %w", networkInterface.Name, err) + } + + if len(address) != 6 || address[0]&1 != 0 || + bytes.Equal(address, []byte{0, 0, 0, 0, 0, 0}) || + bytes.Equal(address, []byte{0x02, 0, 0, 0, 0, 0}) { + return "", fmt.Errorf("macOS redacted the mac address for %q; IOKit returned no usable hardware address", networkInterface.Name) + } + + return address.String(), nil +} + +type macAddressAPI struct { + matching func(uint32, uint32, string) uintptr + service func(uint32, uintptr) uint32 + searchProperty func(uint32, string, uintptr, uintptr, uint32) uintptr + releaseObject func(uint32) int32 + createString func(uintptr, string, uint32) uintptr + release func(uintptr) + typeID func(uintptr) uintptr + dataTypeID func() uintptr + dataLength func(uintptr) int64 + dataBytes func(uintptr) unsafe.Pointer +} + +func hardwareMacAddress(name string) (net.HardwareAddr, error) { + iokit, err := purego.Dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", purego.RTLD_NOW|purego.RTLD_LOCAL) + if err != nil { + return nil, fmt.Errorf("load IOKit: %w", err) + } + + defer func() { _ = purego.Dlclose(iokit) }() + + coreFoundation, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_LOCAL) + if err != nil { + return nil, fmt.Errorf("load CoreFoundation: %w", err) + } + + defer func() { _ = purego.Dlclose(coreFoundation) }() + + var api macAddressAPI + + purego.RegisterLibFunc(&api.matching, iokit, "IOBSDNameMatching") + purego.RegisterLibFunc(&api.service, iokit, "IOServiceGetMatchingService") + purego.RegisterLibFunc(&api.searchProperty, iokit, "IORegistryEntrySearchCFProperty") + purego.RegisterLibFunc(&api.releaseObject, iokit, "IOObjectRelease") + purego.RegisterLibFunc(&api.createString, coreFoundation, "CFStringCreateWithCString") + purego.RegisterLibFunc(&api.release, coreFoundation, "CFRelease") + purego.RegisterLibFunc(&api.typeID, coreFoundation, "CFGetTypeID") + purego.RegisterLibFunc(&api.dataTypeID, coreFoundation, "CFDataGetTypeID") + purego.RegisterLibFunc(&api.dataLength, coreFoundation, "CFDataGetLength") + purego.RegisterLibFunc(&api.dataBytes, coreFoundation, "CFDataGetBytePtr") + + return readHardwareMacAddress(name, api) +} + +func readHardwareMacAddress(name string, api macAddressAPI) (net.HardwareAddr, error) { + matching := api.matching(0, 0, name) + if matching == 0 { + return nil, errors.New("could not create interface matching dictionary") + } + + // IOServiceGetMatchingService consumes the matching dictionary. + service := api.service(0, matching) + if service == 0 { + return nil, errors.New("network interface was not found in IOKit") + } + defer api.releaseObject(service) + + const utf8Encoding = 0x08000100 + + key := api.createString(0, "IOMACAddress", utf8Encoding) + if key == 0 { + return nil, errors.New("could not create hardware address property key") + } + defer api.release(key) + + // Apple's receipt-validation guidance reads IOMACAddress from the interface + // or its parents in the IOService plane. + // https://developer.apple.com/documentation/appstorereceipts/validating-receipts-on-the-device + const iterateRecursivelyAndParents = 0x01 | 0x02 + + property := api.searchProperty(service, "IOService", key, 0, iterateRecursivelyAndParents) + if property == 0 { + return nil, errors.New("hardware address property was not found in IOKit") + } + defer api.release(property) + + if api.typeID(property) != api.dataTypeID() || api.dataLength(property) != 6 { + return nil, errors.New("IOKit hardware address must contain six bytes") + } + + data := api.dataBytes(property) + if data == nil { + return nil, errors.New("IOKit hardware address data is missing") + } + + // Copy the bytes before releasing the CoreFoundation property. + return append(net.HardwareAddr(nil), unsafe.Slice((*byte)(data), 6)...), nil +} diff --git a/pkg/util/machine/mac_address_darwin_test.go b/pkg/util/machine/mac_address_darwin_test.go new file mode 100644 index 0000000..054f04c --- /dev/null +++ b/pkg/util/machine/mac_address_darwin_test.go @@ -0,0 +1,237 @@ +//go:build darwin && !ios + +package machine + +import ( + "errors" + "net" + "unsafe" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("macOS hardware address", func() { + var networkInterface net.Interface + + BeforeEach(func() { + networkInterface = net.Interface{ + Name: "en4", + HardwareAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 0}, + } + }) + + It("preserves an existing unredacted identity without consulting IOKit", func() { + networkInterface.HardwareAddr = net.HardwareAddr{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff} + + address, err := macAddressWithHardwareLookup(networkInterface, func(string) (net.HardwareAddr, error) { + Fail("unexpected IOKit lookup") + + return nil, nil + }) + + Expect(err).ToNot(HaveOccurred()) + Expect(address).To(Equal("aa:bb:cc:dd:ee:ff")) + }) + + It("recovers the hardware address of the selected interface when redacted", func() { + address, err := macAddressWithHardwareLookup(networkInterface, func(name string) (net.HardwareAddr, error) { + Expect(name).To(Equal("en4")) + + return net.HardwareAddr{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, nil + }) + + Expect(err).ToNot(HaveOccurred()) + Expect(address).To(Equal("aa:bb:cc:dd:ee:ff")) + }) + + It("reports unavailable hardware information without returning the placeholder", func() { + lookupErr := errors.New("interface unavailable") + address, err := macAddressWithHardwareLookup(networkInterface, func(string) (net.HardwareAddr, error) { + return nil, lookupErr + }) + + Expect(address).To(BeEmpty()) + Expect(errors.Is(err, lookupErr)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("macOS redacted the mac address for \"en4\"")) + }) + + DescribeTable("rejects an unusable IOKit address", func(hardware net.HardwareAddr) { + address, err := macAddressWithHardwareLookup(networkInterface, func(string) (net.HardwareAddr, error) { + return hardware, nil + }) + + Expect(address).To(BeEmpty()) + Expect(err).To(MatchError(ContainSubstring("IOKit returned no usable hardware address"))) + }, + Entry("missing", nil), + Entry("wrong length", net.HardwareAddr{0xaa, 0xbb}), + Entry("zero", net.HardwareAddr{0, 0, 0, 0, 0, 0}), + Entry("redacted", net.HardwareAddr{0x02, 0, 0, 0, 0, 0}), + Entry("broadcast", net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}), + Entry("multicast", net.HardwareAddr{0x01, 0, 0x5e, 0, 0, 1}), + ) +}) + +var _ = Describe("IOKit hardware address lookup", func() { + const ( + dictionary = uintptr(1) + service = uint32(2) + key = uintptr(3) + property = uintptr(4) + dataType = uintptr(5) + ) + + var ( + api macAddressAPI + data []byte + released []uintptr + releasedObjects []uint32 + ) + + BeforeEach(func() { + data = []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff} + released = nil + releasedObjects = nil + api = macAddressAPI{ + matching: func(port, options uint32, name string) uintptr { + Expect(port).To(BeZero()) + Expect(options).To(BeZero()) + Expect(name).To(Equal("en4")) + + return dictionary + }, + service: func(port uint32, matching uintptr) uint32 { + Expect(port).To(BeZero()) + Expect(matching).To(Equal(dictionary)) + + return service + }, + searchProperty: func(object uint32, plane string, propertyKey, allocator uintptr, options uint32) uintptr { + Expect(object).To(Equal(service)) + Expect(plane).To(Equal("IOService")) + Expect(propertyKey).To(Equal(key)) + Expect(allocator).To(BeZero()) + Expect(options).To(Equal(uint32(3))) + + return property + }, + releaseObject: func(object uint32) int32 { + releasedObjects = append(releasedObjects, object) + + return 0 + }, + createString: func(allocator uintptr, value string, encoding uint32) uintptr { + Expect(allocator).To(BeZero()) + Expect(value).To(Equal("IOMACAddress")) + Expect(encoding).To(Equal(uint32(0x08000100))) + + return key + }, + release: func(object uintptr) { + released = append(released, object) + if object == property { + clear(data) + } + }, + typeID: func(object uintptr) uintptr { + Expect(object).To(Equal(property)) + + return dataType + }, + dataTypeID: func() uintptr { return dataType }, + dataLength: func(object uintptr) int64 { + Expect(object).To(Equal(property)) + + return int64(len(data)) + }, + dataBytes: func(object uintptr) unsafe.Pointer { + Expect(object).To(Equal(property)) + + return unsafe.Pointer(&data[0]) + }, + } + }) + + It("copies the address before releasing the property, key, and interface", func() { + address, err := readHardwareMacAddress("en4", api) + + Expect(err).ToNot(HaveOccurred()) + Expect(address.String()).To(Equal("aa:bb:cc:dd:ee:ff")) + Expect(released).To(Equal([]uintptr{property, key})) + Expect(releasedObjects).To(Equal([]uint32{service})) + }) + + It("handles a missing matching dictionary", func() { + api.matching = func(uint32, uint32, string) uintptr { return 0 } + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("could not create interface matching dictionary")) + Expect(released).To(BeEmpty()) + Expect(releasedObjects).To(BeEmpty()) + }) + + It("handles a missing interface without releasing the consumed dictionary twice", func() { + api.service = func(uint32, uintptr) uint32 { return 0 } + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("network interface was not found in IOKit")) + Expect(released).To(BeEmpty()) + Expect(releasedObjects).To(BeEmpty()) + }) + + It("releases the interface when the property key cannot be created", func() { + api.createString = func(uintptr, string, uint32) uintptr { return 0 } + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("could not create hardware address property key")) + Expect(released).To(BeEmpty()) + Expect(releasedObjects).To(Equal([]uint32{service})) + }) + + It("releases the interface and key when the property is missing", func() { + api.searchProperty = func(uint32, string, uintptr, uintptr, uint32) uintptr { return 0 } + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("hardware address property was not found in IOKit")) + Expect(released).To(Equal([]uintptr{key})) + Expect(releasedObjects).To(Equal([]uint32{service})) + }) + + It("rejects a non-data property without accessing its bytes", func() { + api.typeID = func(uintptr) uintptr { return dataType + 1 } + api.dataLength = nil + api.dataBytes = nil + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("IOKit hardware address must contain six bytes")) + Expect(released).To(Equal([]uintptr{property, key})) + Expect(releasedObjects).To(Equal([]uint32{service})) + }) + + It("rejects an unexpected length without accessing the bytes", func() { + api.dataLength = func(uintptr) int64 { return 8 } + api.dataBytes = nil + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("IOKit hardware address must contain six bytes")) + Expect(released).To(Equal([]uintptr{property, key})) + Expect(releasedObjects).To(Equal([]uint32{service})) + }) + + It("handles missing data without dereferencing a null pointer", func() { + api.dataBytes = func(uintptr) unsafe.Pointer { return nil } + + _, err := readHardwareMacAddress("en4", api) + + Expect(err).To(MatchError("IOKit hardware address data is missing")) + Expect(released).To(Equal([]uintptr{property, key})) + Expect(releasedObjects).To(Equal([]uint32{service})) + }) +}) diff --git a/pkg/util/machine/mac_address_default.go b/pkg/util/machine/mac_address_default.go new file mode 100644 index 0000000..1a5b1bf --- /dev/null +++ b/pkg/util/machine/mac_address_default.go @@ -0,0 +1,9 @@ +//go:build !darwin || ios + +package machine + +import "net" + +func interfaceMacAddress(networkInterface net.Interface) (string, error) { + return networkInterface.HardwareAddr.String(), nil +} diff --git a/pkg/util/machine/machine.go b/pkg/util/machine/machine.go index d2376e2..14796c4 100644 --- a/pkg/util/machine/machine.go +++ b/pkg/util/machine/machine.go @@ -44,7 +44,7 @@ func (*machine) MacAddress() (string, error) { for _, netInterface := range interfaces { addr := netInterface.HardwareAddr.String() if addr != "" { - return addr, nil + return interfaceMacAddress(netInterface) } } diff --git a/pkg/util/machine/machine_test.go b/pkg/util/machine/machine_test.go index 43e6e2f..887f039 100644 --- a/pkg/util/machine/machine_test.go +++ b/pkg/util/machine/machine_test.go @@ -1,6 +1,7 @@ package machine import ( + "runtime" "syscall" "testing" @@ -48,6 +49,10 @@ var _ = Describe("Machine", func() { res, err := machine.MacAddress() Expect(err).ToNot(HaveOccurred()) Expect(res).To(ContainSubstring(":")) + + if runtime.GOOS == "darwin" { + Expect(res).ToNot(Equal("02:00:00:00:00:00")) + } }) }) From 05418556280f18b5aa2f57dfa99e1e0f1d2c6827 Mon Sep 17 00:00:00 2001 From: pdx15 <301492+pdx15@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:28:24 +0000 Subject: [PATCH 15/18] fix: allow downloads of delisted tvos apps Cherry-pick of upstream 387d1a4 (adapted to the fork layout): - platform version lookup failures are now sentinel errors (errPlatformAppNotFound / errPlatformOffersNotFound) and appstore.Error gained Unwrap so errors.Is reaches them through metadata wrappers - Lookup exposes ErrAppNotFound - Download no longer aborts when a tvOS app has no catalog offer: the package is fetched anyway and its declared platform is validated - validatePackagePlatform now runs against the raw .tmp package before patching/renaming, and the .tmp file is removed on failure, so a platform mismatch leaves the previously downloaded file intact - download command falls back to the numeric app ID (with the bundle ID kept on the app) when the bundle lookup reports the app as not found - tests: new 'Downloading delisted tvOS apps' suite (adapted: the fork's redownload endpoint is hardcoded instead of coming from the bag, and the ensureSinfs guard adds a primary+redownload re-request) and a new cmd/download_test.go covering app resolution (adapted to the fork's downloadCmd + dependencies.AppStore injection) Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- cmd/download.go | 11 +- cmd/download_test.go | 104 ++++++++++++ pkg/appstore/appstore_download.go | 22 ++- pkg/appstore/appstore_download_test.go | 158 ++++++++++++++++++ pkg/appstore/appstore_lookup.go | 4 +- pkg/appstore/appstore_lookup_test.go | 2 +- .../appstore_platform_version_lookup.go | 9 +- pkg/appstore/error.go | 4 + 8 files changed, 300 insertions(+), 14 deletions(-) create mode 100644 cmd/download_test.go diff --git a/cmd/download.go b/cmd/download.go index 17dfd3a..38bb0d4 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -41,18 +41,21 @@ func downloadCmd() *cobra.Command { return err } - app := appstore.App{ID: appID} + app := appstore.App{ID: appID, BundleID: bundleID} if bundleID != "" { lookupResult, err := dependencies.AppStore.Lookup(appstore.LookupInput{ Account: acc, BundleID: bundleID, Platform: platform, }) - if err != nil { + if err != nil && (appID == 0 || !errors.Is(err, appstore.ErrAppNotFound)) { return err } - app = lookupResult.App + // Delisted apps may still be downloadable by their numeric ID. + if err == nil { + app = lookupResult.App + } } interactive, _ := cmd.Context().Value(interactiveKey).(bool) @@ -108,7 +111,7 @@ func downloadCmd() *cobra.Command { } cmd.Flags().Int64VarP(&appID, "app-id", "i", 0, "ID of the target iOS app (required)") - cmd.Flags().StringVarP(&bundleID, "bundle-identifier", "b", "", "The bundle identifier of the target iOS app (overrides the app ID)") + cmd.Flags().StringVarP(&bundleID, "bundle-identifier", "b", "", "The bundle identifier of the target iOS app (overrides the app ID when found)") cmd.Flags().StringVarP(&outputPath, "output", "o", "", "The destination path of the downloaded app package") cmd.Flags().StringVar(&externalVersionID, "external-version-id", "", "External version identifier of the target iOS app (defaults to latest version when not specified)") cmd.Flags().StringVar(&platformValue, "platform", "", "Platform to download for: iphone, ipad, or appletv") diff --git a/cmd/download_test.go b/cmd/download_test.go new file mode 100644 index 0000000..6fb6746 --- /dev/null +++ b/cmd/download_test.go @@ -0,0 +1,104 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + + "github.com/majd/ipatool/v2/pkg/appstore" + "github.com/majd/ipatool/v2/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Download command", func() { + Describe("app resolution", func() { + var store *fakeDownloadAppStore + + BeforeEach(func() { + store = &fakeDownloadAppStore{account: appstore.Account{StoreFront: "143441"}} + previousDependencies := dependencies + DeferCleanup(func() { dependencies = previousDependencies }) + dependencies.AppStore = store + dependencies.Logger = log.NewLogger(log.Args{}) + }) + + execute := func(args ...string) error { + cmd := downloadCmd() + cmd.SetArgs(args) + cmd.SetContext(context.WithValue(context.Background(), interactiveKey, false)) + + return cmd.Execute() + } + + It("uses the app ID when the bundle is absent from the catalog", func() { + store.lookupError = fmt.Errorf("lookup: %w", appstore.ErrAppNotFound) + Expect(execute("-i", "42", "-b", "com.example.delisted", "--platform", "appletv")).To(Succeed()) + Expect(store.lookupInputs).To(Equal([]appstore.LookupInput{{ + Account: appstore.Account{StoreFront: "143441"}, + BundleID: "com.example.delisted", + Platform: appstore.PlatformAppleTV, + }})) + Expect(store.downloadInputs).To(HaveLen(1)) + Expect(store.downloadInputs[0].App).To(Equal(appstore.App{ID: 42, BundleID: "com.example.delisted"})) + Expect(store.downloadInputs[0].Platform).To(Equal(appstore.PlatformAppleTV)) + }) + + It("preserves bundle identifier precedence when lookup succeeds", func() { + store.lookupOutput.App = appstore.App{ID: 43, BundleID: "com.example.listed"} + Expect(execute("-i", "42", "-b", "com.example.listed")).To(Succeed()) + Expect(store.downloadInputs).To(HaveLen(1)) + Expect(store.downloadInputs[0].App).To(Equal(store.lookupOutput.App)) + }) + + It("resolves a bundle identifier without an app ID", func() { + store.lookupOutput.App = appstore.App{ID: 43, BundleID: "com.example.listed"} + Expect(execute("-b", "com.example.listed")).To(Succeed()) + Expect(store.downloadInputs).To(HaveLen(1)) + Expect(store.downloadInputs[0].App).To(Equal(store.lookupOutput.App)) + }) + + It("does not look up the bundle for an explicit app ID and version", func() { + Expect(execute("-i", "42", "--platform", "appletv", "--external-version-id", "123456")).To(Succeed()) + Expect(store.lookupInputs).To(BeEmpty()) + Expect(store.downloadInputs).To(HaveLen(1)) + Expect(store.downloadInputs[0].App).To(Equal(appstore.App{ID: 42})) + Expect(store.downloadInputs[0].ExternalVersionID).To(Equal("123456")) + }) + + DescribeTable("preserves lookup errors", func(args []string, lookupError error) { + store.lookupError = lookupError + Expect(execute(args...)).To(MatchError(lookupError)) + Expect(store.downloadInputs).To(BeEmpty()) + }, + Entry("missing bundle without an app ID", []string{"-b", "com.example.delisted"}, appstore.ErrAppNotFound), + Entry("request failure with an app ID", []string{"-i", "42", "-b", "com.example.app"}, errors.New("request failed")), + ) + }) +}) + +type fakeDownloadAppStore struct { + appstore.AppStore + account appstore.Account + downloadInputs []appstore.DownloadInput + lookupInputs []appstore.LookupInput + lookupOutput appstore.LookupOutput + lookupError error +} + +func (s *fakeDownloadAppStore) AccountInfo() (appstore.AccountInfoOutput, error) { + return appstore.AccountInfoOutput{Account: s.account}, nil +} +func (s *fakeDownloadAppStore) Lookup(input appstore.LookupInput) (appstore.LookupOutput, error) { + s.lookupInputs = append(s.lookupInputs, input) + + return s.lookupOutput, s.lookupError +} +func (s *fakeDownloadAppStore) Download(input appstore.DownloadInput) (appstore.DownloadOutput, error) { + s.downloadInputs = append(s.downloadInputs, input) + + return appstore.DownloadOutput{DestinationPath: "/tmp/delisted-test.ipa"}, nil +} +func (*fakeDownloadAppStore) ReplicateSinf(appstore.ReplicateSinfInput) error { + return nil +} diff --git a/pkg/appstore/appstore_download.go b/pkg/appstore/appstore_download.go index 6df81dd..27e6103 100644 --- a/pkg/appstore/appstore_download.go +++ b/pkg/appstore/appstore_download.go @@ -268,7 +268,11 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { externalVersionID := input.ExternalVersionID if externalVersionID == "" && input.Platform == PlatformAppleTV { externalVersionID, err = t.lookupLatestExternalVersionID(input.Account, input.App, input.Platform) - if err != nil { + // Delisted tvOS apps may have no catalog offer but still be available + // for redownload. Validate the returned package's platform below. + missingTVOffer := input.Platform == PlatformAppleTV && + (errors.Is(err, errPlatformAppNotFound) || errors.Is(err, errPlatformOffersNotFound)) + if err != nil && !missingTVOffer { return DownloadOutput{}, fmt.Errorf("failed to resolve platform version: %w", err) } } @@ -309,6 +313,17 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { return DownloadOutput{}, fmt.Errorf("failed to download file: %w", err) } + // Validate the raw package before touching the destination, so a + // platform mismatch leaves any previously downloaded file intact. + err = t.validatePackagePlatform(tmpPath, input.Platform) + if err != nil { + if removeErr := t.os.Remove(tmpPath); removeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to remove invalid package: %w", removeErr)) + } + + return DownloadOutput{}, fmt.Errorf("failed to validate package platform: %w", err) + } + artwork, err := t.downloadArtwork(context.Background(), item.ArtworkURL) if err != nil { return DownloadOutput{}, fmt.Errorf("failed to download artwork: %w", err) @@ -319,11 +334,6 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { return DownloadOutput{}, fmt.Errorf("failed to apply patches: %w", err) } - err = t.validatePackagePlatform(destination, input.Platform) - if err != nil { - return DownloadOutput{}, fmt.Errorf("failed to validate package platform: %w", err) - } - // Read Info.plist once to extract app name and iOS version originalDestination := destination if info, readErr := t.readInfoFromIPA(destination); readErr == nil { diff --git a/pkg/appstore/appstore_download_test.go b/pkg/appstore/appstore_download_test.go index 5285f78..0b7af56 100644 --- a/pkg/appstore/appstore_download_test.go +++ b/pkg/appstore/appstore_download_test.go @@ -2,6 +2,7 @@ package appstore import ( "archive/zip" + "bytes" "errors" "fmt" "io" @@ -9,6 +10,7 @@ import ( gohttp "net/http" "net/url" "os" + "path/filepath" "strings" "time" @@ -760,3 +762,159 @@ var _ = Describe("AppStore (Download)", func() { }) }) }) + +var _ = Describe("Downloading delisted tvOS apps", func() { + var ( + platformClient *http.MockClient[platformVersionLookupResult] + downloadClient *http.MockClient[downloadResult] + httpClient *http.MockClient[interface{}] + store *appstore + input DownloadInput + ) + + missingApp := platformVersionLookupResult{} + missingOffers := platformVersionLookupResult{Results: map[string]platformVersionLookupItem{"42": {}}} + missingVersion := platformVersionLookupResult{Results: map[string]platformVersionLookupItem{ + "42": {Offers: []platformVersionLookupOffer{{}}}, + }} + + BeforeEach(func() { + ctrl := gomock.NewController(GinkgoT()) + platformClient = http.NewMockClient[platformVersionLookupResult](ctrl) + downloadClient = http.NewMockClient[downloadResult](ctrl) + httpClient = http.NewMockClient[interface{}](ctrl) + mockMachine := machine.NewMockMachine(ctrl) + mockMachine.EXPECT().MacAddress().Return("00:11:22:33:44:55", nil) + store = &appstore{ + platformClient: platformClient, + downloadClient: downloadClient, + httpClient: httpClient, + machine: mockMachine, + os: operatingsystem.New(), + } + input = DownloadInput{ + Account: Account{StoreFront: "143441"}, + App: App{ID: 42}, + Platform: PlatformAppleTV, + OutputPath: filepath.Join(GinkgoT().TempDir(), "app.ipa"), + } + }) + + DescribeTable("attempts downloads without a catalog offer and validates the package", + func(catalog platformVersionLookupResult, redownload bool, supportedPlatform string) { + previousOutput := []byte("previous output") + Expect(os.WriteFile(input.OutputPath, previousOutput, 0600)).To(Succeed()) + packageBuffer := new(bytes.Buffer) + writer := zip.NewWriter(packageBuffer) + infoWriter, err := writer.Create("Payload/Test.app/Info.plist") + Expect(err).ToNot(HaveOccurred()) + info, err := plist.Marshal(map[string]interface{}{ + "CFBundleSupportedPlatforms": []string{supportedPlatform}, + }, plist.BinaryFormat) + Expect(err).ToNot(HaveOccurred()) + _, err = infoWriter.Write(info) + Expect(err).ToNot(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + successData := downloadResult{Items: []downloadItemResult{{ + URL: "https://example.test/app.ipa", + Metadata: map[string]interface{}{"bundleShortVersionString": "1.0"}, + }}} + + previous := platformClient.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{ + StatusCode: gohttp.StatusOK, Data: catalog, + }, nil) + + // The raw descriptor carries no sinfs, so the fork's ensureSinfs + // guard re-requests the primary and redownload endpoints once. + checkRequest := func(req http.Request) { + payload := req.Payload.(*http.XMLPayload).Content + Expect(payload).To(HaveKeyWithValue("salableAdamId", input.App.ID)) + Expect(payload).ToNot(HaveKey("externalVersionId")) + Expect(payload).ToNot(HaveKey("appExtVrsId")) + } + primary := func(data downloadResult) *gomock.Call { + return downloadClient.EXPECT().Send(gomock.Any()).Do(checkRequest).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, Data: data, + }, nil) + } + redownload := func(data downloadResult) *gomock.Call { + return downloadClient.EXPECT().Send(gomock.Any()).Do(checkRequest).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, Data: data, + }, nil) + } + + var calls []*gomock.Call + if redownload { + // Empty primary responses fall through to the redownload endpoint. + calls = append(calls, primary(downloadResult{}), primary(downloadResult{}), redownload(successData)) + } else { + calls = append(calls, primary(successData)) + } + calls = append(calls, primary(successData), redownload(successData)) + gomock.InOrder(append([]*gomock.Call{previous}, calls...)) + + httpClient.EXPECT().NewRequest("GET", "https://example.test/app.ipa", nil). + Return(&gohttp.Request{Header: gohttp.Header{}}, nil) + httpClient.EXPECT().Do(gomock.Any()).Return(&gohttp.Response{ + StatusCode: gohttp.StatusOK, + Body: io.NopCloser(bytes.NewReader(packageBuffer.Bytes())), + ContentLength: int64(packageBuffer.Len()), + }, nil) + + out, err := store.Download(input) + if supportedPlatform == "AppleTVOS" { + Expect(err).ToNot(HaveOccurred()) + Expect(out.DestinationPath).To(Equal(input.OutputPath)) + Expect(store.validatePackagePlatform(out.DestinationPath, PlatformAppleTV)).To(Succeed()) + } else { + Expect(err).To(MatchError(ContainSubstring("does not declare AppleTVOS support"))) + Expect(os.ReadFile(input.OutputPath)).To(Equal(previousOutput)) + } + Expect(input.OutputPath + ".tmp").ToNot(BeAnExistingFile()) + }, + Entry("missing app", missingApp, false, "AppleTVOS"), + Entry("missing offers", missingOffers, false, "AppleTVOS"), + Entry("missing app with redownload", missingApp, true, "AppleTVOS"), + Entry("wrong platform from volumeStore", missingApp, false, "iPhoneOS"), + Entry("wrong platform from redownload", missingApp, true, "iPhoneOS"), + ) + + DescribeTable("preserves catalog failures", func(status int, catalog platformVersionLookupResult, sendErr error, message string) { + platformClient.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{ + StatusCode: status, Data: catalog, + }, sendErr) + + _, err := store.Download(input) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("network failure", 0, missingApp, errors.New("connection reset"), "connection reset"), + Entry("invalid response", 200, missingApp, errors.New("invalid JSON"), "invalid JSON"), + Entry("HTTP failure", 503, missingApp, nil, "platform version lookup request failed"), + Entry("offer without version", 200, missingVersion, nil, "no external version id"), + ) + + It("preserves license errors after a missing catalog entry", func() { + platformClient.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK}, nil) + downloadClient.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, + Data: downloadResult{FailureType: FailureTypeLicenseNotFound}, + }, nil) + + _, err := store.Download(input) + Expect(err).To(MatchError(ErrLicenseRequired)) + }) + + It("uses an explicit tvOS version without consulting the catalog", func() { + input.ExternalVersionID = "123456" + downloadClient.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("externalVersionId", input.ExternalVersionID)) + }).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, + Data: downloadResult{FailureType: FailureTypeLicenseNotFound}, + }, nil) + + _, err := store.Download(input) + Expect(err).To(MatchError(ErrLicenseRequired)) + }) +}) diff --git a/pkg/appstore/appstore_lookup.go b/pkg/appstore/appstore_lookup.go index 02570c2..19c3fa5 100644 --- a/pkg/appstore/appstore_lookup.go +++ b/pkg/appstore/appstore_lookup.go @@ -9,6 +9,8 @@ import ( "github.com/majd/ipatool/v2/pkg/http" ) +var ErrAppNotFound = errors.New("app not found") + type LookupInput struct { Account Account BundleID string @@ -41,7 +43,7 @@ func (t *appstore) Lookup(input LookupInput) (LookupOutput, error) { } if len(res.Data.Results) == 0 { - return LookupOutput{}, errors.New("app not found") + return LookupOutput{}, ErrAppNotFound } return LookupOutput{ diff --git a/pkg/appstore/appstore_lookup_test.go b/pkg/appstore/appstore_lookup_test.go index c0b4526..7c3a3da 100644 --- a/pkg/appstore/appstore_lookup_test.go +++ b/pkg/appstore/appstore_lookup_test.go @@ -49,7 +49,7 @@ var _ = Describe("AppStore (Lookup)", func() { StoreFront: "143441", }, }) - Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrAppNotFound)).To(BeTrue()) }) }) diff --git a/pkg/appstore/appstore_platform_version_lookup.go b/pkg/appstore/appstore_platform_version_lookup.go index 3631bd3..f8588c0 100644 --- a/pkg/appstore/appstore_platform_version_lookup.go +++ b/pkg/appstore/appstore_platform_version_lookup.go @@ -12,6 +12,11 @@ import ( "github.com/majd/ipatool/v2/pkg/http" ) +var ( + errPlatformAppNotFound = errors.New("platform version lookup returned no app") + errPlatformOffersNotFound = errors.New("platform version lookup returned no offers") +) + type platformVersionLookupResult struct { Results map[string]platformVersionLookupItem `json:"results,omitempty"` } @@ -98,13 +103,13 @@ func (t *appstore) lookupLatestExternalVersionID(acc Account, app App, platform item, ok := res.Data.Results[strconv.FormatInt(app.ID, 10)] if !ok { - lastErr = NewErrorWithMetadata(errors.New("platform version lookup returned no app"), res) + lastErr = NewErrorWithMetadata(errPlatformAppNotFound, res) continue } if len(item.Offers) == 0 { - lastErr = NewErrorWithMetadata(errors.New("platform version lookup returned no offers"), res) + lastErr = NewErrorWithMetadata(errPlatformOffersNotFound, res) continue } diff --git a/pkg/appstore/error.go b/pkg/appstore/error.go index f35542a..e920fef 100644 --- a/pkg/appstore/error.go +++ b/pkg/appstore/error.go @@ -9,6 +9,10 @@ func (t Error) Error() string { return t.underlyingError.Error() } +func (t Error) Unwrap() error { + return t.underlyingError +} + func NewErrorWithMetadata(err error, metadata interface{}) *Error { return &Error{ underlyingError: err, From f70dbfdc01868dabca140a17a65fd8aad4590b70 Mon Sep 17 00:00:00 2001 From: pdx15 <301492+pdx15@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:39:12 +0000 Subject: [PATCH 16/18] fix: recover downloads via the bag updateProduct endpoint Ports the endpoint-fallback mechanism that upstream introduced in 30b2909 (and extended in 747d66f for macOS and acd9e7a for tvOS), which the fork skipped when it kept its own download flow: - urlBag now parses the redownloadProduct/updateProduct endpoints and fetchURLBag returns the raw bag without the auth-specific SAP config - new appstore_download_product.go: sendUpdateProduct asks the bag's updateProduct endpoint for a pinned version (appExtVrsId) and verifies the response (single item, matching app id / version / bundle id); bag endpoints are validated against the trusted download dispatch domain - fetchDownloadItem takes the platform and, after the redownload endpoint gives up on a pinned version (empty HTTP 500 or a message-only 'no longer available' response), falls back to updateProduct for iphone, ipad, macos, tvOS and unspecified platforms - the fork's redownload URL and its primary-retry + ensureSinfs behaviour are preserved, and the bag is only fetched when the update fallback is actually needed, so existing download/check-download flows and tests are unchanged - new appstore_update_product_test.go covering the fallback: empty-500 and unavailable tvOS pins, unpinned availability responses left as-is, response mismatch and untrusted-endpoint rejection, license error propagation, and missing update endpoint Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- pkg/appstore/appstore_bag.go | 19 +- pkg/appstore/appstore_download.go | 58 ++++- pkg/appstore/appstore_download_product.go | 119 +++++++++++ pkg/appstore/appstore_update_product_test.go | 209 +++++++++++++++++++ 4 files changed, 400 insertions(+), 5 deletions(-) create mode 100644 pkg/appstore/appstore_download_product.go create mode 100644 pkg/appstore/appstore_update_product_test.go diff --git a/pkg/appstore/appstore_bag.go b/pkg/appstore/appstore_bag.go index 29108a2..d9a12bc 100644 --- a/pkg/appstore/appstore_bag.go +++ b/pkg/appstore/appstore_bag.go @@ -42,7 +42,24 @@ type bagResult struct { } type urlBag struct { - AuthEndpoint string `plist:"authenticateAccount,omitempty"` + AuthEndpoint string `plist:"authenticateAccount,omitempty"` + RedownloadEndpoint string `plist:"redownloadProduct,omitempty"` + UpdateEndpoint string `plist:"updateProduct,omitempty"` +} + +// fetchURLBag returns the raw endpoint bag. Download fallbacks only need the +// endpoints and do not require the authentication-specific SAP configuration. +func (t *appstore) fetchURLBag(guid string) (urlBag, error) { + res, err := t.bagClient.Send(t.bagRequest(guid)) + if err != nil { + return urlBag{}, fmt.Errorf("failed to send http request: %w", err) + } + + if res.StatusCode != gohttp.StatusOK { + return urlBag{}, fmt.Errorf("received unexpected status code: %d", res.StatusCode) + } + + return res.Data.URLBag, nil } func (*appstore) bagRequest(guid string) http.Request { diff --git a/pkg/appstore/appstore_download.go b/pkg/appstore/appstore_download.go index 27e6103..32c1ba4 100644 --- a/pkg/appstore/appstore_download.go +++ b/pkg/appstore/appstore_download.go @@ -81,7 +81,7 @@ func (t *appstore) CheckDownload(input CheckDownloadInput) (CheckDownloadOutput, } } - item, err := t.fetchDownloadItem(input.Account, input.App, guid, externalVersionID) + item, err := t.fetchDownloadItem(input.Account, input.App, guid, externalVersionID, input.Platform) if err != nil { return CheckDownloadOutput{}, err } @@ -109,8 +109,11 @@ func (t *appstore) CheckDownload(input CheckDownloadInput) (CheckDownloadOutput, // It is shared by Download and CheckDownload so both apply exactly the same // fallback chain: when volumeStoreDownloadProduct answers with an empty // Items[] (Chrome, Instagram, Microsoft Teams, ...) the request is retried -// once and then the redownload endpoint is consulted. -func (t *appstore) fetchDownloadItem(acc Account, app App, guid string, externalVersionID string) (downloadItemResult, error) { +// once and then the redownload endpoint is consulted. If the redownload +// endpoint cannot serve the pinned version (empty HTTP 500 or a +// "no longer available" message), the bag's updateProduct endpoint is the +// last fallback. +func (t *appstore) fetchDownloadItem(acc Account, app App, guid string, externalVersionID string, platform Platform) (downloadItemResult, error) { req := t.downloadRequest(acc, app, guid, externalVersionID) res, err := t.downloadClient.Send(req) @@ -136,6 +139,15 @@ func (t *appstore) fetchDownloadItem(acc Account, app App, guid string, external if isEmptyResponseError(err) { redownloadReq := t.redownloadRequest(acc, app, guid, externalVersionID) redownloadRes, redownloadErr := t.downloadClient.Send(redownloadReq) + + if item, updateErr, consulted := t.tryUpdateProduct(acc, app, guid, externalVersionID, platform, redownloadRes, redownloadErr); consulted { + if updateErr != nil { + return downloadItemResult{}, updateErr + } + + return t.ensureSinfs(item, req, acc, app, guid, externalVersionID), nil + } + if redownloadErr != nil { return downloadItemResult{}, fmt.Errorf("failed to send redownload request: %w", redownloadErr) } @@ -152,6 +164,44 @@ func (t *appstore) fetchDownloadItem(acc Account, app App, guid string, external return t.ensureSinfs(item, req, acc, app, guid, externalVersionID), nil } +// tryUpdateProduct consults the bag's updateProduct endpoint for the pinned +// version after the redownload endpoint failed to serve it. It reports whether +// the update endpoint was consulted; when it was, updateErr is the final +// outcome of the whole download attempt. +func (t *appstore) tryUpdateProduct(acc Account, app App, guid, externalVersionID string, platform Platform, redownloadRes http.Result[downloadResult], redownloadErr error) (downloadItemResult, error, bool) { + if externalVersionID == "" { + return downloadItemResult{}, nil, false + } + + if platform != "" && platform != PlatformIPhone && platform != PlatformIPad && + platform != PlatformMacOS && platform != PlatformAppleTV { + return downloadItemResult{}, nil, false + } + + gaveUp := isEmptyRedownloadError(redownloadErr) || + (redownloadErr == nil && isUnavailableDownloadProductResponse(redownloadRes)) + if !gaveUp { + return downloadItemResult{}, nil, false + } + + bag, err := t.fetchURLBag(guid) + if err != nil || bag.UpdateEndpoint == "" { + return downloadItemResult{}, nil, false + } + + updateRes, updateErr := t.sendUpdateProduct(bag.UpdateEndpoint, acc, app, guid, externalVersionID) + if updateErr != nil { + return downloadItemResult{}, updateErr, true + } + + item, err := classifyDownloadResponse(updateRes) + if err != nil { + return downloadItemResult{}, err, true + } + + return item, nil, true +} + // ensureSinfs guards against a valid download descriptor that carries no DRM // signatures at all (seen with com.google.GoogleMobile). Such an IPA cannot // be installed on a device, so before spending the bandwidth try to obtain a @@ -286,7 +336,7 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { } } - item, err := t.fetchDownloadItem(input.Account, input.App, guid, externalVersionID) + item, err := t.fetchDownloadItem(input.Account, input.App, guid, externalVersionID, input.Platform) if err != nil { return DownloadOutput{}, err } diff --git a/pkg/appstore/appstore_download_product.go b/pkg/appstore/appstore_download_product.go new file mode 100644 index 0000000..c850116 --- /dev/null +++ b/pkg/appstore/appstore_download_product.go @@ -0,0 +1,119 @@ +package appstore + +import ( + "errors" + "fmt" + gohttp "net/http" + "net/url" + "strings" + + "github.com/majd/ipatool/v2/pkg/http" +) + +const ( + downloadDispatchDomain = "downloaddispatch." + iTunesAPIDomain + updateProductPath = "/up/updateProduct" +) + +// isEmptyRedownloadError reports a redownload endpoint failure that carries no +// payload at all: an empty HTTP 500 body. +func isEmptyRedownloadError(err error) bool { + var unexpected *http.UnexpectedResponseError + + return errors.As(err, &unexpected) && + unexpected.StatusCode == gohttp.StatusInternalServerError && unexpected.Snippet == "" +} + +// Limit recovery to the observed availability response; other customer messages +// and structured failures must retain their normal error handling. +func isUnavailableDownloadProductResponse(res http.Result[downloadResult]) bool { + message := strings.ToLower(strings.TrimSpace(res.Data.CustomerMessage)) + + return res.StatusCode == gohttp.StatusOK && + res.Data.FailureType == "" && len(res.Data.Items) == 0 && + (message == "no longer available" || strings.HasSuffix(message, " no longer available")) +} + +// sendUpdateProduct asks the bag's updateProduct endpoint for a pinned +// version. It can serve pinned iOS, macOS, and tvOS versions when redownload +// returns an empty HTTP 500 or a message-only availability error. The request +// keeps the same session and version selection as the redownload attempt. +func (t *appstore) sendUpdateProduct(endpoint string, acc Account, app App, guid, externalVersionID string) (http.Result[downloadResult], error) { + update, err := newDownloadEndpoint(endpoint, updateProductPath) + if err != nil { + return http.Result[downloadResult]{}, err + } + + payload := map[string]interface{}{ + "creditDisplay": "", + "guid": guid, + "salableAdamId": app.ID, + "serialNumber": "0", + "appExtVrsId": externalVersionID, + } + + res, err := t.downloadClient.Send(http.Request{ + URL: fmt.Sprintf("%s?guid=%s", update.baseURL, guid), + Method: http.MethodPOST, + ResponseFormat: http.ResponseFormatXML, + Headers: map[string]string{ + "Content-Type": "application/x-apple-plist", + "iCloud-DSID": acc.DirectoryServicesID, + "X-Dsid": acc.DirectoryServicesID, + }, + Payload: &http.XMLPayload{ + Content: payload, + }, + }) + if err != nil { + return res, fmt.Errorf("failed to send update request: %w", err) + } + + if res.Data.FailureType != "" { + return res, nil + } + + if res.Data.CustomerMessage != "" { + return res, NewErrorWithMetadata(fmt.Errorf("received update error: %s", res.Data.CustomerMessage), res) + } + + if res.StatusCode != gohttp.StatusOK { + return res, fmt.Errorf("received unexpected update status code: %d", res.StatusCode) + } + + if len(res.Data.Items) != 1 { + return res, errors.New("update response must contain exactly one item") + } + + metadata := res.Data.Items[0].Metadata + if fmt.Sprint(metadata["itemId"]) != fmt.Sprint(app.ID) || + fmt.Sprint(metadata["softwareVersionExternalIdentifier"]) != externalVersionID { + return res, errors.New("update response does not match the requested app or version") + } + + bundleID, ok := metadata["softwareVersionBundleId"].(string) + if !ok || bundleID == "" || (app.BundleID != "" && bundleID != app.BundleID) { + return res, errors.New("update response does not match the requested bundle identifier") + } + + return res, nil +} + +type downloadProductEndpoint struct { + baseURL string +} + +// newDownloadEndpoint validates a bag-provided download endpoint. Only the +// trusted download dispatch domain is accepted. +func newDownloadEndpoint(endpoint, path string) (downloadProductEndpoint, error) { + parsed, err := url.ParseRequestURI(endpoint) + if err != nil || parsed.Scheme != "https" || parsed.Host != downloadDispatchDomain || + parsed.Path != path || parsed.RawPath != "" || parsed.RawQuery != "" || + parsed.ForceQuery || parsed.Fragment != "" || parsed.User != nil { + return downloadProductEndpoint{}, errors.New("invalid download endpoint in bag") + } + + return downloadProductEndpoint{ + baseURL: endpoint, + }, nil +} diff --git a/pkg/appstore/appstore_update_product_test.go b/pkg/appstore/appstore_update_product_test.go new file mode 100644 index 0000000..0cd610e --- /dev/null +++ b/pkg/appstore/appstore_update_product_test.go @@ -0,0 +1,209 @@ +package appstore + +import ( + gohttp "net/http" + + "github.com/majd/ipatool/v2/pkg/http" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" +) + +var _ = Describe("AppStore (Update Product fallback)", func() { + const ( + testGUID = "001122334455" + testVersionID = "818970197" + ) + + var ( + ctrl *gomock.Controller + mockDownload *http.MockClient[downloadResult] + mockBagClient *http.MockClient[bagResult] + store *appstore + acc Account + app App + ) + + validUpdateItem := downloadItemResult{ + URL: "https://cdn/pinned.ipa", + Sinfs: []Sinf{{ID: 0, Data: []byte("sinf")}}, + Metadata: map[string]interface{}{ + "itemId": int64(568903335), + "softwareVersionExternalIdentifier": testVersionID, + "softwareVersionBundleId": "com.example.app", + }, + } + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + mockDownload = http.NewMockClient[downloadResult](ctrl) + mockBagClient = http.NewMockClient[bagResult](ctrl) + store = &appstore{ + downloadClient: mockDownload, + bagClient: mockBagClient, + } + acc = Account{StoreFront: "143441", DirectoryServicesID: "1234"} + app = App{ID: 568903335, BundleID: "com.example.app"} + }) + + AfterEach(func() { + ctrl.Finish() + }) + + expectEmptyPrimary := func(previous *gomock.Call) *gomock.Call { + call := mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, Data: downloadResult{}, + }, nil) + + if previous != nil { + call = call.After(previous) + } + + return call + } + + expectBag := func(previous *gomock.Call, updateEndpoint string) *gomock.Call { + call := mockBagClient.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{ + StatusCode: gohttp.StatusOK, + Data: bagResult{URLBag: urlBag{UpdateEndpoint: updateEndpoint}}, + }, nil) + + return call.After(previous) + } + + expectUpdateRequest := func(previous *gomock.Call, item downloadItemResult, itemErr error) *gomock.Call { + return mockDownload.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://downloaddispatch.itunes.apple.com/up/updateProduct?guid=" + testGUID)) + + payload, ok := req.Payload.(*http.XMLPayload) + Expect(ok).To(BeTrue()) + Expect(payload.Content).To(HaveKeyWithValue("salableAdamId", int64(568903335))) + Expect(payload.Content).To(HaveKeyWithValue("appExtVrsId", testVersionID)) + }).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, Data: downloadResult{Items: []downloadItemResult{item}}, + }, itemErr).After(previous) + } + + // The fallback only kicks in after the primary endpoint came back empty + // twice (first try + retry). + expectEmptyPrimaries := func() *gomock.Call { + first := expectEmptyPrimary(nil) + + return expectEmptyPrimary(first) + } + + It("serves a pinned tvOS version via updateProduct when redownload returns an empty HTTP 500", func() { + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{}, &http.UnexpectedResponseError{ + StatusCode: gohttp.StatusInternalServerError, + }).After(previous) + previous = expectBag(previous, "https://downloaddispatch.itunes.apple.com/up/updateProduct") + expectUpdateRequest(previous, validUpdateItem, nil) + + item, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).ToNot(HaveOccurred()) + Expect(item.URL).To(Equal(validUpdateItem.URL)) + }) + + It("serves a pinned version when redownload reports it as no longer available", func() { + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, + Data: downloadResult{CustomerMessage: "No Longer Available"}, + }, nil).After(previous) + previous = expectBag(previous, "https://downloaddispatch.itunes.apple.com/up/updateProduct") + expectUpdateRequest(previous, validUpdateItem, nil) + + item, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).ToNot(HaveOccurred()) + Expect(item.URL).To(Equal(validUpdateItem.URL)) + }) + + It("preserves an unpinned tvOS availability response without updating", func() { + previous := expectEmptyPrimaries() + mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, + Data: downloadResult{CustomerMessage: "No Longer Available"}, + }, nil).After(previous) + + _, err := store.fetchDownloadItem(acc, app, testGUID, "", PlatformAppleTV) + Expect(err).To(MatchError(ContainSubstring("No Longer Available"))) + }) + + It("rejects an update response that does not match the requested version", func() { + mismatched := validUpdateItem + mismatched.Metadata = map[string]interface{}{ + "itemId": int64(568903335), + "softwareVersionExternalIdentifier": "999999999", + "softwareVersionBundleId": "com.example.app", + } + + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{}, &http.UnexpectedResponseError{ + StatusCode: gohttp.StatusInternalServerError, + }).After(previous) + previous = expectBag(previous, "https://downloaddispatch.itunes.apple.com/up/updateProduct") + expectUpdateRequest(previous, mismatched, nil) + + _, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).To(MatchError("update response does not match the requested app or version")) + }) + + It("rejects an update response with a wrong bundle identifier", func() { + mismatched := validUpdateItem + mismatched.Metadata = map[string]interface{}{ + "itemId": int64(568903335), + "softwareVersionExternalIdentifier": testVersionID, + "softwareVersionBundleId": "com.example.other", + } + + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{}, &http.UnexpectedResponseError{ + StatusCode: gohttp.StatusInternalServerError, + }).After(previous) + previous = expectBag(previous, "https://downloaddispatch.itunes.apple.com/up/updateProduct") + expectUpdateRequest(previous, mismatched, nil) + + _, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).To(MatchError("update response does not match the requested bundle identifier")) + }) + + It("rejects an update endpoint outside the download dispatch domain", func() { + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{}, &http.UnexpectedResponseError{ + StatusCode: gohttp.StatusInternalServerError, + }).After(previous) + previous = expectBag(previous, "https://evil.example.com/up/updateProduct") + + _, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).To(MatchError("invalid download endpoint in bag")) + }) + + It("propagates license errors from the update endpoint", func() { + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{}, &http.UnexpectedResponseError{ + StatusCode: gohttp.StatusInternalServerError, + }).After(previous) + previous = expectBag(previous, "https://downloaddispatch.itunes.apple.com/up/updateProduct") + mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{ + StatusCode: gohttp.StatusOK, + Data: downloadResult{FailureType: FailureTypeLicenseNotFound}, + }, nil).After(previous) + + _, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).To(MatchError(ErrLicenseRequired)) + }) + + It("keeps the redownload error when the bag has no update endpoint", func() { + previous := expectEmptyPrimaries() + previous = mockDownload.EXPECT().Send(gomock.Any()).Return(http.Result[downloadResult]{}, &http.UnexpectedResponseError{ + StatusCode: gohttp.StatusInternalServerError, + }).After(previous) + mockBagClient.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{ + StatusCode: gohttp.StatusOK, Data: bagResult{URLBag: urlBag{}}, + }, nil).After(previous) + + _, err := store.fetchDownloadItem(acc, app, testGUID, testVersionID, PlatformAppleTV) + Expect(err).To(MatchError(ContainSubstring("failed to send redownload request"))) + }) +}) From 78925bc1d58f94c649d256804860bebd5e8c552d Mon Sep 17 00:00:00 2001 From: pdx15 <301492+pdx15@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:49:24 +0000 Subject: [PATCH 17/18] fix: add missing strings import and joinCleanupError helper First CI run (unit tests) failed to compile with: - undefined: strings (appstore_search.go) - undefined: joinCleanupError x3 (appstore_download.go) The helper lives upstream in appstore_replicate_sinf.go; the fork's copy of that file predates it, so append the same implementation. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- pkg/appstore/appstore_replicate_sinf.go | 9 +++++++++ pkg/appstore/appstore_search.go | 1 + 2 files changed, 10 insertions(+) diff --git a/pkg/appstore/appstore_replicate_sinf.go b/pkg/appstore/appstore_replicate_sinf.go index afa0376..0ec2f09 100644 --- a/pkg/appstore/appstore_replicate_sinf.go +++ b/pkg/appstore/appstore_replicate_sinf.go @@ -380,3 +380,12 @@ func (*appstore) readBundleName(reader *zip.ReadCloser) (string, error) { return bundleName, nil } + +func joinCleanupError(err error, message string, cleanupErr error) error { + wrapped := fmt.Errorf("%s: %w", message, cleanupErr) + if err == nil { + return wrapped + } + + return errors.Join(err, wrapped) +} diff --git a/pkg/appstore/appstore_search.go b/pkg/appstore/appstore_search.go index 3114536..408cfd5 100644 --- a/pkg/appstore/appstore_search.go +++ b/pkg/appstore/appstore_search.go @@ -8,6 +8,7 @@ import ( "net/url" "slices" "strconv" + "strings" "github.com/majd/ipatool/v2/pkg/http" ) From 92fad5230cd7d8820766a8f6ce9c8b271822abd7 Mon Sep 17 00:00:00 2001 From: pdx15 <301492+pdx15@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:59:03 +0000 Subject: [PATCH 18/18] fix: repair test compile errors in appstore package - appstore_download_test: local mock helper 'redownload' shadowed the table's bool parameter, breaking declaration and calls; rename the helper to redownloadCall. - appstore_macos_version_lookup_test: the ported test called upstream's sendDownloadProduct method which the fork does not have; test the fork's lookupLatestMacOSExternalVersionID instead (same mocked storefront fetch, same failure entries). Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- pkg/appstore/appstore_download_test.go | 6 +++--- pkg/appstore/appstore_macos_version_lookup_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/appstore/appstore_download_test.go b/pkg/appstore/appstore_download_test.go index 0b7af56..973a75b 100644 --- a/pkg/appstore/appstore_download_test.go +++ b/pkg/appstore/appstore_download_test.go @@ -838,7 +838,7 @@ var _ = Describe("Downloading delisted tvOS apps", func() { StatusCode: gohttp.StatusOK, Data: data, }, nil) } - redownload := func(data downloadResult) *gomock.Call { + redownloadCall := func(data downloadResult) *gomock.Call { return downloadClient.EXPECT().Send(gomock.Any()).Do(checkRequest).Return(http.Result[downloadResult]{ StatusCode: gohttp.StatusOK, Data: data, }, nil) @@ -847,11 +847,11 @@ var _ = Describe("Downloading delisted tvOS apps", func() { var calls []*gomock.Call if redownload { // Empty primary responses fall through to the redownload endpoint. - calls = append(calls, primary(downloadResult{}), primary(downloadResult{}), redownload(successData)) + calls = append(calls, primary(downloadResult{}), primary(downloadResult{}), redownloadCall(successData)) } else { calls = append(calls, primary(successData)) } - calls = append(calls, primary(successData), redownload(successData)) + calls = append(calls, primary(successData), redownloadCall(successData)) gomock.InOrder(append([]*gomock.Call{previous}, calls...)) httpClient.EXPECT().NewRequest("GET", "https://example.test/app.ipa", nil). diff --git a/pkg/appstore/appstore_macos_version_lookup_test.go b/pkg/appstore/appstore_macos_version_lookup_test.go index 3d8a2ec..5452e46 100644 --- a/pkg/appstore/appstore_macos_version_lookup_test.go +++ b/pkg/appstore/appstore_macos_version_lookup_test.go @@ -44,7 +44,7 @@ var _ = Describe("Mac purchase version selection", func() { pages := http.NewMockClient[[]byte](ctrl) store := &appstore{storefrontClient: pages, downloadClient: http.NewMockClient[downloadResult](ctrl)} pages.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: status, Data: body}, requestErr) - _, _, err := store.sendDownloadProduct(Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) + _, err := store.lookupLatestMacOSExternalVersionID(Account{StoreFront: "143443-2,34"}, app) Expect(err).To(HaveOccurred()) }, Entry("HTTP failure", 500, nil, nil),