From 100a2c8fcfb5e0b7689fbfd37576c13bacd62b35 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:57:03 +0000 Subject: [PATCH 1/6] feat: add zstd request compression and ParentRequestID, sync with OpenAPI spec v2-2026-11-09T143926Z, shorten docs and comments Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- CHANGELOG.md | 14 ++ README.md | 10 +- actor.go | 85 ++++------- actor_collection.go | 7 +- actor_env_var.go | 6 +- actor_server.go | 99 +++++++------ actor_server_test.go | 49 +++++++ actor_version.go | 10 +- base.go | 230 +++++++++-------------------- build.go | 12 +- client.go | 66 +++------ common.go | 68 +++------ dataset.go | 61 +++----- dataset_collection.go | 7 +- docs/server_actors.md | 1 + docs/storages.md | 2 +- errors.go | 14 +- go.mod | 5 +- go.sum | 2 + http_client.go | 267 +++++++++++++--------------------- iterator.go | 55 +++---- key_value_store.go | 63 +++----- key_value_store_collection.go | 7 +- log.go | 23 +-- request_queue.go | 35 ++--- request_queue_collection.go | 7 +- run.go | 55 +++---- run_collection.go | 7 +- run_dataset_items.go | 20 +-- schedule.go | 5 +- server_http.go | 9 +- signature.go | 16 +- store_collection.go | 7 +- streamed_log.go | 30 ++-- task.go | 45 ++---- task_collection.go | 7 +- url_normalization.go | 12 +- version.go | 16 +- wait_for_resources.go | 25 ++-- webhook.go | 7 +- webhook_dispatch.go | 7 +- 41 files changed, 563 insertions(+), 910 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de9dccb..46cc8a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to the Apify Go client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.16.0] - 2026-11-10 + +### Added + +- `RequestCompressionZstd` for `WithRequestCompression` and `ActorServerRequestOptions.Compression`. +- `ActorServerRequestOptions.ParentRequestID`, sent as the `X-Actor-Request-ID` header, so that a + Server Actor calling another one passes its caller's pricing tier on. +- `zstd` in the documented `Content-Encoding` values of `SetRecordRawWithEncoding`. + +### Changed + +- Shortened and corrected the documentation and code comments. +- Bumped `APISpecVersion` to `v2-2026-11-09T143926Z` and `ClientVersion` to `0.16.0`. + ## [0.15.0] - 2026-11-06 ### Added diff --git a/README.md b/README.md index a9162aa..1be9c77 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ func main() { | `WithTimeoutMedium` | `30s` | Duration of the "medium" tier — listing, batch and trigger operations (`List`, `Create`, `Start`, `BatchAddRequests`). | | `WithTimeoutLong` | `360s` | Duration of the "long" tier — downloads, uploads and streaming (`ListItems`, `SetRecord`, `Log().Get()`). | | `WithTimeoutMax` | `360s` | Caps the timeout of any single request attempt; a tier configured above it is capped too. `WithTimeout` is an alias for this option, kept for callers of the single-timeout API this client had before tiers. | -| `WithRequestCompression` | `RequestCompressionAuto` | Codec for request-body compression: `RequestCompressionAuto` (brotli, falling back to gzip), `RequestCompressionBrotli`, or `RequestCompressionGzip`. See [Request compression](#request-compression). | +| `WithRequestCompression` | `RequestCompressionAuto` | Codec for request-body compression: `RequestCompressionAuto` (brotli, falling back to gzip), `RequestCompressionBrotli`, `RequestCompressionGzip`, or `RequestCompressionZstd`. See [Request compression](#request-compression). | | `WithUserAgentSuffix` | — | Suffix appended to the `User-Agent` header. | | `WithHTTPBackend` | `DefaultHTTPBackend` | Replaceable HTTP transport. | @@ -156,9 +156,9 @@ request is not aborted client-side while the server is correctly honoring it. Request bodies of 1 KiB or more are compressed before sending, to save upload bandwidth. By default (`RequestCompressionAuto`) the client prefers brotli and falls back to gzip when brotli -errors or does not shrink the body; `WithRequestCompression(RequestCompressionBrotli)` or -`WithRequestCompression(RequestCompressionGzip)` pins it to one codec, with no fallback to the -other (a body that codec does not shrink is still sent uncompressed rather than failing). +errors or does not shrink the body; `WithRequestCompression` with `RequestCompressionBrotli`, `RequestCompressionGzip` or +`RequestCompressionZstd` pins it to one codec, with no fallback (a body that codec does not +shrink is sent uncompressed rather than failing). Compression is also skipped outright for a body whose content type already carries its own compression — images, audio, video, and common archive/office formats (e.g. `.zip`, `.docx`); a raw format that happens to share such a media type, like `image/bmp`, is still compressed. @@ -292,7 +292,7 @@ func main() { - `apify.ClientVersion` — the semantic version of this library. - `apify.APISpecVersion` — the Apify OpenAPI spec version this client was built against - (`v2-2026-11-04T151257Z`). + (`v2-2026-11-09T143926Z`). ### Releasing diff --git a/actor.go b/actor.go index 9afc9f9..804b589 100644 --- a/actor.go +++ b/actor.go @@ -30,20 +30,17 @@ type ActorStartOptions struct { RestartOnError *bool // ForcePermissionLevel overrides the Actor's permission level for this run. ForcePermissionLevel *string - // Webhooks are ad-hoc webhooks to attach to this run. They are serialized to - // base64-encoded JSON as the `webhooks` query parameter, matching the reference clients. + // Webhooks are ad-hoc webhooks to attach to this run. They are serialized to base64-encoded + // JSON as the `webhooks` query parameter. Webhooks []any - // WaitForResources, if set, retries the start every 10 seconds while the API rejects it for - // lack of resources (an *APIError of type "actor-memory-limit-exceeded" or - // "concurrent-runs-limit-exceeded"; both clear as other runs or builds of the account - // finish). Nil (the default) does not retry: the first error is returned as-is. Use - // WaitForResourcesIndefinitely to retry with no time bound, or a number of seconds to stop - // retrying after that long and return the last error. Any other error is returned - // immediately, without retrying. + // WaitForResources retries the start every 10 seconds while the API rejects it for lack of + // resources (an *APIError of type "actor-memory-limit-exceeded" or + // "concurrent-runs-limit-exceeded"). Nil does not retry. Use WaitForResourcesIndefinitely to + // retry without a time bound, or a number of seconds after which the last error is returned. + // Any other error is returned at once. // - // A run that requests more memory than the account's whole memory limit is rejected with - // actor-memory-limit-exceeded as well and never starts, so WaitForResourcesIndefinitely - // retries it forever. + // A run that requests more memory than the account's limit is rejected with + // actor-memory-limit-exceeded and never starts, so WaitForResourcesIndefinitely retries it forever. WaitForResources *int64 // Log configures the redirection of the run's log that [ActorClient.Call] and // [ActorClient.CallRaw] do while they wait (see [StreamedLog]). Nil redirects it to os.Stderr @@ -146,13 +143,9 @@ func (c *ActorClient) Start(ctx context.Context, input any, options ActorStartOp return c.startWithBody(ctx, body, contentTypeJSON, options, c.startTimeout(options)) } -// StartRaw starts the Actor with a raw request body instead of a JSON-serializable value. -// -// Use this for a non-JSON input, e.g. a ZIP archive paired with -// options.ContentType = Ptr("application/zip"); the bytes are sent exactly as given, with no -// JSON serialization. For an object or array input, use [ActorClient.Start] instead, which -// handles the serialization. Mirrors the reference client, whose Actor input accepts a plain -// object, an array of them, or raw bytes. +// StartRaw starts the Actor with a raw request body, for a non-JSON input such as a ZIP archive +// with options.ContentType set to "application/zip". For an object or array input, use +// [ActorClient.Start]. func (c *ActorClient) StartRaw(ctx context.Context, input []byte, options ActorStartOptions) (ActorRun, error) { return c.startWithBody(ctx, input, "application/octet-stream", options, c.startTimeout(options)) } @@ -163,12 +156,9 @@ func (c *ActorClient) startTimeout(options ActorStartOptions) time.Duration { return extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) } -// startWithBody is the shared implementation of Start and StartRaw: it applies the run-start -// options as query parameters and posts the given body, falling back to defaultContentType only -// when the caller did not set options.ContentType. timeout bounds each start request. -// -// A start request that times out is not retried: it may already have created the run, and a -// retry would then start a second, billed one. +// startWithBody starts the Actor with body, which has defaultContentType unless +// options.ContentType is set. A start request that times out is not retried, since it may have +// created a billed run already. func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions, timeout time.Duration) (ActorRun, error) { params := NewQueryParams() options.apply(params) @@ -181,17 +171,13 @@ func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultCon }) } -// Call starts the Actor and waits (client-side polling) for it to finish. +// Call starts the Actor and waits for it to finish. waitSecs bounds the wait, and nil waits +// indefinitely. It returns the finished run, or the running run if the wait ran out. // -// waitSecs bounds the wait; nil waits indefinitely. It returns the finished run (or the -// still-running run if the wait budget was exhausted). +// While it waits, it redirects the run's log to os.Stderr or options.Log.Output, prefixed with the +// Actor name and the run ID (see [StreamedLog]). Set options.Log.Disabled to turn that off. // -// While it waits, it redirects the run's log to os.Stderr (or to options.Log.Output), prefixed -// with the Actor name and the run ID, like the reference client's call(): see [StreamedLog], and -// set options.Log.Disabled to turn it off. -// -// Neither the start request nor the polls have a client-side timeout of their own (matching the -// reference client's call(), whose default timeout is noTimeout): ctx bounds the whole call. +// The start request and the polls have no timeout of their own, so ctx bounds the whole call. func (c *ActorClient) Call(ctx context.Context, input any, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { body, err := marshalInput(input) if err != nil { @@ -233,10 +219,8 @@ func (c *ActorClient) Build(ctx context.Context, versionNumber string, options A return postWithBody[Build](ctx, c.ctx.withoutTimeoutRetry(), "builds", params, nil, contentTypeJSON, timeout) } -// DefaultBuild resolves the Actor's default build and returns a client for it. -// -// waitForFinish optionally bounds how long (in seconds) the API waits for the build to -// finish before responding, matching the reference client's defaultBuild(options). +// DefaultBuild resolves the default build of the Actor and returns a client for it. waitForFinish +// is how many seconds the API may wait for the build to finish before responding. func (c *ActorClient) DefaultBuild(ctx context.Context, waitForFinish *int64) (*BuildClient, error) { if c.ctx.idErr != nil { return nil, c.ctx.idErr @@ -256,24 +240,16 @@ func (c *ActorClient) DefaultBuild(ctx context.Context, waitForFinish *int64) (* return newBuildClient(c.ctx.http, c.baseURL, build.ID), nil } -// ValidateInput validates the given input against the Actor's input schema. -// -// It omits the build parameter, so the API validates against the input schema of the build -// tagged "latest". To validate against a specific build, use [ActorClient.ValidateInputForBuild]. -// -// On success it returns the raw JSON validation result from the API (a JSON object reporting -// whether the input is valid and, if not, the schema violations). +// ValidateInput validates input against the input schema of the build tagged "latest" and returns +// the validation result of the API. To validate against another build, use +// [ActorClient.ValidateInputForBuild]. func (c *ActorClient) ValidateInput(ctx context.Context, input any) (json.RawMessage, error) { return c.ValidateInputForBuild(ctx, input, "") } -// ValidateInputForBuild validates the given input against the input schema of a specific -// Actor build, identified by its tag or number (e.g. "latest", "0.1.2"). An empty build -// omits the parameter, so the API validates against the build tagged "latest" (per the -// OpenAPI specification), equivalent to [ActorClient.ValidateInput]. -// -// On success it returns the raw JSON validation result from the API (a JSON object reporting -// whether the input is valid and, if not, the schema violations). +// ValidateInputForBuild validates input against the input schema of a build, given by its tag or +// number such as "latest" or "0.1.2", and returns the validation result of the API. An empty +// build is the same as [ActorClient.ValidateInput]. func (c *ActorClient) ValidateInputForBuild(ctx context.Context, input any, build string) (json.RawMessage, error) { if c.ctx.idErr != nil { return nil, c.ctx.idErr @@ -302,9 +278,8 @@ func (c *ActorClient) LastRun(status string) *RunClient { return c.LastRunWithOptions(LastRunOptions{Status: status}) } -// LastRunWithOptions returns a client for the last run of this Actor, optionally filtered by -// status and/or origin. See LastRunOptions. Mirrors the reference client's -// lastRun({ status, origin }). +// LastRunWithOptions returns a client for the last run of this Actor, optionally filtered by status +// and/or origin. See LastRunOptions. func (c *ActorClient) LastRunWithOptions(options LastRunOptions) *RunClient { client := newRunClient(c.root, c.ctx.http, c.ctx.subURL(""), "runs", "last") client.setLastRunParams(options) diff --git a/actor_collection.go b/actor_collection.go index e14c9c9..57683c1 100644 --- a/actor_collection.go +++ b/actor_collection.go @@ -40,10 +40,9 @@ func (c *ActorCollectionClient) List(ctx context.Context, options ActorListOptio return listResource[Actor](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the Actors matching the options, fetching pages on -// demand. The options' Limit caps the total number of Actors yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the Actors matching the options, fetching pages on demand. +// The options' Limit caps the total number of Actors yielded (unset means all); the per-page size +// is chunkSize (nil for the server default). func (c *ActorCollectionClient) Iterate(options ActorListOptions, chunkSize *int64) *ListIterator[Actor] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Actor], error) { opts := options diff --git a/actor_env_var.go b/actor_env_var.go index ab7e376..41c6d73 100644 --- a/actor_env_var.go +++ b/actor_env_var.go @@ -17,10 +17,8 @@ func (c *ActorEnvVarCollectionClient) List(ctx context.Context) (PaginationList[ return listResource[ActorEnvVar](ctx, c.ctx, "", NewQueryParams()) } -// Iterate returns a lazy iterator over the version's environment variables. The env-vars -// endpoint is not offset-paginated (it always returns the full set in one response), so there -// is no Limit/chunkSize control; the iterator drains that one page. Mirrors the reference -// client's envVars().list(), which likewise exposes no pagination options. +// Iterate returns an iterator over the version's environment variables. The endpoint returns all +// of them in one response, so there is no Limit or chunkSize. func (c *ActorEnvVarCollectionClient) Iterate() *ListIterator[ActorEnvVar] { return newUnpaginatedIterator(c.List) } diff --git a/actor_server.go b/actor_server.go index e3c4bc9..3513cc0 100644 --- a/actor_server.go +++ b/actor_server.go @@ -14,15 +14,15 @@ import ( "sync" ) +// parentRequestIDHeader carries the ID of the request that a Server Actor is handling. +const parentRequestIDHeader = "X-Actor-Request-ID" + // ErrNotServerActor is returned, wrapped, when an [ActorServerClient] method is used on an Actor // that is not a Server Actor, which is an Actor without a server URL (`standbyUrl`). var ErrNotServerActor = errors.New("the Actor is not a Server Actor") -// ActorServerClient is a client for the server interface of a specific Actor. -// -// A Server Actor (formerly Actor Standby) runs a web server that is called over HTTP, and the -// platform keeps a run ready to answer, so each call skips the run startup. The client sends -// requests to that server and gives its URL and OpenAPI specification. Get an instance with +// ActorServerClient calls the server of a Server Actor (formerly Actor Standby), a web server +// whose run the platform keeps ready, so a call skips the run startup. Get one with // [ActorClient.Server]. It is safe for concurrent use. type ActorServerClient struct { ctx *resourceContext @@ -74,6 +74,11 @@ type ActorServerRequestOptions struct { // unless Headers sets one. A [json.RawMessage] and any other value are sent as JSON, with // Content-Type application/json unless Headers sets one. Body any + // ParentRequestID is the X-Actor-Request-ID header of the request that the calling Server + // Actor is handling. It is sent on as the same header, so that the called Actor charges at the + // pricing tier of that request, and it wins over the same header in Headers. Use a client + // created with the run's APIFY_TOKEN. + ParentRequestID string // Compression compresses a body of 1 KiB or more with the chosen codec. It is nil by // default, which sends the body uncompressed, since the Actor's server may not decode a // compressed body. Set it only if the server decodes the resulting Content-Encoding. A @@ -107,15 +112,12 @@ type ActorServerStream struct { Body io.ReadCloser } -// GetURL returns the URL of the Actor's server. -// -// The URL is the Actor's `standbyUrl`, fetched from the API by the first call of this method, -// [ActorServerClient.GetOpenAPISpec] or [ActorServerClient.Request], and reused by later calls -// on this client. Use it to call the server with another HTTP client, for example one generated -// from the server's OpenAPI specification. +// GetURL returns the URL of the Actor's server, which is its `standbyUrl`. The first call of this +// method, [ActorServerClient.GetOpenAPISpec] or [ActorServerClient.Request] fetches it, and later +// calls reuse it. Use it to call the server with another HTTP client. // -// It returns an *[APIError] for which IsNotFound is true when the Actor does not exist, and an -// error wrapping [ErrNotServerActor] when the Actor is not a Server Actor. +// It returns an *[APIError] with IsNotFound true if the Actor does not exist, and an error +// wrapping [ErrNotServerActor] if it is not a Server Actor. func (c *ActorServerClient) GetURL(ctx context.Context) (string, error) { actor, err := c.getServerActor(ctx) if err != nil { @@ -209,16 +211,10 @@ func isOlderBuild(buildNumber string, target [3]int) bool { } // GetOpenAPISpec returns the OpenAPI specification of the Actor's server, or nil if the Actor's -// definition does not include one. +// definition has none. It is the web server schema of the build that serves the server: the +// `actorStandby.build` of the Actor, or `latest`. // -// The specification is the web server schema from the Actor's definition, taken from the build -// that serves the server interface: the build set in `actorStandby.build`, or `latest` if none -// is set. Use it to generate an HTTP client for the server, and point that client at the URL -// returned by [ActorServerClient.GetURL]. -// -// It returns an *[APIError] for which IsNotFound is true when the Actor does not exist, and an -// error wrapping [ErrNotServerActor] when the Actor is not a Server Actor. It also fails when -// the build that serves the server interface does not exist. +// It returns the errors of [ActorServerClient.GetURL], and an error if that build does not exist. func (c *ActorServerClient) GetOpenAPISpec(ctx context.Context) (json.RawMessage, error) { actor, err := c.fetchSpecActor(ctx) if err != nil { @@ -305,31 +301,22 @@ var httpMethods = map[string]bool{ http.MethodTrace: true, http.MethodPatch: true, } -// Request sends an HTTP request to the Actor's server and returns its response, with the body -// read in full. +// Request sends an HTTP request to the Actor's server and reads the whole response. // -// The request goes to the URL returned by [ActorServerClient.GetURL] and carries the client's -// token in the X-Apify-Authorization header, which the platform consumes, so it never reaches -// the Actor's code. Headers in the options, Authorization included, reach the Actor's server as -// they are. The first request to an idle Actor waits for the platform to start a server run, -// which is why the request uses the "long" timeout tier. As for every method, the context -// can only shorten that bound. +// path is the path on the server, for example "/search", and a fragment is dropped. It is resolved +// against the server URL, so the request cannot go to another host. Pass the query in Params. // -// The response is returned whatever its status, so check StatusCode before using the body. A -// rate limit (429) or Service Unavailable (503) response is retried with the client's retry -// settings, and the last one is returned once the retries run out. The Actor's code can return -// these statuses too, and then it gets the request again. A redirect is returned as it is, -// since following it would send the token to a host the Actor's code picked. A transport error, -// a timeout included, is not retried, since the Actor may have already handled the request. +// The request carries the token in the X-Apify-Authorization header, which the platform removes +// before the Actor sees it. The Headers of the options reach the server as they are. // -// path is the path on the Actor's server, for example "/search". It is always resolved against -// the server URL, so the request, and the token with it, cannot go to another host. Pass the -// query in the options' Params. A fragment (#...) is dropped, since it is never sent to the -// server. +// The response is returned whatever its status, so check StatusCode. A 429 or 503 response is +// retried, and the last one is returned when the retries run out. A redirect is returned as it +// is, so the token never reaches another host. A transport error, a timeout included, is not +// retried, since the Actor may have handled the request. The first request to an idle Actor waits +// for a run to start, so the request uses the "long" timeout tier, which ctx can shorten. // -// It returns an error only when it gets no response: the Actor does not exist (an *[APIError] -// for which IsNotFound is true), the Actor is not a Server Actor ([ErrNotServerActor]), the -// options are invalid, or the request fails or times out. To read the body incrementally, use +// It returns an error only when there is no response: the errors of [ActorServerClient.GetURL], +// invalid options, or a failed or timed-out request. To read the body incrementally, use // [ActorServerClient.RequestStream]. func (c *ActorServerClient) Request(ctx context.Context, path string, options ActorServerRequestOptions) (*ActorServerResponse, error) { resp, err := c.send(ctx, path, options, false) @@ -373,18 +360,35 @@ func (c *ActorServerClient) send(ctx context.Context, path string, options Actor if err != nil { return nil, err } + headers := options.Headers + if options.ParentRequestID != "" { + headers = withHeader(headers, parentRequestIDHeader, options.ParentRequestID) + } return c.ctx.http.callServer(ctx, serverCall{ method: method, url: serverRequestURL(serverURL, path, options.Params), body: body, contentType: contentType, - headers: options.Headers, + headers: headers, compressors: compressors, timeout: c.ctx.longTimeout(), stream: stream, }) } +// withHeader returns a copy of headers with name set to value, replacing the header of that +// name whatever the casing of either. +func withHeader(headers http.Header, name, value string) http.Header { + result := http.Header{} + for key, values := range headers { + if !strings.EqualFold(key, name) { + result[key] = values + } + } + result.Set(name, value) + return result +} + // serverRequestURL joins the server URL, the path and the query. Leading slashes of the path are // dropped and a fragment is cut, so whatever the path looks like (//host, @host, an absolute URL) // it stays a path on the server. @@ -419,12 +423,7 @@ func encodeServerBody(body any) ([]byte, string, error) { } } -// Server returns a client for the server interface of this Actor. -// -// A Server Actor runs a web server, its server interface, that is called over HTTP without -// starting a run for each call. Server Actors were formerly called Actor Standby, and -// [ActorClient.Standby] is a deprecated alias of this method. Every call returns the same -// client, so the server URL it fetches is reused. +// Server returns the [ActorServerClient] of this Actor. Every call returns the same client. func (c *ActorClient) Server() *ActorServerClient { c.serverOnce.Do(func() { c.server = newActorServerClient(c.ctx.http, c.baseURL, c.id) diff --git a/actor_server_test.go b/actor_server_test.go index 7a3e324..9ca522a 100644 --- a/actor_server_test.go +++ b/actor_server_test.go @@ -15,6 +15,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/klauspost/compress/zstd" ) const ( @@ -209,6 +211,34 @@ func TestActorServerRequestSendsMethodPathQueryBodyAndToken(t *testing.T) { } } +func TestActorServerRequestSendsTheParentRequestIDOverTheSameHeader(t *testing.T) { + f := newServerFixture(t) + _, err := f.client().Actor(serverTestActorID).Server().Request(context.Background(), "/", ActorServerRequestOptions{ + Headers: http.Header{"x-actor-request-id": {"stale"}, "X-Other": {"kept"}}, + ParentRequestID: "parent-id", + }) + if err != nil { + t.Fatal(err) + } + r := f.requests(serverTestPrefix + "/")[0] + if got := r.header.Values("X-Actor-Request-ID"); len(got) != 1 || got[0] != "parent-id" { + t.Fatalf("X-Actor-Request-ID: %v", got) + } + if r.header.Get("X-Other") != "kept" { + t.Fatalf("other headers must stay: %v", r.header) + } +} + +func TestActorServerRequestSendsNoRequestIDWithoutAParentRequestID(t *testing.T) { + f := newServerFixture(t) + if _, err := f.client().Actor(serverTestActorID).Server().Request(context.Background(), "/", ActorServerRequestOptions{}); err != nil { + t.Fatal(err) + } + if got := f.requests(serverTestPrefix + "/")[0].header.Get("X-Actor-Request-ID"); got != "" { + t.Fatalf("X-Actor-Request-ID: %q", got) + } +} + func TestActorServerRequestSendsTheCallersAuthorizationAsItIs(t *testing.T) { f := newServerFixture(t) headers := http.Header{"Authorization": {"Bearer actor-token"}} @@ -419,6 +449,25 @@ func TestActorServerRequestCompression(t *testing.T) { t.Fatalf("decompressed body: %q", plain[:20]) } }) + t.Run("zstd on request", func(t *testing.T) { + f := newServerFixture(t) + _, err := f.client().Actor(serverTestActorID).Server().Request(context.Background(), "/items", ActorServerRequestOptions{ + Method: "POST", Body: payload, Compression: Ptr(RequestCompressionZstd), + }) + if err != nil { + t.Fatal(err) + } + r := f.requests(serverTestPrefix + "/")[0] + zr, err := zstd.NewReader(bytes.NewReader(r.body)) + if err != nil || r.header.Get("Content-Encoding") != "zstd" { + t.Fatalf("expected a zstd body, got encoding %q: %v", r.header.Get("Content-Encoding"), err) + } + defer zr.Close() + plain, _ := io.ReadAll(zr) + if !bytes.Contains(plain, []byte(`"items"`)) { + t.Fatalf("decompressed body: %q", plain) + } + }) t.Run("a Content-Type of the caller that is already compressed is not compressed again", func(t *testing.T) { f := newServerFixture(t) body := bytes.Repeat([]byte("x"), 5000) diff --git a/actor_version.go b/actor_version.go index c53344b..ac01cf2 100644 --- a/actor_version.go +++ b/actor_version.go @@ -12,16 +12,14 @@ func newActorVersionCollectionClient(hc *httpClient, actorURL string) *ActorVers return &ActorVersionCollectionClient{ctx: newCollectionContext(hc, actorURL, "versions")} } -// List lists the Actor's versions. The endpoint takes no pagination parameters (per the OpenAPI -// specification, it always returns every version in one response), so neither does this method. +// List lists the Actor's versions. The endpoint returns all of them in one response, so it takes no +// pagination parameters. func (c *ActorVersionCollectionClient) List(ctx context.Context) (PaginationList[ActorVersion], error) { return listResource[ActorVersion](ctx, c.ctx, "", NewQueryParams()) } -// Iterate returns a lazy iterator over the Actor's versions. The versions endpoint is not -// offset-paginated (it always returns every version in one response), so there is no -// Limit/chunkSize control; the iterator drains that one page. Mirrors the reference client's -// versions().list(), which likewise exposes no pagination options. +// Iterate returns an iterator over the Actor's versions. The endpoint returns all of them in one +// response, so there is no Limit or chunkSize. func (c *ActorVersionCollectionClient) Iterate() *ListIterator[ActorVersion] { return newUnpaginatedIterator(c.List) } diff --git a/base.go b/base.go index 88429f6..f669738 100644 --- a/base.go +++ b/base.go @@ -15,45 +15,37 @@ const ( contentTypeJSONCharset = "application/json; charset=utf-8" ) -// How long to wait between polls while waiting for a run/build to finish, and the -// server-side waitForFinish chunk size (the API caps server waiting at 60 seconds). +// Polling constants of [waitForFinish]. const ( + // waitForFinishPollInterval is the pause between polls. waitForFinishPollInterval = 250 * time.Millisecond - waitForFinishRequestSecs = int64(60) - // maxWaitForFinishSecs is the finite upper bound used when the caller asks to wait - // "indefinitely" (waitSecs == nil). The API will not accept "Infinity". 999999 seconds is - // more than 11 days, effectively indefinite while still guaranteeing termination. Mirrors - // the reference client's MAX_WAIT_FOR_FINISH. + // waitForFinishRequestSecs is the longest the API holds one waitForFinish request. + waitForFinishRequestSecs = int64(60) + // maxWaitForFinishSecs stands in for "wait indefinitely", which the API cannot express. maxWaitForFinishSecs = int64(999999) - // maxWaitForMissingJob bounds how long waitForFinish keeps polling a run/build that answers - // 404, to ride out database-replica lag, before giving up with a not-found error instead of - // continuing for the full wait budget above. A successful poll resets this window. Mirrors - // the reference client's MAX_WAIT_WHEN_JOB_NOT_EXIST_MILLIS. + // maxWaitForMissingJob is how long a run or build may keep answering 404 before the wait + // gives up. It covers database replica lag; a successful poll resets it. maxWaitForMissingJob = 3 * time.Second ) -// resourceContext is the resolved context for a resource client: its base URL and the -// shared HTTP client. The free functions in this file implement the CRUD primitives once, -// so each resource client stays small and consistent (DRY). +// resourceContext is the URL and shared HTTP client of a resource client. The functions in this +// file are the request primitives that every resource client builds on. type resourceContext struct { http *httpClient - // url is the fully-qualified base URL of the resource, - // e.g. https://api.apify.com/v2/actors/ID. + // url is the full URL of the resource, for example https://api.apify.com/v2/actors/ID. url string - // baseParams are parameters inherited from a parent resource (e.g. status on last_run). + // baseParams are query parameters inherited from a parent resource, such as status on last_run. baseParams *QueryParams - // apiOrigin is the origin (scheme + host) the API is reached through. + // apiOrigin is the scheme and host the API is reached through. apiOrigin string - // publicOrigin is the origin used to build public, shareable URLs (defaults to apiOrigin). + // publicOrigin is the scheme and host of public URLs. It defaults to apiOrigin. publicOrigin string - // hasOwnID is true when this resource is addressed by its own id (constructed via - // newSingleContext). It is false for a resource reached through a fixed sub-path with no id - // of its own (e.g. a run's default dataset, via newCollectionContext): there, a 404 cannot - // be told apart from the parent being missing, so it must not be swallowed as "absent". + // hasOwnID is true when the resource is addressed by its own ID. A resource reached through a + // fixed sub-path, such as a run's default dataset, has none: its 404 cannot be told apart + // from a missing parent, so it is not reported as "absent". hasOwnID bool - // idErr, if non-nil, is a validation error discovered while building this resource's URL - // (an empty or path-traversing id). It is returned by the first request this context - // attempts, instead of silently sending a malformed request. + // idErr is the validation error of the ID (empty, or a dot segment). The first request returns + // it instead of sending a malformed request. idErr error } @@ -64,10 +56,8 @@ func newCollectionContext(hc *httpClient, baseURL, resourcePath string) *resourc // newSingleContext creates a context for a single resource: {base}/{resourcePath}/{safeID}. // -// id is validated and percent-encoded (see safePathSegment); an invalid id (empty, or a dot -// segment) does not fail synchronously here, because every resource getter in this package -// returns a client struct directly rather than an error. Instead, the resource context -// remembers the error and every CRUD call this client later attempts fails fast with it. +// The getters of resource clients return no error, so an invalid id is kept in idErr and +// returned by the first request. func newSingleContext(hc *httpClient, baseURL, resourcePath, id string) *resourceContext { safeID, err := toSafeID(id) return newResourceContext(hc, baseURL+"/"+resourcePath+"/"+safeID, baseURL, true, err) @@ -94,13 +84,13 @@ func (c *resourceContext) withoutTimeoutRetry() *resourceContext { return &clone } -// withPublicOrigin overrides the origin used when building public URLs. +// withPublicOrigin sets the origin of public URLs. func (c *resourceContext) withPublicOrigin(publicBaseURL string) *resourceContext { c.publicOrigin = originOf(publicBaseURL) return c } -// subURL returns this resource's URL with an optional extra path segment appended. +// subURL returns the resource URL with subPath appended. func (c *resourceContext) subURL(subPath string) string { if subPath == "" { return c.url @@ -108,8 +98,7 @@ func (c *resourceContext) subURL(subPath string) string { return c.url + "/" + subPath } -// publicURL builds the public (shareable) form of this resource's URL with an optional -// extra path segment, swapping the API origin for the configured public origin. +// publicURL returns subURL with the API origin replaced by the public origin. func (c *resourceContext) publicURL(subPath string) string { apiURL := c.subURL(subPath) if c.publicOrigin == c.apiOrigin { @@ -121,21 +110,20 @@ func (c *resourceContext) publicURL(subPath string) string { return apiURL } -// shortTimeout, mediumTimeout and longTimeout return this resource's configured timeout tiers -// (see [timeoutTiers]), read through to the shared httpClient so every resource client picks up -// a [WithTimeoutShort]/[WithTimeoutMedium]/[WithTimeoutLong] override. +// shortTimeout, mediumTimeout and longTimeout return the configured timeout tiers (see +// [timeoutTiers]). func (c *resourceContext) shortTimeout() time.Duration { return c.http.tiers.short } func (c *resourceContext) mediumTimeout() time.Duration { return c.http.tiers.medium } func (c *resourceContext) longTimeout() time.Duration { return c.http.tiers.long } -// mergedParams merges the inherited base params with per-call params. +// mergedParams returns the inherited params extended by params. func (c *resourceContext) mergedParams(params *QueryParams) *QueryParams { merged := c.baseParams.clone() merged.extend(params) return merged } -// originOf extracts the origin (scheme://host[:port]) from a URL, dropping any path. +// originOf returns scheme://host[:port] of a URL. func originOf(rawURL string) string { rest := rawURL scheme := "" @@ -149,20 +137,14 @@ func originOf(rawURL string) string { return scheme + rest } -// getResource performs a GET addressing this client's own resource (subPath "" for the -// resource itself, or a fixed sub-path such as a singleton the resource owns) and unwraps the -// data envelope. A not-found maps to (zero, false, nil) only when the client has its own id -// (c.hasOwnID): there, the 404 unambiguously means this resource is gone. A client reached -// through a fixed sub-path with no id of its own (e.g. a run's default dataset) cannot tell its -// own 404 apart from the parent's, so the error always propagates instead. Mirrors the -// reference client's catchNotFoundForResourceOrThrow. For a 404 that is unambiguous regardless -// of how the client was reached (e.g. a lookup by key or request id), use [getResourceAlways]. +// getResource performs a GET of the resource, or of a fixed sub-path of it, and unwraps the data +// envelope. A 404 gives (zero, false, nil) only if the client has its own ID, since otherwise it +// may mean that the parent is missing. For a 404 that is unambiguous, use [getResourceAlways]. func getResource[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (T, bool, error) { return getResourceWithTimeout[T](ctx, c, subPath, params, c.shortTimeout()) } -// getResourceWithTimeout is [getResource] with the caller's chosen timeout tier, for a lookup -// that is slower than a plain metadata read (see [timeoutTiers]). +// getResourceWithTimeout is [getResource] with the given timeout. func getResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams, timeout time.Duration) (T, bool, error) { var zero T result, err := getResourceRequired[T](ctx, c, subPath, params, timeout) @@ -175,15 +157,11 @@ func getResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subP return result, true, nil } -// maxWaitForFinishHoldSecs caps how much extra time a waitForFinish request parameter can add -// to a request's client-side timeout, matching the reference client's -// MAX_WAIT_FOR_FINISH_HOLD_SECS (the server itself never holds a single request open longer). +// maxWaitForFinishHoldSecs is the longest the server holds a request open for waitForFinish. const maxWaitForFinishHoldSecs = 60 -// extendForServerWait extends base by the time the server may legitimately hold the connection -// open to honor a waitForFinish request parameter (capped at maxWaitForFinishHoldSecs), so the -// client does not abort a request the server is still correctly working on. waitForFinishSecs -// nil or <= 0 returns base unchanged. Mirrors the reference client's timeoutForWaitForFinish. +// extendForServerWait adds the time the server holds a request open for waitForFinishSecs to +// base, so the client does not abort a request that is working as asked. func extendForServerWait(base time.Duration, waitForFinishSecs *int64) time.Duration { if waitForFinishSecs == nil || *waitForFinishSecs <= 0 { return base @@ -192,16 +170,13 @@ func extendForServerWait(base time.Duration, waitForFinishSecs *int64) time.Dura return base + time.Duration(hold)*time.Second } -// getResourceWithServerWait is [getResource] with its timeout extended by -// [extendForServerWait], for a get(waitForFinish) call that asks the server to hold the -// response (RunClient.GetWithWait, BuildClient.GetWithWait). +// getResourceWithServerWait is [getResource] with the timeout extended by [extendForServerWait]. func getResourceWithServerWait[T any](ctx context.Context, c *resourceContext, params *QueryParams, waitForFinishSecs *int64) (T, bool, error) { return getResourceWithTimeout[T](ctx, c, "", params, extendForServerWait(c.shortTimeout(), waitForFinishSecs)) } -// getResourceAlways performs a GET that unwraps the data envelope; a not-found always maps to -// (zero, false, nil), regardless of c.hasOwnID. Used for a lookup that is unambiguous no matter -// how the parent client was reached, such as a request-queue request by id. +// getResourceAlways is [getResource], but a 404 always gives (zero, false, nil). Use it for a +// lookup whose 404 is unambiguous, such as a request-queue request by ID. func getResourceAlways[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (T, bool, error) { var zero T result, err := getResourceRequired[T](ctx, c, subPath, params, c.shortTimeout()) @@ -214,13 +189,8 @@ func getResourceAlways[T any](ctx context.Context, c *resourceContext, subPath s return result, true, nil } -// getResourceRequired performs a GET that unwraps the data envelope and propagates every error, -// including a 404. Used both where a 404 is never ambiguous (list, batch and lookup endpoints) -// and, since [DatasetClient.GetStatistics]/[ScheduleClient.GetLog]/[TaskClient.GetInput], where -// it always means the parent resource is gone. -// -// timeout is the caller's chosen timeout tier (see [timeoutTiers]), or [noRequestTimeout] for a -// request that must not be bounded by one (the polling behind [waitForFinish]). +// getResourceRequired is [getResource], but every error, a 404 included, is returned. timeout may +// be [noRequestTimeout]. func getResourceRequired[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams, timeout time.Duration) (T, error) { var zero T if c.idErr != nil { @@ -234,13 +204,12 @@ func getResourceRequired[T any](ctx context.Context, c *resourceContext, subPath return parseDataEnvelope[T](resp.body) } -// updateResource performs a PUT with a JSON body, unwrapping the data envelope. Metadata writes -// use the "short" timeout tier. +// updateResource performs a PUT with a JSON body and unwraps the data envelope. func updateResource[T any](ctx context.Context, c *resourceContext, subPath string, body any) (T, error) { return updateResourceWithTimeout[T](ctx, c, subPath, body, c.shortTimeout()) } -// updateResourceWithTimeout is [updateResource] with the caller's chosen timeout tier. +// updateResourceWithTimeout is [updateResource] with the given timeout. func updateResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subPath string, body any, timeout time.Duration) (T, error) { var zero T if c.idErr != nil { @@ -258,11 +227,8 @@ func updateResourceWithTimeout[T any](ctx context.Context, c *resourceContext, s return parseDataEnvelope[T](resp.body) } -// deleteResource performs a DELETE addressing this client's own resource. A not-found is -// treated as a successful no-op only when the client has its own id (c.hasOwnID); otherwise the -// error propagates. See [getResource] for why. For a delete that is unambiguous regardless of -// how the parent client was reached (e.g. deleting a key-value-store record by key), use -// [deleteResourceAlways]. +// deleteResource performs a DELETE of the resource. A 404 is a no-op only if the client has its +// own ID (see [getResource]). For an unambiguous 404, use [deleteResourceAlways]. func deleteResource(ctx context.Context, c *resourceContext, subPath string) error { if c.idErr != nil { return c.idErr @@ -275,8 +241,7 @@ func deleteResource(ctx context.Context, c *resourceContext, subPath string) err return nil } -// deleteResourceAlways performs a DELETE; a not-found is always treated as a successful no-op, -// regardless of c.hasOwnID. +// deleteResourceAlways is [deleteResource], but a 404 is always a no-op. func deleteResourceAlways(ctx context.Context, c *resourceContext, subPath string) error { if c.idErr != nil { return c.idErr @@ -289,15 +254,12 @@ func deleteResourceAlways(ctx context.Context, c *resourceContext, subPath strin return nil } -// listResource performs a GET returning a paginated list (data envelope wrapping items). Every -// collection's List uses the "medium" timeout tier. +// listResource performs a GET of a paginated list. func listResource[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (PaginationList[T], error) { return getResourceRequired[PaginationList[T]](ctx, c, subPath, params, c.mediumTimeout()) } -// createResource performs a POST with a JSON body, unwrapping the data envelope. timeout is the -// caller's chosen tier: most Create endpoints are "short", a few (e.g. ActorCollectionClient's) -// are "medium", matching the reference client per resource. +// createResource performs a POST with a JSON body and unwraps the data envelope. func createResource[T any](ctx context.Context, c *resourceContext, params *QueryParams, body any, timeout time.Duration) (T, error) { var zero T if c.idErr != nil { @@ -315,8 +277,7 @@ func createResource[T any](ctx context.Context, c *resourceContext, params *Quer return parseDataEnvelope[T](resp.body) } -// getOrCreateNamed performs a POST that gets-or-creates a named resource -// (POST {collection}?name=...), unwrapping the data envelope. Uses the "short" timeout tier. +// getOrCreateNamed performs POST {collection}?name=..., which gets or creates a named resource. func getOrCreateNamed[T any](ctx context.Context, c *resourceContext, name string) (T, error) { var zero T if c.idErr != nil { @@ -334,9 +295,7 @@ func getOrCreateNamed[T any](ctx context.Context, c *resourceContext, name strin return parseDataEnvelope[T](resp.body) } -// postWithBody performs a POST with a raw body (optional) and content type, unwrapping the -// data envelope. Used where the input is arbitrary user JSON (actor.start, run.metamorph). -// timeout is the caller's chosen tier (trigger/batch operations are typically "medium"). +// postWithBody performs a POST with a raw body and unwraps the data envelope. func postWithBody[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams, body []byte, contentType string, timeout time.Duration) (T, error) { var zero T if c.idErr != nil { @@ -350,9 +309,7 @@ func postWithBody[T any](ctx context.Context, c *resourceContext, subPath string return parseDataEnvelope[T](resp.body) } -// deleteWithBody performs a DELETE with a JSON body (used for batch request deletion), -// unwrapping the data envelope. Uses the "short" timeout tier (matching the reference client's -// batchDeleteRequests). +// deleteWithBody performs a DELETE with a JSON body and unwraps the data envelope. func deleteWithBody[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams, body any) (T, error) { var zero T if c.idErr != nil { @@ -370,11 +327,8 @@ func deleteWithBody[T any](ctx context.Context, c *resourceContext, subPath stri return parseDataEnvelope[T](resp.body) } -// getRaw performs a GET returning the raw response (no data envelope), using the "long" timeout -// tier (a data download). A not-found maps to (nil, nil), regardless of c.hasOwnID: used only -// where that is unambiguous, such as a key-value-store record lookup by key. For a fixed -// sub-path where a 404 always means the parent resource is gone (e.g. dataset statistics), use -// [getRawRequired] instead. +// getRaw performs a GET and returns the raw response. A 404 gives (nil, nil), so use it only for +// an unambiguous 404, such as a record lookup by key. Otherwise use [getRawRequired]. func getRaw(ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (*apiResponse, error) { resp, err := getRawRequired(ctx, c, subPath, params, c.longTimeout()) if err != nil { @@ -386,8 +340,7 @@ func getRaw(ctx context.Context, c *resourceContext, subPath string, params *Que return resp, nil } -// getRawRequired performs a GET returning the raw response (no data envelope) and propagates -// every error, including a 404. timeout is the caller's chosen tier. +// getRawRequired is [getRaw], but every error, a 404 included, is returned. func getRawRequired(ctx context.Context, c *resourceContext, subPath string, params *QueryParams, timeout time.Duration) (*apiResponse, error) { if c.idErr != nil { return nil, c.idErr @@ -396,9 +349,7 @@ func getRawRequired(ctx context.Context, c *resourceContext, subPath string, par return c.http.call(ctx, http.MethodGet, url, nil, "", timeout) } -// headExists performs a HEAD request, returning whether the resource exists, using the "long" -// timeout tier (the reference client's tier for a record lookup). A not-found always maps to -// (false, nil): used only for an unambiguous lookup (a key-value-store record by key). +// headExists performs a HEAD request and reports whether the resource exists. A 404 gives false. func headExists(ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (bool, error) { if c.idErr != nil { return false, c.idErr @@ -414,8 +365,7 @@ func headExists(ctx context.Context, c *resourceContext, subPath string, params return true, nil } -// putRaw performs a PUT with raw bytes and a content type (used for KVS record uploads), using -// the "long" timeout tier (a data upload). +// putRaw performs a PUT with raw bytes. func putRaw(ctx context.Context, c *resourceContext, subPath string, params *QueryParams, body []byte, contentType string) error { if c.idErr != nil { return c.idErr @@ -425,13 +375,9 @@ func putRaw(ctx context.Context, c *resourceContext, subPath string, params *Que return err } -// putRawWithEncoding is putRaw, but body is already compressed under contentEncoding (e.g. -// "gzip", "br"): it is sent as-is and Content-Encoding is set to contentEncoding verbatim, -// unconditionally bypassing the client's own request compression so the body is never -// compressed twice. Used for a key-value-store record upload whose value the caller has -// pre-compressed (see [KeyValueStoreClient.SetRecordRawWithEncoding]). Even with -// contentEncoding == "", this still differs from putRaw: a body of minCompressRequestBytes or -// larger that putRaw would brotli/gzip-compress is sent uncompressed here instead. +// putRawWithEncoding is [putRaw] for a body that is already compressed under contentEncoding. The +// body is sent as it is, and the client's own request compression is skipped, even when +// contentEncoding is empty. func putRawWithEncoding(ctx context.Context, c *resourceContext, subPath string, params *QueryParams, body []byte, contentType, contentEncoding string) error { if c.idErr != nil { return c.idErr @@ -441,20 +387,11 @@ func putRawWithEncoding(ctx context.Context, c *resourceContext, subPath string, return err } -// streamRaw opens a GET request and returns the raw response body as an io.ReadCloser without -// buffering it into memory, bypassing httpClient.call's buffered/retrying transport, along with -// the response headers (e.g. Content-Type, for a caller that needs it — see -// [KeyValueStoreClient.GetRecordStreamWithOptions]). Used where the caller wants to consume a -// (potentially large or live) response incrementally: a streamed log -// ([LogClient.StreamWithOptions]), a streamed dataset export -// ([DatasetClient.DownloadItemsStream]), or a streamed key-value-store record. -// -// Because the response is consumed incrementally it is not retried: a transient failure surfaces -// to the caller directly. The caller is responsible for closing the returned io.ReadCloser. +// streamRaw sends a GET request and returns the response body unread, with the response headers. +// The request is not retried, and the caller closes the body. // -// A non-2xx response is read to completion, closed, and classified into an *APIError. notFound -// controls how a 404 is reported: when true, it resolves to (nil, nil, nil) ("absent"); otherwise -// it is returned as an error. +// A non-2xx response is returned as an *APIError, except that a 404 gives (nil, nil, nil) when +// notFound is true. func streamRaw(ctx context.Context, c *resourceContext, subPath string, params *QueryParams, notFound bool) (io.ReadCloser, http.Header, error) { if c.idErr != nil { return nil, nil, c.idErr @@ -465,8 +402,6 @@ func streamRaw(ctx context.Context, c *resourceContext, subPath string, params * if err != nil { return nil, nil, err } - // Reuse the shared auth/User-Agent header setup so the streaming path can never drift from - // the buffered request path. c.http.applyAuthHeaders(req) resp, err := c.http.backend.Do(req) @@ -474,7 +409,6 @@ func streamRaw(ctx context.Context, c *resourceContext, subPath string, params * return nil, nil, err } if resp.StatusCode >= maxSuccessStatusCode { - // Consume and classify the error body, then close. body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() apiErr := buildAPIError(resp.StatusCode, body, 1, http.MethodGet, extractPath(url)) @@ -489,19 +423,9 @@ func streamRaw(ctx context.Context, c *resourceContext, subPath string, params * // terminalChecker reports whether a fetched resource has reached a terminal state. type terminalChecker[T any] func(*T) bool -// waitForFinish polls a GET endpoint with waitForFinish until the resource reaches a -// terminal state, the wait budget elapses, or the resource keeps returning 404 for longer than -// maxWaitForMissingJob. waitSecs nil means "wait indefinitely", which is implemented as a -// finite but very large bound (maxWaitForFinishSecs) so the loop always terminates. -// -// The overall budget is a pure time bound, evaluated independently of whether the resource is -// currently present. A just-started run/build can transiently return 404 (database-replica -// lag); like the reference client, we treat that as "not yet available" and keep polling, but -// only for up to maxWaitForMissingJob of continuous 404s — a job that keeps answering 404 for -// longer than that does not exist, and polling it for the full budget (which defaults to more -// than 11 days) would hang far longer than a caller would expect. A successful poll resets the -// 404 window. If the resource never became available, the last 404's *APIError is returned -// instead of a successful result. +// waitForFinish polls the resource until it is terminal, waitSecs elapse, or it has answered 404 +// for [maxWaitForMissingJob]. A nil waitSecs waits for [maxWaitForFinishSecs]. When the wait ends +// while the resource is missing, the last 404 error is returned. func waitForFinish[T any](ctx context.Context, c *resourceContext, waitSecs *int64, resourceName string, isTerminal terminalChecker[T]) (T, error) { var zero T @@ -519,8 +443,7 @@ func waitForFinish[T any](ctx context.Context, c *resourceContext, waitSecs *int notFoundErr error ) - // do-while: always perform at least one fetch (matching the reference client), then - // repeat only while the pure time bound has not been reached. + // Fetch at least once, even with a budget of zero. for { elapsed := time.Since(start) remaining := int64((budget - elapsed).Seconds()) @@ -529,11 +452,7 @@ func waitForFinish[T any](ctx context.Context, c *resourceContext, waitSecs *int params := NewQueryParams() params.AddInt("waitForFinish", &requestSecs) - // The poll itself runs with no client-imposed timeout (noRequestTimeout): the server - // may legitimately hold the connection open for up to waitForFinishRequestSecs while - // waiting for the job to finish, so a fixed per-request budget would abort it for doing - // exactly what was asked. The overall wait is still bounded by the pure time budget - // above. + // The server holds the request open on purpose, so a poll has no timeout of its own. res, err := getResourceRequired[T](ctx, c, "", params, noRequestTimeout) ok := err == nil if err != nil && (!c.hasOwnID || !isNotFound(err)) { @@ -547,11 +466,7 @@ func waitForFinish[T any](ctx context.Context, c *resourceContext, waitSecs *int return resource, nil } } else { - // A transient 404 is not immediately fatal: keep polling, but only for up to - // maxWaitForMissingJob of continuous 404s. present is cleared so that, if the budget - // runs out while currently 404 (even if the resource was seen earlier), the final - // check below reports not-found rather than a stale last-known state — matching the - // reference client, which resets its job to undefined on every 404. + // A 404 may be replica lag, so keep polling until it has lasted maxWaitForMissingJob. present = false if notFoundSince.IsZero() { notFoundSince = time.Now() @@ -562,7 +477,6 @@ func waitForFinish[T any](ctx context.Context, c *resourceContext, waitSecs *int } } - // Pure time bound: stop once the budget is exhausted, regardless of presence. if time.Since(start) >= budget { break } @@ -573,14 +487,11 @@ func waitForFinish[T any](ctx context.Context, c *resourceContext, waitSecs *int } if present { - // Budget exhausted but the latest poll succeeded: return its state (e.g. still running). - // Mirrors the reference client returning the last fetched job. + // The wait ran out, so this is the last state, for example still running. return resource, nil } - // The budget ran out while the resource was absent (currently 404, whether or not it was - // seen earlier): notFoundErr is always set at this point, since present can only be false - // after at least one swallowed 404. + // The wait ran out on a 404. return zero, notFoundErr } @@ -598,8 +509,7 @@ func minInt64(a, b int64) int64 { return b } -// mustMarshal serializes v to JSON, returning nil on error. Used for small, internally -// constructed bodies (maps of primitives) that cannot fail to marshal in practice. +// mustMarshal serializes v to JSON for bodies built from primitives, which cannot fail. func mustMarshal(v any) []byte { data, _ := json.Marshal(v) return data diff --git a/build.go b/build.go index e9635cd..e974399 100644 --- a/build.go +++ b/build.go @@ -28,10 +28,9 @@ func (c *BuildCollectionClient) List(ctx context.Context, options ListOptions) ( return listResource[Build](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the builds matching the options, fetching pages on -// demand. The options' Limit caps the total number of builds yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the builds matching the options, fetching pages on demand. +// The options' Limit caps the total number of builds yielded (unset means all); the per-page size +// is chunkSize (nil for the server default). func (c *BuildCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Build] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Build], error) { opts := options @@ -61,9 +60,8 @@ func (c *BuildClient) Get(ctx context.Context) (Build, bool, error) { return c.GetWithWait(ctx, nil) } -// GetWithWait fetches the build, optionally asking the API to wait up to waitForFinishSecs -// seconds (max 60) for the build to finish before responding. Pass nil for an immediate -// fetch. Mirrors the reference client's get({ waitForFinish }). +// GetWithWait fetches the build, optionally asking the API to wait up to waitForFinishSecs seconds +// (max 60) for the build to finish before responding. Pass nil for an immediate fetch. func (c *BuildClient) GetWithWait(ctx context.Context, waitForFinishSecs *int64) (Build, bool, error) { params := NewQueryParams() params.AddInt("waitForFinish", waitForFinishSecs) diff --git a/client.go b/client.go index e7fa007..6de4077 100644 --- a/client.go +++ b/client.go @@ -1,12 +1,8 @@ -// Package apify is the official, idiomatic Go client for the Apify API -// (https://docs.apify.com/api/v2). +// Package apify is the Go client of the Apify API (https://docs.apify.com/api/v2). The README +// states that the client is experimental and generated and maintained by AI. // -// See the top-level README for the AI-generated/AI-maintained disclaimer. -// -// It provides a resource-oriented interface that mirrors the official JavaScript and Rust -// clients: start from an [ApifyClient], then drill down into resources (Actors, runs, -// datasets, key-value stores, request queues, tasks, schedules, webhooks, the store, users -// and logs). +// Start from an [ApifyClient] and go on to the resource clients: Actors, runs, datasets, +// key-value stores, request queues, tasks, schedules, webhooks, the Store, users and logs. // // # Quick start // @@ -24,10 +20,10 @@ // # Architecture // // - Public interface: [ApifyClient] and the resource clients it returns. -// - Replaceable transport: the [HTTPBackend] interface, with a default -// [DefaultHTTPBackend]. Swap it via [WithHTTPBackend]. -// - Cross-cutting behaviour (auth, User-Agent, retries with exponential backoff, -// timeouts) lives in the internal HTTP client and is applied to every request. +// - Transport: the [HTTPBackend] interface, with [DefaultHTTPBackend] as the default. Replace it +// with [WithHTTPBackend]. +// - Authentication, the User-Agent header, retries and timeouts are applied to every request by +// the internal HTTP client. package apify import ( @@ -38,7 +34,7 @@ import ( "time" ) -// Default configuration values, matching the reference clients. +// Default configuration values. const ( // defaultBaseURL is the default base URL of the Apify API (without the /v2 suffix). defaultBaseURL = "https://api.apify.com" @@ -46,8 +42,7 @@ const ( defaultMaxRetries = 8 // defaultMinDelayBetweenRetries is the default minimum delay between retries. defaultMinDelayBetweenRetries = 500 * time.Millisecond - // Default durations of the timeout tiers, matching the reference client's - // DEFAULT_TIMEOUT_SHORT_SECS/MEDIUM/LONG/MAX. + // Default durations of the timeout tiers. defaultTimeoutShort = 5 * time.Second defaultTimeoutMedium = 30 * time.Second defaultTimeoutLong = 360 * time.Second @@ -117,13 +112,7 @@ func WithMinDelayBetweenRetries(d time.Duration) Option { return func(c *clientConfig) { c.minDelayBetweenRetries = d } } -// WithTimeout caps the timeout of any single request attempt (default 360s), same as -// [WithTimeoutMax]. It bounds the growth of a tier's timeout across retries, and a tier -// configured above the cap is capped too. Kept as an alias for the single-timeout option this -// client had before timeout tiers ([WithTimeoutShort]/[WithTimeoutMedium]/[WithTimeoutLong]); -// an existing caller of WithTimeout keeps every request capped at the same value as before, -// while gaining the lower tier defaults below that cap (5s/30s/360s) in place of the previous -// flat 360s for every request. +// WithTimeout is an alias of [WithTimeoutMax], kept from before timeout tiers existed. func WithTimeout(d time.Duration) Option { return WithTimeoutMax(d) } @@ -161,8 +150,8 @@ func WithUserAgentSuffix(suffix string) Option { // WithRequestCompression selects which codec compresses request bodies (default // [RequestCompressionAuto]: brotli, falling back to gzip). A caller that wants a specific -// codec, with no fallback to the other one, can pass [RequestCompressionBrotli] or -// [RequestCompressionGzip]. +// codec, with no fallback to another one, can pass [RequestCompressionBrotli], +// [RequestCompressionGzip] or [RequestCompressionZstd]. func WithRequestCompression(algo RequestCompression) Option { return func(c *clientConfig) { c.requestCompression = algo } } @@ -191,11 +180,8 @@ func defaultIsAtHome() bool { return os.Getenv(envIsAtHome) != "" } -// NewClient creates a client configured by the given functional options. -// -// Authentication is supplied via [WithToken]; without it the client can still call -// endpoints that do not require authentication. Other options ([WithBaseURL], -// [WithMaxRetries], [WithTimeout], [WithHTTPBackend], ...) tune the transport and behaviour. +// NewClient creates a client from the options. Without [WithToken] it can call only the endpoints +// that need no token. // // client := apify.NewClient(apify.WithToken("my-api-token")) func NewClient(opts ...Option) *ApifyClient { @@ -229,8 +215,8 @@ func NewClient(opts ...Option) *ApifyClient { minDelayBetweenRetries: cfg.minDelayBetweenRetries, timeoutMax: cfg.timeoutMax, }, - // A tier configured above timeoutMax is capped at it, same as the reference client, - // so WithTimeoutMax always bounds every request regardless of its tier. + // A tier configured above timeoutMax is capped at it, so WithTimeoutMax always bounds every + // request regardless of its tier. tiers: timeoutTiers{ short: minDuration(cfg.timeoutShort, cfg.timeoutMax), medium: minDuration(cfg.timeoutMedium, cfg.timeoutMax), @@ -253,13 +239,8 @@ func NewClient(opts ...Option) *ApifyClient { // it is already present there (see [toAPIBaseURL]). const apiVersionPath = "/v2" -// toAPIBaseURL appends apiVersionPath to url unless it is already the final path segment, -// matching the reference client's toApiBaseUrl. -// -// A trailing slash (or several) is trimmed first, so "https://host/v2/" and "https://host/v2//" -// both normalize to "https://host/v2" rather than growing a double slash. The check looks only -// at the URL's path, not at the string as a whole: "https://v2" has no path ("v2" is the host), -// so it still becomes "https://v2/v2" even though the raw string ends in "v2". +// toAPIBaseURL appends apiVersionPath to rawURL unless the path already ends with it. Trailing +// slashes are trimmed first. Only the path is checked, so "https://v2" becomes "https://v2/v2". func toAPIBaseURL(rawURL string) string { authority, path := splitAuthority(rawURL) path = strings.TrimRight(path, "/") @@ -430,12 +411,9 @@ func (c *ApifyClient) User(id string) *UserClient { return newUserClient(c.http, c.baseURL, id) } -// SetStatusMessage sets the status message of the current Actor run. -// -// This convenience method updates the run identified by the ACTOR_RUN_ID environment -// variable, so it only works when called from inside an Actor run. If isTerminal is true, -// the message becomes final and won't be overwritten. It returns an error if ACTOR_RUN_ID -// is not set. +// SetStatusMessage sets the status message of the current Actor run, which the ACTOR_RUN_ID +// environment variable identifies, so it works only inside an Actor run. A terminal message is +// final and is not overwritten. It returns an error if ACTOR_RUN_ID is not set. func (c *ApifyClient) SetStatusMessage(ctx context.Context, message string, isTerminal bool) (ActorRun, error) { runID := os.Getenv("ACTOR_RUN_ID") if runID == "" { diff --git a/common.go b/common.go index 8f92b4e..1db18c2 100644 --- a/common.go +++ b/common.go @@ -24,12 +24,7 @@ func parseDataEnvelope[T any](body []byte) (T, error) { const notFoundStatusCode = 404 -// isNotFound reports whether err is an API error with a 404 status, mirroring the reference -// clients' NotFoundError: a get/delete/head on a missing resource should resolve to "absent" -// rather than raise. Any 404 counts, regardless of its machine-readable `type` — the reference -// clients used to swallow only the "record-not-found"/"record-or-token-not-found" types, but -// that let an unrelated 404 (e.g. a typo'd sub-path) pass silently for "not found" too, so v3 -// dropped the type check. +// isNotFound reports whether err is an API error with status 404, whatever its type. func isNotFound(err error) bool { apiErr, ok := AsAPIError(err) if !ok { @@ -180,12 +175,8 @@ func (o StorageListOptions) apply(q *QueryParams) { // client.Actors().List(ctx, apify.ActorListOptions{My: apify.Ptr(true), Limit: apify.Ptr(int64(10))}) func Ptr[T any](v T) *T { return &v } -// PaginationList is a single page of an offset/limit-paginated list. -// -// The pagination metadata (Total, Offset, Limit, Count, Desc) accompanies the Items slice. -// Note: Total reflects the API's reported total, which can briefly lag immediately after a -// write (e.g. right after PushItems) because the count is computed asynchronously — re-read -// after a short delay if you need an exact post-write total. +// PaginationList is one page of an offset and limit paginated list. Total can lag right after a +// write, for example after PushItems, because the API counts asynchronously. type PaginationList[T any] struct { // Total is the total number of items available across all pages. Total int64 `json:"total"` @@ -193,12 +184,10 @@ type PaginationList[T any] struct { Offset int64 `json:"offset"` // Limit is the maximum number of items the API would return for this request. Limit int64 `json:"limit"` - // Count is the number of entries on this page. On a page from [DatasetClient.ListItems] it is - // the number of dataset rows the API scanned for the page (the X-Apify-Pagination-Count - // header), or the number of items returned when that is larger. Filters (Clean, SkipEmpty, - // SkipHidden) can leave it above len(Items). The next page starts at - // Offset + min(Count, Limit); the cap matters only with Unwind, which can return more items - // than the rows scanned. + // Count is the number of entries on the page. For a page of [DatasetClient.ListItems] it is the + // number of dataset rows the API scanned (the X-Apify-Pagination-Count header), or the number of + // items returned if that is larger. Filters (Clean, SkipEmpty, SkipHidden) can leave it above + // len(Items). The next page starts at Offset + min(Count, Limit). Count int64 `json:"count"` // Desc reports whether the items are in descending order. Desc bool `json:"desc"` @@ -206,16 +195,8 @@ type PaginationList[T any] struct { Items []T `json:"items"` } -// osTokenOverrides maps Go's runtime.GOOS values to the platform names the reference Apify JS -// client emits in the User-Agent OS token. That token comes from Node's os.platform(), which -// spells a few platforms differently from Go's runtime.GOOS: -// - "windows" -> "win32" -// - "solaris" -> "sunos", and Go's "illumos" (a Solaris/OpenSolaris derivative that Node/libuv -// also reports as "sunos") -> "sunos" -// - "ios" -> "darwin" (Node/libuv reports Apple platforms uniformly as "darwin") -// -// Every other GOOS value already matches os.platform() verbatim (e.g. "linux", "darwin", -// "android", "freebsd", "openbsd", "aix"), so it is passed through unchanged. +// osTokenOverrides maps the runtime.GOOS values that Node's os.platform() spells differently, +// which the reference client puts in its User-Agent. Other values are used as they are. var osTokenOverrides = map[string]string{ "windows": "win32", "solaris": "sunos", @@ -233,20 +214,14 @@ func platformToken(goos string) string { return goos } -// osToken returns the User-Agent OS token for the current platform. runtime.GOOS is the idiomatic -// Go source of the platform identifier; platformToken aligns the two names that Go spells -// differently from the reference clients. +// osToken returns the OS token of the User-Agent, which is runtime.GOOS with the names that Node +// spells differently replaced (see [osTokenOverrides]). func osToken() string { return platformToken(runtime.GOOS) } -// BuildUserAgent builds the User-Agent header value mandated by the client requirements: -// `ApifyClient/{version} ({os}; {language version}); isAtHome/{isAtHome}`. -// -// isAtHome is driven solely by the platform's APIFY_IS_AT_HOME environment variable (matching -// the requirements and the reference JS client, which reads it via @apify/consts) and is -// rendered lowercase (true/false). The {os} token uses osToken so it matches the platform names -// the reference clients emit. +// BuildUserAgent builds the User-Agent value `ApifyClient/{version} ({os}; {language version}); +// isAtHome/{isAtHome}`. isAtHome comes from the APIFY_IS_AT_HOME environment variable. func BuildUserAgent(suffix string, isAtHomeFn func() bool) string { atHome := "false" if isAtHomeFn() { @@ -264,22 +239,15 @@ func goVersion() string { return strings.TrimPrefix(runtime.Version(), "go") } -// toSafeID rewrites a resource id into the `username~resourcename` form Apify's URLs use -// (every `/` becomes `~`, matching the reference client's replaceAll) and validates the result -// as a URL path segment. See [safePathSegment] for what that rejects and why. +// toSafeID rewrites a resource ID into the `username~resourcename` form of Apify URLs, where every +// `/` becomes `~`, and validates it with [safePathSegment]. func toSafeID(id string) (string, error) { return safePathSegment(strings.ReplaceAll(id, "/", "~")) } -// safePathSegment validates value as a single URL path segment and percent-encodes it. Used -// directly to validate a value interpolated into the path below the resource's own id (a -// record key, a request id), and by [toSafeID] for the id itself. -// -// An empty string or a dot segment ("." or "..") is rejected outright rather than encoded: -// percent-encoding alone is not enough to stop a path-traversal attempt, because a URL parser -// (or an intermediate proxy) can decode and resolve dot segments after this client has built -// the request path, collapsing e.g. "records/%2E%2E" to the parent endpoint exactly like a -// literal "records/..". Mirrors the reference client's toPathSegment. +// safePathSegment percent-encodes value as one URL path segment. It rejects an empty value and +// the dot segments "." and "..", since a proxy can resolve an encoded dot segment to the parent +// path. func safePathSegment(value string) (string, error) { if value == "" { return "", errors.New("resource id must not be empty") diff --git a/dataset.go b/dataset.go index a216a96..5b819e3 100644 --- a/dataset.go +++ b/dataset.go @@ -156,12 +156,8 @@ func (c *DatasetClient) Delete(ctx context.Context) error { return deleteResource(ctx, c.ctx, "") } -// ListDatasetItems lists a single page of items from the dataset, decoding each into T -// (e.g. json.RawMessage or a struct). -// -// The dataset items endpoint returns a bare JSON array (not a data envelope) and reports -// pagination via X-Apify-Pagination-* headers, which are surfaced in the returned -// [PaginationList]. +// ListDatasetItems lists one page of the dataset items, decoding each item into T, for example a +// struct or json.RawMessage. The pagination headers fill the returned [PaginationList]. func ListDatasetItems[T any](ctx context.Context, c *DatasetClient, options DatasetListItemsOptions) (PaginationList[T], error) { var result PaginationList[T] params := NewQueryParams() @@ -183,13 +179,10 @@ func ListDatasetItems[T any](ctx context.Context, c *DatasetClient, options Data result.Total = headerInt(resp.header, "X-Apify-Pagination-Total", returned) result.Offset = headerInt(resp.header, "X-Apify-Pagination-Offset", 0) result.Limit = headerInt(resp.header, "X-Apify-Pagination-Limit", returned) - // The API applies Offset/Limit to the dataset's rows before Clean/SkipEmpty/SkipHidden/ - // Unwind reshape them, so the items returned can land on either side of the rows scanned. - // X-Apify-Pagination-Count reports the scanned number, and Count carries it so that - // IterateDatasetItems advances and stops by it: a filter can't make it re-scan rows a - // previous page already covered, and Unwind can't make it skip rows. The scanned number is - // derived from the dataset's item count, which lags a fresh push, so it can fall below the - // items returned; the larger of the two is taken. + // The API applies Offset and Limit to the rows before the filters and Unwind reshape them, so + // the items returned can differ in number from the rows scanned. Count is the scanned number, + // and the iterators advance by it. It is derived from the item count, which lags a fresh push, + // so the larger of it and the items returned is used. result.Count = maxInt64(headerInt(resp.header, "X-Apify-Pagination-Count", returned), returned) if options.Desc != nil { result.Desc = *options.Desc @@ -203,10 +196,9 @@ func (c *DatasetClient) ListItems(ctx context.Context, options DatasetListItemsO return ListDatasetItems[json.RawMessage](ctx, c, options) } -// IterateDatasetItems returns a lazy iterator over the dataset's items, decoding each into T -// and fetching pages on demand. The options' Limit caps the total number of items yielded -// (unset means all); the per-page size is chunkSize (nil for the server default). Mirrors the -// reference client's iterable listItems(). +// IterateDatasetItems returns a lazy iterator over the dataset's items, decoding each into T and +// fetching pages on demand. The options' Limit caps the total number of items yielded (unset means +// all); the per-page size is chunkSize (nil for the server default). func IterateDatasetItems[T any](c *DatasetClient, options DatasetListItemsOptions, chunkSize *int64) *ListIterator[T] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[T], error) { opts := options @@ -241,14 +233,8 @@ func (c *DatasetClient) DownloadItems(ctx context.Context, format DownloadItemsF return resp.body, nil } -// DownloadItemsStream is [DatasetClient.DownloadItems], but returns a Readable stream of the -// serialized export instead of buffering it all into memory, so a large export can be processed -// in chunks (e.g. piped straight to a file via io.Copy). The caller is responsible for closing -// the returned [io.ReadCloser]. -// -// Because the response is consumed incrementally it bypasses the buffered/retrying transport, so -// a transient failure surfaces to the caller directly instead of being retried. Mirrors the -// reference client's downloadItems(format, { stream: true }). +// DownloadItemsStream is [DatasetClient.DownloadItems], but returns the export unread, so that a +// large export can be processed in chunks. The caller closes it. The request is not retried. func (c *DatasetClient) DownloadItemsStream(ctx context.Context, format DownloadItemsFormat, options DatasetDownloadOptions) (io.ReadCloser, error) { params := NewQueryParams() fmtStr := string(format) @@ -258,14 +244,11 @@ func (c *DatasetClient) DownloadItemsStream(ctx context.Context, format Download return stream, err } -// PushItems pushes one or more items to the dataset. items must serialize to a JSON object or -// an array of objects, the only shapes the API accepts: anything else (a bare string, number or -// boolean, an array holding a non-object, nil) fails with an error before any request is sent. -// A value that is already serialized JSON can be passed as a [json.RawMessage]; a plain Go string -// is serialized as a JSON string, which is not a valid item. +// PushItems pushes items to the dataset. items must serialize to a JSON object or an array of +// objects, otherwise PushItems returns an error before sending a request. A string is serialized +// as a JSON string, so pass a [json.RawMessage] for JSON that is already serialized. // -// A push that times out is not retried: the API may already have stored the items, and a retry -// would then store them a second time. +// A push that times out is not retried, since the API may have stored the items already. func (c *DatasetClient) PushItems(ctx context.Context, items any) error { body, err := json.Marshal(items) if err != nil { @@ -317,15 +300,11 @@ func (c *DatasetClient) GetStatistics(ctx context.Context) (json.RawMessage, err return parseDataEnvelope[json.RawMessage](resp.body) } -// CreateItemsPublicURL builds a public URL for downloading this dataset's items, served as -// JSON. To request another export format, use [DatasetClient.CreateItemsPublicURLWithFormat]. +// CreateItemsPublicURL returns a public URL of the dataset items as JSON. For another format, use +// [DatasetClient.CreateItemsPublicURLWithFormat]. // -// It mirrors the reference client's createItemsPublicUrl: it fetches the dataset, and if the -// dataset exposes a URL-signing secret key (i.e. it is private), appends an HMAC-SHA256 -// signature so the URL grants access without an API token. expiresInSecs optionally bounds -// the validity of a signed URL (nil for non-expiring); it must be positive, and a zero or -// negative value is an error returned before any request is sent. The URL is built from the -// configured public base URL. +// If the dataset has a URL signing secret key, the URL is signed, so it works without a token. +// expiresInSecs is the validity of a signed URL, nil for no expiry, and it must be positive. func (c *DatasetClient) CreateItemsPublicURL(ctx context.Context, options DatasetListItemsOptions, expiresInSecs *int64) (string, error) { return c.CreateItemsPublicURLWithFormat(ctx, options, expiresInSecs, "") } @@ -345,7 +324,7 @@ func (c *DatasetClient) CreateItemsPublicURLWithFormat(ctx context.Context, opti params.AddString("format", &fmtStr) } - // Fetched with the "long" timeout tier, the reference client's tier for this call. + // The "long" timeout tier is used for this fetch. dataset, present, err := getResourceWithTimeout[Dataset](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) if err != nil { return "", err diff --git a/dataset_collection.go b/dataset_collection.go index 9cadc6a..8c78771 100644 --- a/dataset_collection.go +++ b/dataset_collection.go @@ -18,10 +18,9 @@ func (c *DatasetCollectionClient) List(ctx context.Context, options StorageListO return listResource[Dataset](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the datasets matching the options, fetching pages on -// demand. The options' Limit caps the total number of datasets yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the datasets matching the options, fetching pages on demand. +// The options' Limit caps the total number of datasets yielded (unset means all); the per-page size +// is chunkSize (nil for the server default). func (c *DatasetCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[Dataset] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Dataset], error) { opts := options diff --git a/docs/server_actors.md b/docs/server_actors.md index 911f939..29b999c 100644 --- a/docs/server_actors.md +++ b/docs/server_actors.md @@ -50,6 +50,7 @@ fmt.Println(results) | `Params` | `url.Values` | Query parameters. | | `Headers` | `http.Header` | Additional headers. They reach the Actor's server as they are and win over the headers the client sets; a header with no values removes the client's header of that name. | | `Body` | `any` | A `string` or `[]byte` is sent as it is. A `json.RawMessage` and any other value are sent as JSON with `Content-Type: application/json`, unless `Headers` sets a content type. | +| `ParentRequestID` | `string` | The `X-Actor-Request-ID` header of the request your Server Actor is handling. It is sent on as the same header, so the called Actor charges at the pricing tier of that request, and it wins over the same header in `Headers`. Use a client created with the run's `APIFY_TOKEN`. | | `Compression` | `*apify.RequestCompression` | Compresses a body of 1 KiB or more. Nil by default, which sends the body uncompressed, since the Actor's server may not decode a compressed body. Set it only if the server decodes the resulting `Content-Encoding`. A `Content-Encoding` header in `Headers` sends the body untouched. | The response has the `StatusCode`, the `Header` and the `Body` bytes, and `JSON(&v)` decodes the diff --git a/docs/storages.md b/docs/storages.md index 6022f69..00bdac0 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -175,7 +175,7 @@ Single store: `client.KeyValueStore(id)`: | `GetRecordStreamWithOptions(ctx, key, GetRecordOptions)` | Stream with options. | | `SetRecordRaw(ctx, key, value []byte, contentType string) error` | Write raw bytes, sent as they are: the client does not parse or convert the value, whatever the content type (so `application/json` bytes must already be valid JSON). | | `SetRecordRawWithOptions(ctx, key, value []byte, contentType string, SetRecordOptions) error` | Like `SetRecordRaw`, with `ContentEncoding` (the value is already compressed, as with `SetRecordRawWithEncoding`) and `DoNotRetryTimeouts` (an upload that times out fails at once instead of being retried). | -| `SetRecordRawWithEncoding(ctx, key, value []byte, contentType, contentEncoding string) error` | Write raw bytes that are already compressed (`contentEncoding` e.g. `"gzip"`, `"br"`, `"deflate"`): sent as-is under `Content-Encoding`, skipping the client's own request compression so the value is never compressed twice. | +| `SetRecordRawWithEncoding(ctx, key, value []byte, contentType, contentEncoding string) error` | Write raw bytes that are already compressed (`contentEncoding` e.g. `"gzip"`, `"br"`, `"zstd"`, `"deflate"`): sent as-is under `Content-Encoding`, skipping the client's own request compression so the value is never compressed twice. | | `SetRecordJSON(ctx, key, value any) error` | Write JSON. | | `SetRecordJSONWithOptions(ctx, key, value any, SetRecordOptions) error` | Like `SetRecordJSON`, with the per-call `SetRecordOptions`. | | `RecordExists(ctx, key) (bool, error)` | HEAD check. | diff --git a/errors.go b/errors.go index 1dbc944..5618ee9 100644 --- a/errors.go +++ b/errors.go @@ -6,12 +6,7 @@ import ( "fmt" ) -// APIError is returned for HTTP requests that reach the Apify API but receive a -// non-success status code. -// -// It mirrors the `ApifyApiError` of the reference JavaScript client and exposes the -// parsed error Type, the human-readable Message, the HTTP StatusCode, the number of -// the final Attempt, and the request HTTPMethod/Path. +// APIError is returned for a request that reaches the API and gets a non-success status. type APIError struct { // StatusCode is the HTTP status code of the error response. StatusCode int @@ -39,12 +34,7 @@ const ( statusServerErrorMin = 500 ) -// IsInvalidRequest reports whether this is an HTTP 400 Bad Request, typically because the -// request failed validation. -// -// The reference clients throw a distinct InvalidRequestError subclass for this status; since Go -// has no exception hierarchy to mirror, these Is* predicates are the idiomatic equivalent -// classification on the one APIError type. +// IsInvalidRequest reports whether the status is 400 Bad Request. func (e *APIError) IsInvalidRequest() bool { return e.StatusCode == statusInvalidRequest } // IsUnauthorized reports whether this is an HTTP 401 Unauthorized: the token is missing or diff --git a/go.mod b/go.mod index c9a8d11..68b644e 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,7 @@ go 1.23 toolchain go1.26.10 -require github.com/andybalholm/brotli v1.2.2 +require ( + github.com/andybalholm/brotli v1.2.2 + github.com/klauspost/compress v1.18.0 +) diff --git a/go.sum b/go.sum index 80d4b3a..4750299 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= diff --git a/http_client.go b/http_client.go index 9bb69cb..da6fdb2 100644 --- a/http_client.go +++ b/http_client.go @@ -13,45 +13,35 @@ import ( "time" "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" ) -// Status-code thresholds used to classify responses. Named constants so the retry -// policy has no magic numbers (see the language-agnostic coding rules). +// Status codes and limits of the retry and compression policy. const ( - // rateLimitExceededStatusCode is returned when the per-resource rate limit is hit. + // rateLimitExceededStatusCode is the status of an exceeded rate limit. rateLimitExceededStatusCode = 429 - // minServerErrorStatusCode and above are treated as retryable internal errors. + // minServerErrorStatusCode is the lowest status that is retried as a server error. minServerErrorStatusCode = 500 - // maxSuccessStatusCode: responses with status < this are treated as success. + // maxSuccessStatusCode is the lowest status that is not a success. maxSuccessStatusCode = 300 - // backoffFactor is the multiplier applied to the inter-retry delay after each - // attempt (exponential backoff). Matches the reference client's async-retry factor of 2. + // backoffFactor multiplies the delay between retries after each attempt. backoffFactor = 2 - // minCompressRequestBytes is the smallest request body worth compressing. Below this - // size the CPU cost and Content-Encoding header overhead outweigh the bandwidth saved, - // so the body is sent as-is. Matches the reference client's MIN_COMPRESS_BYTES threshold. + // minCompressRequestBytes is the smallest request body that is compressed. minCompressRequestBytes = 1024 ) -// Content-Encoding values for compressed request bodies. The reference client prefers brotli -// and falls back to gzip, so this client supports both: brotli (best ratio) is tried first and -// gzip is the fallback when brotli is unavailable or fails to shrink the payload. +// Content-Encoding values of compressed request bodies. const ( - // contentEncodingBrotli is the preferred Content-Encoding (Brotli, RFC 7932). contentEncodingBrotli = "br" - // contentEncodingGzip is the fallback Content-Encoding (gzip, RFC 1952). - contentEncodingGzip = "gzip" + contentEncodingGzip = "gzip" + contentEncodingZstd = "zstd" ) -// HTTPBackend is the replaceable transport contract of the client. +// HTTPBackend is the replaceable transport of the client. It sends one request and returns the +// raw response. Authentication, the User-Agent header, retries and decoding are done by the client. // -// Implementations are responsible only for sending a single request and returning the -// raw response. Authentication, the User-Agent header, retries and (de)serialization are -// handled by httpClient, so a backend only needs to perform one network round-trip. -// -// A non-2xx HTTP status is NOT an error at this layer — return it as a normal -// *http.Response. Only transport-level failures (connection refused, DNS, timeout) -// should be returned as an error. +// A non-2xx status is a normal *http.Response. Return an error only for a transport failure such +// as a refused connection, a DNS error or a timeout. // // A backend that follows redirects must not follow them for a request that [FollowsRedirects] // reports false for: such a request carries credentials meant for its original host only. @@ -60,71 +50,60 @@ type HTTPBackend interface { Do(req *http.Request) (*http.Response, error) } -// DefaultHTTPBackend is the default HTTPBackend implementation, backed by an -// *http.Client. The per-attempt timeout is applied via the request context by the -// orchestrating client, so this backend uses no client-level timeout of its own. +// DefaultHTTPBackend is the default [HTTPBackend], backed by an *http.Client. The client applies +// the timeout of each attempt through the request context, so the backend sets none. type DefaultHTTPBackend struct { client *http.Client } -// NewDefaultHTTPBackend creates a backend with a sensible default *http.Client that -// keeps connections alive for connection-pool reuse. +// NewDefaultHTTPBackend creates a [DefaultHTTPBackend] with a default *http.Client. func NewDefaultHTTPBackend() *DefaultHTTPBackend { return &DefaultHTTPBackend{client: &http.Client{}} } -// NewHTTPBackendWithClient wraps a caller-provided *http.Client, useful for sharing a -// connection pool or customizing proxy/TLS settings. +// NewHTTPBackendWithClient creates a [DefaultHTTPBackend] around client, for example to share a +// connection pool or to set proxy and TLS options. func NewHTTPBackendWithClient(client *http.Client) *DefaultHTTPBackend { return &DefaultHTTPBackend{client: client} } -// Do implements HTTPBackend. It follows redirects like any *http.Client, except for a request -// that must not (see [FollowsRedirects]), whose redirect response is returned as it is. +// Do implements [HTTPBackend]. It follows redirects, except for a request that [FollowsRedirects] +// reports false for, whose redirect response is returned as it is. func (b *DefaultHTTPBackend) Do(req *http.Request) (*http.Response, error) { if FollowsRedirects(req) { return b.client.Do(req) } - // A copy shares the transport and the cookie jar, so the connection pool stays shared. + // The copy shares the transport, so the connection pool is shared. client := *b.client client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } return client.Do(req) } -// retryConfig configures the retry/timeout behaviour of the httpClient. +// retryConfig is the retry policy of the httpClient. type retryConfig struct { - // maxRetries is the maximum number of retries (the request is attempted up to - // maxRetries+1 times). + // maxRetries is the number of retries, so a request is attempted up to maxRetries+1 times. maxRetries int - // minDelayBetweenRetries is doubled on each subsequent retry (exponential backoff). + // minDelayBetweenRetries is the first delay between attempts. It doubles on each retry. minDelayBetweenRetries time.Duration - // timeoutMax caps the timeout of any single request attempt. It bounds the growth of a - // tier's timeout across retries (see attemptTimeout) and the inter-retry delay alike. + // timeoutMax caps the timeout of one attempt and the delay between attempts. timeoutMax time.Duration } -// timeoutTiers holds the configurable default request timeouts. Every exported method is -// assigned the tier that fits the expected duration of its request — short for simple metadata -// reads/writes, medium for listing/batch/trigger operations, long for downloads, uploads and -// streaming — matching the reference client's timeout tiers. A caller that needs a different -// bound for one call already has full control via the context.Context every method takes (see -// the "Cancellation" section of the README): canceling it, or giving it an earlier deadline via -// context.WithTimeout, overrides whichever tier the method would otherwise use, since -// doAttempt's context.WithTimeout always resolves to the earlier of the two deadlines. +// timeoutTiers are the default request timeouts. Each method uses the tier that fits its +// request: short for a metadata read or write, medium for a list, batch or trigger call, long for +// a download, upload or stream. A context deadline can shorten the tier but not extend it. type timeoutTiers struct { short time.Duration medium time.Duration long time.Duration } -// noRequestTimeout means "no client-imposed timeout for this request" — only the caller's -// context bounds it. Used for the polling requests behind WaitForFinish/Call, which must not be -// aborted by a fixed per-request budget while legitimately waiting for a job to finish. +// noRequestTimeout leaves a request bounded by the caller's context alone. The polling behind +// WaitForFinish and Call uses it, since the server holds those requests open on purpose. const noRequestTimeout time.Duration = 0 -// httpClient is the orchestrating HTTP client shared by every resource client. It owns -// the backend, the optional API token, the User-Agent, and the retry/timeout policy, and -// applies them to every request. It is safe for concurrent use and cheap to copy. +// httpClient is the HTTP client shared by every resource client. It applies the token, the +// User-Agent, the retry policy and the timeouts to each request. It is safe for concurrent use. type httpClient struct { backend HTTPBackend token string @@ -132,63 +111,48 @@ type httpClient struct { retry retryConfig tiers timeoutTiers compressors []requestCompressor - // doNotRetryTimeouts makes a request attempt that exceeds its timeout fail at once instead of - // being retried. Set it (through [httpClient.withoutTimeoutRetry]) for a request that is not - // idempotent, where a timed-out attempt may already have taken effect on the server, so - // repeating it could create a second run or store the same dataset items twice. + // doNotRetryTimeouts fails an attempt that timed out instead of retrying it. Set it for a + // request that is not idempotent, where the attempt may have taken effect already. doNotRetryTimeouts bool } -// withoutTimeoutRetry returns a copy of c that does not retry a request attempt that timed out. -// Other retryable failures (a 429 or 5xx response, a transport error that is not a timeout) are -// still retried. +// withoutTimeoutRetry returns a copy of c that does not retry a timed-out attempt. Other +// retryable failures are still retried. func (c *httpClient) withoutTimeoutRetry() *httpClient { clone := *c clone.doNotRetryTimeouts = true return &clone } -// apiResponse is the parsed result of a single API call: the status, headers and the -// fully-buffered response body. +// apiResponse is the status, headers and body of a response. type apiResponse struct { statusCode int header http.Header body []byte } -// call sends req with authentication, the User-Agent header and the retry policy applied, -// returning the first successful response or the final error. -// -// body, if non-nil, is the raw request body; it is re-sent on each retry (we buffer it). -// contentType, if non-empty, sets the Content-Type header. baseTimeout is the per-endpoint -// base timeout; it grows with each attempt up to the client's overall timeout budget. +// call sends a request with the token, the User-Agent header and the retry policy, and returns +// the first successful response or the final error. baseTimeout grows with each attempt, up to +// the client's maximum. func (c *httpClient) call(ctx context.Context, method, url string, body []byte, contentType string, baseTimeout time.Duration) (*apiResponse, error) { return c.callWithHeaders(ctx, method, url, body, contentType, nil, baseTimeout) } -// callWithHeaders is like call but additionally sets the given extra request headers (e.g. -// the charge idempotency key). They are applied on every attempt. +// callWithHeaders is [httpClient.call] with extra request headers, set on every attempt. func (c *httpClient) callWithHeaders(ctx context.Context, method, url string, body []byte, contentType string, extraHeaders map[string]string, baseTimeout time.Duration) (*apiResponse, error) { - // Compress the body once (not per attempt): it is identical on every retry. contentEncoding - // is "" when the body was left uncompressed (too small, already compressed, or compression - // did not shrink it). + // The body is the same on every attempt, so it is compressed once. sendBody, contentEncoding := maybeCompressRequestBody(body, contentType, c.compressors) return c.doRetryLoop(ctx, method, url, sendBody, contentType, contentEncoding, extraHeaders, baseTimeout) } -// callPreEncoded is like call, but body is sent exactly as given, under contentEncoding (set as -// the Content-Encoding header when non-empty): the client's own request compression (see -// maybeCompressRequestBody) is skipped unconditionally, so a pre-compressed body (e.g. a -// pre-compressed key-value-store record) is never compressed a second time. Even with -// contentEncoding == "", this still differs from call: a body of minCompressRequestBytes or -// larger that call would brotli/gzip-compress is sent uncompressed here instead. +// callPreEncoded is [httpClient.call] for a body that is already compressed under contentEncoding. +// The body is sent as it is, and the client's own compression is skipped even when contentEncoding +// is empty. func (c *httpClient) callPreEncoded(ctx context.Context, method, url string, body []byte, contentType, contentEncoding string, baseTimeout time.Duration) (*apiResponse, error) { return c.doRetryLoop(ctx, method, url, body, contentType, contentEncoding, nil, baseTimeout) } -// doRetryLoop is the shared retry/backoff loop behind call, callWithHeaders and callPreEncoded: -// it sends sendBody (already compressed/encoded, if at all) under contentEncoding, retrying -// retryable failures with exponential backoff. +// doRetryLoop sends the request and retries retryable failures with exponential backoff. func (c *httpClient) doRetryLoop(ctx context.Context, method, url string, sendBody []byte, contentType, contentEncoding string, extraHeaders map[string]string, baseTimeout time.Duration) (*apiResponse, error) { delay := c.retry.minDelayBetweenRetries maxAttempts := c.retry.maxRetries + 1 @@ -209,7 +173,6 @@ func (c *httpClient) doRetryLoop(ctx context.Context, method, url string, sendBo retryable = isStatusRetryable(resp.statusCode) } - // Give up immediately on non-retryable errors or after the last attempt. if !retryable || attempt == maxAttempts { return nil, lastErr } @@ -222,16 +185,9 @@ func (c *httpClient) doRetryLoop(ctx context.Context, method, url string, sendBo return nil, lastErr } -// backoff sleeps before the next attempt of a retried request and doubles delay for the one -// after, matching the reference client's async-retry factor of 2. It returns false when ctx ended -// during the sleep. Two intentional and documented divergences from the reference JS client: -// - Jitter: the actual sleep is drawn uniformly from [delay, 2*delay). The reference does not -// jitter, but the Apify API docs explicitly prescribe it ("wait for a period of time chosen -// randomly from the interval DELAY to 2*DELAY milliseconds") to avoid synchronized retries -// (thundering herd), so we follow the API guidance here. -// - Cap: the base delay is capped at the overall request timeout as a safety net (the -// reference leaves it uncapped). With the default settings this cap is never reached within -// maxRetries, so it does not change observable behaviour. +// backoff sleeps before the next attempt and doubles delay. It returns false when ctx ended during +// the sleep. The sleep is random in [delay, 2*delay), as the Apify API documentation prescribes, +// and the delay is capped at the maximum request timeout. func (c *httpClient) backoff(ctx context.Context, delay *time.Duration) bool { if !sleepWithContext(ctx, randomizedDelay(*delay)) { return false @@ -240,17 +196,13 @@ func (c *httpClient) backoff(ctx context.Context, delay *time.Duration) bool { return true } -// applyAuthHeaders sets the User-Agent and (if a token is configured) Authorization headers -// that every request to the API must carry. Together with [httpClient.applyAuthHeadersAs] it is -// the single place those headers are set, so the buffered request path and the raw streaming -// path (LogClient.Stream) stay in sync. +// applyAuthHeaders sets the User-Agent header and, if a token is configured, the Authorization +// header. func (c *httpClient) applyAuthHeaders(req *http.Request) { c.applyAuthHeadersAs(req, "Authorization") } -// applyAuthHeadersAs is [httpClient.applyAuthHeaders] with the token in the header called -// tokenHeader, for the requests whose Authorization header is not the client's to use (see -// serverAuthorizationHeader). +// applyAuthHeadersAs is [httpClient.applyAuthHeaders] with the token in tokenHeader. func (c *httpClient) applyAuthHeadersAs(req *http.Request, tokenHeader string) { req.Header.Set("User-Agent", c.userAgent) if c.token != "" { @@ -258,9 +210,8 @@ func (c *httpClient) applyAuthHeadersAs(req *http.Request, tokenHeader string) { } } -// newRequest builds a request of the client: the body, the User-Agent header, the token in -// tokenHeader and the content headers. A non-empty contentEncoding says that body is already -// compressed with it. +// newRequest builds a request with the User-Agent header, the token in tokenHeader and the +// content headers. A non-empty contentEncoding says that body is compressed with it. func (c *httpClient) newRequest(ctx context.Context, method, url string, body []byte, contentType, contentEncoding, tokenHeader string) (*http.Request, error) { var reader io.Reader if body != nil { @@ -280,10 +231,7 @@ func (c *httpClient) newRequest(ctx context.Context, method, url string, body [] return req, nil } -// doAttempt performs a single HTTP round-trip with the given per-attempt timeout. -// -// contentEncoding, when non-empty, is set as the Content-Encoding header (the body is already -// compressed with that encoding by the caller). +// doAttempt performs one round-trip with the given timeout. func (c *httpClient) doAttempt(ctx context.Context, method, url string, body []byte, contentType, contentEncoding string, extraHeaders map[string]string, timeout time.Duration) (*apiResponse, error) { attemptCtx := ctx cancel := func() {} @@ -294,7 +242,7 @@ func (c *httpClient) doAttempt(ctx context.Context, method, url string, body []b req, err := c.newRequest(attemptCtx, method, url, body, contentType, contentEncoding, "Authorization") if err != nil { - // A malformed request is a programming/argument error, not a transport error. + // An invalid request is a caller error, so it is not retried. return nil, &APIError{StatusCode: 0, Type: "invalid-request", Message: err.Error(), Attempt: 1, HTTPMethod: method, Path: extractPath(url)} } for k, v := range extraHeaders { @@ -309,20 +257,16 @@ func (c *httpClient) doAttempt(ctx context.Context, method, url string, body []b data, err := io.ReadAll(resp.Body) if err != nil { - // Reading the body failed mid-stream; treat as a retryable transport error. + // A failed read is a retryable transport error. return nil, err } return &apiResponse{statusCode: resp.StatusCode, header: resp.Header, body: data}, nil } -// requestCompressor pairs a Content-Encoding value with the function that produces a body -// in that encoding. Splitting the encodings this way lets maybeCompressRequestBody try them -// in preference order and keeps each codec independently testable. +// requestCompressor is a codec: the Content-Encoding value it produces and its compress function. type requestCompressor struct { - // encoding is the Content-Encoding header value this codec produces (e.g. "br", "gzip"). encoding string - // compress returns the compressed form of body, or an error if that codec is unavailable. compress func(body []byte) ([]byte, error) } @@ -330,25 +274,27 @@ type requestCompressor struct { type RequestCompression int const ( - // RequestCompressionAuto prefers brotli (typically a better ratio) and falls back to gzip - // when brotli errors or fails to shrink the body; this is the default. Matches the - // reference client's brotli-then-gzip preference. + // RequestCompressionAuto uses brotli and falls back to gzip when brotli fails or does not + // shrink the body. It is the default. RequestCompressionAuto RequestCompression = iota - // RequestCompressionBrotli always uses brotli; a body it does not shrink is sent - // uncompressed rather than falling back to gzip. + // RequestCompressionBrotli uses brotli only. A body it does not shrink is sent uncompressed. RequestCompressionBrotli - // RequestCompressionGzip always uses gzip; a body it does not shrink is sent uncompressed. + // RequestCompressionGzip uses gzip only. A body it does not shrink is sent uncompressed. RequestCompressionGzip + // RequestCompressionZstd uses zstd at its default level only. A body it does not shrink is + // sent uncompressed. + RequestCompressionZstd ) -// compressorsFor returns the codec(s) maybeCompressRequestBody tries, in preference order, for -// the given [RequestCompression] choice. +// compressorsFor returns the codecs of algo in the order they are tried. func compressorsFor(algo RequestCompression) []requestCompressor { switch algo { case RequestCompressionBrotli: return []requestCompressor{{encoding: contentEncodingBrotli, compress: brotliCompress}} case RequestCompressionGzip: return []requestCompressor{{encoding: contentEncodingGzip, compress: gzipCompress}} + case RequestCompressionZstd: + return []requestCompressor{{encoding: contentEncodingZstd, compress: zstdCompress}} default: return []requestCompressor{ {encoding: contentEncodingBrotli, compress: brotliCompress}, @@ -357,9 +303,8 @@ func compressorsFor(algo RequestCompression) []requestCompressor { } } -// compressBody runs body through the streaming writer built by newWriter and returns the -// compressed bytes. newWriter's WriteCloser must flush all buffered output on Close. This is the -// shared body of the per-codec helpers, which differ only in the writer they construct. +// compressBody compresses body with the writer that newWriter creates. The writer must flush on +// Close. func compressBody(body []byte, newWriter func(io.Writer) io.WriteCloser) ([]byte, error) { var buf bytes.Buffer w := newWriter(&buf) @@ -382,16 +327,19 @@ func gzipCompress(body []byte) ([]byte, error) { return compressBody(body, func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }) } -// maybeCompressRequestBody compresses body when it is large enough to be worth it and its -// content type is not already compressed, returning the bytes to send and the Content-Encoding -// value ("" when the body is sent uncompressed). -// -// This mirrors the reference client, which compresses request payloads above a 1 KiB threshold -// to save upload bandwidth. compressors is the configured codec list (see -// [RequestCompression]/[WithRequestCompression]), tried in order; if a codec errors or does not -// actually shrink the body (e.g. an already-compressed payload the content type did not flag), -// the next one is tried, and if none help the original body is sent uncompressed rather than -// failing the request. +// zstdCompress compresses body with zstd (Content-Encoding: zstd). +func zstdCompress(body []byte) ([]byte, error) { + return compressBody(body, func(w io.Writer) io.WriteCloser { + // NewWriter fails only for invalid options, and none are passed. + encoder, _ := zstd.NewWriter(w) + return encoder + }) +} + +// maybeCompressRequestBody compresses body if it is at least [minCompressRequestBytes] long and +// its content type is compressible. It tries the compressors in order and uses the first that +// shrinks the body. It returns the bytes to send and the Content-Encoding value, which is empty +// when the body is sent as it is. func maybeCompressRequestBody(body []byte, contentType string, compressors []requestCompressor) ([]byte, string) { if len(body) < minCompressRequestBytes { return body, "" @@ -402,15 +350,10 @@ func maybeCompressRequestBody(body []byte, contentType string, compressors []req return compressWith(body, compressors) } -// Media-type prefixes whose payloads already carry their own compression, so running them -// through brotli/gzip burns CPU and memory for a result that is usually no smaller — and the -// request also keeps the intended Content-Type, rather than becoming an unreadable -// Content-Encoding: br blob the destination may not expect for these formats. Matches the -// reference client's ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES. +// alreadyCompressedContentTypePrefixes are the media types whose payloads are already compressed. var alreadyCompressedContentTypePrefixes = []string{"audio/", "image/", "video/"} -// Exact media types (outside the prefixes above) whose payloads already carry their own -// compression. Matches the reference client's ALREADY_COMPRESSED_MEDIA_TYPES. +// alreadyCompressedContentTypes are further media types whose payloads are already compressed. var alreadyCompressedContentTypes = map[string]bool{ "application/epub+zip": true, "application/gzip": true, @@ -433,8 +376,7 @@ var alreadyCompressedContentTypes = map[string]bool{ "font/woff2": true, } -// Uncompressed media types that sit under an already-compressed prefix above, so compressing -// them still pays off. Matches the reference client's COMPRESSIBLE_MEDIA_TYPES. +// compressibleContentTypes are uncompressed media types under an already-compressed prefix. var compressibleContentTypes = map[string]bool{ "audio/aiff": true, "audio/basic": true, @@ -454,18 +396,12 @@ var compressibleContentTypes = map[string]bool{ "image/x-ms-bmp": true, } -// Structured-syntax suffixes that mark a media type as text even under an already-compressed -// prefix (e.g. image/svg+xml). Matches the reference client's COMPRESSIBLE_MEDIA_TYPE_SUFFIXES. +// compressibleContentTypeSuffixes mark a media type as text, for example image/svg+xml. var compressibleContentTypeSuffixes = []string{"+json", "+xml"} -// isCompressibleContentType decides whether a request body with the given Content-Type is -// worth compressing. -// -// Images, audio, video and archives already carry their own compression: running them through -// brotli or gzip burns CPU, holds a second full copy of the body in memory, and usually produces -// output no smaller than the input (sometimes larger). A format that is raw despite such a media -// type, e.g. image/bmp or audio/wav, is still compressed. An empty Content-Type is assumed -// compressible. Matches the reference client's isCompressibleContentType. +// isCompressibleContentType reports whether a body of the given Content-Type is worth compressing. +// Images, audio, video and archives are not, except for raw formats such as image/bmp. An empty +// Content-Type is compressible. func isCompressibleContentType(contentType string) bool { if contentType == "" { return true @@ -504,7 +440,7 @@ func compressWith(body []byte, compressors []requestCompressor) ([]byte, string) for _, c := range compressors { out, err := c.compress(body) if err != nil { - continue // codec unavailable/failed: fall back to the next one + continue } if len(out) < len(body) { return out, c.encoding @@ -513,12 +449,8 @@ func compressWith(body []byte, compressors []requestCompressor) ([]byte, string) return body, "" } -// attemptTimeout returns min(overall, base * 2^(attempt-1)). -// -// The first attempt uses the per-endpoint base timeout; each retry doubles it so a -// slow-but-progressing connection gets more time, while never exceeding the overall budget. -// base == noRequestTimeout is returned unchanged: such a request grows no timeout of its own -// across retries, relying solely on the caller's context. +// attemptTimeout returns base doubled for each retry, capped at the maximum timeout. A base of +// [noRequestTimeout] stays unbounded. func (c *httpClient) attemptTimeout(base time.Duration, attempt int) time.Duration { if base == noRequestTimeout { return noRequestTimeout @@ -533,15 +465,13 @@ func (c *httpClient) attemptTimeout(base time.Duration, attempt int) time.Durati return minDuration(scaled, c.retry.timeoutMax) } -// isStatusRetryable reports whether a non-success status should be retried. We retry 429 -// (rate limit) and 5xx (internal server errors); other 4xx statuses are caller errors. +// isStatusRetryable reports whether a status is retried: 429 and 5xx. func isStatusRetryable(status int) bool { return status == rateLimitExceededStatusCode || status >= minServerErrorStatusCode } -// isTransportErrorRetryable reports whether a transport-level error should be retried. -// Network failures and timeouts are retryable; an *APIError raised from a malformed -// request (invalid argument) is not. +// isTransportErrorRetryable reports whether a transport error is retried. An *APIError, which +// comes from an invalid request, is not. func isTransportErrorRetryable(err error) bool { if _, ok := AsAPIError(err); ok { return false @@ -559,8 +489,7 @@ func isTimeoutError(err error) bool { return errors.As(err, &netErr) && netErr.Timeout() } -// randomizedDelay returns a delay chosen randomly from [delay, 2*delay), matching the -// exponential-backoff-with-jitter algorithm described in the Apify API docs. +// randomizedDelay returns a random delay in [delay, 2*delay). func randomizedDelay(delay time.Duration) time.Duration { if delay <= 0 { return delay diff --git a/iterator.go b/iterator.go index a5e6089..9cf7812 100644 --- a/iterator.go +++ b/iterator.go @@ -2,21 +2,13 @@ package apify import "context" -// ListIterator lazily iterates over an offset/limit-paginated collection, fetching one page -// at a time on demand. Obtain one from a collection client's Iterate method and drain it by -// calling Next until it returns (nil, nil). +// ListIterator lazily iterates over a paginated collection, fetching one page at a time. Get one +// from the Iterate method of a collection client, and call Next until it returns (nil, nil). // -// Its end-user semantics match the reference JS client's iterable list(): the list options' -// Limit is a cap on the total number of items yielded across all pages (unset means "all -// matching items"), the page size is the separate chunkSize argument passed to Iterate (unset -// means the API's default page size, capped by the remaining Limit; dataset items have no default -// limit, so iterating them without chunkSize or Limit loads the whole dataset in one response), -// and a caller-set Offset on the options is honored as the starting -// point (iteration begins there and yields at most Limit items from that offset onward). This -// keeps the two clients consistent for callers reasoning about offset/limit/chunk behaviour. -// -// The Limit uses the "0 == unset" convention: a nil Limit, or Limit set to ptr(0), yields the -// whole collection (no cap), rather than zero items. +// The Limit of the list options caps the total number of items yielded, and nil or 0 means all of +// them. The chunkSize argument of Iterate is the page size, which defaults to the page size of the +// API, so iterating dataset items without a chunkSize or Limit loads the whole dataset at once. +// The Offset of the options is the starting point. type ListIterator[T any] struct { // fetch fetches one page starting at offset. limit is the per-page limit to request // (0 means "unset", i.e. let the server choose). The collection's Iterate method bakes the @@ -48,12 +40,8 @@ func newListIterator[T any](limit, chunkSize *int64, startOffset int64, fetch fu return &ListIterator[T]{fetch: fetch, limit: limit, chunkSize: chunkSize, offset: startOffset} } -// newUnpaginatedIterator builds a ListIterator for an endpoint that ignores offset/limit and -// always returns every item in one response (e.g. the Actor version and environment-variable -// collections — see ActorVersionCollectionClient.Iterate and ActorEnvVarCollectionClient.Iterate). -// It fetches once and never asks for a second page. Matches the reference client's -// listUnpaginatedResources: under the general paging algorithm, an endpoint like this would -// return the same non-empty page forever, since it never "scans zero rows". +// newUnpaginatedIterator iterates an endpoint that ignores offset and limit and returns every item +// in one response, such as the Actor versions. It fetches once. func newUnpaginatedIterator[T any](fetch func(ctx context.Context) (PaginationList[T], error)) *ListIterator[T] { return &ListIterator[T]{ unpaginated: true, @@ -77,20 +65,13 @@ func (it *ListIterator[T]) Next(ctx context.Context) (*T, error) { return &item, nil } -// loadPage loads the next page into the buffer, following the same offset/limit/chunkSize -// arithmetic as the reference client's _listPaginated: the first page requests -// min(limit, chunkSize) items, and each subsequent page requests min(remaining, chunkSize) -// items, where remaining only exists when the caller set a Limit. Iteration ends when a page -// scans no rows, or (if a Limit was set) once that many items have been yielded — never by -// consulting the reported Total, which goes stale the moment the listed resource grows or -// shrinks while the loop runs (e.g. a dataset a running Actor keeps pushing to). Draining an -// uncapped iterator therefore always ends with one extra request that finds an empty page. +// loadPage loads the next page. The first page requests min(limit, chunkSize) items, and later +// pages min(remaining, chunkSize), where remaining exists only if a Limit is set. Iteration ends +// when a page scans no rows or the Limit is reached, never by Total, which goes stale while the +// resource changes. An uncapped iterator therefore ends with one request for an empty page. // -// Advancing the offset uses the larger of the page's Count and its item count (see scannedRows), -// capped at the limit requested for that page. A page of dataset items can return more or fewer items -// than the rows it scanned (see ListDatasetItems), and its Count is the scanned number. The -// cap matters because Unwind can make the item count exceed what was requested; advancing past -// the requested limit would skip rows the next request should have read. +// The offset advances by the larger of the page's Count and its item count (see scannedRows), +// capped at the limit requested for the page, so that Unwind cannot make it skip rows. func (it *ListIterator[T]) loadPage(ctx context.Context) error { if it.unpaginated { return it.loadUnpaginatedPage(ctx) @@ -142,9 +123,8 @@ func (it *ListIterator[T]) loadUnpaginatedPage(ctx context.Context) error { return nil } -// scannedRows returns the number of rows page covers: its Count, or the returned item count -// when that is larger, capped at requestedLimit (0 meaning "unset", i.e. no cap). Matches the -// reference client's scannedRows. +// scannedRows returns the number of rows page covers: its Count, or the returned item count when +// that is larger, capped at requestedLimit (0 meaning "unset", i.e. no cap). func scannedRows[T any](page PaginationList[T], requestedLimit int64) int64 { scanned := maxInt64(page.Count, int64(len(page.Items))) if requestedLimit > 0 && scanned > requestedLimit { @@ -180,8 +160,7 @@ func (it *ListIterator[T]) chunkVal() int64 { return *it.chunkSize } -// minForLimitParam mirrors the reference client's minForLimitParam: it treats 0 as "unset" and -// returns the smaller of the two defined values, or 0 when both are unset. +// minForLimitParam func minForLimitParam(a, b int64) int64 { if a <= 0 { return maxInt64(b, 0) diff --git a/key_value_store.go b/key_value_store.go index 8b70321..9cd00b8 100644 --- a/key_value_store.go +++ b/key_value_store.go @@ -64,8 +64,7 @@ func (c *KeyValueStoreClient) Get(ctx context.Context) (KeyValueStore, bool, err return getResource[KeyValueStore](ctx, c.ctx, "", NewQueryParams()) } -// Update updates the store metadata (e.g. name) and returns the updated object. Uses the "long" -// timeout tier, the reference client's tier for this call. +// Update updates the store metadata, such as its name, and returns the updated store. func (c *KeyValueStoreClient) Update(ctx context.Context, newFields any) (KeyValueStore, error) { return updateResourceWithTimeout[KeyValueStore](ctx, c.ctx, "", newFields, c.ctx.longTimeout()) } @@ -82,18 +81,11 @@ func (c *KeyValueStoreClient) ListKeys(ctx context.Context, options ListKeysOpti return getResourceRequired[KeyValueStoreKeysPage](ctx, c.ctx, "keys", params, c.ctx.mediumTimeout()) } -// IterateKeys returns a lazy iterator over the store's keys, fetching one page at a time on -// demand via the cursor-based keys endpoint (exclusiveStartKey / nextExclusiveStartKey). It -// mirrors the reference client's async-iterable listKeys() and follows this client's iteration -// convention (like the collection Iterate helpers): the options' Limit caps the total number of -// keys yielded across all pages (unset means all keys), the per-page size is the separate -// chunkSize argument (nil for the server default), Prefix/Collection/Signature filter every -// page, and ExclusiveStartKey sets the key to start listing after. Limit uses the "0 == unset" -// convention: a nil Limit, or Limit set to ptr(0), yields all keys rather than zero keys. -// -// Because keys are cursor-paginated (not offset/limit paginated) it uses its own -// KeyValueStoreKeysIterator rather than the generic ListIterator, sharing the cursor mechanics -// of RequestQueueClient.PaginateRequests. +// IterateKeys returns an iterator over the keys of the store, fetching one page at a time. The +// Limit of the options caps the total number of keys, and nil or 0 means all of them. chunkSize +// is the page size, nil for the default. Prefix, Collection and Signature filter every page, and +// ExclusiveStartKey is the key to start after. Keys are paginated by cursor, so it does not use +// [ListIterator]. func (c *KeyValueStoreClient) IterateKeys(options ListKeysOptions, chunkSize *int64) *KeyValueStoreKeysIterator { return &KeyValueStoreKeysIterator{client: c, options: options, chunkSize: chunkSize, nextStartKey: options.ExclusiveStartKey} } @@ -202,12 +194,9 @@ func (c *KeyValueStoreClient) RecordExists(ctx context.Context, key string) (boo return headExists(ctx, c.ctx, "records/"+safeKey, NewQueryParams()) } -// GetRecord fetches a record by key, or (nil, false, nil) if it does not exist. The value -// holds the raw bytes; the content type is reported in the record. -// -// Like the reference client, it requests the record as an attachment (sent on the wire as -// attachment=1, the truthy form this client's bool serializer uses) so the API returns the -// record's raw bytes directly rather than redirecting. Use [GetRecordWithOptions] to override. +// GetRecord fetches a record by key, or returns (nil, false, nil) if there is none. The value is +// the raw bytes. It requests the record as an attachment, so the API returns the bytes instead +// of redirecting. Use [KeyValueStoreClient.GetRecordWithOptions] to change that. func (c *KeyValueStoreClient) GetRecord(ctx context.Context, key string) (*KeyValueStoreRecord, bool, error) { return c.GetRecordWithOptions(ctx, key, GetRecordOptions{Attachment: ptrBool(true)}) } @@ -237,10 +226,9 @@ func (c *KeyValueStoreClient) GetRecordWithOptions(ctx context.Context, key stri }, true, nil } -// GetRecordStream is [KeyValueStoreClient.GetRecord], but returns the value as a stream instead -// of buffering it into memory, so a large record can be read incrementally. The caller is -// responsible for closing the returned stream. Mirrors the reference client's -// getRecord(key, { stream: true }). +// GetRecordStream is [KeyValueStoreClient.GetRecord], but returns the value as a stream instead of +// buffering it into memory, so a large record can be read incrementally. The caller is responsible +// for closing the returned stream. func (c *KeyValueStoreClient) GetRecordStream(ctx context.Context, key string) (*KeyValueStoreRecordStream, bool, error) { return c.GetRecordStreamWithOptions(ctx, key, GetRecordOptions{Attachment: ptrBool(true)}) } @@ -277,13 +265,10 @@ func (c *KeyValueStoreClient) SetRecordRaw(ctx context.Context, key string, valu return c.setRecord(ctx, key, value, contentType, "", false, false) } -// SetRecordRawWithEncoding is like [KeyValueStoreClient.SetRecordRaw], for a value that is -// already compressed: contentEncoding (e.g. "gzip", "br", "deflate") is sent as the -// Content-Encoding header and value is uploaded exactly as given. This call always skips the -// client's own request compression, so the bytes are never compressed twice — unlike -// SetRecordRaw, which compresses a large-enough body of its own accord. Use SetRecordRaw (or -// contentEncoding == "") for a value that is not pre-compressed. Mirrors the reference client's -// setRecord({ contentEncoding }). +// SetRecordRawWithEncoding is [KeyValueStoreClient.SetRecordRaw] for a value that is already +// compressed. contentEncoding, such as "gzip", "br", "zstd" or "deflate", is sent as the +// Content-Encoding header, and the value is uploaded as it is, without the client's own +// compression. func (c *KeyValueStoreClient) SetRecordRawWithEncoding(ctx context.Context, key string, value []byte, contentType, contentEncoding string) error { return c.setRecord(ctx, key, value, contentType, contentEncoding, true, false) } @@ -310,14 +295,14 @@ func (c *KeyValueStoreClient) setRecord(ctx context.Context, key string, value [ // SetRecordOptions configures [KeyValueStoreClient.SetRecordRawWithOptions] and // [KeyValueStoreClient.SetRecordJSONWithOptions]. type SetRecordOptions struct { - // ContentEncoding, when not empty (e.g. "gzip", "br", "deflate"), says the value is already + // ContentEncoding, when not empty (e.g. "gzip", "br", "zstd", "deflate"), says the value is already // compressed under that encoding: it is uploaded exactly as given, with the Content-Encoding // header, and the client's own request compression is skipped. See // [KeyValueStoreClient.SetRecordRawWithEncoding]. ContentEncoding string // DoNotRetryTimeouts, if true, makes an upload attempt that times out fail at once instead of - // being retried (other failures are still retried). Use it when storing the record twice - // would be harmful. Matches the reference client's setRecord({ doNotRetryTimeouts }). + // being retried (other failures are still retried). Use it when storing the record twice would + // be harmful. DoNotRetryTimeouts bool } @@ -351,17 +336,13 @@ func (c *KeyValueStoreClient) DeleteRecord(ctx context.Context, key string) erro return deleteResourceAlways(ctx, c.ctx, "records/"+safeKey) } -// getForPublicURL fetches the store to read its URL-signing secret, with the "long" timeout tier -// the reference client uses for its public-URL builders. +// getForPublicURL fetches the store to read its URL signing secret. func (c *KeyValueStoreClient) getForPublicURL(ctx context.Context) (KeyValueStore, bool, error) { return getResourceWithTimeout[KeyValueStore](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) } -// GetRecordPublicURL builds a public URL for fetching the given record. -// -// It mirrors the reference client: it fetches the store, and if the store exposes a -// URL-signing secret key (i.e. it is private), appends an HMAC-SHA256 signature so the URL -// grants access without an API token. The URL is built from the configured public base URL. +// GetRecordPublicURL returns a public URL of the record. If the store has a URL signing secret key, +// the URL is signed, so it works without a token. The URL uses the configured public base URL. func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string) (string, error) { safeKey, err := safePathSegment(key) if err != nil { diff --git a/key_value_store_collection.go b/key_value_store_collection.go index 7fc01d4..3eb6807 100644 --- a/key_value_store_collection.go +++ b/key_value_store_collection.go @@ -19,10 +19,9 @@ func (c *KeyValueStoreCollectionClient) List(ctx context.Context, options Storag return listResource[KeyValueStore](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the key-value stores matching the options, fetching -// pages on demand. The options' Limit caps the total number of stores yielded (unset means -// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference -// client's iterable list(). +// Iterate returns a lazy iterator over the key-value stores matching the options, fetching pages on +// demand. The options' Limit caps the total number of stores yielded (unset means all); the per- +// page size is chunkSize (nil for the server default). func (c *KeyValueStoreCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[KeyValueStore] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[KeyValueStore], error) { opts := options diff --git a/log.go b/log.go index 572edfb..2a48f62 100644 --- a/log.go +++ b/log.go @@ -49,12 +49,8 @@ func (c *LogClient) GetWithOptions(ctx context.Context, options LogOptions) (str return string(body), ok, err } -// GetBuffer fetches the entire log as raw bytes, or (nil, false, nil) if the log does not exist. -// -// It is the byte-oriented counterpart of [LogClient.Get]: both fetch the whole log in one -// response (for incremental reading, use [LogClient.Stream]), and GetBuffer only changes the -// return type to the undecoded bytes, e.g. for writing the log straight to a file. See -// [LogClient.Get] for how a 404 is handled. +// GetBuffer fetches the whole log as bytes, or returns (nil, false, nil) if there is none. It is +// [LogClient.Get] without the decoding, for example to write the log to a file. func (c *LogClient) GetBuffer(ctx context.Context) ([]byte, bool, error) { return c.GetBufferWithOptions(ctx, LogOptions{}) } @@ -74,18 +70,11 @@ func (c *LogClient) GetBufferWithOptions(ctx context.Context, options LogOptions return resp.body, true, nil } -// Stream opens a live, streaming connection to the log and returns a reader over the log -// bytes, or (nil, nil) if the log does not exist. The caller is responsible for closing the -// returned [io.ReadCloser]. -// -// Unlike [LogClient.Get], this bypasses the buffered/retrying transport so the log can be -// followed in real time as the run produces it (the `stream=1` query parameter). Because the -// response is consumed incrementally, it is not retried; transient failures surface to the -// caller. This mirrors the reference clients' streamed-log behaviour used for log redirection. +// Stream returns a reader over the live log, or (nil, nil) if there is no log. The caller closes +// it. The request is not retried. // -// A 404 resolves to (nil, nil) only when this client names the log by its own id -// (client.Log(id)): a chained client without one, such as run.Log() or build.Log(), returns an -// error instead, since the run or build itself may be what is missing. +// A 404 gives (nil, nil) only for client.Log(id). For run.Log() and build.Log() it is an error, +// since the run or build itself may be missing. func (c *LogClient) Stream(ctx context.Context) (io.ReadCloser, error) { return c.StreamWithOptions(ctx, LogOptions{}) } diff --git a/request_queue.go b/request_queue.go index 2ff316e..a122ebd 100644 --- a/request_queue.go +++ b/request_queue.go @@ -167,18 +167,14 @@ func (c *RequestQueueClient) ListAndLockHead(ctx context.Context, lockSecs int64 return postWithBody[json.RawMessage](ctx, c.ctx, "head/lock", params, nil, "", c.ctx.mediumTimeout()) } -// maxRequestsPerBatchOperation is the API limit on requests per batch call. Larger inputs -// are split into chunks of this size, matching the reference client. +// maxRequestsPerBatchOperation is the API limit on requests per batch call. Larger inputs are split +// into chunks of this size. const maxRequestsPerBatchOperation = 25 -// maxPayloadSizeBytes is the API's maximum request body size for a batch-requests call, -// matching the published @apify/consts MAX_PAYLOAD_SIZE_BYTES (9 MiB; apify/apify-shared-js). +// maxPayloadSizeBytes is the maximum request body size of a batch-requests call, 9 MiB. const maxPayloadSizeBytes = 9437184 -// payloadSizeLimitBytes shaves a small safety margin off maxPayloadSizeBytes (0.01%, matching -// the reference client's SAFETY_BUFFER_PERCENT), so a batch this client assembles stays under -// the server's limit even accounting for any rounding differences between the two. -// 9437184 * 0.0001 = 943.7184, rounded up to 944; 9437184 - 944 = 9436240. +// payloadSizeLimitBytes is maxPayloadSizeBytes less a 0.01% safety margin, rounded up to 944 bytes. const payloadSizeLimitBytes = maxPayloadSizeBytes - 944 // BatchAddResult is the typed result of [RequestQueueClient.BatchAddRequests]: the requests @@ -256,16 +252,10 @@ func joinAsJSONArray(batch []serializedRequest) []byte { return buf.Bytes() } -// BatchAddRequests adds multiple requests to the queue. If forefront is true, they are added -// to the front of the queue. -// -// The input is serialized once and split into batches that respect both the API's 25-request -// and (effective, after a small safety margin) 9 MiB payload-size limits; the whole call fails -// before anything is sent if any single request is too large for a batch of its own. The -// per-batch results are merged into a single [BatchAddResult]. Each batch is still subject to -// the client's standard retry policy, but unlike the reference client this does not itself -// retry a batch's unprocessed requests — inspect [BatchAddResult.UnprocessedRequests] and retry -// those explicitly if needed. +// BatchAddRequests adds requests to the queue, at the front if forefront is true. It splits them +// into batches within the API limits of 25 requests and 9 MiB, and merges the results. It fails +// before sending anything if one request is too large for a batch. Unlike the reference client, +// it does not retry unprocessed requests, so check [BatchAddResult.UnprocessedRequests]. func (c *RequestQueueClient) BatchAddRequests(ctx context.Context, requests []RequestQueueRequest, forefront bool) (BatchAddResult, error) { var merged BatchAddResult serialized, err := serializeRequestsForBatch(requests) @@ -388,11 +378,10 @@ func (c *RequestQueueClient) PaginateRequests(pageLimit *int64) *RequestQueueReq return c.PaginateRequestsWithOptions(ListRequestsOptions{}, pageLimit) } -// PaginateRequestsWithOptions is [RequestQueueClient.PaginateRequests] with listing options, like -// the reference client's paginateRequests: Limit caps the number of requests yielded across all -// pages (nil or ptr(0) means all of them), Cursor (or the deprecated ExclusiveStartID) sets where -// to start, and Filter applies to every page. The page size is the separate chunkSize argument -// (nil for the server default). Invalid options are reported by the first call of Next. +// PaginateRequestsWithOptions is [RequestQueueClient.PaginateRequests] with listing options. Limit +// caps the number of requests yielded, and nil or 0 means all. Cursor, or the deprecated +// ExclusiveStartID, is where to start, and Filter applies to every page. chunkSize is the page size, +// nil for the default. The first call of Next reports invalid options. func (c *RequestQueueClient) PaginateRequestsWithOptions(options ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator { return &RequestQueueRequestsIterator{client: c, options: options, chunkSize: chunkSize} } diff --git a/request_queue_collection.go b/request_queue_collection.go index 0a88cfb..1db29dc 100644 --- a/request_queue_collection.go +++ b/request_queue_collection.go @@ -19,10 +19,9 @@ func (c *RequestQueueCollectionClient) List(ctx context.Context, options Storage return listResource[RequestQueue](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the request queues matching the options, fetching -// pages on demand. The options' Limit caps the total number of queues yielded (unset means -// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference -// client's iterable list(). +// Iterate returns a lazy iterator over the request queues matching the options, fetching pages on +// demand. The options' Limit caps the total number of queues yielded (unset means all); the per- +// page size is chunkSize (nil for the server default). func (c *RequestQueueCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[RequestQueue] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[RequestQueue], error) { opts := options diff --git a/run.go b/run.go index f587e61..1ac71c3 100644 --- a/run.go +++ b/run.go @@ -41,10 +41,10 @@ type RunChargeOptions struct { EventName string // Count is the number of times to charge the event (defaults to 1). Count *int64 - // IdempotencyKey deduplicates the charge across retries. If empty, one is auto-generated - // as "{runId}-{eventName}-{timestampMillis}-{random}", matching the reference client. The - // API requires this header on every charge request; the key expires 3 minutes after the - // charge, so a later request reusing it starts a new charge rather than being deduplicated. + // IdempotencyKey deduplicates the charge across retries. If empty, one is auto-generated as + // "{runId}-{eventName}-{timestampMillis}-{random}". The API requires this header on every + // charge request; the key expires 3 minutes after the charge, so a later request reusing it + // starts a new charge rather than being deduplicated. IdempotencyKey string // RequestID is the ID of the Server Actor request being charged, taken from the // `X-Actor-Request-ID` header of the incoming request. Required for multi-tenant Server @@ -70,11 +70,8 @@ func newRunClient(root *ApifyClient, hc *httpClient, baseURL, resourcePath, id s } } -// LastRunOptions filters which "last" run the ActorClient.LastRunWithOptions / -// TaskClient.LastRunWithOptions accessors resolve to. An empty field leaves that filter unset. -// -// Origin is threaded to the runs/last endpoint as a documented query parameter of that endpoint in -// the OpenAPI spec, mirroring the reference client (lastRun({ origin })). +// LastRunOptions filters which last run [ActorClient.LastRunWithOptions] and +// [TaskClient.LastRunWithOptions] resolve to. An empty field sets no filter. type LastRunOptions struct { // Status filters by run status (e.g. "SUCCEEDED", "FAILED", "RUNNING"). Status string @@ -98,9 +95,9 @@ func (c *RunClient) Get(ctx context.Context) (ActorRun, bool, error) { return c.GetWithWait(ctx, nil) } -// GetWithWait fetches the run, optionally asking the API to wait up to waitForFinishSecs -// seconds (max 60) for the run to reach a terminal state before responding. Pass nil for an -// immediate fetch. Mirrors the reference client's get({ waitForFinish }). +// GetWithWait fetches the run, optionally asking the API to wait up to waitForFinishSecs seconds +// (max 60) for the run to reach a terminal state before responding. Pass nil for an immediate +// fetch. func (c *RunClient) GetWithWait(ctx context.Context, waitForFinishSecs *int64) (ActorRun, bool, error) { params := NewQueryParams() params.AddInt("waitForFinish", waitForFinishSecs) @@ -117,10 +114,9 @@ func (c *RunClient) Delete(ctx context.Context) error { return deleteResource(ctx, c.ctx, "") } -// Abort aborts the run. If gracefully points to true, the run is sent a signal so it can -// finish the current request before terminating; if false it is aborted immediately. Pass -// nil to omit the parameter entirely and let the server apply its default (immediate abort), -// matching the reference client's optional `gracefully` option. +// Abort aborts the run. If gracefully points to true, the run is sent a signal so it can finish the +// current request before terminating; if false it is aborted immediately. Pass nil to omit the +// parameter entirely and let the server apply its default (immediate abort). func (c *RunClient) Abort(ctx context.Context, gracefully *bool) (ActorRun, error) { params := NewQueryParams() params.AddBool("gracefully", gracefully) @@ -147,12 +143,8 @@ func (c *RunClient) Metamorph(ctx context.Context, targetActorID string, input a return c.metamorphWithBody(ctx, targetActorID, body, contentTypeJSON, options) } -// MetamorphRaw transforms the run into a run of another Actor (metamorph) with a raw request -// body instead of a JSON-serializable value. -// -// The bytes are sent exactly as given, with no JSON serialization — pair this with -// options.ContentType for a non-JSON input. For an object or array input, use -// [RunClient.Metamorph] instead. +// MetamorphRaw is [RunClient.Metamorph] with a raw request body, for a non-JSON input. Set +// options.ContentType. func (c *RunClient) MetamorphRaw(ctx context.Context, targetActorID string, input []byte, options MetamorphOptions) (ActorRun, error) { return c.metamorphWithBody(ctx, targetActorID, input, "application/octet-stream", options) } @@ -191,12 +183,9 @@ func (c *RunClient) Resurrect(ctx context.Context, options RunResurrectOptions) return postWithBody[ActorRun](ctx, c.ctx, "resurrect", params, nil, "", c.ctx.mediumTimeout()) } -// Charge charges for a pay-per-event Actor run: it records occurrences of a named event. -// Only meaningful for runs of pay-per-event Actors. -// -// An idempotency key is always sent (auto-generated if not provided), so a charge that is -// retried by the transport is applied at most once, matching the reference client. The API -// requires this header and expires the key 3 minutes after the charge. +// Charge records occurrences of an event of a pay-per-event Actor run. It always sends an +// idempotency key, generated if you set none, so a charge retried by the client is applied once. +// The API expires the key after 3 minutes. func (c *RunClient) Charge(ctx context.Context, options RunChargeOptions) error { if c.ctx.idErr != nil { return c.ctx.idErr @@ -224,8 +213,8 @@ func (c *RunClient) Charge(ctx context.Context, options RunChargeOptions) error const chargeIdempotencyHeader = "idempotency-key" // generateIdempotencyKey builds a per-charge idempotency key of the form -// "{runId}-{eventName}-{timestampMillis}-{random}", matching the reference client. It need -// not be cryptographically secure — only unique enough to avoid collisions within a request. +// "{runId}-{eventName}-{timestampMillis}-{random}". It need not be cryptographically secure — only +// unique enough to avoid collisions within a request. func (c *RunClient) generateIdempotencyKey(eventName string) string { return c.id + "-" + eventName + "-" + strconv.FormatInt(time.Now().UnixMilli(), 10) + "-" + @@ -270,11 +259,7 @@ func (c *RunClient) Log() *LogClient { return client } -// GetStreamedLog opens a live stream of this run's raw log. The caller must close the returned -// reader. It is a convenience wrapper over Log().StreamWithOptions with raw=true. -// -// Despite the name it is not the counterpart of the reference client's getStreamedLog(), which -// redirects the log message by message: that is [RunClient.NewStreamedLog]. +// GetStreamedLog opens the raw log of the run as a stream. The caller closes it. // // Deprecated: use [RunClient.NewStreamedLog] to redirect the log, or Log().StreamWithOptions for // the raw stream. diff --git a/run_collection.go b/run_collection.go index 8dc8ea9..42a9380 100644 --- a/run_collection.go +++ b/run_collection.go @@ -46,10 +46,9 @@ func (c *RunCollectionClient) List(ctx context.Context, options ListOptions, fil return listResource[ActorRun](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the runs matching the options and filter, fetching -// pages on demand. The options' Limit caps the total number of runs yielded (unset means all); -// the per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the runs matching the options and filter, fetching pages on +// demand. The options' Limit caps the total number of runs yielded (unset means all); the per-page +// size is chunkSize (nil for the server default). func (c *RunCollectionClient) Iterate(options ListOptions, filter RunListOptions, chunkSize *int64) *ListIterator[ActorRun] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorRun], error) { opts := options diff --git a/run_dataset_items.go b/run_dataset_items.go index bdfa701..9166ff7 100644 --- a/run_dataset_items.go +++ b/run_dataset_items.go @@ -8,8 +8,7 @@ import ( const ( // defaultRunIterateChunkSize is the number of dataset rows requested per API call when - // [RunIterateDatasetItemsOptions.ChunkSize] is unset. Matches the reference client's - // DEFAULT_ITERATE_CHUNK_SIZE. + // [RunIterateDatasetItemsOptions.ChunkSize] is unset. defaultRunIterateChunkSize = int64(1000) // defaultRunIteratePollIntervalSecs is how long the iterator waits for the run to finish // between polls when [RunIterateDatasetItemsOptions.PollIntervalSecs] is unset. @@ -52,17 +51,14 @@ const ( runIterateDone ) -// RunDatasetItemsIterator iterates over the items of a run's default dataset while the run is -// still producing them. Obtain one from [RunClient.IterateDatasetItems] or -// [IterateRunDatasetItems] and drain it by calling Next until it returns (nil, nil). +// RunDatasetItemsIterator iterates over the items of the default dataset of a run while the run +// produces them. Get one from [RunClient.IterateDatasetItems] or [IterateRunDatasetItems], and +// call Next until it returns (nil, nil). // -// While the run has not finished, each round yields the rows below the dataset's itemCount and -// then waits up to PollIntervalSecs for the run to finish, so the last rows are read as soon as it -// does. Each page is requested with a limit that ends at itemCount, so it covers exactly the -// rows it asks for, whatever the filters or Unwind do to the items. itemCount lags a few seconds -// behind the pushed items, so once the run reaches a terminal status the rows past it are read a -// page at a time until none are left. On a client from LastRun, the iterator sticks to the run -// its first request resolves to. Matches the reference client's iterateDatasetItems(). +// While the run is active, each round reads the rows below the dataset's itemCount and then waits +// up to PollIntervalSecs for the run to finish. itemCount lags the pushed items, so after the run +// finishes the iterator reads page by page until no rows are left. On a client from LastRun, it +// stays with the run that its first request resolved to. type RunDatasetItemsIterator[T any] struct { run *RunClient options RunIterateDatasetItemsOptions diff --git a/schedule.go b/schedule.go index 8daa1eb..36f7b3e 100644 --- a/schedule.go +++ b/schedule.go @@ -20,9 +20,8 @@ func (c *ScheduleCollectionClient) List(ctx context.Context, options ListOptions } // Iterate returns a lazy iterator over the schedules matching the options, fetching pages on -// demand. The options' Limit caps the total number of schedules yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// demand. The options' Limit caps the total number of schedules yielded (unset means all); the per- +// page size is chunkSize (nil for the server default). func (c *ScheduleCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Schedule] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Schedule], error) { opts := options diff --git a/server_http.go b/server_http.go index cf766eb..d7ce518 100644 --- a/server_http.go +++ b/server_http.go @@ -69,12 +69,9 @@ type serverResponse struct { stream io.ReadCloser } -// callServer sends call to an Actor's server and returns its response whatever the status. -// -// It differs from [httpClient.call] where the server is not the API: an error status is -// returned instead of raised, only 429 and 503 are retried, a transport error (a timeout -// included) is never retried since the Actor may have handled the request already, the token -// goes in X-Apify-Authorization, and redirects are not followed. +// callServer sends call to an Actor's server and returns the response whatever its status. Unlike +// [httpClient.call], it retries only 429 and 503, never retries a transport error, sends the +// token in X-Apify-Authorization and does not follow redirects. func (c *httpClient) callServer(ctx context.Context, call serverCall) (*serverResponse, error) { // Header names are case-insensitive, so canonicalize the caller's once: the lookups below // and the headers on the wire then agree on what the caller set. A name with no values stays, diff --git a/signature.go b/signature.go index 84f9551..6a5bb38 100644 --- a/signature.go +++ b/signature.go @@ -10,11 +10,9 @@ import ( "time" ) -// Constants for the Apify storage-content signing scheme, matching the platform's -// @apify/utilities implementation that the reference clients rely on. +// Constants of the storage-content signing scheme of @apify/utilities. const ( - // storageContentSignatureVersion is the version tag embedded in signatures. The - // reference clients rely on the upstream default of "0". + // storageContentSignatureVersion is the version tag embedded in signatures. storageContentSignatureVersion = "0" // hmacSignatureHexLen is the number of leading hex characters of the HMAC digest used. hmacSignatureHexLen = 30 @@ -63,14 +61,8 @@ func toBase62(value *big.Int) string { return string(digits) } -// signStorageContent builds a storage-content signature for a resource's public URL, -// byte-for-byte compatible with @apify/utilities createStorageContentSignature. -// -// It signs the message "{version}.{expiresAtMillis}.{resourceId}" (expiresAtMillis is the -// absolute expiry in ms, or 0 for a non-expiring URL) with createHmacSignature, then -// returns the base64url (no padding) encoding of "{version}.{expiresAtMillis}.{hmac}". -// -// expiresInSecs is optional (nil for a non-expiring URL). +// signStorageContent signs the public URL of a storage resource, compatibly with +// createStorageContentSignature of @apify/utilities. expiresInSecs is nil for no expiry. func signStorageContent(secretKey, resourceID string, expiresInSecs *int64) string { var expiresAtMillis int64 if expiresInSecs != nil { diff --git a/store_collection.go b/store_collection.go index 06a6063..148183f 100644 --- a/store_collection.go +++ b/store_collection.go @@ -64,10 +64,9 @@ func (c *StoreCollectionClient) List(ctx context.Context, options StoreListOptio // It is the generic [ListIterator] specialized to Store Actors; drain it with Next. type StoreActorIterator = ListIterator[ActorStoreListItem] -// Iterate returns a lazy iterator over the Store Actors matching the options, fetching pages -// on demand. The options' Limit caps the total number of Actors yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the Store Actors matching the options, fetching pages on +// demand. The options' Limit caps the total number of Actors yielded (unset means all); the per- +// page size is chunkSize (nil for the server default). func (c *StoreCollectionClient) Iterate(options StoreListOptions, chunkSize *int64) *StoreActorIterator { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorStoreListItem], error) { opts := options diff --git a/streamed_log.go b/streamed_log.go index 9730bb8..ce8d842 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -17,12 +17,11 @@ const ( // emptyLogStreamRetry is how long a [StreamedLog] waits before it reopens a log stream that // delivered nothing: the API serves the log of a run that has not logged anything yet as an // empty stream that ends at once. The pause doubles up to emptyLogStreamMaxRetry, which bounds - // the request rate while a run waits long to start. Matches the reference client's - // EMPTY_LOG_STREAM_RETRY_MILLIS and EMPTY_LOG_STREAM_MAX_RETRY_MILLIS. + // the request rate while a run waits long to start. emptyLogStreamRetry = 500 * time.Millisecond emptyLogStreamMaxRetry = 5 * time.Second // stopGrace is how long [StreamedLog.Stop] waits for the messages already in flight before it - // aborts the log stream. Matches the reference client's STOP_GRACE_MILLIS. + // aborts the log stream. stopGrace = time.Second // logRedirectPrefixSeparator ends the default message prefix, which names the Actor and the run. logRedirectPrefixSeparator = " -> " @@ -60,14 +59,11 @@ type StreamedLogOptions struct { FromStart *bool } -// StreamedLog redirects the live log of an Actor run to a writer, message by message. Obtain one -// from [RunClient.NewStreamedLog], begin the redirection with [StreamedLog.Start] and end it with -// [StreamedLog.Stop]. Matches the reference client's StreamedLog. -// -// Each message of the log, which starts with the timestamp the platform puts in front of it, is -// written on its own line, prefixed. Messages that were already logged when the redirection was -// created are skipped if FromStart is false. The log of a run that has not logged anything yet is -// reopened until it delivers its first bytes. +// StreamedLog redirects the live log of an Actor run to a writer, one prefixed line per message. +// Get one from [RunClient.NewStreamedLog], and begin and end the redirection with +// [StreamedLog.Start] and [StreamedLog.Stop]. Messages logged before the redirection was created +// are skipped unless FromStart is true. A log that is still empty is reopened until it delivers +// its first bytes. type StreamedLog struct { log *LogClient output io.Writer @@ -156,14 +152,10 @@ func (s *StreamedLog) Start(ctx context.Context) error { return nil } -// Stop ends the redirection. It gives the messages already in flight up to one second, then -// aborts the log stream; if the stream delivered nothing yet, it reads the whole log in one -// request instead, so a short run that finished before the stream connected is not lost. It -// returns the error that made the redirection stop early, if any. ctx bounds the wait for it, -// the final read of the whole log included: when ctx is done first, Stop gives up on the rest of -// the log and returns the error of ctx, unless the context given to Start ended first, which ends -// the redirection on purpose; when both end at about the same time, either result is possible. It -// returns an error if the redirection is not running or is already being stopped. +// Stop ends the redirection and returns the error that stopped it early, if any. Messages in +// flight get up to one second. If the stream delivered nothing, Stop reads the whole log in one +// request, so that a short run is not lost. ctx bounds the wait, the final read included. It +// returns an error if the redirection is not running or is already stopping. func (s *StreamedLog) Stop(ctx context.Context) error { s.mu.Lock() if !s.running { diff --git a/task.go b/task.go index e31aabb..61c7a52 100644 --- a/task.go +++ b/task.go @@ -35,32 +35,22 @@ func (c *TaskClient) Delete(ctx context.Context) error { return deleteResource(ctx, c.ctx, "") } -// Publish publishes the task on its public landing page by setting IsPublic through Update. -// -// The task's Actor must be public and the task must already have its public display -// configuration (PublicConfig) set up. Requires write permission to both the task and its -// Actor. Publishing an already published task does nothing. -// -// An Actor can have at most 10 published tasks, and an account at most 100; contact Apify -// support to raise either limit. +// Publish publishes the task on its public page by setting IsPublic. The Actor must be public and +// the task must have its PublicConfig. Publishing a published task does nothing. An Actor can have +// 10 published tasks and an account 100. func (c *TaskClient) Publish(ctx context.Context) (Task, error) { return c.Update(ctx, map[string]any{"isPublic": true}) } -// Unpublish unpublishes the task from its public landing page by setting IsPublic through -// Update. -// -// The public display configuration (PublicConfig) is preserved, so the task can be published -// again without re-entering it. Like Publish, Unpublish requires write permission to both the -// task and its Actor. Unpublishing a task that is not published does nothing. +// Unpublish removes the task from its public page by clearing IsPublic. PublicConfig is kept. +// Unpublishing a task that is not published does nothing. func (c *TaskClient) Unpublish(ctx context.Context) (Task, error) { return c.Update(ctx, map[string]any{"isPublic": false}) } -// TaskStartOptions configures starting a task run ([TaskClient.Start]/[TaskClient.Call]). -// -// It mirrors [ActorStartOptions] but omits the fields the task run endpoint does not accept -// (the Actor-only `contentType` and `forcePermissionLevel`), matching the reference client. +// TaskStartOptions configures [TaskClient.Start] and [TaskClient.Call]. It is [ActorStartOptions] +// without the fields that the task run endpoint does not accept, `contentType` and +// `forcePermissionLevel`. type TaskStartOptions struct { // Build is the tag or number of the build to run (e.g. "latest", "0.1.2"). Build *string @@ -78,10 +68,8 @@ type TaskStartOptions struct { RestartOnError *bool // Webhooks are ad-hoc webhooks to attach to this run (base64-encoded JSON). Webhooks []any - // WaitForResources, if set, retries the start every 10 seconds while the API rejects it for - // lack of resources. See [ActorStartOptions.WaitForResources] for the full semantics (same - // error types, cooldown and WaitForResourcesIndefinitely sentinel; this field mirrors it for - // task starts). + // WaitForResources retries the start while the API rejects it for lack of resources. See + // [ActorStartOptions.WaitForResources]. WaitForResources *int64 } @@ -121,11 +109,9 @@ func (c *TaskClient) startWithTimeout(ctx context.Context, input any, options Ta }) } -// Call starts the task and waits (client-side polling) for it to finish. waitSecs bounds the -// wait; nil waits indefinitely. -// -// Neither the start request nor the polls have a client-side timeout of their own (matching the -// reference client's call(), whose default timeout is noTimeout): ctx bounds the whole call. +// Call starts the task and waits for it to finish. waitSecs bounds the wait, and nil waits +// indefinitely. The start request and the polls have no timeout of their own, so ctx bounds the +// whole call. func (c *TaskClient) Call(ctx context.Context, input any, options TaskStartOptions, waitSecs *int64) (ActorRun, error) { run, err := c.startWithTimeout(ctx, input, options, noRequestTimeout) if err != nil { @@ -171,9 +157,8 @@ func (c *TaskClient) LastRun(status string) *RunClient { return c.LastRunWithOptions(LastRunOptions{Status: status}) } -// LastRunWithOptions returns a client for the last run of this task, optionally filtered by -// status and/or origin. See LastRunOptions. Mirrors the reference client's -// lastRun({ status, origin }). +// LastRunWithOptions returns a client for the last run of this task, optionally filtered by status +// and/or origin. See LastRunOptions. func (c *TaskClient) LastRunWithOptions(options LastRunOptions) *RunClient { client := newRunClient(c.root, c.ctx.http, c.ctx.subURL(""), "runs", "last") client.setLastRunParams(options) diff --git a/task_collection.go b/task_collection.go index 6da22a1..ac6468f 100644 --- a/task_collection.go +++ b/task_collection.go @@ -19,10 +19,9 @@ func (c *TaskCollectionClient) List(ctx context.Context, options ListOptions) (P return listResource[Task](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the tasks matching the options, fetching pages on -// demand. The options' Limit caps the total number of tasks yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the tasks matching the options, fetching pages on demand. +// The options' Limit caps the total number of tasks yielded (unset means all); the per-page size is +// chunkSize (nil for the server default). func (c *TaskCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Task] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Task], error) { opts := options diff --git a/url_normalization.go b/url_normalization.go index 8c2c21a..a2d1cf4 100644 --- a/url_normalization.go +++ b/url_normalization.go @@ -11,13 +11,11 @@ import ( "unicode/utf8" ) -// The fields the OpenAPI specification declares as `format: uri` are returned in a normalized -// form, close to the WHATWG URL serialization that the JavaScript client returns: the scheme and -// host are lowercased, a non-ASCII host is converted to punycode, a numeric (IPv4) or IPv6 host is -// written in its canonical form, the default port is dropped, an empty path becomes "/", dot -// segments are resolved and characters that are unsafe in a URL are percent-encoded. It is built -// for the http(s), ws(s), ftp and file URLs the API returns. The host of any other scheme is kept -// as given. It does not apply the full Unicode (UTS #46) mapping to a non-ASCII host. +// The URL fields of the API are returned close to the WHATWG URL serialization of the JavaScript +// client: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IP host is +// canonical, the default port is dropped, an empty path becomes "/", dot segments are resolved and +// unsafe characters are percent-encoded. This covers http(s), ws(s), ftp and file URLs. The host of +// another scheme is kept, and the full Unicode (UTS #46) mapping is not applied. // defaultPorts maps a special URL scheme to its default port, which normalization drops. var defaultPorts = map[string]string{ diff --git a/version.go b/version.go index c6e3194..c38ac2d 100644 --- a/version.go +++ b/version.go @@ -1,13 +1,9 @@ package apify -// ClientVersion is the semantic version of this client library. -// -// It follows Semantic Versioning (https://semver.org/). Changes to the public -// interface (other than additive ones) are considered breaking changes. -const ClientVersion = "0.15.0" +// ClientVersion is the semantic version of this client. A change to the public interface other +// than an addition is a breaking change. +const ClientVersion = "0.16.0" -// APISpecVersion is the version of the Apify OpenAPI specification that this -// client was generated and verified against. -// -// It corresponds to the `info.version` field of the Apify OpenAPI document. -const APISpecVersion = "v2-2026-11-04T151257Z" +// APISpecVersion is the `info.version` of the Apify OpenAPI specification that this client was +// last updated against. +const APISpecVersion = "v2-2026-11-09T143926Z" diff --git a/wait_for_resources.go b/wait_for_resources.go index f2192ec..df4e187 100644 --- a/wait_for_resources.go +++ b/wait_for_resources.go @@ -6,32 +6,25 @@ import ( "time" ) -// resourceLimitErrorTypes are the Apify API error types a run/task start is rejected with while -// the account has no free memory or concurrent-run slot for it. Both clear as other runs or -// builds of the account finish. Matches the reference client's RESOURCE_LIMIT_ERROR_TYPES. +// resourceLimitErrorTypes are the error types of a start rejected for lack of memory or concurrent +// run slots. var resourceLimitErrorTypes = map[string]bool{ "actor-memory-limit-exceeded": true, "concurrent-runs-limit-exceeded": true, } -// waitForResourcesCooldown is the delay between two attempts to start a run rejected for lack of -// resources. Matches the reference client's WAIT_FOR_RESOURCES_COOLDOWN_MILLIS. +// waitForResourcesCooldown is the delay between attempts to start a run. const waitForResourcesCooldown = 10 * time.Second // WaitForResourcesIndefinitely, used as [ActorStartOptions.WaitForResources] or -// [TaskStartOptions.WaitForResources], retries a resource-limited start forever instead of -// bounding the retrying to a number of seconds. Matches the reference client's -// `waitForResources: true`. +// [TaskStartOptions.WaitForResources], retries a resource-limited start forever instead of bounding +// the retrying to a number of seconds. const WaitForResourcesIndefinitely = int64(-1) -// startWaitingForResources calls start, retrying it every waitForResourcesCooldown while it -// fails with a resource-limit error (`actor-memory-limit-exceeded` or -// `concurrent-runs-limit-exceeded`), until it succeeds or the waitForResources budget elapses. -// Any other error is returned immediately, without retrying. -// -// waitForResources nil disables retrying, so the first error (of any kind) is returned -// immediately. [WaitForResourcesIndefinitely] retries with no time bound. Any other value is the -// number of seconds to keep retrying for (must be >= 0) before returning the last error. +// startWaitingForResources calls start again every waitForResourcesCooldown while it fails with a +// resource-limit error, until it succeeds or waitForResources seconds pass. Any other error is +// returned at once. A nil waitForResources does not retry, and [WaitForResourcesIndefinitely] +// retries without a time bound. func startWaitingForResources[T any](ctx context.Context, waitForResources *int64, start func() (T, error)) (T, error) { if waitForResources == nil { return start() diff --git a/webhook.go b/webhook.go index a513ccb..08f5fd1 100644 --- a/webhook.go +++ b/webhook.go @@ -25,10 +25,9 @@ func (c *WebhookCollectionClient) List(ctx context.Context, options ListOptions) return listResource[Webhook](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the webhooks matching the options, fetching pages on -// demand. The options' Limit caps the total number of webhooks yielded (unset means all); the -// per-page size is chunkSize (nil for the server default). Mirrors the reference client's -// iterable list(). +// Iterate returns a lazy iterator over the webhooks matching the options, fetching pages on demand. +// The options' Limit caps the total number of webhooks yielded (unset means all); the per-page size +// is chunkSize (nil for the server default). func (c *WebhookCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Webhook] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Webhook], error) { opts := options diff --git a/webhook_dispatch.go b/webhook_dispatch.go index 50f3704..951829c 100644 --- a/webhook_dispatch.go +++ b/webhook_dispatch.go @@ -25,10 +25,9 @@ func (c *WebhookDispatchCollectionClient) List(ctx context.Context, options List return listResource[WebhookDispatch](ctx, c.ctx, "", params) } -// Iterate returns a lazy iterator over the webhook dispatches matching the options, fetching -// pages on demand. The options' Limit caps the total number of dispatches yielded (unset means -// all); the per-page size is chunkSize (nil for the server default). Mirrors the reference -// client's iterable list(). +// Iterate returns a lazy iterator over the webhook dispatches matching the options, fetching pages +// on demand. The options' Limit caps the total number of dispatches yielded (unset means all); the +// per-page size is chunkSize (nil for the server default). func (c *WebhookDispatchCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[WebhookDispatch] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[WebhookDispatch], error) { opts := options From 503bd600ae9d6ca7e148df53711a4465f47b93da Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:14:57 +0000 Subject: [PATCH 2/6] feat: retry unprocessed batch requests, share the zstd encoder, finish the docs and comments cleanup Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- .github/workflows/go-integration-tests.yml | 48 +- .github/workflows/go-publish.yml | 44 +- CHANGELOG.md | 8 +- README.md | 74 +- actor_env_var_test.go | 17 + actor_server.go | 9 +- actor_server_test.go | 4 +- base.go | 2 +- client.go | 5 +- client_meta_test.go | 15 +- dataset_items_test.go | 79 ++ docs/server_actors.md | 25 + docs/storages.md | 7 +- docs/tasks.md | 5 +- docs/webhooks.md | 3 +- errors_test.go | 63 ++ examples/create_build_run_actor/main.go | 3 +- http_client.go | 29 +- iterator.go | 11 +- iterator_test.go | 3 +- key_value_store_test.go | 14 + models.go | 2 + not_found_test.go | 88 +++ path_validation_test.go | 60 ++ request_queue.go | 184 ++++- request_queue_batch_test.go | 281 +++++++ request_queue_test.go | 64 ++ review_fixes_test.go | 836 --------------------- run_test.go | 141 ++++ streamed_log.go | 10 +- task.go | 10 +- tests/examples_test.go | 4 +- tests/helpers_test.go | 2 +- tests/iterate_test.go | 4 +- timeout_tier_test.go | 61 +- unit_http_test.go | 33 + user_test.go | 47 ++ wait_for_finish_test.go | 162 ++++ 38 files changed, 1429 insertions(+), 1028 deletions(-) create mode 100644 actor_env_var_test.go create mode 100644 dataset_items_test.go create mode 100644 errors_test.go create mode 100644 not_found_test.go create mode 100644 path_validation_test.go create mode 100644 request_queue_batch_test.go create mode 100644 request_queue_test.go delete mode 100644 review_fixes_test.go create mode 100644 run_test.go create mode 100644 user_test.go create mode 100644 wait_for_finish_test.go diff --git a/.github/workflows/go-integration-tests.yml b/.github/workflows/go-integration-tests.yml index feaf21c..b7e0a8f 100644 --- a/.github/workflows/go-integration-tests.yml +++ b/.github/workflows/go-integration-tests.yml @@ -1,7 +1,6 @@ name: Go integration tests -# Language-specific workflow: only runs for the Go client. Triggers on PRs to master that -# touch Go client, test, or example code, and can be dispatched manually from any branch. +# Runs for changes to Go code, docs or this workflow, and can be dispatched from any branch. on: pull_request: branches: [master] @@ -9,15 +8,14 @@ on: - '**/*.go' - 'go.mod' - 'go.sum' - # The "Test examples" step validates the in-documentation snippets, so doc changes must - # re-run the workflow even though Markdown is not Go code. + # The "Test examples" step checks the snippets of the docs. - 'docs/**' - 'README.md' - '.github/workflows/go-integration-tests.yml' - '.github/scripts/assert-tests-ran.sh' workflow_dispatch: -# Avoid concurrent runs of the same ref racing on the shared test account. +# Runs of one ref share the test account, so they must not overlap. concurrency: group: go-integration-${{ github.ref }} cancel-in-progress: true @@ -32,8 +30,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - # Patched toolchain (Go 1.26.10+): earlier releases still trust the compromised - # intermediate CA certificates. Keep in step with the `toolchain` line in go.mod. + # Earlier releases trust compromised CA certificates. Keep in step with go.mod. go-version: '1.26.10' cache: true @@ -49,13 +46,8 @@ jobs: - name: Vet run: go vet ./... - # The idiomatic Go linter. `go vet` only catches a narrow set of correctness issues; - # golangci-lint bundles staticcheck and other analyzers, satisfying the coding-rules - # mandate that linting/type-checking run in CI. Config lives in .golangci.yml. - name: Lint (golangci-lint) - # Use the v7 action and pin a golangci-lint v2 release: .golangci.yml uses the - # v2 config schema, which the v6 action's v1.x binary rejects ("additional - # properties 'version'/'default' not allowed"). + # .golangci.yml uses the v2 schema, which needs the v7 action. uses: golangci/golangci-lint-action@v7 with: version: v2.15.0 @@ -64,13 +56,10 @@ jobs: run: go build ./... - name: Unit tests - # The unit tests are offline (mock HTTP backend) and prove the retry/error/signature - # logic without hitting the API. + # Offline tests with a mock HTTP backend. run: go test . -v - # Fail fast if the integration-test secret is missing or empty. Without this guard the - # integration tests silently "pass" (they skip when APIFY_TOKEN is unset), so a green - # run would not prove the API logic actually executed. + # Without the token the integration tests skip and the run would be green. - name: Require APIFY_TOKEN secret env: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} @@ -82,36 +71,19 @@ jobs: - name: Integration tests env: - # The integration-test token is stored as a repository secret. APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} - # Limit parallelism to be gentle on the shared test account. The documentation example - # programs (TestExample*) and the in-documentation snippet checks (TestDocSnippets*) are - # exercised by the standalone "Test examples" step below, so they are skipped here to - # keep the two concerns separate. - # - # -json is parsed by assert-tests-ran.sh so the step fails if zero tests actually ran or - # every test skipped (a filter that matches nothing, a package with no test functions, - # or an all-skip run would otherwise exit green and hide that the API logic never ran). + # The example and snippet tests run in "Test examples". assert-tests-ran.sh fails the step + # if no test ran or all were skipped. run: | set -o pipefail go test ./tests/ -json -timeout 900s -p 1 -parallel 4 -skip '^(TestExample|TestDocSnippets)' \ | tee integration.jsonl ./.github/scripts/assert-tests-ran.sh integration.jsonl "Integration tests" - # Standalone CI step that verifies the documentation examples actually work. It runs the - # example programs in examples/ end-to-end against the live API (via the TestExample* - # smoke tests in tests/examples_test.go, each of which executes `go run ./examples/`) - # and checks that every in-documentation code snippet is valid, runnable, and gofmt- - # formatted (the TestDocSnippets* tests, which extract every ```go block from the README - # and docs/ and compile each one). Both are required by the documentation requirements: - # each documentation example has a CI test that actually runs the code, and each snippet - # must be valid, runnable and properly formatted. + # Runs the programs in examples/ against the live API and compiles the snippets of the docs. - name: Test examples env: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} - # Match the integration-test thread cap so the example programs (which hit the live - # account) stay gentle on the shared test account. As above, -json is asserted so the - # step fails if the example/snippet tests did not actually run or all skipped. run: | set -o pipefail go test ./tests/ -json -timeout 900s -p 1 -parallel 4 -run '^(TestExample|TestDocSnippets)' \ diff --git a/.github/workflows/go-publish.yml b/.github/workflows/go-publish.yml index 18da0f5..eafd386 100644 --- a/.github/workflows/go-publish.yml +++ b/.github/workflows/go-publish.yml @@ -1,13 +1,7 @@ name: Publish Go client -# Language-specific publish workflow for the Go client. Go has no central package registry to -# upload an archive to: a module is "published" by pushing a semver git tag (vX.Y.Z), after which -# the Go module proxy (proxy.golang.org) and pkg.go.dev pick it up on first request. This workflow -# is therefore the language-specific distribution standard for Go. -# -# It is triggered manually only (workflow_dispatch) so a maintainer deliberately decides when a -# release is cut. The version is taken from the single source of truth (the ClientVersion constant -# in version.go) so the published tag can never disagree with the in-code client version. +# Publishes the Go client by pushing the tag v, which the Go module proxy and +# pkg.go.dev pick up. It runs only when a maintainer triggers it. on: workflow_dispatch: inputs: @@ -16,14 +10,12 @@ on: type: boolean default: false -# Never allow two publish runs to race; pushing two release tags concurrently is hard to undo. +# Two release tags pushed at once are hard to undo. concurrency: group: go-publish cancel-in-progress: false -# The default GITHUB_TOKEN needs write access to push the tag and create the release. No other -# credentials are required: the Go module proxy reads the public repo directly, so there is no -# registry token to configure. +# The GITHUB_TOKEN pushes the tag and creates the release. No other credentials are needed. permissions: contents: write @@ -34,12 +26,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - # Full history so the tag points at the real commit and `git describe`-style tooling works. + # Full history, so that the tag points at the real commit. fetch-depth: 0 - # A release tag is immutable once the Go module proxy has fetched it, so a release must only - # ever be cut from master. Refuse to run from any other ref, even when dispatched manually, - # so a maintainer cannot accidentally publish an unmerged feature-branch commit as a version. + # A tag is immutable once the module proxy has fetched it, so release from master only. - name: Require master branch run: | if [ "${GITHUB_REF}" != "refs/heads/master" ]; then @@ -50,12 +40,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - # Patched toolchain (Go 1.26.10+): earlier releases still trust the compromised - # intermediate CA certificates. Keep in step with the `toolchain` line in go.mod. + # Earlier releases trust compromised CA certificates. Keep in step with go.mod. go-version: '1.26.10' cache: true - # Gate the release on the same quality bar as CI so a broken build can never be tagged. + # The checks of CI. - name: Check formatting run: | unformatted=$(gofmt -l .) @@ -77,12 +66,10 @@ jobs: run: go build ./... - name: Unit tests - # Offline (mock HTTP backend) tests; prove the retry/error/signature logic without the API. + # Offline tests with a mock HTTP backend. run: go test . -v - # Derive the release tag from the single source of truth (version.go). Keeping the tag in lock - # step with the in-code ClientVersion means consumers who read the constant and consumers who - # `go get` a tag always see the same version. + # The tag is the ClientVersion of version.go. - name: Resolve version from version.go id: version run: | @@ -98,8 +85,7 @@ jobs: echo "tag=v${version}" >> "$GITHUB_OUTPUT" echo "Resolved release tag: v${version}" - # Fail early (before creating any tag) if this version was already released, so a publish run - # can never silently no-op or clobber an existing release. + # Fail before creating anything if the version was released already. - name: Ensure tag does not already exist env: TAG: ${{ steps.version.outputs.tag }} @@ -123,8 +109,6 @@ jobs: git tag -a "${TAG}" -m "Release ${TAG}" git push origin "${TAG}" - # Create a GitHub release for the tag. Uses the default GITHUB_TOKEN from repository secrets; - # no personal access token or registry credential is needed. - name: Create GitHub release if: ${{ github.event.inputs.dry_run != 'true' }} env: @@ -135,8 +119,7 @@ jobs: --title "${TAG}" \ --notes "Apify Go client ${TAG}. See CHANGELOG.md for details." - # Prompt the public Go module proxy to fetch the new version. This is what makes the release - # discoverable on pkg.go.dev; it requires no authentication because the repository is public. + # Asks the Go module proxy to fetch the version, which lists it on pkg.go.dev. - name: Trigger Go module proxy indexing if: ${{ github.event.inputs.dry_run != 'true' }} env: @@ -144,7 +127,6 @@ jobs: run: | module=$(go list -m) echo "Requesting ${module}@${TAG} from the Go module proxy to trigger indexing." - # Best-effort: the proxy may take a moment to see the freshly pushed tag, so a failure here - # is not fatal — the module is still published via the tag and will be indexed on first use. + # The proxy may not see the tag yet. That is not fatal, since the tag publishes the module. curl -fsSL "https://proxy.golang.org/${module}/@v/${TAG}.info" || \ echo "::warning::Proxy did not return the version yet; pkg.go.dev will index it on first request." diff --git a/CHANGELOG.md b/CHANGELOG.md index 46cc8a0..3023612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,10 +13,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ActorServerRequestOptions.ParentRequestID`, sent as the `X-Actor-Request-ID` header, so that a Server Actor calling another one passes its caller's pricing tier on. - `zstd` in the documented `Content-Encoding` values of `SetRecordRawWithEncoding`. +- `RequestQueueClient.BatchAddRequestsWithOptions` with `BatchAddRequestsOptions`. + `BatchAddRequests` now sends the requests that the API did not process again (3 retries by + default) and sends up to 5 batches at once. If a batch fails with an error, the error is + returned together with the result of all batches. +- `RequestQueueOperationInfo.UniqueKey`. ### Changed -- Shortened and corrected the documentation and code comments. +- Shortened and corrected the documentation and code comments, and split the tests of + `review_fixes_test.go` by subject. - Bumped `APISpecVersion` to `v2-2026-11-09T143926Z` and `ClientVersion` to `0.16.0`. ## [0.15.0] - 2026-11-06 diff --git a/README.md b/README.md index 1be9c77..da39e95 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,12 @@ An idiomatic Go client for the [Apify API](https://docs.apify.com/api/v2). -It provides a resource-oriented interface that mirrors the official -[JavaScript](https://github.com/apify/apify-client-js) and Rust clients: start from an -`ApifyClient`, then drill down into resources (Actors, runs, datasets, key-value stores, -request queues, tasks, schedules, webhooks, the store, users and logs). +Start from an `ApifyClient` and go on to the resources: Actors, runs, datasets, key-value +stores, request queues, tasks, schedules, webhooks, the Store, users and logs. ## Features -- Resource-oriented API surface consistent with the reference clients. +- One client per resource, with the same method names as the [JavaScript client](https://github.com/apify/apify-client-js). - Transparent authentication, `User-Agent` header, retries with exponential backoff, and per-request timeouts applied to every call. - Replaceable HTTP transport (the `HTTPBackend` interface) with a default implementation. @@ -21,8 +19,8 @@ request queues, tasks, schedules, webhooks, the store, users and logs). and request-queue iterators, live iteration of a running Actor's dataset items, dataset export, signed public URLs, request-queue locking, requests to Server Actors, and more. - Forward-compatible models that keep unknown API fields in an `Extra` map. -- A single third-party dependency (`github.com/andybalholm/brotli`, used for Brotli - request-body compression); everything else is the Go standard library. +- Two third-party dependencies for request-body compression, `github.com/andybalholm/brotli` and + `github.com/klauspost/compress` (zstd). Everything else is the Go standard library. ## Installation @@ -158,23 +156,20 @@ Request bodies of 1 KiB or more are compressed before sending, to save upload ba default (`RequestCompressionAuto`) the client prefers brotli and falls back to gzip when brotli errors or does not shrink the body; `WithRequestCompression` with `RequestCompressionBrotli`, `RequestCompressionGzip` or `RequestCompressionZstd` pins it to one codec, with no fallback (a body that codec does not -shrink is sent uncompressed rather than failing). -Compression is also skipped outright for a body whose content type already carries its own -compression — images, audio, video, and common archive/office formats (e.g. `.zip`, `.docx`); -a raw format that happens to share such a media type, like `image/bmp`, is still compressed. +shrink is sent uncompressed rather than failing). A body whose content type is already +compressed, such as images, audio, video and archives, is not compressed, except for raw formats +like `image/bmp`. The `User-Agent` header reports an `isAtHome` flag indicating whether the client runs on the Apify platform. It is driven solely by the `APIFY_IS_AT_HOME` environment variable (the same -variable the JavaScript reference client reads); if it is set to a non-empty value, the flag is -`true`, otherwise `false`. +variable); if it is set to a non-empty value, the flag is `true`, otherwise `false`. ### URL fields The fields the API specification declares as URLs (`consoleUrl`, `containerUrl`, `itemsPublicUrl`, `keysPublicUrl`, `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, the webhook `requestUrl`, a run's `output` links, a Store Actor's `url` and `userPictureUrl`, and a user profile's -`pictureUrl` and `websiteUrl`) are returned in a normalized form, close to the WHATWG URL serialization that the JavaScript -client returns: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IPv4 or +`pictureUrl` and `websiteUrl`) are returned in a normalized form, close to the WHATWG URL serialization: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IPv4 or IPv6 host is written canonically, a default port is dropped, an empty path becomes `/`, dot segments are resolved, and unsafe characters are percent-encoded. It covers the http(s), ws(s), ftp and file URLs the API returns; it does not apply the full Unicode (UTS #46) mapping to a @@ -225,9 +220,7 @@ affected accessors. | `Path` | `string` | Request path of the endpoint (URL excluding origin). | It also exposes `IsInvalidRequest()`, `IsUnauthorized()`, `IsForbidden()`, `IsNotFound()`, -`IsConflict()`, `IsRateLimited()` and `IsServerError()` — status-classification predicates, the -idiomatic equivalent of the reference clients' `InvalidRequestError`/`NotFoundError`/... -subclasses, since Go has no exception hierarchy to mirror them with. +`IsConflict()`, `IsRateLimited()` and `IsServerError()`, which classify the status. ```go user, ok, err := client.Me().Get(ctx) @@ -244,10 +237,8 @@ if !ok { ### Cancellation -The reference JavaScript client accepts an `AbortSignal` option on most methods so a caller can -cancel an in-flight request. Every method here already takes a `context.Context`: cancel it (or -let a `context.WithTimeout`/`context.WithDeadline` elapse) to stop the underlying request the -same way, with no separate option needed. +Every method takes a `context.Context`. Cancel it, or let its timeout or deadline pass, to stop +the request. ## Custom HTTP transport @@ -296,20 +287,14 @@ func main() { ### Releasing -Go modules are distributed by pushing a semver git tag — there is no separate package registry to -upload to. The [`Publish Go client`](.github/workflows/go-publish.yml) workflow is the release -mechanism: a maintainer triggers it manually (`workflow_dispatch`), it runs the same quality gate -as CI, then creates and pushes the `v` tag, opens a GitHub release, and asks the Go -module proxy to index the new version so it appears on -[pkg.go.dev](https://pkg.go.dev/github.com/apify/apify-client-go). The tag is derived from -`ClientVersion` in [`version.go`](version.go), so bump that constant before releasing. The workflow -uses only the built-in `GITHUB_TOKEN`; no extra credentials are required. - -There is no "Trusted Publisher" step: that mechanism applies to registries that authenticate -uploads (e.g. PyPI, npm, crates.io). Go has no central registry and no upload step — a module is -published purely by pushing a git tag that the public module proxy reads — so there is no token or -trusted-publisher relationship to configure. The workflow therefore relies only on the repository's -built-in `GITHUB_TOKEN` to push the tag and open the release. +A Go module is published by pushing a semver git tag. A maintainer triggers the +[`Publish Go client`](.github/workflows/go-publish.yml) workflow manually. It runs the checks of CI, +pushes the `v` tag, opens a GitHub release and asks the Go module proxy to index +the version for [pkg.go.dev](https://pkg.go.dev/github.com/apify/apify-client-go). Bump +`ClientVersion` in [`version.go`](version.go) before releasing. + +Go has no registry to upload to, so there is no Trusted Publisher step, and the workflow needs only +the built-in `GITHUB_TOKEN`. ## Examples @@ -342,21 +327,14 @@ Per-resource guides are in [`docs/`](docs): ## Scope -The client implements only documented API endpoints. Matching the JavaScript reference (and -the Rust sibling) for cross-client consistency, the following documented endpoints are -**intentionally not implemented**: +The client implements only documented API endpoints. These documented endpoints are not +implemented: - Synchronous run endpoints (`run-sync`, `run-sync-get-dataset-items`). - The keyed-`POST` record aliases. -- Cryptographic tools: `POST /v2/tools/encode-and-sign` and `POST /v2/tools/decode-and-verify`. - These are server-side conveniences for the same HMAC signing this client already performs - locally in `signature.go`; the reference clients do not expose them, so the Go client omits - them too for parity. (Should a future requirement need them, they can be added alongside - `signature.go`.) -- `/v2/browser-info`. Not exposed by the reference clients; omitted for parity. - -This is a deliberate, parity-driven decision, not an accidental gap. See the CHANGELOG for the -same note. +- `POST /v2/tools/encode-and-sign` and `POST /v2/tools/decode-and-verify`. The client signs + public URLs locally. +- `/v2/browser-info`. ## License diff --git a/actor_env_var_test.go b/actor_env_var_test.go new file mode 100644 index 0000000..33eb018 --- /dev/null +++ b/actor_env_var_test.go @@ -0,0 +1,17 @@ +package apify + +import ( + "encoding/json" + "testing" +) + +// The API requires `value` on env var create/update, so an empty value must still be sent. +func TestActorEnvVarSendsEmptyValue(t *testing.T) { + body, err := json.Marshal(ActorEnvVar{Name: "EMPTY"}) + if err != nil { + t.Fatal(err) + } + if string(body) != `{"name":"EMPTY","value":""}` { + t.Fatalf("unexpected body: %s", body) + } +} diff --git a/actor_server.go b/actor_server.go index 3513cc0..6c993d0 100644 --- a/actor_server.go +++ b/actor_server.go @@ -310,10 +310,11 @@ var httpMethods = map[string]bool{ // before the Actor sees it. The Headers of the options reach the server as they are. // // The response is returned whatever its status, so check StatusCode. A 429 or 503 response is -// retried, and the last one is returned when the retries run out. A redirect is returned as it -// is, so the token never reaches another host. A transport error, a timeout included, is not -// retried, since the Actor may have handled the request. The first request to an idle Actor waits -// for a run to start, so the request uses the "long" timeout tier, which ctx can shorten. +// retried, and the last one is returned when the retries run out. The Actor's own code can return +// these statuses too, and then it gets the request again. A redirect is returned as it is, so the +// token never reaches another host. A transport error, a timeout included, is not retried, since +// the Actor may have handled the request. The first request to an idle Actor waits for a run to +// start, so the request uses the "long" timeout tier, which ctx can shorten. // // It returns an error only when there is no response: the errors of [ActorServerClient.GetURL], // invalid options, or a failed or timed-out request. To read the body incrementally, use diff --git a/actor_server_test.go b/actor_server_test.go index 9ca522a..314136a 100644 --- a/actor_server_test.go +++ b/actor_server_test.go @@ -463,8 +463,8 @@ func TestActorServerRequestCompression(t *testing.T) { t.Fatalf("expected a zstd body, got encoding %q: %v", r.header.Get("Content-Encoding"), err) } defer zr.Close() - plain, _ := io.ReadAll(zr) - if !bytes.Contains(plain, []byte(`"items"`)) { + plain, err := io.ReadAll(zr) + if err != nil || !bytes.Contains(plain, []byte(`"items"`)) { t.Fatalf("decompressed body: %q", plain) } }) diff --git a/base.go b/base.go index f669738..d5c6317 100644 --- a/base.go +++ b/base.go @@ -408,7 +408,7 @@ func streamRaw(ctx context.Context, c *resourceContext, subPath string, params * if err != nil { return nil, nil, err } - if resp.StatusCode >= maxSuccessStatusCode { + if resp.StatusCode >= minNonSuccessStatusCode { body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() apiErr := buildAPIError(resp.StatusCode, body, 1, http.MethodGet, extractPath(url)) diff --git a/client.go b/client.go index 6de4077..4c3f39c 100644 --- a/client.go +++ b/client.go @@ -1,5 +1,4 @@ -// Package apify is the Go client of the Apify API (https://docs.apify.com/api/v2). The README -// states that the client is experimental and generated and maintained by AI. +// Package apify is the Go client of the Apify API (https://docs.apify.com/api/v2). // // Start from an [ApifyClient] and go on to the resource clients: Actors, runs, datasets, // key-value stores, request queues, tasks, schedules, webhooks, the Store, users and logs. @@ -151,7 +150,7 @@ func WithUserAgentSuffix(suffix string) Option { // WithRequestCompression selects which codec compresses request bodies (default // [RequestCompressionAuto]: brotli, falling back to gzip). A caller that wants a specific // codec, with no fallback to another one, can pass [RequestCompressionBrotli], -// [RequestCompressionGzip] or [RequestCompressionZstd]. +// [RequestCompressionGzip], or [RequestCompressionZstd]. func WithRequestCompression(algo RequestCompression) Option { return func(c *clientConfig) { c.requestCompression = algo } } diff --git a/client_meta_test.go b/client_meta_test.go index b595e4a..72dd665 100644 --- a/client_meta_test.go +++ b/client_meta_test.go @@ -34,10 +34,8 @@ func TestUserAgentFormat(t *testing.T) { } } -// The User-Agent OS token must exactly match the value Node's os.platform() (used by the -// reference JS client) reports for the running platform. Go's runtime.GOOS spells a few -// platforms differently, so those are mapped: windows->win32, solaris/illumos->sunos, -// ios->darwin. Every other GOOS value already matches os.platform() and passes through unchanged. +// The OS token of the User-Agent must match what Node's os.platform() reports. Go spells a few +// platforms differently: windows is win32, solaris and illumos are sunos, and ios is darwin. func TestPlatformToken(t *testing.T) { cases := map[string]string{ // Go spellings that diverge from os.platform() and must be remapped. @@ -82,8 +80,7 @@ func TestUserAgentIsAtHomeFlag(t *testing.T) { } } -// defaultIsAtHome must be driven solely by APIFY_IS_AT_HOME (per the requirements and the JS -// reference); a bare isAtHome env var must NOT affect it. +// defaultIsAtHome reads APIFY_IS_AT_HOME only, not a bare isAtHome variable. func TestDefaultIsAtHomeReadsOnlyApifyIsAtHome(t *testing.T) { t.Setenv(envIsAtHome, "") t.Setenv("isAtHome", "") @@ -126,10 +123,8 @@ func TestBaseURLOverrideAppendsV2(t *testing.T) { } } -// WithBaseURL appends /v2 when missing, but does not double it when the caller already -// included it (e.g. a URL read back from APIBaseURL and passed to a new client) — mirrors the -// reference client's toApiBaseUrl test matrix exactly, so the two clients agree on every case, -// including the "https://v2" host-vs-path edge case. +// WithBaseURL appends /v2 when it is missing and does not double it, including for the +// "https://v2" edge case, where v2 is the host and not the path. func TestBaseURLAppendsV2Idempotently(t *testing.T) { cases := []struct { input string diff --git a/dataset_items_test.go b/dataset_items_test.go new file mode 100644 index 0000000..095aa7b --- /dev/null +++ b/dataset_items_test.go @@ -0,0 +1,79 @@ +package apify + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" +) + +// DatasetClient.CreateItemsPublicURLWithFormat adds a `format` query parameter, and +// CreateItemsPublicURL (unchanged) omits it, defaulting to the endpoint's own json default. +func TestCreateItemsPublicURLWithFormat(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":{"id":"ds1"}}`)} + client := testClient(backend, 0) + dataset := client.Dataset("ds1") + + url, err := dataset.CreateItemsPublicURLWithFormat(context.Background(), DatasetListItemsOptions{}, nil, FormatCSV) + if err != nil { + t.Fatalf("create items public url with format: %v", err) + } + if !strings.Contains(url, "format=csv") { + t.Fatalf("expected format=csv in url, got %q", url) + } + + url, err = dataset.CreateItemsPublicURL(context.Background(), DatasetListItemsOptions{}, nil) + if err != nil { + t.Fatalf("create items public url: %v", err) + } + if strings.Contains(url, "format=") { + t.Fatalf("CreateItemsPublicURL must not set a format param, url was: %q", url) + } +} + +// IterateDatasetItems must advance and stop by the number of rows scanned +// (X-Apify-Pagination-Count), not the number of items a page returned: a filter can make a page +// return fewer items than the rows it covered (even zero, on a page that still has more rows +// behind it), and the iterator must not stop early in that case. +func TestIterateDatasetItemsUsesScannedCount(t *testing.T) { + // pageHeaders builds the X-Apify-Pagination-* headers for a page that scanned `scanned` rows + // of a dataset of `total` rows, starting at `offset`, having requested up to `limit`. + pageHeaders := func(total, offset, limit, scanned int64) http.Header { + h := http.Header{} + h.Set("X-Apify-Pagination-Total", itoa(total)) + h.Set("X-Apify-Pagination-Offset", itoa(offset)) + h.Set("X-Apify-Pagination-Limit", itoa(limit)) + h.Set("X-Apify-Pagination-Count", itoa(scanned)) + return h + } + backend := &mockBackend{responses: []mockResponse{ + // Page 1: the API scanned 5 rows (offset 0-4) but a filter dropped all of them. + {status: 200, body: `[]`, headers: pageHeaders(10, 0, 5, 5)}, + // Page 2: scanned the remaining 5 rows (offset 5-9), all kept. + {status: 200, body: `[1,2,3,4,5]`, headers: pageHeaders(10, 5, 5, 5)}, + // Page 3: no Limit was set, so the iterator does not stop after page 2's 5 items; it + // needs this trailing empty page (scanned 0) to confirm there is nothing left. + {status: 200, body: `[]`, headers: pageHeaders(10, 10, 5, 0)}, + }} + client := testClient(backend, 0) + + it := client.Dataset("ds1").IterateItems(DatasetListItemsOptions{}, Ptr(int64(5))) + var items []json.RawMessage + for { + item, err := it.Next(context.Background()) + if err != nil { + t.Fatalf("iterate: %v", err) + } + if item == nil { + break + } + items = append(items, *item) + } + if len(items) != 5 { + t.Fatalf("expected 5 items across both non-empty pages, got %d", len(items)) + } + if backend.calls != 3 { + t.Fatalf("expected exactly 3 page fetches (including the trailing empty one), got %d", backend.calls) + } +} diff --git a/docs/server_actors.md b/docs/server_actors.md index 29b999c..d4d69bf 100644 --- a/docs/server_actors.md +++ b/docs/server_actors.md @@ -42,6 +42,31 @@ if err := response.JSON(&results); err != nil { fmt.Println(results) ``` +## Calling a Server Actor from a Server Actor + +A multi-tenant Server Actor shares its runs among its users, so the platform tells the requests +apart by the `X-Actor-Request-ID` header of each request it forwards to the Actor. When your +Server Actor calls another one while handling a request, pass that header in `ParentRequestID`. +The client sends it as the same header, so the called Actor charges at the pricing tier of your +incoming request. Use a client created with the run's `APIFY_TOKEN`. For details, see +[Compose multi-tenant Server Actors](https://docs.apify.com/platform/actors/development/programming-interface/standby#compose-server-actors). + +```go +handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response, err := client.Actor("username/other-actor").Server().Request(r.Context(), "/search", apify.ActorServerRequestOptions{ + Params: url.Values{"q": {"apify"}}, + ParentRequestID: r.Header.Get("X-Actor-Request-ID"), + }) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + w.WriteHeader(response.StatusCode) + _, _ = w.Write(response.Body) +}) +_ = handler +``` + ## Request options | Field | Type | Meaning | diff --git a/docs/storages.md b/docs/storages.md index 00bdac0..9c890fd 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -290,8 +290,9 @@ Single queue: `client.RequestQueue(id)`: | `ListRequests(ctx, ListRequestsOptions) (json.RawMessage, error)` | Paginated list. Returns the raw API response (`{ "items": [...], "nextCursor": ... }`); unmarshal into your own type, or use `PaginateRequests` for typed iteration. | | `PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator` | Lazy iterator over all requests (yields `*RequestQueueRequest`). | | `PaginateRequestsWithOptions(ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator` | The same with `Limit` (caps the total across pages), `Cursor` (where to start) and `Filter` (every page); `chunkSize` is the page size. | -| `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests, auto-split into batches that respect the API's 25-request and (effective) 9 MiB payload-size limits. Fails before sending anything if any single request is too large for a batch of its own. Unlike the reference client, does not itself retry a batch's `UnprocessedRequests` — inspect the result and retry those explicitly if needed. | -| `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload — a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`), matching the reference client's `batchDeleteRequests`. Returns the raw API response. | +| `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests with the default `BatchAddRequestsOptions`. See `BatchAddRequestsWithOptions`. | +| `BatchAddRequestsWithOptions(ctx, []RequestQueueRequest, BatchAddRequestsOptions) (BatchAddResult, error)` | Add many requests, split into batches within the API limits of 25 requests and 9 MiB, with up to `MaxParallel` (5) batches sent at once. The requests that the API does not process are sent again, up to `MaxUnprocessedRequestsRetries` (3) times, after a delay that starts at `MinDelayBetweenUnprocessedRequestsRetries` (500ms) and doubles. Fails before sending anything if one request is too large for a batch. If a batch fails with an error, the error is returned together with the result of all batches, in which the requests of that batch that were not added are unprocessed. | +| `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload — a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`). Returns the raw API response. | | `ListAndLockHead(ctx, lockSecs int64, limit *int64) (json.RawMessage, error)` | Fetch the head of the queue and lock the returned requests for `lockSecs` seconds. Returns the raw API response. | | `ProlongRequestLock(ctx, id string, lockSecs int64, forefront bool) (json.RawMessage, error)` | Extend the lock on a request by `lockSecs` seconds; `forefront` controls re-queue position when the lock expires. | | `DeleteRequestLock(ctx, id string, forefront bool) error` | Release the lock on a request without modifying it. | @@ -356,7 +357,7 @@ Return types: | Field | Type | Meaning | |---|---|---| | `ProcessedRequests` | `[]RequestQueueOperationInfo` | Requests the API successfully added. | -| `UnprocessedRequests` | `[]RequestQueueRequest` | Requests the API did not process. | +| `UnprocessedRequests` | `[]RequestQueueRequest` | Requests that are still unprocessed after the retries. | ```go rq, _ := client.RequestQueues().GetOrCreate(ctx, "") diff --git a/docs/tasks.md b/docs/tasks.md index ce63c70..d905c3a 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -71,9 +71,8 @@ if err != nil { run, err := client.Task(task.ID).Call(ctx, nil, apify.TaskStartOptions{}, apify.Ptr(int64(120))) ``` -`TaskStartOptions` mirrors `ActorStartOptions` (see [actors.md](actors.md)) but omits the -Actor-only `ContentType` and `ForcePermissionLevel`, which the task run endpoint does not -accept. +`TaskStartOptions` is `ActorStartOptions` (see [actors.md](actors.md)) without the `ContentType` +and `ForcePermissionLevel` fields, which the task run endpoint does not accept. `LastRun(status)` / `LastRunWithOptions(LastRunOptions{Status, Origin})` resolve the task's most recent run, optionally narrowed by status and/or origin. `LastRunOptions` is the same type used diff --git a/docs/webhooks.md b/docs/webhooks.md index 0a39be6..e2473c9 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -15,8 +15,7 @@ Webhooks notify external services when events occur. Access the webhook collecti An Actor's or task's webhooks are also listable via `client.Actor(id).Webhooks()` / `client.Task(id).Webhooks()`. -`Create` (and `Update`) take a free-form JSON object (`any`) that mirrors the webhook shape in -the Apify API, so it can carry any field the API accepts. The commonly used input keys are: +`Create` (and `Update`) take a JSON object (`any`) with the webhook fields of the Apify API. The commonly used input keys are: | Key | Type | Meaning | |---|---|---| diff --git a/errors_test.go b/errors_test.go new file mode 100644 index 0000000..cba4d97 --- /dev/null +++ b/errors_test.go @@ -0,0 +1,63 @@ +package apify + +import ( + "context" + "testing" +) + +// ApiError's Is* status-classification predicates match the response status, and only that +// status's predicate is true. +func TestAPIErrorStatusPredicates(t *testing.T) { + errorFor := func(status int) *APIError { + backend := &mockBackend{responses: constant(status, `{"error":{"type":"x","message":"m"}}`)} + client := testClient(backend, 0) + _, err := client.Me().MonthlyUsage(context.Background()) + apiErr, ok := AsAPIError(err) + if !ok { + t.Fatalf("expected an *APIError for status %d, got %v", status, err) + } + return apiErr + } + + notFoundErr := errorFor(404) + if !notFoundErr.IsNotFound() { + t.Error("expected IsNotFound() for a 404") + } + for _, predicate := range []struct { + name string + ok bool + }{ + {"IsInvalidRequest", notFoundErr.IsInvalidRequest()}, + {"IsUnauthorized", notFoundErr.IsUnauthorized()}, + {"IsForbidden", notFoundErr.IsForbidden()}, + {"IsConflict", notFoundErr.IsConflict()}, + {"IsRateLimited", notFoundErr.IsRateLimited()}, + {"IsServerError", notFoundErr.IsServerError()}, + } { + if predicate.ok { + t.Errorf("expected %s() to be false for a 404", predicate.name) + } + } + + if !errorFor(400).IsInvalidRequest() { + t.Error("expected IsInvalidRequest() for a 400") + } + if !errorFor(401).IsUnauthorized() { + t.Error("expected IsUnauthorized() for a 401") + } + if !errorFor(403).IsForbidden() { + t.Error("expected IsForbidden() for a 403") + } + if !errorFor(409).IsConflict() { + t.Error("expected IsConflict() for a 409") + } + if !errorFor(429).IsRateLimited() { + t.Error("expected IsRateLimited() for a 429") + } + if !errorFor(500).IsServerError() { + t.Error("expected IsServerError() for a 500") + } + if !errorFor(503).IsServerError() { + t.Error("expected IsServerError() for a 503") + } +} diff --git a/examples/create_build_run_actor/main.go b/examples/create_build_run_actor/main.go index ea1c5d9..6e01d00 100644 --- a/examples/create_build_run_actor/main.go +++ b/examples/create_build_run_actor/main.go @@ -23,8 +23,7 @@ func main() { ctx := context.Background() // Actor names are unique per account, so use a random suffix to stay collision-free when - // this example runs concurrently (e.g. in CI, or alongside the sibling clients' examples on - // the same test account). + // this example runs concurrently, for example in CI. var suffix [6]byte _, _ = rand.Read(suffix[:]) name := "go-example-" + hex.EncodeToString(suffix[:]) diff --git a/http_client.go b/http_client.go index da6fdb2..1631b51 100644 --- a/http_client.go +++ b/http_client.go @@ -22,8 +22,8 @@ const ( rateLimitExceededStatusCode = 429 // minServerErrorStatusCode is the lowest status that is retried as a server error. minServerErrorStatusCode = 500 - // maxSuccessStatusCode is the lowest status that is not a success. - maxSuccessStatusCode = 300 + // minNonSuccessStatusCode is the lowest status that is not a success. + minNonSuccessStatusCode = 300 // backoffFactor multiplies the delay between retries after each attempt. backoffFactor = 2 // minCompressRequestBytes is the smallest request body that is compressed. @@ -166,7 +166,7 @@ func (c *httpClient) doRetryLoop(ctx context.Context, method, url string, sendBo if err != nil { lastErr = err retryable = isTransportErrorRetryable(err) && (!c.doNotRetryTimeouts || !isTimeoutError(err)) - } else if resp.statusCode < maxSuccessStatusCode { + } else if resp.statusCode < minNonSuccessStatusCode { return resp, nil } else { lastErr = buildAPIError(resp.statusCode, resp.body, attempt, method, path) @@ -281,7 +281,7 @@ const ( RequestCompressionBrotli // RequestCompressionGzip uses gzip only. A body it does not shrink is sent uncompressed. RequestCompressionGzip - // RequestCompressionZstd uses zstd at its default level only. A body it does not shrink is + // RequestCompressionZstd uses zstd only, at its default level. A body it does not shrink is // sent uncompressed. RequestCompressionZstd ) @@ -327,13 +327,22 @@ func gzipCompress(body []byte) ([]byte, error) { return compressBody(body, func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }) } +// zstdEncoder compresses every zstd request body. A nil writer makes it a stateless encoder that is +// safe for concurrent use, which is cheaper than a new encoder for each request. +var zstdEncoder = newZstdEncoder() + +func newZstdEncoder() *zstd.Encoder { + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + if err != nil { + // Only invalid options fail, and these are constant. + panic(err) + } + return encoder +} + // zstdCompress compresses body with zstd (Content-Encoding: zstd). func zstdCompress(body []byte) ([]byte, error) { - return compressBody(body, func(w io.Writer) io.WriteCloser { - // NewWriter fails only for invalid options, and none are passed. - encoder, _ := zstd.NewWriter(w) - return encoder - }) + return zstdEncoder.EncodeAll(body, nil), nil } // maybeCompressRequestBody compresses body if it is at least [minCompressRequestBytes] long and @@ -457,7 +466,7 @@ func (c *httpClient) attemptTimeout(base time.Duration, attempt int) time.Durati } scaled := base for i := 1; i < attempt; i++ { - scaled *= 2 + scaled *= backoffFactor if scaled >= c.retry.timeoutMax { return c.retry.timeoutMax } diff --git a/iterator.go b/iterator.go index 9cf7812..7aa9ca0 100644 --- a/iterator.go +++ b/iterator.go @@ -6,9 +6,10 @@ import "context" // from the Iterate method of a collection client, and call Next until it returns (nil, nil). // // The Limit of the list options caps the total number of items yielded, and nil or 0 means all of -// them. The chunkSize argument of Iterate is the page size, which defaults to the page size of the -// API, so iterating dataset items without a chunkSize or Limit loads the whole dataset at once. -// The Offset of the options is the starting point. +// them. The chunkSize argument of Iterate is the page size. If it is unset, the page size is the +// default of the API, capped by the remaining Limit. Dataset items have no default limit, so +// iterating them without a chunkSize or Limit loads the whole dataset in one response. The Offset +// of the options is where the iteration starts. type ListIterator[T any] struct { // fetch fetches one page starting at offset. limit is the per-page limit to request // (0 means "unset", i.e. let the server choose). The collection's Iterate method bakes the @@ -134,8 +135,8 @@ func scannedRows[T any](page PaginationList[T], requestedLimit int64) int64 { } // remainingLimitParam returns the number of items still allowed by the caller's Limit, for use -// as a minForLimitParam input (0 meaning "unset"). Only meaningful once hasLimit is set; loadPage -// never calls it otherwise. +// minForLimitParam returns the smaller of a and b, ignoring a value of 0 or below, which means +// unset. It returns 0 when neither is set. func (it *ListIterator[T]) remainingLimitParam() int64 { if !it.hasLimit { return 0 diff --git a/iterator_test.go b/iterator_test.go index 1745a64..6f484ce 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -158,8 +158,7 @@ func TestListIteratorEmptyCollection(t *testing.T) { } func TestListIteratorHonorsStartOffset(t *testing.T) { - // Caller-set Offset=4 on a backing collection of 10, page size 3: iteration must start at - // item 4 and yield 4..9 (6 items), mirroring the reference's options.offset start point. + // An Offset of 4 on a collection of 10 with a page size of 3 starts at item 4 and yields 4..9. var calls [][2]int64 it := newListIterator(nil, ptrInt64(3), 4, stubFetcher(10, 1000, &calls)) got := drainInts(t, it) diff --git a/key_value_store_test.go b/key_value_store_test.go index cc3a3a4..0b27bd6 100644 --- a/key_value_store_test.go +++ b/key_value_store_test.go @@ -29,3 +29,17 @@ func TestSetRecordRawWithOptionsSendsTheContentEncoding(t *testing.T) { t.Fatal("without a ContentEncoding the client compresses a large value of its own accord") } } + +// GetRecord requests the record as an attachment by default. +func TestGetRecordDefaultsAttachment(t *testing.T) { + backend := &mockBackend{responses: constant(200, "raw-bytes")} + client := testClient(backend, 0) + + _, _, err := client.KeyValueStore("store1").GetRecord(context.Background(), "OUTPUT") + if err != nil { + t.Fatalf("get record: %v", err) + } + if !strings.Contains(backend.lastURL, "attachment=1") { + t.Fatalf("expected attachment=1 in URL, got %q", backend.lastURL) + } +} diff --git a/models.go b/models.go index d770a54..1bd7303 100644 --- a/models.go +++ b/models.go @@ -416,6 +416,8 @@ func (r *RequestQueueRequest) UnmarshalJSON(data []byte) error { type RequestQueueOperationInfo struct { // RequestID is the ID of the affected request. RequestID string `json:"requestId"` + // UniqueKey is the unique key of the affected request. + UniqueKey string `json:"uniqueKey"` // WasAlreadyPresent reports whether the request was already in the queue. WasAlreadyPresent bool `json:"wasAlreadyPresent"` // WasAlreadyHandled reports whether the request had already been handled. diff --git a/not_found_test.go b/not_found_test.go new file mode 100644 index 0000000..915181f --- /dev/null +++ b/not_found_test.go @@ -0,0 +1,88 @@ +package apify + +import ( + "context" + "testing" +) + +// Get and Delete on a resource with its own ID treat a 404 of any `type` as absent. +func TestGetSwallowsAnyNotFoundType(t *testing.T) { + backend := &mockBackend{responses: constant(404, `{"error":{"type":"some-other-error","message":"gone"}}`)} + client := testClient(backend, 0) + + _, present, err := client.Actor("missing").Get(context.Background()) + if err != nil { + t.Fatalf("expected a 404 of any type to resolve to absent, got error: %v", err) + } + if present { + t.Fatal("expected present=false for a 404 response") + } +} + +// Get/Delete on a client reached through a fixed sub-path with no id of its own (e.g. a run's +// default dataset) must NOT swallow a 404: the response cannot tell the run apart from the +// dataset as what is missing. +func TestNestedResourceGetPropagates404(t *testing.T) { + backend := &mockBackend{responses: notFound()} + client := testClient(backend, 0) + + _, _, err := client.Run("run1").Dataset().Get(context.Background()) + if err == nil { + t.Fatal("expected a 404 on a nested dataset client to propagate as an error") + } + apiErr, ok := AsAPIError(err) + if !ok || apiErr.StatusCode != 404 { + t.Fatalf("expected a 404 *APIError, got %v", err) + } + + err = client.Run("run1").Dataset().Delete(context.Background()) + if err == nil { + t.Fatal("expected a 404 on a nested dataset client's Delete to propagate as an error") + } +} + +// GetRecord/RecordExists on a nested key-value store (no id of its own) must still swallow a +// 404: unlike the store's own Get/Delete, a record lookup by key is never ambiguous. +func TestNestedKeyValueStoreRecordLookupStillSwallows404(t *testing.T) { + backend := &mockBackend{responses: notFound()} + client := testClient(backend, 0) + store := client.Run("run1").KeyValueStore() + + record, present, err := store.GetRecord(context.Background(), "OUTPUT") + if err != nil { + t.Fatalf("expected a missing record to resolve to absent, got error: %v", err) + } + if present || record != nil { + t.Fatalf("expected present=false and a nil record, got present=%v record=%v", present, record) + } + + exists, err := store.RecordExists(context.Background(), "OUTPUT") + if err != nil { + t.Fatalf("expected RecordExists to resolve false, got error: %v", err) + } + if exists { + t.Fatal("expected exists=false for a 404 response") + } +} + +// DatasetClient.GetStatistics, TaskClient.GetInput and ScheduleClient.GetLog must propagate a +// 404 as an error rather than an "absent" result: it always means the parent resource is gone. +func TestSingletonSubResourcesPropagate404(t *testing.T) { + cases := []struct { + name string + call func(c *ApifyClient) error + }{ + {"dataset statistics", func(c *ApifyClient) error { _, err := c.Dataset("ds1").GetStatistics(context.Background()); return err }}, + {"task input", func(c *ApifyClient) error { _, err := c.Task("task1").GetInput(context.Background()); return err }}, + {"schedule log", func(c *ApifyClient) error { _, err := c.Schedule("sch1").GetLog(context.Background()); return err }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &mockBackend{responses: notFound()} + client := testClient(backend, 0) + if err := c.call(client); err == nil { + t.Fatal("expected a 404 to propagate as an error") + } + }) + } +} diff --git a/path_validation_test.go b/path_validation_test.go new file mode 100644 index 0000000..a753af8 --- /dev/null +++ b/path_validation_test.go @@ -0,0 +1,60 @@ +package apify + +import ( + "context" + "testing" +) + +// A resource id, record key or request id that is a dot segment (".." or ".") must be rejected +// before any request is sent: a URL parser or proxy can resolve it after the path is built, +// letting it escape to a different endpoint (path traversal). +func TestPathTraversalRejected(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + + cases := []struct { + name string + call func() error + }{ + {"resource id", func() error { _, _, err := client.Actor("..").Get(context.Background()); return err }}, + {"record key", func() error { + _, _, err := client.KeyValueStore("store1").GetRecord(context.Background(), "..") + return err + }}, + {"request id", func() error { + _, _, err := client.RequestQueue("q1").GetRequest(context.Background(), "..") + return err + }}, + {"prolong request lock", func() error { + _, err := client.RequestQueue("q1").ProlongRequestLock(context.Background(), "..", 30, false) + return err + }}, + {"delete request lock", func() error { + return client.RequestQueue("q1").DeleteRequestLock(context.Background(), "..", false) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if err := c.call(); err == nil { + t.Fatal("expected an error for a dot-segment id, got nil") + } + }) + } + if backend.calls != 0 { + t.Fatalf("a dot-segment id must be rejected before any API call, got %d calls", backend.calls) + } +} + +// An empty resource id must be rejected the same way as a dot segment, not silently read as +// "the whole collection" (e.g. an empty Actor version number). +func TestEmptyIDRejected(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + + if _, _, err := client.Actor("some-actor").Version("").Get(context.Background()); err == nil { + t.Fatal("expected an error for an empty version number, got nil") + } + if backend.calls != 0 { + t.Fatalf("an empty id must be rejected before any API call, got %d calls", backend.calls) + } +} diff --git a/request_queue.go b/request_queue.go index a122ebd..0e54b5e 100644 --- a/request_queue.go +++ b/request_queue.go @@ -6,7 +6,11 @@ import ( "encoding/json" "errors" "fmt" + "math" + "math/rand" "net/http" + "sync" + "time" ) // ListRequestsOptions configures [RequestQueueClient.ListRequests]. @@ -252,28 +256,186 @@ func joinAsJSONArray(batch []serializedRequest) []byte { return buf.Bytes() } -// BatchAddRequests adds requests to the queue, at the front if forefront is true. It splits them -// into batches within the API limits of 25 requests and 9 MiB, and merges the results. It fails -// before sending anything if one request is too large for a batch. Unlike the reference client, -// it does not retry unprocessed requests, so check [BatchAddResult.UnprocessedRequests]. +// Defaults of [BatchAddRequestsOptions]. +const ( + // defaultMaxUnprocessedRequestsRetries is the default number of retries of the unprocessed + // requests of a batch. + defaultMaxUnprocessedRequestsRetries = 3 + // defaultMaxParallelBatches is the default number of batches that are sent at once. + defaultMaxParallelBatches = 5 + // defaultMinDelayBetweenUnprocessedRequestsRetries is the default base delay between retries. + defaultMinDelayBetweenUnprocessedRequestsRetries = 500 * time.Millisecond +) + +// BatchAddRequestsOptions configures [RequestQueueClient.BatchAddRequestsWithOptions]. A nil +// field has its default value. +type BatchAddRequestsOptions struct { + // Forefront adds the requests to the front of the queue. + Forefront bool + // MaxUnprocessedRequestsRetries is how many times the requests that the API did not process + // are sent again. Defaults to 3. + MaxUnprocessedRequestsRetries *int64 + // MaxParallel is how many batches are sent at once. Defaults to 5. + MaxParallel *int64 + // MinDelayBetweenUnprocessedRequestsRetries is the base delay before a retry. It doubles with + // each retry and is randomized by up to 100%. Defaults to 500ms. + MinDelayBetweenUnprocessedRequestsRetries *time.Duration +} + +// batchAddSettings are the resolved [BatchAddRequestsOptions]. +type batchAddSettings struct { + forefront bool + maxRetries int + maxParallel int + minDelay time.Duration +} + +// resolve fills in the defaults and rejects values that the API cannot make sense of. +func (o BatchAddRequestsOptions) resolve() (batchAddSettings, error) { + settings := batchAddSettings{ + forefront: o.Forefront, + maxRetries: defaultMaxUnprocessedRequestsRetries, + maxParallel: defaultMaxParallelBatches, + minDelay: defaultMinDelayBetweenUnprocessedRequestsRetries, + } + if o.MaxUnprocessedRequestsRetries != nil { + if *o.MaxUnprocessedRequestsRetries < 0 { + return settings, errors.New("BatchAddRequestsOptions.MaxUnprocessedRequestsRetries must not be negative") + } + settings.maxRetries = int(*o.MaxUnprocessedRequestsRetries) + } + if o.MaxParallel != nil { + if *o.MaxParallel < 1 { + return settings, errors.New("BatchAddRequestsOptions.MaxParallel must be at least 1") + } + settings.maxParallel = int(*o.MaxParallel) + } + if o.MinDelayBetweenUnprocessedRequestsRetries != nil { + if *o.MinDelayBetweenUnprocessedRequestsRetries < 0 { + return settings, errors.New("BatchAddRequestsOptions.MinDelayBetweenUnprocessedRequestsRetries must not be negative") + } + settings.minDelay = *o.MinDelayBetweenUnprocessedRequestsRetries + } + return settings, nil +} + +// BatchAddRequests adds requests to the queue, at the front if forefront is true, with the default +// [BatchAddRequestsOptions]. See [RequestQueueClient.BatchAddRequestsWithOptions]. func (c *RequestQueueClient) BatchAddRequests(ctx context.Context, requests []RequestQueueRequest, forefront bool) (BatchAddResult, error) { + return c.BatchAddRequestsWithOptions(ctx, requests, BatchAddRequestsOptions{Forefront: forefront}) +} + +// BatchAddRequestsWithOptions adds requests to the queue. It splits them into batches within the +// API limits of 25 requests and 9 MiB, sends up to MaxParallel batches at once, and sends the +// requests that the API did not process again, up to MaxUnprocessedRequestsRetries times. It fails +// before sending anything if one request is too large for a batch. +// +// The result lists the requests that were added and those that are still unprocessed. When a batch +// fails with an error, the error is returned with the result of all batches, in which the requests +// of the failed batch that were not added are unprocessed. When ctx ends, the error of ctx is +// returned. +func (c *RequestQueueClient) BatchAddRequestsWithOptions(ctx context.Context, requests []RequestQueueRequest, options BatchAddRequestsOptions) (BatchAddResult, error) { var merged BatchAddResult + settings, err := options.resolve() + if err != nil { + return merged, err + } serialized, err := serializeRequestsForBatch(requests) if err != nil { return merged, err } - for _, batch := range splitIntoBatches(serialized, maxRequestsPerBatchOperation, payloadSizeLimitBytes) { - chunkResult, err := c.batchAddChunk(ctx, batch, forefront) + batches := splitIntoBatches(serialized, maxRequestsPerBatchOperation, payloadSizeLimitBytes) + + results := make([]batchOutcome, len(batches)) + slots := make(chan struct{}, settings.maxParallel) + var wg sync.WaitGroup + for i, batch := range batches { + slots <- struct{}{} + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-slots }() + results[i] = c.addBatchWithRetries(ctx, batch, settings) + }() + } + wg.Wait() + + var firstErr error + for _, outcome := range results { + merged.ProcessedRequests = append(merged.ProcessedRequests, outcome.result.ProcessedRequests...) + merged.UnprocessedRequests = append(merged.UnprocessedRequests, outcome.result.UnprocessedRequests...) + if firstErr == nil { + firstErr = outcome.err + } + } + return merged, firstErr +} + +// batchOutcome is the result of one batch and the error that ended it early, if any. +type batchOutcome struct { + result BatchAddResult + err error +} + +// addBatchWithRetries sends batch and then the requests that the API did not process, until all +// are added or the retries are used up. +func (c *RequestQueueClient) addBatchWithRetries(ctx context.Context, batch []serializedRequest, settings batchAddSettings) batchOutcome { + var outcome batchOutcome + added := map[string]bool{} + remaining := batch + for attempt := 0; attempt <= settings.maxRetries; attempt++ { + response, err := c.batchAddChunk(ctx, remaining, settings.forefront) if err != nil { - return merged, err + outcome.err = err + outcome.result.UnprocessedRequests = unprocessedOf(remaining) + return outcome } - merged.ProcessedRequests = append(merged.ProcessedRequests, chunkResult.ProcessedRequests...) - merged.UnprocessedRequests = append(merged.UnprocessedRequests, chunkResult.UnprocessedRequests...) + outcome.result.ProcessedRequests = append(outcome.result.ProcessedRequests, response.ProcessedRequests...) + outcome.result.UnprocessedRequests = response.UnprocessedRequests + for _, processed := range response.ProcessedRequests { + added[processed.UniqueKey] = true + } + remaining = withoutAdded(batch, added) + if len(remaining) == 0 || attempt == settings.maxRetries { + return outcome + } + if !sleepWithContext(ctx, unprocessedRetryDelay(settings.minDelay, attempt)) { + outcome.err = ctx.Err() + outcome.result.UnprocessedRequests = unprocessedOf(remaining) + return outcome + } + } + return outcome +} + +// unprocessedRetryDelay returns the delay before the retry that follows attempt: the base delay +// doubled for each earlier retry, times a random factor in [1, 2). +func unprocessedRetryDelay(minDelay time.Duration, attempt int) time.Duration { + return time.Duration((1 + rand.Float64()) * float64(minDelay) * math.Pow(backoffFactor, float64(attempt))) +} + +// withoutAdded returns the requests of batch whose unique key is not in added. +func withoutAdded(batch []serializedRequest, added map[string]bool) []serializedRequest { + var remaining []serializedRequest + for _, s := range batch { + if !added[s.original.UniqueKey] { + remaining = append(remaining, s) + } + } + return remaining +} + +// unprocessedOf returns the unprocessed form of requests, which has only a method, a unique key +// and a URL. +func unprocessedOf(requests []serializedRequest) []RequestQueueRequest { + unprocessed := make([]RequestQueueRequest, len(requests)) + for i, s := range requests { + unprocessed[i] = RequestQueueRequest{Method: s.original.Method, UniqueKey: s.original.UniqueKey, URL: s.original.URL} } - return merged, nil + return unprocessed } -// batchAddChunk sends a single, pre-serialized and pre-sized batch and parses the typed result. +// batchAddChunk sends one batch and parses the result. func (c *RequestQueueClient) batchAddChunk(ctx context.Context, batch []serializedRequest, forefront bool) (BatchAddResult, error) { params := NewQueryParams() params.AddBool("forefront", &forefront) diff --git a/request_queue_batch_test.go b/request_queue_batch_test.go new file mode 100644 index 0000000..2734180 --- /dev/null +++ b/request_queue_batch_test.go @@ -0,0 +1,281 @@ +package apify + +import ( + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" +) + +// batchBackend answers batch-add requests like the API: it adds every request except the ones +// that reject says to leave unprocessed. +type batchBackend struct { + mu sync.Mutex + calls int + bodies [][]RequestQueueRequest + inflight atomic.Int64 + peak atomic.Int64 + // reject returns the unique keys that the call with the given index leaves unprocessed. + reject func(call int, requests []RequestQueueRequest) map[string]bool + // fail returns the error of the call with the given index, or nil. + fail func(call int) error + // delay is how long each call takes. + delay time.Duration +} + +func (b *batchBackend) Do(req *http.Request) (*http.Response, error) { + data := decodeBody(req) + var requests []RequestQueueRequest + if err := json.Unmarshal(data, &requests); err != nil { + return nil, err + } + current := b.inflight.Add(1) + defer b.inflight.Add(-1) + for peak := b.peak.Load(); current > peak && !b.peak.CompareAndSwap(peak, current); peak = b.peak.Load() { + } + if b.delay > 0 { + time.Sleep(b.delay) + } + + b.mu.Lock() + call := b.calls + b.calls++ + b.bodies = append(b.bodies, requests) + b.mu.Unlock() + + if b.fail != nil { + if err := b.fail(call); err != nil { + return nil, err + } + } + var rejected map[string]bool + if b.reject != nil { + rejected = b.reject(call, requests) + } + processed, unprocessed := []map[string]any{}, []map[string]any{} + for _, r := range requests { + if rejected[r.UniqueKey] { + unprocessed = append(unprocessed, map[string]any{"uniqueKey": r.UniqueKey, "url": r.URL, "method": r.Method}) + } else { + processed = append(processed, map[string]any{"requestId": "id-" + r.UniqueKey, "uniqueKey": r.UniqueKey}) + } + } + body, _ := json.Marshal(map[string]any{"data": map[string]any{"processedRequests": processed, "unprocessedRequests": unprocessed}}) + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(string(body)))}, nil +} + +// decodeBody reads the body of req, which the client may have compressed. +func decodeBody(req *http.Request) []byte { + var reader io.Reader = req.Body + switch req.Header.Get("Content-Encoding") { + case "br": + reader = brotli.NewReader(req.Body) + case "gzip": + reader, _ = gzip.NewReader(req.Body) + case "zstd": + reader, _ = zstd.NewReader(req.Body) + } + data, _ := io.ReadAll(reader) + return data +} + +func uniqueRequests(n int) []RequestQueueRequest { + requests := make([]RequestQueueRequest, n) + for i := range requests { + key := "key-" + string(rune('a'+i/26)) + string(rune('a'+i%26)) + requests[i] = RequestQueueRequest{URL: "https://example.com/" + key, UniqueKey: key, Method: "GET"} + } + return requests +} + +// fastBatchOptions retries without a noticeable delay. +func fastBatchOptions() BatchAddRequestsOptions { + return BatchAddRequestsOptions{MinDelayBetweenUnprocessedRequestsRetries: Ptr(time.Millisecond)} +} + +func TestBatchAddRequestsSplitsInputIntoBatchesOfTheAPILimit(t *testing.T) { + backend := &batchBackend{} + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequests(context.Background(), uniqueRequests(60), false) + if err != nil { + t.Fatal(err) + } + if backend.calls != 3 || len(result.ProcessedRequests) != 60 || len(result.UnprocessedRequests) != 0 { + t.Fatalf("calls %d, processed %d, unprocessed %d", backend.calls, len(result.ProcessedRequests), len(result.UnprocessedRequests)) + } +} + +func TestBatchAddRequestsRetriesTheUnprocessedRequestsOnly(t *testing.T) { + backend := &batchBackend{reject: func(call int, requests []RequestQueueRequest) map[string]bool { + if call == 0 { + return map[string]bool{"key-aa": true, "key-ab": true} + } + return nil + }} + options := fastBatchOptions() + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(5), options) + if err != nil { + t.Fatal(err) + } + if backend.calls != 2 { + t.Fatalf("expected one retry, got %d calls", backend.calls) + } + if retried := backend.bodies[1]; len(retried) != 2 || retried[0].UniqueKey != "key-aa" || retried[1].UniqueKey != "key-ab" { + t.Fatalf("the retry must send the unprocessed requests only, got %+v", retried) + } + if len(result.ProcessedRequests) != 5 || len(result.UnprocessedRequests) != 0 { + t.Fatalf("processed %d, unprocessed %d", len(result.ProcessedRequests), len(result.UnprocessedRequests)) + } +} + +func TestBatchAddRequestsReportsWhatIsStillUnprocessedWhenTheRetriesRunOut(t *testing.T) { + backend := &batchBackend{reject: func(call int, requests []RequestQueueRequest) map[string]bool { + return map[string]bool{"key-aa": true} + }} + options := fastBatchOptions() + options.MaxUnprocessedRequestsRetries = Ptr(int64(2)) + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(3), options) + if err != nil { + t.Fatal(err) + } + if backend.calls != 3 { + t.Fatalf("expected the first attempt and 2 retries, got %d calls", backend.calls) + } + if len(result.ProcessedRequests) != 2 || len(result.UnprocessedRequests) != 1 || result.UnprocessedRequests[0].UniqueKey != "key-aa" { + t.Fatalf("result: %+v", result) + } +} + +func TestBatchAddRequestsDoesNotRetryWithZeroRetries(t *testing.T) { + backend := &batchBackend{reject: func(int, []RequestQueueRequest) map[string]bool { return map[string]bool{"key-aa": true} }} + options := fastBatchOptions() + options.MaxUnprocessedRequestsRetries = Ptr(int64(0)) + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(3), options) + if err != nil || backend.calls != 1 || len(result.UnprocessedRequests) != 1 { + t.Fatalf("calls %d, unprocessed %d, err %v", backend.calls, len(result.UnprocessedRequests), err) + } +} + +func TestBatchAddRequestsSendsAtMostMaxParallelBatchesAtOnce(t *testing.T) { + backend := &batchBackend{delay: 20 * time.Millisecond} + options := BatchAddRequestsOptions{MaxParallel: Ptr(int64(2))} + if _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(125), options); err != nil { + t.Fatal(err) + } + if backend.calls != 5 || backend.peak.Load() != 2 { + t.Fatalf("calls %d, peak parallelism %d", backend.calls, backend.peak.Load()) + } +} + +func TestBatchAddRequestsReturnsTheErrorWithTheResultOfTheOtherBatches(t *testing.T) { + failure := errors.New("connection refused") + backend := &batchBackend{fail: func(call int) error { + if call == 0 { + return failure + } + return nil + }} + options := BatchAddRequestsOptions{MaxParallel: Ptr(int64(1))} + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(30), options) + if !errors.Is(err, failure) { + t.Fatalf("err = %v", err) + } + if len(result.UnprocessedRequests) != 25 || len(result.ProcessedRequests) != 5 { + t.Fatalf("the failed batch must be unprocessed and the other one added: processed %d, unprocessed %d", + len(result.ProcessedRequests), len(result.UnprocessedRequests)) + } + if first := result.UnprocessedRequests[0]; first.UniqueKey != "key-aa" || first.URL == "" || first.Method != "GET" { + t.Fatalf("an unprocessed request keeps its method, unique key and URL: %+v", first) + } +} + +func TestBatchAddRequestsStopsRetryingWhenTheContextEnds(t *testing.T) { + backend := &batchBackend{reject: func(int, []RequestQueueRequest) map[string]bool { return map[string]bool{"key-aa": true} }} + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + options := BatchAddRequestsOptions{MinDelayBetweenUnprocessedRequestsRetries: Ptr(time.Hour)} + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(ctx, uniqueRequests(2), options) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v", err) + } + if len(result.ProcessedRequests) != 1 || len(result.UnprocessedRequests) != 1 { + t.Fatalf("processed %d, unprocessed %d", len(result.ProcessedRequests), len(result.UnprocessedRequests)) + } +} + +func TestBatchAddRequestsRejectsInvalidOptionsBeforeSendingAnything(t *testing.T) { + cases := map[string]BatchAddRequestsOptions{ + "negative retries": {MaxUnprocessedRequestsRetries: Ptr(int64(-1))}, + "zero parallelism": {MaxParallel: Ptr(int64(0))}, + "negative delay": {MinDelayBetweenUnprocessedRequestsRetries: Ptr(-time.Second)}, + } + for name, options := range cases { + t.Run(name, func(t *testing.T) { + backend := &batchBackend{} + if _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(1), options); err == nil || backend.calls != 0 { + t.Fatalf("err = %v, calls = %d", err, backend.calls) + } + }) + } +} + +func TestBatchAddRequestsSendsForefront(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":{"processedRequests":[],"unprocessedRequests":[]}}`)} + options := BatchAddRequestsOptions{Forefront: true, MaxUnprocessedRequestsRetries: Ptr(int64(0))} + if _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(1), options); err != nil { + t.Fatal(err) + } + if !strings.Contains(backend.lastURL, "forefront=1") { + t.Fatalf("URL: %s", backend.lastURL) + } +} + +func TestBatchAddRequestsRejectsAnOversizedRequestBeforeSendingAnything(t *testing.T) { + backend := &batchBackend{} + requests := uniqueRequests(30) + // A URL this long exceeds payloadSizeLimitBytes, so the request cannot fit a batch of its own. + requests[29].URL = "https://example.com/" + strings.Repeat("a", payloadSizeLimitBytes) + + _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequests(context.Background(), requests, false) + if err == nil || !strings.Contains(err.Error(), "index 29") { + t.Fatalf("expected an error naming index 29, got %v", err) + } + if backend.calls != 0 { + t.Fatalf("expected no batch to be sent, got %d calls", backend.calls) + } +} + +func TestBatchAddRequestsSplitsABatchThatExceedsTheByteLimit(t *testing.T) { + backend := &batchBackend{} + big := strings.Repeat("a", payloadSizeLimitBytes/2) + requests := []RequestQueueRequest{ + {URL: "https://example.com/" + big, UniqueKey: "one"}, + {URL: "https://example.com/" + big, UniqueKey: "two"}, + } + if _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequests(context.Background(), requests, false); err != nil { + t.Fatal(err) + } + if backend.calls != 2 { + t.Fatalf("expected 2 batches, got %d calls", backend.calls) + } +} + +func TestBatchAddRequestsSendsEachBatchAsOneJSONArray(t *testing.T) { + backend := &batchBackend{} + requests := uniqueRequests(2) + if _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequests(context.Background(), requests, false); err != nil { + t.Fatal(err) + } + if got := backend.bodies[0]; len(got) != 2 || got[0].URL != requests[0].URL || got[1].URL != requests[1].URL { + t.Fatalf("batch body: %+v", got) + } +} diff --git a/request_queue_test.go b/request_queue_test.go new file mode 100644 index 0000000..eade27e --- /dev/null +++ b/request_queue_test.go @@ -0,0 +1,64 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +// ListRequests must reject mutually-exclusive options and an invalid Filter before any API call. +func TestListRequestsValidation(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + queue := client.RequestQueue("q1") + + start, cursor := "a", "b" + if _, err := queue.ListRequests(context.Background(), ListRequestsOptions{ExclusiveStartID: &start, Cursor: &cursor}); err == nil { + t.Fatal("expected error for mutually-exclusive ExclusiveStartID and Cursor") + } + if _, err := queue.ListRequests(context.Background(), ListRequestsOptions{Filter: []string{"bogus"}}); err == nil { + t.Fatal("expected error for invalid Filter value") + } + if backend.calls != 0 { + t.Fatalf("validation must short-circuit before any API call, got %d calls", backend.calls) + } +} + +// Every RequestQueueClient method that sends a request — including the ones that bypass the +// shared base.go CRUD helpers (ProlongRequestLock, DeleteRequestLock) — must fail fast on a +// queue built with an invalid id, instead of sending a request built from it. +func TestRequestQueueLockMethodsFailFastOnInvalidQueueID(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + queue := client.RequestQueue("..") + + if _, err := queue.ProlongRequestLock(context.Background(), "req1", 30, false); err == nil { + t.Fatal("expected ProlongRequestLock to fail fast on an invalid queue id") + } + if err := queue.DeleteRequestLock(context.Background(), "req1", false); err == nil { + t.Fatal("expected DeleteRequestLock to fail fast on an invalid queue id") + } + if backend.calls != 0 { + t.Fatalf("an invalid queue id must be rejected before any API call, got %d calls", backend.calls) + } +} + +// ListRequests sends a Filter with several values as one comma-joined query parameter. +func TestListRequestsFilterCommaJoined(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + queue := client.RequestQueue("q1") + + if _, err := queue.ListRequests(context.Background(), ListRequestsOptions{ + Filter: []string{RequestFilterLocked, RequestFilterPending}, + }); err != nil { + t.Fatalf("ListRequests with multi-value Filter: %v", err) + } + if backend.calls != 1 { + t.Fatalf("expected exactly one API call, got %d", backend.calls) + } + gotURL := backend.lastURL + if !strings.Contains(gotURL, "filter=locked%2Cpending") && !strings.Contains(gotURL, "filter=locked,pending") { + t.Fatalf("expected filter sent comma-joined as a single param, got URL: %s", gotURL) + } +} diff --git a/review_fixes_test.go b/review_fixes_test.go deleted file mode 100644 index 9981a0d..0000000 --- a/review_fixes_test.go +++ /dev/null @@ -1,836 +0,0 @@ -package apify - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "strings" - "testing" - "time" -) - -// okData is a constant 200 response with an empty data envelope, reused by the offline -// tests below. -func okData() []mockResponse { return constant(200, `{"data":{}}`) } - -// Charge must always send an idempotency-key header so a transport retry cannot double-charge. -func TestChargeSendsIdempotencyKey(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - - err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "my-event"}) - if err != nil { - t.Fatalf("charge: %v", err) - } - key := backend.lastHeaders.Get("idempotency-key") - if key == "" { - t.Fatal("expected an idempotency-key header to be sent") - } - if !strings.HasPrefix(key, "run123-my-event-") { - t.Fatalf("idempotency key should embed run id and event name, got %q", key) - } - if !strings.Contains(backend.lastBody, `"eventName":"my-event"`) { - t.Fatalf("unexpected charge body: %s", backend.lastBody) - } - if !strings.Contains(backend.lastURL, "/actor-runs/run123/charge") { - t.Fatalf("expected charge request against /actor-runs/run123/charge, got %q", backend.lastURL) - } -} - -// A caller-supplied idempotency key must be used verbatim. -func TestChargeHonorsExplicitIdempotencyKey(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - - err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "e", IdempotencyKey: "fixed-key"}) - if err != nil { - t.Fatalf("charge: %v", err) - } - if got := backend.lastHeaders.Get("idempotency-key"); got != "fixed-key" { - t.Fatalf("expected explicit idempotency key, got %q", got) - } -} - -// RequestID must be sent as requestId in the charge body, and omitted when empty. -func TestChargeSendsRequestID(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - - err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "e", RequestID: "req-1"}) - if err != nil { - t.Fatalf("charge: %v", err) - } - if !strings.Contains(backend.lastBody, `"requestId":"req-1"`) || !strings.Contains(backend.lastBody, `"count":1`) { - t.Fatalf("unexpected charge body: %s", backend.lastBody) - } - - if err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "e"}); err != nil { - t.Fatalf("charge: %v", err) - } - if strings.Contains(backend.lastBody, "requestId") { - t.Fatalf("requestId must be omitted when empty, got %s", backend.lastBody) - } -} - -// GetRecord must default attachment=true (matching the reference client). -func TestGetRecordDefaultsAttachment(t *testing.T) { - backend := &mockBackend{responses: constant(200, "raw-bytes")} - client := testClient(backend, 0) - - _, _, err := client.KeyValueStore("store1").GetRecord(context.Background(), "OUTPUT") - if err != nil { - t.Fatalf("get record: %v", err) - } - if !strings.Contains(backend.lastURL, "attachment=1") { - t.Fatalf("expected attachment=1 in URL, got %q", backend.lastURL) - } -} - -// BatchAddRequests must split inputs larger than the API's 25-per-batch limit into chunks. -func TestBatchAddRequestsChunks(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"processedRequests":[],"unprocessedRequests":[]}}`)} - client := testClient(backend, 0) - - requests := make([]RequestQueueRequest, 60) // 60 -> 25 + 25 + 10 = 3 chunks - for i := range requests { - requests[i] = RequestQueueRequest{URL: "https://example.com"} - } - if _, err := client.RequestQueue("q1").BatchAddRequests(context.Background(), requests, false); err != nil { - t.Fatalf("batch add: %v", err) - } - if backend.calls != 3 { - t.Fatalf("expected 3 batch calls for 60 requests, got %d", backend.calls) - } -} - -// Run.GetWithWait must forward the waitForFinish query parameter. -func TestRunGetWithWaitForwardsParam(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"id":"r1"}}`)} - client := testClient(backend, 0) - - secs := int64(30) - if _, _, err := client.Run("r1").GetWithWait(context.Background(), &secs); err != nil { - t.Fatalf("get with wait: %v", err) - } - if !strings.Contains(backend.lastURL, "waitForFinish=30") { - t.Fatalf("expected waitForFinish=30 in URL, got %q", backend.lastURL) - } -} - -// GetStreamedLog must open a raw, streaming log request (stream=1 & raw=1). -func TestGetStreamedLogParams(t *testing.T) { - backend := &mockBackend{responses: constant(200, "live log bytes")} - client := testClient(backend, 0) - - stream, err := client.Run("r1").GetStreamedLog(context.Background()) - if err != nil { - t.Fatalf("get streamed log: %v", err) - } - defer func() { _ = stream.Close() }() - if !strings.Contains(backend.lastURL, "stream=1") { - t.Fatalf("expected stream=1 in URL, got %q", backend.lastURL) - } - if !strings.Contains(backend.lastURL, "raw=1") { - t.Fatalf("expected raw=1 in URL, got %q", backend.lastURL) - } -} - -// notFound is a constant 404 response, used to simulate a resource that never becomes -// available (e.g. database-replica lag on a just-started run that never resolves). -func notFound() []mockResponse { - return constant(404, `{"error":{"type":"record-not-found","message":"missing"}}`) -} - -// A bounded WaitForFinish against a resource that stays 404 must terminate when the budget -// is exhausted and return the 404 as an *APIError (not a successful result), matching the -// reference client's waitForFinish() now rejecting with a NotFoundError. -func TestWaitForFinishBoundedReturnsDescriptiveErrorOn404(t *testing.T) { - backend := &mockBackend{responses: notFound()} - client := testClient(backend, 0) - - zeroSecs := int64(0) - _, err := client.Run("r1").WaitForFinish(context.Background(), &zeroSecs) - if err == nil { - t.Fatal("expected an error when the run never becomes available") - } - var apiErr *APIError - if !errors.As(err, &apiErr) || !apiErr.IsNotFound() { - t.Fatalf("expected a not-found *APIError, got %v (%T)", err, err) - } - // 404 must not be retried by the transport, and the budget=0 path does exactly one fetch. - if backend.calls != 1 { - t.Fatalf("expected exactly 1 fetch for a zero-budget wait, got %d", backend.calls) - } -} - -// WaitForFinish must give up on a run/build that keeps answering 404 once that has gone on for -// longer than the 404 grace period (3s), well before the overall wait budget — matching the -// reference client, which otherwise polled a missing job for its default budget of >11 days. -func TestWaitForFinishGivesUpAfterGracePeriodOnPersistent404(t *testing.T) { - backend := &mockBackend{responses: notFound()} - client := testClient(backend, 0) - - generousSecs := int64(30) - start := time.Now() - _, err := client.Run("r1").WaitForFinish(context.Background(), &generousSecs) - elapsed := time.Since(start) - - var apiErr *APIError - if !errors.As(err, &apiErr) || !apiErr.IsNotFound() { - t.Fatalf("expected a not-found *APIError, got %v (%T)", err, err) - } - // Must stop around the 3s grace period, nowhere near the 30s budget. - if elapsed < maxWaitForMissingJob || elapsed > maxWaitForMissingJob+2*time.Second { - t.Fatalf("expected to give up around the %s grace period, took %s", maxWaitForMissingJob, elapsed) - } -} - -// A successful poll in between resets the 404 grace period, so a run/build that goes -// 404 -> briefly visible -> 404 again is not penalized for the earlier 404 streak. -func TestWaitForFinishResetsGracePeriodOnSuccessfulPoll(t *testing.T) { - notFoundResp := mockResponse{status: 404, body: `{"error":{"type":"record-not-found","message":"missing"}}`} - runningResp := mockResponse{status: 200, body: `{"data":{"id":"r1","status":"RUNNING"}}`} - succeededResp := mockResponse{status: 200, body: `{"data":{"id":"r1","status":"SUCCEEDED"}}`} - backend := &mockBackend{responses: []mockResponse{ - notFoundResp, notFoundResp, notFoundResp, runningResp, - notFoundResp, notFoundResp, notFoundResp, succeededResp, - }} - client := testClient(backend, 0) - - generousSecs := int64(30) - run, err := client.Run("r1").WaitForFinish(context.Background(), &generousSecs) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if run.Status != "SUCCEEDED" { - t.Fatalf("expected the final SUCCEEDED run, got status %q", run.Status) - } - if backend.calls != 8 { - t.Fatalf("expected exactly 8 polls (both 404 streaks fully consumed), got %d", backend.calls) - } -} - -// If the run was seen once and then answers 404 for the rest of a short wait budget (the budget -// elapsing well inside the 3s grace period, not because of it), WaitForFinish must report -// not-found rather than returning the stale last-known state: present is cleared on every 404, -// matching the reference client resetting its job to undefined on every 404. -func TestWaitForFinishReturnsNotFoundWhenBudgetExhaustedWhileCurrently404(t *testing.T) { - runningResp := mockResponse{status: 200, body: `{"data":{"id":"r1","status":"RUNNING"}}`} - notFoundResp := mockResponse{status: 404, body: `{"error":{"type":"record-not-found","message":"missing"}}`} - // Only the first poll succeeds; every poll after that 404s, clamped by mockBackend for as - // long as the (sub-3s) budget keeps the loop going. - backend := &mockBackend{responses: []mockResponse{runningResp, notFoundResp}} - client := testClient(backend, 0) - - oneSec := int64(1) - _, err := client.Run("r1").WaitForFinish(context.Background(), &oneSec) - var apiErr *APIError - if !errors.As(err, &apiErr) || !apiErr.IsNotFound() { - t.Fatalf("expected a not-found *APIError (not the stale RUNNING run), got %v", err) - } - if backend.calls < 2 { - t.Fatalf("expected at least 2 polls (the success, then at least one 404), got %d", backend.calls) - } -} - -// An INDEFINITE WaitForFinish (nil waitSecs) against a permanent 404 must NOT hang forever: -// it polls through 404s on a pure time bound, so a cancelled context unblocks it promptly. -// This is the regression test for the waitForFinish hang bug. -func TestWaitForFinishIndefiniteDoesNotHangOn404(t *testing.T) { - backend := &mockBackend{responses: notFound()} - client := testClient(backend, 0) - - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - - done := make(chan error, 1) - go func() { - _, err := client.Run("r1").WaitForFinish(ctx, nil) - done <- err - }() - - select { - case err := <-done: - // It must return because the context was cancelled (or, in principle, the budget), - // never spin indefinitely. - if err == nil { - t.Fatal("expected a non-nil error (context cancelled), got nil") - } - case <-time.After(5 * time.Second): - t.Fatal("WaitForFinish hung on a permanent 404 during an indefinite wait") - } - // Must have polled at least once through the 404 (proving 404 is not treated as fatal). - if backend.calls < 1 { - t.Fatalf("expected at least one poll through the 404, got %d", backend.calls) - } -} - -// ListRequests must reject mutually-exclusive options and an invalid Filter before any API call. -func TestListRequestsValidation(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - queue := client.RequestQueue("q1") - - start, cursor := "a", "b" - if _, err := queue.ListRequests(context.Background(), ListRequestsOptions{ExclusiveStartID: &start, Cursor: &cursor}); err == nil { - t.Fatal("expected error for mutually-exclusive ExclusiveStartID and Cursor") - } - if _, err := queue.ListRequests(context.Background(), ListRequestsOptions{Filter: []string{"bogus"}}); err == nil { - t.Fatal("expected error for invalid Filter value") - } - if backend.calls != 0 { - t.Fatalf("validation must short-circuit before any API call, got %d calls", backend.calls) - } -} - -// A resource id, record key or request id that is a dot segment (".." or ".") must be rejected -// before any request is sent: a URL parser or proxy can resolve it after the path is built, -// letting it escape to a different endpoint (path traversal). -func TestPathTraversalRejected(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - - cases := []struct { - name string - call func() error - }{ - {"resource id", func() error { _, _, err := client.Actor("..").Get(context.Background()); return err }}, - {"record key", func() error { - _, _, err := client.KeyValueStore("store1").GetRecord(context.Background(), "..") - return err - }}, - {"request id", func() error { - _, _, err := client.RequestQueue("q1").GetRequest(context.Background(), "..") - return err - }}, - {"prolong request lock", func() error { - _, err := client.RequestQueue("q1").ProlongRequestLock(context.Background(), "..", 30, false) - return err - }}, - {"delete request lock", func() error { - return client.RequestQueue("q1").DeleteRequestLock(context.Background(), "..", false) - }}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - if err := c.call(); err == nil { - t.Fatal("expected an error for a dot-segment id, got nil") - } - }) - } - if backend.calls != 0 { - t.Fatalf("a dot-segment id must be rejected before any API call, got %d calls", backend.calls) - } -} - -// Every RequestQueueClient method that sends a request — including the ones that bypass the -// shared base.go CRUD helpers (ProlongRequestLock, DeleteRequestLock) — must fail fast on a -// queue built with an invalid id, instead of sending a request built from it. -func TestRequestQueueLockMethodsFailFastOnInvalidQueueID(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - queue := client.RequestQueue("..") - - if _, err := queue.ProlongRequestLock(context.Background(), "req1", 30, false); err == nil { - t.Fatal("expected ProlongRequestLock to fail fast on an invalid queue id") - } - if err := queue.DeleteRequestLock(context.Background(), "req1", false); err == nil { - t.Fatal("expected DeleteRequestLock to fail fast on an invalid queue id") - } - if backend.calls != 0 { - t.Fatalf("an invalid queue id must be rejected before any API call, got %d calls", backend.calls) - } -} - -// An empty resource id must be rejected the same way as a dot segment, not silently read as -// "the whole collection" (e.g. an empty Actor version number). -func TestEmptyIDRejected(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - - if _, _, err := client.Actor("some-actor").Version("").Get(context.Background()); err == nil { - t.Fatal("expected an error for an empty version number, got nil") - } - if backend.calls != 0 { - t.Fatalf("an empty id must be rejected before any API call, got %d calls", backend.calls) - } -} - -// Get/Delete on a resource addressed by its own id must swallow a 404 of any `type`, not just -// "record-not-found"/"record-or-token-not-found" (matching the reference client's NotFoundError, -// which classifies solely by status). -func TestGetSwallowsAnyNotFoundType(t *testing.T) { - backend := &mockBackend{responses: constant(404, `{"error":{"type":"some-other-error","message":"gone"}}`)} - client := testClient(backend, 0) - - _, present, err := client.Actor("missing").Get(context.Background()) - if err != nil { - t.Fatalf("expected a 404 of any type to resolve to absent, got error: %v", err) - } - if present { - t.Fatal("expected present=false for a 404 response") - } -} - -// Get/Delete on a client reached through a fixed sub-path with no id of its own (e.g. a run's -// default dataset) must NOT swallow a 404: the response cannot tell the run apart from the -// dataset as what is missing. -func TestNestedResourceGetPropagates404(t *testing.T) { - backend := &mockBackend{responses: notFound()} - client := testClient(backend, 0) - - _, _, err := client.Run("run1").Dataset().Get(context.Background()) - if err == nil { - t.Fatal("expected a 404 on a nested dataset client to propagate as an error") - } - apiErr, ok := AsAPIError(err) - if !ok || apiErr.StatusCode != 404 { - t.Fatalf("expected a 404 *APIError, got %v", err) - } - - err = client.Run("run1").Dataset().Delete(context.Background()) - if err == nil { - t.Fatal("expected a 404 on a nested dataset client's Delete to propagate as an error") - } -} - -// GetRecord/RecordExists on a nested key-value store (no id of its own) must still swallow a -// 404: unlike the store's own Get/Delete, a record lookup by key is never ambiguous. -func TestNestedKeyValueStoreRecordLookupStillSwallows404(t *testing.T) { - backend := &mockBackend{responses: notFound()} - client := testClient(backend, 0) - store := client.Run("run1").KeyValueStore() - - record, present, err := store.GetRecord(context.Background(), "OUTPUT") - if err != nil { - t.Fatalf("expected a missing record to resolve to absent, got error: %v", err) - } - if present || record != nil { - t.Fatalf("expected present=false and a nil record, got present=%v record=%v", present, record) - } - - exists, err := store.RecordExists(context.Background(), "OUTPUT") - if err != nil { - t.Fatalf("expected RecordExists to resolve false, got error: %v", err) - } - if exists { - t.Fatal("expected exists=false for a 404 response") - } -} - -// DatasetClient.GetStatistics, TaskClient.GetInput and ScheduleClient.GetLog must propagate a -// 404 as an error rather than an "absent" result: it always means the parent resource is gone. -func TestSingletonSubResourcesPropagate404(t *testing.T) { - cases := []struct { - name string - call func(c *ApifyClient) error - }{ - {"dataset statistics", func(c *ApifyClient) error { _, err := c.Dataset("ds1").GetStatistics(context.Background()); return err }}, - {"task input", func(c *ApifyClient) error { _, err := c.Task("task1").GetInput(context.Background()); return err }}, - {"schedule log", func(c *ApifyClient) error { _, err := c.Schedule("sch1").GetLog(context.Background()); return err }}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - backend := &mockBackend{responses: notFound()} - client := testClient(backend, 0) - if err := c.call(client); err == nil { - t.Fatal("expected a 404 to propagate as an error") - } - }) - } -} - -// MonthlyUsageWithOptions sends the date query parameter only when it is set. -func TestMonthlyUsageWithOptionsDate(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - - if _, err := client.Me().MonthlyUsageWithOptions(context.Background(), MonthlyUsageOptions{Date: "2026-09-14"}); err != nil { - t.Fatalf("monthly usage: %v", err) - } - if !strings.Contains(backend.lastURL, "/users/me/usage/monthly") || !strings.Contains(backend.lastURL, "date=2026-09-14") { - t.Fatalf("expected date query parameter, got %q", backend.lastURL) - } - if _, err := client.Me().MonthlyUsage(context.Background()); err != nil { - t.Fatalf("monthly usage: %v", err) - } - if strings.Contains(backend.lastURL, "date=") { - t.Fatalf("date must be omitted when empty, got %q", backend.lastURL) - } -} - -// The usage and limits endpoints are documented for a user by ID or username as well, so a -// client from User(id) calls them instead of failing. -func TestUserLimitsAndUsageByID(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - ctx := context.Background() - - if _, err := client.User("someone").MonthlyUsageWithOptions(ctx, MonthlyUsageOptions{Date: "2026-09-14"}); err != nil { - t.Fatalf("monthly usage by ID: %v", err) - } - if !strings.Contains(backend.lastURL, "/users/someone/usage/monthly") || !strings.Contains(backend.lastURL, "date=2026-09-14") { - t.Fatalf("unexpected monthly usage URL %q", backend.lastURL) - } - if _, err := client.User("someone").Limits(ctx); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { - t.Fatalf("limits by ID: %v, URL %q", err, backend.lastURL) - } - if err := client.User("someone").UpdateLimits(ctx, map[string]any{"maxMonthlyUsageUsd": 5}); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { - t.Fatalf("update limits by ID: %v, URL %q", err, backend.lastURL) - } -} - -// ApiError's Is* status-classification predicates match the response status, and only that -// status's predicate is true. -func TestAPIErrorStatusPredicates(t *testing.T) { - errorFor := func(status int) *APIError { - backend := &mockBackend{responses: constant(status, `{"error":{"type":"x","message":"m"}}`)} - client := testClient(backend, 0) - _, err := client.Me().MonthlyUsage(context.Background()) - apiErr, ok := AsAPIError(err) - if !ok { - t.Fatalf("expected an *APIError for status %d, got %v", status, err) - } - return apiErr - } - - notFoundErr := errorFor(404) - if !notFoundErr.IsNotFound() { - t.Error("expected IsNotFound() for a 404") - } - for _, predicate := range []struct { - name string - ok bool - }{ - {"IsInvalidRequest", notFoundErr.IsInvalidRequest()}, - {"IsUnauthorized", notFoundErr.IsUnauthorized()}, - {"IsForbidden", notFoundErr.IsForbidden()}, - {"IsConflict", notFoundErr.IsConflict()}, - {"IsRateLimited", notFoundErr.IsRateLimited()}, - {"IsServerError", notFoundErr.IsServerError()}, - } { - if predicate.ok { - t.Errorf("expected %s() to be false for a 404", predicate.name) - } - } - - if !errorFor(400).IsInvalidRequest() { - t.Error("expected IsInvalidRequest() for a 400") - } - if !errorFor(401).IsUnauthorized() { - t.Error("expected IsUnauthorized() for a 401") - } - if !errorFor(403).IsForbidden() { - t.Error("expected IsForbidden() for a 403") - } - if !errorFor(409).IsConflict() { - t.Error("expected IsConflict() for a 409") - } - if !errorFor(429).IsRateLimited() { - t.Error("expected IsRateLimited() for a 429") - } - if !errorFor(500).IsServerError() { - t.Error("expected IsServerError() for a 500") - } - if !errorFor(503).IsServerError() { - t.Error("expected IsServerError() for a 503") - } -} - -// ActorClient.StartRaw sends the given bytes verbatim (no JSON serialization), defaulting to -// application/octet-stream when the caller does not set options.ContentType. -func TestStartRawSendsBytesVerbatimWithDefaultContentType(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"id":"run1"}}`)} - client := testClient(backend, 0) - - input := []byte("not json, just bytes: \x00\x01\x02") - if _, err := client.Actor("me/actor").StartRaw(context.Background(), input, ActorStartOptions{}); err != nil { - t.Fatalf("start raw: %v", err) - } - if got := backend.lastHeaders.Get("Content-Type"); got != "application/octet-stream" { - t.Fatalf("expected default content type application/octet-stream, got %q", got) - } - if backend.lastBody != string(input) { - t.Fatalf("expected the raw bytes to be sent verbatim, got %q", backend.lastBody) - } -} - -// RunClient.MetamorphRaw sends the given bytes verbatim and still forwards targetActorId/build -// as query parameters like Metamorph. -func TestMetamorphRawSendsBytesVerbatim(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"id":"run1"}}`)} - client := testClient(backend, 0) - - input := []byte("raw metamorph input") - _, err := client.Run("run1").MetamorphRaw(context.Background(), "other/actor", input, MetamorphOptions{Build: "1.0"}) - if err != nil { - t.Fatalf("metamorph raw: %v", err) - } - if got := backend.lastHeaders.Get("Content-Type"); got != "application/octet-stream" { - t.Fatalf("expected default content type application/octet-stream, got %q", got) - } - if backend.lastBody != string(input) { - t.Fatalf("expected the raw bytes to be sent verbatim, got %q", backend.lastBody) - } - if !strings.Contains(backend.lastURL, "targetActorId=other~actor") { - t.Fatalf("expected targetActorId in URL, got %q", backend.lastURL) - } - if !strings.Contains(backend.lastURL, "build=1.0") { - t.Fatalf("expected build in URL, got %q", backend.lastURL) - } -} - -// DatasetClient.CreateItemsPublicURLWithFormat adds a `format` query parameter, and -// CreateItemsPublicURL (unchanged) omits it, defaulting to the endpoint's own json default. -func TestCreateItemsPublicURLWithFormat(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"id":"ds1"}}`)} - client := testClient(backend, 0) - dataset := client.Dataset("ds1") - - url, err := dataset.CreateItemsPublicURLWithFormat(context.Background(), DatasetListItemsOptions{}, nil, FormatCSV) - if err != nil { - t.Fatalf("create items public url with format: %v", err) - } - if !strings.Contains(url, "format=csv") { - t.Fatalf("expected format=csv in url, got %q", url) - } - - url, err = dataset.CreateItemsPublicURL(context.Background(), DatasetListItemsOptions{}, nil) - if err != nil { - t.Fatalf("create items public url: %v", err) - } - if strings.Contains(url, "format=") { - t.Fatalf("CreateItemsPublicURL must not set a format param, url was: %q", url) - } -} - -// IterateDatasetItems must advance and stop by the number of rows scanned -// (X-Apify-Pagination-Count), not the number of items a page returned: a filter can make a page -// return fewer items than the rows it covered (even zero, on a page that still has more rows -// behind it), and the iterator must not stop early in that case. -func TestIterateDatasetItemsUsesScannedCount(t *testing.T) { - // pageHeaders builds the X-Apify-Pagination-* headers for a page that scanned `scanned` rows - // of a dataset of `total` rows, starting at `offset`, having requested up to `limit`. - pageHeaders := func(total, offset, limit, scanned int64) http.Header { - h := http.Header{} - h.Set("X-Apify-Pagination-Total", itoa(total)) - h.Set("X-Apify-Pagination-Offset", itoa(offset)) - h.Set("X-Apify-Pagination-Limit", itoa(limit)) - h.Set("X-Apify-Pagination-Count", itoa(scanned)) - return h - } - backend := &mockBackend{responses: []mockResponse{ - // Page 1: the API scanned 5 rows (offset 0-4) but a filter dropped all of them. - {status: 200, body: `[]`, headers: pageHeaders(10, 0, 5, 5)}, - // Page 2: scanned the remaining 5 rows (offset 5-9), all kept. - {status: 200, body: `[1,2,3,4,5]`, headers: pageHeaders(10, 5, 5, 5)}, - // Page 3: no Limit was set, so the iterator does not stop after page 2's 5 items; it - // needs this trailing empty page (scanned 0) to confirm there is nothing left. - {status: 200, body: `[]`, headers: pageHeaders(10, 10, 5, 0)}, - }} - client := testClient(backend, 0) - - it := client.Dataset("ds1").IterateItems(DatasetListItemsOptions{}, Ptr(int64(5))) - var items []json.RawMessage - for { - item, err := it.Next(context.Background()) - if err != nil { - t.Fatalf("iterate: %v", err) - } - if item == nil { - break - } - items = append(items, *item) - } - if len(items) != 5 { - t.Fatalf("expected 5 items across both non-empty pages, got %d", len(items)) - } - if backend.calls != 3 { - t.Fatalf("expected exactly 3 page fetches (including the trailing empty one), got %d", backend.calls) - } -} - -// extendForServerWait extends the base timeout by the requested server-side wait (capped at -// maxWaitForFinishHoldSecs), matching the reference client's timeoutForWaitForFinish. -func TestExtendForServerWait(t *testing.T) { - cases := []struct { - name string - base time.Duration - waitForFinishSec *int64 - want time.Duration - }{ - {"nil wait leaves base unchanged", 5 * time.Second, nil, 5 * time.Second}, - {"zero wait leaves base unchanged", 5 * time.Second, Ptr(int64(0)), 5 * time.Second}, - {"negative wait leaves base unchanged", 5 * time.Second, Ptr(int64(-1)), 5 * time.Second}, - {"wait under the cap adds its full duration", 5 * time.Second, Ptr(int64(30)), 35 * time.Second}, - {"wait over the cap is capped at 60s", 5 * time.Second, Ptr(int64(999)), 65 * time.Second}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - if got := extendForServerWait(c.base, c.waitForFinishSec); got != c.want { - t.Errorf("extendForServerWait(%v, %v) = %v, want %v", c.base, c.waitForFinishSec, got, c.want) - } - }) - } -} - -// blockingBackend is an HTTPBackend that blocks until the request's context is done, then -// returns the context's error. It simulates a slow/unresponsive server for exercising -// client-side timeout behavior deterministically (no real network or sleep-based flakiness). -type blockingBackend struct{} - -func (blockingBackend) Do(req *http.Request) (*http.Response, error) { - <-req.Context().Done() - return nil, req.Context().Err() -} - -// A metadata call (Get, which uses the "short" tier) must be aborted by the client once the -// short tier elapses, even though the caller's own context has no deadline of its own. -func TestShortTierTimesOutMetadataCall(t *testing.T) { - client := NewClient( - WithToken("t"), - WithHTTPBackend(blockingBackend{}), - WithMaxRetries(0), - WithTimeoutShort(30*time.Millisecond), - ) - - start := time.Now() - _, _, err := client.Actor("some-actor").Get(context.Background()) - elapsed := time.Since(start) - - if err == nil { - t.Fatal("expected a timeout error from a backend that never responds") - } - if elapsed > 2*time.Second { - t.Fatalf("expected the short tier (30ms) to abort the request quickly, took %v", elapsed) - } -} - -// WaitForFinish's polling request must NOT be bounded by the short tier (it uses -// noRequestTimeout): with an unresponsive backend, it is the caller's own context deadline that -// ends the call, not a tiny configured short tier. -func TestWaitForFinishPollIgnoresShortTier(t *testing.T) { - client := NewClient( - WithToken("t"), - WithHTTPBackend(blockingBackend{}), - WithMaxRetries(0), - WithTimeoutShort(10*time.Millisecond), - ) - - ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) - defer cancel() - - start := time.Now() - _, err := client.Run("run1").WaitForFinish(ctx, Ptr(int64(1))) - elapsed := time.Since(start) - - if err == nil { - t.Fatal("expected an error once the caller's context deadline is reached") - } - // If the 10ms short tier were (wrongly) applied to the poll, this would return almost - // instantly instead of waiting for the ~150ms caller deadline. - if elapsed < 100*time.Millisecond { - t.Fatalf("expected the poll to run until the caller's context deadline (~150ms), returned after %v", elapsed) - } -} - -// BatchAddRequests must reject an oversized request before sending anything: earlier batches -// before the oversized one must not go out. -func TestBatchAddRequestsRejectsOversizedRequestBeforeSendingAnything(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"processedRequests":[],"unprocessedRequests":[]}}`)} - client := testClient(backend, 0) - - requests := make([]RequestQueueRequest, 30) // more than one batch's worth - for i := range requests { - requests[i] = RequestQueueRequest{URL: "https://example.com"} - } - // A URL alone this long exceeds payloadSizeLimitBytes, so this one request can never fit a - // batch of its own. - requests[29].URL = "https://example.com/" + strings.Repeat("a", payloadSizeLimitBytes) - - _, err := client.RequestQueue("q1").BatchAddRequests(context.Background(), requests, false) - if err == nil { - t.Fatal("expected an error for an oversized request") - } - if !strings.Contains(err.Error(), "index 29") { - t.Fatalf("expected the error to name the request's index (29), got: %v", err) - } - if backend.calls != 0 { - t.Fatalf("an oversized request must be rejected before any batch is sent, got %d calls", backend.calls) - } -} - -// BatchAddRequests must split a batch early when the next request would push it over the byte -// limit, even though both fit the 25-request count limit. -func TestBatchAddRequestsSplitsByByteSize(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"processedRequests":[],"unprocessedRequests":[]}}`)} - client := testClient(backend, 0) - - // Two requests whose combined JSON exceeds the payload limit, but each fits alone. - big := strings.Repeat("a", payloadSizeLimitBytes/2) - requests := []RequestQueueRequest{ - {URL: "https://example.com/" + big}, - {URL: "https://example.com/" + big}, - } - - result, err := client.RequestQueue("q1").BatchAddRequests(context.Background(), requests, false) - if err != nil { - t.Fatalf("batch add: %v", err) - } - _ = result - if backend.calls != 2 { - t.Fatalf("expected the two oversized-together requests to split into 2 batches, got %d calls", backend.calls) - } -} - -// BatchAddRequests sends each batch's body as a single JSON array assembled from the requests' -// pre-serialized JSON, not a re-marshaled slice. -func TestBatchAddRequestsSendsJSONArrayBody(t *testing.T) { - backend := &mockBackend{responses: constant(200, `{"data":{"processedRequests":[],"unprocessedRequests":[]}}`)} - client := testClient(backend, 0) - - requests := []RequestQueueRequest{{URL: "https://example.com/a"}, {URL: "https://example.com/b"}} - if _, err := client.RequestQueue("q1").BatchAddRequests(context.Background(), requests, false); err != nil { - t.Fatalf("batch add: %v", err) - } - var decoded []RequestQueueRequest - if err := json.Unmarshal([]byte(backend.lastBody), &decoded); err != nil { - t.Fatalf("batch body is not a valid JSON array: %v (body: %s)", err, backend.lastBody) - } - if len(decoded) != 2 || decoded[0].URL != requests[0].URL || decoded[1].URL != requests[1].URL { - t.Fatalf("unexpected decoded batch body: %+v", decoded) - } -} - -// ListRequests must serialize a multi-value Filter as a single comma-joined query parameter, -// matching the spec (array of enum locked|pending, style=form explode=false) and the JS reference. -func TestListRequestsFilterCommaJoined(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - queue := client.RequestQueue("q1") - - if _, err := queue.ListRequests(context.Background(), ListRequestsOptions{ - Filter: []string{RequestFilterLocked, RequestFilterPending}, - }); err != nil { - t.Fatalf("ListRequests with multi-value Filter: %v", err) - } - if backend.calls != 1 { - t.Fatalf("expected exactly one API call, got %d", backend.calls) - } - gotURL := backend.lastURL - if !strings.Contains(gotURL, "filter=locked%2Cpending") && !strings.Contains(gotURL, "filter=locked,pending") { - t.Fatalf("expected filter sent comma-joined as a single param, got URL: %s", gotURL) - } -} - -// The API requires `value` on env var create/update, so an empty value must still be sent. -func TestActorEnvVarSendsEmptyValue(t *testing.T) { - body, err := json.Marshal(ActorEnvVar{Name: "EMPTY"}) - if err != nil { - t.Fatal(err) - } - if string(body) != `{"name":"EMPTY","value":""}` { - t.Fatalf("unexpected body: %s", body) - } -} diff --git a/run_test.go b/run_test.go new file mode 100644 index 0000000..776eee2 --- /dev/null +++ b/run_test.go @@ -0,0 +1,141 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +// Charge must always send an idempotency-key header so a transport retry cannot double-charge. +func TestChargeSendsIdempotencyKey(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + + err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "my-event"}) + if err != nil { + t.Fatalf("charge: %v", err) + } + key := backend.lastHeaders.Get("idempotency-key") + if key == "" { + t.Fatal("expected an idempotency-key header to be sent") + } + if !strings.HasPrefix(key, "run123-my-event-") { + t.Fatalf("idempotency key should embed run id and event name, got %q", key) + } + if !strings.Contains(backend.lastBody, `"eventName":"my-event"`) { + t.Fatalf("unexpected charge body: %s", backend.lastBody) + } + if !strings.Contains(backend.lastURL, "/actor-runs/run123/charge") { + t.Fatalf("expected charge request against /actor-runs/run123/charge, got %q", backend.lastURL) + } +} + +// A caller-supplied idempotency key must be used verbatim. +func TestChargeHonorsExplicitIdempotencyKey(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + + err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "e", IdempotencyKey: "fixed-key"}) + if err != nil { + t.Fatalf("charge: %v", err) + } + if got := backend.lastHeaders.Get("idempotency-key"); got != "fixed-key" { + t.Fatalf("expected explicit idempotency key, got %q", got) + } +} + +// RequestID must be sent as requestId in the charge body, and omitted when empty. +func TestChargeSendsRequestID(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + + err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "e", RequestID: "req-1"}) + if err != nil { + t.Fatalf("charge: %v", err) + } + if !strings.Contains(backend.lastBody, `"requestId":"req-1"`) || !strings.Contains(backend.lastBody, `"count":1`) { + t.Fatalf("unexpected charge body: %s", backend.lastBody) + } + + if err := client.Run("run123").Charge(context.Background(), RunChargeOptions{EventName: "e"}); err != nil { + t.Fatalf("charge: %v", err) + } + if strings.Contains(backend.lastBody, "requestId") { + t.Fatalf("requestId must be omitted when empty, got %s", backend.lastBody) + } +} + +// Run.GetWithWait must forward the waitForFinish query parameter. +func TestRunGetWithWaitForwardsParam(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":{"id":"r1"}}`)} + client := testClient(backend, 0) + + secs := int64(30) + if _, _, err := client.Run("r1").GetWithWait(context.Background(), &secs); err != nil { + t.Fatalf("get with wait: %v", err) + } + if !strings.Contains(backend.lastURL, "waitForFinish=30") { + t.Fatalf("expected waitForFinish=30 in URL, got %q", backend.lastURL) + } +} + +// GetStreamedLog must open a raw, streaming log request (stream=1 & raw=1). +func TestGetStreamedLogParams(t *testing.T) { + backend := &mockBackend{responses: constant(200, "live log bytes")} + client := testClient(backend, 0) + + stream, err := client.Run("r1").GetStreamedLog(context.Background()) + if err != nil { + t.Fatalf("get streamed log: %v", err) + } + defer func() { _ = stream.Close() }() + if !strings.Contains(backend.lastURL, "stream=1") { + t.Fatalf("expected stream=1 in URL, got %q", backend.lastURL) + } + if !strings.Contains(backend.lastURL, "raw=1") { + t.Fatalf("expected raw=1 in URL, got %q", backend.lastURL) + } +} + +// ActorClient.StartRaw sends the given bytes verbatim (no JSON serialization), defaulting to +// application/octet-stream when the caller does not set options.ContentType. +func TestStartRawSendsBytesVerbatimWithDefaultContentType(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":{"id":"run1"}}`)} + client := testClient(backend, 0) + + input := []byte("not json, just bytes: \x00\x01\x02") + if _, err := client.Actor("me/actor").StartRaw(context.Background(), input, ActorStartOptions{}); err != nil { + t.Fatalf("start raw: %v", err) + } + if got := backend.lastHeaders.Get("Content-Type"); got != "application/octet-stream" { + t.Fatalf("expected default content type application/octet-stream, got %q", got) + } + if backend.lastBody != string(input) { + t.Fatalf("expected the raw bytes to be sent verbatim, got %q", backend.lastBody) + } +} + +// RunClient.MetamorphRaw sends the given bytes verbatim and still forwards targetActorId/build +// as query parameters like Metamorph. +func TestMetamorphRawSendsBytesVerbatim(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":{"id":"run1"}}`)} + client := testClient(backend, 0) + + input := []byte("raw metamorph input") + _, err := client.Run("run1").MetamorphRaw(context.Background(), "other/actor", input, MetamorphOptions{Build: "1.0"}) + if err != nil { + t.Fatalf("metamorph raw: %v", err) + } + if got := backend.lastHeaders.Get("Content-Type"); got != "application/octet-stream" { + t.Fatalf("expected default content type application/octet-stream, got %q", got) + } + if backend.lastBody != string(input) { + t.Fatalf("expected the raw bytes to be sent verbatim, got %q", backend.lastBody) + } + if !strings.Contains(backend.lastURL, "targetActorId=other~actor") { + t.Fatalf("expected targetActorId in URL, got %q", backend.lastURL) + } + if !strings.Contains(backend.lastURL, "build=1.0") { + t.Fatalf("expected build in URL, got %q", backend.lastURL) + } +} diff --git a/streamed_log.go b/streamed_log.go index ce8d842..c7d23cb 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -62,8 +62,8 @@ type StreamedLogOptions struct { // StreamedLog redirects the live log of an Actor run to a writer, one prefixed line per message. // Get one from [RunClient.NewStreamedLog], and begin and end the redirection with // [StreamedLog.Start] and [StreamedLog.Stop]. Messages logged before the redirection was created -// are skipped unless FromStart is true. A log that is still empty is reopened until it delivers -// its first bytes. +// are skipped if FromStart is false. A log that is still empty is reopened until it delivers its +// first bytes. type StreamedLog struct { log *LogClient output io.Writer @@ -154,8 +154,10 @@ func (s *StreamedLog) Start(ctx context.Context) error { // Stop ends the redirection and returns the error that stopped it early, if any. Messages in // flight get up to one second. If the stream delivered nothing, Stop reads the whole log in one -// request, so that a short run is not lost. ctx bounds the wait, the final read included. It -// returns an error if the redirection is not running or is already stopping. +// request, so that a short run is not lost. ctx bounds the wait, the final read included: when it +// ends first, Stop returns its error, unless the context given to Start ended first, which ends +// the redirection on purpose. It returns an error if the redirection is not running or is already +// stopping. func (s *StreamedLog) Stop(ctx context.Context) error { s.mu.Lock() if !s.running { diff --git a/task.go b/task.go index 61c7a52..9b7929a 100644 --- a/task.go +++ b/task.go @@ -35,14 +35,16 @@ func (c *TaskClient) Delete(ctx context.Context) error { return deleteResource(ctx, c.ctx, "") } -// Publish publishes the task on its public page by setting IsPublic. The Actor must be public and -// the task must have its PublicConfig. Publishing a published task does nothing. An Actor can have -// 10 published tasks and an account 100. +// Publish publishes the task on its public page by setting IsPublic. The Actor must be public, and +// the task must have its PublicConfig. It needs write permission to both the task and its Actor. +// Publishing a published task does nothing. An Actor can have at most 10 published tasks and an +// account at most 100. Apify support can raise both limits. func (c *TaskClient) Publish(ctx context.Context) (Task, error) { return c.Update(ctx, map[string]any{"isPublic": true}) } -// Unpublish removes the task from its public page by clearing IsPublic. PublicConfig is kept. +// Unpublish removes the task from its public page by clearing IsPublic. PublicConfig is kept, so +// the task can be published again. It needs write permission to both the task and its Actor. // Unpublishing a task that is not published does nothing. func (c *TaskClient) Unpublish(ctx context.Context) (Task, error) { return c.Update(ctx, map[string]any{"isPublic": false}) diff --git a/tests/examples_test.go b/tests/examples_test.go index b4bb013..ed0118d 100644 --- a/tests/examples_test.go +++ b/tests/examples_test.go @@ -6,9 +6,7 @@ import ( "testing" ) -// runExample runs `go run ./examples/` as a subprocess and fails the test if it does -// not exit successfully. Each documentation example is exercised this way, mirroring the -// reference clients' example smoke tests, so the docs stay runnable. +// runExample runs `go run ./examples/` and fails the test if it does not exit successfully. func runExample(t *testing.T, name string) { t.Helper() if os.Getenv("APIFY_TOKEN") == "" { diff --git a/tests/helpers_test.go b/tests/helpers_test.go index e6edce9..57c9836 100644 --- a/tests/helpers_test.go +++ b/tests/helpers_test.go @@ -21,7 +21,7 @@ import ( apify "github.com/apify/apify-client-go" ) -// defaultAPIURL mirrors the integration-test contract fallback. +// defaultAPIURL is the API URL when APIFY_API_URL is not set. const defaultAPIURL = "https://api.apify.com/v2" // resolveBaseURL derives the client base_url from an optional APIFY_API_URL. diff --git a/tests/iterate_test.go b/tests/iterate_test.go index a24a93f..0b1c28c 100644 --- a/tests/iterate_test.go +++ b/tests/iterate_test.go @@ -460,8 +460,8 @@ func TestIterateDatasetItems(t *testing.T) { t.Fatalf("expected to iterate %d items, got %d", total, count) } - // Limit is a cap on the total number of items yielded (not the page size), matching the - // reference client. With Limit=3 and a page size of 2, iteration must stop at exactly 3. + // Limit caps the total number of items, not the page size: with a Limit of 3 and a page size of + // 2, iteration stops at exactly 3. const wantCap = 3 capped := dataset.IterateItems(apify.DatasetListItemsOptions{Limit: ptr(int64(wantCap))}, ptr(int64(2))) capCount := 0 diff --git a/timeout_tier_test.go b/timeout_tier_test.go index 9fe4b6a..e61e033 100644 --- a/timeout_tier_test.go +++ b/timeout_tier_test.go @@ -2,11 +2,13 @@ package apify import ( "context" + "net/http" "testing" + "time" ) -// The calls the reference client puts on the long tier must not be cut off by a short tier: here -// the short tier (tinyTier) is shorter than the response time, and the calls still succeed. +// The calls on the long tier are not cut off by a shorter short tier: here the short tier +// (tinyTier) is shorter than the response time, and the calls still succeed. func TestKeyValueStoreAndPublicURLCallsUseTheLongTier(t *testing.T) { ctx := context.Background() store := `{"data":{"id":"s","urlSigningSecretKey":"secret"}}` @@ -47,3 +49,58 @@ func TestKeyValueStoreAndPublicURLCallsUseTheLongTier(t *testing.T) { }) } } + +// extendForServerWait adds the server-side wait to the base timeout, capped at +// maxWaitForFinishHoldSecs. +func TestExtendForServerWait(t *testing.T) { + cases := []struct { + name string + base time.Duration + waitForFinishSec *int64 + want time.Duration + }{ + {"nil wait leaves base unchanged", 5 * time.Second, nil, 5 * time.Second}, + {"zero wait leaves base unchanged", 5 * time.Second, Ptr(int64(0)), 5 * time.Second}, + {"negative wait leaves base unchanged", 5 * time.Second, Ptr(int64(-1)), 5 * time.Second}, + {"wait under the cap adds its full duration", 5 * time.Second, Ptr(int64(30)), 35 * time.Second}, + {"wait over the cap is capped at 60s", 5 * time.Second, Ptr(int64(999)), 65 * time.Second}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := extendForServerWait(c.base, c.waitForFinishSec); got != c.want { + t.Errorf("extendForServerWait(%v, %v) = %v, want %v", c.base, c.waitForFinishSec, got, c.want) + } + }) + } +} + +// blockingBackend blocks until the request's context is done and returns its error, which +// simulates an unresponsive server. +type blockingBackend struct{} + +func (blockingBackend) Do(req *http.Request) (*http.Response, error) { + <-req.Context().Done() + return nil, req.Context().Err() +} + +// A metadata call (Get, which uses the "short" tier) must be aborted by the client once the +// short tier elapses, even though the caller's own context has no deadline of its own. +func TestShortTierTimesOutMetadataCall(t *testing.T) { + client := NewClient( + WithToken("t"), + WithHTTPBackend(blockingBackend{}), + WithMaxRetries(0), + WithTimeoutShort(30*time.Millisecond), + ) + + start := time.Now() + _, _, err := client.Actor("some-actor").Get(context.Background()) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected a timeout error from a backend that never responds") + } + if elapsed > 2*time.Second { + t.Fatalf("expected the short tier (30ms) to abort the request quickly, took %v", elapsed) + } +} diff --git a/unit_http_test.go b/unit_http_test.go index 0712e36..c0a832e 100644 --- a/unit_http_test.go +++ b/unit_http_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" ) // mockBackend is a deterministic HTTPBackend for offline unit tests. It serves a queue of @@ -311,6 +312,34 @@ func TestMaybeCompressRequestBody_CompressibleExceptionUnderPrefix(t *testing.T) } } +// WithRequestCompression(RequestCompressionZstd) compresses a large body with zstd. +func TestWithRequestCompressionZstd(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":{}}`)} + client := NewClient( + WithToken("t"), + WithHTTPBackend(backend), + WithMaxRetries(0), + WithRequestCompression(RequestCompressionZstd), + ) + + large := strings.Repeat("compress me ", 200) + if _, err := client.Actor("some-actor").Update(context.Background(), map[string]any{"v": large}); err != nil { + t.Fatalf("update: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got != contentEncodingZstd { + t.Fatalf("expected Content-Encoding zstd, got %q", got) + } + decoder, err := zstd.NewReader(strings.NewReader(backend.lastBody)) + if err != nil { + t.Fatal(err) + } + defer decoder.Close() + plain, err := io.ReadAll(decoder) + if err != nil || !strings.Contains(string(plain), large) { + t.Fatalf("the body is not valid zstd of the request: %v", err) + } +} + // WithRequestCompression(RequestCompressionGzip) must use gzip only, even for a body brotli // would otherwise have compressed first. func TestWithRequestCompressionGzipOnly(t *testing.T) { @@ -666,3 +695,7 @@ func TestZeroRetriesSingleAttempt(t *testing.T) { t.Fatalf("expected exactly 1 attempt with max_retries=0, got %d", backend.calls) } } + +// okData is a constant 200 response with an empty data envelope, reused by the offline +// tests below. +func okData() []mockResponse { return constant(200, `{"data":{}}`) } diff --git a/user_test.go b/user_test.go new file mode 100644 index 0000000..9d033a9 --- /dev/null +++ b/user_test.go @@ -0,0 +1,47 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +// MonthlyUsageWithOptions sends the date query parameter only when it is set. +func TestMonthlyUsageWithOptionsDate(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + + if _, err := client.Me().MonthlyUsageWithOptions(context.Background(), MonthlyUsageOptions{Date: "2026-09-14"}); err != nil { + t.Fatalf("monthly usage: %v", err) + } + if !strings.Contains(backend.lastURL, "/users/me/usage/monthly") || !strings.Contains(backend.lastURL, "date=2026-09-14") { + t.Fatalf("expected date query parameter, got %q", backend.lastURL) + } + if _, err := client.Me().MonthlyUsage(context.Background()); err != nil { + t.Fatalf("monthly usage: %v", err) + } + if strings.Contains(backend.lastURL, "date=") { + t.Fatalf("date must be omitted when empty, got %q", backend.lastURL) + } +} + +// The usage and limits endpoints are documented for a user by ID or username as well, so a +// client from User(id) calls them instead of failing. +func TestUserLimitsAndUsageByID(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + ctx := context.Background() + + if _, err := client.User("someone").MonthlyUsageWithOptions(ctx, MonthlyUsageOptions{Date: "2026-09-14"}); err != nil { + t.Fatalf("monthly usage by ID: %v", err) + } + if !strings.Contains(backend.lastURL, "/users/someone/usage/monthly") || !strings.Contains(backend.lastURL, "date=2026-09-14") { + t.Fatalf("unexpected monthly usage URL %q", backend.lastURL) + } + if _, err := client.User("someone").Limits(ctx); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { + t.Fatalf("limits by ID: %v, URL %q", err, backend.lastURL) + } + if err := client.User("someone").UpdateLimits(ctx, map[string]any{"maxMonthlyUsageUsd": 5}); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { + t.Fatalf("update limits by ID: %v, URL %q", err, backend.lastURL) + } +} diff --git a/wait_for_finish_test.go b/wait_for_finish_test.go new file mode 100644 index 0000000..d4f2f2f --- /dev/null +++ b/wait_for_finish_test.go @@ -0,0 +1,162 @@ +package apify + +import ( + "context" + "errors" + "testing" + "time" +) + +// notFound is a constant 404 response, used to simulate a resource that never becomes +// available (e.g. database-replica lag on a just-started run that never resolves). +func notFound() []mockResponse { + return constant(404, `{"error":{"type":"record-not-found","message":"missing"}}`) +} + +// A bounded WaitForFinish on a resource that stays 404 ends when the budget runs out and returns +// the 404 as an *APIError. +func TestWaitForFinishBoundedReturnsDescriptiveErrorOn404(t *testing.T) { + backend := &mockBackend{responses: notFound()} + client := testClient(backend, 0) + + zeroSecs := int64(0) + _, err := client.Run("r1").WaitForFinish(context.Background(), &zeroSecs) + if err == nil { + t.Fatal("expected an error when the run never becomes available") + } + var apiErr *APIError + if !errors.As(err, &apiErr) || !apiErr.IsNotFound() { + t.Fatalf("expected a not-found *APIError, got %v (%T)", err, err) + } + // 404 must not be retried by the transport, and the budget=0 path does exactly one fetch. + if backend.calls != 1 { + t.Fatalf("expected exactly 1 fetch for a zero-budget wait, got %d", backend.calls) + } +} + +// WaitForFinish gives up on a resource that keeps answering 404 for the grace period (3s), long +// before the overall wait budget. +func TestWaitForFinishGivesUpAfterGracePeriodOnPersistent404(t *testing.T) { + backend := &mockBackend{responses: notFound()} + client := testClient(backend, 0) + + generousSecs := int64(30) + start := time.Now() + _, err := client.Run("r1").WaitForFinish(context.Background(), &generousSecs) + elapsed := time.Since(start) + + var apiErr *APIError + if !errors.As(err, &apiErr) || !apiErr.IsNotFound() { + t.Fatalf("expected a not-found *APIError, got %v (%T)", err, err) + } + // Must stop around the 3s grace period, nowhere near the 30s budget. + if elapsed < maxWaitForMissingJob || elapsed > maxWaitForMissingJob+2*time.Second { + t.Fatalf("expected to give up around the %s grace period, took %s", maxWaitForMissingJob, elapsed) + } +} + +// A successful poll in between resets the 404 grace period, so a run/build that goes +// 404 -> briefly visible -> 404 again is not penalized for the earlier 404 streak. +func TestWaitForFinishResetsGracePeriodOnSuccessfulPoll(t *testing.T) { + notFoundResp := mockResponse{status: 404, body: `{"error":{"type":"record-not-found","message":"missing"}}`} + runningResp := mockResponse{status: 200, body: `{"data":{"id":"r1","status":"RUNNING"}}`} + succeededResp := mockResponse{status: 200, body: `{"data":{"id":"r1","status":"SUCCEEDED"}}`} + backend := &mockBackend{responses: []mockResponse{ + notFoundResp, notFoundResp, notFoundResp, runningResp, + notFoundResp, notFoundResp, notFoundResp, succeededResp, + }} + client := testClient(backend, 0) + + generousSecs := int64(30) + run, err := client.Run("r1").WaitForFinish(context.Background(), &generousSecs) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if run.Status != "SUCCEEDED" { + t.Fatalf("expected the final SUCCEEDED run, got status %q", run.Status) + } + if backend.calls != 8 { + t.Fatalf("expected exactly 8 polls (both 404 streaks fully consumed), got %d", backend.calls) + } +} + +// A run that was seen once and then answers 404 until the wait budget runs out, which is within +// the grace period, is reported as not found and not as its stale last state. +func TestWaitForFinishReturnsNotFoundWhenBudgetExhaustedWhileCurrently404(t *testing.T) { + runningResp := mockResponse{status: 200, body: `{"data":{"id":"r1","status":"RUNNING"}}`} + notFoundResp := mockResponse{status: 404, body: `{"error":{"type":"record-not-found","message":"missing"}}`} + // Only the first poll succeeds; every poll after that 404s, clamped by mockBackend for as + // long as the (sub-3s) budget keeps the loop going. + backend := &mockBackend{responses: []mockResponse{runningResp, notFoundResp}} + client := testClient(backend, 0) + + oneSec := int64(1) + _, err := client.Run("r1").WaitForFinish(context.Background(), &oneSec) + var apiErr *APIError + if !errors.As(err, &apiErr) || !apiErr.IsNotFound() { + t.Fatalf("expected a not-found *APIError (not the stale RUNNING run), got %v", err) + } + if backend.calls < 2 { + t.Fatalf("expected at least 2 polls (the success, then at least one 404), got %d", backend.calls) + } +} + +// An INDEFINITE WaitForFinish (nil waitSecs) against a permanent 404 must NOT hang forever: +// it polls through 404s on a pure time bound, so a cancelled context unblocks it promptly. +// This is the regression test for the waitForFinish hang bug. +func TestWaitForFinishIndefiniteDoesNotHangOn404(t *testing.T) { + backend := &mockBackend{responses: notFound()} + client := testClient(backend, 0) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + done := make(chan error, 1) + go func() { + _, err := client.Run("r1").WaitForFinish(ctx, nil) + done <- err + }() + + select { + case err := <-done: + // It must return because the context was cancelled (or, in principle, the budget), + // never spin indefinitely. + if err == nil { + t.Fatal("expected a non-nil error (context cancelled), got nil") + } + case <-time.After(5 * time.Second): + t.Fatal("WaitForFinish hung on a permanent 404 during an indefinite wait") + } + // Must have polled at least once through the 404 (proving 404 is not treated as fatal). + if backend.calls < 1 { + t.Fatalf("expected at least one poll through the 404, got %d", backend.calls) + } +} + +// WaitForFinish's polling request must NOT be bounded by the short tier (it uses +// noRequestTimeout): with an unresponsive backend, it is the caller's own context deadline that +// ends the call, not a tiny configured short tier. +func TestWaitForFinishPollIgnoresShortTier(t *testing.T) { + client := NewClient( + WithToken("t"), + WithHTTPBackend(blockingBackend{}), + WithMaxRetries(0), + WithTimeoutShort(10*time.Millisecond), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := client.Run("run1").WaitForFinish(ctx, Ptr(int64(1))) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected an error once the caller's context deadline is reached") + } + // If the 10ms short tier were (wrongly) applied to the poll, this would return almost + // instantly instead of waiting for the ~150ms caller deadline. + if elapsed < 100*time.Millisecond { + t.Fatalf("expected the poll to run until the caller's context deadline (~150ms), returned after %v", elapsed) + } +} From 75b1e80b065cf69290811df6ab51686b05ff0217 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:15:13 +0000 Subject: [PATCH 3/6] docs: rewrap per-page comments Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- key_value_store_collection.go | 4 ++-- request_queue_collection.go | 4 ++-- schedule.go | 4 ++-- store_collection.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/key_value_store_collection.go b/key_value_store_collection.go index 3eb6807..82fd159 100644 --- a/key_value_store_collection.go +++ b/key_value_store_collection.go @@ -20,8 +20,8 @@ func (c *KeyValueStoreCollectionClient) List(ctx context.Context, options Storag } // Iterate returns a lazy iterator over the key-value stores matching the options, fetching pages on -// demand. The options' Limit caps the total number of stores yielded (unset means all); the per- -// page size is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of stores yielded (unset means all); the page size +// is chunkSize (nil for the server default). func (c *KeyValueStoreCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[KeyValueStore] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[KeyValueStore], error) { opts := options diff --git a/request_queue_collection.go b/request_queue_collection.go index 1db29dc..78a4413 100644 --- a/request_queue_collection.go +++ b/request_queue_collection.go @@ -20,8 +20,8 @@ func (c *RequestQueueCollectionClient) List(ctx context.Context, options Storage } // Iterate returns a lazy iterator over the request queues matching the options, fetching pages on -// demand. The options' Limit caps the total number of queues yielded (unset means all); the per- -// page size is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of queues yielded (unset means all); the page size +// is chunkSize (nil for the server default). func (c *RequestQueueCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[RequestQueue] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[RequestQueue], error) { opts := options diff --git a/schedule.go b/schedule.go index 36f7b3e..b3b20f1 100644 --- a/schedule.go +++ b/schedule.go @@ -20,8 +20,8 @@ func (c *ScheduleCollectionClient) List(ctx context.Context, options ListOptions } // Iterate returns a lazy iterator over the schedules matching the options, fetching pages on -// demand. The options' Limit caps the total number of schedules yielded (unset means all); the per- -// page size is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of schedules yielded (unset means all); the page size +// is chunkSize (nil for the server default). func (c *ScheduleCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Schedule] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Schedule], error) { opts := options diff --git a/store_collection.go b/store_collection.go index 148183f..427d409 100644 --- a/store_collection.go +++ b/store_collection.go @@ -65,8 +65,8 @@ func (c *StoreCollectionClient) List(ctx context.Context, options StoreListOptio type StoreActorIterator = ListIterator[ActorStoreListItem] // Iterate returns a lazy iterator over the Store Actors matching the options, fetching pages on -// demand. The options' Limit caps the total number of Actors yielded (unset means all); the per- -// page size is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of Actors yielded (unset means all); the page size +// is chunkSize (nil for the server default). func (c *StoreCollectionClient) Iterate(options StoreListOptions, chunkSize *int64) *StoreActorIterator { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorStoreListItem], error) { opts := options From 809398de8f9137e326bdad30c02cf2965ea327cb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:31:39 +0000 Subject: [PATCH 4/6] fix: report failed batches as unprocessed, share the zstd encoder across goroutines, cap the retry delay, fix docs Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- .github/workflows/go-integration-tests.yml | 2 +- CHANGELOG.md | 13 ++-- README.md | 17 +++-- client.go | 2 +- docs/storages.md | 2 +- docs/webhooks.md | 3 +- http_client.go | 8 +- iterator.go | 6 +- key_value_store_collection.go | 4 +- models.go | 2 +- request_queue.go | 44 +++++++---- request_queue_batch_test.go | 88 +++++++++++++++++++--- request_queue_collection.go | 4 +- run.go | 3 +- schedule.go | 4 +- store_collection.go | 4 +- unit_http_test.go | 17 +++++ wait_for_finish_test.go | 1 - 18 files changed, 160 insertions(+), 64 deletions(-) diff --git a/.github/workflows/go-integration-tests.yml b/.github/workflows/go-integration-tests.yml index b7e0a8f..d088049 100644 --- a/.github/workflows/go-integration-tests.yml +++ b/.github/workflows/go-integration-tests.yml @@ -1,6 +1,6 @@ name: Go integration tests -# Runs for changes to Go code, docs or this workflow, and can be dispatched from any branch. +# Runs for changes to Go code, docs, or this workflow, and can be dispatched from any branch. on: pull_request: branches: [master] diff --git a/CHANGELOG.md b/CHANGELOG.md index 3023612..10178ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,16 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ActorServerRequestOptions.ParentRequestID`, sent as the `X-Actor-Request-ID` header, so that a Server Actor calling another one passes its caller's pricing tier on. - `zstd` in the documented `Content-Encoding` values of `SetRecordRawWithEncoding`. -- `RequestQueueClient.BatchAddRequestsWithOptions` with `BatchAddRequestsOptions`. - `BatchAddRequests` now sends the requests that the API did not process again (3 retries by - default) and sends up to 5 batches at once. If a batch fails with an error, the error is - returned together with the result of all batches. -- `RequestQueueOperationInfo.UniqueKey`. +- `RequestQueueClient.BatchAddRequestsWithOptions` and `BatchAddRequestsOptions`. +- `RequestQueueOperationInfo.UniqueKey`, set by batch add results. ### Changed -- Shortened and corrected the documentation and code comments, and split the tests of - `review_fixes_test.go` by subject. +- `BatchAddRequests` sends the requests that the API did not process again, 3 times by default, + and sends up to 5 batches at once. A batch that fails with an API or transport error leaves its + requests that were not added in `UnprocessedRequests`, and it is not an error of the call. +- Shortened and corrected the documentation and code comments. - Bumped `APISpecVersion` to `v2-2026-11-09T143926Z` and `ClientVersion` to `0.16.0`. ## [0.15.0] - 2026-11-06 diff --git a/README.md b/README.md index da39e95..54fb562 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ An idiomatic Go client for the [Apify API](https://docs.apify.com/api/v2). Start from an `ApifyClient` and go on to the resources: Actors, runs, datasets, key-value -stores, request queues, tasks, schedules, webhooks, the Store, users and logs. +stores, request queues, tasks, schedules, webhooks, the Store, users, and logs. ## Features @@ -154,22 +154,23 @@ request is not aborted client-side while the server is correctly honoring it. Request bodies of 1 KiB or more are compressed before sending, to save upload bandwidth. By default (`RequestCompressionAuto`) the client prefers brotli and falls back to gzip when brotli -errors or does not shrink the body; `WithRequestCompression` with `RequestCompressionBrotli`, `RequestCompressionGzip` or -`RequestCompressionZstd` pins it to one codec, with no fallback (a body that codec does not -shrink is sent uncompressed rather than failing). A body whose content type is already -compressed, such as images, audio, video and archives, is not compressed, except for raw formats +errors or does not shrink the body. `WithRequestCompression` with `RequestCompressionBrotli`, +`RequestCompressionGzip`, or `RequestCompressionZstd` pins it to one codec, with no fallback: a +body that codec does not shrink is sent uncompressed. A body whose content type is already +compressed, such as images, audio, video, and archives, is not compressed, except for raw formats like `image/bmp`. The `User-Agent` header reports an `isAtHome` flag indicating whether the client runs on the -Apify platform. It is driven solely by the `APIFY_IS_AT_HOME` environment variable (the same -variable); if it is set to a non-empty value, the flag is `true`, otherwise `false`. +Apify platform. It is driven solely by the `APIFY_IS_AT_HOME` environment variable: if it is set +to a non-empty value, the flag is `true`, otherwise `false`. ### URL fields The fields the API specification declares as URLs (`consoleUrl`, `containerUrl`, `itemsPublicUrl`, `keysPublicUrl`, `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, the webhook `requestUrl`, a run's `output` links, a Store Actor's `url` and `userPictureUrl`, and a user profile's -`pictureUrl` and `websiteUrl`) are returned in a normalized form, close to the WHATWG URL serialization: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IPv4 or +`pictureUrl` and `websiteUrl`) are returned in a normalized form, close to the WHATWG URL +serialization: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IPv4 or IPv6 host is written canonically, a default port is dropped, an empty path becomes `/`, dot segments are resolved, and unsafe characters are percent-encoded. It covers the http(s), ws(s), ftp and file URLs the API returns; it does not apply the full Unicode (UTS #46) mapping to a diff --git a/client.go b/client.go index 4c3f39c..7a3619c 100644 --- a/client.go +++ b/client.go @@ -1,7 +1,7 @@ // Package apify is the Go client of the Apify API (https://docs.apify.com/api/v2). // // Start from an [ApifyClient] and go on to the resource clients: Actors, runs, datasets, -// key-value stores, request queues, tasks, schedules, webhooks, the Store, users and logs. +// key-value stores, request queues, tasks, schedules, webhooks, the Store, users, and logs. // // # Quick start // diff --git a/docs/storages.md b/docs/storages.md index 9c890fd..9578a24 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -292,7 +292,7 @@ Single queue: `client.RequestQueue(id)`: | `PaginateRequestsWithOptions(ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator` | The same with `Limit` (caps the total across pages), `Cursor` (where to start) and `Filter` (every page); `chunkSize` is the page size. | | `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests with the default `BatchAddRequestsOptions`. See `BatchAddRequestsWithOptions`. | | `BatchAddRequestsWithOptions(ctx, []RequestQueueRequest, BatchAddRequestsOptions) (BatchAddResult, error)` | Add many requests, split into batches within the API limits of 25 requests and 9 MiB, with up to `MaxParallel` (5) batches sent at once. The requests that the API does not process are sent again, up to `MaxUnprocessedRequestsRetries` (3) times, after a delay that starts at `MinDelayBetweenUnprocessedRequestsRetries` (500ms) and doubles. Fails before sending anything if one request is too large for a batch. If a batch fails with an error, the error is returned together with the result of all batches, in which the requests of that batch that were not added are unprocessed. | -| `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload — a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`). Returns the raw API response. | +| `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload: a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`). Returns the raw API response. | | `ListAndLockHead(ctx, lockSecs int64, limit *int64) (json.RawMessage, error)` | Fetch the head of the queue and lock the returned requests for `lockSecs` seconds. Returns the raw API response. | | `ProlongRequestLock(ctx, id string, lockSecs int64, forefront bool) (json.RawMessage, error)` | Extend the lock on a request by `lockSecs` seconds; `forefront` controls re-queue position when the lock expires. | | `DeleteRequestLock(ctx, id string, forefront bool) error` | Release the lock on a request without modifying it. | diff --git a/docs/webhooks.md b/docs/webhooks.md index e2473c9..6e4b4bb 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -15,7 +15,8 @@ Webhooks notify external services when events occur. Access the webhook collecti An Actor's or task's webhooks are also listable via `client.Actor(id).Webhooks()` / `client.Task(id).Webhooks()`. -`Create` (and `Update`) take a JSON object (`any`) with the webhook fields of the Apify API. The commonly used input keys are: +`Create` (and `Update`) take a JSON object (`any`) with the webhook fields of the Apify API. +The commonly used input keys are: | Key | Type | Meaning | |---|---|---| diff --git a/http_client.go b/http_client.go index 1631b51..321da26 100644 --- a/http_client.go +++ b/http_client.go @@ -327,14 +327,14 @@ func gzipCompress(body []byte) ([]byte, error) { return compressBody(body, func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }) } -// zstdEncoder compresses every zstd request body. A nil writer makes it a stateless encoder that is -// safe for concurrent use, which is cheaper than a new encoder for each request. +// zstdEncoder compresses every zstd request body with EncodeAll, which is safe for concurrent use +// and cheaper than a new encoder for each request. var zstdEncoder = newZstdEncoder() func newZstdEncoder() *zstd.Encoder { - encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + encoder, err := zstd.NewWriter(nil) if err != nil { - // Only invalid options fail, and these are constant. + // Only invalid options fail, and none are passed. panic(err) } return encoder diff --git a/iterator.go b/iterator.go index 7aa9ca0..7e396f3 100644 --- a/iterator.go +++ b/iterator.go @@ -135,8 +135,7 @@ func scannedRows[T any](page PaginationList[T], requestedLimit int64) int64 { } // remainingLimitParam returns the number of items still allowed by the caller's Limit, for use -// minForLimitParam returns the smaller of a and b, ignoring a value of 0 or below, which means -// unset. It returns 0 when neither is set. +// as a minForLimitParam input, where 0 means unset. It is meaningful only once hasLimit is set. func (it *ListIterator[T]) remainingLimitParam() int64 { if !it.hasLimit { return 0 @@ -161,7 +160,8 @@ func (it *ListIterator[T]) chunkVal() int64 { return *it.chunkSize } -// minForLimitParam +// minForLimitParam returns the smaller of a and b, ignoring a value of 0 or below, which means +// unset. It returns 0 when neither is set. func minForLimitParam(a, b int64) int64 { if a <= 0 { return maxInt64(b, 0) diff --git a/key_value_store_collection.go b/key_value_store_collection.go index 82fd159..182845b 100644 --- a/key_value_store_collection.go +++ b/key_value_store_collection.go @@ -20,8 +20,8 @@ func (c *KeyValueStoreCollectionClient) List(ctx context.Context, options Storag } // Iterate returns a lazy iterator over the key-value stores matching the options, fetching pages on -// demand. The options' Limit caps the total number of stores yielded (unset means all); the page size -// is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of stores yielded (unset means all); the page +// size is chunkSize (nil for the server default). func (c *KeyValueStoreCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[KeyValueStore] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[KeyValueStore], error) { opts := options diff --git a/models.go b/models.go index 1bd7303..ba9f52f 100644 --- a/models.go +++ b/models.go @@ -416,7 +416,7 @@ func (r *RequestQueueRequest) UnmarshalJSON(data []byte) error { type RequestQueueOperationInfo struct { // RequestID is the ID of the affected request. RequestID string `json:"requestId"` - // UniqueKey is the unique key of the affected request. + // UniqueKey is the unique key of the affected request. Only the results of a batch add set it. UniqueKey string `json:"uniqueKey"` // WasAlreadyPresent reports whether the request was already in the queue. WasAlreadyPresent bool `json:"wasAlreadyPresent"` diff --git a/request_queue.go b/request_queue.go index 0e54b5e..d1ff47a 100644 --- a/request_queue.go +++ b/request_queue.go @@ -6,8 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "math" - "math/rand" "net/http" "sync" "time" @@ -290,7 +288,7 @@ type batchAddSettings struct { minDelay time.Duration } -// resolve fills in the defaults and rejects values that the API cannot make sense of. +// resolve fills in the defaults and rejects invalid values. func (o BatchAddRequestsOptions) resolve() (batchAddSettings, error) { settings := batchAddSettings{ forefront: o.Forefront, @@ -330,10 +328,10 @@ func (c *RequestQueueClient) BatchAddRequests(ctx context.Context, requests []Re // requests that the API did not process again, up to MaxUnprocessedRequestsRetries times. It fails // before sending anything if one request is too large for a batch. // -// The result lists the requests that were added and those that are still unprocessed. When a batch -// fails with an error, the error is returned with the result of all batches, in which the requests -// of the failed batch that were not added are unprocessed. When ctx ends, the error of ctx is -// returned. +// The result lists the requests that were added and those that are still unprocessed. A batch +// that fails with an API or transport error is not an error of the call: its requests that were +// not added are unprocessed. The call returns an error for a response that cannot be decoded, and +// the error of ctx, with the result so far, when ctx ends. func (c *RequestQueueClient) BatchAddRequestsWithOptions(ctx context.Context, requests []RequestQueueRequest, options BatchAddRequestsOptions) (BatchAddResult, error) { var merged BatchAddResult settings, err := options.resolve() @@ -371,7 +369,7 @@ func (c *RequestQueueClient) BatchAddRequestsWithOptions(ctx context.Context, re return merged, firstErr } -// batchOutcome is the result of one batch and the error that ended it early, if any. +// batchOutcome is the result of one batch and the error that fails the whole call, if any. type batchOutcome struct { result BatchAddResult err error @@ -386,8 +384,11 @@ func (c *RequestQueueClient) addBatchWithRetries(ctx context.Context, batch []se for attempt := 0; attempt <= settings.maxRetries; attempt++ { response, err := c.batchAddChunk(ctx, remaining, settings.forefront) if err != nil { - outcome.err = err + // An API or transport error leaves the rest of the batch unprocessed. outcome.result.UnprocessedRequests = unprocessedOf(remaining) + if ctx.Err() != nil || isResponseDecodingError(err) { + outcome.err = err + } return outcome } outcome.result.ProcessedRequests = append(outcome.result.ProcessedRequests, response.ProcessedRequests...) @@ -399,7 +400,7 @@ func (c *RequestQueueClient) addBatchWithRetries(ctx context.Context, batch []se if len(remaining) == 0 || attempt == settings.maxRetries { return outcome } - if !sleepWithContext(ctx, unprocessedRetryDelay(settings.minDelay, attempt)) { + if !sleepWithContext(ctx, c.unprocessedRetryDelay(settings.minDelay, attempt)) { outcome.err = ctx.Err() outcome.result.UnprocessedRequests = unprocessedOf(remaining) return outcome @@ -408,10 +409,21 @@ func (c *RequestQueueClient) addBatchWithRetries(ctx context.Context, batch []se return outcome } -// unprocessedRetryDelay returns the delay before the retry that follows attempt: the base delay -// doubled for each earlier retry, times a random factor in [1, 2). -func unprocessedRetryDelay(minDelay time.Duration, attempt int) time.Duration { - return time.Duration((1 + rand.Float64()) * float64(minDelay) * math.Pow(backoffFactor, float64(attempt))) +// unprocessedRetryDelay returns the delay before the retry that follows attempt: minDelay doubled +// for each earlier retry, capped at the maximum request timeout and randomized. +func (c *RequestQueueClient) unprocessedRetryDelay(minDelay time.Duration, attempt int) time.Duration { + delay := minDelay + for i := 0; i < attempt; i++ { + delay = minDuration(delay*backoffFactor, c.ctx.http.retry.timeoutMax) + } + return randomizedDelay(delay) +} + +// isResponseDecodingError reports whether err comes from decoding a response body. +func isResponseDecodingError(err error) bool { + var syntaxErr *json.SyntaxError + var typeErr *json.UnmarshalTypeError + return errors.As(err, &syntaxErr) || errors.As(err, &typeErr) } // withoutAdded returns the requests of batch whose unique key is not in added. @@ -542,8 +554,8 @@ func (c *RequestQueueClient) PaginateRequests(pageLimit *int64) *RequestQueueReq // PaginateRequestsWithOptions is [RequestQueueClient.PaginateRequests] with listing options. Limit // caps the number of requests yielded, and nil or 0 means all. Cursor, or the deprecated -// ExclusiveStartID, is where to start, and Filter applies to every page. chunkSize is the page size, -// nil for the default. The first call of Next reports invalid options. +// ExclusiveStartID, is where to start, and Filter applies to every page. chunkSize is the page +// size, nil for the default. The first call of Next reports invalid options. func (c *RequestQueueClient) PaginateRequestsWithOptions(options ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator { return &RequestQueueRequestsIterator{client: c, options: options, chunkSize: chunkSize} } diff --git a/request_queue_batch_test.go b/request_queue_batch_test.go index 2734180..27738e8 100644 --- a/request_queue_batch_test.go +++ b/request_queue_batch_test.go @@ -81,11 +81,23 @@ func decodeBody(req *http.Request) []byte { case "br": reader = brotli.NewReader(req.Body) case "gzip": - reader, _ = gzip.NewReader(req.Body) + gz, err := gzip.NewReader(req.Body) + if err != nil { + return nil + } + reader = gz case "zstd": - reader, _ = zstd.NewReader(req.Body) + zr, err := zstd.NewReader(req.Body) + if err != nil { + return nil + } + defer zr.Close() + reader = zr + } + data, err := io.ReadAll(reader) + if err != nil { + return nil } - data, _ := io.ReadAll(reader) return data } @@ -176,28 +188,84 @@ func TestBatchAddRequestsSendsAtMostMaxParallelBatchesAtOnce(t *testing.T) { } } -func TestBatchAddRequestsReturnsTheErrorWithTheResultOfTheOtherBatches(t *testing.T) { - failure := errors.New("connection refused") +func TestBatchAddRequestsLeavesTheRequestsOfAFailedBatchUnprocessed(t *testing.T) { backend := &batchBackend{fail: func(call int) error { if call == 0 { - return failure + return errors.New("connection refused") } return nil }} options := BatchAddRequestsOptions{MaxParallel: Ptr(int64(1))} result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(30), options) - if !errors.Is(err, failure) { - t.Fatalf("err = %v", err) + if err != nil { + t.Fatalf("a failed batch is not an error of the call: %v", err) } if len(result.UnprocessedRequests) != 25 || len(result.ProcessedRequests) != 5 { - t.Fatalf("the failed batch must be unprocessed and the other one added: processed %d, unprocessed %d", - len(result.ProcessedRequests), len(result.UnprocessedRequests)) + t.Fatalf("processed %d, unprocessed %d", len(result.ProcessedRequests), len(result.UnprocessedRequests)) } if first := result.UnprocessedRequests[0]; first.UniqueKey != "key-aa" || first.URL == "" || first.Method != "GET" { t.Fatalf("an unprocessed request keeps its method, unique key and URL: %+v", first) } } +func TestBatchAddRequestsKeepsWhatWasAddedWhenARetryFails(t *testing.T) { + backend := &batchBackend{ + reject: func(call int, requests []RequestQueueRequest) map[string]bool { + if call == 0 { + return map[string]bool{"key-ab": true, "key-ac": true} + } + return nil + }, + fail: func(call int) error { + if call == 1 { + return errors.New("connection reset") + } + return nil + }, + } + result, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequestsWithOptions(context.Background(), uniqueRequests(4), fastBatchOptions()) + if err != nil { + t.Fatal(err) + } + if len(result.ProcessedRequests) != 2 || len(result.UnprocessedRequests) != 2 { + t.Fatalf("processed %d, unprocessed %d", len(result.ProcessedRequests), len(result.UnprocessedRequests)) + } + for _, r := range result.UnprocessedRequests { + if r.UniqueKey != "key-ab" && r.UniqueKey != "key-ac" { + t.Fatalf("an added request is reported as unprocessed: %+v", r) + } + } +} + +func TestBatchAddRequestsReturnsTheErrorWhenTheContextEndsDuringARequest(t *testing.T) { + backend := &batchBackend{delay: 100 * time.Millisecond, fail: func(int) error { return context.DeadlineExceeded }} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequests(ctx, uniqueRequests(2), false) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v", err) + } +} + +func TestBatchAddRequestsReturnsAnErrorForAnUndecodableResponse(t *testing.T) { + backend := &mockBackend{responses: constant(200, `{"data":`)} + _, err := testClient(backend, 0).RequestQueue("q1").BatchAddRequests(context.Background(), uniqueRequests(1), false) + if err == nil { + t.Fatal("expected an error") + } +} + +func TestUnprocessedRetryDelayStaysWithinTheCap(t *testing.T) { + client := testClient(&batchBackend{}, 0).RequestQueue("q1") + minDelay := 500 * time.Millisecond + for _, attempt := range []int{0, 1, 5, 40, 200} { + delay := client.unprocessedRetryDelay(minDelay, attempt) + if delay < minDelay || delay >= 2*client.ctx.http.retry.timeoutMax { + t.Fatalf("attempt %d: delay %v", attempt, delay) + } + } +} + func TestBatchAddRequestsStopsRetryingWhenTheContextEnds(t *testing.T) { backend := &batchBackend{reject: func(int, []RequestQueueRequest) map[string]bool { return map[string]bool{"key-aa": true} }} ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) diff --git a/request_queue_collection.go b/request_queue_collection.go index 78a4413..29baf08 100644 --- a/request_queue_collection.go +++ b/request_queue_collection.go @@ -20,8 +20,8 @@ func (c *RequestQueueCollectionClient) List(ctx context.Context, options Storage } // Iterate returns a lazy iterator over the request queues matching the options, fetching pages on -// demand. The options' Limit caps the total number of queues yielded (unset means all); the page size -// is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of queues yielded (unset means all); the page +// size is chunkSize (nil for the server default). func (c *RequestQueueCollectionClient) Iterate(options StorageListOptions, chunkSize *int64) *ListIterator[RequestQueue] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[RequestQueue], error) { opts := options diff --git a/run.go b/run.go index 1ac71c3..0e0bf4f 100644 --- a/run.go +++ b/run.go @@ -213,8 +213,7 @@ func (c *RunClient) Charge(ctx context.Context, options RunChargeOptions) error const chargeIdempotencyHeader = "idempotency-key" // generateIdempotencyKey builds a per-charge idempotency key of the form -// "{runId}-{eventName}-{timestampMillis}-{random}". It need not be cryptographically secure — only -// unique enough to avoid collisions within a request. +// "{runId}-{eventName}-{timestampMillis}-{random}". It must be unique, not secure. func (c *RunClient) generateIdempotencyKey(eventName string) string { return c.id + "-" + eventName + "-" + strconv.FormatInt(time.Now().UnixMilli(), 10) + "-" + diff --git a/schedule.go b/schedule.go index b3b20f1..166ae87 100644 --- a/schedule.go +++ b/schedule.go @@ -20,8 +20,8 @@ func (c *ScheduleCollectionClient) List(ctx context.Context, options ListOptions } // Iterate returns a lazy iterator over the schedules matching the options, fetching pages on -// demand. The options' Limit caps the total number of schedules yielded (unset means all); the page size -// is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of schedules yielded (unset means all); the page +// size is chunkSize (nil for the server default). func (c *ScheduleCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[Schedule] { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[Schedule], error) { opts := options diff --git a/store_collection.go b/store_collection.go index 427d409..8c48ee6 100644 --- a/store_collection.go +++ b/store_collection.go @@ -65,8 +65,8 @@ func (c *StoreCollectionClient) List(ctx context.Context, options StoreListOptio type StoreActorIterator = ListIterator[ActorStoreListItem] // Iterate returns a lazy iterator over the Store Actors matching the options, fetching pages on -// demand. The options' Limit caps the total number of Actors yielded (unset means all); the page size -// is chunkSize (nil for the server default). +// demand. The options' Limit caps the total number of Actors yielded (unset means all); the page +// size is chunkSize (nil for the server default). func (c *StoreCollectionClient) Iterate(options StoreListOptions, chunkSize *int64) *StoreActorIterator { return newListIterator(options.Limit, chunkSize, offsetVal(options.Offset), func(ctx context.Context, offset, limit int64) (PaginationList[ActorStoreListItem], error) { opts := options diff --git a/unit_http_test.go b/unit_http_test.go index c0a832e..d9e98d2 100644 --- a/unit_http_test.go +++ b/unit_http_test.go @@ -340,6 +340,23 @@ func TestWithRequestCompressionZstd(t *testing.T) { } } +// zstdCompress is safe for concurrent use and round-trips. +func TestZstdCompressConcurrently(t *testing.T) { + body := []byte(strings.Repeat("compress me ", 500)) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + out, err := zstdCompress(body) + if err != nil || len(out) >= len(body) { + t.Errorf("compressed %d bytes to %d: %v", len(body), len(out), err) + } + }() + } + wg.Wait() +} + // WithRequestCompression(RequestCompressionGzip) must use gzip only, even for a body brotli // would otherwise have compressed first. func TestWithRequestCompressionGzipOnly(t *testing.T) { diff --git a/wait_for_finish_test.go b/wait_for_finish_test.go index d4f2f2f..55347d6 100644 --- a/wait_for_finish_test.go +++ b/wait_for_finish_test.go @@ -103,7 +103,6 @@ func TestWaitForFinishReturnsNotFoundWhenBudgetExhaustedWhileCurrently404(t *tes // An INDEFINITE WaitForFinish (nil waitSecs) against a permanent 404 must NOT hang forever: // it polls through 404s on a pure time bound, so a cancelled context unblocks it promptly. -// This is the regression test for the waitForFinish hang bug. func TestWaitForFinishIndefiniteDoesNotHangOn404(t *testing.T) { backend := &mockBackend{responses: notFound()} client := testClient(backend, 0) From 9884c98341d74c508f789fb5b2c960b5ca273172 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:39:24 +0000 Subject: [PATCH 5/6] fix: return the queue ID error from BatchAddRequests, cap the first retry delay, document the batch behavior Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- CHANGELOG.md | 5 +++-- docs/storages.md | 3 ++- request_queue.go | 10 +++++++--- request_queue_batch_test.go | 20 +++++++++++++++----- unit_http_test.go | 2 +- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10178ef..441ad1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,9 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- `BatchAddRequests` sends the requests that the API did not process again, 3 times by default, +- `BatchAddRequests` retries the requests that the API did not process, up to 3 times by default, and sends up to 5 batches at once. A batch that fails with an API or transport error leaves its - requests that were not added in `UnprocessedRequests`, and it is not an error of the call. + requests that were not added in `UnprocessedRequests`, and it is not an error of the call. An + undecodable response or an ended context is still an error. - Shortened and corrected the documentation and code comments. - Bumped `APISpecVersion` to `v2-2026-11-09T143926Z` and `ClientVersion` to `0.16.0`. diff --git a/docs/storages.md b/docs/storages.md index 9578a24..fd4545e 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -291,7 +291,7 @@ Single queue: `client.RequestQueue(id)`: | `PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator` | Lazy iterator over all requests (yields `*RequestQueueRequest`). | | `PaginateRequestsWithOptions(ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator` | The same with `Limit` (caps the total across pages), `Cursor` (where to start) and `Filter` (every page); `chunkSize` is the page size. | | `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests with the default `BatchAddRequestsOptions`. See `BatchAddRequestsWithOptions`. | -| `BatchAddRequestsWithOptions(ctx, []RequestQueueRequest, BatchAddRequestsOptions) (BatchAddResult, error)` | Add many requests, split into batches within the API limits of 25 requests and 9 MiB, with up to `MaxParallel` (5) batches sent at once. The requests that the API does not process are sent again, up to `MaxUnprocessedRequestsRetries` (3) times, after a delay that starts at `MinDelayBetweenUnprocessedRequestsRetries` (500ms) and doubles. Fails before sending anything if one request is too large for a batch. If a batch fails with an error, the error is returned together with the result of all batches, in which the requests of that batch that were not added are unprocessed. | +| `BatchAddRequestsWithOptions(ctx, []RequestQueueRequest, BatchAddRequestsOptions) (BatchAddResult, error)` | Add many requests, split into batches within the API limits of 25 requests and 9 MiB, with up to `MaxParallel` (5) batches sent at once. The requests that the API does not process are sent again, up to `MaxUnprocessedRequestsRetries` (3) times, after a delay that starts at `MinDelayBetweenUnprocessedRequestsRetries` (500ms) and doubles. Fails before sending anything if one request is too large for a batch. A batch that fails with an API or transport error is not an error of the call: its requests that were not added are unprocessed, and the error is not reported. To find out why requests stay unprocessed, add one with `AddRequest`. The call returns an error, with the result so far, when a response cannot be decoded or `ctx` ends. | | `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload: a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`). Returns the raw API response. | | `ListAndLockHead(ctx, lockSecs int64, limit *int64) (json.RawMessage, error)` | Fetch the head of the queue and lock the returned requests for `lockSecs` seconds. Returns the raw API response. | | `ProlongRequestLock(ctx, id string, lockSecs int64, forefront bool) (json.RawMessage, error)` | Extend the lock on a request by `lockSecs` seconds; `forefront` controls re-queue position when the lock expires. | @@ -351,6 +351,7 @@ Return types: | `RequestID` | `string` | ID of the affected request. | | `WasAlreadyPresent` | `bool` | Whether the request was already in the queue. | | `WasAlreadyHandled` | `bool` | Whether the request had already been handled. | +| `UniqueKey` | `string` | Unique key of the affected request. Only the results of a batch add set it. | `BatchAddRequests` returns a `BatchAddResult`: diff --git a/request_queue.go b/request_queue.go index d1ff47a..05f17cd 100644 --- a/request_queue.go +++ b/request_queue.go @@ -330,10 +330,14 @@ func (c *RequestQueueClient) BatchAddRequests(ctx context.Context, requests []Re // // The result lists the requests that were added and those that are still unprocessed. A batch // that fails with an API or transport error is not an error of the call: its requests that were -// not added are unprocessed. The call returns an error for a response that cannot be decoded, and -// the error of ctx, with the result so far, when ctx ends. +// not added are unprocessed, and the error is not reported. To find out why requests stay +// unprocessed, add one with [RequestQueueClient.AddRequest]. The call returns an error, with the +// result so far, when a response cannot be decoded or ctx ends. func (c *RequestQueueClient) BatchAddRequestsWithOptions(ctx context.Context, requests []RequestQueueRequest, options BatchAddRequestsOptions) (BatchAddResult, error) { var merged BatchAddResult + if c.ctx.idErr != nil { + return merged, c.ctx.idErr + } settings, err := options.resolve() if err != nil { return merged, err @@ -412,7 +416,7 @@ func (c *RequestQueueClient) addBatchWithRetries(ctx context.Context, batch []se // unprocessedRetryDelay returns the delay before the retry that follows attempt: minDelay doubled // for each earlier retry, capped at the maximum request timeout and randomized. func (c *RequestQueueClient) unprocessedRetryDelay(minDelay time.Duration, attempt int) time.Duration { - delay := minDelay + delay := minDuration(minDelay, c.ctx.http.retry.timeoutMax) for i := 0; i < attempt; i++ { delay = minDuration(delay*backoffFactor, c.ctx.http.retry.timeoutMax) } diff --git a/request_queue_batch_test.go b/request_queue_batch_test.go index 27738e8..cff4894 100644 --- a/request_queue_batch_test.go +++ b/request_queue_batch_test.go @@ -257,15 +257,25 @@ func TestBatchAddRequestsReturnsAnErrorForAnUndecodableResponse(t *testing.T) { func TestUnprocessedRetryDelayStaysWithinTheCap(t *testing.T) { client := testClient(&batchBackend{}, 0).RequestQueue("q1") - minDelay := 500 * time.Millisecond - for _, attempt := range []int{0, 1, 5, 40, 200} { - delay := client.unprocessedRetryDelay(minDelay, attempt) - if delay < minDelay || delay >= 2*client.ctx.http.retry.timeoutMax { - t.Fatalf("attempt %d: delay %v", attempt, delay) + timeoutMax := client.ctx.http.retry.timeoutMax + for _, minDelay := range []time.Duration{500 * time.Millisecond, 2 * timeoutMax} { + for _, attempt := range []int{0, 1, 5, 40, 200} { + delay := client.unprocessedRetryDelay(minDelay, attempt) + if delay <= 0 || delay >= 2*timeoutMax { + t.Fatalf("minDelay %v, attempt %d: delay %v", minDelay, attempt, delay) + } } } } +func TestBatchAddRequestsReturnsTheErrorOfAnInvalidQueueID(t *testing.T) { + backend := &batchBackend{} + _, err := testClient(backend, 0).RequestQueue("").BatchAddRequests(context.Background(), uniqueRequests(2), false) + if err == nil || backend.calls != 0 { + t.Fatalf("err = %v, calls = %d", err, backend.calls) + } +} + func TestBatchAddRequestsStopsRetryingWhenTheContextEnds(t *testing.T) { backend := &batchBackend{reject: func(int, []RequestQueueRequest) map[string]bool { return map[string]bool{"key-aa": true} }} ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) diff --git a/unit_http_test.go b/unit_http_test.go index d9e98d2..172ced4 100644 --- a/unit_http_test.go +++ b/unit_http_test.go @@ -340,7 +340,7 @@ func TestWithRequestCompressionZstd(t *testing.T) { } } -// zstdCompress is safe for concurrent use and round-trips. +// zstdCompress is safe for concurrent use and shrinks a compressible body. func TestZstdCompressConcurrently(t *testing.T) { body := []byte(strings.Repeat("compress me ", 500)) var wg sync.WaitGroup From 3f9e3bca250f5a3c21e686ebaa02e2b2ba379f51 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 14:42:48 +0000 Subject: [PATCH 6/6] docs: document the retry delay cap and tighten its test Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- docs/storages.md | 2 +- request_queue.go | 2 +- request_queue_batch_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/storages.md b/docs/storages.md index fd4545e..b7a273f 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -291,7 +291,7 @@ Single queue: `client.RequestQueue(id)`: | `PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator` | Lazy iterator over all requests (yields `*RequestQueueRequest`). | | `PaginateRequestsWithOptions(ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator` | The same with `Limit` (caps the total across pages), `Cursor` (where to start) and `Filter` (every page); `chunkSize` is the page size. | | `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests with the default `BatchAddRequestsOptions`. See `BatchAddRequestsWithOptions`. | -| `BatchAddRequestsWithOptions(ctx, []RequestQueueRequest, BatchAddRequestsOptions) (BatchAddResult, error)` | Add many requests, split into batches within the API limits of 25 requests and 9 MiB, with up to `MaxParallel` (5) batches sent at once. The requests that the API does not process are sent again, up to `MaxUnprocessedRequestsRetries` (3) times, after a delay that starts at `MinDelayBetweenUnprocessedRequestsRetries` (500ms) and doubles. Fails before sending anything if one request is too large for a batch. A batch that fails with an API or transport error is not an error of the call: its requests that were not added are unprocessed, and the error is not reported. To find out why requests stay unprocessed, add one with `AddRequest`. The call returns an error, with the result so far, when a response cannot be decoded or `ctx` ends. | +| `BatchAddRequestsWithOptions(ctx, []RequestQueueRequest, BatchAddRequestsOptions) (BatchAddResult, error)` | Add many requests, split into batches within the API limits of 25 requests and 9 MiB, with up to `MaxParallel` (5) batches sent at once. The requests that the API does not process are sent again, up to `MaxUnprocessedRequestsRetries` (3) times, after a delay that starts at `MinDelayBetweenUnprocessedRequestsRetries` (500ms), doubles, and is capped at `WithTimeoutMax`. Fails before sending anything if one request is too large for a batch. A batch that fails with an API or transport error is not an error of the call: its requests that were not added are unprocessed, and the error is not reported. To find out why requests stay unprocessed, add one with `AddRequest`. The call returns an error, with the result so far, when a response cannot be decoded or `ctx` ends. | | `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload: a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`). Returns the raw API response. | | `ListAndLockHead(ctx, lockSecs int64, limit *int64) (json.RawMessage, error)` | Fetch the head of the queue and lock the returned requests for `lockSecs` seconds. Returns the raw API response. | | `ProlongRequestLock(ctx, id string, lockSecs int64, forefront bool) (json.RawMessage, error)` | Extend the lock on a request by `lockSecs` seconds; `forefront` controls re-queue position when the lock expires. | diff --git a/request_queue.go b/request_queue.go index 05f17cd..a5bca74 100644 --- a/request_queue.go +++ b/request_queue.go @@ -276,7 +276,7 @@ type BatchAddRequestsOptions struct { // MaxParallel is how many batches are sent at once. Defaults to 5. MaxParallel *int64 // MinDelayBetweenUnprocessedRequestsRetries is the base delay before a retry. It doubles with - // each retry and is randomized by up to 100%. Defaults to 500ms. + // each retry, is capped at [WithTimeoutMax], and is randomized by up to 100%. Defaults to 500ms. MinDelayBetweenUnprocessedRequestsRetries *time.Duration } diff --git a/request_queue_batch_test.go b/request_queue_batch_test.go index cff4894..afd896f 100644 --- a/request_queue_batch_test.go +++ b/request_queue_batch_test.go @@ -261,7 +261,7 @@ func TestUnprocessedRetryDelayStaysWithinTheCap(t *testing.T) { for _, minDelay := range []time.Duration{500 * time.Millisecond, 2 * timeoutMax} { for _, attempt := range []int{0, 1, 5, 40, 200} { delay := client.unprocessedRetryDelay(minDelay, attempt) - if delay <= 0 || delay >= 2*timeoutMax { + if delay < minDuration(minDelay, timeoutMax) || delay >= 2*timeoutMax { t.Fatalf("minDelay %v, attempt %d: delay %v", minDelay, attempt, delay) } }