diff --git a/CHANGELOG.md b/CHANGELOG.md index e8a8f7c..171a1e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,39 @@ All notable changes to the Apify Go client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.14.0] - 2026-10-30 + +### Added + +- `RunListOptions.Deployment` and `StoreListOptions.IncludeInputSchema` list parameters. +- `UserClient` limits and monthly usage methods work on `client.User(id)` too, not only `Me()`. +- `StreamedLog` (`RunClient.NewStreamedLog`) redirects a run's log to a writer; `ActorClient.Call` + and `CallRaw` use it by default, `ActorStartOptions.Log` configures or disables it. +- `RunClient.IterateDatasetItems` and `IterateRunDatasetItems[T]` iterate a run's dataset while it + is still being filled. +- `KeyValueStoreClient.SetRecordRawWithOptions` and `SetRecordJSONWithOptions`. +- `RequestQueueClient.PaginateRequestsWithOptions` iterates with `Limit`, `Cursor` and `Filter`. +- Typed URL fields on the models (`ConsoleURL`, `ItemsPublicURL`, `RecordPublicURL`, + `User.Profile`, `WebhookDispatch.Webhook` and others), returned normalized. + +### Changed + +- Bumped `APISpecVersion` to `v2-2026-10-29T100357Z` and `ClientVersion` to `0.14.0`. +- **Breaking:** `ActorVersionCollectionClient.List` and `Iterate` take no options. +- The typed URL fields are no longer in the `Extra` map of their model. +- `ListRequestsOptions.ExclusiveStartID` is deprecated in favor of `Cursor`. +- `RunClient.GetStreamedLog` is deprecated: it returns the raw stream, use `NewStreamedLog`. +- `Call` starts the run without a client-side timeout. +- Requests that are not idempotent (`Start`, `Build`, `PushItems`) are not retried after a timeout. +- Some `KeyValueStoreClient` and `DatasetClient` methods use the "long" timeout tier. +- Storages, log and public URLs of a `LastRun` carry its `status` and `origin` filters. +- Documented that `GetRecord` and `SetRecordRaw` handle raw bytes. + +### Fixed + +- `DatasetClient.PushItems` rejects values that are not a JSON object or an array of objects. +- Methods that lacked an ID check now fail on an empty or dot-segment ID. + ## [0.13.1] - 2026-10-26 ### Changed diff --git a/README.md b/README.md index 40190dd..d582c0d 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ request queues, tasks, schedules, webhooks, the store, users and logs). - Transparent authentication, `User-Agent` header, retries with exponential backoff, and per-request timeouts applied to every call. - Replaceable HTTP transport (the `HTTPBackend` interface) with a default implementation. -- Convenience helpers: start-and-wait, build/run polling, lazy store and request-queue - iterators, dataset export, signed public URLs, request-queue locking, and more. +- Convenience helpers: start-and-wait with live log redirection, build/run polling, lazy store + and request-queue iterators, live iteration of a running Actor's dataset items, dataset + export, signed public URLs, request-queue locking, and more. - Forward-compatible models that keep unknown API fields in an `Extra` map. - A single third-party dependency (`github.com/andybalholm/brotli`, used for Brotli request-body compression); everything else is the Go standard library. @@ -124,18 +125,27 @@ func main() { ### Timeouts Every request the client sends is assigned one of three timeout tiers, matching the kind of -operation its method performs — `short` for a metadata read/write, `medium` for a listing, -batch or trigger call, `long` for a download, upload or streaming call — except the polling -requests behind `WaitForFinish`/`Call`, which run with no client-imposed timeout of their own -(the pure time budget `waitSecs`/`timeoutSecs` already bounds the overall wait). The duration of -each tier is configured on the client (see the table above); a timeout still grows with each -retry, capped at `WithTimeoutMax`. - -A single call that needs a different bound already has full control via the -`context.Context` every method takes: canceling it, or giving it an earlier deadline with -`context.WithTimeout`, overrides whichever tier the method would otherwise use (the client's -internal per-attempt deadline is always the *later* of the two to apply — i.e. -`context.WithTimeout` composes with whichever is sooner). +operation its method performs: `short` for a metadata read/write, `medium` for a listing, batch or +trigger call, `long` for a download, upload or streaming call. The duration of each tier is +configured on the client (see the table above); a timeout still grows with each retry, capped at +`WithTimeoutMax`. + +Two kinds of request have no client-side timeout of their own: the polling requests behind +`WaitForFinish` and `Call`, and the start request of `Call`. `waitSecs` bounds how long `Call` and +`WaitForFinish` wait for the run, and the `context.Context` bounds the whole call. + +Every method takes a `context.Context`, and its deadline or cancellation can only shorten a +request: the request ends when its tier's timeout or the context's deadline passes, whichever is +sooner. To give one call a shorter bound than its tier, pass a context created with +`context.WithTimeout`; a context cannot extend a tier, so raise the tier (`WithTimeoutShort`, +`WithTimeoutMedium`, `WithTimeoutLong`, `WithTimeoutMax`) for a call that needs longer. + +A request that timed out is retried like any other transient failure, with a longer timeout, +except one that is not idempotent, where the timed-out attempt may already have taken effect: +starting a run (`Start`, `StartRaw`, a task's `Start`), building an Actor (`Build`), pushing +dataset items (`PushItems`) and, when `SetRecordOptions.DoNotRetryTimeouts` is set, storing a +record. Those fail on the first timeout. The other failures (a 429 or 5xx response, a network +error) are retried for them as well. A method with a `waitForFinishSecs`/`WaitForFinish` parameter that asks the API to hold the response (e.g. `RunClient.GetWithWait`, `ActorClient.Start`'s `WaitForFinish` option) extends @@ -158,6 +168,20 @@ Apify platform. It is driven solely by the `APIFY_IS_AT_HOME` environment variab variable the JavaScript reference client reads); if it is set to a non-empty value, the flag is `true`, otherwise `false`. +### URL fields + +The fields the API specification declares as URLs (`consoleUrl`, `containerUrl`, `itemsPublicUrl`, +`keysPublicUrl`, `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, the webhook `requestUrl`, a +run's `output` links, a Store Actor's `url` and `userPictureUrl`, and a user profile's +`pictureUrl` and `websiteUrl`) are returned in a normalized form, close to the WHATWG URL serialization that the JavaScript +client returns: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IPv4 or +IPv6 host is written canonically, a default port is dropped, an empty path becomes `/`, dot +segments are resolved, and unsafe characters are percent-encoded. It covers the http(s), ws(s), +ftp and file URLs the API returns; it does not apply the full Unicode (UTS #46) mapping to a +non-ASCII host. Compare a field against a stored URL after normalizing the stored one the same +way (for example by reading it back from the API). A value that is not an absolute URL makes +decoding the response fail. + ## Resource clients | Accessor | Returns | Purpose | @@ -173,7 +197,7 @@ variable the JavaScript reference client reads); if it is set to a non-empty val | `client.Webhooks()` / `client.Webhook(id)` | `*WebhookCollectionClient` / `*WebhookClient` | List/create webhooks; manage and test a single webhook. | | `client.WebhookDispatches()` / `client.WebhookDispatch(id)` | `*WebhookDispatchCollectionClient` / `*WebhookDispatchClient` | List/inspect webhook dispatches. | | `client.Store()` | `*StoreCollectionClient` | Browse and iterate the Apify Store. | -| `client.Me()` / `client.User(id)` | `*UserClient` | Account details, usage and limits (account scope for `Me()`). | +| `client.Me()` / `client.User(id)` | `*UserClient` | Account details, usage and limits (for your own account or your organization with `User(id)`). | | `client.Log(buildOrRunID)` | `*LogClient` | Read or stream a build/run log. | ## Error handling @@ -258,7 +282,7 @@ func main() { - `apify.ClientVersion` — the semantic version of this library. - `apify.APISpecVersion` — the Apify OpenAPI spec version this client was built against - (`v2-2026-10-26T112451Z`). + (`v2-2026-10-29T100357Z`). ### Releasing @@ -295,7 +319,7 @@ APIFY_TOKEN= go run ./examples/run_store_actor | `run_store_actor` | Run a Store Actor, wait for it, and read its default dataset. | | `run_and_last_run_storages` | Start a run, wait for it to finish, then fetch the Actor's last run and its storages. | | `iterate_store` | Lazily iterate Actors in the Apify Store. | -| `log_redirection` | Start an Actor and stream its log in real time. | +| `log_redirection` | Run an Actor and redirect its log to stdout in real time. | | `create_build_run_actor` | Create an Actor, build it, run it, and print the run log. | | `public_build_no_token` | Fetch a public Actor's default build with an unauthenticated client (no token). | diff --git a/actor.go b/actor.go index 0ad71fa..67f7b67 100644 --- a/actor.go +++ b/actor.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "net/http" + "time" ) // ActorStartOptions configures starting an Actor or task run @@ -43,6 +44,11 @@ type ActorStartOptions struct { // actor-memory-limit-exceeded as well and never starts, so WaitForResourcesIndefinitely // retries it forever. WaitForResources *int64 + // Log configures the redirection of the run's log that [ActorClient.Call] and + // [ActorClient.CallRaw] do while they wait (see [StreamedLog]). Nil redirects it to os.Stderr + // with the default prefix; set Log.Disabled to turn the redirection off. Start and StartRaw + // ignore it. + Log *StreamedLogOptions } func (o ActorStartOptions) apply(q *QueryParams) { @@ -133,7 +139,7 @@ func (c *ActorClient) Start(ctx context.Context, input any, options ActorStartOp if err != nil { return ActorRun{}, err } - return c.startWithBody(ctx, body, contentTypeJSON, options) + return c.startWithBody(ctx, body, contentTypeJSON, options, c.startTimeout(options)) } // StartRaw starts the Actor with a raw request body instead of a JSON-serializable value. @@ -144,22 +150,30 @@ func (c *ActorClient) Start(ctx context.Context, input any, options ActorStartOp // handles the serialization. Mirrors the reference client, whose Actor input accepts a plain // object, an array of them, or raw bytes. func (c *ActorClient) StartRaw(ctx context.Context, input []byte, options ActorStartOptions) (ActorRun, error) { - return c.startWithBody(ctx, input, "application/octet-stream", options) + return c.startWithBody(ctx, input, "application/octet-stream", options, c.startTimeout(options)) +} + +// startTimeout is the timeout of a start request: the "medium" tier, extended to cover the time +// the API may hold the response for options.WaitForFinish. +func (c *ActorClient) startTimeout(options ActorStartOptions) time.Duration { + return extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) } // startWithBody is the shared implementation of Start and StartRaw: it applies the run-start // options as query parameters and posts the given body, falling back to defaultContentType only -// when the caller did not set options.ContentType. -func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions) (ActorRun, error) { +// when the caller did not set options.ContentType. timeout bounds each start request. +// +// A start request that times out is not retried: it may already have created the run, and a +// retry would then start a second, billed one. +func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions, timeout time.Duration) (ActorRun, error) { params := NewQueryParams() options.apply(params) contentType := defaultContentType if options.ContentType != nil && *options.ContentType != "" { contentType = *options.ContentType } - timeout := extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) return startWaitingForResources(ctx, options.WaitForResources, func() (ActorRun, error) { - return postWithBody[ActorRun](ctx, c.ctx, "runs", params, body, contentType, timeout) + return postWithBody[ActorRun](ctx, c.ctx.withoutTimeoutRetry(), "runs", params, body, contentType, timeout) }) } @@ -167,23 +181,39 @@ func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultCon // // waitSecs bounds the wait; nil waits indefinitely. It returns the finished run (or the // still-running run if the wait budget was exhausted). +// +// While it waits, it redirects the run's log to os.Stderr (or to options.Log.Output), prefixed +// with the Actor name and the run ID, like the reference client's call(): see [StreamedLog], and +// set options.Log.Disabled to turn it off. +// +// Neither the start request nor the polls have a client-side timeout of their own (matching the +// reference client's call(), whose default timeout is noTimeout): ctx bounds the whole call. func (c *ActorClient) Call(ctx context.Context, input any, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { - run, err := c.Start(ctx, input, options) + body, err := marshalInput(input) if err != nil { return ActorRun{}, err } - // Use the root client's run client so polling targets the canonical run route. - return c.root.Run(run.ID).WaitForFinish(ctx, waitSecs) + return c.callWithBody(ctx, body, contentTypeJSON, options, waitSecs) } // CallRaw starts the Actor with a raw request body ([ActorClient.StartRaw]) and waits for it to // finish, exactly like [ActorClient.Call] but for a non-JSON input. func (c *ActorClient) CallRaw(ctx context.Context, input []byte, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { - run, err := c.StartRaw(ctx, input, options) + return c.callWithBody(ctx, input, "application/octet-stream", options, waitSecs) +} + +// callWithBody is the shared implementation of Call and CallRaw. +func (c *ActorClient) callWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { + run, err := c.startWithBody(ctx, body, defaultContentType, options, noRequestTimeout) if err != nil { return ActorRun{}, err } - return c.root.Run(run.ID).WaitForFinish(ctx, waitSecs) + // Use the root client's run client so polling targets the canonical run route. + runClient := c.root.Run(run.ID) + streamed := startLogRedirection(ctx, runClient, options.Log) + finished, err := runClient.WaitForFinish(ctx, waitSecs) + stopLogRedirection(ctx, streamed, options.Log) + return finished, err } // Build builds the given version of the Actor and returns the created build. @@ -195,7 +225,8 @@ func (c *ActorClient) Build(ctx context.Context, versionNumber string, options A AddBool("useCache", options.UseCache). AddInt("waitForFinish", options.WaitForFinish) timeout := extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) - return postWithBody[Build](ctx, c.ctx, "builds", params, nil, contentTypeJSON, timeout) + // A build that timed out is not retried: it may already have been created. + return postWithBody[Build](ctx, c.ctx.withoutTimeoutRetry(), "builds", params, nil, contentTypeJSON, timeout) } // DefaultBuild resolves the Actor's default build and returns a client for it. @@ -203,6 +234,9 @@ func (c *ActorClient) Build(ctx context.Context, versionNumber string, options A // waitForFinish optionally bounds how long (in seconds) the API waits for the build to // finish before responding, matching the reference client's defaultBuild(options). func (c *ActorClient) DefaultBuild(ctx context.Context, waitForFinish *int64) (*BuildClient, error) { + if c.ctx.idErr != nil { + return nil, c.ctx.idErr + } params := NewQueryParams() params.AddInt("waitForFinish", waitForFinish) url := params.applyToURL(c.ctx.subURL("builds/default")) @@ -237,6 +271,9 @@ func (c *ActorClient) ValidateInput(ctx context.Context, input any) (json.RawMes // On success it returns the raw JSON validation result from the API (a JSON object reporting // whether the input is valid and, if not, the schema violations). func (c *ActorClient) ValidateInputForBuild(ctx context.Context, input any, build string) (json.RawMessage, error) { + if c.ctx.idErr != nil { + return nil, c.ctx.idErr + } body, err := json.Marshal(input) if err != nil { return nil, err diff --git a/actor_version.go b/actor_version.go index bf050d8..c53344b 100644 --- a/actor_version.go +++ b/actor_version.go @@ -12,22 +12,18 @@ func newActorVersionCollectionClient(hc *httpClient, actorURL string) *ActorVers return &ActorVersionCollectionClient{ctx: newCollectionContext(hc, actorURL, "versions")} } -// List lists the Actor's versions. -func (c *ActorVersionCollectionClient) List(ctx context.Context, options ListOptions) (PaginationList[ActorVersion], error) { - params := NewQueryParams() - options.apply(params) - return listResource[ActorVersion](ctx, c.ctx, "", params) +// List lists the Actor's versions. The endpoint takes no pagination parameters (per the OpenAPI +// specification, it always returns every version in one response), so neither does this method. +func (c *ActorVersionCollectionClient) List(ctx context.Context) (PaginationList[ActorVersion], error) { + return listResource[ActorVersion](ctx, c.ctx, "", NewQueryParams()) } -// Iterate returns a lazy iterator over the Actor's versions. The versions endpoint ignores -// offset/limit/desc (per the OpenAPI specification, it always returns every version in one -// response), so options and chunkSize are accepted only for interface symmetry with the other -// collections' Iterate and have no effect; the iterator drains a single page. Mirrors the -// reference client's versions().list(), which likewise exposes no pagination options. -func (c *ActorVersionCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[ActorVersion] { - return newUnpaginatedIterator(func(ctx context.Context) (PaginationList[ActorVersion], error) { - return c.List(ctx, options) - }) +// Iterate returns a lazy iterator over the Actor's versions. The versions endpoint is not +// offset-paginated (it always returns every version in one response), so there is no +// Limit/chunkSize control; the iterator drains that one page. Mirrors the reference client's +// versions().list(), which likewise exposes no pagination options. +func (c *ActorVersionCollectionClient) Iterate() *ListIterator[ActorVersion] { + return newUnpaginatedIterator(c.List) } // Create creates a new Actor version. version is any JSON-serializable version definition. diff --git a/base.go b/base.go index 6d69d25..88429f6 100644 --- a/base.go +++ b/base.go @@ -86,6 +86,14 @@ func newResourceContext(hc *httpClient, url, baseURL string, hasOwnID bool, idEr } } +// withoutTimeoutRetry returns a copy of this context whose requests do not retry an attempt that +// timed out (see [httpClient.withoutTimeoutRetry]). Use it for a request that is not idempotent. +func (c *resourceContext) withoutTimeoutRetry() *resourceContext { + clone := *c + clone.http = c.http.withoutTimeoutRetry() + return &clone +} + // withPublicOrigin overrides the origin used when building public URLs. func (c *resourceContext) withPublicOrigin(publicBaseURL string) *resourceContext { c.publicOrigin = originOf(publicBaseURL) @@ -150,8 +158,14 @@ func originOf(rawURL string) string { // reference client's catchNotFoundForResourceOrThrow. For a 404 that is unambiguous regardless // of how the client was reached (e.g. a lookup by key or request id), use [getResourceAlways]. func getResource[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (T, bool, error) { + return getResourceWithTimeout[T](ctx, c, subPath, params, c.shortTimeout()) +} + +// getResourceWithTimeout is [getResource] with the caller's chosen timeout tier, for a lookup +// that is slower than a plain metadata read (see [timeoutTiers]). +func getResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams, timeout time.Duration) (T, bool, error) { var zero T - result, err := getResourceRequired[T](ctx, c, subPath, params, c.shortTimeout()) + result, err := getResourceRequired[T](ctx, c, subPath, params, timeout) if err != nil { if c.hasOwnID && isNotFound(err) { return zero, false, nil @@ -182,16 +196,7 @@ func extendForServerWait(base time.Duration, waitForFinishSecs *int64) time.Dura // [extendForServerWait], for a get(waitForFinish) call that asks the server to hold the // response (RunClient.GetWithWait, BuildClient.GetWithWait). func getResourceWithServerWait[T any](ctx context.Context, c *resourceContext, params *QueryParams, waitForFinishSecs *int64) (T, bool, error) { - var zero T - timeout := extendForServerWait(c.shortTimeout(), waitForFinishSecs) - result, err := getResourceRequired[T](ctx, c, "", params, timeout) - if err != nil { - if c.hasOwnID && isNotFound(err) { - return zero, false, nil - } - return zero, false, err - } - return result, true, nil + return getResourceWithTimeout[T](ctx, c, "", params, extendForServerWait(c.shortTimeout(), waitForFinishSecs)) } // getResourceAlways performs a GET that unwraps the data envelope; a not-found always maps to @@ -232,6 +237,11 @@ func getResourceRequired[T any](ctx context.Context, c *resourceContext, subPath // updateResource performs a PUT with a JSON body, unwrapping the data envelope. Metadata writes // use the "short" timeout tier. func updateResource[T any](ctx context.Context, c *resourceContext, subPath string, body any) (T, error) { + return updateResourceWithTimeout[T](ctx, c, subPath, body, c.shortTimeout()) +} + +// updateResourceWithTimeout is [updateResource] with the caller's chosen timeout tier. +func updateResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subPath string, body any, timeout time.Duration) (T, error) { var zero T if c.idErr != nil { return zero, c.idErr @@ -241,7 +251,7 @@ func updateResource[T any](ctx context.Context, c *resourceContext, subPath stri return zero, err } url := c.mergedParams(NewQueryParams()).applyToURL(c.subURL(subPath)) - resp, err := c.http.call(ctx, http.MethodPut, url, data, contentTypeJSON, c.shortTimeout()) + resp, err := c.http.call(ctx, http.MethodPut, url, data, contentTypeJSON, timeout) if err != nil { return zero, err } @@ -386,15 +396,15 @@ func getRawRequired(ctx context.Context, c *resourceContext, subPath string, par return c.http.call(ctx, http.MethodGet, url, nil, "", timeout) } -// headExists performs a HEAD request, returning whether the resource exists, using the "short" -// timeout tier. A not-found always maps to (false, nil): used only for an unambiguous lookup (a -// key-value-store record by key). +// headExists performs a HEAD request, returning whether the resource exists, using the "long" +// timeout tier (the reference client's tier for a record lookup). A not-found always maps to +// (false, nil): used only for an unambiguous lookup (a key-value-store record by key). func headExists(ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (bool, error) { if c.idErr != nil { return false, c.idErr } url := c.mergedParams(params).applyToURL(c.subURL(subPath)) - _, err := c.http.call(ctx, http.MethodHead, url, nil, "", c.shortTimeout()) + _, err := c.http.call(ctx, http.MethodHead, url, nil, "", c.longTimeout()) if err != nil { if isNotFound(err) { return false, nil diff --git a/call_test.go b/call_test.go new file mode 100644 index 0000000..0662daa --- /dev/null +++ b/call_test.go @@ -0,0 +1,63 @@ +package apify + +import ( + "context" + "testing" + "time" +) + +// noLogRedirection turns the log redirection of Call off, so a test counts only the requests of +// the call itself. +var noLogRedirection = &StreamedLogOptions{Disabled: true} + +// Call has no client-side timeout of its own: a start slower than the medium tier must neither be +// aborted nor repeated (which would start a second, billed run). Start itself is bound by it. +func TestCallStartsWithoutATimeout(t *testing.T) { + run := `{"data":{"id":"run1","status":"SUCCEEDED"}}` + t.Run("Actor Call", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + got, err := slowClient(backend).Actor("a").Call(context.Background(), map[string]any{}, ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))) + if err != nil || got.ID != "run1" { + t.Fatalf("Call: %+v, %v", got, err) + } + if n := backend.calls.Load(); n != 2 { // the start and one poll + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Actor CallRaw", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Actor("a").CallRaw(context.Background(), []byte("x"), ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))); err != nil { + t.Fatalf("CallRaw: %v", err) + } + if n := backend.calls.Load(); n != 2 { + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Task Call", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Task("t").Call(context.Background(), nil, TaskStartOptions{}, Ptr(int64(0))); err != nil { + t.Fatalf("Task Call: %v", err) + } + if n := backend.calls.Load(); n != 2 { + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Start stays bound by the medium tier", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Actor("a").Start(context.Background(), nil, ActorStartOptions{}); err == nil { + t.Fatal("expected Start to time out") + } + }) + t.Run("ctx still bounds Call", func(t *testing.T) { + backend := &slowBackend{delay: time.Minute, body: run} + ctx, cancel := context.WithTimeout(context.Background(), 3*tinyTier) + defer cancel() + started := time.Now() + if _, err := slowClient(backend).Actor("a").Call(ctx, nil, ActorStartOptions{Log: noLogRedirection}, nil); err == nil { + t.Fatal("expected an error once ctx is done") + } + if time.Since(started) > 5*time.Second { + t.Fatal("Call did not stop at the ctx deadline") + } + }) +} diff --git a/dataset.go b/dataset.go index 79c83af..7551aba 100644 --- a/dataset.go +++ b/dataset.go @@ -1,8 +1,11 @@ package apify import ( + "bytes" "context" "encoding/json" + "errors" + "fmt" "io" "net/http" "strconv" @@ -163,7 +166,10 @@ func ListDatasetItems[T any](ctx context.Context, c *DatasetClient, options Data var result PaginationList[T] params := NewQueryParams() options.apply(params) - url := params.applyToURL(c.ctx.subURL("items")) + if c.ctx.idErr != nil { + return result, c.ctx.idErr + } + url := c.ctx.mergedParams(params).applyToURL(c.ctx.subURL("items")) resp, err := c.ctx.http.call(ctx, http.MethodGet, url, nil, "", c.ctx.longTimeout()) if err != nil { return result, err @@ -252,17 +258,52 @@ func (c *DatasetClient) DownloadItemsStream(ctx context.Context, format Download return stream, err } -// PushItems pushes one or more items to the dataset. items must serialize to a JSON object -// or an array of objects. +// PushItems pushes one or more items to the dataset. items must serialize to a JSON object or +// an array of objects, the only shapes the API accepts: anything else (a bare string, number or +// boolean, an array holding a non-object, nil) fails with an error before any request is sent. +// A value that is already serialized JSON can be passed as a [json.RawMessage]; a plain Go string +// is serialized as a JSON string, which is not a valid item. +// +// A push that times out is not retried: the API may already have stored the items, and a retry +// would then store them a second time. func (c *DatasetClient) PushItems(ctx context.Context, items any) error { body, err := json.Marshal(items) if err != nil { return err } - _, err = c.ctx.http.call(ctx, http.MethodPost, c.ctx.subURL("items"), body, contentTypeJSONCharset, c.ctx.mediumTimeout()) + if err := checkItemsShape(body); err != nil { + return err + } + if c.ctx.idErr != nil { + return c.ctx.idErr + } + url := c.ctx.mergedParams(NewQueryParams()).applyToURL(c.ctx.subURL("items")) + _, err = c.ctx.withoutTimeoutRetry().http.call(ctx, http.MethodPost, url, body, contentTypeJSONCharset, c.ctx.mediumTimeout()) return err } +// checkItemsShape reports whether body, a JSON document, is an object or an array of objects. +func checkItemsShape(body []byte) error { + trimmed := bytes.TrimSpace(body) + switch { + case bytes.HasPrefix(trimmed, []byte("{")): + return nil + case bytes.HasPrefix(trimmed, []byte("[")): + var elements []json.RawMessage + if err := json.Unmarshal(trimmed, &elements); err != nil { + return err + } + for i, element := range elements { + if !bytes.HasPrefix(bytes.TrimSpace(element), []byte("{")) { + return fmt.Errorf("PushItems: item %d is not a JSON object; the API only accepts objects", i) + } + } + return nil + default: + return errors.New("PushItems: items must be a JSON object or an array of objects") + } +} + // GetStatistics returns statistical information about the dataset. // // A 404 here always means the dataset itself is gone (the same way a plain Get would report @@ -300,7 +341,8 @@ func (c *DatasetClient) CreateItemsPublicURLWithFormat(ctx context.Context, opti params.AddString("format", &fmtStr) } - dataset, present, err := c.Get(ctx) + // Fetched with the "long" timeout tier, the reference client's tier for this call. + dataset, present, err := getResourceWithTimeout[Dataset](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) if err != nil { return "", err } @@ -310,7 +352,7 @@ func (c *DatasetClient) CreateItemsPublicURLWithFormat(ctx context.Context, opti params.AddString("signature", &sig) } } - return params.applyToURL(c.ctx.publicURL("items")), nil + return c.ctx.mergedParams(params).applyToURL(c.ctx.publicURL("items")), nil } // headerInt parses an integer HTTP header value, returning fallback if absent or invalid. diff --git a/dataset_push_test.go b/dataset_push_test.go new file mode 100644 index 0000000..51c2a66 --- /dev/null +++ b/dataset_push_test.go @@ -0,0 +1,73 @@ +package apify + +import ( + "context" + "encoding/json" + "testing" +) + +func TestPushItemsAcceptsOnlyObjectsAndArraysOfObjects(t *testing.T) { + type item struct { + A int `json:"a"` + } + valid := map[string]any{ + "map": map[string]any{"a": 1}, + "struct": item{A: 1}, + "struct pointer": &item{A: 1}, + "slice of structs": []item{{A: 1}, {A: 2}}, + "slice of maps": []map[string]any{{"a": 1}}, + "empty slice": []item{}, + "slice of any": []any{map[string]any{"a": 1}, item{A: 2}}, + "raw JSON object": rawJSON(`{"a":1}`), + "raw JSON array": rawJSON(`[{"a":1},{"a":2}]`), + "padded raw JSON obj": rawJSON(" \n{\"a\":1} "), + } + for name, items := range valid { + t.Run("accepts "+name, func(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err != nil { + t.Fatalf("PushItems rejected a valid shape: %v", err) + } + if backend.calls != 1 { + t.Fatalf("expected 1 request, got %d", backend.calls) + } + }) + } + + invalid := map[string]any{ + "string": "[1,2]", + "[]byte": []byte(`{"a":1}`), + "number": 1, + "bool": true, + "nil": nil, + "slice of strings": []string{"a", "b"}, + "slice of numbers": []int{1, 2}, + "mixed slice": []any{map[string]any{"a": 1}, "b"}, + "slice of slices": [][]int{{1}}, + "raw JSON string": rawJSON(`"a"`), + "raw JSON null element": rawJSON(`[{"a":1},null]`), + } + for name, items := range invalid { + t.Run("rejects "+name, func(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err == nil { + t.Fatal("PushItems accepted a shape the API rejects") + } + if backend.calls != 0 { + t.Fatalf("nothing may be sent for an invalid shape, got %d requests", backend.calls) + } + }) + } +} + +func TestPushItemsRejectsAnInvalidDatasetIDBeforeSending(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("").PushItems(context.Background(), map[string]any{"a": 1}); err == nil { + t.Fatal("expected an error for an empty dataset ID") + } + if backend.calls != 0 { + t.Fatalf("nothing may be sent for an invalid ID, got %d requests", backend.calls) + } +} + +func rawJSON(s string) json.RawMessage { return json.RawMessage(s) } diff --git a/docs/README.md b/docs/README.md index 54a1d98..99cbe1f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -105,7 +105,7 @@ the [storages](storages.md) page. - [Actors](actors.md) — Actors, versions, environment variables. - [Builds](builds.md) — Actor builds. -- [Runs](runs.md) — Actor runs and their default storages. +- [Runs](runs.md) — Actor runs, their default storages, log redirection and live iteration of dataset items. - [Tasks](tasks.md) — Actor tasks. - [Storages](storages.md) — Datasets, key-value stores, request queues. - [Schedules](schedules.md) — Schedules. diff --git a/docs/actors.md b/docs/actors.md index 87d85ef..cee353d 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -82,7 +82,7 @@ The `Actor` value returned by `Get`/`Create`/`Update` and listed by `List`: | `Delete(ctx) error` | Delete the Actor. | | `Start(ctx, input any, ActorStartOptions) (ActorRun, error)` | Start a run, return immediately. | | `StartRaw(ctx, input []byte, ActorStartOptions) (ActorRun, error)` | Like `Start`, but sends `input` as a raw request body (no JSON serialization) — for a non-JSON input such as a ZIP archive, paired with `ActorStartOptions.ContentType`. | -| `Call(ctx, input any, ActorStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait for the run to finish. `waitSecs` bounds the client-side wait; pass `nil` to wait indefinitely (until the run reaches a terminal state). | +| `Call(ctx, input any, ActorStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait for the run to finish, redirecting the run's log to `os.Stderr` while it waits (see `ActorStartOptions.Log`). `waitSecs` bounds the client-side wait; pass `nil` to wait indefinitely (until the run reaches a terminal state). Neither the start request nor the polls have a client-side timeout: `ctx` bounds the call. | | `CallRaw(ctx, input []byte, ActorStartOptions, waitSecs *int64) (ActorRun, error)` | `StartRaw` followed by waiting for the run to finish, like `Call`. | | `Build(ctx, versionNumber string, ActorBuildOptions) (Build, error)` | Build a version. | | `DefaultBuild(ctx, waitForFinish *int64) (*BuildClient, error)` | Resolve the default build. | @@ -155,6 +155,7 @@ fmt.Println(string(result)) // raw JSON validation result | `RestartOnError` | `*bool` | Restart the run if it fails. | | `ForcePermissionLevel` | `*string` | Override the Actor's permission level for this run. Accepted values: `LIMITED_PERMISSIONS`, `FULL_PERMISSIONS`. | | `Webhooks` | `[]any` | Ad-hoc webhooks to attach to this run. Each element is a map describing one webhook: `{"eventTypes": []string, "requestUrl": string, "payloadTemplate": string}` (same shape as a webhook definition; `eventTypes` are the `WebhookEventType` values listed in [webhooks.md](webhooks.md)). | +| `Log` | `*StreamedLogOptions` | Only for `Call`/`CallRaw`: how the run's log is redirected while they wait. `nil` redirects it to `os.Stderr`, with every message prefixed by `"{Actor name} runId:{run ID} -> "`. Set `Output` (an `io.Writer`), `Prefix` or `FromStart` to change that, or `Disabled` to turn the redirection off. See [`StreamedLog`](runs.md#log-redirection). `Start` and `StartRaw` ignore it. | | `WaitForResources` | `*int64` | Retry the start every 10 seconds while the API rejects it for lack of resources (`actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`). `apify.WaitForResourcesIndefinitely` retries until the run starts; any other value is the number of seconds to retry for before returning the last error. `nil` (the default) does not retry. Any other error is returned immediately. | `ActorBuildOptions` (all fields optional): @@ -197,7 +198,7 @@ run, err := client.Actor("apify/hello-world").Call(ctx, | Method | Description | | --- | --- | -| `Versions().List(ctx, ListOptions)` / `Versions().Iterate(ListOptions, chunkSize *int64)` / `Versions().Create(ctx, def)` | List/iterate/create versions. | +| `Versions().List(ctx)` / `Versions().Iterate()` / `Versions().Create(ctx, def)` | List/iterate/create versions. The endpoint returns every version in one response and takes no pagination parameters, so neither method takes options. | | `Version(n).Get/Update/Delete(ctx)` | Manage a single version. | | `Version(n).EnvVars().List(ctx)` / `.Iterate()` / `.Create(ctx, ActorEnvVar)` | List/iterate/create env vars. | | `Version(n).EnvVar(name).Get/Update/Delete(ctx)` | Manage a single env var. | diff --git a/docs/misc.md b/docs/misc.md index 6f4fcb4..5cc6a2b 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -23,6 +23,7 @@ Browse public Actors with `client.Store()`: | `IncludeUnrunnableActors` | `*bool` | Include Actors the current user cannot run. | | `AllowsAgenticUsers` | `*bool` | Filter to Actors that allow agentic users. | | `ResponseFormat` | `*string` | Select the response format. Accepted values: `full`, `agent`. | +| `IncludeInputSchema` | `*bool` | Add to each Actor an `inputSchema` field with a compact version of its default build's input schema. With it set, the API defaults `Limit` to 10 and does not accept a larger one. | Each item is an `ActorStoreListItem`: @@ -32,6 +33,8 @@ Each item is an `ActorStoreListItem`: | `Name` | `string` | Technical name of the Actor. | | `Username` | `string` | Username of the Actor's owner. | | `Title` | `string` | Human-readable title (may be empty; fall back to `Name`). | +| `URL` | `*string` | URL of the Actor's page in the Apify Store, normalized. `nil` when there is none. | +| `UserPictureURL` | `*string` | URL of the owner's profile picture, normalized. `nil` when there is none. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (forward compatibility). | ```go @@ -56,10 +59,10 @@ for { | Method | Description | | --- | --- | | `Get(ctx) (User, bool, error)` | Account details (private for `Me()`, public otherwise). | -| `MonthlyUsage(ctx) (json.RawMessage, error)` | Current month's usage (`Me()` only). | -| `MonthlyUsageWithOptions(ctx, MonthlyUsageOptions) (json.RawMessage, error)` | Usage for the period containing `Date` (`YYYY-MM-DD`; empty means current) (`Me()` only). | -| `Limits(ctx) (json.RawMessage, error)` | Current account's limits (`Me()` only). | -| `UpdateLimits(ctx, newLimits any) error` | Update the account's limits (`Me()` only). | +| `MonthlyUsage(ctx) (json.RawMessage, error)` | Current month's usage. | +| `MonthlyUsageWithOptions(ctx, MonthlyUsageOptions) (json.RawMessage, error)` | Usage for the period containing `Date` (`YYYY-MM-DD`; empty means current). | +| `Limits(ctx) (json.RawMessage, error)` | The account's limits. | +| `UpdateLimits(ctx, newLimits any) error` | Update the account's limits. | The `User` value returned by `Get` carries the account's public fields; for `Me()` the API also returns private account details, which are preserved in `Extra`: @@ -68,9 +71,10 @@ also returns private account details, which are preserved in `Extra`: |---|---|---| | `ID` | `string` | Unique user ID. | | `Username` | `string` | The user's username. | +| `Profile` | `*UserProfile` | The public profile: `Name`, `PictureURL` and `WebsiteURL` (both `*string`, normalized URLs), and an `Extra` map for the other fields. `nil` if the API does not return one. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (private details for `Me()`). | -The usage/limits methods return an error if called on a non-`Me()` client. +The usage and limits methods work on `Me()` and on `User(id)` (by ID or username). The API serves them only for your own account or for the organization your API token belongs to, and rejects tokens with limited permissions. `MonthlyUsage` reports usage for the current month and returns the raw JSON usage report from the API — a JSON object with the account's usage breakdown and totals for the period. @@ -113,14 +117,15 @@ fmt.Println(string(usage)) // raw JSON usage report for the current month | `Raw` | `*bool` | Return the unprocessed log content (no platform post-processing). | | `Download` | `*bool` | Set `Content-Disposition` so the log is served as a download. | -For convenient live redirection of a run's log, `client.Run(id).GetStreamedLog(ctx)` returns -a raw live stream directly. +`client.Run(id).GetStreamedLog(ctx)` is deprecated: despite the name it returns the raw live +stream, like `Log().StreamWithOptions` with `Raw` set. To redirect the log message by message to +a writer, with a prefix, use `NewStreamedLog`; see [log redirection](runs.md#log-redirection). ```go // Read the whole log. logText, ok, err := client.Run(runID).Log().Get(ctx) -// Or stream it live (log redirection). +// Or stream it live, as raw bytes. stream, err := client.Run(runID).Log().Stream(ctx) if err != nil { log.Fatal(err) diff --git a/docs/runs.md b/docs/runs.md index b2787c6..16e63cb 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -18,8 +18,9 @@ with `client.Run(id)`, and an Actor's or task's runs with `client.Actor(id).Runs | `Status` | `[]string` | Filter by one or more run statuses; sent as a comma-separated list. Values are the eight `ActorJobStatus` values: `READY`, `RUNNING`, `SUCCEEDED`, `FAILED`, `TIMING-OUT`, `TIMED-OUT`, `ABORTING`, `ABORTED`. | | `StartedAfter` | `*string` | Only runs started after this ISO-8601 timestamp. | | `StartedBefore` | `*string` | Only runs started before this ISO-8601 timestamp. | +| `Deployment` | `[]string` | Only Actor Standby runs served by these pools (`SINGLE_TENANT`, `MULTI_TENANT_FREE_USERS`, `MULTI_TENANT_PAYING_USERS`, `MULTI_TENANT_OWNER_PAID`); sent as a comma-separated list. The API requires `Status` to hold only non-terminal statuses (`READY`, `RUNNING`, `TIMING-OUT`, `ABORTING`) with it. | -The time filters apply only to Actor- and task-scoped collections. +`StartedAfter`, `StartedBefore` and `Deployment` apply only to the account-wide (`client.Runs()`) and Actor-scoped (`client.Actor(id).Runs()`) collections, not to a task's. ## Single run @@ -40,7 +41,13 @@ The time filters apply only to Actor- and task-scoped collections. | `KeyValueStore() *KeyValueStoreClient` | The run's default key-value store. | | `RequestQueue() *RequestQueueClient` | The run's default request queue. | | `Log() *LogClient` | The run's log. | -| `GetStreamedLog(ctx) (io.ReadCloser, error)` | Live stream of the run's raw log (log redirection). | +| `GetStreamedLog(ctx) (io.ReadCloser, error)` | Deprecated: a raw live stream of the run's log, as bytes, despite the name. Use `NewStreamedLog` to redirect the log, or `Log().StreamWithOptions` for the raw stream. | +| `NewStreamedLog(ctx, StreamedLogOptions) (*StreamedLog, bool, error)` | Redirects the run's log message by message to a writer; see [Log redirection](#log-redirection). The bool is `false` for a run that does not exist (with the default prefix). | +| `IterateDatasetItems(RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[json.RawMessage]` | Iterates over the items of the run's default dataset while the run is still producing them; see [Live iteration of dataset items](#live-iteration-of-dataset-items). | + +On a client from `LastRun(...)`, `Dataset()`, `KeyValueStore()`, `RequestQueue()` and `Log()` carry +the same `status` and `origin` filters, so they address the storages and the log of the run the +filters resolve to, not those of the newest run of any status. > **Missing resources.** `Dataset()`/`KeyValueStore()`/`RequestQueue()`/`Log()` address a > sub-resource with no ID of its own, so a 404 from `Get`/`Delete`/`Log().Get`/`Log().Stream` @@ -112,9 +119,88 @@ run's metadata. The commonly used fields: | `DefaultDatasetID` | `string` | ID of the run's default dataset. | | `DefaultKeyValueStoreID` | `string` | ID of the run's default key-value store. | | `DefaultRequestQueueID` | `string` | ID of the run's default request queue. | -| `ContainerURL` | `string` | URL of the run's container (for live access). | +| `ContainerURL` | `string` | URL of the run's container (for live access), normalized. | +| `ConsoleURL` | `string` | URL of the run in the Apify Console, normalized. Empty on items from `List`/`Iterate`. | +| `Output` | `map[string]string` | Output links of the run (name to normalized URL), generated from the output schema of the Actor build. `nil` if the build defines none, and on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (forward compatibility). | `ActorRun.IsTerminal()` reports whether a run has finished. Status message convenience: `client.SetStatusMessage(ctx, message, isTerminal)` updates the current run identified by the `ACTOR_RUN_ID` environment variable. + +## Log redirection + +`ActorClient.Call` redirects the log of the run while it waits (set `ActorStartOptions.Log` to +change where it goes or turn it off). To redirect the log of any run, create a `StreamedLog`: + +```go +streamed, found, err := client.Run(runID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) +if err != nil { + log.Fatal(err) +} +if !found { + log.Fatal("run not found") +} +if err := streamed.Start(ctx); err != nil { + log.Fatal(err) +} +// ... wait for the run to finish ... +if err := streamed.Stop(ctx); err != nil { + log.Fatal(err) +} +``` + +`StreamedLogOptions` (all fields optional): + +| Field | Type | Meaning | +|---|---|---| +| `Output` | `io.Writer` | Receives every message, one per line, with the prefix. Defaults to `os.Stderr`. Writes are serialized. | +| `Prefix` | `*string` | Written in front of every message. The default is `"{Actor name} runId:{run ID} -> "`, which costs two requests (the run and its Actor); a non-nil `Prefix`, an empty one included, skips them. | +| `FromStart` | `*bool` | `false` drops the messages that were logged before the redirection was created. Default `true`. | +| `Disabled` | `bool` | Turns the redirection off in `ActorStartOptions.Log`. | + +The log is split into messages at the timestamps the platform puts in front of each of them, so a +message that spans several lines is written as one, even when its lines arrive in separate reads; +the last message is therefore written when the next one starts or the log ends. `Start` begins the +redirection in the background, and it also ends when the `ctx` it got is done. `Stop` gives the +messages in flight up to one second, then aborts the stream; if the stream delivered nothing yet, +it reads the whole log in one request, so a short run that finished before the stream connected is +not lost. The `ctx` given to `Stop` bounds that wait and that read: when it is done first, `Stop` +returns its error and the rest of the log is not redirected, unless the `ctx` given to `Start` +ended first, which ends the redirection on purpose and is not an error. When both end at about the +same time, either result is possible. A second `Stop` while one is running returns an error. A log +that has no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, +until its first bytes arrive. `Stop` returns the error that ended the redirection early, if any; +`Call` reports it on the redirection's output and goes on. + +## Live iteration of dataset items + +`RunClient.IterateDatasetItems` (and the generic `apify.IterateRunDatasetItems[T]`, which decodes +every item into `T`) iterates over the items of the run's default dataset *while the run is still +producing them*. While the run has not finished, each round yields the rows below the dataset's +`itemCount` and then waits up to `PollIntervalSecs` for the run to finish, so the last rows are +read as soon as it does. `itemCount` lags a few seconds behind the pushed items, so once the run +has finished the rows past it are read a page at a time until none are left. On a client from +`LastRun(...)`, the iterator sticks to the run its first request resolves to. + +`RunIterateDatasetItemsOptions` (all fields optional): + +| Field | Type | Meaning | +|---|---|---| +| `Items` | `DatasetListItemsOptions` | Filters and projections applied to every page (see [storages.md](storages.md)). `Items.Offset` is the number of rows to skip (default 0) and `Items.Limit` the maximum number of dataset rows to scan (default no limit): fewer items are yielded when filters drop some, more when `Unwind` splits a row into several. `Items.Desc` is not supported, because the rows are read by offset from the oldest; it makes `Next` fail. | +| `ChunkSize` | `*int64` | Maximum number of dataset rows requested per API call. Default 1000. | +| `PollIntervalSecs` | `*int64` | How long to wait for the run to finish between polls, in seconds. Default 5. | + +```go +it := client.Run(runID).IterateDatasetItems(apify.RunIterateDatasetItemsOptions{}) +for { + item, err := it.Next(ctx) + if err != nil { + log.Fatal(err) + } + if item == nil { + break // every item has been read + } + fmt.Println(string(*item)) +} +``` diff --git a/docs/storages.md b/docs/storages.md index 57d3327..4d2c671 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -19,7 +19,7 @@ Single dataset: `client.Dataset(id)`: | `Delete(ctx) error` | Delete the dataset. | | `ListItems(ctx, DatasetListItemsOptions) (PaginationList[json.RawMessage], error)` | Read items. | | `IterateItems(DatasetListItemsOptions, chunkSize *int64) *ListIterator[json.RawMessage]` | Lazy iterator over items (`Limit` caps total, `chunkSize` is page size); `IterateDatasetItems[T]` decodes into your type. | -| `PushItems(ctx, items any) error` | Append one item or a slice of items. | +| `PushItems(ctx, items any) error` | Append one item or a slice of items. They must serialize to a JSON object or an array of objects, the only shapes the API accepts; anything else (a string, a number, a slice of strings, `nil`) fails before any request is sent. Pass a `json.RawMessage` for already serialized JSON. A push that times out is not retried, because the API may already have stored the items. | | `DownloadItems(ctx, DownloadItemsFormat, DatasetDownloadOptions) ([]byte, error)` | Export items (JSON, JSONL, CSV, XLSX, XML, RSS, HTML — see the format constants below). | | `DownloadItemsStream(ctx, DownloadItemsFormat, DatasetDownloadOptions) (io.ReadCloser, error)` | Like `DownloadItems`, but as a stream for a large export; close when done. | | `GetStatistics(ctx) (json.RawMessage, error)` | Dataset statistics. | @@ -36,6 +36,8 @@ The `Dataset` value returned by `Get`/`GetOrCreate`/`Update` and listed by `List | `CreatedAt` | `*time.Time` | When the dataset was created. | | `ModifiedAt` | `*time.Time` | When the dataset was last modified. | | `ItemCount` | `int64` | Number of items currently stored. | +| `ConsoleURL` | `string` | URL of the dataset in the Apify Console (normalized). Empty on items from `List`/`Iterate`. | +| `ItemsPublicURL` | `string` | Public URL of the dataset's items (normalized); signed for a private dataset. Empty on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | `DatasetListItemsOptions` controls filtering, projection, and pagination of the items @@ -167,13 +169,15 @@ Single store: `client.KeyValueStore(id)`: | `Delete(ctx) error` | Delete the store. | | `ListKeys(ctx, ListKeysOptions) (KeyValueStoreKeysPage, error)` | List one page of keys. | | `IterateKeys(ListKeysOptions, chunkSize *int64) *KeyValueStoreKeysIterator` | Lazy iterator over keys (cursor-based). `Limit` caps the total yielded; `chunkSize` is the page size; `Prefix`/`Collection`/`Signature` filter every page; `ExclusiveStartKey` sets where to start. | -| `GetRecord(ctx, key) (*KeyValueStoreRecord, bool, error)` | Read a record. | +| `GetRecord(ctx, key) (*KeyValueStoreRecord, bool, error)` | Read a record. The value is the raw bytes, never parsed: decode it according to the `ContentType` the API reports. | | `GetRecordWithOptions(ctx, key, GetRecordOptions)` | Read with options. | | `GetRecordStream(ctx, key) (*KeyValueStoreRecordStream, bool, error)` | Like `GetRecord`, but the value is a stream for a large record; close when done. | | `GetRecordStreamWithOptions(ctx, key, GetRecordOptions)` | Stream with options. | -| `SetRecordRaw(ctx, key, value []byte, contentType string) error` | Write raw bytes. | +| `SetRecordRaw(ctx, key, value []byte, contentType string) error` | Write raw bytes, sent as they are: the client does not parse or convert the value, whatever the content type (so `application/json` bytes must already be valid JSON). | +| `SetRecordRawWithOptions(ctx, key, value []byte, contentType string, SetRecordOptions) error` | Like `SetRecordRaw`, with `ContentEncoding` (the value is already compressed, as with `SetRecordRawWithEncoding`) and `DoNotRetryTimeouts` (an upload that times out fails at once instead of being retried). | | `SetRecordRawWithEncoding(ctx, key, value []byte, contentType, contentEncoding string) error` | Write raw bytes that are already compressed (`contentEncoding` e.g. `"gzip"`, `"br"`, `"deflate"`): sent as-is under `Content-Encoding`, skipping the client's own request compression so the value is never compressed twice. | | `SetRecordJSON(ctx, key, value any) error` | Write JSON. | +| `SetRecordJSONWithOptions(ctx, key, value any, SetRecordOptions) error` | Like `SetRecordJSON`, with the per-call `SetRecordOptions`. | | `RecordExists(ctx, key) (bool, error)` | HEAD check. | | `DeleteRecord(ctx, key) error` | Delete a record. | | `GetRecordPublicURL(ctx, key) (string, error)` | Signed public record URL. | @@ -188,6 +192,9 @@ The `KeyValueStore` value returned by `Get`/`GetOrCreate`/`Update` and listed by | `UserID` | `string` | ID of the user who owns the store. | | `CreatedAt` | `*time.Time` | When the store was created. | | `ModifiedAt` | `*time.Time` | When the store was last modified. | +| `ConsoleURL` | `string` | URL of the store in the Apify Console (normalized). Empty on items from `List`/`Iterate`, which do not carry it. | +| `KeysPublicURL` | `string` | Public URL listing the store's keys (normalized); signed for a private store. Empty on items from `List`/`Iterate`. | +| `RecordsPublicURL` | `string` | Public URL prefix of the store's records (normalized). Empty on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | Option structs (all fields optional): @@ -226,8 +233,9 @@ when done. | `NextExclusiveStartKey` | `string` | Key to pass to fetch the next page. | | `Items` | `[]KeyValueStoreKey` | The listed keys. | -Each `KeyValueStoreKey` element has `Key` (`string`) and `Size` (`int64`, the record size in -bytes), plus an `Extra` (`map[string]json.RawMessage`) catch-all. +Each `KeyValueStoreKey` element has `Key` (`string`), `Size` (`int64`, the record size in +bytes) and `RecordPublicURL` (`string`, the public URL of the record, normalized), plus an +`Extra` (`map[string]json.RawMessage`) catch-all. ```go store, _ := client.KeyValueStores().GetOrCreate(ctx, "") @@ -281,6 +289,7 @@ Single queue: `client.RequestQueue(id)`: | `DeleteRequest(ctx, id) error` | Delete a request. | | `ListRequests(ctx, ListRequestsOptions) (json.RawMessage, error)` | Paginated list. Returns the raw API response (`{ "items": [...], "nextCursor": ... }`); unmarshal into your own type, or use `PaginateRequests` for typed iteration. | | `PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator` | Lazy iterator over all requests (yields `*RequestQueueRequest`). | +| `PaginateRequestsWithOptions(ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator` | The same with `Limit` (caps the total across pages), `Cursor` (where to start) and `Filter` (every page); `chunkSize` is the page size. | | `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests, auto-split into batches that respect the API's 25-request and (effective) 9 MiB payload-size limits. Fails before sending anything if any single request is too large for a batch of its own. Unlike the reference client, does not itself retry a batch's `UnprocessedRequests` — inspect the result and retry those explicitly if needed. | | `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload — a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`), matching the reference client's `batchDeleteRequests`. Returns the raw API response. | | `ListAndLockHead(ctx, lockSecs int64, limit *int64) (json.RawMessage, error)` | Fetch the head of the queue and lock the returned requests for `lockSecs` seconds. Returns the raw API response. | @@ -299,6 +308,7 @@ The `RequestQueue` value returned by `Get`/`GetOrCreate`/`Update` and listed by | `CreatedAt` | `*time.Time` | When the queue was created. | | `ModifiedAt` | `*time.Time` | When the queue was last modified. | | `TotalRequestCount` | `int64` | Total number of requests ever added. | +| `ConsoleURL` | `string` | URL of the queue in the Apify Console (normalized). Empty on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | `RequestQueueRequest` is the request payload/record. `URL` is required; `ID` is assigned by the @@ -371,6 +381,22 @@ for { } fmt.Println(req.URL) } + +// Iterate over at most 50 pending requests, 25 per page. +pending := queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ + Limit: apify.Ptr(int64(50)), + Filter: []string{apify.RequestFilterPending}, +}, apify.Ptr(int64(25))) +for { + req, err := pending.Next(ctx) + if err != nil { + log.Fatal(err) + } + if req == nil { + break + } + fmt.Println(req.URL) +} ``` `StorageListOptions` is shared by the three storage collections' `List` methods. All fields are diff --git a/docs/tasks.md b/docs/tasks.md index 1568848..ce63c70 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -25,6 +25,7 @@ A task is a pre-configured Actor run with stored input. Access the task collecti | `ModifiedAt` | `*time.Time` | When the task was last modified. | | `IsPublic` | `*bool` | Whether the task is published on its public landing page, derived from `PublicConfig.PublishedAt`; use `Publish`/`Unpublish` to change it. | | `PublicConfig` | `*TaskPublicConfig` | Public-facing display configuration of the landing page, set once the task has been configured for publishing. | +| `StandbyURL` | `*string` | URL of the task's Actor Standby endpoint, normalized. `nil` when the task has none, and on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | ### `TaskPublicConfig` fields @@ -49,7 +50,7 @@ A task is a pre-configured Actor run with stored input. Access the task collecti | `Publish(ctx) (Task, error)` | Publish the task on its public landing page. An Actor can have at most 10 published tasks, and an account at most 100 (contact Apify support to raise either limit). | | `Unpublish(ctx) (Task, error)` | Unpublish the task from its public landing page. | | `Start(ctx, input any, TaskStartOptions) (ActorRun, error)` | Start a run (input overrides stored input). | -| `Call(ctx, input any, TaskStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait. | +| `Call(ctx, input any, TaskStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait. Neither the start request nor the polls have a client-side timeout: `ctx` bounds the call. | | `GetInput(ctx) (json.RawMessage, error)` | Fetch the stored input. | | `UpdateInput(ctx, input any) (json.RawMessage, error)` | Replace the stored input. | | `LastRun(status string) *RunClient` | Client for the last run (optional status filter). | diff --git a/docs/webhooks.md b/docs/webhooks.md index 20beb81..0a39be6 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -51,7 +51,7 @@ The same values apply to the ad-hoc `ActorStartOptions.Webhooks` element (see |---|---|---| | `ID` | `string` | Unique webhook ID. | | `UserID` | `string` | ID of the user who owns the webhook. | -| `RequestURL` | `string` | URL the webhook posts to. | +| `RequestURL` | `string` | URL the webhook posts to, normalized. Empty when the API returns `null`. | | `EventTypes` | `[]string` | Events that trigger the webhook. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | @@ -61,6 +61,7 @@ The same values apply to the ad-hoc `ActorStartOptions.Webhooks` element (see |---|---|---| | `ID` | `string` | Unique dispatch ID. | | `WebhookID` | `string` | ID of the webhook that produced this dispatch. | +| `Webhook` | `*WebhookDispatchWebhookSummary` | Summary of the webhook as it was when the dispatch was created: `ID`, `ActionType`, `IsAdHoc`, `RequestURL` (normalized), and an `Extra` map. `nil` if the API does not return it. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | ## Single webhook diff --git a/examples/log_redirection/main.go b/examples/log_redirection/main.go index ab4c220..f24d93f 100644 --- a/examples/log_redirection/main.go +++ b/examples/log_redirection/main.go @@ -1,5 +1,5 @@ -// Command log_redirection starts an Actor without waiting, then streams its log output to -// stdout in real time (log redirection). +// Command log_redirection runs an Actor and redirects its log to stdout in real time while it +// runs. Call does this by default (to os.Stderr); here the output is set to stdout. // // Run with: // @@ -8,7 +8,7 @@ package main import ( "context" - "io" + "fmt" "log" "os" @@ -20,20 +20,29 @@ func main() { client := exampleclient.New() ctx := context.Background() - // Start the Actor and return immediately (do not wait for it to finish). - run, err := client.Actor("apify/hello-world").Start(ctx, nil, apify.ActorStartOptions{}) + // Every message of the run's log is printed as the run produces it, prefixed with the Actor + // name and the run ID, until the run finishes. + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{ + Log: &apify.StreamedLogOptions{Output: os.Stdout}, + }, nil) if err != nil { - log.Fatalf("start actor: %v", err) + log.Fatalf("call actor: %v", err) } + fmt.Println("run finished with status", run.Status) - // Open a live (raw) log stream and copy it to stdout as the run produces output. - stream, err := client.Run(run.ID).GetStreamedLog(ctx) + // To redirect the log of a run that was started elsewhere, create the redirection for it and + // stop it once the run is done. Starting it for a finished run reads the whole log. + streamed, found, err := client.Run(run.ID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) if err != nil { - log.Fatalf("open log stream: %v", err) + log.Fatalf("create log redirection: %v", err) } - defer func() { _ = stream.Close() }() - - if _, err := io.Copy(os.Stdout, stream); err != nil { - log.Fatalf("copy log: %v", err) + if !found { + log.Fatal("run not found") + } + if err := streamed.Start(ctx); err != nil { + log.Fatalf("start log redirection: %v", err) + } + if err := streamed.Stop(ctx); err != nil { + log.Fatalf("stop log redirection: %v", err) } } diff --git a/http_client.go b/http_client.go index 5d79716..94a0013 100644 --- a/http_client.go +++ b/http_client.go @@ -4,8 +4,10 @@ import ( "bytes" "compress/gzip" "context" + "errors" "io" "math/rand" + "net" "net/http" "strings" "time" @@ -120,6 +122,20 @@ type httpClient struct { retry retryConfig tiers timeoutTiers compressors []requestCompressor + // doNotRetryTimeouts makes a request attempt that exceeds its timeout fail at once instead of + // being retried. Set it (through [httpClient.withoutTimeoutRetry]) for a request that is not + // idempotent, where a timed-out attempt may already have taken effect on the server, so + // repeating it could create a second run or store the same dataset items twice. + doNotRetryTimeouts bool +} + +// withoutTimeoutRetry returns a copy of c that does not retry a request attempt that timed out. +// Other retryable failures (a 429 or 5xx response, a transport error that is not a timeout) are +// still retried. +func (c *httpClient) withoutTimeoutRetry() *httpClient { + clone := *c + clone.doNotRetryTimeouts = true + return &clone } // apiResponse is the parsed result of a single API call: the status, headers and the @@ -175,7 +191,7 @@ func (c *httpClient) doRetryLoop(ctx context.Context, method, url string, sendBo var retryable bool if err != nil { lastErr = err - retryable = isTransportErrorRetryable(err) + retryable = isTransportErrorRetryable(err) && (!c.doNotRetryTimeouts || !isTimeoutError(err)) } else if resp.statusCode < maxSuccessStatusCode { return resp, nil } else { @@ -498,6 +514,16 @@ func isTransportErrorRetryable(err error) bool { return true } +// isTimeoutError reports whether err means a request attempt ran out of time: the attempt's own +// context deadline, or a timeout reported by the transport. +func isTimeoutError(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + // randomizedDelay returns a delay chosen randomly from [delay, 2*delay), matching the // exponential-backoff-with-jitter algorithm described in the Apify API docs. func randomizedDelay(delay time.Duration) time.Duration { diff --git a/key_value_store.go b/key_value_store.go index 34e63cc..787aca2 100644 --- a/key_value_store.go +++ b/key_value_store.go @@ -64,9 +64,10 @@ func (c *KeyValueStoreClient) Get(ctx context.Context) (KeyValueStore, bool, err return getResource[KeyValueStore](ctx, c.ctx, "", NewQueryParams()) } -// Update updates the store metadata (e.g. name) and returns the updated object. +// Update updates the store metadata (e.g. name) and returns the updated object. Uses the "long" +// timeout tier, the reference client's tier for this call. func (c *KeyValueStoreClient) Update(ctx context.Context, newFields any) (KeyValueStore, error) { - return updateResource[KeyValueStore](ctx, c.ctx, "", newFields) + return updateResourceWithTimeout[KeyValueStore](ctx, c.ctx, "", newFields, c.ctx.longTimeout()) } // Delete deletes the store. @@ -269,13 +270,11 @@ func (c *KeyValueStoreClient) GetRecordStreamWithOptions(ctx context.Context, ke }, true, nil } -// SetRecordRaw stores a record with raw bytes and the given content type. +// SetRecordRaw stores a record with raw bytes and the given content type. The bytes are sent as +// they are, whatever the content type: the client neither parses nor converts them, so a value +// stored as application/json must already be valid JSON. func (c *KeyValueStoreClient) SetRecordRaw(ctx context.Context, key string, value []byte, contentType string) error { - safeKey, err := safePathSegment(key) - if err != nil { - return err - } - return putRaw(ctx, c.ctx, "records/"+safeKey, NewQueryParams(), value, contentType) + return c.setRecord(ctx, key, value, contentType, "", false, false) } // SetRecordRawWithEncoding is like [KeyValueStoreClient.SetRecordRaw], for a value that is @@ -286,20 +285,61 @@ func (c *KeyValueStoreClient) SetRecordRaw(ctx context.Context, key string, valu // contentEncoding == "") for a value that is not pre-compressed. Mirrors the reference client's // setRecord({ contentEncoding }). func (c *KeyValueStoreClient) SetRecordRawWithEncoding(ctx context.Context, key string, value []byte, contentType, contentEncoding string) error { + return c.setRecord(ctx, key, value, contentType, contentEncoding, true, false) +} + +// setRecord is the shared implementation of the SetRecord methods. preEncoded sends value exactly +// as given under contentEncoding (which may be empty), skipping the client's own request +// compression; doNotRetryTimeouts makes an upload attempt that timed out fail at once. +func (c *KeyValueStoreClient) setRecord(ctx context.Context, key string, value []byte, contentType, contentEncoding string, preEncoded, doNotRetryTimeouts bool) error { safeKey, err := safePathSegment(key) if err != nil { return err } - return putRawWithEncoding(ctx, c.ctx, "records/"+safeKey, NewQueryParams(), value, contentType, contentEncoding) + resource := c.ctx + if doNotRetryTimeouts { + resource = resource.withoutTimeoutRetry() + } + subPath := "records/" + safeKey + if preEncoded { + return putRawWithEncoding(ctx, resource, subPath, NewQueryParams(), value, contentType, contentEncoding) + } + return putRaw(ctx, resource, subPath, NewQueryParams(), value, contentType) +} + +// SetRecordOptions configures [KeyValueStoreClient.SetRecordRawWithOptions] and +// [KeyValueStoreClient.SetRecordJSONWithOptions]. +type SetRecordOptions struct { + // ContentEncoding, when not empty (e.g. "gzip", "br", "deflate"), says the value is already + // compressed under that encoding: it is uploaded exactly as given, with the Content-Encoding + // header, and the client's own request compression is skipped. See + // [KeyValueStoreClient.SetRecordRawWithEncoding]. + ContentEncoding string + // DoNotRetryTimeouts, if true, makes an upload attempt that times out fail at once instead of + // being retried (other failures are still retried). Use it when storing the record twice + // would be harmful. Matches the reference client's setRecord({ doNotRetryTimeouts }). + DoNotRetryTimeouts bool +} + +// SetRecordRawWithOptions is like [KeyValueStoreClient.SetRecordRaw], with the per-call +// [SetRecordOptions]. +func (c *KeyValueStoreClient) SetRecordRawWithOptions(ctx context.Context, key string, value []byte, contentType string, options SetRecordOptions) error { + return c.setRecord(ctx, key, value, contentType, options.ContentEncoding, options.ContentEncoding != "", options.DoNotRetryTimeouts) } // SetRecordJSON stores a record holding the JSON serialization of value. func (c *KeyValueStoreClient) SetRecordJSON(ctx context.Context, key string, value any) error { + return c.SetRecordJSONWithOptions(ctx, key, value, SetRecordOptions{}) +} + +// SetRecordJSONWithOptions is like [KeyValueStoreClient.SetRecordJSON], with the per-call +// [SetRecordOptions]. +func (c *KeyValueStoreClient) SetRecordJSONWithOptions(ctx context.Context, key string, value any, options SetRecordOptions) error { data, err := json.Marshal(value) if err != nil { return err } - return c.SetRecordRaw(ctx, key, data, contentTypeJSONCharset) + return c.SetRecordRawWithOptions(ctx, key, data, contentTypeJSONCharset, options) } // DeleteRecord deletes a record by key. @@ -311,6 +351,12 @@ func (c *KeyValueStoreClient) DeleteRecord(ctx context.Context, key string) erro return deleteResourceAlways(ctx, c.ctx, "records/"+safeKey) } +// getForPublicURL fetches the store to read its URL-signing secret, with the "long" timeout tier +// the reference client uses for its public-URL builders. +func (c *KeyValueStoreClient) getForPublicURL(ctx context.Context) (KeyValueStore, bool, error) { + return getResourceWithTimeout[KeyValueStore](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) +} + // GetRecordPublicURL builds a public URL for fetching the given record. // // It mirrors the reference client: it fetches the store, and if the store exposes a @@ -322,7 +368,7 @@ func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string return "", err } params := NewQueryParams() - store, present, err := c.Get(ctx) + store, present, err := c.getForPublicURL(ctx) if err != nil { return "", err } @@ -332,7 +378,7 @@ func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string params.AddString("signature", &sig) } } - return params.applyToURL(c.ctx.publicURL("records/" + safeKey)), nil + return c.ctx.mergedParams(params).applyToURL(c.ctx.publicURL("records/" + safeKey)), nil } // CreateKeysPublicURL builds a public URL for listing this store's keys. @@ -341,7 +387,7 @@ func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string // optionally bounds the validity of a signed URL (nil for non-expiring). func (c *KeyValueStoreClient) CreateKeysPublicURL(ctx context.Context, expiresInSecs *int64) (string, error) { params := NewQueryParams() - store, present, err := c.Get(ctx) + store, present, err := c.getForPublicURL(ctx) if err != nil { return "", err } @@ -351,5 +397,5 @@ func (c *KeyValueStoreClient) CreateKeysPublicURL(ctx context.Context, expiresIn params.AddString("signature", &sig) } } - return params.applyToURL(c.ctx.publicURL("keys")), nil + return c.ctx.mergedParams(params).applyToURL(c.ctx.publicURL("keys")), nil } diff --git a/key_value_store_test.go b/key_value_store_test.go new file mode 100644 index 0000000..cc3a3a4 --- /dev/null +++ b/key_value_store_test.go @@ -0,0 +1,31 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +func TestSetRecordRawWithOptionsSendsTheContentEncoding(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + value := []byte(strings.Repeat("x", 2*minCompressRequestBytes)) + + err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{ContentEncoding: "gzip"}) + if err != nil { + t.Fatalf("SetRecordRawWithOptions: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got != "gzip" { + t.Fatalf("expected Content-Encoding gzip, got %q", got) + } + if backend.lastBody != string(value) { + t.Fatal("a pre-compressed value must be uploaded as given, never compressed again") + } + + if err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{}); err != nil { + t.Fatalf("SetRecordRawWithOptions: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got == "" { + t.Fatal("without a ContentEncoding the client compresses a large value of its own accord") + } +} diff --git a/last_run_test.go b/last_run_test.go new file mode 100644 index 0000000..5fe8a45 --- /dev/null +++ b/last_run_test.go @@ -0,0 +1,99 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +// The storages and the log of a run reached through LastRun carry its status and origin filters, +// so they address the storages and the log of the same run, not those of the newest run of any +// status. +func TestStoragesAndLogOfAFilteredLastRunCarryItsFilters(t *testing.T) { + ctx := context.Background() + lastRun := func(client *ApifyClient) *RunClient { + return client.Actor("a").LastRunWithOptions(LastRunOptions{Status: "SUCCEEDED", Origin: "API"}) + } + taskLastRun := func(client *ApifyClient) *RunClient { + return client.Task("t").LastRunWithOptions(LastRunOptions{Status: "SUCCEEDED", Origin: "API"}) + } + + cases := map[string]func(client *ApifyClient) error{ + "dataset get": func(c *ApifyClient) error { _, _, err := lastRun(c).Dataset().Get(ctx); return err }, + "dataset items": func(c *ApifyClient) error { + _, err := lastRun(c).Dataset().ListItems(ctx, DatasetListItemsOptions{}) + return err + }, + "dataset download": func(c *ApifyClient) error { + _, err := lastRun(c).Dataset().DownloadItems(ctx, "json", DatasetDownloadOptions{}) + return err + }, + "dataset push": func(c *ApifyClient) error { + return lastRun(c).Dataset().PushItems(ctx, map[string]any{"a": 1}) + }, + "dataset public URL": func(c *ApifyClient) error { + _, err := lastRun(c).Dataset().CreateItemsPublicURL(ctx, DatasetListItemsOptions{}, nil) + return err + }, + "store get": func(c *ApifyClient) error { _, _, err := lastRun(c).KeyValueStore().Get(ctx); return err }, + "store keys": func(c *ApifyClient) error { + _, err := lastRun(c).KeyValueStore().ListKeys(ctx, ListKeysOptions{}) + return err + }, + "store record": func(c *ApifyClient) error { + _, _, err := lastRun(c).KeyValueStore().GetRecord(ctx, "OUTPUT") + return err + }, + "store record public URL": func(c *ApifyClient) error { + _, err := lastRun(c).KeyValueStore().GetRecordPublicURL(ctx, "OUTPUT") + return err + }, + "store keys public URL": func(c *ApifyClient) error { + _, err := lastRun(c).KeyValueStore().CreateKeysPublicURL(ctx, nil) + return err + }, + "queue get": func(c *ApifyClient) error { _, _, err := lastRun(c).RequestQueue().Get(ctx); return err }, + "queue head": func(c *ApifyClient) error { _, err := lastRun(c).RequestQueue().ListHead(ctx, nil); return err }, + "log get": func(c *ApifyClient) error { _, _, err := lastRun(c).Log().Get(ctx); return err }, + "log stream": func(c *ApifyClient) error { + stream, err := lastRun(c).GetStreamedLog(ctx) + if err == nil { + _ = stream.Close() + } + return err + }, + "task last run dataset": func(c *ApifyClient) error { _, _, err := taskLastRun(c).Dataset().Get(ctx); return err }, + "task last run log": func(c *ApifyClient) error { _, _, err := taskLastRun(c).Log().Get(ctx); return err }, + } + for name, call := range cases { + t.Run(name, func(t *testing.T) { + body := `{"data":{"id":"x","urlSigningSecretKey":"s"}}` + if name == "dataset items" { + body = `[]` // the items endpoint answers with a bare array + } + backend := &mockBackend{responses: []mockResponse{{status: 200, body: body}}} + if err := call(testClient(backend, 0)); err != nil { + t.Fatalf("call failed: %v", err) + } + if !strings.Contains(backend.lastURL, "/runs/last/") { + t.Fatalf("expected a runs/last URL, got %q", backend.lastURL) + } + for _, want := range []string{"status=SUCCEEDED", "origin=API"} { + if !strings.Contains(backend.lastURL, want) { + t.Errorf("%s is missing from %q", want, backend.lastURL) + } + } + }) + } +} + +// A storage reached through a run that has no filters does not gain any. +func TestStoragesOfAnUnfilteredRunHaveNoFilterParams(t *testing.T) { + backend := &mockBackend{responses: okData()} + if _, _, err := testClient(backend, 0).Run("r").Dataset().Get(context.Background()); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "?") { + t.Errorf("unexpected query string in %q", backend.lastURL) + } +} diff --git a/models.go b/models.go index 3049599..b8d238e 100644 --- a/models.go +++ b/models.go @@ -2,6 +2,7 @@ package apify import ( "encoding/json" + "errors" "io" "time" ) @@ -96,8 +97,13 @@ type ActorRun struct { DefaultKeyValueStoreID string `json:"defaultKeyValueStoreId"` // DefaultRequestQueueID is the ID of the run's default request queue. DefaultRequestQueueID string `json:"defaultRequestQueueId"` - // ContainerURL is the URL of the run's container (for live access). + // ContainerURL is the URL of the run's container (for live access), normalized. ContainerURL string `json:"containerUrl"` + // ConsoleURL is the URL of the run in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` + // Output holds the output links of the run (name to URL), generated from the output schema + // of the Actor build and normalized. Nil if the build defines no output schema. + Output map[string]string `json:"output,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -105,7 +111,14 @@ type ActorRun struct { func (r *ActorRun) UnmarshalJSON(data []byte) error { type alias ActorRun known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(r), known, &r.Extra) + if err := unmarshalWithExtra(data, (*alias)(r), known, &r.Extra); err != nil { + return err + } + return errors.Join( + normalizeURLField("containerUrl", &r.ContainerURL), + normalizeURLField("consoleUrl", &r.ConsoleURL), + normalizeURLMap("output", r.Output), + ) } // IsTerminal reports whether the run has reached a terminal (finished) status. @@ -174,6 +187,9 @@ type Task struct { // PublicConfig is the public-facing display configuration of the task's landing page, set // when the task has been configured for publishing (nil otherwise). PublicConfig *TaskPublicConfig `json:"publicConfig,omitempty"` + // StandbyURL is the URL of the task's Actor Standby endpoint, normalized. Nil when the task + // has none. + StandbyURL *string `json:"standbyUrl,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -181,7 +197,10 @@ type Task struct { func (t *Task) UnmarshalJSON(data []byte) error { type alias Task known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(t), known, &t.Extra) + if err := unmarshalWithExtra(data, (*alias)(t), known, &t.Extra); err != nil { + return err + } + return normalizeOptionalURLField("standbyUrl", &t.StandbyURL) } // TaskPublicConfig is the public-facing display configuration of a task's public landing page. @@ -220,6 +239,11 @@ type Dataset struct { ModifiedAt *time.Time `json:"modifiedAt"` // ItemCount is the number of items currently stored. ItemCount int64 `json:"itemCount"` + // ConsoleURL is the URL of the dataset in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` + // ItemsPublicURL is the public URL of the dataset's items, normalized. It carries a + // signature for a private dataset. + ItemsPublicURL string `json:"itemsPublicUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -227,7 +251,13 @@ type Dataset struct { func (d *Dataset) UnmarshalJSON(data []byte) error { type alias Dataset known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(d), known, &d.Extra) + if err := unmarshalWithExtra(data, (*alias)(d), known, &d.Extra); err != nil { + return err + } + return errors.Join( + normalizeURLField("consoleUrl", &d.ConsoleURL), + normalizeURLField("itemsPublicUrl", &d.ItemsPublicURL), + ) } // KeyValueStore stores arbitrary data records. @@ -242,6 +272,13 @@ type KeyValueStore struct { CreatedAt *time.Time `json:"createdAt"` // ModifiedAt is when the store was last modified. ModifiedAt *time.Time `json:"modifiedAt"` + // ConsoleURL is the URL of the store in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` + // KeysPublicURL is the public URL listing the store's keys, normalized. It carries a + // signature for a private store. + KeysPublicURL string `json:"keysPublicUrl"` + // RecordsPublicURL is the public URL prefix of the store's records, normalized. + RecordsPublicURL string `json:"recordsPublicUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -249,7 +286,14 @@ type KeyValueStore struct { func (k *KeyValueStore) UnmarshalJSON(data []byte) error { type alias KeyValueStore known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(k), known, &k.Extra) + if err := unmarshalWithExtra(data, (*alias)(k), known, &k.Extra); err != nil { + return err + } + return errors.Join( + normalizeURLField("consoleUrl", &k.ConsoleURL), + normalizeURLField("keysPublicUrl", &k.KeysPublicURL), + normalizeURLField("recordsPublicUrl", &k.RecordsPublicURL), + ) } // KeyValueStoreKey is a single key listed from a key-value store. @@ -258,6 +302,9 @@ type KeyValueStoreKey struct { Key string `json:"key"` // Size is the record size in bytes. Size int64 `json:"size"` + // RecordPublicURL is the public URL of the record, normalized. It carries a signature for a + // private store. + RecordPublicURL string `json:"recordPublicUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -265,7 +312,10 @@ type KeyValueStoreKey struct { func (k *KeyValueStoreKey) UnmarshalJSON(data []byte) error { type alias KeyValueStoreKey known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(k), known, &k.Extra) + if err := unmarshalWithExtra(data, (*alias)(k), known, &k.Extra); err != nil { + return err + } + return normalizeURLField("recordPublicUrl", &k.RecordPublicURL) } // KeyValueStoreKeysPage is a page of keys from a key-value store. @@ -319,6 +369,8 @@ type RequestQueue struct { ModifiedAt *time.Time `json:"modifiedAt"` // TotalRequestCount is the total number of requests ever added. TotalRequestCount int64 `json:"totalRequestCount"` + // ConsoleURL is the URL of the queue in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -326,7 +378,10 @@ type RequestQueue struct { func (q *RequestQueue) UnmarshalJSON(data []byte) error { type alias RequestQueue known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(q), known, &q.Extra) + if err := unmarshalWithExtra(data, (*alias)(q), known, &q.Extra); err != nil { + return err + } + return normalizeURLField("consoleUrl", &q.ConsoleURL) } // RequestQueueRequest is a single request stored in a request queue. @@ -418,7 +473,8 @@ type Webhook struct { ID string `json:"id"` // UserID is the ID of the user who owns the webhook. UserID string `json:"userId"` - // RequestURL is the URL the webhook posts to. + // RequestURL is the URL the webhook posts to, normalized. Empty for a webhook that is not + // an HTTP one (the API returns null). RequestURL string `json:"requestUrl"` // EventTypes are the events that trigger the webhook. EventTypes []string `json:"eventTypes"` @@ -429,7 +485,10 @@ type Webhook struct { func (w *Webhook) UnmarshalJSON(data []byte) error { type alias Webhook known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(w), known, &w.Extra) + if err := unmarshalWithExtra(data, (*alias)(w), known, &w.Extra); err != nil { + return err + } + return normalizeURLField("requestUrl", &w.RequestURL) } // WebhookDispatch is a single invocation of a webhook. @@ -438,6 +497,9 @@ type WebhookDispatch struct { ID string `json:"id"` // WebhookID is the ID of the webhook that produced this dispatch. WebhookID string `json:"webhookId"` + // Webhook is a summary of the webhook as it was when the dispatch was created. Nil if the + // API does not return it. + Webhook *WebhookDispatchWebhookSummary `json:"webhook,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -448,12 +510,37 @@ func (d *WebhookDispatch) UnmarshalJSON(data []byte) error { return unmarshalWithExtra(data, (*alias)(d), known, &d.Extra) } +// WebhookDispatchWebhookSummary is the summary of a webhook embedded in a [WebhookDispatch]. +type WebhookDispatchWebhookSummary struct { + // ID is the ID of the webhook (the API calls the field `_id`). + ID string `json:"_id"` + // ActionType is the kind of action the webhook runs, e.g. HTTP request or a Slack message. + ActionType string `json:"actionType"` + // IsAdHoc reports whether the webhook was attached to a single run instead of being saved. + IsAdHoc bool `json:"isAdHoc"` + // RequestURL is the URL the webhook posts to, normalized. Empty when there is none. + RequestURL string `json:"requestUrl"` + // Extra holds any other fields returned by the API. + Extra Extra `json:"-"` +} + +func (w *WebhookDispatchWebhookSummary) UnmarshalJSON(data []byte) error { + type alias WebhookDispatchWebhookSummary + known := knownJSONKeys(alias{}) + if err := unmarshalWithExtra(data, (*alias)(w), known, &w.Extra); err != nil { + return err + } + return normalizeURLField("requestUrl", &w.RequestURL) +} + // User is an Apify user account. type User struct { // ID is the unique user ID. ID string `json:"id"` // Username is the user's username. Username string `json:"username"` + // Profile is the user's public profile. Nil if the API does not return it. + Profile *UserProfile `json:"profile,omitempty"` // Extra holds any other fields returned by the API (private details for "me"). Extra Extra `json:"-"` } @@ -464,6 +551,30 @@ func (u *User) UnmarshalJSON(data []byte) error { return unmarshalWithExtra(data, (*alias)(u), known, &u.Extra) } +// UserProfile is the public profile of a [User]. +type UserProfile struct { + // Name is the user's display name. + Name string `json:"name"` + // PictureURL is the URL of the user's profile picture, normalized. Nil when there is none. + PictureURL *string `json:"pictureUrl,omitempty"` + // WebsiteURL is the URL of the user's website, normalized. Nil when there is none. + WebsiteURL *string `json:"websiteUrl,omitempty"` + // Extra holds any other fields returned by the API. + Extra Extra `json:"-"` +} + +func (p *UserProfile) UnmarshalJSON(data []byte) error { + type alias UserProfile + known := knownJSONKeys(alias{}) + if err := unmarshalWithExtra(data, (*alias)(p), known, &p.Extra); err != nil { + return err + } + return errors.Join( + normalizeOptionalURLField("pictureUrl", &p.PictureURL), + normalizeOptionalURLField("websiteUrl", &p.WebsiteURL), + ) +} + // ActorStoreListItem is an Actor as listed in the Apify Store. type ActorStoreListItem struct { // ID is the unique Actor ID. @@ -474,6 +585,11 @@ type ActorStoreListItem struct { Username string `json:"username"` // Title is the human-readable title. Title string `json:"title"` + // URL is the URL of the Actor's page in the Apify Store, normalized. Nil when there is none. + URL *string `json:"url,omitempty"` + // UserPictureURL is the URL of the owner's profile picture, normalized. Nil when there is + // none. + UserPictureURL *string `json:"userPictureUrl,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -481,7 +597,13 @@ type ActorStoreListItem struct { func (a *ActorStoreListItem) UnmarshalJSON(data []byte) error { type alias ActorStoreListItem known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(a), known, &a.Extra) + if err := unmarshalWithExtra(data, (*alias)(a), known, &a.Extra); err != nil { + return err + } + return errors.Join( + normalizeOptionalURLField("url", &a.URL), + normalizeOptionalURLField("userPictureUrl", &a.UserPictureURL), + ) } // ActorVersion is a single version of an Actor. diff --git a/models_test.go b/models_test.go new file mode 100644 index 0000000..610107e --- /dev/null +++ b/models_test.go @@ -0,0 +1,31 @@ +package apify + +import ( + "context" + "testing" +) + +// The API returns a pending request with a null handledAt, and a schedule whose Console editor +// state was dropped with a null uiState: both parse, the null kept in Extra. +func TestNullFieldsOfRequestsAndSchedulesParse(t *testing.T) { + var request RequestQueueRequest + mustUnmarshal(t, `{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}`, &request) + if request.ID != "r1" || request.UniqueKey != "k" || string(request.Extra["handledAt"]) != "null" { + t.Errorf("unexpected request %+v", request) + } + if request.Method != "" || request.UserData != nil { + t.Errorf("the optional fields of a minimal request stay empty: %+v", request) + } + + var schedule Schedule + mustUnmarshal(t, `{"id":"s1","userId":"u","name":"n","cronExpression":"* * * * *","isEnabled":true,"uiState":null}`, &schedule) + if schedule.ID != "s1" || string(schedule.Extra["uiState"]) != "null" { + t.Errorf("unexpected schedule %+v", schedule) + } + + backend := &mockBackend{responses: constant(200, `{"data":{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}}`)} + got, present, err := testClient(backend, 0).RequestQueue("q").GetRequest(context.Background(), "r1") + if err != nil || !present || got.ID != "r1" { + t.Fatalf("GetRequest of a pending request: %+v, %v, %v", got, present, err) + } +} diff --git a/request_queue.go b/request_queue.go index 0ed6ead..2ff316e 100644 --- a/request_queue.go +++ b/request_queue.go @@ -14,6 +14,8 @@ type ListRequestsOptions struct { // Limit is the maximum number of requests to return. Limit *int64 // ExclusiveStartID lists requests after this ID. + // + // Deprecated: the API deprecated the exclusiveStartId parameter. Use Cursor. ExclusiveStartID *string // Cursor is an opaque pagination cursor (alternative to ExclusiveStartID). Cursor *string @@ -380,20 +382,33 @@ func (c *RequestQueueClient) UnlockRequests(ctx context.Context) (json.RawMessag } // PaginateRequests returns a lazy iterator over all requests in the queue, fetching pages -// of up to pageLimit requests at a time (nil for the server default). +// of up to pageLimit requests at a time (nil for the server default). To cap the total, start +// from a cursor or filter the states, use [RequestQueueClient.PaginateRequestsWithOptions]. func (c *RequestQueueClient) PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator { - return &RequestQueueRequestsIterator{client: c, pageLimit: pageLimit} + return c.PaginateRequestsWithOptions(ListRequestsOptions{}, pageLimit) +} + +// PaginateRequestsWithOptions is [RequestQueueClient.PaginateRequests] with listing options, like +// the reference client's paginateRequests: Limit caps the number of requests yielded across all +// pages (nil or ptr(0) means all of them), Cursor (or the deprecated ExclusiveStartID) sets where +// to start, and Filter applies to every page. The page size is the separate chunkSize argument +// (nil for the server default). Invalid options are reported by the first call of Next. +func (c *RequestQueueClient) PaginateRequestsWithOptions(options ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator { + return &RequestQueueRequestsIterator{client: c, options: options, chunkSize: chunkSize} } // RequestQueueRequestsIterator lazily iterates over a request queue's requests, fetching one // page at a time via the cursor-based listing endpoint. type RequestQueueRequestsIterator struct { client *RequestQueueClient - pageLimit *int64 + options ListRequestsOptions + chunkSize *int64 buffer []RequestQueueRequest pos int nextCursor string + capped bool // whether a total cap (options.Limit) is in effect + remaining int64 // requests the cap still allows; only meaningful when capped started bool exhausted bool } @@ -407,7 +422,7 @@ type requestsPage struct { // Next returns the next request, or (nil, nil) when the iterator is exhausted. func (it *RequestQueueRequestsIterator) Next(ctx context.Context) (*RequestQueueRequest, error) { for it.pos >= len(it.buffer) { - if it.exhausted || (it.started && it.nextCursor == "") { + if it.exhausted { return nil, nil } if err := it.fetchPage(ctx); err != nil { @@ -419,13 +434,33 @@ func (it *RequestQueueRequestsIterator) Next(ctx context.Context) (*RequestQueue return &req, nil } -// fetchPage loads the next page of requests into the buffer. +// fetchPage loads the next page of requests into the buffer. The page size is the smaller of the +// remaining total cap and the chunk size, so the last page is never over-fetched. func (it *RequestQueueRequestsIterator) fetchPage(ctx context.Context) error { - params := NewQueryParams() - params.AddInt("limit", it.pageLimit) - if it.nextCursor != "" { - params.AddString("cursor", &it.nextCursor) + opts := it.options + if !it.started { + if err := opts.validate(); err != nil { + return err + } + if l := opts.Limit; l != nil && *l > 0 { + it.capped = true + it.remaining = *l + } + } else { + opts.ExclusiveStartID = nil + opts.Cursor = &it.nextCursor + } + var capLeft, chunk int64 + if it.capped { + capLeft = it.remaining } + if it.chunkSize != nil { + chunk = *it.chunkSize + } + opts.Limit = pageLimitPtr(minForLimitParam(capLeft, chunk)) + + params := NewQueryParams() + opts.apply(params) it.client.withClientKey(params) raw, err := getResourceRequired[requestsPage](ctx, it.client.ctx, "requests", params, it.client.ctx.mediumTimeout()) @@ -435,12 +470,18 @@ func (it *RequestQueueRequestsIterator) fetchPage(ctx context.Context) error { it.started = true it.buffer = raw.Items it.pos = 0 + it.nextCursor = "" if raw.NextCursor != nil { it.nextCursor = *raw.NextCursor - } else { - it.nextCursor = "" } - if len(raw.Items) == 0 && it.nextCursor == "" { + if it.capped { + it.remaining -= int64(len(it.buffer)) + if it.remaining < 0 { // a server that returned more than asked for + it.buffer = it.buffer[:int64(len(it.buffer))+it.remaining] + it.remaining = 0 + } + } + if it.nextCursor == "" || (it.capped && it.remaining == 0) { it.exhausted = true } return nil diff --git a/review_fixes_test.go b/review_fixes_test.go index a290bfd..9981a0d 100644 --- a/review_fixes_test.go +++ b/review_fixes_test.go @@ -458,8 +458,26 @@ func TestMonthlyUsageWithOptionsDate(t *testing.T) { if strings.Contains(backend.lastURL, "date=") { t.Fatalf("date must be omitted when empty, got %q", backend.lastURL) } - if _, err := client.User("someone").MonthlyUsageWithOptions(context.Background(), MonthlyUsageOptions{}); err == nil { - t.Fatal("expected an error for a non-me client") +} + +// The usage and limits endpoints are documented for a user by ID or username as well, so a +// client from User(id) calls them instead of failing. +func TestUserLimitsAndUsageByID(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + ctx := context.Background() + + if _, err := client.User("someone").MonthlyUsageWithOptions(ctx, MonthlyUsageOptions{Date: "2026-09-14"}); err != nil { + t.Fatalf("monthly usage by ID: %v", err) + } + if !strings.Contains(backend.lastURL, "/users/someone/usage/monthly") || !strings.Contains(backend.lastURL, "date=2026-09-14") { + t.Fatalf("unexpected monthly usage URL %q", backend.lastURL) + } + if _, err := client.User("someone").Limits(ctx); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { + t.Fatalf("limits by ID: %v, URL %q", err, backend.lastURL) + } + if err := client.User("someone").UpdateLimits(ctx, map[string]any{"maxMonthlyUsageUsd": 5}); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { + t.Fatalf("update limits by ID: %v, URL %q", err, backend.lastURL) } } diff --git a/run.go b/run.go index 4079067..f587e61 100644 --- a/run.go +++ b/run.go @@ -198,6 +198,9 @@ func (c *RunClient) Resurrect(ctx context.Context, options RunResurrectOptions) // retried by the transport is applied at most once, matching the reference client. The API // requires this header and expires the key 3 minutes after the charge. func (c *RunClient) Charge(ctx context.Context, options RunChargeOptions) error { + if c.ctx.idErr != nil { + return c.ctx.idErr + } count := int64(1) if options.Count != nil { count = *options.Count @@ -238,31 +241,43 @@ func (c *RunClient) WaitForFinish(ctx context.Context, waitSecs *int64) (ActorRu return waitForFinish[ActorRun](ctx, c.ctx, waitSecs, "run", func(r *ActorRun) bool { return r.IsTerminal() }) } -// Dataset returns a client for this run's default dataset. +// Dataset returns a client for this run's default dataset. On a client from LastRun, it carries +// the same status and origin filters, so it addresses the dataset of the same run. func (c *RunClient) Dataset() *DatasetClient { - return newDatasetNestedClient(c.ctx.http, c.ctx.subURL(""), "dataset") + client := newDatasetNestedClient(c.ctx.http, c.ctx.subURL(""), "dataset") + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // KeyValueStore returns a client for this run's default key-value store. func (c *RunClient) KeyValueStore() *KeyValueStoreClient { - return newKeyValueStoreNestedClient(c.ctx.http, c.ctx.subURL(""), "key-value-store") + client := newKeyValueStoreNestedClient(c.ctx.http, c.ctx.subURL(""), "key-value-store") + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // RequestQueue returns a client for this run's default request queue. func (c *RunClient) RequestQueue() *RequestQueueClient { - return newRequestQueueNestedClient(c.ctx.http, c.ctx.subURL(""), "request-queue") + client := newRequestQueueNestedClient(c.ctx.http, c.ctx.subURL(""), "request-queue") + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // Log returns a client for accessing this run's log. func (c *RunClient) Log() *LogClient { - return newNestedLogClient(c.ctx.http, c.ctx.subURL("")) + client := newNestedLogClient(c.ctx.http, c.ctx.subURL("")) + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } -// GetStreamedLog opens a live stream of this run's raw log, for convenient log redirection. +// GetStreamedLog opens a live stream of this run's raw log. The caller must close the returned +// reader. It is a convenience wrapper over Log().StreamWithOptions with raw=true. +// +// Despite the name it is not the counterpart of the reference client's getStreamedLog(), which +// redirects the log message by message: that is [RunClient.NewStreamedLog]. // -// It is a convenience wrapper over Log().StreamWithOptions with raw=true (matching the -// reference client's getStreamedLog, which streams raw log content). The caller must close -// the returned reader. +// Deprecated: use [RunClient.NewStreamedLog] to redirect the log, or Log().StreamWithOptions for +// the raw stream. func (c *RunClient) GetStreamedLog(ctx context.Context) (io.ReadCloser, error) { raw := true return c.Log().StreamWithOptions(ctx, LogOptions{Raw: &raw}) diff --git a/run_collection.go b/run_collection.go index a801c9f..8dc8ea9 100644 --- a/run_collection.go +++ b/run_collection.go @@ -3,8 +3,8 @@ package apify import "context" // RunListOptions adds run-specific filters on top of [ListOptions] for -// [RunCollectionClient.List]. The startedAfter/startedBefore filters are only honoured by -// the Actor-scoped and task-scoped run collections. +// [RunCollectionClient.List]. The StartedAfter, StartedBefore and Deployment filters are only +// honoured by the account-wide and Actor-scoped run collections, not by a task's. type RunListOptions struct { // Status filters by one or more run statuses (e.g. "SUCCEEDED", "RUNNING"). Sent as a // comma-separated list, as the API accepts. @@ -13,12 +13,18 @@ type RunListOptions struct { StartedAfter *string // StartedBefore filters to runs started before this ISO-8601 timestamp. StartedBefore *string + // Deployment filters Actor Standby runs by the pool that serves them, as a list of + // SINGLE_TENANT, MULTI_TENANT_FREE_USERS, MULTI_TENANT_PAYING_USERS or MULTI_TENANT_OWNER_PAID. + // Sent as a comma-separated list. The API requires Status to hold only non-terminal statuses + // (READY, RUNNING, TIMING-OUT or ABORTING) when it is set. + Deployment []string } func (o RunListOptions) apply(q *QueryParams) { q.AddCSV("status", o.Status). AddString("startedAfter", o.StartedAfter). - AddString("startedBefore", o.StartedBefore) + AddString("startedBefore", o.StartedBefore). + AddCSV("deployment", o.Deployment) } // RunCollectionClient is a client for a run collection: the account-wide collection diff --git a/run_collection_test.go b/run_collection_test.go new file mode 100644 index 0000000..cade32b --- /dev/null +++ b/run_collection_test.go @@ -0,0 +1,35 @@ +package apify + +import ( + "context" + "net/url" + "strings" + "testing" +) + +func TestRunListsSendTheDeploymentFilter(t *testing.T) { + ctx := context.Background() + filter := RunListOptions{Status: []string{"RUNNING"}, Deployment: []string{"SINGLE_TENANT", "MULTI_TENANT_FREE_USERS"}} + + for name, list := range map[string]func(client *ApifyClient) error{ + "account runs": func(c *ApifyClient) error { _, err := c.Runs().List(ctx, ListOptions{}, filter); return err }, + "actor runs": func(c *ApifyClient) error { _, err := c.Actor("a").Runs().List(ctx, ListOptions{}, filter); return err }, + } { + backend := &mockBackend{responses: okData()} + if err := list(testClient(backend, 0)); err != nil { + t.Fatalf("%s: %v", name, err) + } + query, _ := url.ParseQuery(strings.SplitN(backend.lastURL, "?", 2)[1]) + if got := query.Get("deployment"); got != "SINGLE_TENANT,MULTI_TENANT_FREE_USERS" { + t.Errorf("%s: deployment = %q", name, got) + } + } + + backend := &mockBackend{responses: okData()} + if _, err := testClient(backend, 0).Runs().List(ctx, ListOptions{}, RunListOptions{}); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "deployment") { + t.Errorf("deployment must be omitted when unset, got %q", backend.lastURL) + } +} diff --git a/run_dataset_items.go b/run_dataset_items.go new file mode 100644 index 0000000..2db48ca --- /dev/null +++ b/run_dataset_items.go @@ -0,0 +1,278 @@ +package apify + +import ( + "context" + "encoding/json" + "errors" +) + +const ( + // defaultRunIterateChunkSize is the number of dataset rows requested per API call when + // [RunIterateDatasetItemsOptions.ChunkSize] is unset. Matches the reference client's + // DEFAULT_ITERATE_CHUNK_SIZE. + defaultRunIterateChunkSize = int64(1000) + // defaultRunIteratePollIntervalSecs is how long the iterator waits for the run to finish + // between polls when [RunIterateDatasetItemsOptions.PollIntervalSecs] is unset. + defaultRunIteratePollIntervalSecs = int64(5) +) + +// errRunIterateDesc is returned when the items options ask for newest-first order. +var errRunIterateDesc = errors.New("RunIterateDatasetItemsOptions: Items.Desc is not supported: the iterator reads rows by offset from the oldest, which a descending order would count from the newest") + +// RunIterateDatasetItemsOptions configures [RunClient.IterateDatasetItems] and +// [IterateRunDatasetItems]. +type RunIterateDatasetItemsOptions struct { + // Items are the filters and projections applied to every page, the same as for + // [DatasetClient.ListItems]. Items.Offset is the number of rows to skip from the beginning + // (default 0), and Items.Limit the maximum number of dataset rows to scan (default no limit): + // fewer items are yielded when filters drop some, more when Unwind splits a row into several. + // Items.Desc is not supported, because the rows are read by offset from the oldest. + Items DatasetListItemsOptions + // ChunkSize is the maximum number of dataset rows requested per API call (default 1000). + ChunkSize *int64 + // PollIntervalSecs is how long to wait for the run to finish between polls, in seconds + // (default 5). + PollIntervalSecs *int64 +} + +// runIteratePhase is the stage a [RunDatasetItemsIterator] is in. +type runIteratePhase int + +const ( + // runIterateStart: nothing has been requested yet. + runIterateStart runIteratePhase = iota + // runIterateLive: the run may still be producing items. Each round reads the rows below the + // dataset's itemCount, then waits for the run to finish. + runIterateLive + // runIterateTail: the run has finished. The rows past itemCount (which lags a few seconds + // behind the pushed items) are read a page at a time until none are left. + runIterateTail + // runIterateDone: the iterator is exhausted. + runIterateDone +) + +// RunDatasetItemsIterator iterates over the items of a run's default dataset while the run is +// still producing them. Obtain one from [RunClient.IterateDatasetItems] or +// [IterateRunDatasetItems] and drain it by calling Next until it returns (nil, nil). +// +// While the run has not finished, each round yields the rows below the dataset's itemCount and +// then waits up to PollIntervalSecs for the run to finish, so the last rows are read as soon as it +// does. Each page is requested with a limit that ends at itemCount, so it covers exactly the +// rows it asks for, whatever the filters or Unwind do to the items. itemCount lags a few seconds +// behind the pushed items, so once the run reaches a terminal status the rows past it are read a +// page at a time until none are left. On a client from LastRun, the iterator sticks to the run +// its first request resolves to. Matches the reference client's iterateDatasetItems(). +type RunDatasetItemsIterator[T any] struct { + run *RunClient + options RunIterateDatasetItemsOptions + + pageSize int64 + pollSecs int64 + position int64 + end int64 + hasEnd bool + + phase runIteratePhase + dataset *DatasetClient + runClient *RunClient + currentRun ActorRun + runPresent bool + // roundFinished is whether the run was already finished when the current live round began. + roundFinished bool + // roundItemCount is the number of rows the current live round reads up to. + roundItemCount int64 + roundStarted bool + + buffer []T + pos int +} + +// IterateRunDatasetItems returns an iterator over the items of the run's default dataset, each +// decoded into T, that follows the dataset while the run is still producing items. See +// [RunDatasetItemsIterator]. +func IterateRunDatasetItems[T any](c *RunClient, options RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[T] { + it := &RunDatasetItemsIterator[T]{ + run: c, + options: options, + pageSize: defaultRunIterateChunkSize, + pollSecs: defaultRunIteratePollIntervalSecs, + position: offsetVal(options.Items.Offset), + } + if options.ChunkSize != nil && *options.ChunkSize > 0 { + it.pageSize = *options.ChunkSize + } + if options.PollIntervalSecs != nil && *options.PollIntervalSecs >= 0 { + it.pollSecs = *options.PollIntervalSecs + } + if options.Items.Limit != nil && *options.Items.Limit > 0 { + it.hasEnd = true + it.end = it.position + *options.Items.Limit + } + return it +} + +// IterateDatasetItems is [IterateRunDatasetItems] with every item decoded into a +// json.RawMessage. See [RunDatasetItemsIterator]. +func (c *RunClient) IterateDatasetItems(options RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[json.RawMessage] { + return IterateRunDatasetItems[json.RawMessage](c, options) +} + +// Next returns the next item, or (nil, nil) once every item has been read (or the Limit of +// scanned rows is reached). It calls the API only when the current in-memory page is used up. +func (it *RunDatasetItemsIterator[T]) Next(ctx context.Context) (*T, error) { + for it.pos >= len(it.buffer) { + if it.phase == runIterateDone { + return nil, nil + } + if err := it.advance(ctx); err != nil { + return nil, err + } + } + item := it.buffer[it.pos] + it.pos++ + return &item, nil +} + +// advance performs the next step of the iteration, which loads the next page into the buffer, or +// moves to the next phase. +func (it *RunDatasetItemsIterator[T]) advance(ctx context.Context) error { + it.buffer, it.pos = nil, 0 + switch it.phase { + case runIterateStart: + return it.start(ctx) + case runIterateLive: + return it.liveStep(ctx) + case runIterateTail: + return it.tailStep(ctx) + default: + return nil + } +} + +// start resolves the run and the client of its default dataset. +func (it *RunDatasetItemsIterator[T]) start(ctx context.Context) error { + if it.options.Items.Desc != nil && *it.options.Items.Desc { + return errRunIterateDesc + } + it.runClient = it.run + run, present, err := getResourceWithTimeout[ActorRun](ctx, it.run.ctx, "", NewQueryParams(), it.run.ctx.longTimeout()) + if err != nil { + return err + } + it.currentRun, it.runPresent = run, present + // A LastRun client resolves runs/last on each request, so a newer run would swap the dataset + // mid-iteration: pin the iteration to the run the first request resolved to. + if present && run.ID != it.run.id { + it.runClient = it.run.root.Run(run.ID) + } + it.dataset = it.runClient.Dataset() + it.phase = runIterateLive + return nil +} + +// liveStep reads one page of the current live round, or ends the round: it waits for the run to +// finish and starts another one, or moves on to the tail. +func (it *RunDatasetItemsIterator[T]) liveStep(ctx context.Context) error { + if !it.roundStarted { + it.roundFinished = !it.runPresent || it.currentRun.IsTerminal() + dataset, present, err := getResourceWithTimeout[Dataset](ctx, it.dataset.ctx, "", NewQueryParams(), it.dataset.ctx.longTimeout()) + if err != nil { + return err + } + it.roundItemCount = 0 + if present { + it.roundItemCount = dataset.ItemCount + } + if it.hasEnd { + it.roundItemCount = minInt64(it.roundItemCount, it.end) + } + it.roundStarted = true + } + + if it.position < it.roundItemCount { + pageLimit := minInt64(it.pageSize, it.roundItemCount-it.position) + items, _, err := it.listPage(ctx, it.position, pageLimit) + if err != nil { + return err + } + it.buffer = items + it.position += pageLimit + return nil + } + + it.roundStarted = false + if it.hasEnd && it.position >= it.end { + it.phase = runIterateDone + return nil + } + if it.roundFinished { + it.phase = runIterateTail + return nil + } + run, err := it.runClient.WaitForFinish(ctx, &it.pollSecs) + if err != nil { + return err + } + it.currentRun, it.runPresent = run, true + return nil +} + +// tailStep reads one page past itemCount after the run has finished. Only an empty page ends the +// iteration, since filters can shorten a full one. +func (it *RunDatasetItemsIterator[T]) tailStep(ctx context.Context) error { + pageLimit := it.pageSize + if it.hasEnd { + pageLimit = minInt64(it.pageSize, it.end-it.position) + } + items, scanned, err := it.listPage(ctx, it.position, pageLimit) + if err != nil { + return err + } + it.buffer = items + + // A page that Clean, SkipEmpty or Unwind emptied past a lagging itemCount reports no scanned + // rows either, so a plain read confirms the dataset really has no more rows. + if len(items) == 0 && scanned == 0 { + confirmed, err := it.confirmNoMoreRows(ctx) + if err != nil { + return err + } + if confirmed { + it.phase = runIterateDone + return nil + } + } + it.position += pageLimit + if it.hasEnd && it.position >= it.end { + it.phase = runIterateDone + } + return nil +} + +// confirmNoMoreRows reports whether the dataset has no row at the current position. Without a +// filter that can empty a page, an empty page already means that. +func (it *RunDatasetItemsIterator[T]) confirmNoMoreRows(ctx context.Context) (bool, error) { + items := it.options.Items + filtered := (items.Clean != nil && *items.Clean) || (items.SkipEmpty != nil && *items.SkipEmpty) || len(items.Unwind) > 0 + if !filtered { + return true, nil + } + plain, err := ListDatasetItems[json.RawMessage](ctx, it.dataset, DatasetListItemsOptions{Offset: &it.position, Limit: Ptr(int64(1))}) + if err != nil { + return false, err + } + return len(plain.Items) == 0, nil +} + +// listPage reads the page of the dataset that starts at offset and holds at most limit rows, +// returning its items and the number of rows the API scanned for it. +func (it *RunDatasetItemsIterator[T]) listPage(ctx context.Context, offset, limit int64) ([]T, int64, error) { + options := it.options.Items + options.Offset = &offset + options.Limit = &limit + page, err := ListDatasetItems[T](ctx, it.dataset, options) + if err != nil { + return nil, 0, err + } + return page.Items, page.scanned, nil +} diff --git a/run_dataset_items_test.go b/run_dataset_items_test.go new file mode 100644 index 0000000..74b6ba5 --- /dev/null +++ b/run_dataset_items_test.go @@ -0,0 +1,233 @@ +package apify + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "testing" +) + +// fakeRunBackend simulates a run that keeps pushing items to its default dataset. Each answer to +// a run poll (a GET of the run or a waitForFinish) moves the simulation one step: runSteps[i] is +// the run status and itemCounts[i] the dataset's itemCount after it, and every item row exists +// from the start of the test, so a page past itemCount still returns rows, like the lagging +// itemCount of the real API. +type fakeRunBackend struct { + mu sync.Mutex + runID string + runSteps []string + itemCounts []int64 + totalRows int + step int + requests []string +} + +func (b *fakeRunBackend) Do(req *http.Request) (*http.Response, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.requests = append(b.requests, req.URL.RequestURI()) + path := req.URL.Path + query := req.URL.Query() + respond := func(body string, header http.Header) (*http.Response, error) { + if header == nil { + header = http.Header{} + } + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader([]byte(body)))}, nil + } + current := min(b.step, len(b.runSteps)-1) + switch { + case strings.HasSuffix(path, "/dataset/items"): + offset, _ := strconv.Atoi(query.Get("offset")) + limit, _ := strconv.Atoi(query.Get("limit")) + var rows []map[string]int + for i := offset; i < offset+limit && i < b.totalRows; i++ { + rows = append(rows, map[string]int{"n": i}) + } + body, _ := json.Marshal(rows) + if rows == nil { + body = []byte("[]") + } + header := http.Header{} + header.Set("X-Apify-Pagination-Count", strconv.Itoa(len(rows))) + return respond(string(body), header) + case strings.HasSuffix(path, "/dataset"): + return respond(fmt.Sprintf(`{"data":{"id":"d","itemCount":%d}}`, b.itemCounts[current]), nil) + case strings.HasSuffix(path, "/actor-runs/"+b.runID), strings.HasSuffix(path, "/runs/last"): + if query.Get("waitForFinish") != "" { + b.step++ + current = min(b.step, len(b.runSteps)-1) + } + return respond(fmt.Sprintf(`{"data":{"id":%q,"status":%q}}`, b.runID, b.runSteps[current]), nil) + } + return nil, fmt.Errorf("unexpected request %s", req.URL) +} + +func drainRunItems(t *testing.T, it *RunDatasetItemsIterator[map[string]int]) []int { + t.Helper() + var got []int + for { + item, err := it.Next(context.Background()) + if err != nil { + t.Fatalf("Next: %v", err) + } + if item == nil { + return got + } + got = append(got, (*item)["n"]) + } +} + +func sequence(from, to int) []int { + var out []int + for i := from; i < to; i++ { + out = append(out, i) + } + return out +} + +func equalInts(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestIterateDatasetItemsReadsAFinishedRun(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{5}, totalRows: 5} + client := testClient(backend, 0) + got := drainRunItems(t, IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{})) + if !equalInts(got, sequence(0, 5)) { + t.Fatalf("got %v", got) + } +} + +// While the run is running, the iterator reads up to itemCount, waits for the run to finish, and +// then reads the rows past a stale itemCount until a page is empty. +func TestIterateDatasetItemsFollowsARunningRun(t *testing.T) { + backend := &fakeRunBackend{ + runID: "run1", + runSteps: []string{"RUNNING", "RUNNING", "SUCCEEDED"}, + itemCounts: []int64{2, 4, 4}, + totalRows: 7, // itemCount stays at 4 although 7 rows exist: the tail catches up + } + client := testClient(backend, 0) + it := IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{ + ChunkSize: Ptr(int64(3)), + PollIntervalSecs: Ptr(int64(1)), + }) + got := drainRunItems(t, it) + if !equalInts(got, sequence(0, 7)) { + t.Fatalf("got %v", got) + } + polls := 0 + for _, r := range backend.requests { + if strings.Contains(r, "/actor-runs/run1?waitForFinish=") { + polls++ + } + } + if polls == 0 { + t.Fatalf("expected the iterator to wait for the run to finish, got %v", backend.requests) + } + // The first round stops at the itemCount of 2 although 3 rows fit in a chunk. + if !strings.Contains(backend.requests[2], "offset=0&limit=2") { + t.Fatalf("the first page must end at itemCount: %v", backend.requests) + } + for _, r := range backend.requests { + if strings.Contains(r, "/dataset/items") && strings.Contains(r, "limit=") { + query, _ := url.ParseQuery(strings.SplitN(r, "?", 2)[1]) + if limit, _ := strconv.Atoi(query.Get("limit")); limit > 3 { + t.Fatalf("a page asked for %d rows, more than the chunk size 3: %s", limit, r) + } + } + } +} + +func TestIterateDatasetItemsHonorsOffsetAndLimit(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{10}, totalRows: 10} + client := testClient(backend, 0) + it := IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{ + Items: DatasetListItemsOptions{Offset: Ptr(int64(2)), Limit: Ptr(int64(5))}, + ChunkSize: Ptr(int64(2)), + }) + if got := drainRunItems(t, it); !equalInts(got, sequence(2, 7)) { + t.Fatalf("got %v, want rows 2..6", got) + } +} + +// The tail reads past itemCount even when the run is finished before the first request, up to the +// Limit. +func TestIterateDatasetItemsLimitEndsTheTail(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{0}, totalRows: 10} + client := testClient(backend, 0) + it := IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{ + Items: DatasetListItemsOptions{Limit: Ptr(int64(4))}, + ChunkSize: Ptr(int64(3)), + }) + if got := drainRunItems(t, it); !equalInts(got, sequence(0, 4)) { + t.Fatalf("got %v, want rows 0..3", got) + } +} + +// A LastRun client resolves runs/last on each request: the iteration pins itself to the run its +// first request resolved to. +func TestIterateDatasetItemsPinsALastRunClient(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{3}, totalRows: 3} + client := testClient(backend, 0) + got := drainRunItems(t, IterateRunDatasetItems[map[string]int](client.Actor("a").LastRun("SUCCEEDED"), RunIterateDatasetItemsOptions{})) + if !equalInts(got, sequence(0, 3)) { + t.Fatalf("got %v", got) + } + for _, r := range backend.requests[1:] { + if strings.Contains(r, "/runs/last") { + t.Fatalf("after resolving the run, the requests must name it, not runs/last: %s", r) + } + } + if !strings.Contains(backend.requests[0], "/runs/last") || !strings.Contains(backend.requests[0], "status=SUCCEEDED") { + t.Fatalf("the first request resolves the last run: %s", backend.requests[0]) + } +} + +func TestIterateDatasetItemsRejectsDescendingOrder(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{3}, totalRows: 3} + client := testClient(backend, 0) + it := client.Run("run1").IterateDatasetItems(RunIterateDatasetItemsOptions{Items: DatasetListItemsOptions{Desc: Ptr(true)}}) + for range 2 { + if _, err := it.Next(context.Background()); err == nil { + t.Fatal("expected an error for a descending order") + } + } + if len(backend.requests) != 0 { + t.Fatalf("no request may be sent, got %v", backend.requests) + } +} + +func TestIterateDatasetItemsDecodesIntoRawMessages(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{2}, totalRows: 2} + it := testClient(backend, 0).Run("run1").IterateDatasetItems(RunIterateDatasetItemsOptions{}) + var raws []string + for { + item, err := it.Next(context.Background()) + if err != nil { + t.Fatal(err) + } + if item == nil { + break + } + raws = append(raws, string(*item)) + } + if strings.Join(raws, ",") != `{"n":0},{"n":1}` { + t.Fatalf("got %v", raws) + } +} diff --git a/store_collection.go b/store_collection.go index 04bf7e2..06a6063 100644 --- a/store_collection.go +++ b/store_collection.go @@ -24,6 +24,10 @@ type StoreListOptions struct { AllowsAgenticUsers *bool // ResponseFormat selects the response format. ResponseFormat *string + // IncludeInputSchema, if true, adds to each listed Actor an inputSchema field with a compact + // version of its default build's input schema (property types and required properties only). + // With it set, the API defaults Limit to 10 and does not accept a larger one. + IncludeInputSchema *bool } func (o StoreListOptions) apply(q *QueryParams) { @@ -36,7 +40,8 @@ func (o StoreListOptions) apply(q *QueryParams) { AddString("pricingModel", o.PricingModel). AddBool("includeUnrunnableActors", o.IncludeUnrunnableActors). AddBool("allowsAgenticUsers", o.AllowsAgenticUsers). - AddString("responseFormat", o.ResponseFormat) + AddString("responseFormat", o.ResponseFormat). + AddBool("includeInputSchema", o.IncludeInputSchema) } // StoreCollectionClient is a client for browsing the Apify Store (GET /v2/store). diff --git a/store_collection_test.go b/store_collection_test.go new file mode 100644 index 0000000..0b9e469 --- /dev/null +++ b/store_collection_test.go @@ -0,0 +1,24 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +func TestStoreListSendsIncludeInputSchema(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + if _, err := client.Store().List(context.Background(), StoreListOptions{IncludeInputSchema: Ptr(true)}); err != nil { + t.Fatal(err) + } + if !strings.Contains(backend.lastURL, "includeInputSchema=1") { + t.Errorf("expected includeInputSchema=1, got %q", backend.lastURL) + } + if _, err := client.Store().List(context.Background(), StoreListOptions{}); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "includeInputSchema") { + t.Errorf("includeInputSchema must be omitted when unset, got %q", backend.lastURL) + } +} diff --git a/streamed_log.go b/streamed_log.go new file mode 100644 index 0000000..9730bb8 --- /dev/null +++ b/streamed_log.go @@ -0,0 +1,445 @@ +package apify + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" + "time" +) + +const ( + // emptyLogStreamRetry is how long a [StreamedLog] waits before it reopens a log stream that + // delivered nothing: the API serves the log of a run that has not logged anything yet as an + // empty stream that ends at once. The pause doubles up to emptyLogStreamMaxRetry, which bounds + // the request rate while a run waits long to start. Matches the reference client's + // EMPTY_LOG_STREAM_RETRY_MILLIS and EMPTY_LOG_STREAM_MAX_RETRY_MILLIS. + emptyLogStreamRetry = 500 * time.Millisecond + emptyLogStreamMaxRetry = 5 * time.Second + // stopGrace is how long [StreamedLog.Stop] waits for the messages already in flight before it + // aborts the log stream. Matches the reference client's STOP_GRACE_MILLIS. + stopGrace = time.Second + // logRedirectPrefixSeparator ends the default message prefix, which names the Actor and the run. + logRedirectPrefixSeparator = " -> " + // logTimestampPattern matches the ISO timestamp, 24 characters long, that starts a message. + logTimestampPattern = `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z` + // logMarkerOverlapBytes is the length of a timestamp together with the end of line before it. + logMarkerOverlapBytes = 25 + // logReadChunkBytes is the size of the buffer the log stream is read through. + logReadChunkBytes = 32 * 1024 +) + +// logMessageMarker matches the ISO timestamp that starts every message of an Actor run log. The +// log is split into messages at these markers, so a message that spans several lines (a stack +// trace, say) is redirected as one. +var logMessageMarker = regexp.MustCompile(`(?:\n|^)(` + logTimestampPattern + `)`) + +// logMessageStart matches the timestamp that starts a message in the middle of the buffer, with +// the end of line before it. +var logMessageStart = regexp.MustCompile(`\n` + logTimestampPattern) + +// StreamedLogOptions configures [RunClient.NewStreamedLog] and, through +// [ActorStartOptions.Log], the log redirection of [ActorClient.Call]. +type StreamedLogOptions struct { + // Disabled turns the log redirection off. It only has an effect in [ActorStartOptions.Log]. + Disabled bool + // Output receives every redirected message, one per line, prefixed with Prefix. Defaults to + // os.Stderr. Writes are serialized, so the writer need not be safe for concurrent use. + Output io.Writer + // Prefix is written in front of every message. Defaults to "{Actor name} runId:{run ID} -> ", + // which costs two requests (the run and its Actor); a non-nil Prefix (an empty one included) + // skips them. + Prefix *string + // FromStart controls whether messages logged before the redirection was created are + // redirected too (default true). With false, they are dropped. + FromStart *bool +} + +// StreamedLog redirects the live log of an Actor run to a writer, message by message. Obtain one +// from [RunClient.NewStreamedLog], begin the redirection with [StreamedLog.Start] and end it with +// [StreamedLog.Stop]. Matches the reference client's StreamedLog. +// +// Each message of the log, which starts with the timestamp the platform puts in front of it, is +// written on its own line, prefixed. Messages that were already logged when the redirection was +// created are skipped if FromStart is false. The log of a run that has not logged anything yet is +// reopened until it delivers its first bytes. +type StreamedLog struct { + log *LogClient + output io.Writer + prefix string + startedAt time.Time + fromStart bool + + mu sync.Mutex // serializes writes to output and guards the fields below + running bool + stopping bool + startCtx context.Context // the context of the Start call, which ends the redirection + stopCtx context.Context // the context of the Stop call, which bounds the final read + cancelStream context.CancelFunc + wake chan struct{} + done chan error + buffer bytes.Buffer + scanned int // how much of the buffer was searched for message timestamps +} + +// NewStreamedLog creates the redirection of this run's log, which does nothing until +// [StreamedLog.Start]. With the default prefix it fetches the run and its Actor to name them, and +// the bool reports whether the run exists: it is false (with a nil [StreamedLog]) for a run that +// does not. A non-nil [StreamedLogOptions.Prefix] skips the lookups, so the bool is then always +// true. +func (c *RunClient) NewStreamedLog(ctx context.Context, options StreamedLogOptions) (*StreamedLog, bool, error) { + prefix := "" + if options.Prefix != nil { + prefix = *options.Prefix + } else { + run, present, err := getResourceWithTimeout[ActorRun](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) + if err != nil || !present { + return nil, false, err + } + actorName := "" + if run.ActID != "" { + actorClient := c.root.Actor(run.ActID) + actor, found, err := getResourceWithTimeout[Actor](ctx, actorClient.ctx, "", NewQueryParams(), actorClient.ctx.longTimeout()) + if err != nil { + return nil, false, err + } + if found { + actorName = actor.Name + } + } + parts := []string{"runId:" + run.ID} + if actorName != "" { + parts = append([]string{actorName}, parts...) + } + prefix = strings.Join(parts, " ") + logRedirectPrefixSeparator + } + + output := options.Output + if output == nil { + output = os.Stderr + } + return &StreamedLog{ + log: c.Log(), + output: output, + prefix: prefix, + startedAt: time.Now(), + fromStart: options.FromStart == nil || *options.FromStart, + }, true, nil +} + +// Start begins the redirection in the background. The redirection also ends when ctx is done. +// It returns an error if the redirection is already running. +func (s *StreamedLog) Start(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.running { + return errors.New("StreamedLog: already started") + } + streamCtx, cancel := context.WithCancel(ctx) + s.running = true + s.stopping = false + s.startCtx = ctx + s.stopCtx = nil + s.cancelStream = cancel + s.wake = make(chan struct{}) + s.done = make(chan error, 1) + s.buffer.Reset() + s.scanned = 0 + go func() { + s.done <- s.streamLog(ctx, streamCtx) + }() + return nil +} + +// Stop ends the redirection. It gives the messages already in flight up to one second, then +// aborts the log stream; if the stream delivered nothing yet, it reads the whole log in one +// request instead, so a short run that finished before the stream connected is not lost. It +// returns the error that made the redirection stop early, if any. ctx bounds the wait for it, +// the final read of the whole log included: when ctx is done first, Stop gives up on the rest of +// the log and returns the error of ctx, unless the context given to Start ended first, which ends +// the redirection on purpose; when both end at about the same time, either result is possible. It +// returns an error if the redirection is not running or is already being stopped. +func (s *StreamedLog) Stop(ctx context.Context) error { + s.mu.Lock() + if !s.running { + s.mu.Unlock() + return errors.New("StreamedLog: not started") + } + if s.stopping { + s.mu.Unlock() + return errors.New("StreamedLog: already stopping") + } + s.stopping = true + s.stopCtx = ctx + close(s.wake) + cancel, done, startCtx := s.cancelStream, s.done, s.startCtx + s.mu.Unlock() + + timer := time.AfterFunc(stopGrace, cancel) + defer timer.Stop() + var err error + select { + case err = <-done: + case <-ctx.Done(): + select { + case err = <-done: // it finished as the context ended + default: + // The redirection is still running: cutting it off loses the rest of the log, which + // the caller must be able to tell from a complete one. + cancel() + if err = <-done; err == nil && startCtx.Err() == nil { + err = ctx.Err() + } + } + } + cancel() + // A read cut short by the context of this call reports a plain cancellation: name the cause. + if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, context.Canceled) { + err = ctxErr + } + + s.mu.Lock() + s.running = false + s.mu.Unlock() + return err +} + +// finalReadContext returns the context of the final read of the whole log: it ends with the +// context of the redirection and with the one of the Stop call. +func (s *StreamedLog) finalReadContext(parent context.Context) (context.Context, context.CancelFunc) { + s.mu.Lock() + stopCtx := s.stopCtx + s.mu.Unlock() + readCtx, cancel := context.WithCancel(parent) + stopWatching := context.AfterFunc(stopCtx, cancel) + return readCtx, func() { + stopWatching() + cancel() + } +} + +// logReadError wraps the failure of the final read of the whole log, which leaves the log +// unredirected. Unlike a stream that is cut on purpose, it is never swallowed. +type logReadError struct{ err error } + +func (e *logReadError) Error() string { return e.err.Error() } +func (e *logReadError) Unwrap() error { return e.err } + +// isContextError reports whether err is the cancellation or the deadline of a context. +func isContextError(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + +func (s *StreamedLog) isStopping() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.stopping +} + +// streamLog redirects the log until the stream ends for good. parent is the context of the whole +// redirection, and streamCtx the one of the log stream, which Stop cancels after its grace period. +// It returns nil when the redirection was ended on purpose. +func (s *StreamedLog) streamLog(parent, streamCtx context.Context) error { + err := s.redirect(parent, streamCtx) + var lost *logReadError + if errors.As(err, &lost) && (!isContextError(lost.err) || parent.Err() == nil) { + // The log was not redirected: the caller must hear of it, unless the cause is the end of + // the caller's own context, which ends the redirection on purpose. + return lost.err + } + if err != nil && streamCtx.Err() != nil && isContextError(err) { + return nil // ended on purpose: by Stop, or because the caller's context is done + } + return err +} + +func (s *StreamedLog) redirect(parent, streamCtx context.Context) error { + raw := true + options := LogOptions{Raw: &raw} + retry := emptyLogStreamRetry + gotBytes := false + + for !gotBytes { + if s.isStopping() { + // The stream's context ends with the grace period, which must not cut this read + // short: it ends with the context of the caller's Stop instead. + readCtx, cancelRead := s.finalReadContext(parent) + body, _, err := s.log.GetBufferWithOptions(readCtx, options) + cancelRead() + if err != nil { + return &logReadError{err: err} + } + s.consume(body) + break + } + stream, err := s.log.StreamWithOptions(streamCtx, options) + if err != nil { + if s.isStopping() && parent.Err() == nil { + continue // the grace period ended while connecting: read the log in one request + } + return err + } + if stream == nil { + return nil // the log does not exist + } + if s.isStopping() { + _ = stream.Close() + continue + } + var readErr error + gotBytes, readErr = s.consumeStream(stream) + _ = stream.Close() + if readErr != nil && streamCtx.Err() == nil { + s.flush() + return readErr + } + if !gotBytes { + select { + case <-time.After(retry): + case <-s.wake: + case <-streamCtx.Done(): + } + retry = min(retry*2, emptyLogStreamMaxRetry) + } + } + s.flush() + return nil +} + +// consumeStream redirects the messages of stream and reports whether it delivered any byte. It +// reads until the end of the stream or an error; Stop ends it by cancelling the stream after its +// grace period, so the messages in flight are not lost. +func (s *StreamedLog) consumeStream(stream io.Reader) (bool, error) { + chunk := make([]byte, logReadChunkBytes) + gotBytes := false + for { + n, err := stream.Read(chunk) + if n > 0 { + gotBytes = true + s.consume(chunk[:n]) + } + if err != nil { + if errors.Is(err, io.EOF) { + return gotBytes, nil + } + return gotBytes, err + } + } +} + +// consume adds data to the buffer and redirects every message that is known to be complete: the +// ones followed by the timestamp of the next message. The last message stays in the buffer, since +// its continuation lines (a stack trace, say) may arrive in a later read; [StreamedLog.flush] +// writes it when the log ends. Only the new data is searched for timestamps, so a large message +// is not scanned again on every read. +func (s *StreamedLog) consume(data []byte) { + from := max(s.scanned-logMarkerOverlapBytes, 0) // a timestamp may be split across reads + s.buffer.Write(data) + buffered := s.buffer.Bytes() + s.scanned = len(buffered) + found := logMessageStart.FindAllIndex(buffered[from:], -1) + if len(found) == 0 { + return + } + // The newest timestamp starts the message that stays; everything before it is complete. + last := from + found[len(found)-1][0] + if last == 0 { + return + } + complete := string(buffered[:last]) + s.buffer.Next(last) + s.scanned = s.buffer.Len() + s.redirectMessages(complete) +} + +// flush redirects whatever is left in the buffer when the log ends: the last message, or a last +// line that has no timestamp. +func (s *StreamedLog) flush() { + text := s.buffer.String() + s.buffer.Reset() + s.scanned = 0 + switch { + case strings.TrimSpace(text) == "": + case logMessageMarker.MatchString(text): + s.redirectMessages(text) + default: + s.write(strings.TrimSpace(text)) + } +} + +// redirectMessages splits text into messages at their timestamps and writes them. +func (s *StreamedLog) redirectMessages(text string) { + markers := logMessageMarker.FindAllStringSubmatchIndex(text, -1) + for i, marker := range markers { + contentEnd := len(text) + if i+1 < len(markers) { + contentEnd = markers[i+1][0] + } + timestamp := text[marker[2]:marker[3]] + if !s.fromStart { + if logged, err := time.Parse(time.RFC3339Nano, timestamp); err == nil && logged.Before(s.startedAt) { + continue + } + } + s.write(strings.TrimSpace(timestamp + text[marker[3]:contentEnd])) + } +} + +// write writes one prefixed message to the output. +func (s *StreamedLog) write(message string) { + s.mu.Lock() + defer s.mu.Unlock() + _, _ = fmt.Fprintf(s.output, "%s%s\n", s.prefix, message) +} + +// startLogRedirection starts redirecting the log of run, as [ActorClient.Call] does by default. +// A failure never fails the call: it is reported to the redirection's output and the call goes on +// without redirected logs. It returns nil when the redirection is disabled or could not start. +func startLogRedirection(ctx context.Context, run *RunClient, options *StreamedLogOptions) *StreamedLog { + settings := StreamedLogOptions{} + if options != nil { + settings = *options + } + if settings.Disabled { + return nil + } + streamed, found, err := run.NewStreamedLog(ctx, settings) + if err == nil && found { + err = streamed.Start(ctx) + } + if err != nil { + reportLogRedirectionError(settings.Output, err) + return nil + } + if !found { + return nil // the run is gone: there is no log to redirect + } + return streamed +} + +// stopLogRedirection ends the redirection started by [startLogRedirection], if there is one. +func stopLogRedirection(ctx context.Context, streamed *StreamedLog, options *StreamedLogOptions) { + if streamed == nil { + return + } + if err := streamed.Stop(ctx); err != nil { + var output io.Writer + if options != nil { + output = options.Output + } + reportLogRedirectionError(output, err) + } +} + +// reportLogRedirectionError tells the user that the log redirection ended early. +func reportLogRedirectionError(output io.Writer, err error) { + if output == nil { + output = os.Stderr + } + _, _ = fmt.Fprintf(output, "Log redirection stopped due to error: %v\n", err) +} diff --git a/streamed_log_test.go b/streamed_log_test.go new file mode 100644 index 0000000..851822f --- /dev/null +++ b/streamed_log_test.go @@ -0,0 +1,735 @@ +package apify + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" +) + +// syncBuffer is a bytes.Buffer that is safe to read while the redirection writes to it. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// logBackend serves a run's log: streamBodies[i] answers the i-th request for the live stream +// (the last one repeats), and fullLog answers the read of the whole log. Run and Actor lookups +// answer the fixed documents below. +type logBackend struct { + mu sync.Mutex + streamBodies []string + streamErr error + fullLog string + streamCalls int + fullLogCalls int + rawOnStream bool + streamChunks []string // when set, the stream serves these pieces, one per read + streamTailErr error // makes the stream fail with this error after its body + // blockFullLog makes the read of the whole log wait until its request is canceled. + blockFullLog bool + // holdStream keeps the live stream open after its body, until the request is canceled. + holdStream bool + // fullLogFailure, when set, makes the read of the whole log fail with this status after + // fullLogDelay. + fullLogFailure int + fullLogDelay time.Duration +} + +type failingReader struct { + data io.Reader + err error +} + +func (r *failingReader) Read(p []byte) (int, error) { + n, err := r.data.Read(p) + if errors.Is(err, io.EOF) { + return n, r.err + } + return n, err +} + +// holdingReader serves data, then blocks until ctx is done, like an open log stream of a run that +// is still running but logs nothing. +type holdingReader struct { + data io.Reader + ctx context.Context +} + +func (r *holdingReader) Read(p []byte) (int, error) { + n, err := r.data.Read(p) + if errors.Is(err, io.EOF) && n == 0 { + <-r.ctx.Done() + return 0, r.ctx.Err() + } + if errors.Is(err, io.EOF) { + return n, nil + } + return n, err +} + +// chunkReader serves each of its chunks in a read of its own, like a log stream whose data +// arrives in separate pieces. +type chunkReader struct{ chunks []string } + +func (r *chunkReader) Read(p []byte) (int, error) { + if len(r.chunks) == 0 { + return 0, io.EOF + } + n := copy(p, r.chunks[0]) + r.chunks[0] = r.chunks[0][n:] + if r.chunks[0] == "" { + r.chunks = r.chunks[1:] + } + return n, nil +} + +func (b *logBackend) fullLogReads() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.fullLogCalls +} + +func (b *logBackend) Do(req *http.Request) (*http.Response, error) { + b.mu.Lock() + defer b.mu.Unlock() + ok := func(body io.Reader) (*http.Response, error) { + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(body)}, nil + } + path := req.URL.Path + switch { + case strings.HasSuffix(path, "/log"): + if req.URL.Query().Get("stream") != "" { + b.rawOnStream = req.URL.Query().Get("raw") == "1" + if b.streamErr != nil { + return nil, b.streamErr + } + body := b.streamBodies[min(b.streamCalls, len(b.streamBodies)-1)] + b.streamCalls++ + var reader io.Reader = strings.NewReader(body) + if b.streamChunks != nil { + reader = &chunkReader{chunks: append([]string(nil), b.streamChunks...)} + } + if b.holdStream { + reader = &holdingReader{data: reader, ctx: req.Context()} + } + if b.streamTailErr != nil { + reader = &failingReader{data: reader, err: b.streamTailErr} + } + return ok(reader) + } + b.fullLogCalls++ + if b.blockFullLog { + b.mu.Unlock() + <-req.Context().Done() + b.mu.Lock() + return nil, req.Context().Err() + } + if b.fullLogFailure != 0 { + b.mu.Unlock() + time.Sleep(b.fullLogDelay) + b.mu.Lock() + return &http.Response{StatusCode: b.fullLogFailure, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"error":{"type":"forbidden","message":"no"}}`))}, nil + } + return ok(strings.NewReader(b.fullLog)) + case strings.HasSuffix(path, "/actors/actor1"): + return ok(strings.NewReader(`{"data":{"id":"actor1","name":"hello-world"}}`)) + case strings.Contains(path, "/actor-runs/"): + return ok(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"RUNNING"}}`)) + } + return nil, errors.New("unexpected request " + req.URL.String()) +} + +const sampleLog = "2026-10-30T10:00:00.000Z ACTOR: Starting\n" + + "2026-10-30T10:00:01.000Z first line\nsecond line of the same message\n" + + "2026-10-30T10:00:02.000Z last message\n" + +func TestStreamedLogRedirectsMessagesWithThePrefixOfTheActorAndTheRun(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + out := &syncBuffer{} + ctx := context.Background() + + streamed, found, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out}) + if err != nil || !found { + t.Fatalf("NewStreamedLog: %v, %v, %v", streamed, found, err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + + const prefix = "hello-world runId:run1 -> " + want := prefix + "2026-10-30T10:00:00.000Z ACTOR: Starting\n" + + prefix + "2026-10-30T10:00:01.000Z first line\nsecond line of the same message\n" + + prefix + "2026-10-30T10:00:02.000Z last message\n" + if got := out.String(); got != want { + t.Fatalf("redirected log:\n%q\nwant:\n%q", got, want) + } + if !backend.rawOnStream { + t.Fatal("the raw log must be streamed") + } +} + +func TestStreamedLogPrefixOverrideSkipsTheLookups(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + out := &syncBuffer{} + ctx := context.Background() + streamed, found, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil || !found { + t.Fatalf("NewStreamedLog: %v, %v, %v", streamed, found, err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(out.String(), "2026-10-30T10:00:00.000Z ACTOR: Starting\n") { + t.Fatalf("an empty prefix is used as it is: %q", out.String()) + } +} + +func TestStreamedLogDropsMessagesBeforeTheStartWhenNotFromStart(t *testing.T) { + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + log := "2026-10-30T10:00:00.000Z old\n" + future + " new\n" + backend := &logBackend{streamBodies: []string{log}} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(""), FromStart: Ptr(false)}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "new") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "old") { + t.Fatalf("a message logged before the redirection was created must be dropped: %q", out.String()) + } +} + +// The API serves the log of a run that has not logged anything as an empty stream that ends at +// once: the redirection reopens it until the first bytes arrive. +func TestStreamedLogReopensAnEmptyStream(t *testing.T) { + backend := &logBackend{streamBodies: []string{"", "", sampleLog}} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if backend.streamCalls != 3 { + t.Fatalf("expected the empty stream to be reopened twice, got %d stream requests", backend.streamCalls) + } +} + +// A run that finished before the stream delivered anything is not lost: stopping reads the whole +// log in one request. +func TestStreamedLogStoppedBeforeTheFirstBytesReadsTheWholeLog(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, fullLog: sampleLog} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + if err := streamed.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if backend.fullLogCalls != 1 { + t.Fatalf("expected one read of the whole log, got %d", backend.fullLogCalls) + } + if !strings.Contains(out.String(), "last message") || strings.Count(out.String(), "\n") < 4 { + t.Fatalf("the whole log must be redirected: %q", out.String()) + } +} + +func TestStreamedLogFlushesALastLineWithoutAnEndOfLine(t *testing.T) { + backend := &logBackend{streamBodies: []string{"2026-10-30T10:00:00.000Z one\n2026-10-30T10:00:01.000Z unterminated"}} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(">")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "unterminated") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if got := out.String(); got != ">2026-10-30T10:00:00.000Z one\n>2026-10-30T10:00:01.000Z unterminated\n" { + t.Fatalf("got %q", got) + } +} + +func TestStreamedLogStartAndStopMustBePaired(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Stop(ctx); err == nil { + t.Fatal("Stop before Start must fail") + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err == nil { + t.Fatal("a second Start must fail") + } + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } +} + +// Two goroutines may call Stop at once: one stops the redirection, the other gets an error +// instead of closing a channel twice. +func TestStreamedLogConcurrentStopDoesNotPanic(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, fullLog: sampleLog} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + const callers = 8 + results := make(chan error, callers) + for range callers { + go func() { results <- streamed.Stop(ctx) }() + } + succeeded := 0 + for range callers { + if <-results == nil { + succeeded++ + } + } + if succeeded != 1 { + t.Fatalf("exactly one Stop must succeed, %d did", succeeded) + } +} + +// The context of Stop bounds the final read of the whole log, not only the wait for the stream. +func TestStreamedLogStopContextBoundsTheReadOfTheWholeLog(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + stopCtx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + started := time.Now() + err = streamed.Stop(stopCtx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected the deadline error of the Stop context, got %v", err) + } + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Fatalf("Stop must return once its context is done, took %v", elapsed) + } +} + +// Stop with a context that is already done does not turn a log that was redirected completely +// into an error: nothing was cut short. +func TestStreamedLogStopWithADoneContextAfterACompleteLogIsNotAnError(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + out := &syncBuffer{} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + done, cancel := context.WithCancel(context.Background()) + cancel() + if err := streamed.Stop(done); err != nil { + t.Fatalf("a complete log is not an error, got %v", err) + } +} + +// A Call whose context ends while it waits does not report a failed redirection after a log that +// was redirected completely. +func TestCallWhoseContextEndsDoesNotReportAFalseRedirectionError(t *testing.T) { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, fullLog: sampleLog}, pollDelay: 300 * time.Millisecond} + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + _, err := testClient(backend, 0).Actor("actor1").Call(ctx, nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) + if err == nil { + t.Fatal("expected the call to end with the error of its context") + } + if strings.Contains(out.String(), "Log redirection stopped due to error") { + t.Fatalf("a complete log is not a redirection error: %q", out.String()) + } +} + +// A failed final read of the whole log is reported even when it fails after the grace period of +// Stop: the log is lost, and the caller must hear of it. +func TestStreamedLogStopReportsAFailedReadOfTheWholeLog(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, fullLogFailure: http.StatusForbidden, fullLogDelay: stopGrace + 500*time.Millisecond} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + err = streamed.Stop(context.Background()) + apiErr, ok := AsAPIError(err) + if !ok || apiErr.StatusCode != http.StatusForbidden { + t.Fatalf("expected the 403 of the final read, got %v", err) + } +} + +// Stop with a context that ends while the stream is still open and has delivered lines cuts the +// log off: it returns the error of its context, so a truncated log is not mistaken for a complete +// one. +func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} + out := &syncBuffer{} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := streamed.Stop(stopCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected the error of the Stop context for a log that was cut off, got %v", err) + } +} + +// A message whose continuation lines arrive in a later read is redirected whole, and so is one +// whose timestamp is split across reads. +func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, holdStream: true, streamChunks: []string{ + "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n", + " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:00:02.000", + "Z split\n", + }} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + // The stream stays open: the message before the split timestamp is redirected only if the + // timestamp is recognized across the two reads, not when the log ends. + waitFor(t, func() bool { return strings.Contains(out.String(), "next") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + want := "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:00:02.000Z split\n" + if got := out.String(); got != want { + t.Fatalf("redirected log:\n%q\nwant:\n%q", got, want) + } +} + +// Stop promises the messages in flight up to a second: a stream that is still delivering when +// Stop is called is read to its end, not just to the end of the current read. +func TestStreamedLogStopKeepsReadingTheMessagesInFlight(t *testing.T) { + const lines = 20000 + var log strings.Builder + for i := 0; i < lines; i++ { + fmt.Fprintf(&log, "2026-10-30T10:00:00.000Z message %d\n", i) + } + backend := &logBackend{streamBodies: []string{log.String()}} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "message 0\n") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if got := strings.Count(out.String(), "\n"); got != lines { + t.Fatalf("redirected %d of %d messages", got, lines) + } +} + +// Without a Stop context to cut it off, a live stream ends after the grace period: that is how a +// redirection ends normally, and it is not an error. +func TestStreamedLogStopOfALiveStreamAfterTheGracePeriodIsNotAnError(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} + out := &syncBuffer{} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + if err := streamed.Stop(context.Background()); err != nil { + t.Fatalf("a normal stop is not an error, got %v", err) + } +} + +// The context given to Start ends the redirection on purpose, so its end is not an error of Stop, +// whether it ends before Stop or during the final read of the whole log. +func TestStreamedLogStartContextEndingIsNotAnErrorOfStop(t *testing.T) { + t.Run("before Stop", func(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + parent, cancelParent := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + cancelParent() + time.Sleep(50 * time.Millisecond) + if err := streamed.Stop(context.Background()); err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) + t.Run("during the final read", func(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + parent, cancelParent := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + stopped := make(chan error, 1) + go func() { stopped <- streamed.Stop(context.Background()) }() + waitFor(t, func() bool { return backend.fullLogReads() == 1 }) + cancelParent() + if err := <-stopped; err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) +} + +// The end of the context given to Start ends the redirection on purpose: Stop with the same +// context, as Call does, reports no error for an open stream that is cut off by it. +func TestStreamedLogStopAfterTheStartContextEndedIsNotAnError(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} + out := &syncBuffer{} + parent, cancel := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) + cancel() + if err := streamed.Stop(parent); err != nil { + t.Fatalf("the end of the Start context is not an error, got %v", err) + } +} + +// A Call whose context ends while the log stream is still open, and has delivered lines, reports +// no redirection error. It is a smoke test of the whole path: whether the redirection is still +// running when the call stops it depends on timing, so the deterministic guard of this behavior is +// TestStreamedLogStopAfterTheStartContextEndedIsNotAnError. +func TestCallWhoseContextEndsWithAnOpenLogStreamReportsNoError(t *testing.T) { + for range 10 { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, holdStream: true}, pollDelay: time.Minute} + ctx, cancel := context.WithCancel(context.Background()) + go func() { + defer cancel() + deadline := time.Now().Add(10 * time.Second) + for !strings.Contains(out.String(), "first line") && time.Now().Before(deadline) { + time.Sleep(2 * time.Millisecond) + } + }() + _, err := testClient(backend, 0).Actor("actor1").Call(ctx, nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) + if err == nil { + t.Fatal("expected the call to end with the error of its context") + } + if strings.Contains(out.String(), "Log redirection stopped due to error") { + t.Fatalf("the end of the context of the call is not a redirection error: %q", out.String()) + } + } +} + +// Stop never returns a bare cancellation: the Start context ending just after the Stop context +// (as Call does) gives either the error of the Stop context or nil. +func TestStreamedLogStopNeverReturnsABareCancellation(t *testing.T) { + for range 40 { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + parent, cancelParent := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + stopCtx, cancelStop := context.WithTimeout(context.Background(), 20*time.Millisecond) + context.AfterFunc(stopCtx, cancelParent) + err = streamed.Stop(stopCtx) + cancelStop() + if err != nil && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected nil or the error of the Stop context, got %v", err) + } + } +} + +func TestNewStreamedLogOfAMissingRunIsNotFound(t *testing.T) { + backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} + streamed, found, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) + if err != nil || found || streamed != nil { + t.Fatalf("expected (nil, false, nil), got %v, %v, %v", streamed, found, err) + } +} + +// Call redirects the log of the run by default, and the call goes on when the redirection fails. +func TestCallRedirectsTheLogOfTheRun(t *testing.T) { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, fullLog: sampleLog}} + got, err := testClient(backend, 0).Actor("actor1").Call(context.Background(), nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out}}, Ptr(int64(0))) + if err != nil || got.Status != "SUCCEEDED" { + t.Fatalf("Call: %+v, %v", got, err) + } + if !strings.Contains(out.String(), "hello-world runId:run1 -> 2026-10-30T10:00:02.000Z last message") { + t.Fatalf("the log must be redirected with the prefix of the Actor and the run: %q", out.String()) + } +} + +func TestCallWithDisabledLogRedirectionRequestsNoLog(t *testing.T) { + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, fullLog: sampleLog}} + _, err := testClient(backend, 0).Actor("actor1").Call(context.Background(), nil, ActorStartOptions{Log: &StreamedLogOptions{Disabled: true}}, Ptr(int64(0))) + if err != nil { + t.Fatal(err) + } + if backend.streamCalls != 0 || backend.fullLogCalls != 0 { + t.Fatalf("no log may be requested, got %d stream and %d full reads", backend.streamCalls, backend.fullLogCalls) + } +} + +func TestACallGoesOnWhenTheLogRedirectionFails(t *testing.T) { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, streamErr: errors.New("boom")}, pollDelay: 100 * time.Millisecond} + got, err := testClient(backend, 0).Actor("actor1").Call(context.Background(), nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) + if err != nil || got.Status != "SUCCEEDED" { + t.Fatalf("a failing log redirection must not fail the call: %+v, %v", got, err) + } + if !strings.Contains(out.String(), "Log redirection stopped due to error") || !strings.Contains(out.String(), "boom") { + t.Fatalf("the failure must be reported: %q", out.String()) + } +} + +// callLogBackend also serves the start of a run, and answers every poll of it as finished after +// pollDelay, which leaves the log redirection time to run before the call stops it. +type callLogBackend struct { + logBackend + pollDelay time.Duration +} + +func (b *callLogBackend) Do(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/runs") { + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"RUNNING"}}`))}, nil + } + if strings.Contains(req.URL.Path, "/actor-runs/") && !strings.HasSuffix(req.URL.Path, "/log") && req.URL.Query().Get("waitForFinish") != "" { + select { + case <-time.After(b.pollDelay): + case <-req.Context().Done(): + return nil, req.Context().Err() + } + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"SUCCEEDED"}}`))}, nil + } + return b.logBackend.Do(req) +} + +func waitFor(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatal("timed out waiting for the condition") + } + time.Sleep(5 * time.Millisecond) + } +} + +// One large message must not be scanned again on every read: Stop gives the log only a second. +// The bound is loose, to allow for the race detector. +func TestStreamedLogRedirectsALargeMessageQuickly(t *testing.T) { + const line = "line 1\n" + const lines = 4 << 20 / len(line) + var log strings.Builder + log.WriteString("2026-10-30T10:00:00.000Z big\n") + for i := 0; i < lines; i++ { + log.WriteString(line) + } + log.WriteString("2026-10-30T10:00:01.000Z next\n") + out := &syncBuffer{} + streamed := &StreamedLog{output: out, fromStart: true} + body := []byte(log.String()) + + started := time.Now() + for len(body) > 0 { + n := min(logReadChunkBytes, len(body)) + streamed.consume(body[:n]) + body = body[n:] + } + streamed.flush() + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Fatalf("redirecting 4 MiB took %v", elapsed) + } + if got := strings.Count(out.String(), "\n"); got != lines+2 { + t.Fatalf("redirected %d lines, want %d", got, lines+2) + } +} diff --git a/task.go b/task.go index 2b72294..e31aabb 100644 --- a/task.go +++ b/task.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "time" ) // TaskClient is a client for a specific Actor task. @@ -99,23 +100,34 @@ func (o TaskStartOptions) apply(q *QueryParams) { // Start starts the task and returns immediately with the created run. input optionally // overrides the task's stored input (nil to use the stored input). +// +// A start request that times out is not retried: it may already have created the run, and a +// retry would then start a second, billed one. func (c *TaskClient) Start(ctx context.Context, input any, options TaskStartOptions) (ActorRun, error) { + return c.startWithTimeout(ctx, input, options, extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish)) +} + +// startWithTimeout is the shared implementation of Start and Call: it starts the task with the +// given timeout for each start request. +func (c *TaskClient) startWithTimeout(ctx context.Context, input any, options TaskStartOptions, timeout time.Duration) (ActorRun, error) { params := NewQueryParams() options.apply(params) body, err := marshalInput(input) if err != nil { return ActorRun{}, err } - timeout := extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) return startWaitingForResources(ctx, options.WaitForResources, func() (ActorRun, error) { - return postWithBody[ActorRun](ctx, c.ctx, "runs", params, body, contentTypeJSON, timeout) + return postWithBody[ActorRun](ctx, c.ctx.withoutTimeoutRetry(), "runs", params, body, contentTypeJSON, timeout) }) } // Call starts the task and waits (client-side polling) for it to finish. waitSecs bounds the // wait; nil waits indefinitely. +// +// Neither the start request nor the polls have a client-side timeout of their own (matching the +// reference client's call(), whose default timeout is noTimeout): ctx bounds the whole call. func (c *TaskClient) Call(ctx context.Context, input any, options TaskStartOptions, waitSecs *int64) (ActorRun, error) { - run, err := c.Start(ctx, input, options) + run, err := c.startWithTimeout(ctx, input, options, noRequestTimeout) if err != nil { return ActorRun{}, err } @@ -136,6 +148,9 @@ func (c *TaskClient) GetInput(ctx context.Context) (json.RawMessage, error) { // UpdateInput replaces the task's stored input and returns the updated input. func (c *TaskClient) UpdateInput(ctx context.Context, input any) (json.RawMessage, error) { + if c.ctx.idErr != nil { + return nil, c.ctx.idErr + } data, err := json.Marshal(input) if err != nil { return nil, err diff --git a/tests/actor_run_test.go b/tests/actor_run_test.go index 86c33d3..f8d748b 100644 --- a/tests/actor_run_test.go +++ b/tests/actor_run_test.go @@ -1,6 +1,8 @@ package apify_test import ( + "bytes" + "strings" "testing" apify "github.com/apify/apify-client-go" @@ -145,3 +147,229 @@ func TestLastRunAccess(t *testing.T) { t.Fatalf("expected last succeeded run filtered by origin, got %q", lastRunByOrigin.Status) } } + +// TestCallRedirectsTheRunLog runs hello-world through Call and checks that its log was redirected +// to the configured output, prefixed with the Actor name and the run ID. +func TestCallRedirectsTheRunLog(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var out bytes.Buffer + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{ + Log: &apify.StreamedLogOptions{Output: &out}, + }, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + if want := "hello-world runId:" + run.ID + " -> "; !strings.Contains(out.String(), want) { + t.Fatalf("expected the redirected log to carry the prefix %q, got %q", want, out.String()) + } +} + +// TestRunAndTaskStoragesAndLogCanBeRead covers the simple GETs of a finished run: the run-scoped +// default storages (dataset, key-value store, request queue), the log by run id, and the default +// build, webhook listings and last run of the Actor and of a task. +func TestRunAndTaskStoragesAndLogCanBeRead(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + actor := client.Actor("apify/hello-world") + run, err := actor.Call(ctx, nil, apify.ActorStartOptions{Log: &apify.StreamedLogOptions{Disabled: true}}, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + runClient := client.Run(run.ID) + + if logText, ok, err := client.Log(run.ID).Get(ctx); err != nil || !ok || logText == "" { + t.Fatalf("Log(%s).Get: ok=%v err=%v len=%d", run.ID, ok, err, len(logText)) + } + + if ds, ok, err := runClient.Dataset().Get(ctx); err != nil || !ok || ds.ID != run.DefaultDatasetID { + t.Fatalf("run dataset Get: %+v ok=%v err=%v", ds, ok, err) + } + if _, err := runClient.Dataset().GetStatistics(ctx); err != nil { + t.Fatalf("run dataset GetStatistics: %v", err) + } + if store, ok, err := runClient.KeyValueStore().Get(ctx); err != nil || !ok || store.ID != run.DefaultKeyValueStoreID { + t.Fatalf("run key-value store Get: %+v ok=%v err=%v", store, ok, err) + } + if _, err := runClient.KeyValueStore().ListKeys(ctx, apify.ListKeysOptions{}); err != nil { + t.Fatalf("run key-value store ListKeys: %v", err) + } + if queue, ok, err := runClient.RequestQueue().Get(ctx); err != nil || !ok || queue.ID != run.DefaultRequestQueueID { + t.Fatalf("run request queue Get: %+v ok=%v err=%v", queue, ok, err) + } + if _, err := runClient.RequestQueue().ListHead(ctx, nil); err != nil { + t.Fatalf("run request queue ListHead: %v", err) + } + if _, err := runClient.RequestQueue().ListRequests(ctx, apify.ListRequestsOptions{}); err != nil { + t.Fatalf("run request queue ListRequests: %v", err) + } + + if _, err := actor.DefaultBuild(ctx, nil); err != nil { + t.Fatalf("DefaultBuild: %v", err) + } + if _, err := actor.Webhooks().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}); err != nil { + t.Fatalf("actor webhooks List: %v", err) + } + + task, err := client.Tasks().Create(ctx, taskDef(uniqueName("task-last-run"))) + if err != nil { + t.Fatalf("create task: %v", err) + } + defer func() { _ = client.Task(task.ID).Delete(ctx) }() + taskRun, err := client.Task(task.ID).Call(ctx, nil, apify.TaskStartOptions{}, ptr(int64(120))) + if err != nil { + t.Fatalf("call task: %v", err) + } + if _, err := client.Task(task.ID).Webhooks().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}); err != nil { + t.Fatalf("task webhooks List: %v", err) + } + last, ok, err := client.Task(task.ID).LastRun("SUCCEEDED").Get(ctx) + if err != nil || !ok || last.ID != taskRun.ID { + t.Fatalf("task LastRun: %+v ok=%v err=%v", last, ok, err) + } + if logText, ok, err := client.Task(task.ID).LastRun("SUCCEEDED").Log().Get(ctx); err != nil || !ok || logText == "" { + t.Fatalf("task last run log: ok=%v err=%v len=%d", ok, err, len(logText)) + } + // The storages of a filtered last run are those of the run the filter resolves to. + if ds, ok, err := client.Task(task.ID).LastRun("SUCCEEDED").Dataset().Get(ctx); err != nil || !ok || ds.ID != taskRun.DefaultDatasetID { + t.Fatalf("task last run dataset: %+v ok=%v err=%v", ds, ok, err) + } +} + +// TestModelURLFieldsAreNormalized checks, against the real API, that the URL fields of a run and +// of its storages come back in their normalized form: an absolute URL with a path. +func TestModelURLFieldsAreNormalized(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{Log: &apify.StreamedLogOptions{Disabled: true}}, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + got, ok, err := client.Run(run.ID).Get(ctx) + if err != nil || !ok { + t.Fatalf("get run: ok=%v err=%v", ok, err) + } + for name, value := range map[string]string{"consoleUrl": got.ConsoleURL, "containerUrl": got.ContainerURL} { + if value == "" { + continue // the API does not return the field for this run + } + if !strings.HasPrefix(value, "https://") || strings.Count(value, "/") < 3 { + t.Errorf("%s is not a normalized absolute URL: %q", name, value) + } + } + if got.ConsoleURL == "" { + t.Error("the API is expected to return the consoleUrl of a run") + } + + ds, ok, err := client.Dataset(run.DefaultDatasetID).Get(ctx) + if err != nil || !ok { + t.Fatalf("get dataset: ok=%v err=%v", ok, err) + } + if !strings.HasPrefix(ds.ConsoleURL, "https://") || ds.ItemsPublicURL == "" { + t.Errorf("unexpected dataset URLs: %q, %q", ds.ConsoleURL, ds.ItemsPublicURL) + } + store, ok, err := client.KeyValueStore(run.DefaultKeyValueStoreID).Get(ctx) + if err != nil || !ok { + t.Fatalf("get key-value store: ok=%v err=%v", ok, err) + } + if !strings.HasPrefix(store.ConsoleURL, "https://") || store.KeysPublicURL == "" || store.RecordsPublicURL == "" { + t.Errorf("unexpected key-value store URLs: %+v", store) + } +} + +// TestIterateRunDatasetItems pushes items into the default dataset of a finished run, then reads +// them back through RunClient.IterateDatasetItems, which must also catch the rows past a lagging +// itemCount once the run is finished. +func TestIterateRunDatasetItems(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{Log: &apify.StreamedLogOptions{Disabled: true}}, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + runClient := client.Run(run.ID) + const total = 7 + items := make([]map[string]any, total) + for i := range items { + items[i] = map[string]any{"n": i} + } + if err := runClient.Dataset().PushItems(ctx, items); err != nil { + t.Fatalf("push items: %v", err) + } + + // hello-world pushes an item of its own, which has no "n" and is left out below. + type item struct { + N *int `json:"n"` + } + it := apify.IterateRunDatasetItems[item](runClient, apify.RunIterateDatasetItemsOptions{ + ChunkSize: ptr(int64(3)), + PollIntervalSecs: ptr(int64(1)), + }) + var got []int + for { + next, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate run dataset items: %v", err) + } + if next == nil { + break + } + if next.N != nil { + got = append(got, *next.N) + } + } + if len(got) != total { + t.Fatalf("expected the %d pushed items, got %v", total, got) + } + for i, n := range got { + if n != i { + t.Fatalf("items out of order: %v", got) + } + } + + // A Limit caps the rows scanned. + limited := runClient.IterateDatasetItems(apify.RunIterateDatasetItemsOptions{ + Items: apify.DatasetListItemsOptions{Limit: ptr(int64(3))}, + }) + count := 0 + for { + next, err := limited.Next(ctx) + if err != nil { + t.Fatalf("iterate with offset and limit: %v", err) + } + if next == nil { + break + } + count++ + } + if count != 3 { + t.Fatalf("expected 3 items with limit 3, got %d", count) + } +} + +// TestRunListDeploymentFilter lists the Actor Standby runs of a pool through the account-wide +// and Actor-scoped run collections: the filter is accepted (the result may well be empty). +func TestRunListDeploymentFilter(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + filter := apify.RunListOptions{ + Status: []string{"READY", "RUNNING"}, + Deployment: []string{"SINGLE_TENANT", "MULTI_TENANT_FREE_USERS"}, + } + if _, err := client.Runs().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}, filter); err != nil { + t.Fatalf("Runs().List with a deployment filter: %v", err) + } + if _, err := client.Actor("apify/hello-world").Runs().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}, filter); err != nil { + t.Fatalf("Actor.Runs().List with a deployment filter: %v", err) + } +} diff --git a/tests/actor_test.go b/tests/actor_test.go index 9d31c69..3ba4ba0 100644 --- a/tests/actor_test.go +++ b/tests/actor_test.go @@ -97,7 +97,7 @@ func TestActorCRUDFlow(t *testing.T) { if _, err := actor.Builds().List(ctx, apify.ListOptions{}); err != nil { t.Fatalf("builds list: %v", err) } - if _, err := actor.Versions().List(ctx, apify.ListOptions{}); err != nil { + if _, err := actor.Versions().List(ctx); err != nil { t.Fatalf("versions list: %v", err) } } @@ -131,7 +131,7 @@ func TestActorVersionCRUDFlow(t *testing.T) { if _, ok, err := actor.Version("0.1").Get(ctx); err != nil || !ok { t.Fatalf("get version: ok=%v err=%v", ok, err) } - if _, err := actor.Versions().List(ctx, apify.ListOptions{}); err != nil { + if _, err := actor.Versions().List(ctx); err != nil { t.Fatalf("list versions: %v", err) } if _, err := actor.Version("0.1").Update(ctx, map[string]any{"buildTag": "beta", "sourceType": "SOURCE_FILES", "sourceFiles": []any{}}); err != nil { diff --git a/tests/iterate_test.go b/tests/iterate_test.go index ede6fec..a24a93f 100644 --- a/tests/iterate_test.go +++ b/tests/iterate_test.go @@ -13,8 +13,8 @@ import ( // consistent, so a resource can be missing from the first listing right after creation; the // helper re-iterates a fresh iterator until it converges. const ( - iterFindAttempts = 8 - iterFindBackoff = time.Second + iterFindAttempts = 32 + iterFindBackoff = 3 * time.Second ) // iterateFindIDs repeatedly drains a fresh iterator (from makeIter) and checks that every id @@ -305,7 +305,7 @@ func TestIterateActorVersions(t *testing.T) { } iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.ActorVersion] { - return actor.Versions().Iterate(smallPageDesc(), ptr(int64(1))) + return actor.Versions().Iterate() }, func(v *apify.ActorVersion) string { return v.VersionNumber }, []string{"0.0", "0.1"}, 100) } diff --git a/tests/request_queue_test.go b/tests/request_queue_test.go index 4565735..9b0dbfc 100644 --- a/tests/request_queue_test.go +++ b/tests/request_queue_test.go @@ -1,6 +1,7 @@ package apify_test import ( + "encoding/json" "fmt" "testing" @@ -131,6 +132,74 @@ func TestRequestQueuePaginateMultiplePages(t *testing.T) { } } +// TestRequestQueuePaginateRequestsWithOptions validates the total cap, the filter and the cursor +// of the iterator over the requests. +func TestRequestQueuePaginateRequestsWithOptions(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + rq, err := client.RequestQueues().GetOrCreate(ctx, uniqueName("rq-page-opts")) + if err != nil { + t.Fatalf("get-or-create: %v", err) + } + defer func() { _ = client.RequestQueue(rq.ID).Delete(ctx) }() + queue := client.RequestQueue(rq.ID) + for i := 0; i < 5; i++ { + url := fmt.Sprintf("https://example.com/%d", i) + if _, err := queue.AddRequest(ctx, apify.RequestQueueRequest{URL: url, UniqueKey: url}, false); err != nil { + t.Fatalf("add request %d: %v", i, err) + } + } + + drain := func(it *apify.RequestQueueRequestsIterator) []string { + t.Helper() + var urls []string + for { + req, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate: %v", err) + } + if req == nil { + return urls + } + urls = append(urls, req.URL) + } + } + + // Limit caps the total across pages (3 of 5 with 2 per page), and Filter applies to each page. + pending := drain(queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ + Limit: ptr(int64(3)), Filter: []string{apify.RequestFilterPending}, + }, ptr(int64(2)))) + if len(pending) != 3 { + t.Fatalf("expected 3 pending requests, got %v", pending) + } + if locked := drain(queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ + Filter: []string{apify.RequestFilterLocked}, + }, nil)); len(locked) != 0 { + t.Fatalf("expected no locked requests, got %v", locked) + } + + // Cursor starts the iteration after the first page. + raw, err := queue.ListRequests(ctx, apify.ListRequestsOptions{Limit: ptr(int64(2))}) + if err != nil { + t.Fatalf("list requests: %v", err) + } + var first struct { + NextCursor string `json:"nextCursor"` + } + if err := json.Unmarshal(raw, &first); err != nil || first.NextCursor == "" { + t.Fatalf("expected a next cursor in %s (%v)", raw, err) + } + if rest := drain(queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{Cursor: &first.NextCursor}, nil)); len(rest) != 3 { + t.Fatalf("expected 3 requests after the cursor, got %v", rest) + } + + if _, err := queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{Filter: []string{"bogus"}}, nil).Next(ctx); err == nil { + t.Fatal("expected an invalid filter to be rejected") + } +} + // TestRequestQueueBatchAddRequests exercises the chunked batch-add path with more than the // 25-per-call API limit, verifying all requests are accepted across chunks. func TestRequestQueueBatchAddRequests(t *testing.T) { diff --git a/tests/store_test.go b/tests/store_test.go index d58fdf1..32ce870 100644 --- a/tests/store_test.go +++ b/tests/store_test.go @@ -44,3 +44,22 @@ func TestIterateStore(t *testing.T) { t.Fatalf("expected to iterate at least 12 store actors, got %d", count) } } + +// TestListStoreIncludeInputSchema asks the Store to add the compact input schema to every Actor: +// with it set, the API defaults the limit to 10 and accepts nothing above it. +func TestListStoreIncludeInputSchema(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + page, err := client.Store().List(ctx, apify.StoreListOptions{Limit: ptr(int64(3)), IncludeInputSchema: ptr(true)}) + if err != nil { + t.Fatalf("Store().List with includeInputSchema: %v", err) + } + if len(page.Items) == 0 { + t.Fatal("expected at least one Store Actor") + } + if len(page.Items) > 10 { + t.Fatalf("with includeInputSchema the API returns at most 10 Actors, got %d", len(page.Items)) + } +} diff --git a/tests/user_test.go b/tests/user_test.go index 26ce317..10b7815 100644 --- a/tests/user_test.go +++ b/tests/user_test.go @@ -1,6 +1,9 @@ package apify_test -import "testing" +import ( + "strings" + "testing" +) func TestGetOwnAccount(t *testing.T) { client := requireClient(t) @@ -66,3 +69,52 @@ func TestGetLimits(t *testing.T) { t.Fatal("expected a non-empty limits object") } } + +// TestGetLimitsAndUsageByUsername reads the limits and the monthly usage through the by-username +// client: the API serves them for your own account by ID or username as well as through "me". +func TestGetLimitsAndUsageByUsername(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + me, ok, err := client.Me().Get(ctx) + if err != nil || !ok || me.Username == "" { + t.Fatalf("Me().Get: ok=%v err=%v username=%q", ok, err, me.Username) + } + user := client.User(me.Username) + limits, err := user.Limits(ctx) + if err != nil { + t.Fatalf("User(%q).Limits: %v", me.Username, err) + } + if len(limits) == 0 { + t.Fatal("expected a non-empty limits object") + } + usage, err := user.MonthlyUsage(ctx) + if err != nil { + t.Fatalf("User(%q).MonthlyUsage: %v", me.Username, err) + } + if len(usage) == 0 { + t.Fatal("expected a non-empty monthly usage object") + } +} + +// TestUserProfileURLsAreNormalized checks that the profile of the own account is parsed and its +// URL fields, when set, are absolute URLs. +func TestUserProfileURLsAreNormalized(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + me, ok, err := client.Me().Get(ctx) + if err != nil || !ok { + t.Fatalf("Me().Get: ok=%v err=%v", ok, err) + } + if me.Profile == nil { + t.Fatal("expected the profile of the own account") + } + for name, value := range map[string]*string{"pictureUrl": me.Profile.PictureURL, "websiteUrl": me.Profile.WebsiteURL} { + if value != nil && !strings.Contains(*value, "://") { + t.Errorf("%s is not an absolute URL: %q", name, *value) + } + } +} diff --git a/timeout_retry_test.go b/timeout_retry_test.go new file mode 100644 index 0000000..6a427d7 --- /dev/null +++ b/timeout_retry_test.go @@ -0,0 +1,148 @@ +package apify + +import ( + "bytes" + "context" + "io" + "net/http" + "sync/atomic" + "testing" + "time" +) + +// slowBackend answers every request with a 200 and body after delay, unless the request's +// context ends first (then it fails with the context's error, as a timed-out transport does). It +// counts the requests it received and keeps the last URL. +type slowBackend struct { + delay time.Duration + body string + calls atomic.Int64 + url atomic.Value +} + +func (b *slowBackend) Do(req *http.Request) (*http.Response, error) { + b.calls.Add(1) + b.url.Store(req.URL.String()) + select { + case <-time.After(b.delay): + case <-req.Context().Done(): + return nil, req.Context().Err() + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader([]byte(b.body)))}, nil +} + +const ( + // slowResponseDelay is how long slowBackend takes to answer: longer than the tiny tier used in + // the tests below, so a request bound by that tier times out, but short enough to keep them fast. + slowResponseDelay = 400 * time.Millisecond + // tinyTier is the timeout tier of the test clients below. + tinyTier = 40 * time.Millisecond + // retriesAllowed is the number of retries of the test clients below, so a retried request + // shows up as extra calls. + retriesAllowed = 3 +) + +func slowClient(backend HTTPBackend, options ...Option) *ApifyClient { + base := []Option{ + WithToken("t"), + WithHTTPBackend(backend), + WithMaxRetries(retriesAllowed), + WithMinDelayBetweenRetries(time.Millisecond), + WithTimeoutShort(tinyTier), + WithTimeoutMedium(tinyTier), + } + return NewClient(append(base, options...)...) +} + +// A request that is not idempotent must not be retried after it timed out: it may already have +// taken effect on the server. +func TestNonIdempotentRequestsAreNotRetriedAfterATimeout(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"Actor Start", func(c *ApifyClient) error { + _, err := c.Actor("a").Start(ctx, map[string]any{}, ActorStartOptions{}) + return err + }}, + {"Actor StartRaw", func(c *ApifyClient) error { + _, err := c.Actor("a").StartRaw(ctx, []byte("x"), ActorStartOptions{}) + return err + }}, + {"Actor Build", func(c *ApifyClient) error { + _, err := c.Actor("a").Build(ctx, "0.1", ActorBuildOptions{}) + return err + }}, + {"Task Start", func(c *ApifyClient) error { + _, err := c.Task("t").Start(ctx, nil, TaskStartOptions{}) + return err + }}, + {"Dataset PushItems", func(c *ApifyClient) error { + return c.Dataset("d").PushItems(ctx, map[string]any{"a": 1}) + }}, + {"KeyValueStore SetRecordRawWithOptions", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordRawWithOptions(ctx, "k", []byte("v"), "text/plain", SetRecordOptions{DoNotRetryTimeouts: true}) + }}, + {"KeyValueStore SetRecordJSONWithOptions", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordJSONWithOptions(ctx, "k", map[string]any{"a": 1}, SetRecordOptions{DoNotRetryTimeouts: true}) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: `{"data":{}}`} + // The record upload and the dataset push run on tiers the tiny medium one covers; the + // upload uses the long tier, so shrink it too. + if err := c.run(slowClient(backend, WithTimeoutLong(tinyTier))); err == nil { + t.Fatal("expected a timeout error") + } + if got := backend.calls.Load(); got != 1 { + t.Fatalf("a timed-out request must not be retried, got %d attempts", got) + } + }) + } +} + +// The reverse of the test above: an idempotent request is retried after a timeout, and so is a +// record upload that did not ask for DoNotRetryTimeouts. +func TestIdempotentRequestsAreRetriedAfterATimeout(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"Actor Get", func(c *ApifyClient) error { + _, _, err := c.Actor("a").Get(ctx) + return err + }}, + {"KeyValueStore SetRecordRaw", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordRaw(ctx, "k", []byte("v"), "text/plain") + }}, + {"KeyValueStore SetRecordRawWithOptions", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordRawWithOptions(ctx, "k", []byte("v"), "text/plain", SetRecordOptions{}) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: `{"data":{}}`} + if err := c.run(slowClient(backend, WithTimeoutLong(tinyTier))); err == nil { + t.Fatal("expected a timeout error") + } + if got := backend.calls.Load(); got != retriesAllowed+1 { + t.Fatalf("expected %d attempts, got %d", retriesAllowed+1, got) + } + }) + } +} + +// A non-idempotent request still retries failures that are not timeouts (here a 500). +func TestNonIdempotentRequestsStillRetryServerErrors(t *testing.T) { + backend := &mockBackend{responses: constant(500, `{"error":{"type":"x","message":"m"}}`)} + client := testClient(backend, 2) + if err := client.Dataset("d").PushItems(context.Background(), map[string]any{"a": 1}); err == nil { + t.Fatal("expected an error") + } + if backend.calls != 3 { + t.Fatalf("expected 3 attempts, got %d", backend.calls) + } +} diff --git a/timeout_tier_test.go b/timeout_tier_test.go new file mode 100644 index 0000000..9fe4b6a --- /dev/null +++ b/timeout_tier_test.go @@ -0,0 +1,49 @@ +package apify + +import ( + "context" + "testing" +) + +// The calls the reference client puts on the long tier must not be cut off by a short tier: here +// the short tier (tinyTier) is shorter than the response time, and the calls still succeed. +func TestKeyValueStoreAndPublicURLCallsUseTheLongTier(t *testing.T) { + ctx := context.Background() + store := `{"data":{"id":"s","urlSigningSecretKey":"secret"}}` + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"KeyValueStore Update", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").Update(ctx, map[string]any{"name": "n"}) + return err + }}, + {"KeyValueStore RecordExists", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").RecordExists(ctx, "k") + return err + }}, + {"KeyValueStore GetRecordPublicURL", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").GetRecordPublicURL(ctx, "k") + return err + }}, + {"KeyValueStore CreateKeysPublicURL", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").CreateKeysPublicURL(ctx, nil) + return err + }}, + {"Dataset CreateItemsPublicURL", func(c *ApifyClient) error { + _, err := c.Dataset("d").CreateItemsPublicURL(ctx, DatasetListItemsOptions{}, nil) + return err + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: store} + if err := c.run(slowClient(backend)); err != nil { + t.Fatalf("a long-tier call must outlive the %v short and medium tiers: %v", tinyTier, err) + } + if got := backend.calls.Load(); got != 1 { + t.Fatalf("expected a single request, got %d", got) + } + }) + } +} diff --git a/url_normalization.go b/url_normalization.go new file mode 100644 index 0000000..8c2c21a --- /dev/null +++ b/url_normalization.go @@ -0,0 +1,667 @@ +package apify + +import ( + "errors" + "fmt" + "net/netip" + "regexp" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// The fields the OpenAPI specification declares as `format: uri` are returned in a normalized +// form, close to the WHATWG URL serialization that the JavaScript client returns: the scheme and +// host are lowercased, a non-ASCII host is converted to punycode, a numeric (IPv4) or IPv6 host is +// written in its canonical form, the default port is dropped, an empty path becomes "/", dot +// segments are resolved and characters that are unsafe in a URL are percent-encoded. It is built +// for the http(s), ws(s), ftp and file URLs the API returns. The host of any other scheme is kept +// as given. It does not apply the full Unicode (UTS #46) mapping to a non-ASCII host. + +// defaultPorts maps a special URL scheme to its default port, which normalization drops. +var defaultPorts = map[string]string{ + "http": "80", + "https": "443", + "ws": "80", + "wss": "443", + "ftp": "21", +} + +// urlSchemePattern matches the scheme at the start of an absolute URL. +var urlSchemePattern = regexp.MustCompile(`^([A-Za-z][A-Za-z0-9+.-]*):`) + +// maxPort is the largest valid TCP/UDP port number. +const maxPort = 65535 + +// Bytes that are percent-encoded in each URL component, in addition to every control character, +// space and non-ASCII byte (the WHATWG percent-encode sets). +const ( + unsafePathBytes = `"#<>?` + "`{}" + unsafeQueryBytes = `"#<>'` + unsafeOpaqueQueryByte = `"#<>` + unsafeFragmentBytes = `"<>` + "`" + unsafeUserinfoBytes = `"#<>?` + "`{}/:;=@[\\]^|" +) + +// forbiddenHostBytes are the ASCII characters a host may not contain (besides control characters +// and the space). +const forbiddenHostBytes = "#%/:<>?@[\\]^|" + +// ipv4PartCount is the number of parts of a dotted IPv4 address, and ipv4PartBits the bits of +// each; ipv6PieceCount is the number of 16-bit pieces of an IPv6 address. +const ( + ipv4PartCount = 4 + ipv4PartBits = 8 + ipv6PieceCount = 8 + maxIPv4Part = 1<= 0 { + rest, fragment, hasFragment = rest[:i], rest[i+1:], true + } + var query string + hasQuery := false + if i := strings.IndexByte(rest, '?'); i >= 0 { + rest, query, hasQuery = rest[:i], rest[i+1:], true + } + if special { + rest = strings.ReplaceAll(rest, `\`, "/") + } + + var out strings.Builder + out.WriteString(scheme) + out.WriteByte(':') + path := rest + hasAuthority := special + switch { + case special && !isFile: + // A special URL always has an authority, whatever number of slashes (even none) follows + // its scheme. + authority := strings.TrimLeft(rest, "/") + end := strings.IndexByte(authority, '/') + if end < 0 { + end = len(authority) + } + path = authority[end:] + host, err := normalizeAuthority(authority[:end], authorityRules{defaultPort: defaultPorts[scheme], special: true, allowPort: true, allowUserinfo: true}) + if err != nil { + return "", fmt.Errorf("invalid URL %q: %w", raw, err) + } + out.WriteString("//") + out.WriteString(host) + case isFile: + host := "" + path = rest + if authority, ok := strings.CutPrefix(rest, "//"); ok { + end := strings.IndexByte(authority, '/') + if end < 0 { + end = len(authority) + } + path = authority[end:] + if authority[:end] != "" { + var err error + if host, err = normalizeAuthority(authority[:end], authorityRules{special: true}); err != nil { + return "", fmt.Errorf("invalid URL %q: %w", raw, err) + } + } + if host == "localhost" { + host = "" + } + } + out.WriteString("//") + out.WriteString(host) + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + default: + if authority, ok := strings.CutPrefix(rest, "//"); ok { + hasAuthority = true + end := strings.IndexByte(authority, '/') + if end < 0 { + end = len(authority) + } + path = authority[end:] + host, err := normalizeAuthority(authority[:end], authorityRules{allowPort: true, allowUserinfo: true}) + if err != nil { + return "", fmt.Errorf("invalid URL %q: %w", raw, err) + } + out.WriteString("//") + out.WriteString(host) + } + } + + if special && path == "" { + path = "/" + } + opaquePath := !special && !strings.HasPrefix(path, "/") + if !opaquePath { + path = removeDotSegments(path) + if !hasAuthority && strings.HasPrefix(path, "//") { + // Without this marker the path would read as the start of an authority. + out.WriteString("/.") + } + out.WriteString(percentEncode(path, unsafePathBytes)) + } else { + out.WriteString(percentEncodeControls(path)) + } + if hasQuery { + unsafe := unsafeQueryBytes + if !special { + unsafe = unsafeOpaqueQueryByte + } + out.WriteByte('?') + out.WriteString(percentEncode(query, unsafe)) + } + if hasFragment { + out.WriteByte('#') + out.WriteString(percentEncode(fragment, unsafeFragmentBytes)) + } + return out.String(), nil +} + +// authorityRules says how the authority of a URL is normalized. +type authorityRules struct { + // defaultPort is the port that is dropped as the default one of the scheme. + defaultPort string + // special is true for the special schemes, whose host is a domain, IPv4 or IPv6 address. The + // host of any other scheme is opaque and kept as given. + special bool + // allowPort is false for a scheme whose URLs have no port (file), and allowUserinfo for one + // whose URLs have no credentials (file). + allowPort bool + allowUserinfo bool +} + +// normalizeAuthority normalizes the "userinfo@host:port" part of a URL. +func normalizeAuthority(authority string, rules authorityRules) (string, error) { + var userinfo string + if i := strings.LastIndexByte(authority, '@'); i >= 0 { + if !rules.allowUserinfo { + return "", errors.New("credentials are not allowed") + } + userinfo, authority = authority[:i], authority[i+1:] + } + host, port := authority, "" + hasPortSeparator := false + if strings.HasPrefix(authority, "[") { + end := strings.IndexByte(authority, ']') + if end < 0 { + return "", errors.New("unterminated IPv6 address") + } + tail := authority[end+1:] + if tail != "" && !strings.HasPrefix(tail, ":") { + return "", errors.New("unexpected text after the IPv6 address") + } + address, err := netip.ParseAddr(authority[1:end]) + if err != nil || !address.Is6() || address.Zone() != "" { + return "", fmt.Errorf("invalid IPv6 address %q", authority[1:end]) + } + host, port = serializeIPv6(address), strings.TrimPrefix(tail, ":") + hasPortSeparator = tail != "" + } else { + if i := strings.LastIndexByte(authority, ':'); i >= 0 { + host, port, hasPortSeparator = authority[:i], authority[i+1:], true + } + var err error + if host, err = normalizeHost(host, rules.special); err != nil { + return "", err + } + } + if hasPortSeparator && !rules.allowPort { + return "", errors.New("a port is not allowed") + } + if port != "" { + number, err := parsePort(port) + if err != nil { + return "", fmt.Errorf("invalid port %q", port) + } + port = strconv.Itoa(number) + if port == rules.defaultPort { + port = "" + } + } + + var out strings.Builder + username, password, _ := strings.Cut(userinfo, ":") + if username != "" || password != "" { + out.WriteString(percentEncode(username, unsafeUserinfoBytes)) + if password != "" { + out.WriteByte(':') + out.WriteString(percentEncode(password, unsafeUserinfoBytes)) + } + out.WriteByte('@') + } + out.WriteString(host) + if port != "" { + out.WriteByte(':') + out.WriteString(port) + } + return out.String(), nil +} + +// parsePort parses a port: ASCII digits only, at most maxPort. +func parsePort(port string) (int, error) { + for i := 0; i < len(port); i++ { + if port[i] < '0' || port[i] > '9' { + return 0, errors.New("not a number") + } + } + number, err := strconv.Atoi(port) + if err != nil || number > maxPort { + return 0, errors.New("out of range") + } + return number, nil +} + +// normalizeHost normalizes a host name: percent-decoded, lowercased, converted to punycode, and +// an IPv4 address (in any of its numeric notations) written as four decimal parts. An empty host +// is an error for a special scheme. With special false the host is opaque and kept as given. +func normalizeHost(host string, special bool) (string, error) { + if !special { + return percentEncode(host, ""), nil + } + decoded, err := percentDecode(host) + if err != nil { + return "", fmt.Errorf("invalid host %q: %w", host, err) + } + if decoded == "" { + return "", errors.New("empty host") + } + if strings.ContainsFunc(decoded, isForbiddenHostRune) { + return "", fmt.Errorf("invalid host %q", host) + } + ascii := hostToASCII(strings.ToLower(decoded)) + if endsInANumber(ascii) { + return parseIPv4(ascii) + } + return ascii, nil +} + +// isForbiddenHostRune reports whether r may not appear in a host. +func isForbiddenHostRune(r rune) bool { + return r <= ' ' || r == 0x7F || strings.ContainsRune(forbiddenHostBytes, r) +} + +// percentDecode decodes the percent escapes of s, which must make valid UTF-8. +func percentDecode(s string) (string, error) { + if !strings.Contains(s, "%") { + return s, nil + } + decoded := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + if s[i] != '%' { + decoded = append(decoded, s[i]) + continue + } + if i+3 > len(s) { + decoded = append(decoded, s[i]) + continue + } + value, err := strconv.ParseUint(s[i+1:i+3], 16, 8) + if err != nil { + decoded = append(decoded, s[i]) + continue + } + decoded = append(decoded, byte(value)) + i += 2 + } + if !utf8.Valid(decoded) { + return "", errors.New("not valid UTF-8") + } + return string(decoded), nil +} + +// removeDotSegments resolves the "." and ".." segments of an absolute path. +func removeDotSegments(path string) string { + if path == "" { + return path + } + segments := strings.Split(path, "/") + kept := make([]string, 0, len(segments)) + for i, segment := range segments { + last := i == len(segments)-1 + switch strings.ToLower(segment) { + case ".", "%2e": + if last { + kept = append(kept, "") + } + case "..", ".%2e", "%2e.", "%2e%2e": + if len(kept) > 1 { + kept = kept[:len(kept)-1] + } + if last { + kept = append(kept, "") + } + default: + kept = append(kept, segment) + } + } + return strings.Join(kept, "/") +} + +// percentEncode percent-encodes every byte of s that is a control character, a space, a +// non-ASCII byte or listed in unsafe. An existing percent escape is kept as it is. +func percentEncode(s, unsafe string) string { + const upperHex = "0123456789ABCDEF" + var out strings.Builder + for i := 0; i < len(s); i++ { + b := s[i] + if b <= ' ' || b >= 0x7F || strings.IndexByte(unsafe, b) >= 0 { + out.WriteByte('%') + out.WriteByte(upperHex[b>>4]) + out.WriteByte(upperHex[b&0x0F]) + continue + } + out.WriteByte(b) + } + return out.String() +} + +// percentEncodeControls percent-encodes only the control characters and the non-ASCII bytes of s, +// as is done for the opaque path of a URL such as mailto:. +func percentEncodeControls(s string) string { + const upperHex = "0123456789ABCDEF" + var out strings.Builder + for i := 0; i < len(s); i++ { + b := s[i] + if b < ' ' || b >= 0x7F { + out.WriteByte('%') + out.WriteByte(upperHex[b>>4]) + out.WriteByte(upperHex[b&0x0F]) + continue + } + out.WriteByte(b) + } + return out.String() +} + +// serializeIPv6 writes an IPv6 address as WHATWG does: lowercase hexadecimal pieces without +// leading zeros, the longest run of zero pieces (of at least two) compressed to "::", in brackets. +func serializeIPv6(address netip.Addr) string { + bytes := address.As16() + var pieces [ipv6PieceCount]uint16 + for i := range pieces { + pieces[i] = uint16(bytes[2*i])<<8 | uint16(bytes[2*i+1]) + } + compressStart, compressLength := -1, 0 + for i := 0; i < ipv6PieceCount; { + if pieces[i] != 0 { + i++ + continue + } + j := i + for j < ipv6PieceCount && pieces[j] == 0 { + j++ + } + if j-i > compressLength && j-i >= 2 { + compressStart, compressLength = i, j-i + } + i = j + } + var out strings.Builder + out.WriteByte('[') + for i := 0; i < ipv6PieceCount; i++ { + if i == compressStart { + if i == 0 { + out.WriteString("::") + } else { + out.WriteByte(':') + } + i += compressLength - 1 + continue + } + out.WriteString(strconv.FormatUint(uint64(pieces[i]), 16)) + if i != ipv6PieceCount-1 { + out.WriteByte(':') + } + } + out.WriteByte(']') + return out.String() +} + +// endsInANumber reports whether the last label of host is a number (decimal, 0x hexadecimal or +// 0 octal), which makes the whole host an IPv4 address. +func endsInANumber(host string) bool { + labels := strings.Split(host, ".") + if labels[len(labels)-1] == "" { + if len(labels) == 1 { + return false + } + labels = labels[:len(labels)-1] + } + last := labels[len(labels)-1] + if last != "" && strings.Trim(last, "0123456789") == "" { + return true + } + _, ok := parseIPv4Part(last) + return ok +} + +// parseIPv4Part parses one part of an IPv4 address in decimal, 0x hexadecimal or 0 octal. +func parseIPv4Part(part string) (uint64, bool) { + if part == "" { + return 0, false + } + base := 10 + digits := part + switch { + case len(part) >= 2 && (part[:2] == "0x" || part[:2] == "0X"): + base, digits = 16, part[2:] + case len(part) >= 2 && part[0] == '0': + base, digits = 8, part[1:] + } + if digits == "" { + return 0, true // "0x" is zero + } + value, err := strconv.ParseUint(digits, base, 64) + return value, err == nil +} + +// parseIPv4 parses an IPv4 host in any of its numeric notations and writes it as four decimal +// parts. A host that ends in a number but is not a valid address is an error. +func parseIPv4(host string) (string, error) { + parts := strings.Split(host, ".") + if parts[len(parts)-1] == "" && len(parts) > 1 { + parts = parts[:len(parts)-1] + } + if len(parts) > ipv4PartCount { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + numbers := make([]uint64, len(parts)) + for i, part := range parts { + value, ok := parseIPv4Part(part) + if !ok { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + numbers[i] = value + } + for _, number := range numbers[:len(numbers)-1] { + if number > maxIPv4Part { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + } + last := numbers[len(numbers)-1] + if last >= 1<<(ipv4PartBits*uint(ipv4PartCount-len(numbers)+1)) { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + address := last + for i, number := range numbers[:len(numbers)-1] { + address += number << (ipv4PartBits * uint(ipv4PartCount-1-i)) + } + octets := make([]string, ipv4PartCount) + for i := range octets { + octets[i] = strconv.FormatUint(address>>(ipv4PartBits*uint(ipv4PartCount-1-i))&maxIPv4Part, 10) + } + return strings.Join(octets, "."), nil +} + +// Punycode (RFC 3492) parameters. +const ( + punycodeBase = 36 + punycodeTMin = 1 + punycodeTMax = 26 + punycodeSkew = 38 + punycodeDamp = 700 + punycodeInitialBias = 72 + punycodeInitialN = 128 + punycodePrefix = "xn--" +) + +// hostToASCII converts every non-ASCII label of a lowercased host to its punycode form. +func hostToASCII(host string) string { + labels := strings.Split(host, ".") + for i, label := range labels { + if !isASCII(label) { + labels[i] = punycodePrefix + punycodeEncode(label) + } + } + return strings.Join(labels, ".") +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// punycodeEncode encodes a label as punycode (RFC 3492), without the "xn--" prefix. +func punycodeEncode(label string) string { + runes := []rune(label) + var out []byte + for _, r := range runes { + if r < punycodeInitialN { + out = append(out, byte(r)) + } + } + basicCount := len(out) + handled := basicCount + if basicCount > 0 { + out = append(out, '-') + } + n, delta, bias := rune(punycodeInitialN), 0, punycodeInitialBias + for handled < len(runes) { + next := rune(unicode.MaxRune + 1) + for _, r := range runes { + if r >= n && r < next { + next = r + } + } + delta += int(next-n) * (handled + 1) + n = next + for _, r := range runes { + if r < n { + delta++ + } + if r != n { + continue + } + q := delta + for k := punycodeBase; ; k += punycodeBase { + threshold := k - bias + if threshold < punycodeTMin { + threshold = punycodeTMin + } else if threshold > punycodeTMax { + threshold = punycodeTMax + } + if q < threshold { + break + } + out = append(out, punycodeDigit(threshold+(q-threshold)%(punycodeBase-threshold))) + q = (q - threshold) / (punycodeBase - threshold) + } + out = append(out, punycodeDigit(q)) + bias = adaptPunycodeBias(delta, handled+1, handled == basicCount) + delta = 0 + handled++ + } + delta++ + n++ + } + return string(out) +} + +// punycodeDigit maps 0..35 to "a".."z", "0".."9". +func punycodeDigit(d int) byte { + const letters = 26 + if d < letters { + return byte('a' + d) + } + return byte('0' + d - letters) +} + +// adaptPunycodeBias is the bias adaptation function of RFC 3492, section 6.1. +func adaptPunycodeBias(delta, points int, first bool) int { + if first { + delta /= punycodeDamp + } else { + delta /= 2 + } + delta += delta / points + k := 0 + for delta > ((punycodeBase-punycodeTMin)*punycodeTMax)/2 { + delta /= punycodeBase - punycodeTMin + k += punycodeBase + } + return k + (punycodeBase-punycodeTMin+1)*delta/(delta+punycodeSkew) +} + +// normalizeURLField normalizes the URL held in *field in place. An empty value (a missing or +// null field) is left as it is; any other value that is not an absolute URL is an error naming +// the field. +func normalizeURLField(name string, field *string) error { + if *field == "" { + return nil + } + normalized, err := normalizeURL(*field) + if err != nil { + return fmt.Errorf("field %q: %w", name, err) + } + *field = normalized + return nil +} + +// normalizeOptionalURLField is [normalizeURLField] for a nullable field held as a *string. +func normalizeOptionalURLField(name string, field **string) error { + if *field == nil { + return nil + } + return normalizeURLField(name, *field) +} + +// normalizeURLMap normalizes every value of a map of URLs in place. +func normalizeURLMap(name string, urls map[string]string) error { + for key, value := range urls { + if err := normalizeURLField(name+"."+key, &value); err != nil { + return err + } + urls[key] = value + } + return nil +} diff --git a/url_normalization_test.go b/url_normalization_test.go new file mode 100644 index 0000000..3ee5f5b --- /dev/null +++ b/url_normalization_test.go @@ -0,0 +1,182 @@ +package apify + +import ( + "encoding/json" + "testing" +) + +func TestNormalizeURL(t *testing.T) { + cases := []struct{ raw, want string }{ + // The scheme and host are lowercased and the default port dropped. + {"HTTPS://Run-1.Example.COM:443", "https://run-1.example.com/"}, + {"http://example.com:80/a", "http://example.com/a"}, + {"https://example.com:8443/a", "https://example.com:8443/a"}, + // An empty path becomes "/", an existing path is kept. + {"https://example.com", "https://example.com/"}, + {"https://example.com/v2/datasets/abc/items?format=json", "https://example.com/v2/datasets/abc/items?format=json"}, + {"https://example.com?q=1", "https://example.com/?q=1"}, + // A non-ASCII host becomes punycode. + {"https://bücher.example/", "https://xn--bcher-kva.example/"}, + // Unsafe characters are percent-encoded; existing escapes are kept. + {"https://example.com/a b?q=1 2#f g", "https://example.com/a%20b?q=1%202#f%20g"}, + {"https://example.com/a%20b/é", "https://example.com/a%20b/%C3%A9"}, + {"https://example.com/a?x='\"'", "https://example.com/a?x=%27%22%27"}, + // Dot segments are resolved. + {"https://example.com/a/b/../c/./d", "https://example.com/a/c/d"}, + {"https://example.com/a/..", "https://example.com/"}, + // Userinfo and IPv6 hosts survive. + {"https://user:pa ss@Example.com/", "https://user:pa%20ss@example.com/"}, + {"http://[::1]:80/x", "http://[::1]/x"}, + // The host of a URL that is not special is kept as given; its path is still resolved. + {"mailto:Someone@Example.com", "mailto:Someone@Example.com"}, + {"mailto:a b", "mailto:a b"}, + {"foo://Example.COM/A", "foo://Example.COM/A"}, + {"foo://host/a/../b", "foo://host/b"}, + {"foo:/a/./b", "foo:/a/b"}, + {"foo:/.//p", "foo:/.//p"}, + {"foo:/a/..//p", "foo:/.//p"}, + {"foo://h/.//p", "foo://h//p"}, + // An empty password, and an empty userinfo, are dropped. + {"https://u:@ex.com/", "https://u@ex.com/"}, + {"https://:@ex.com/", "https://ex.com/"}, + {"https://:p@ex.com/", "https://:p@ex.com/"}, + // A special URL has an authority, whatever the number of slashes after its scheme. + {"https:foo", "https://foo/"}, + {"https:///foo/a", "https://foo/a"}, + {"https:\\\\foo\\a", "https://foo/a"}, + // The host is percent-decoded, and numeric hosts are written canonically. + {"https://ex%41mple.com/", "https://example.com/"}, + {"http://0x7f.1/", "http://127.0.0.1/"}, + {"http://127.1/", "http://127.0.0.1/"}, + {"http://2130706433/", "http://127.0.0.1/"}, + {"http://[2001:db8:0:0:0:0:0:1]/", "http://[2001:db8::1]/"}, + {"http://[0:0:0:0:0:0:0:0]/", "http://[::]/"}, + {"http://[2001:db8:0:1:0:0:0:1]:8080/", "http://[2001:db8:0:1::1]:8080/"}, + // file URLs. + {"file:///tmp/a", "file:///tmp/a"}, + {"file://localhost/tmp/a", "file:///tmp/a"}, + {"file:tmp/a", "file:///tmp/a"}, + // Tabs and line breaks are removed. + {"https://exa\tmple.com/pa\nth", "https://example.com/path"}, + // More punycode. + {"https://m\u00fcnchen.example/", "https://xn--mnchen-3ya.example/"}, + {"https://\u65e5\u672c\u8a9e.jp/", "https://xn--wgv71a119e.jp/"}, + {"https://\u30c9\u30e1\u30a4\u30f3\u540d\u4f8b.example/", "https://xn--eckwd4c7cu47r2wf.example/"}, + // Surrounding whitespace is dropped, a backslash is a slash. + {" https://example.com\\a ", "https://example.com/a"}, + } + for _, c := range cases { + got, err := normalizeURL(c.raw) + if err != nil { + t.Errorf("normalizeURL(%q) failed: %v", c.raw, err) + continue + } + if got != c.want { + t.Errorf("normalizeURL(%q) = %q, want %q", c.raw, got, c.want) + } + again, err := normalizeURL(got) + if err != nil || again != got { + t.Errorf("normalizeURL is not idempotent for %q: got %q, %v", got, again, err) + } + } +} + +func TestNormalizeURLRejectsInvalidURLs(t *testing.T) { + for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "http://1.2.3.4.5/", "http://256.1.1.1/", "http://foo.123/", "https://%ff.example/", "http://ex.com:+80/", "http://ex.com:-0/", "http://[fe80::1%25eth0]/", "file://host:80/x", "file://host:/x", "file://u@h/x", "file://u:p@h/x"} { + if got, err := normalizeURL(raw); err == nil { + t.Errorf("normalizeURL(%q) = %q, want an error", raw, got) + } + } +} + +func TestEveryURLFieldOfTheModelsIsNormalized(t *testing.T) { + const raw = "HTTP://Example.COM:80" + const want = "http://example.com/" + + var run ActorRun + mustUnmarshal(t, `{"id":"r","containerUrl":"`+raw+`","consoleUrl":"`+raw+`","output":{"results":"`+raw+`"}}`, &run) + if run.ContainerURL != want || run.ConsoleURL != want || run.Output["results"] != want { + t.Errorf("run URLs not normalized: %+v", run) + } + + var task Task + mustUnmarshal(t, `{"id":"t","standbyUrl":"`+raw+`"}`, &task) + if task.StandbyURL == nil || *task.StandbyURL != want { + t.Errorf("task standbyUrl not normalized: %v", task.StandbyURL) + } + var noStandby Task + mustUnmarshal(t, `{"id":"t","standbyUrl":null}`, &noStandby) + if noStandby.StandbyURL != nil { + t.Errorf("a null standbyUrl must stay nil, got %q", *noStandby.StandbyURL) + } + + var dataset Dataset + mustUnmarshal(t, `{"id":"d","consoleUrl":"`+raw+`","itemsPublicUrl":"`+raw+`"}`, &dataset) + if dataset.ConsoleURL != want || dataset.ItemsPublicURL != want { + t.Errorf("dataset URLs not normalized: %+v", dataset) + } + + var store KeyValueStore + mustUnmarshal(t, `{"id":"s","consoleUrl":"`+raw+`","keysPublicUrl":"`+raw+`","recordsPublicUrl":"`+raw+`"}`, &store) + if store.ConsoleURL != want || store.KeysPublicURL != want || store.RecordsPublicURL != want { + t.Errorf("store URLs not normalized: %+v", store) + } + + var key KeyValueStoreKey + mustUnmarshal(t, `{"key":"k","size":1,"recordPublicUrl":"`+raw+`"}`, &key) + if key.RecordPublicURL != want { + t.Errorf("key recordPublicUrl not normalized: %q", key.RecordPublicURL) + } + + var queue RequestQueue + mustUnmarshal(t, `{"id":"q","consoleUrl":"`+raw+`"}`, &queue) + if queue.ConsoleURL != want { + t.Errorf("queue consoleUrl not normalized: %q", queue.ConsoleURL) + } + + var webhook Webhook + mustUnmarshal(t, `{"id":"w","requestUrl":"`+raw+`"}`, &webhook) + if webhook.RequestURL != want { + t.Errorf("webhook requestUrl not normalized: %q", webhook.RequestURL) + } + var noURL Webhook + mustUnmarshal(t, `{"id":"w","requestUrl":null}`, &noURL) + if noURL.RequestURL != "" { + t.Errorf("a null requestUrl must stay empty, got %q", noURL.RequestURL) + } + + var dispatch WebhookDispatch + mustUnmarshal(t, `{"id":"x","webhookId":"w","webhook":{"_id":"w","actionType":"HTTP","isAdHoc":false,"requestUrl":"`+raw+`","slackMessage":{}}}`, &dispatch) + if dispatch.Webhook == nil || dispatch.Webhook.RequestURL != want || dispatch.Webhook.ID != "w" { + t.Errorf("dispatch webhook summary not parsed and normalized: %+v", dispatch.Webhook) + } + if _, ok := dispatch.Webhook.Extra["slackMessage"]; !ok { + t.Errorf("an unmodelled field of the summary must land in Extra: %v", dispatch.Webhook.Extra) + } + + var item ActorStoreListItem + mustUnmarshal(t, `{"id":"a","url":"`+raw+`","userPictureUrl":"`+raw+`"}`, &item) + if item.URL == nil || *item.URL != want || item.UserPictureURL == nil || *item.UserPictureURL != want { + t.Errorf("store item URLs not normalized: %+v", item) + } + + var user User + mustUnmarshal(t, `{"id":"u","username":"me","profile":{"name":"N","pictureUrl":"`+raw+`","websiteUrl":null}}`, &user) + if user.Profile == nil || user.Profile.PictureURL == nil || *user.Profile.PictureURL != want || user.Profile.WebsiteURL != nil { + t.Errorf("user profile URLs not normalized: %+v", user.Profile) + } +} + +func TestAnInvalidURLFieldFailsTheResponse(t *testing.T) { + var run ActorRun + if err := json.Unmarshal([]byte(`{"id":"r","consoleUrl":"not a url"}`), &run); err == nil { + t.Fatal("expected an error for an invalid URL field") + } +} + +func mustUnmarshal(t *testing.T, data string, v any) { + t.Helper() + if err := json.Unmarshal([]byte(data), v); err != nil { + t.Fatalf("unmarshal %s: %v", data, err) + } +} diff --git a/user.go b/user.go index 6f1d5e2..4b60eb4 100644 --- a/user.go +++ b/user.go @@ -3,37 +3,29 @@ package apify import ( "context" "encoding/json" - "errors" "net/http" ) // UserClient is a client for accessing user data (/v2/users/{userId} or /v2/users/me). // -// For the current user ("me"), it also exposes account usage and limits. Those endpoints -// only exist for "me" and return an error if called on another user's client. +// It also exposes account usage and limits. For another user than the current one, the API only +// serves them for your own account (by ID or username) or for the organization your API token +// belongs to, and rejects tokens with limited permissions. type UserClient struct { - ctx *resourceContext - isMe bool + ctx *resourceContext } func newUserClient(hc *httpClient, baseURL, id string) *UserClient { - return &UserClient{ - ctx: newSingleContext(hc, baseURL, "users", id), - isMe: id == meUserPlaceholder, - } + return &UserClient{ctx: newSingleContext(hc, baseURL, "users", id)} } -// errNotMe is returned by the me-only methods when called on another user's client. -var errNotMe = errors.New("this operation is only available for the current user (use Me())") - // Get fetches the user. For "me" it returns private account details; for other users it // returns the public profile. The bool reports whether the user exists. func (c *UserClient) Get(ctx context.Context) (User, bool, error) { return getResource[User](ctx, c.ctx, "", NewQueryParams()) } -// MonthlyUsage fetches the current account's monthly usage for the current month. Only -// available for "me". +// MonthlyUsage fetches the account's monthly usage for the current month. // // It returns the raw JSON usage report from the API (a JSON object with the account's usage // breakdown and totals for the period). @@ -49,11 +41,8 @@ type MonthlyUsageOptions struct { } // MonthlyUsageWithOptions is like [UserClient.MonthlyUsage] but can select another period -// with [MonthlyUsageOptions.Date]. Only available for "me". +// with [MonthlyUsageOptions.Date]. func (c *UserClient) MonthlyUsageWithOptions(ctx context.Context, options MonthlyUsageOptions) (json.RawMessage, error) { - if !c.isMe { - return nil, errNotMe - } params := NewQueryParams() if options.Date != "" { params.AddString("date", &options.Date) @@ -61,18 +50,15 @@ func (c *UserClient) MonthlyUsageWithOptions(ctx context.Context, options Monthl return getResourceRequired[json.RawMessage](ctx, c.ctx, "usage/monthly", params, c.ctx.shortTimeout()) } -// Limits fetches the current account's resource limits. Only available for "me". +// Limits fetches the account's resource limits. func (c *UserClient) Limits(ctx context.Context) (json.RawMessage, error) { - if !c.isMe { - return nil, errNotMe - } return getResourceRequired[json.RawMessage](ctx, c.ctx, "limits", NewQueryParams(), c.ctx.shortTimeout()) } -// UpdateLimits updates the current account's resource limits. Only available for "me". +// UpdateLimits updates the account's resource limits. func (c *UserClient) UpdateLimits(ctx context.Context, newLimits any) error { - if !c.isMe { - return errNotMe + if c.ctx.idErr != nil { + return c.ctx.idErr } data, err := json.Marshal(newLimits) if err != nil { diff --git a/version.go b/version.go index bb1dd5b..eeaa11e 100644 --- a/version.go +++ b/version.go @@ -4,10 +4,10 @@ package apify // // It follows Semantic Versioning (https://semver.org/). Changes to the public // interface (other than additive ones) are considered breaking changes. -const ClientVersion = "0.13.1" +const ClientVersion = "0.14.0" // APISpecVersion is the version of the Apify OpenAPI specification that this // client was generated and verified against. // // It corresponds to the `info.version` field of the Apify OpenAPI document. -const APISpecVersion = "v2-2026-10-26T112451Z" +const APISpecVersion = "v2-2026-10-29T100357Z"