From 348853c156635cc743308e24a54971254a726720 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 12:19:38 +0000 Subject: [PATCH 01/12] feat: sync Go client with Apify OpenAPI spec v2-2026-10-29T100357Z and port the apify-client-js v3 changes --- CHANGELOG.md | 53 +++++ README.md | 40 +++- actor.go | 61 ++++-- base.go | 42 ++-- dataset.go | 54 ++++- docs/README.md | 2 +- docs/actors.md | 3 +- docs/misc.md | 19 +- docs/runs.md | 84 +++++++- docs/storages.md | 16 +- docs/tasks.md | 3 +- docs/webhooks.md | 3 +- examples/log_redirection/main.go | 32 +-- go.mod | 9 +- go.sum | 4 + http_client.go | 28 ++- key_value_store.go | 58 ++++- models.go | 142 +++++++++++- review_fixes_test.go | 22 +- run.go | 22 +- run_collection.go | 11 +- run_dataset_items.go | 278 ++++++++++++++++++++++++ run_dataset_items_test.go | 233 ++++++++++++++++++++ spec_sync_test.go | 169 +++++++++++++++ store_collection.go | 7 +- streamed_log.go | 358 +++++++++++++++++++++++++++++++ streamed_log_test.go | 325 ++++++++++++++++++++++++++++ task.go | 21 +- tests/actor_run_test.go | 228 ++++++++++++++++++++ tests/iterate_test.go | 4 +- tests/store_test.go | 19 ++ tests/user_test.go | 54 ++++- timeout_retry_test.go | 339 +++++++++++++++++++++++++++++ url_normalization.go | 273 +++++++++++++++++++++++ url_normalization_test.go | 149 +++++++++++++ user.go | 36 +--- version.go | 4 +- 37 files changed, 3067 insertions(+), 138 deletions(-) create mode 100644 run_dataset_items.go create mode 100644 run_dataset_items_test.go create mode 100644 spec_sync_test.go create mode 100644 streamed_log.go create mode 100644 streamed_log_test.go create mode 100644 timeout_retry_test.go create mode 100644 url_normalization.go create mode 100644 url_normalization_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e8a8f7c..fa20827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,59 @@ All notable changes to the Apify Go client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.14.0] - 2026-10-30 + +### Added + +- `RunListOptions.Deployment` filters the account-wide and Actor-scoped run lists by the Actor + Standby pool that serves a run; `StoreListOptions.IncludeInputSchema` adds each Actor's compact + input schema to the Store listing. +- `UserClient.Limits`, `UpdateLimits` and `MonthlyUsage*` work on `client.User(id)` too (the API + serves them for your own account or your organization by ID or username); they no longer fail + on a client that is not `Me()`. +- `StreamedLog` (`RunClient.NewStreamedLog`, `Start`, `Stop`, `StreamedLogOptions`) redirects a + run's log message by message to a writer with a prefix. `ActorClient.Call` and `CallRaw` use it + by default, writing to `os.Stderr` with the prefix `"{Actor name} runId:{run ID} -> "`; + `ActorStartOptions.Log` changes the output or turns it off (`Disabled`). +- `RunClient.IterateDatasetItems` and `IterateRunDatasetItems[T]` iterate over the items of a + run's default dataset while the run is still producing them. +- `KeyValueStoreClient.SetRecordRawWithOptions` and `SetRecordJSONWithOptions` with + `SetRecordOptions` (`ContentEncoding`, `DoNotRetryTimeouts`). +- Typed, normalized URL fields on the models: `ActorRun.ConsoleURL`/`Output`, `Dataset.ConsoleURL`/ + `ItemsPublicURL`, `KeyValueStore.ConsoleURL`/`KeysPublicURL`/`RecordsPublicURL`, + `KeyValueStoreKey.RecordPublicURL`, `RequestQueue.ConsoleURL`, `Task.StandbyURL`, + `ActorStoreListItem.URL`/`UserPictureURL`, `User.Profile` (`UserProfile`) and + `WebhookDispatch.Webhook` (`WebhookDispatchWebhookSummary`). The URL fields the API specification + declares (these, `ActorRun.ContainerURL` and `Webhook.RequestURL`) are returned normalized: the + scheme and host are lowercased, a non-ASCII host becomes punycode, a default port is dropped, an + empty path becomes `/`, dot segments are resolved and unsafe characters are percent-encoded. A + value that is not an absolute URL fails decoding the response. + +### Changed + +- Bumped `APISpecVersion` to `v2-2026-10-29T100357Z` and `ClientVersion` to `0.14.0`. +- `ActorClient.Call`/`CallRaw` and `TaskClient.Call` start the run without a client-side timeout, + so a slow start is no longer aborted and repeated, which started a second run. +- A request that timed out is no longer retried when it is not idempotent: `Start`, `StartRaw` + and `Build` of an Actor, a task's `Start`, and `PushItems`. +- `KeyValueStoreClient.Update`, `RecordExists`, `GetRecordPublicURL`, `CreateKeysPublicURL` and + `DatasetClient.CreateItemsPublicURL*` use the "long" timeout tier, like the reference client. +- The storages and the log of a run reached through `LastRun`/`LastRunWithOptions`, and their + public URLs, carry its `status` and `origin` filters, so they address the run the filters + resolve to instead of the newest run of any status. +- Added `golang.org/x/net` as a dependency, for the punycode conversion of URL hosts. +- Documented that `GetRecord` returns the raw bytes without parsing them and that `SetRecordRaw` + sends them as they are, whatever the content type. + +### Fixed + +- `DatasetClient.PushItems` rejects, before sending, a value that is not a JSON object or an array + of objects, the only shapes the API accepts. A plain Go string, number, `nil` or a slice of + strings used to be sent and rejected by the API. +- `DatasetClient.ListItems`, `ActorClient.DefaultBuild` and `ValidateInputForBuild`, + `TaskClient.UpdateInput`, `UserClient.UpdateLimits` and `RunClient.Charge` now fail on an empty or + dot-segment ID like every other method, instead of sending a malformed request. + ## [0.13.1] - 2026-10-26 ### Changed diff --git a/README.md b/README.md index 40190dd..1f5d9bc 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,13 @@ request queues, tasks, schedules, webhooks, the store, users and logs). - Transparent authentication, `User-Agent` header, retries with exponential backoff, and per-request timeouts applied to every call. - Replaceable HTTP transport (the `HTTPBackend` interface) with a default implementation. -- Convenience helpers: start-and-wait, build/run polling, lazy store and request-queue - iterators, dataset export, signed public URLs, request-queue locking, and more. +- Convenience helpers: start-and-wait with live log redirection, build/run polling, lazy store + and request-queue iterators, live iteration of a running Actor's dataset items, dataset + export, signed public URLs, request-queue locking, and more. - Forward-compatible models that keep unknown API fields in an `Extra` map. -- A single third-party dependency (`github.com/andybalholm/brotli`, used for Brotli - request-body compression); everything else is the Go standard library. +- Two third-party dependencies: `github.com/andybalholm/brotli` (Brotli request-body + compression) and `golang.org/x/net/idna` (punycode conversion of hosts when normalizing URL + fields); everything else is the Go standard library. ## Installation @@ -126,8 +128,9 @@ func main() { Every request the client sends is assigned one of three timeout tiers, matching the kind of operation its method performs — `short` for a metadata read/write, `medium` for a listing, batch or trigger call, `long` for a download, upload or streaming call — except the polling -requests behind `WaitForFinish`/`Call`, which run with no client-imposed timeout of their own -(the pure time budget `waitSecs`/`timeoutSecs` already bounds the overall wait). The duration of +requests behind `WaitForFinish`/`Call`, and the start request of `Call`, which run with no +client-imposed timeout of their own (the time budget `waitSecs` bounds the overall wait, and the +`context.Context` the call itself). The duration of each tier is configured on the client (see the table above); a timeout still grows with each retry, capped at `WithTimeoutMax`. @@ -137,6 +140,13 @@ A single call that needs a different bound already has full control via the internal per-attempt deadline is always the *later* of the two to apply — i.e. `context.WithTimeout` composes with whichever is sooner). +A request that timed out is retried like any other transient failure, with a longer timeout, +except one that is not idempotent, where the timed-out attempt may already have taken effect: +starting a run (`Start`, `StartRaw`, a task's `Start`), building an Actor (`Build`), pushing +dataset items (`PushItems`) and, when `SetRecordOptions.DoNotRetryTimeouts` is set, storing a +record. Those fail on the first timeout. The other failures (a 429 or 5xx response, a network +error) are retried for them as well. + A method with a `waitForFinishSecs`/`WaitForFinish` parameter that asks the API to hold the response (e.g. `RunClient.GetWithWait`, `ActorClient.Start`'s `WaitForFinish` option) extends its tier's timeout by the requested wait (capped at 60s, the server's own hold limit), so the @@ -158,6 +168,18 @@ Apify platform. It is driven solely by the `APIFY_IS_AT_HOME` environment variab variable the JavaScript reference client reads); if it is set to a non-empty value, the flag is `true`, otherwise `false`. +### URL fields + +The fields the API specification declares as URLs (`consoleUrl`, `containerUrl`, `itemsPublicUrl`, +`keysPublicUrl`, `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, the webhook `requestUrl`, a +run's `output` links, a Store Actor's `url` and `userPictureUrl`, and a user profile's +`pictureUrl` and `websiteUrl`) are returned in a normalized form, so every Apify client returns +the same string for the same field: the scheme and host are lowercased, a non-ASCII host becomes +punycode, a default port is dropped, an empty path becomes `/`, dot segments are resolved, and +unsafe characters are percent-encoded. Compare a field against a stored URL after normalizing +the stored one the same way (for example by reading it back from the API). A value that is not +an absolute URL makes decoding the response fail. + ## Resource clients | Accessor | Returns | Purpose | @@ -173,7 +195,7 @@ variable the JavaScript reference client reads); if it is set to a non-empty val | `client.Webhooks()` / `client.Webhook(id)` | `*WebhookCollectionClient` / `*WebhookClient` | List/create webhooks; manage and test a single webhook. | | `client.WebhookDispatches()` / `client.WebhookDispatch(id)` | `*WebhookDispatchCollectionClient` / `*WebhookDispatchClient` | List/inspect webhook dispatches. | | `client.Store()` | `*StoreCollectionClient` | Browse and iterate the Apify Store. | -| `client.Me()` / `client.User(id)` | `*UserClient` | Account details, usage and limits (account scope for `Me()`). | +| `client.Me()` / `client.User(id)` | `*UserClient` | Account details, usage and limits (for your own account or your organization with `User(id)`). | | `client.Log(buildOrRunID)` | `*LogClient` | Read or stream a build/run log. | ## Error handling @@ -258,7 +280,7 @@ func main() { - `apify.ClientVersion` — the semantic version of this library. - `apify.APISpecVersion` — the Apify OpenAPI spec version this client was built against - (`v2-2026-10-26T112451Z`). + (`v2-2026-10-29T100357Z`). ### Releasing @@ -295,7 +317,7 @@ APIFY_TOKEN= go run ./examples/run_store_actor | `run_store_actor` | Run a Store Actor, wait for it, and read its default dataset. | | `run_and_last_run_storages` | Start a run, wait for it to finish, then fetch the Actor's last run and its storages. | | `iterate_store` | Lazily iterate Actors in the Apify Store. | -| `log_redirection` | Start an Actor and stream its log in real time. | +| `log_redirection` | Run an Actor and redirect its log to stdout in real time. | | `create_build_run_actor` | Create an Actor, build it, run it, and print the run log. | | `public_build_no_token` | Fetch a public Actor's default build with an unauthenticated client (no token). | diff --git a/actor.go b/actor.go index 0ad71fa..67f7b67 100644 --- a/actor.go +++ b/actor.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "net/http" + "time" ) // ActorStartOptions configures starting an Actor or task run @@ -43,6 +44,11 @@ type ActorStartOptions struct { // actor-memory-limit-exceeded as well and never starts, so WaitForResourcesIndefinitely // retries it forever. WaitForResources *int64 + // Log configures the redirection of the run's log that [ActorClient.Call] and + // [ActorClient.CallRaw] do while they wait (see [StreamedLog]). Nil redirects it to os.Stderr + // with the default prefix; set Log.Disabled to turn the redirection off. Start and StartRaw + // ignore it. + Log *StreamedLogOptions } func (o ActorStartOptions) apply(q *QueryParams) { @@ -133,7 +139,7 @@ func (c *ActorClient) Start(ctx context.Context, input any, options ActorStartOp if err != nil { return ActorRun{}, err } - return c.startWithBody(ctx, body, contentTypeJSON, options) + return c.startWithBody(ctx, body, contentTypeJSON, options, c.startTimeout(options)) } // StartRaw starts the Actor with a raw request body instead of a JSON-serializable value. @@ -144,22 +150,30 @@ func (c *ActorClient) Start(ctx context.Context, input any, options ActorStartOp // handles the serialization. Mirrors the reference client, whose Actor input accepts a plain // object, an array of them, or raw bytes. func (c *ActorClient) StartRaw(ctx context.Context, input []byte, options ActorStartOptions) (ActorRun, error) { - return c.startWithBody(ctx, input, "application/octet-stream", options) + return c.startWithBody(ctx, input, "application/octet-stream", options, c.startTimeout(options)) +} + +// startTimeout is the timeout of a start request: the "medium" tier, extended to cover the time +// the API may hold the response for options.WaitForFinish. +func (c *ActorClient) startTimeout(options ActorStartOptions) time.Duration { + return extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) } // startWithBody is the shared implementation of Start and StartRaw: it applies the run-start // options as query parameters and posts the given body, falling back to defaultContentType only -// when the caller did not set options.ContentType. -func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions) (ActorRun, error) { +// when the caller did not set options.ContentType. timeout bounds each start request. +// +// A start request that times out is not retried: it may already have created the run, and a +// retry would then start a second, billed one. +func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions, timeout time.Duration) (ActorRun, error) { params := NewQueryParams() options.apply(params) contentType := defaultContentType if options.ContentType != nil && *options.ContentType != "" { contentType = *options.ContentType } - timeout := extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) return startWaitingForResources(ctx, options.WaitForResources, func() (ActorRun, error) { - return postWithBody[ActorRun](ctx, c.ctx, "runs", params, body, contentType, timeout) + return postWithBody[ActorRun](ctx, c.ctx.withoutTimeoutRetry(), "runs", params, body, contentType, timeout) }) } @@ -167,23 +181,39 @@ func (c *ActorClient) startWithBody(ctx context.Context, body []byte, defaultCon // // waitSecs bounds the wait; nil waits indefinitely. It returns the finished run (or the // still-running run if the wait budget was exhausted). +// +// While it waits, it redirects the run's log to os.Stderr (or to options.Log.Output), prefixed +// with the Actor name and the run ID, like the reference client's call(): see [StreamedLog], and +// set options.Log.Disabled to turn it off. +// +// Neither the start request nor the polls have a client-side timeout of their own (matching the +// reference client's call(), whose default timeout is noTimeout): ctx bounds the whole call. func (c *ActorClient) Call(ctx context.Context, input any, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { - run, err := c.Start(ctx, input, options) + body, err := marshalInput(input) if err != nil { return ActorRun{}, err } - // Use the root client's run client so polling targets the canonical run route. - return c.root.Run(run.ID).WaitForFinish(ctx, waitSecs) + return c.callWithBody(ctx, body, contentTypeJSON, options, waitSecs) } // CallRaw starts the Actor with a raw request body ([ActorClient.StartRaw]) and waits for it to // finish, exactly like [ActorClient.Call] but for a non-JSON input. func (c *ActorClient) CallRaw(ctx context.Context, input []byte, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { - run, err := c.StartRaw(ctx, input, options) + return c.callWithBody(ctx, input, "application/octet-stream", options, waitSecs) +} + +// callWithBody is the shared implementation of Call and CallRaw. +func (c *ActorClient) callWithBody(ctx context.Context, body []byte, defaultContentType string, options ActorStartOptions, waitSecs *int64) (ActorRun, error) { + run, err := c.startWithBody(ctx, body, defaultContentType, options, noRequestTimeout) if err != nil { return ActorRun{}, err } - return c.root.Run(run.ID).WaitForFinish(ctx, waitSecs) + // Use the root client's run client so polling targets the canonical run route. + runClient := c.root.Run(run.ID) + streamed := startLogRedirection(ctx, runClient, options.Log) + finished, err := runClient.WaitForFinish(ctx, waitSecs) + stopLogRedirection(ctx, streamed, options.Log) + return finished, err } // Build builds the given version of the Actor and returns the created build. @@ -195,7 +225,8 @@ func (c *ActorClient) Build(ctx context.Context, versionNumber string, options A AddBool("useCache", options.UseCache). AddInt("waitForFinish", options.WaitForFinish) timeout := extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) - return postWithBody[Build](ctx, c.ctx, "builds", params, nil, contentTypeJSON, timeout) + // A build that timed out is not retried: it may already have been created. + return postWithBody[Build](ctx, c.ctx.withoutTimeoutRetry(), "builds", params, nil, contentTypeJSON, timeout) } // DefaultBuild resolves the Actor's default build and returns a client for it. @@ -203,6 +234,9 @@ func (c *ActorClient) Build(ctx context.Context, versionNumber string, options A // waitForFinish optionally bounds how long (in seconds) the API waits for the build to // finish before responding, matching the reference client's defaultBuild(options). func (c *ActorClient) DefaultBuild(ctx context.Context, waitForFinish *int64) (*BuildClient, error) { + if c.ctx.idErr != nil { + return nil, c.ctx.idErr + } params := NewQueryParams() params.AddInt("waitForFinish", waitForFinish) url := params.applyToURL(c.ctx.subURL("builds/default")) @@ -237,6 +271,9 @@ func (c *ActorClient) ValidateInput(ctx context.Context, input any) (json.RawMes // On success it returns the raw JSON validation result from the API (a JSON object reporting // whether the input is valid and, if not, the schema violations). func (c *ActorClient) ValidateInputForBuild(ctx context.Context, input any, build string) (json.RawMessage, error) { + if c.ctx.idErr != nil { + return nil, c.ctx.idErr + } body, err := json.Marshal(input) if err != nil { return nil, err diff --git a/base.go b/base.go index 6d69d25..88429f6 100644 --- a/base.go +++ b/base.go @@ -86,6 +86,14 @@ func newResourceContext(hc *httpClient, url, baseURL string, hasOwnID bool, idEr } } +// withoutTimeoutRetry returns a copy of this context whose requests do not retry an attempt that +// timed out (see [httpClient.withoutTimeoutRetry]). Use it for a request that is not idempotent. +func (c *resourceContext) withoutTimeoutRetry() *resourceContext { + clone := *c + clone.http = c.http.withoutTimeoutRetry() + return &clone +} + // withPublicOrigin overrides the origin used when building public URLs. func (c *resourceContext) withPublicOrigin(publicBaseURL string) *resourceContext { c.publicOrigin = originOf(publicBaseURL) @@ -150,8 +158,14 @@ func originOf(rawURL string) string { // reference client's catchNotFoundForResourceOrThrow. For a 404 that is unambiguous regardless // of how the client was reached (e.g. a lookup by key or request id), use [getResourceAlways]. func getResource[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (T, bool, error) { + return getResourceWithTimeout[T](ctx, c, subPath, params, c.shortTimeout()) +} + +// getResourceWithTimeout is [getResource] with the caller's chosen timeout tier, for a lookup +// that is slower than a plain metadata read (see [timeoutTiers]). +func getResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subPath string, params *QueryParams, timeout time.Duration) (T, bool, error) { var zero T - result, err := getResourceRequired[T](ctx, c, subPath, params, c.shortTimeout()) + result, err := getResourceRequired[T](ctx, c, subPath, params, timeout) if err != nil { if c.hasOwnID && isNotFound(err) { return zero, false, nil @@ -182,16 +196,7 @@ func extendForServerWait(base time.Duration, waitForFinishSecs *int64) time.Dura // [extendForServerWait], for a get(waitForFinish) call that asks the server to hold the // response (RunClient.GetWithWait, BuildClient.GetWithWait). func getResourceWithServerWait[T any](ctx context.Context, c *resourceContext, params *QueryParams, waitForFinishSecs *int64) (T, bool, error) { - var zero T - timeout := extendForServerWait(c.shortTimeout(), waitForFinishSecs) - result, err := getResourceRequired[T](ctx, c, "", params, timeout) - if err != nil { - if c.hasOwnID && isNotFound(err) { - return zero, false, nil - } - return zero, false, err - } - return result, true, nil + return getResourceWithTimeout[T](ctx, c, "", params, extendForServerWait(c.shortTimeout(), waitForFinishSecs)) } // getResourceAlways performs a GET that unwraps the data envelope; a not-found always maps to @@ -232,6 +237,11 @@ func getResourceRequired[T any](ctx context.Context, c *resourceContext, subPath // updateResource performs a PUT with a JSON body, unwrapping the data envelope. Metadata writes // use the "short" timeout tier. func updateResource[T any](ctx context.Context, c *resourceContext, subPath string, body any) (T, error) { + return updateResourceWithTimeout[T](ctx, c, subPath, body, c.shortTimeout()) +} + +// updateResourceWithTimeout is [updateResource] with the caller's chosen timeout tier. +func updateResourceWithTimeout[T any](ctx context.Context, c *resourceContext, subPath string, body any, timeout time.Duration) (T, error) { var zero T if c.idErr != nil { return zero, c.idErr @@ -241,7 +251,7 @@ func updateResource[T any](ctx context.Context, c *resourceContext, subPath stri return zero, err } url := c.mergedParams(NewQueryParams()).applyToURL(c.subURL(subPath)) - resp, err := c.http.call(ctx, http.MethodPut, url, data, contentTypeJSON, c.shortTimeout()) + resp, err := c.http.call(ctx, http.MethodPut, url, data, contentTypeJSON, timeout) if err != nil { return zero, err } @@ -386,15 +396,15 @@ func getRawRequired(ctx context.Context, c *resourceContext, subPath string, par return c.http.call(ctx, http.MethodGet, url, nil, "", timeout) } -// headExists performs a HEAD request, returning whether the resource exists, using the "short" -// timeout tier. A not-found always maps to (false, nil): used only for an unambiguous lookup (a -// key-value-store record by key). +// headExists performs a HEAD request, returning whether the resource exists, using the "long" +// timeout tier (the reference client's tier for a record lookup). A not-found always maps to +// (false, nil): used only for an unambiguous lookup (a key-value-store record by key). func headExists(ctx context.Context, c *resourceContext, subPath string, params *QueryParams) (bool, error) { if c.idErr != nil { return false, c.idErr } url := c.mergedParams(params).applyToURL(c.subURL(subPath)) - _, err := c.http.call(ctx, http.MethodHead, url, nil, "", c.shortTimeout()) + _, err := c.http.call(ctx, http.MethodHead, url, nil, "", c.longTimeout()) if err != nil { if isNotFound(err) { return false, nil diff --git a/dataset.go b/dataset.go index 79c83af..7551aba 100644 --- a/dataset.go +++ b/dataset.go @@ -1,8 +1,11 @@ package apify import ( + "bytes" "context" "encoding/json" + "errors" + "fmt" "io" "net/http" "strconv" @@ -163,7 +166,10 @@ func ListDatasetItems[T any](ctx context.Context, c *DatasetClient, options Data var result PaginationList[T] params := NewQueryParams() options.apply(params) - url := params.applyToURL(c.ctx.subURL("items")) + if c.ctx.idErr != nil { + return result, c.ctx.idErr + } + url := c.ctx.mergedParams(params).applyToURL(c.ctx.subURL("items")) resp, err := c.ctx.http.call(ctx, http.MethodGet, url, nil, "", c.ctx.longTimeout()) if err != nil { return result, err @@ -252,17 +258,52 @@ func (c *DatasetClient) DownloadItemsStream(ctx context.Context, format Download return stream, err } -// PushItems pushes one or more items to the dataset. items must serialize to a JSON object -// or an array of objects. +// PushItems pushes one or more items to the dataset. items must serialize to a JSON object or +// an array of objects, the only shapes the API accepts: anything else (a bare string, number or +// boolean, an array holding a non-object, nil) fails with an error before any request is sent. +// A value that is already serialized JSON can be passed as a [json.RawMessage]; a plain Go string +// is serialized as a JSON string, which is not a valid item. +// +// A push that times out is not retried: the API may already have stored the items, and a retry +// would then store them a second time. func (c *DatasetClient) PushItems(ctx context.Context, items any) error { body, err := json.Marshal(items) if err != nil { return err } - _, err = c.ctx.http.call(ctx, http.MethodPost, c.ctx.subURL("items"), body, contentTypeJSONCharset, c.ctx.mediumTimeout()) + if err := checkItemsShape(body); err != nil { + return err + } + if c.ctx.idErr != nil { + return c.ctx.idErr + } + url := c.ctx.mergedParams(NewQueryParams()).applyToURL(c.ctx.subURL("items")) + _, err = c.ctx.withoutTimeoutRetry().http.call(ctx, http.MethodPost, url, body, contentTypeJSONCharset, c.ctx.mediumTimeout()) return err } +// checkItemsShape reports whether body, a JSON document, is an object or an array of objects. +func checkItemsShape(body []byte) error { + trimmed := bytes.TrimSpace(body) + switch { + case bytes.HasPrefix(trimmed, []byte("{")): + return nil + case bytes.HasPrefix(trimmed, []byte("[")): + var elements []json.RawMessage + if err := json.Unmarshal(trimmed, &elements); err != nil { + return err + } + for i, element := range elements { + if !bytes.HasPrefix(bytes.TrimSpace(element), []byte("{")) { + return fmt.Errorf("PushItems: item %d is not a JSON object; the API only accepts objects", i) + } + } + return nil + default: + return errors.New("PushItems: items must be a JSON object or an array of objects") + } +} + // GetStatistics returns statistical information about the dataset. // // A 404 here always means the dataset itself is gone (the same way a plain Get would report @@ -300,7 +341,8 @@ func (c *DatasetClient) CreateItemsPublicURLWithFormat(ctx context.Context, opti params.AddString("format", &fmtStr) } - dataset, present, err := c.Get(ctx) + // Fetched with the "long" timeout tier, the reference client's tier for this call. + dataset, present, err := getResourceWithTimeout[Dataset](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) if err != nil { return "", err } @@ -310,7 +352,7 @@ func (c *DatasetClient) CreateItemsPublicURLWithFormat(ctx context.Context, opti params.AddString("signature", &sig) } } - return params.applyToURL(c.ctx.publicURL("items")), nil + return c.ctx.mergedParams(params).applyToURL(c.ctx.publicURL("items")), nil } // headerInt parses an integer HTTP header value, returning fallback if absent or invalid. diff --git a/docs/README.md b/docs/README.md index 54a1d98..99cbe1f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -105,7 +105,7 @@ the [storages](storages.md) page. - [Actors](actors.md) — Actors, versions, environment variables. - [Builds](builds.md) — Actor builds. -- [Runs](runs.md) — Actor runs and their default storages. +- [Runs](runs.md) — Actor runs, their default storages, log redirection and live iteration of dataset items. - [Tasks](tasks.md) — Actor tasks. - [Storages](storages.md) — Datasets, key-value stores, request queues. - [Schedules](schedules.md) — Schedules. diff --git a/docs/actors.md b/docs/actors.md index 87d85ef..d49d9ac 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -82,7 +82,7 @@ The `Actor` value returned by `Get`/`Create`/`Update` and listed by `List`: | `Delete(ctx) error` | Delete the Actor. | | `Start(ctx, input any, ActorStartOptions) (ActorRun, error)` | Start a run, return immediately. | | `StartRaw(ctx, input []byte, ActorStartOptions) (ActorRun, error)` | Like `Start`, but sends `input` as a raw request body (no JSON serialization) — for a non-JSON input such as a ZIP archive, paired with `ActorStartOptions.ContentType`. | -| `Call(ctx, input any, ActorStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait for the run to finish. `waitSecs` bounds the client-side wait; pass `nil` to wait indefinitely (until the run reaches a terminal state). | +| `Call(ctx, input any, ActorStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait for the run to finish, redirecting the run's log to `os.Stderr` while it waits (see `ActorStartOptions.Log`). `waitSecs` bounds the client-side wait; pass `nil` to wait indefinitely (until the run reaches a terminal state). Neither the start request nor the polls have a client-side timeout: `ctx` bounds the call. | | `CallRaw(ctx, input []byte, ActorStartOptions, waitSecs *int64) (ActorRun, error)` | `StartRaw` followed by waiting for the run to finish, like `Call`. | | `Build(ctx, versionNumber string, ActorBuildOptions) (Build, error)` | Build a version. | | `DefaultBuild(ctx, waitForFinish *int64) (*BuildClient, error)` | Resolve the default build. | @@ -155,6 +155,7 @@ fmt.Println(string(result)) // raw JSON validation result | `RestartOnError` | `*bool` | Restart the run if it fails. | | `ForcePermissionLevel` | `*string` | Override the Actor's permission level for this run. Accepted values: `LIMITED_PERMISSIONS`, `FULL_PERMISSIONS`. | | `Webhooks` | `[]any` | Ad-hoc webhooks to attach to this run. Each element is a map describing one webhook: `{"eventTypes": []string, "requestUrl": string, "payloadTemplate": string}` (same shape as a webhook definition; `eventTypes` are the `WebhookEventType` values listed in [webhooks.md](webhooks.md)). | +| `Log` | `*StreamedLogOptions` | Only for `Call`/`CallRaw`: how the run's log is redirected while they wait. `nil` redirects it to `os.Stderr`, with every message prefixed by `"{Actor name} runId:{run ID} -> "`. Set `Output` (an `io.Writer`), `Prefix` or `FromStart` to change that, or `Disabled` to turn the redirection off. See [`StreamedLog`](runs.md#log-redirection). `Start` and `StartRaw` ignore it. | | `WaitForResources` | `*int64` | Retry the start every 10 seconds while the API rejects it for lack of resources (`actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`). `apify.WaitForResourcesIndefinitely` retries until the run starts; any other value is the number of seconds to retry for before returning the last error. `nil` (the default) does not retry. Any other error is returned immediately. | `ActorBuildOptions` (all fields optional): diff --git a/docs/misc.md b/docs/misc.md index 6f4fcb4..c996193 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -23,6 +23,7 @@ Browse public Actors with `client.Store()`: | `IncludeUnrunnableActors` | `*bool` | Include Actors the current user cannot run. | | `AllowsAgenticUsers` | `*bool` | Filter to Actors that allow agentic users. | | `ResponseFormat` | `*string` | Select the response format. Accepted values: `full`, `agent`. | +| `IncludeInputSchema` | `*bool` | Add to each Actor an `inputSchema` field with a compact version of its default build's input schema. With it set, the API defaults `Limit` to 10 and does not accept a larger one. | Each item is an `ActorStoreListItem`: @@ -32,6 +33,8 @@ Each item is an `ActorStoreListItem`: | `Name` | `string` | Technical name of the Actor. | | `Username` | `string` | Username of the Actor's owner. | | `Title` | `string` | Human-readable title (may be empty; fall back to `Name`). | +| `URL` | `*string` | URL of the Actor's page in the Apify Store, normalized. `nil` when there is none. | +| `UserPictureURL` | `*string` | URL of the owner's profile picture, normalized. `nil` when there is none. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (forward compatibility). | ```go @@ -56,10 +59,10 @@ for { | Method | Description | | --- | --- | | `Get(ctx) (User, bool, error)` | Account details (private for `Me()`, public otherwise). | -| `MonthlyUsage(ctx) (json.RawMessage, error)` | Current month's usage (`Me()` only). | -| `MonthlyUsageWithOptions(ctx, MonthlyUsageOptions) (json.RawMessage, error)` | Usage for the period containing `Date` (`YYYY-MM-DD`; empty means current) (`Me()` only). | -| `Limits(ctx) (json.RawMessage, error)` | Current account's limits (`Me()` only). | -| `UpdateLimits(ctx, newLimits any) error` | Update the account's limits (`Me()` only). | +| `MonthlyUsage(ctx) (json.RawMessage, error)` | Current month's usage. | +| `MonthlyUsageWithOptions(ctx, MonthlyUsageOptions) (json.RawMessage, error)` | Usage for the period containing `Date` (`YYYY-MM-DD`; empty means current). | +| `Limits(ctx) (json.RawMessage, error)` | The account's limits. | +| `UpdateLimits(ctx, newLimits any) error` | Update the account's limits. | The `User` value returned by `Get` carries the account's public fields; for `Me()` the API also returns private account details, which are preserved in `Extra`: @@ -68,9 +71,10 @@ also returns private account details, which are preserved in `Extra`: |---|---|---| | `ID` | `string` | Unique user ID. | | `Username` | `string` | The user's username. | +| `Profile` | `*UserProfile` | The public profile: `Name`, `PictureURL` and `WebsiteURL` (both `*string`, normalized URLs), and an `Extra` map for the other fields. `nil` if the API does not return one. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (private details for `Me()`). | -The usage/limits methods return an error if called on a non-`Me()` client. +The usage and limits methods work on `Me()` and on `User(id)` (by ID or username). The API serves them only for your own account or for the organization your API token belongs to, and rejects tokens with limited permissions. `MonthlyUsage` reports usage for the current month and returns the raw JSON usage report from the API — a JSON object with the account's usage breakdown and totals for the period. @@ -113,8 +117,9 @@ fmt.Println(string(usage)) // raw JSON usage report for the current month | `Raw` | `*bool` | Return the unprocessed log content (no platform post-processing). | | `Download` | `*bool` | Set `Content-Disposition` so the log is served as a download. | -For convenient live redirection of a run's log, `client.Run(id).GetStreamedLog(ctx)` returns -a raw live stream directly. +`client.Run(id).GetStreamedLog(ctx)` returns a raw live stream of the run's log directly. To +redirect the log message by message to a writer, with a prefix, see +[log redirection](runs.md#log-redirection). ```go // Read the whole log. diff --git a/docs/runs.md b/docs/runs.md index b2787c6..17e07e3 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -18,8 +18,9 @@ with `client.Run(id)`, and an Actor's or task's runs with `client.Actor(id).Runs | `Status` | `[]string` | Filter by one or more run statuses; sent as a comma-separated list. Values are the eight `ActorJobStatus` values: `READY`, `RUNNING`, `SUCCEEDED`, `FAILED`, `TIMING-OUT`, `TIMED-OUT`, `ABORTING`, `ABORTED`. | | `StartedAfter` | `*string` | Only runs started after this ISO-8601 timestamp. | | `StartedBefore` | `*string` | Only runs started before this ISO-8601 timestamp. | +| `Deployment` | `[]string` | Only Actor Standby runs served by these pools (`SINGLE_TENANT`, `MULTI_TENANT_FREE_USERS`, `MULTI_TENANT_PAYING_USERS`, `MULTI_TENANT_OWNER_PAID`); sent as a comma-separated list. The API requires `Status` to hold only non-terminal statuses (`READY`, `RUNNING`, `TIMING-OUT`, `ABORTING`) with it. | -The time filters apply only to Actor- and task-scoped collections. +The time filters apply only to Actor- and task-scoped collections, and `Deployment` only to the account-wide and Actor-scoped ones. ## Single run @@ -40,7 +41,13 @@ The time filters apply only to Actor- and task-scoped collections. | `KeyValueStore() *KeyValueStoreClient` | The run's default key-value store. | | `RequestQueue() *RequestQueueClient` | The run's default request queue. | | `Log() *LogClient` | The run's log. | -| `GetStreamedLog(ctx) (io.ReadCloser, error)` | Live stream of the run's raw log (log redirection). | +| `GetStreamedLog(ctx) (io.ReadCloser, error)` | Live stream of the run's raw log, as bytes. | +| `NewStreamedLog(ctx, StreamedLogOptions) (*StreamedLog, error)` | Redirects the run's log message by message to a writer; see [Log redirection](#log-redirection). `(nil, nil)` for a run that does not exist (with the default prefix). | +| `IterateDatasetItems(RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[json.RawMessage]` | Iterates over the items of the run's default dataset while the run is still producing them; see [Live iteration of dataset items](#live-iteration-of-dataset-items). | + +On a client from `LastRun(...)`, `Dataset()`, `KeyValueStore()`, `RequestQueue()` and `Log()` carry +the same `status` and `origin` filters, so they address the storages and the log of the run the +filters resolve to, not those of the newest run of any status. > **Missing resources.** `Dataset()`/`KeyValueStore()`/`RequestQueue()`/`Log()` address a > sub-resource with no ID of its own, so a 404 from `Get`/`Delete`/`Log().Get`/`Log().Stream` @@ -112,9 +119,80 @@ run's metadata. The commonly used fields: | `DefaultDatasetID` | `string` | ID of the run's default dataset. | | `DefaultKeyValueStoreID` | `string` | ID of the run's default key-value store. | | `DefaultRequestQueueID` | `string` | ID of the run's default request queue. | -| `ContainerURL` | `string` | URL of the run's container (for live access). | +| `ContainerURL` | `string` | URL of the run's container (for live access), normalized. | +| `ConsoleURL` | `string` | URL of the run in the Apify Console, normalized. | +| `Output` | `map[string]string` | Output links of the run (name to normalized URL), generated from the output schema of the Actor build. `nil` if the build defines none. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (forward compatibility). | `ActorRun.IsTerminal()` reports whether a run has finished. Status message convenience: `client.SetStatusMessage(ctx, message, isTerminal)` updates the current run identified by the `ACTOR_RUN_ID` environment variable. + +## Log redirection + +`ActorClient.Call` redirects the log of the run while it waits (set `ActorStartOptions.Log` to +change where it goes or turn it off). To redirect the log of any run, create a `StreamedLog`: + +```go +streamed, err := client.Run(runID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) +if err != nil { + log.Fatal(err) +} +if err := streamed.Start(ctx); err != nil { + log.Fatal(err) +} +// ... wait for the run to finish ... +if err := streamed.Stop(ctx); err != nil { + log.Fatal(err) +} +``` + +`StreamedLogOptions` (all fields optional): + +| Field | Type | Meaning | +|---|---|---| +| `Output` | `io.Writer` | Receives every message, one per line, with the prefix. Defaults to `os.Stderr`. Writes are serialized. | +| `Prefix` | `*string` | Written in front of every message. The default is `"{Actor name} runId:{run ID} -> "`, which costs two requests (the run and its Actor); a non-nil `Prefix`, an empty one included, skips them. | +| `FromStart` | `*bool` | `false` drops the messages that were logged before the redirection was created. Default `true`. | +| `Disabled` | `bool` | Turns the redirection off in `ActorStartOptions.Log`. | + +The log is split into messages at the timestamps the platform puts in front of each of them, so a +message that spans several lines is written as one. `Start` begins the redirection in the +background, and it also ends when the `ctx` it got is done. `Stop` gives the messages in flight up +to one second, then aborts the stream; if the stream delivered nothing yet, it reads the whole log +in one request, so a short run that finished before the stream connected is not lost. A log that +has no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, +until its first bytes arrive. `Stop` returns the error that ended the redirection early, if any; +`Call` reports it on the redirection's output and goes on. + +## Live iteration of dataset items + +`RunClient.IterateDatasetItems` (and the generic `apify.IterateRunDatasetItems[T]`, which decodes +every item into `T`) iterates over the items of the run's default dataset *while the run is still +producing them*. While the run has not finished, each round yields the rows below the dataset's +`itemCount` and then waits up to `PollIntervalSecs` for the run to finish, so the last rows are +read as soon as it does. `itemCount` lags a few seconds behind the pushed items, so once the run +has finished the rows past it are read a page at a time until none are left. On a client from +`LastRun(...)`, the iterator sticks to the run its first request resolves to. + +`RunIterateDatasetItemsOptions` (all fields optional): + +| Field | Type | Meaning | +|---|---|---| +| `Items` | `DatasetListItemsOptions` | Filters and projections applied to every page (see [storages.md](storages.md)). `Items.Offset` is the number of rows to skip (default 0) and `Items.Limit` the maximum number of dataset rows to scan (default no limit): fewer items are yielded when filters drop some, more when `Unwind` splits a row into several. `Items.Desc` is not supported, because the rows are read by offset from the oldest; it makes `Next` fail. | +| `ChunkSize` | `*int64` | Maximum number of dataset rows requested per API call. Default 1000. | +| `PollIntervalSecs` | `*int64` | How long to wait for the run to finish between polls, in seconds. Default 5. | + +```go +it := client.Run(runID).IterateDatasetItems(apify.RunIterateDatasetItemsOptions{}) +for { + item, err := it.Next(ctx) + if err != nil { + log.Fatal(err) + } + if item == nil { + break // every item has been read + } + fmt.Println(string(*item)) +} +``` diff --git a/docs/storages.md b/docs/storages.md index 57d3327..ef4d561 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -19,7 +19,7 @@ Single dataset: `client.Dataset(id)`: | `Delete(ctx) error` | Delete the dataset. | | `ListItems(ctx, DatasetListItemsOptions) (PaginationList[json.RawMessage], error)` | Read items. | | `IterateItems(DatasetListItemsOptions, chunkSize *int64) *ListIterator[json.RawMessage]` | Lazy iterator over items (`Limit` caps total, `chunkSize` is page size); `IterateDatasetItems[T]` decodes into your type. | -| `PushItems(ctx, items any) error` | Append one item or a slice of items. | +| `PushItems(ctx, items any) error` | Append one item or a slice of items. They must serialize to a JSON object or an array of objects, the only shapes the API accepts; anything else (a string, a number, a slice of strings, `nil`) fails before any request is sent. Pass a `json.RawMessage` for already serialized JSON. A push that times out is not retried, because the API may already have stored the items. | | `DownloadItems(ctx, DownloadItemsFormat, DatasetDownloadOptions) ([]byte, error)` | Export items (JSON, JSONL, CSV, XLSX, XML, RSS, HTML — see the format constants below). | | `DownloadItemsStream(ctx, DownloadItemsFormat, DatasetDownloadOptions) (io.ReadCloser, error)` | Like `DownloadItems`, but as a stream for a large export; close when done. | | `GetStatistics(ctx) (json.RawMessage, error)` | Dataset statistics. | @@ -36,6 +36,8 @@ The `Dataset` value returned by `Get`/`GetOrCreate`/`Update` and listed by `List | `CreatedAt` | `*time.Time` | When the dataset was created. | | `ModifiedAt` | `*time.Time` | When the dataset was last modified. | | `ItemCount` | `int64` | Number of items currently stored. | +| `ConsoleURL` | `string` | URL of the dataset in the Apify Console (normalized). | +| `ItemsPublicURL` | `string` | Public URL of the dataset's items (normalized); signed for a private dataset. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | `DatasetListItemsOptions` controls filtering, projection, and pagination of the items @@ -167,13 +169,15 @@ Single store: `client.KeyValueStore(id)`: | `Delete(ctx) error` | Delete the store. | | `ListKeys(ctx, ListKeysOptions) (KeyValueStoreKeysPage, error)` | List one page of keys. | | `IterateKeys(ListKeysOptions, chunkSize *int64) *KeyValueStoreKeysIterator` | Lazy iterator over keys (cursor-based). `Limit` caps the total yielded; `chunkSize` is the page size; `Prefix`/`Collection`/`Signature` filter every page; `ExclusiveStartKey` sets where to start. | -| `GetRecord(ctx, key) (*KeyValueStoreRecord, bool, error)` | Read a record. | +| `GetRecord(ctx, key) (*KeyValueStoreRecord, bool, error)` | Read a record. The value is the raw bytes, never parsed: decode it according to the `ContentType` the API reports. | | `GetRecordWithOptions(ctx, key, GetRecordOptions)` | Read with options. | | `GetRecordStream(ctx, key) (*KeyValueStoreRecordStream, bool, error)` | Like `GetRecord`, but the value is a stream for a large record; close when done. | | `GetRecordStreamWithOptions(ctx, key, GetRecordOptions)` | Stream with options. | -| `SetRecordRaw(ctx, key, value []byte, contentType string) error` | Write raw bytes. | +| `SetRecordRaw(ctx, key, value []byte, contentType string) error` | Write raw bytes, sent as they are: the client does not parse or convert the value, whatever the content type (so `application/json` bytes must already be valid JSON). | +| `SetRecordRawWithOptions(ctx, key, value []byte, contentType string, SetRecordOptions) error` | Like `SetRecordRaw`, with `ContentEncoding` (the value is already compressed, as with `SetRecordRawWithEncoding`) and `DoNotRetryTimeouts` (an upload that times out fails at once instead of being retried). | | `SetRecordRawWithEncoding(ctx, key, value []byte, contentType, contentEncoding string) error` | Write raw bytes that are already compressed (`contentEncoding` e.g. `"gzip"`, `"br"`, `"deflate"`): sent as-is under `Content-Encoding`, skipping the client's own request compression so the value is never compressed twice. | | `SetRecordJSON(ctx, key, value any) error` | Write JSON. | +| `SetRecordJSONWithOptions(ctx, key, value any, SetRecordOptions) error` | Like `SetRecordJSON`, with the per-call `SetRecordOptions`. | | `RecordExists(ctx, key) (bool, error)` | HEAD check. | | `DeleteRecord(ctx, key) error` | Delete a record. | | `GetRecordPublicURL(ctx, key) (string, error)` | Signed public record URL. | @@ -226,8 +230,9 @@ when done. | `NextExclusiveStartKey` | `string` | Key to pass to fetch the next page. | | `Items` | `[]KeyValueStoreKey` | The listed keys. | -Each `KeyValueStoreKey` element has `Key` (`string`) and `Size` (`int64`, the record size in -bytes), plus an `Extra` (`map[string]json.RawMessage`) catch-all. +Each `KeyValueStoreKey` element has `Key` (`string`), `Size` (`int64`, the record size in +bytes) and `RecordPublicURL` (`string`, the public URL of the record, normalized), plus an +`Extra` (`map[string]json.RawMessage`) catch-all. ```go store, _ := client.KeyValueStores().GetOrCreate(ctx, "") @@ -299,6 +304,7 @@ The `RequestQueue` value returned by `Get`/`GetOrCreate`/`Update` and listed by | `CreatedAt` | `*time.Time` | When the queue was created. | | `ModifiedAt` | `*time.Time` | When the queue was last modified. | | `TotalRequestCount` | `int64` | Total number of requests ever added. | +| `ConsoleURL` | `string` | URL of the queue in the Apify Console (normalized). | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | `RequestQueueRequest` is the request payload/record. `URL` is required; `ID` is assigned by the diff --git a/docs/tasks.md b/docs/tasks.md index 1568848..c4f7f6b 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -25,6 +25,7 @@ A task is a pre-configured Actor run with stored input. Access the task collecti | `ModifiedAt` | `*time.Time` | When the task was last modified. | | `IsPublic` | `*bool` | Whether the task is published on its public landing page, derived from `PublicConfig.PublishedAt`; use `Publish`/`Unpublish` to change it. | | `PublicConfig` | `*TaskPublicConfig` | Public-facing display configuration of the landing page, set once the task has been configured for publishing. | +| `StandbyURL` | `*string` | URL of the task's Actor Standby endpoint, normalized. `nil` when the task has none. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | ### `TaskPublicConfig` fields @@ -49,7 +50,7 @@ A task is a pre-configured Actor run with stored input. Access the task collecti | `Publish(ctx) (Task, error)` | Publish the task on its public landing page. An Actor can have at most 10 published tasks, and an account at most 100 (contact Apify support to raise either limit). | | `Unpublish(ctx) (Task, error)` | Unpublish the task from its public landing page. | | `Start(ctx, input any, TaskStartOptions) (ActorRun, error)` | Start a run (input overrides stored input). | -| `Call(ctx, input any, TaskStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait. | +| `Call(ctx, input any, TaskStartOptions, waitSecs *int64) (ActorRun, error)` | Start and wait. Neither the start request nor the polls have a client-side timeout: `ctx` bounds the call. | | `GetInput(ctx) (json.RawMessage, error)` | Fetch the stored input. | | `UpdateInput(ctx, input any) (json.RawMessage, error)` | Replace the stored input. | | `LastRun(status string) *RunClient` | Client for the last run (optional status filter). | diff --git a/docs/webhooks.md b/docs/webhooks.md index 20beb81..0a39be6 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -51,7 +51,7 @@ The same values apply to the ad-hoc `ActorStartOptions.Webhooks` element (see |---|---|---| | `ID` | `string` | Unique webhook ID. | | `UserID` | `string` | ID of the user who owns the webhook. | -| `RequestURL` | `string` | URL the webhook posts to. | +| `RequestURL` | `string` | URL the webhook posts to, normalized. Empty when the API returns `null`. | | `EventTypes` | `[]string` | Events that trigger the webhook. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | @@ -61,6 +61,7 @@ The same values apply to the ad-hoc `ActorStartOptions.Webhooks` element (see |---|---|---| | `ID` | `string` | Unique dispatch ID. | | `WebhookID` | `string` | ID of the webhook that produced this dispatch. | +| `Webhook` | `*WebhookDispatchWebhookSummary` | Summary of the webhook as it was when the dispatch was created: `ID`, `ActionType`, `IsAdHoc`, `RequestURL` (normalized), and an `Extra` map. `nil` if the API does not return it. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | ## Single webhook diff --git a/examples/log_redirection/main.go b/examples/log_redirection/main.go index ab4c220..49a68bc 100644 --- a/examples/log_redirection/main.go +++ b/examples/log_redirection/main.go @@ -1,5 +1,5 @@ -// Command log_redirection starts an Actor without waiting, then streams its log output to -// stdout in real time (log redirection). +// Command log_redirection runs an Actor and redirects its log to stdout in real time while it +// runs. Call does this by default (to os.Stderr); here the output is set to stdout. // // Run with: // @@ -8,7 +8,7 @@ package main import ( "context" - "io" + "fmt" "log" "os" @@ -20,20 +20,26 @@ func main() { client := exampleclient.New() ctx := context.Background() - // Start the Actor and return immediately (do not wait for it to finish). - run, err := client.Actor("apify/hello-world").Start(ctx, nil, apify.ActorStartOptions{}) + // Every message of the run's log is printed as the run produces it, prefixed with the Actor + // name and the run ID, until the run finishes. + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{ + Log: &apify.StreamedLogOptions{Output: os.Stdout}, + }, nil) if err != nil { - log.Fatalf("start actor: %v", err) + log.Fatalf("call actor: %v", err) } + fmt.Println("run finished with status", run.Status) - // Open a live (raw) log stream and copy it to stdout as the run produces output. - stream, err := client.Run(run.ID).GetStreamedLog(ctx) + // To redirect the log of a run that was started elsewhere, create the redirection for it and + // stop it once the run is done. Starting it for a finished run reads the whole log. + streamed, err := client.Run(run.ID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) if err != nil { - log.Fatalf("open log stream: %v", err) + log.Fatalf("create log redirection: %v", err) } - defer func() { _ = stream.Close() }() - - if _, err := io.Copy(os.Stdout, stream); err != nil { - log.Fatalf("copy log: %v", err) + if err := streamed.Start(ctx); err != nil { + log.Fatalf("start log redirection: %v", err) + } + if err := streamed.Stop(ctx); err != nil { + log.Fatalf("stop log redirection: %v", err) } } diff --git a/go.mod b/go.mod index c9a8d11..91c6357 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,12 @@ module github.com/apify/apify-client-go -go 1.23 +go 1.23.0 toolchain go1.26.10 -require github.com/andybalholm/brotli v1.2.2 +require ( + github.com/andybalholm/brotli v1.2.2 + golang.org/x/net v0.38.0 +) + +require golang.org/x/text v0.23.0 // indirect diff --git a/go.sum b/go.sum index 80d4b3a..4f9b9e0 100644 --- a/go.sum +++ b/go.sum @@ -2,3 +2,7 @@ github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtn github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= diff --git a/http_client.go b/http_client.go index 5d79716..94a0013 100644 --- a/http_client.go +++ b/http_client.go @@ -4,8 +4,10 @@ import ( "bytes" "compress/gzip" "context" + "errors" "io" "math/rand" + "net" "net/http" "strings" "time" @@ -120,6 +122,20 @@ type httpClient struct { retry retryConfig tiers timeoutTiers compressors []requestCompressor + // doNotRetryTimeouts makes a request attempt that exceeds its timeout fail at once instead of + // being retried. Set it (through [httpClient.withoutTimeoutRetry]) for a request that is not + // idempotent, where a timed-out attempt may already have taken effect on the server, so + // repeating it could create a second run or store the same dataset items twice. + doNotRetryTimeouts bool +} + +// withoutTimeoutRetry returns a copy of c that does not retry a request attempt that timed out. +// Other retryable failures (a 429 or 5xx response, a transport error that is not a timeout) are +// still retried. +func (c *httpClient) withoutTimeoutRetry() *httpClient { + clone := *c + clone.doNotRetryTimeouts = true + return &clone } // apiResponse is the parsed result of a single API call: the status, headers and the @@ -175,7 +191,7 @@ func (c *httpClient) doRetryLoop(ctx context.Context, method, url string, sendBo var retryable bool if err != nil { lastErr = err - retryable = isTransportErrorRetryable(err) + retryable = isTransportErrorRetryable(err) && (!c.doNotRetryTimeouts || !isTimeoutError(err)) } else if resp.statusCode < maxSuccessStatusCode { return resp, nil } else { @@ -498,6 +514,16 @@ func isTransportErrorRetryable(err error) bool { return true } +// isTimeoutError reports whether err means a request attempt ran out of time: the attempt's own +// context deadline, or a timeout reported by the transport. +func isTimeoutError(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + // randomizedDelay returns a delay chosen randomly from [delay, 2*delay), matching the // exponential-backoff-with-jitter algorithm described in the Apify API docs. func randomizedDelay(delay time.Duration) time.Duration { diff --git a/key_value_store.go b/key_value_store.go index 34e63cc..a0787aa 100644 --- a/key_value_store.go +++ b/key_value_store.go @@ -64,9 +64,10 @@ func (c *KeyValueStoreClient) Get(ctx context.Context) (KeyValueStore, bool, err return getResource[KeyValueStore](ctx, c.ctx, "", NewQueryParams()) } -// Update updates the store metadata (e.g. name) and returns the updated object. +// Update updates the store metadata (e.g. name) and returns the updated object. Uses the "long" +// timeout tier, the reference client's tier for this call. func (c *KeyValueStoreClient) Update(ctx context.Context, newFields any) (KeyValueStore, error) { - return updateResource[KeyValueStore](ctx, c.ctx, "", newFields) + return updateResourceWithTimeout[KeyValueStore](ctx, c.ctx, "", newFields, c.ctx.longTimeout()) } // Delete deletes the store. @@ -293,13 +294,50 @@ func (c *KeyValueStoreClient) SetRecordRawWithEncoding(ctx context.Context, key return putRawWithEncoding(ctx, c.ctx, "records/"+safeKey, NewQueryParams(), value, contentType, contentEncoding) } +// SetRecordOptions configures [KeyValueStoreClient.SetRecordRawWithOptions] and +// [KeyValueStoreClient.SetRecordJSONWithOptions]. +type SetRecordOptions struct { + // ContentEncoding, when not empty (e.g. "gzip", "br", "deflate"), says the value is already + // compressed under that encoding: it is uploaded exactly as given, with the Content-Encoding + // header, and the client's own request compression is skipped. See + // [KeyValueStoreClient.SetRecordRawWithEncoding]. + ContentEncoding string + // DoNotRetryTimeouts, if true, makes an upload attempt that times out fail at once instead of + // being retried (other failures are still retried). Use it when storing the record twice + // would be harmful. Matches the reference client's setRecord({ doNotRetryTimeouts }). + DoNotRetryTimeouts bool +} + +// SetRecordRawWithOptions is like [KeyValueStoreClient.SetRecordRaw], with the per-call +// [SetRecordOptions]. +func (c *KeyValueStoreClient) SetRecordRawWithOptions(ctx context.Context, key string, value []byte, contentType string, options SetRecordOptions) error { + safeKey, err := safePathSegment(key) + if err != nil { + return err + } + resource := c.ctx + if options.DoNotRetryTimeouts { + resource = resource.withoutTimeoutRetry() + } + if options.ContentEncoding != "" { + return putRawWithEncoding(ctx, resource, "records/"+safeKey, NewQueryParams(), value, contentType, options.ContentEncoding) + } + return putRaw(ctx, resource, "records/"+safeKey, NewQueryParams(), value, contentType) +} + // SetRecordJSON stores a record holding the JSON serialization of value. func (c *KeyValueStoreClient) SetRecordJSON(ctx context.Context, key string, value any) error { + return c.SetRecordJSONWithOptions(ctx, key, value, SetRecordOptions{}) +} + +// SetRecordJSONWithOptions is like [KeyValueStoreClient.SetRecordJSON], with the per-call +// [SetRecordOptions]. +func (c *KeyValueStoreClient) SetRecordJSONWithOptions(ctx context.Context, key string, value any, options SetRecordOptions) error { data, err := json.Marshal(value) if err != nil { return err } - return c.SetRecordRaw(ctx, key, data, contentTypeJSONCharset) + return c.SetRecordRawWithOptions(ctx, key, data, contentTypeJSONCharset, options) } // DeleteRecord deletes a record by key. @@ -311,6 +349,12 @@ func (c *KeyValueStoreClient) DeleteRecord(ctx context.Context, key string) erro return deleteResourceAlways(ctx, c.ctx, "records/"+safeKey) } +// getForPublicURL fetches the store to read its URL-signing secret, with the "long" timeout tier +// the reference client uses for its public-URL builders. +func (c *KeyValueStoreClient) getForPublicURL(ctx context.Context) (KeyValueStore, bool, error) { + return getResourceWithTimeout[KeyValueStore](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) +} + // GetRecordPublicURL builds a public URL for fetching the given record. // // It mirrors the reference client: it fetches the store, and if the store exposes a @@ -322,7 +366,7 @@ func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string return "", err } params := NewQueryParams() - store, present, err := c.Get(ctx) + store, present, err := c.getForPublicURL(ctx) if err != nil { return "", err } @@ -332,7 +376,7 @@ func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string params.AddString("signature", &sig) } } - return params.applyToURL(c.ctx.publicURL("records/" + safeKey)), nil + return c.ctx.mergedParams(params).applyToURL(c.ctx.publicURL("records/" + safeKey)), nil } // CreateKeysPublicURL builds a public URL for listing this store's keys. @@ -341,7 +385,7 @@ func (c *KeyValueStoreClient) GetRecordPublicURL(ctx context.Context, key string // optionally bounds the validity of a signed URL (nil for non-expiring). func (c *KeyValueStoreClient) CreateKeysPublicURL(ctx context.Context, expiresInSecs *int64) (string, error) { params := NewQueryParams() - store, present, err := c.Get(ctx) + store, present, err := c.getForPublicURL(ctx) if err != nil { return "", err } @@ -351,5 +395,5 @@ func (c *KeyValueStoreClient) CreateKeysPublicURL(ctx context.Context, expiresIn params.AddString("signature", &sig) } } - return params.applyToURL(c.ctx.publicURL("keys")), nil + return c.ctx.mergedParams(params).applyToURL(c.ctx.publicURL("keys")), nil } diff --git a/models.go b/models.go index 3049599..b8d238e 100644 --- a/models.go +++ b/models.go @@ -2,6 +2,7 @@ package apify import ( "encoding/json" + "errors" "io" "time" ) @@ -96,8 +97,13 @@ type ActorRun struct { DefaultKeyValueStoreID string `json:"defaultKeyValueStoreId"` // DefaultRequestQueueID is the ID of the run's default request queue. DefaultRequestQueueID string `json:"defaultRequestQueueId"` - // ContainerURL is the URL of the run's container (for live access). + // ContainerURL is the URL of the run's container (for live access), normalized. ContainerURL string `json:"containerUrl"` + // ConsoleURL is the URL of the run in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` + // Output holds the output links of the run (name to URL), generated from the output schema + // of the Actor build and normalized. Nil if the build defines no output schema. + Output map[string]string `json:"output,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -105,7 +111,14 @@ type ActorRun struct { func (r *ActorRun) UnmarshalJSON(data []byte) error { type alias ActorRun known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(r), known, &r.Extra) + if err := unmarshalWithExtra(data, (*alias)(r), known, &r.Extra); err != nil { + return err + } + return errors.Join( + normalizeURLField("containerUrl", &r.ContainerURL), + normalizeURLField("consoleUrl", &r.ConsoleURL), + normalizeURLMap("output", r.Output), + ) } // IsTerminal reports whether the run has reached a terminal (finished) status. @@ -174,6 +187,9 @@ type Task struct { // PublicConfig is the public-facing display configuration of the task's landing page, set // when the task has been configured for publishing (nil otherwise). PublicConfig *TaskPublicConfig `json:"publicConfig,omitempty"` + // StandbyURL is the URL of the task's Actor Standby endpoint, normalized. Nil when the task + // has none. + StandbyURL *string `json:"standbyUrl,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -181,7 +197,10 @@ type Task struct { func (t *Task) UnmarshalJSON(data []byte) error { type alias Task known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(t), known, &t.Extra) + if err := unmarshalWithExtra(data, (*alias)(t), known, &t.Extra); err != nil { + return err + } + return normalizeOptionalURLField("standbyUrl", &t.StandbyURL) } // TaskPublicConfig is the public-facing display configuration of a task's public landing page. @@ -220,6 +239,11 @@ type Dataset struct { ModifiedAt *time.Time `json:"modifiedAt"` // ItemCount is the number of items currently stored. ItemCount int64 `json:"itemCount"` + // ConsoleURL is the URL of the dataset in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` + // ItemsPublicURL is the public URL of the dataset's items, normalized. It carries a + // signature for a private dataset. + ItemsPublicURL string `json:"itemsPublicUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -227,7 +251,13 @@ type Dataset struct { func (d *Dataset) UnmarshalJSON(data []byte) error { type alias Dataset known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(d), known, &d.Extra) + if err := unmarshalWithExtra(data, (*alias)(d), known, &d.Extra); err != nil { + return err + } + return errors.Join( + normalizeURLField("consoleUrl", &d.ConsoleURL), + normalizeURLField("itemsPublicUrl", &d.ItemsPublicURL), + ) } // KeyValueStore stores arbitrary data records. @@ -242,6 +272,13 @@ type KeyValueStore struct { CreatedAt *time.Time `json:"createdAt"` // ModifiedAt is when the store was last modified. ModifiedAt *time.Time `json:"modifiedAt"` + // ConsoleURL is the URL of the store in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` + // KeysPublicURL is the public URL listing the store's keys, normalized. It carries a + // signature for a private store. + KeysPublicURL string `json:"keysPublicUrl"` + // RecordsPublicURL is the public URL prefix of the store's records, normalized. + RecordsPublicURL string `json:"recordsPublicUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -249,7 +286,14 @@ type KeyValueStore struct { func (k *KeyValueStore) UnmarshalJSON(data []byte) error { type alias KeyValueStore known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(k), known, &k.Extra) + if err := unmarshalWithExtra(data, (*alias)(k), known, &k.Extra); err != nil { + return err + } + return errors.Join( + normalizeURLField("consoleUrl", &k.ConsoleURL), + normalizeURLField("keysPublicUrl", &k.KeysPublicURL), + normalizeURLField("recordsPublicUrl", &k.RecordsPublicURL), + ) } // KeyValueStoreKey is a single key listed from a key-value store. @@ -258,6 +302,9 @@ type KeyValueStoreKey struct { Key string `json:"key"` // Size is the record size in bytes. Size int64 `json:"size"` + // RecordPublicURL is the public URL of the record, normalized. It carries a signature for a + // private store. + RecordPublicURL string `json:"recordPublicUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -265,7 +312,10 @@ type KeyValueStoreKey struct { func (k *KeyValueStoreKey) UnmarshalJSON(data []byte) error { type alias KeyValueStoreKey known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(k), known, &k.Extra) + if err := unmarshalWithExtra(data, (*alias)(k), known, &k.Extra); err != nil { + return err + } + return normalizeURLField("recordPublicUrl", &k.RecordPublicURL) } // KeyValueStoreKeysPage is a page of keys from a key-value store. @@ -319,6 +369,8 @@ type RequestQueue struct { ModifiedAt *time.Time `json:"modifiedAt"` // TotalRequestCount is the total number of requests ever added. TotalRequestCount int64 `json:"totalRequestCount"` + // ConsoleURL is the URL of the queue in the Apify Console, normalized. + ConsoleURL string `json:"consoleUrl"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -326,7 +378,10 @@ type RequestQueue struct { func (q *RequestQueue) UnmarshalJSON(data []byte) error { type alias RequestQueue known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(q), known, &q.Extra) + if err := unmarshalWithExtra(data, (*alias)(q), known, &q.Extra); err != nil { + return err + } + return normalizeURLField("consoleUrl", &q.ConsoleURL) } // RequestQueueRequest is a single request stored in a request queue. @@ -418,7 +473,8 @@ type Webhook struct { ID string `json:"id"` // UserID is the ID of the user who owns the webhook. UserID string `json:"userId"` - // RequestURL is the URL the webhook posts to. + // RequestURL is the URL the webhook posts to, normalized. Empty for a webhook that is not + // an HTTP one (the API returns null). RequestURL string `json:"requestUrl"` // EventTypes are the events that trigger the webhook. EventTypes []string `json:"eventTypes"` @@ -429,7 +485,10 @@ type Webhook struct { func (w *Webhook) UnmarshalJSON(data []byte) error { type alias Webhook known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(w), known, &w.Extra) + if err := unmarshalWithExtra(data, (*alias)(w), known, &w.Extra); err != nil { + return err + } + return normalizeURLField("requestUrl", &w.RequestURL) } // WebhookDispatch is a single invocation of a webhook. @@ -438,6 +497,9 @@ type WebhookDispatch struct { ID string `json:"id"` // WebhookID is the ID of the webhook that produced this dispatch. WebhookID string `json:"webhookId"` + // Webhook is a summary of the webhook as it was when the dispatch was created. Nil if the + // API does not return it. + Webhook *WebhookDispatchWebhookSummary `json:"webhook,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -448,12 +510,37 @@ func (d *WebhookDispatch) UnmarshalJSON(data []byte) error { return unmarshalWithExtra(data, (*alias)(d), known, &d.Extra) } +// WebhookDispatchWebhookSummary is the summary of a webhook embedded in a [WebhookDispatch]. +type WebhookDispatchWebhookSummary struct { + // ID is the ID of the webhook (the API calls the field `_id`). + ID string `json:"_id"` + // ActionType is the kind of action the webhook runs, e.g. HTTP request or a Slack message. + ActionType string `json:"actionType"` + // IsAdHoc reports whether the webhook was attached to a single run instead of being saved. + IsAdHoc bool `json:"isAdHoc"` + // RequestURL is the URL the webhook posts to, normalized. Empty when there is none. + RequestURL string `json:"requestUrl"` + // Extra holds any other fields returned by the API. + Extra Extra `json:"-"` +} + +func (w *WebhookDispatchWebhookSummary) UnmarshalJSON(data []byte) error { + type alias WebhookDispatchWebhookSummary + known := knownJSONKeys(alias{}) + if err := unmarshalWithExtra(data, (*alias)(w), known, &w.Extra); err != nil { + return err + } + return normalizeURLField("requestUrl", &w.RequestURL) +} + // User is an Apify user account. type User struct { // ID is the unique user ID. ID string `json:"id"` // Username is the user's username. Username string `json:"username"` + // Profile is the user's public profile. Nil if the API does not return it. + Profile *UserProfile `json:"profile,omitempty"` // Extra holds any other fields returned by the API (private details for "me"). Extra Extra `json:"-"` } @@ -464,6 +551,30 @@ func (u *User) UnmarshalJSON(data []byte) error { return unmarshalWithExtra(data, (*alias)(u), known, &u.Extra) } +// UserProfile is the public profile of a [User]. +type UserProfile struct { + // Name is the user's display name. + Name string `json:"name"` + // PictureURL is the URL of the user's profile picture, normalized. Nil when there is none. + PictureURL *string `json:"pictureUrl,omitempty"` + // WebsiteURL is the URL of the user's website, normalized. Nil when there is none. + WebsiteURL *string `json:"websiteUrl,omitempty"` + // Extra holds any other fields returned by the API. + Extra Extra `json:"-"` +} + +func (p *UserProfile) UnmarshalJSON(data []byte) error { + type alias UserProfile + known := knownJSONKeys(alias{}) + if err := unmarshalWithExtra(data, (*alias)(p), known, &p.Extra); err != nil { + return err + } + return errors.Join( + normalizeOptionalURLField("pictureUrl", &p.PictureURL), + normalizeOptionalURLField("websiteUrl", &p.WebsiteURL), + ) +} + // ActorStoreListItem is an Actor as listed in the Apify Store. type ActorStoreListItem struct { // ID is the unique Actor ID. @@ -474,6 +585,11 @@ type ActorStoreListItem struct { Username string `json:"username"` // Title is the human-readable title. Title string `json:"title"` + // URL is the URL of the Actor's page in the Apify Store, normalized. Nil when there is none. + URL *string `json:"url,omitempty"` + // UserPictureURL is the URL of the owner's profile picture, normalized. Nil when there is + // none. + UserPictureURL *string `json:"userPictureUrl,omitempty"` // Extra holds any other fields returned by the API. Extra Extra `json:"-"` } @@ -481,7 +597,13 @@ type ActorStoreListItem struct { func (a *ActorStoreListItem) UnmarshalJSON(data []byte) error { type alias ActorStoreListItem known := knownJSONKeys(alias{}) - return unmarshalWithExtra(data, (*alias)(a), known, &a.Extra) + if err := unmarshalWithExtra(data, (*alias)(a), known, &a.Extra); err != nil { + return err + } + return errors.Join( + normalizeOptionalURLField("url", &a.URL), + normalizeOptionalURLField("userPictureUrl", &a.UserPictureURL), + ) } // ActorVersion is a single version of an Actor. diff --git a/review_fixes_test.go b/review_fixes_test.go index a290bfd..9981a0d 100644 --- a/review_fixes_test.go +++ b/review_fixes_test.go @@ -458,8 +458,26 @@ func TestMonthlyUsageWithOptionsDate(t *testing.T) { if strings.Contains(backend.lastURL, "date=") { t.Fatalf("date must be omitted when empty, got %q", backend.lastURL) } - if _, err := client.User("someone").MonthlyUsageWithOptions(context.Background(), MonthlyUsageOptions{}); err == nil { - t.Fatal("expected an error for a non-me client") +} + +// The usage and limits endpoints are documented for a user by ID or username as well, so a +// client from User(id) calls them instead of failing. +func TestUserLimitsAndUsageByID(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + ctx := context.Background() + + if _, err := client.User("someone").MonthlyUsageWithOptions(ctx, MonthlyUsageOptions{Date: "2026-09-14"}); err != nil { + t.Fatalf("monthly usage by ID: %v", err) + } + if !strings.Contains(backend.lastURL, "/users/someone/usage/monthly") || !strings.Contains(backend.lastURL, "date=2026-09-14") { + t.Fatalf("unexpected monthly usage URL %q", backend.lastURL) + } + if _, err := client.User("someone").Limits(ctx); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { + t.Fatalf("limits by ID: %v, URL %q", err, backend.lastURL) + } + if err := client.User("someone").UpdateLimits(ctx, map[string]any{"maxMonthlyUsageUsd": 5}); err != nil || !strings.HasSuffix(backend.lastURL, "/users/someone/limits") { + t.Fatalf("update limits by ID: %v, URL %q", err, backend.lastURL) } } diff --git a/run.go b/run.go index 4079067..009fdb4 100644 --- a/run.go +++ b/run.go @@ -198,6 +198,9 @@ func (c *RunClient) Resurrect(ctx context.Context, options RunResurrectOptions) // retried by the transport is applied at most once, matching the reference client. The API // requires this header and expires the key 3 minutes after the charge. func (c *RunClient) Charge(ctx context.Context, options RunChargeOptions) error { + if c.ctx.idErr != nil { + return c.ctx.idErr + } count := int64(1) if options.Count != nil { count = *options.Count @@ -238,24 +241,33 @@ func (c *RunClient) WaitForFinish(ctx context.Context, waitSecs *int64) (ActorRu return waitForFinish[ActorRun](ctx, c.ctx, waitSecs, "run", func(r *ActorRun) bool { return r.IsTerminal() }) } -// Dataset returns a client for this run's default dataset. +// Dataset returns a client for this run's default dataset. On a client from LastRun, it carries +// the same status and origin filters, so it addresses the dataset of the same run. func (c *RunClient) Dataset() *DatasetClient { - return newDatasetNestedClient(c.ctx.http, c.ctx.subURL(""), "dataset") + client := newDatasetNestedClient(c.ctx.http, c.ctx.subURL(""), "dataset") + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // KeyValueStore returns a client for this run's default key-value store. func (c *RunClient) KeyValueStore() *KeyValueStoreClient { - return newKeyValueStoreNestedClient(c.ctx.http, c.ctx.subURL(""), "key-value-store") + client := newKeyValueStoreNestedClient(c.ctx.http, c.ctx.subURL(""), "key-value-store") + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // RequestQueue returns a client for this run's default request queue. func (c *RunClient) RequestQueue() *RequestQueueClient { - return newRequestQueueNestedClient(c.ctx.http, c.ctx.subURL(""), "request-queue") + client := newRequestQueueNestedClient(c.ctx.http, c.ctx.subURL(""), "request-queue") + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // Log returns a client for accessing this run's log. func (c *RunClient) Log() *LogClient { - return newNestedLogClient(c.ctx.http, c.ctx.subURL("")) + client := newNestedLogClient(c.ctx.http, c.ctx.subURL("")) + client.ctx.baseParams = c.ctx.baseParams.clone() + return client } // GetStreamedLog opens a live stream of this run's raw log, for convenient log redirection. diff --git a/run_collection.go b/run_collection.go index a801c9f..38a7fa9 100644 --- a/run_collection.go +++ b/run_collection.go @@ -4,7 +4,8 @@ import "context" // RunListOptions adds run-specific filters on top of [ListOptions] for // [RunCollectionClient.List]. The startedAfter/startedBefore filters are only honoured by -// the Actor-scoped and task-scoped run collections. +// the Actor-scoped and task-scoped run collections, and the deployment filter only by the +// account-wide and Actor-scoped ones. type RunListOptions struct { // Status filters by one or more run statuses (e.g. "SUCCEEDED", "RUNNING"). Sent as a // comma-separated list, as the API accepts. @@ -13,12 +14,18 @@ type RunListOptions struct { StartedAfter *string // StartedBefore filters to runs started before this ISO-8601 timestamp. StartedBefore *string + // Deployment filters Actor Standby runs by the pool that serves them, as a list of + // SINGLE_TENANT, MULTI_TENANT_FREE_USERS, MULTI_TENANT_PAYING_USERS or MULTI_TENANT_OWNER_PAID. + // Sent as a comma-separated list. The API requires Status to hold only non-terminal statuses + // (READY, RUNNING, TIMING-OUT or ABORTING) when it is set. + Deployment []string } func (o RunListOptions) apply(q *QueryParams) { q.AddCSV("status", o.Status). AddString("startedAfter", o.StartedAfter). - AddString("startedBefore", o.StartedBefore) + AddString("startedBefore", o.StartedBefore). + AddCSV("deployment", o.Deployment) } // RunCollectionClient is a client for a run collection: the account-wide collection diff --git a/run_dataset_items.go b/run_dataset_items.go new file mode 100644 index 0000000..2db48ca --- /dev/null +++ b/run_dataset_items.go @@ -0,0 +1,278 @@ +package apify + +import ( + "context" + "encoding/json" + "errors" +) + +const ( + // defaultRunIterateChunkSize is the number of dataset rows requested per API call when + // [RunIterateDatasetItemsOptions.ChunkSize] is unset. Matches the reference client's + // DEFAULT_ITERATE_CHUNK_SIZE. + defaultRunIterateChunkSize = int64(1000) + // defaultRunIteratePollIntervalSecs is how long the iterator waits for the run to finish + // between polls when [RunIterateDatasetItemsOptions.PollIntervalSecs] is unset. + defaultRunIteratePollIntervalSecs = int64(5) +) + +// errRunIterateDesc is returned when the items options ask for newest-first order. +var errRunIterateDesc = errors.New("RunIterateDatasetItemsOptions: Items.Desc is not supported: the iterator reads rows by offset from the oldest, which a descending order would count from the newest") + +// RunIterateDatasetItemsOptions configures [RunClient.IterateDatasetItems] and +// [IterateRunDatasetItems]. +type RunIterateDatasetItemsOptions struct { + // Items are the filters and projections applied to every page, the same as for + // [DatasetClient.ListItems]. Items.Offset is the number of rows to skip from the beginning + // (default 0), and Items.Limit the maximum number of dataset rows to scan (default no limit): + // fewer items are yielded when filters drop some, more when Unwind splits a row into several. + // Items.Desc is not supported, because the rows are read by offset from the oldest. + Items DatasetListItemsOptions + // ChunkSize is the maximum number of dataset rows requested per API call (default 1000). + ChunkSize *int64 + // PollIntervalSecs is how long to wait for the run to finish between polls, in seconds + // (default 5). + PollIntervalSecs *int64 +} + +// runIteratePhase is the stage a [RunDatasetItemsIterator] is in. +type runIteratePhase int + +const ( + // runIterateStart: nothing has been requested yet. + runIterateStart runIteratePhase = iota + // runIterateLive: the run may still be producing items. Each round reads the rows below the + // dataset's itemCount, then waits for the run to finish. + runIterateLive + // runIterateTail: the run has finished. The rows past itemCount (which lags a few seconds + // behind the pushed items) are read a page at a time until none are left. + runIterateTail + // runIterateDone: the iterator is exhausted. + runIterateDone +) + +// RunDatasetItemsIterator iterates over the items of a run's default dataset while the run is +// still producing them. Obtain one from [RunClient.IterateDatasetItems] or +// [IterateRunDatasetItems] and drain it by calling Next until it returns (nil, nil). +// +// While the run has not finished, each round yields the rows below the dataset's itemCount and +// then waits up to PollIntervalSecs for the run to finish, so the last rows are read as soon as it +// does. Each page is requested with a limit that ends at itemCount, so it covers exactly the +// rows it asks for, whatever the filters or Unwind do to the items. itemCount lags a few seconds +// behind the pushed items, so once the run reaches a terminal status the rows past it are read a +// page at a time until none are left. On a client from LastRun, the iterator sticks to the run +// its first request resolves to. Matches the reference client's iterateDatasetItems(). +type RunDatasetItemsIterator[T any] struct { + run *RunClient + options RunIterateDatasetItemsOptions + + pageSize int64 + pollSecs int64 + position int64 + end int64 + hasEnd bool + + phase runIteratePhase + dataset *DatasetClient + runClient *RunClient + currentRun ActorRun + runPresent bool + // roundFinished is whether the run was already finished when the current live round began. + roundFinished bool + // roundItemCount is the number of rows the current live round reads up to. + roundItemCount int64 + roundStarted bool + + buffer []T + pos int +} + +// IterateRunDatasetItems returns an iterator over the items of the run's default dataset, each +// decoded into T, that follows the dataset while the run is still producing items. See +// [RunDatasetItemsIterator]. +func IterateRunDatasetItems[T any](c *RunClient, options RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[T] { + it := &RunDatasetItemsIterator[T]{ + run: c, + options: options, + pageSize: defaultRunIterateChunkSize, + pollSecs: defaultRunIteratePollIntervalSecs, + position: offsetVal(options.Items.Offset), + } + if options.ChunkSize != nil && *options.ChunkSize > 0 { + it.pageSize = *options.ChunkSize + } + if options.PollIntervalSecs != nil && *options.PollIntervalSecs >= 0 { + it.pollSecs = *options.PollIntervalSecs + } + if options.Items.Limit != nil && *options.Items.Limit > 0 { + it.hasEnd = true + it.end = it.position + *options.Items.Limit + } + return it +} + +// IterateDatasetItems is [IterateRunDatasetItems] with every item decoded into a +// json.RawMessage. See [RunDatasetItemsIterator]. +func (c *RunClient) IterateDatasetItems(options RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[json.RawMessage] { + return IterateRunDatasetItems[json.RawMessage](c, options) +} + +// Next returns the next item, or (nil, nil) once every item has been read (or the Limit of +// scanned rows is reached). It calls the API only when the current in-memory page is used up. +func (it *RunDatasetItemsIterator[T]) Next(ctx context.Context) (*T, error) { + for it.pos >= len(it.buffer) { + if it.phase == runIterateDone { + return nil, nil + } + if err := it.advance(ctx); err != nil { + return nil, err + } + } + item := it.buffer[it.pos] + it.pos++ + return &item, nil +} + +// advance performs the next step of the iteration, which loads the next page into the buffer, or +// moves to the next phase. +func (it *RunDatasetItemsIterator[T]) advance(ctx context.Context) error { + it.buffer, it.pos = nil, 0 + switch it.phase { + case runIterateStart: + return it.start(ctx) + case runIterateLive: + return it.liveStep(ctx) + case runIterateTail: + return it.tailStep(ctx) + default: + return nil + } +} + +// start resolves the run and the client of its default dataset. +func (it *RunDatasetItemsIterator[T]) start(ctx context.Context) error { + if it.options.Items.Desc != nil && *it.options.Items.Desc { + return errRunIterateDesc + } + it.runClient = it.run + run, present, err := getResourceWithTimeout[ActorRun](ctx, it.run.ctx, "", NewQueryParams(), it.run.ctx.longTimeout()) + if err != nil { + return err + } + it.currentRun, it.runPresent = run, present + // A LastRun client resolves runs/last on each request, so a newer run would swap the dataset + // mid-iteration: pin the iteration to the run the first request resolved to. + if present && run.ID != it.run.id { + it.runClient = it.run.root.Run(run.ID) + } + it.dataset = it.runClient.Dataset() + it.phase = runIterateLive + return nil +} + +// liveStep reads one page of the current live round, or ends the round: it waits for the run to +// finish and starts another one, or moves on to the tail. +func (it *RunDatasetItemsIterator[T]) liveStep(ctx context.Context) error { + if !it.roundStarted { + it.roundFinished = !it.runPresent || it.currentRun.IsTerminal() + dataset, present, err := getResourceWithTimeout[Dataset](ctx, it.dataset.ctx, "", NewQueryParams(), it.dataset.ctx.longTimeout()) + if err != nil { + return err + } + it.roundItemCount = 0 + if present { + it.roundItemCount = dataset.ItemCount + } + if it.hasEnd { + it.roundItemCount = minInt64(it.roundItemCount, it.end) + } + it.roundStarted = true + } + + if it.position < it.roundItemCount { + pageLimit := minInt64(it.pageSize, it.roundItemCount-it.position) + items, _, err := it.listPage(ctx, it.position, pageLimit) + if err != nil { + return err + } + it.buffer = items + it.position += pageLimit + return nil + } + + it.roundStarted = false + if it.hasEnd && it.position >= it.end { + it.phase = runIterateDone + return nil + } + if it.roundFinished { + it.phase = runIterateTail + return nil + } + run, err := it.runClient.WaitForFinish(ctx, &it.pollSecs) + if err != nil { + return err + } + it.currentRun, it.runPresent = run, true + return nil +} + +// tailStep reads one page past itemCount after the run has finished. Only an empty page ends the +// iteration, since filters can shorten a full one. +func (it *RunDatasetItemsIterator[T]) tailStep(ctx context.Context) error { + pageLimit := it.pageSize + if it.hasEnd { + pageLimit = minInt64(it.pageSize, it.end-it.position) + } + items, scanned, err := it.listPage(ctx, it.position, pageLimit) + if err != nil { + return err + } + it.buffer = items + + // A page that Clean, SkipEmpty or Unwind emptied past a lagging itemCount reports no scanned + // rows either, so a plain read confirms the dataset really has no more rows. + if len(items) == 0 && scanned == 0 { + confirmed, err := it.confirmNoMoreRows(ctx) + if err != nil { + return err + } + if confirmed { + it.phase = runIterateDone + return nil + } + } + it.position += pageLimit + if it.hasEnd && it.position >= it.end { + it.phase = runIterateDone + } + return nil +} + +// confirmNoMoreRows reports whether the dataset has no row at the current position. Without a +// filter that can empty a page, an empty page already means that. +func (it *RunDatasetItemsIterator[T]) confirmNoMoreRows(ctx context.Context) (bool, error) { + items := it.options.Items + filtered := (items.Clean != nil && *items.Clean) || (items.SkipEmpty != nil && *items.SkipEmpty) || len(items.Unwind) > 0 + if !filtered { + return true, nil + } + plain, err := ListDatasetItems[json.RawMessage](ctx, it.dataset, DatasetListItemsOptions{Offset: &it.position, Limit: Ptr(int64(1))}) + if err != nil { + return false, err + } + return len(plain.Items) == 0, nil +} + +// listPage reads the page of the dataset that starts at offset and holds at most limit rows, +// returning its items and the number of rows the API scanned for it. +func (it *RunDatasetItemsIterator[T]) listPage(ctx context.Context, offset, limit int64) ([]T, int64, error) { + options := it.options.Items + options.Offset = &offset + options.Limit = &limit + page, err := ListDatasetItems[T](ctx, it.dataset, options) + if err != nil { + return nil, 0, err + } + return page.Items, page.scanned, nil +} diff --git a/run_dataset_items_test.go b/run_dataset_items_test.go new file mode 100644 index 0000000..74b6ba5 --- /dev/null +++ b/run_dataset_items_test.go @@ -0,0 +1,233 @@ +package apify + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "testing" +) + +// fakeRunBackend simulates a run that keeps pushing items to its default dataset. Each answer to +// a run poll (a GET of the run or a waitForFinish) moves the simulation one step: runSteps[i] is +// the run status and itemCounts[i] the dataset's itemCount after it, and every item row exists +// from the start of the test, so a page past itemCount still returns rows, like the lagging +// itemCount of the real API. +type fakeRunBackend struct { + mu sync.Mutex + runID string + runSteps []string + itemCounts []int64 + totalRows int + step int + requests []string +} + +func (b *fakeRunBackend) Do(req *http.Request) (*http.Response, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.requests = append(b.requests, req.URL.RequestURI()) + path := req.URL.Path + query := req.URL.Query() + respond := func(body string, header http.Header) (*http.Response, error) { + if header == nil { + header = http.Header{} + } + return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(bytes.NewReader([]byte(body)))}, nil + } + current := min(b.step, len(b.runSteps)-1) + switch { + case strings.HasSuffix(path, "/dataset/items"): + offset, _ := strconv.Atoi(query.Get("offset")) + limit, _ := strconv.Atoi(query.Get("limit")) + var rows []map[string]int + for i := offset; i < offset+limit && i < b.totalRows; i++ { + rows = append(rows, map[string]int{"n": i}) + } + body, _ := json.Marshal(rows) + if rows == nil { + body = []byte("[]") + } + header := http.Header{} + header.Set("X-Apify-Pagination-Count", strconv.Itoa(len(rows))) + return respond(string(body), header) + case strings.HasSuffix(path, "/dataset"): + return respond(fmt.Sprintf(`{"data":{"id":"d","itemCount":%d}}`, b.itemCounts[current]), nil) + case strings.HasSuffix(path, "/actor-runs/"+b.runID), strings.HasSuffix(path, "/runs/last"): + if query.Get("waitForFinish") != "" { + b.step++ + current = min(b.step, len(b.runSteps)-1) + } + return respond(fmt.Sprintf(`{"data":{"id":%q,"status":%q}}`, b.runID, b.runSteps[current]), nil) + } + return nil, fmt.Errorf("unexpected request %s", req.URL) +} + +func drainRunItems(t *testing.T, it *RunDatasetItemsIterator[map[string]int]) []int { + t.Helper() + var got []int + for { + item, err := it.Next(context.Background()) + if err != nil { + t.Fatalf("Next: %v", err) + } + if item == nil { + return got + } + got = append(got, (*item)["n"]) + } +} + +func sequence(from, to int) []int { + var out []int + for i := from; i < to; i++ { + out = append(out, i) + } + return out +} + +func equalInts(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestIterateDatasetItemsReadsAFinishedRun(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{5}, totalRows: 5} + client := testClient(backend, 0) + got := drainRunItems(t, IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{})) + if !equalInts(got, sequence(0, 5)) { + t.Fatalf("got %v", got) + } +} + +// While the run is running, the iterator reads up to itemCount, waits for the run to finish, and +// then reads the rows past a stale itemCount until a page is empty. +func TestIterateDatasetItemsFollowsARunningRun(t *testing.T) { + backend := &fakeRunBackend{ + runID: "run1", + runSteps: []string{"RUNNING", "RUNNING", "SUCCEEDED"}, + itemCounts: []int64{2, 4, 4}, + totalRows: 7, // itemCount stays at 4 although 7 rows exist: the tail catches up + } + client := testClient(backend, 0) + it := IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{ + ChunkSize: Ptr(int64(3)), + PollIntervalSecs: Ptr(int64(1)), + }) + got := drainRunItems(t, it) + if !equalInts(got, sequence(0, 7)) { + t.Fatalf("got %v", got) + } + polls := 0 + for _, r := range backend.requests { + if strings.Contains(r, "/actor-runs/run1?waitForFinish=") { + polls++ + } + } + if polls == 0 { + t.Fatalf("expected the iterator to wait for the run to finish, got %v", backend.requests) + } + // The first round stops at the itemCount of 2 although 3 rows fit in a chunk. + if !strings.Contains(backend.requests[2], "offset=0&limit=2") { + t.Fatalf("the first page must end at itemCount: %v", backend.requests) + } + for _, r := range backend.requests { + if strings.Contains(r, "/dataset/items") && strings.Contains(r, "limit=") { + query, _ := url.ParseQuery(strings.SplitN(r, "?", 2)[1]) + if limit, _ := strconv.Atoi(query.Get("limit")); limit > 3 { + t.Fatalf("a page asked for %d rows, more than the chunk size 3: %s", limit, r) + } + } + } +} + +func TestIterateDatasetItemsHonorsOffsetAndLimit(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{10}, totalRows: 10} + client := testClient(backend, 0) + it := IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{ + Items: DatasetListItemsOptions{Offset: Ptr(int64(2)), Limit: Ptr(int64(5))}, + ChunkSize: Ptr(int64(2)), + }) + if got := drainRunItems(t, it); !equalInts(got, sequence(2, 7)) { + t.Fatalf("got %v, want rows 2..6", got) + } +} + +// The tail reads past itemCount even when the run is finished before the first request, up to the +// Limit. +func TestIterateDatasetItemsLimitEndsTheTail(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{0}, totalRows: 10} + client := testClient(backend, 0) + it := IterateRunDatasetItems[map[string]int](client.Run("run1"), RunIterateDatasetItemsOptions{ + Items: DatasetListItemsOptions{Limit: Ptr(int64(4))}, + ChunkSize: Ptr(int64(3)), + }) + if got := drainRunItems(t, it); !equalInts(got, sequence(0, 4)) { + t.Fatalf("got %v, want rows 0..3", got) + } +} + +// A LastRun client resolves runs/last on each request: the iteration pins itself to the run its +// first request resolved to. +func TestIterateDatasetItemsPinsALastRunClient(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{3}, totalRows: 3} + client := testClient(backend, 0) + got := drainRunItems(t, IterateRunDatasetItems[map[string]int](client.Actor("a").LastRun("SUCCEEDED"), RunIterateDatasetItemsOptions{})) + if !equalInts(got, sequence(0, 3)) { + t.Fatalf("got %v", got) + } + for _, r := range backend.requests[1:] { + if strings.Contains(r, "/runs/last") { + t.Fatalf("after resolving the run, the requests must name it, not runs/last: %s", r) + } + } + if !strings.Contains(backend.requests[0], "/runs/last") || !strings.Contains(backend.requests[0], "status=SUCCEEDED") { + t.Fatalf("the first request resolves the last run: %s", backend.requests[0]) + } +} + +func TestIterateDatasetItemsRejectsDescendingOrder(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{3}, totalRows: 3} + client := testClient(backend, 0) + it := client.Run("run1").IterateDatasetItems(RunIterateDatasetItemsOptions{Items: DatasetListItemsOptions{Desc: Ptr(true)}}) + for range 2 { + if _, err := it.Next(context.Background()); err == nil { + t.Fatal("expected an error for a descending order") + } + } + if len(backend.requests) != 0 { + t.Fatalf("no request may be sent, got %v", backend.requests) + } +} + +func TestIterateDatasetItemsDecodesIntoRawMessages(t *testing.T) { + backend := &fakeRunBackend{runID: "run1", runSteps: []string{"SUCCEEDED"}, itemCounts: []int64{2}, totalRows: 2} + it := testClient(backend, 0).Run("run1").IterateDatasetItems(RunIterateDatasetItemsOptions{}) + var raws []string + for { + item, err := it.Next(context.Background()) + if err != nil { + t.Fatal(err) + } + if item == nil { + break + } + raws = append(raws, string(*item)) + } + if strings.Join(raws, ",") != `{"n":0},{"n":1}` { + t.Fatalf("got %v", raws) + } +} diff --git a/spec_sync_test.go b/spec_sync_test.go new file mode 100644 index 0000000..f124038 --- /dev/null +++ b/spec_sync_test.go @@ -0,0 +1,169 @@ +package apify + +import ( + "context" + "net/url" + "strings" + "testing" +) + +func TestRunListsSendTheDeploymentFilter(t *testing.T) { + ctx := context.Background() + filter := RunListOptions{Status: []string{"RUNNING"}, Deployment: []string{"SINGLE_TENANT", "MULTI_TENANT_FREE_USERS"}} + + for name, list := range map[string]func(client *ApifyClient) error{ + "account runs": func(c *ApifyClient) error { _, err := c.Runs().List(ctx, ListOptions{}, filter); return err }, + "actor runs": func(c *ApifyClient) error { _, err := c.Actor("a").Runs().List(ctx, ListOptions{}, filter); return err }, + } { + backend := &mockBackend{responses: okData()} + if err := list(testClient(backend, 0)); err != nil { + t.Fatalf("%s: %v", name, err) + } + query, _ := url.ParseQuery(strings.SplitN(backend.lastURL, "?", 2)[1]) + if got := query.Get("deployment"); got != "SINGLE_TENANT,MULTI_TENANT_FREE_USERS" { + t.Errorf("%s: deployment = %q", name, got) + } + } + + backend := &mockBackend{responses: okData()} + if _, err := testClient(backend, 0).Runs().List(ctx, ListOptions{}, RunListOptions{}); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "deployment") { + t.Errorf("deployment must be omitted when unset, got %q", backend.lastURL) + } +} + +func TestStoreListSendsIncludeInputSchema(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + if _, err := client.Store().List(context.Background(), StoreListOptions{IncludeInputSchema: Ptr(true)}); err != nil { + t.Fatal(err) + } + if !strings.Contains(backend.lastURL, "includeInputSchema=1") { + t.Errorf("expected includeInputSchema=1, got %q", backend.lastURL) + } + if _, err := client.Store().List(context.Background(), StoreListOptions{}); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "includeInputSchema") { + t.Errorf("includeInputSchema must be omitted when unset, got %q", backend.lastURL) + } +} + +// The storages and the log of a run reached through LastRun carry its status and origin filters, +// so they address the storages and the log of the same run, not those of the newest run of any +// status. +func TestStoragesAndLogOfAFilteredLastRunCarryItsFilters(t *testing.T) { + ctx := context.Background() + lastRun := func(client *ApifyClient) *RunClient { + return client.Actor("a").LastRunWithOptions(LastRunOptions{Status: "SUCCEEDED", Origin: "API"}) + } + taskLastRun := func(client *ApifyClient) *RunClient { + return client.Task("t").LastRunWithOptions(LastRunOptions{Status: "SUCCEEDED", Origin: "API"}) + } + + cases := map[string]func(client *ApifyClient) error{ + "dataset get": func(c *ApifyClient) error { _, _, err := lastRun(c).Dataset().Get(ctx); return err }, + "dataset items": func(c *ApifyClient) error { + _, err := lastRun(c).Dataset().ListItems(ctx, DatasetListItemsOptions{}) + return err + }, + "dataset download": func(c *ApifyClient) error { + _, err := lastRun(c).Dataset().DownloadItems(ctx, "json", DatasetDownloadOptions{}) + return err + }, + "dataset push": func(c *ApifyClient) error { + return lastRun(c).Dataset().PushItems(ctx, map[string]any{"a": 1}) + }, + "dataset public URL": func(c *ApifyClient) error { + _, err := lastRun(c).Dataset().CreateItemsPublicURL(ctx, DatasetListItemsOptions{}, nil) + return err + }, + "store get": func(c *ApifyClient) error { _, _, err := lastRun(c).KeyValueStore().Get(ctx); return err }, + "store keys": func(c *ApifyClient) error { + _, err := lastRun(c).KeyValueStore().ListKeys(ctx, ListKeysOptions{}) + return err + }, + "store record": func(c *ApifyClient) error { + _, _, err := lastRun(c).KeyValueStore().GetRecord(ctx, "OUTPUT") + return err + }, + "store record public URL": func(c *ApifyClient) error { + _, err := lastRun(c).KeyValueStore().GetRecordPublicURL(ctx, "OUTPUT") + return err + }, + "store keys public URL": func(c *ApifyClient) error { + _, err := lastRun(c).KeyValueStore().CreateKeysPublicURL(ctx, nil) + return err + }, + "queue get": func(c *ApifyClient) error { _, _, err := lastRun(c).RequestQueue().Get(ctx); return err }, + "queue head": func(c *ApifyClient) error { _, err := lastRun(c).RequestQueue().ListHead(ctx, nil); return err }, + "log get": func(c *ApifyClient) error { _, _, err := lastRun(c).Log().Get(ctx); return err }, + "log stream": func(c *ApifyClient) error { + stream, err := lastRun(c).GetStreamedLog(ctx) + if err == nil { + _ = stream.Close() + } + return err + }, + "task last run dataset": func(c *ApifyClient) error { _, _, err := taskLastRun(c).Dataset().Get(ctx); return err }, + "task last run log": func(c *ApifyClient) error { _, _, err := taskLastRun(c).Log().Get(ctx); return err }, + } + for name, call := range cases { + t.Run(name, func(t *testing.T) { + body := `{"data":{"id":"x","urlSigningSecretKey":"s"}}` + if name == "dataset items" { + body = `[]` // the items endpoint answers with a bare array + } + backend := &mockBackend{responses: []mockResponse{{status: 200, body: body}}} + if err := call(testClient(backend, 0)); err != nil { + t.Fatalf("call failed: %v", err) + } + if !strings.Contains(backend.lastURL, "/runs/last/") { + t.Fatalf("expected a runs/last URL, got %q", backend.lastURL) + } + for _, want := range []string{"status=SUCCEEDED", "origin=API"} { + if !strings.Contains(backend.lastURL, want) { + t.Errorf("%s is missing from %q", want, backend.lastURL) + } + } + }) + } +} + +// A storage reached through a run that has no filters does not gain any. +func TestStoragesOfAnUnfilteredRunHaveNoFilterParams(t *testing.T) { + backend := &mockBackend{responses: okData()} + if _, _, err := testClient(backend, 0).Run("r").Dataset().Get(context.Background()); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "?") { + t.Errorf("unexpected query string in %q", backend.lastURL) + } +} + +// The API returns a pending request with a null handledAt, and a schedule whose Console editor +// state was dropped with a null uiState: both parse, the null kept in Extra. +func TestNullFieldsOfRequestsAndSchedulesParse(t *testing.T) { + var request RequestQueueRequest + mustUnmarshal(t, `{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}`, &request) + if request.ID != "r1" || request.UniqueKey != "k" || string(request.Extra["handledAt"]) != "null" { + t.Errorf("unexpected request %+v", request) + } + if request.Method != "" || request.UserData != nil { + t.Errorf("the optional fields of a minimal request stay empty: %+v", request) + } + + var schedule Schedule + mustUnmarshal(t, `{"id":"s1","userId":"u","name":"n","cronExpression":"* * * * *","isEnabled":true,"uiState":null}`, &schedule) + if schedule.ID != "s1" || string(schedule.Extra["uiState"]) != "null" { + t.Errorf("unexpected schedule %+v", schedule) + } + + backend := &mockBackend{responses: constant(200, `{"data":{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}}`)} + got, present, err := testClient(backend, 0).RequestQueue("q").GetRequest(context.Background(), "r1") + if err != nil || !present || got.ID != "r1" { + t.Fatalf("GetRequest of a pending request: %+v, %v, %v", got, present, err) + } +} diff --git a/store_collection.go b/store_collection.go index 04bf7e2..06a6063 100644 --- a/store_collection.go +++ b/store_collection.go @@ -24,6 +24,10 @@ type StoreListOptions struct { AllowsAgenticUsers *bool // ResponseFormat selects the response format. ResponseFormat *string + // IncludeInputSchema, if true, adds to each listed Actor an inputSchema field with a compact + // version of its default build's input schema (property types and required properties only). + // With it set, the API defaults Limit to 10 and does not accept a larger one. + IncludeInputSchema *bool } func (o StoreListOptions) apply(q *QueryParams) { @@ -36,7 +40,8 @@ func (o StoreListOptions) apply(q *QueryParams) { AddString("pricingModel", o.PricingModel). AddBool("includeUnrunnableActors", o.IncludeUnrunnableActors). AddBool("allowsAgenticUsers", o.AllowsAgenticUsers). - AddString("responseFormat", o.ResponseFormat) + AddString("responseFormat", o.ResponseFormat). + AddBool("includeInputSchema", o.IncludeInputSchema) } // StoreCollectionClient is a client for browsing the Apify Store (GET /v2/store). diff --git a/streamed_log.go b/streamed_log.go new file mode 100644 index 0000000..d11f769 --- /dev/null +++ b/streamed_log.go @@ -0,0 +1,358 @@ +package apify + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" + "time" +) + +const ( + // emptyLogStreamRetry is how long a [StreamedLog] waits before it reopens a log stream that + // delivered nothing: the API serves the log of a run that has not logged anything yet as an + // empty stream that ends at once. The pause doubles up to emptyLogStreamMaxRetry, which bounds + // the request rate while a run waits long to start. Matches the reference client's + // EMPTY_LOG_STREAM_RETRY_MILLIS and EMPTY_LOG_STREAM_MAX_RETRY_MILLIS. + emptyLogStreamRetry = 500 * time.Millisecond + emptyLogStreamMaxRetry = 5 * time.Second + // stopGrace is how long [StreamedLog.Stop] waits for the messages already in flight before it + // aborts the log stream. Matches the reference client's STOP_GRACE_MILLIS. + stopGrace = time.Second + // logRedirectPrefixSeparator ends the default message prefix, which names the Actor and the run. + logRedirectPrefixSeparator = " -> " + // logReadChunkBytes is the size of the buffer the log stream is read through. + logReadChunkBytes = 32 * 1024 +) + +// logMessageMarker matches the ISO timestamp that starts every message of an Actor run log. The +// log is split into messages at these markers, so a message that spans several lines (a stack +// trace, say) is redirected as one. +var logMessageMarker = regexp.MustCompile(`(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)`) + +// StreamedLogOptions configures [RunClient.NewStreamedLog] and, through +// [ActorStartOptions.Log], the log redirection of [ActorClient.Call]. +type StreamedLogOptions struct { + // Disabled turns the log redirection off. It only has an effect in [ActorStartOptions.Log]. + Disabled bool + // Output receives every redirected message, one per line, prefixed with Prefix. Defaults to + // os.Stderr. Writes are serialized, so the writer need not be safe for concurrent use. + Output io.Writer + // Prefix is written in front of every message. Defaults to "{Actor name} runId:{run ID} -> ", + // which costs two requests (the run and its Actor); a non-nil Prefix (an empty one included) + // skips them. + Prefix *string + // FromStart controls whether messages logged before the redirection was created are + // redirected too (default true). With false, they are dropped. + FromStart *bool +} + +// StreamedLog redirects the live log of an Actor run to a writer, message by message. Obtain one +// from [RunClient.NewStreamedLog], begin the redirection with [StreamedLog.Start] and end it with +// [StreamedLog.Stop]. Matches the reference client's StreamedLog. +// +// Each message of the log, which starts with the timestamp the platform puts in front of it, is +// written on its own line, prefixed. Messages that were already logged when the redirection was +// created are skipped if FromStart is false. The log of a run that has not logged anything yet is +// reopened until it delivers its first bytes. +type StreamedLog struct { + log *LogClient + output io.Writer + prefix string + startedAt time.Time + fromStart bool + + mu sync.Mutex // serializes writes to output and guards the fields below + running bool + stopping bool + cancelStream context.CancelFunc + wake chan struct{} + done chan error + buffer bytes.Buffer +} + +// NewStreamedLog creates the redirection of this run's log, which does nothing until +// [StreamedLog.Start]. With the default prefix it fetches the run and its Actor to name them, and +// it returns (nil, nil) if the run does not exist. +func (c *RunClient) NewStreamedLog(ctx context.Context, options StreamedLogOptions) (*StreamedLog, error) { + prefix := "" + if options.Prefix != nil { + prefix = *options.Prefix + } else { + run, present, err := getResourceWithTimeout[ActorRun](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) + if err != nil || !present { + return nil, err + } + actorName := "" + if run.ActID != "" { + actor, found, err := c.root.Actor(run.ActID).Get(ctx) + if err != nil { + return nil, err + } + if found { + actorName = actor.Name + } + } + parts := []string{"runId:" + run.ID} + if actorName != "" { + parts = append([]string{actorName}, parts...) + } + prefix = strings.Join(parts, " ") + logRedirectPrefixSeparator + } + + output := options.Output + if output == nil { + output = os.Stderr + } + return &StreamedLog{ + log: c.Log(), + output: output, + prefix: prefix, + startedAt: time.Now(), + fromStart: options.FromStart == nil || *options.FromStart, + }, nil +} + +// Start begins the redirection in the background. The redirection also ends when ctx is done. +// It returns an error if the redirection is already running. +func (s *StreamedLog) Start(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.running { + return errors.New("StreamedLog: already started") + } + streamCtx, cancel := context.WithCancel(ctx) + s.running = true + s.stopping = false + s.cancelStream = cancel + s.wake = make(chan struct{}) + s.done = make(chan error, 1) + s.buffer.Reset() + go func() { + s.done <- s.streamLog(ctx, streamCtx) + }() + return nil +} + +// Stop ends the redirection. It gives the messages already in flight up to one second, then +// aborts the log stream; if the stream delivered nothing yet, it reads the whole log in one +// request instead, so a short run that finished before the stream connected is not lost. It +// returns the error that made the redirection stop early, if any. ctx bounds the final read of +// the whole log. +func (s *StreamedLog) Stop(ctx context.Context) error { + s.mu.Lock() + if !s.running { + s.mu.Unlock() + return errors.New("StreamedLog: not started") + } + s.stopping = true + close(s.wake) + cancel, done := s.cancelStream, s.done + s.mu.Unlock() + + timer := time.AfterFunc(stopGrace, cancel) + defer timer.Stop() + var err error + select { + case err = <-done: + case <-ctx.Done(): + cancel() + err = <-done + } + cancel() + + s.mu.Lock() + s.running = false + s.mu.Unlock() + return err +} + +func (s *StreamedLog) isStopping() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.stopping +} + +// streamLog redirects the log until the stream ends for good. parent is the context of the whole +// redirection, and streamCtx the one of the log stream, which Stop cancels after its grace period. +// It returns nil when the redirection was ended on purpose. +func (s *StreamedLog) streamLog(parent, streamCtx context.Context) error { + err := s.redirect(parent, streamCtx) + if err != nil && streamCtx.Err() != nil { + return nil // ended on purpose: by Stop, or because the caller's context is done + } + return err +} + +func (s *StreamedLog) redirect(parent, streamCtx context.Context) error { + raw := true + options := LogOptions{Raw: &raw} + retry := emptyLogStreamRetry + gotBytes := false + + for !gotBytes { + if s.isStopping() { + // The stream's context ends with the grace period, which must not cut this read short. + body, _, err := s.log.GetBufferWithOptions(parent, options) + if err != nil { + return err + } + s.consume(body) + break + } + stream, err := s.log.StreamWithOptions(streamCtx, options) + if err != nil { + if s.isStopping() && parent.Err() == nil { + continue // the grace period ended while connecting: read the log in one request + } + return err + } + if stream == nil { + return nil // the log does not exist + } + if s.isStopping() { + _ = stream.Close() + continue + } + var readErr error + gotBytes, readErr = s.consumeStream(stream) + _ = stream.Close() + if readErr != nil && streamCtx.Err() == nil { + return readErr + } + if !gotBytes { + select { + case <-time.After(retry): + case <-s.wake: + case <-streamCtx.Done(): + } + retry = min(retry*2, emptyLogStreamMaxRetry) + } + } + s.flush() + return nil +} + +// consumeStream redirects the complete messages of stream and reports whether it delivered any +// byte. It stops at the end of the stream, an error, or when the redirection is being stopped. +func (s *StreamedLog) consumeStream(stream io.Reader) (bool, error) { + chunk := make([]byte, logReadChunkBytes) + gotBytes := false + for { + n, err := stream.Read(chunk) + if n > 0 { + gotBytes = true + s.consume(chunk[:n]) + } + if err != nil { + if errors.Is(err, io.EOF) { + return gotBytes, nil + } + return gotBytes, err + } + if s.isStopping() { + return gotBytes, nil + } + } +} + +// consume adds data to the buffer and redirects every complete message in it: everything before +// the last end of line. An incomplete last line stays in the buffer for the next chunk. +func (s *StreamedLog) consume(data []byte) { + s.buffer.Write(data) + buffered := s.buffer.Bytes() + lastEOL := bytes.LastIndexByte(buffered, '\n') + if lastEOL < 0 { + return + } + complete := string(buffered[:lastEOL]) + rest := append([]byte(nil), buffered[lastEOL:]...) + s.buffer.Reset() + s.buffer.Write(rest) + s.redirectMessages(complete) +} + +// flush redirects whatever is left in the buffer when the log ends: an incomplete last message, +// or a last line without an end of line. +func (s *StreamedLog) flush() { + last := strings.TrimSpace(s.buffer.String()) + s.buffer.Reset() + if last != "" { + s.write(last) + } +} + +// redirectMessages splits the complete part of the log into messages at their timestamps and +// writes them. +func (s *StreamedLog) redirectMessages(text string) { + markers := logMessageMarker.FindAllStringSubmatchIndex(text, -1) + for i, marker := range markers { + contentEnd := len(text) + if i+1 < len(markers) { + contentEnd = markers[i+1][0] + } + timestamp := text[marker[2]:marker[3]] + if !s.fromStart { + if logged, err := time.Parse(time.RFC3339Nano, timestamp); err == nil && logged.Before(s.startedAt) { + continue + } + } + s.write(strings.TrimSpace(timestamp + text[marker[3]:contentEnd])) + } +} + +// write writes one prefixed message to the output. +func (s *StreamedLog) write(message string) { + s.mu.Lock() + defer s.mu.Unlock() + _, _ = fmt.Fprintf(s.output, "%s%s\n", s.prefix, message) +} + +// startLogRedirection starts redirecting the log of run, as [ActorClient.Call] does by default. +// A failure never fails the call: it is reported to the redirection's output and the call goes on +// without redirected logs. It returns nil when the redirection is disabled or could not start. +func startLogRedirection(ctx context.Context, run *RunClient, options *StreamedLogOptions) *StreamedLog { + settings := StreamedLogOptions{} + if options != nil { + settings = *options + } + if settings.Disabled { + return nil + } + streamed, err := run.NewStreamedLog(ctx, settings) + if err == nil && streamed != nil { + err = streamed.Start(ctx) + } + if err != nil { + reportLogRedirectionError(settings.Output, err) + return nil + } + return streamed +} + +// stopLogRedirection ends the redirection started by [startLogRedirection], if there is one. +func stopLogRedirection(ctx context.Context, streamed *StreamedLog, options *StreamedLogOptions) { + if streamed == nil { + return + } + if err := streamed.Stop(ctx); err != nil { + var output io.Writer + if options != nil { + output = options.Output + } + reportLogRedirectionError(output, err) + } +} + +// reportLogRedirectionError tells the user that the log redirection ended early. +func reportLogRedirectionError(output io.Writer, err error) { + if output == nil { + output = os.Stderr + } + _, _ = fmt.Fprintf(output, "Log redirection stopped due to error: %v\n", err) +} diff --git a/streamed_log_test.go b/streamed_log_test.go new file mode 100644 index 0000000..975ec5a --- /dev/null +++ b/streamed_log_test.go @@ -0,0 +1,325 @@ +package apify + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" +) + +// syncBuffer is a bytes.Buffer that is safe to read while the redirection writes to it. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// logBackend serves a run's log: streamBodies[i] answers the i-th request for the live stream +// (the last one repeats), and fullLog answers the read of the whole log. Run and Actor lookups +// answer the fixed documents below. +type logBackend struct { + mu sync.Mutex + streamBodies []string + streamErr error + fullLog string + streamCalls int + fullLogCalls int + rawOnStream bool + streamTailErr error // makes the stream fail with this error after its body +} + +type failingReader struct { + data io.Reader + err error +} + +func (r *failingReader) Read(p []byte) (int, error) { + n, err := r.data.Read(p) + if errors.Is(err, io.EOF) { + return n, r.err + } + return n, err +} + +func (b *logBackend) Do(req *http.Request) (*http.Response, error) { + b.mu.Lock() + defer b.mu.Unlock() + ok := func(body io.Reader) (*http.Response, error) { + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(body)}, nil + } + path := req.URL.Path + switch { + case strings.HasSuffix(path, "/log"): + if req.URL.Query().Get("stream") != "" { + b.rawOnStream = req.URL.Query().Get("raw") == "1" + if b.streamErr != nil { + return nil, b.streamErr + } + body := b.streamBodies[min(b.streamCalls, len(b.streamBodies)-1)] + b.streamCalls++ + var reader io.Reader = strings.NewReader(body) + if b.streamTailErr != nil { + reader = &failingReader{data: reader, err: b.streamTailErr} + } + return ok(reader) + } + b.fullLogCalls++ + return ok(strings.NewReader(b.fullLog)) + case strings.HasSuffix(path, "/actors/actor1"): + return ok(strings.NewReader(`{"data":{"id":"actor1","name":"hello-world"}}`)) + case strings.Contains(path, "/actor-runs/"): + return ok(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"RUNNING"}}`)) + } + return nil, errors.New("unexpected request " + req.URL.String()) +} + +const sampleLog = "2026-10-30T10:00:00.000Z ACTOR: Starting\n" + + "2026-10-30T10:00:01.000Z first line\nsecond line of the same message\n" + + "2026-10-30T10:00:02.000Z last message\n" + +func TestStreamedLogRedirectsMessagesWithThePrefixOfTheActorAndTheRun(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + out := &syncBuffer{} + ctx := context.Background() + + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out}) + if err != nil || streamed == nil { + t.Fatalf("NewStreamedLog: %v, %v", streamed, err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + + const prefix = "hello-world runId:run1 -> " + want := prefix + "2026-10-30T10:00:00.000Z ACTOR: Starting\n" + + prefix + "2026-10-30T10:00:01.000Z first line\nsecond line of the same message\n" + + prefix + "2026-10-30T10:00:02.000Z last message\n" + if got := out.String(); got != want { + t.Fatalf("redirected log:\n%q\nwant:\n%q", got, want) + } + if !backend.rawOnStream { + t.Fatal("the raw log must be streamed") + } +} + +func TestStreamedLogPrefixOverrideSkipsTheLookups(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + out := &syncBuffer{} + ctx := context.Background() + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil || streamed == nil { + t.Fatalf("NewStreamedLog: %v, %v", streamed, err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(out.String(), "2026-10-30T10:00:00.000Z ACTOR: Starting\n") { + t.Fatalf("an empty prefix is used as it is: %q", out.String()) + } +} + +func TestStreamedLogDropsMessagesBeforeTheStartWhenNotFromStart(t *testing.T) { + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + log := "2026-10-30T10:00:00.000Z old\n" + future + " new\n" + backend := &logBackend{streamBodies: []string{log}} + out := &syncBuffer{} + ctx := context.Background() + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(""), FromStart: Ptr(false)}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "new") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "old") { + t.Fatalf("a message logged before the redirection was created must be dropped: %q", out.String()) + } +} + +// The API serves the log of a run that has not logged anything as an empty stream that ends at +// once: the redirection reopens it until the first bytes arrive. +func TestStreamedLogReopensAnEmptyStream(t *testing.T) { + backend := &logBackend{streamBodies: []string{"", "", sampleLog}} + out := &syncBuffer{} + ctx := context.Background() + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if backend.streamCalls != 3 { + t.Fatalf("expected the empty stream to be reopened twice, got %d stream requests", backend.streamCalls) + } +} + +// A run that finished before the stream delivered anything is not lost: stopping reads the whole +// log in one request. +func TestStreamedLogStoppedBeforeTheFirstBytesReadsTheWholeLog(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, fullLog: sampleLog} + out := &syncBuffer{} + ctx := context.Background() + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + if err := streamed.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if backend.fullLogCalls != 1 { + t.Fatalf("expected one read of the whole log, got %d", backend.fullLogCalls) + } + if !strings.Contains(out.String(), "last message") || strings.Count(out.String(), "\n") < 4 { + t.Fatalf("the whole log must be redirected: %q", out.String()) + } +} + +func TestStreamedLogFlushesALastLineWithoutAnEndOfLine(t *testing.T) { + backend := &logBackend{streamBodies: []string{"2026-10-30T10:00:00.000Z one\n2026-10-30T10:00:01.000Z unterminated"}} + out := &syncBuffer{} + ctx := context.Background() + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(">")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "unterminated") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if got := out.String(); got != ">2026-10-30T10:00:00.000Z one\n>2026-10-30T10:00:01.000Z unterminated\n" { + t.Fatalf("got %q", got) + } +} + +func TestStreamedLogStartAndStopMustBePaired(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + ctx := context.Background() + streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Stop(ctx); err == nil { + t.Fatal("Stop before Start must fail") + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err == nil { + t.Fatal("a second Start must fail") + } + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } +} + +func TestNewStreamedLogOfAMissingRunIsNil(t *testing.T) { + backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} + streamed, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) + if err != nil || streamed != nil { + t.Fatalf("expected (nil, nil), got %v, %v", streamed, err) + } +} + +// Call redirects the log of the run by default, and the call goes on when the redirection fails. +func TestCallRedirectsTheLogOfTheRun(t *testing.T) { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, fullLog: sampleLog}} + got, err := testClient(backend, 0).Actor("actor1").Call(context.Background(), nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out}}, Ptr(int64(0))) + if err != nil || got.Status != "SUCCEEDED" { + t.Fatalf("Call: %+v, %v", got, err) + } + if !strings.Contains(out.String(), "hello-world runId:run1 -> 2026-10-30T10:00:02.000Z last message") { + t.Fatalf("the log must be redirected with the prefix of the Actor and the run: %q", out.String()) + } +} + +func TestCallWithDisabledLogRedirectionRequestsNoLog(t *testing.T) { + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, fullLog: sampleLog}} + _, err := testClient(backend, 0).Actor("actor1").Call(context.Background(), nil, ActorStartOptions{Log: &StreamedLogOptions{Disabled: true}}, Ptr(int64(0))) + if err != nil { + t.Fatal(err) + } + if backend.streamCalls != 0 || backend.fullLogCalls != 0 { + t.Fatalf("no log may be requested, got %d stream and %d full reads", backend.streamCalls, backend.fullLogCalls) + } +} + +func TestACallGoesOnWhenTheLogRedirectionFails(t *testing.T) { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, streamErr: errors.New("boom")}, pollDelay: 100 * time.Millisecond} + got, err := testClient(backend, 0).Actor("actor1").Call(context.Background(), nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) + if err != nil || got.Status != "SUCCEEDED" { + t.Fatalf("a failing log redirection must not fail the call: %+v, %v", got, err) + } + if !strings.Contains(out.String(), "Log redirection stopped due to error") || !strings.Contains(out.String(), "boom") { + t.Fatalf("the failure must be reported: %q", out.String()) + } +} + +// callLogBackend also serves the start of a run, and answers every poll of it as finished after +// pollDelay, which leaves the log redirection time to run before the call stops it. +type callLogBackend struct { + logBackend + pollDelay time.Duration +} + +func (b *callLogBackend) Do(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/runs") { + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"RUNNING"}}`))}, nil + } + if strings.Contains(req.URL.Path, "/actor-runs/") && !strings.HasSuffix(req.URL.Path, "/log") && req.URL.Query().Get("waitForFinish") != "" { + time.Sleep(b.pollDelay) + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"SUCCEEDED"}}`))}, nil + } + return b.logBackend.Do(req) +} + +func waitFor(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatal("timed out waiting for the condition") + } + time.Sleep(5 * time.Millisecond) + } +} diff --git a/task.go b/task.go index 2b72294..e31aabb 100644 --- a/task.go +++ b/task.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "time" ) // TaskClient is a client for a specific Actor task. @@ -99,23 +100,34 @@ func (o TaskStartOptions) apply(q *QueryParams) { // Start starts the task and returns immediately with the created run. input optionally // overrides the task's stored input (nil to use the stored input). +// +// A start request that times out is not retried: it may already have created the run, and a +// retry would then start a second, billed one. func (c *TaskClient) Start(ctx context.Context, input any, options TaskStartOptions) (ActorRun, error) { + return c.startWithTimeout(ctx, input, options, extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish)) +} + +// startWithTimeout is the shared implementation of Start and Call: it starts the task with the +// given timeout for each start request. +func (c *TaskClient) startWithTimeout(ctx context.Context, input any, options TaskStartOptions, timeout time.Duration) (ActorRun, error) { params := NewQueryParams() options.apply(params) body, err := marshalInput(input) if err != nil { return ActorRun{}, err } - timeout := extendForServerWait(c.ctx.mediumTimeout(), options.WaitForFinish) return startWaitingForResources(ctx, options.WaitForResources, func() (ActorRun, error) { - return postWithBody[ActorRun](ctx, c.ctx, "runs", params, body, contentTypeJSON, timeout) + return postWithBody[ActorRun](ctx, c.ctx.withoutTimeoutRetry(), "runs", params, body, contentTypeJSON, timeout) }) } // Call starts the task and waits (client-side polling) for it to finish. waitSecs bounds the // wait; nil waits indefinitely. +// +// Neither the start request nor the polls have a client-side timeout of their own (matching the +// reference client's call(), whose default timeout is noTimeout): ctx bounds the whole call. func (c *TaskClient) Call(ctx context.Context, input any, options TaskStartOptions, waitSecs *int64) (ActorRun, error) { - run, err := c.Start(ctx, input, options) + run, err := c.startWithTimeout(ctx, input, options, noRequestTimeout) if err != nil { return ActorRun{}, err } @@ -136,6 +148,9 @@ func (c *TaskClient) GetInput(ctx context.Context) (json.RawMessage, error) { // UpdateInput replaces the task's stored input and returns the updated input. func (c *TaskClient) UpdateInput(ctx context.Context, input any) (json.RawMessage, error) { + if c.ctx.idErr != nil { + return nil, c.ctx.idErr + } data, err := json.Marshal(input) if err != nil { return nil, err diff --git a/tests/actor_run_test.go b/tests/actor_run_test.go index 86c33d3..f8d748b 100644 --- a/tests/actor_run_test.go +++ b/tests/actor_run_test.go @@ -1,6 +1,8 @@ package apify_test import ( + "bytes" + "strings" "testing" apify "github.com/apify/apify-client-go" @@ -145,3 +147,229 @@ func TestLastRunAccess(t *testing.T) { t.Fatalf("expected last succeeded run filtered by origin, got %q", lastRunByOrigin.Status) } } + +// TestCallRedirectsTheRunLog runs hello-world through Call and checks that its log was redirected +// to the configured output, prefixed with the Actor name and the run ID. +func TestCallRedirectsTheRunLog(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + var out bytes.Buffer + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{ + Log: &apify.StreamedLogOptions{Output: &out}, + }, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + if want := "hello-world runId:" + run.ID + " -> "; !strings.Contains(out.String(), want) { + t.Fatalf("expected the redirected log to carry the prefix %q, got %q", want, out.String()) + } +} + +// TestRunAndTaskStoragesAndLogCanBeRead covers the simple GETs of a finished run: the run-scoped +// default storages (dataset, key-value store, request queue), the log by run id, and the default +// build, webhook listings and last run of the Actor and of a task. +func TestRunAndTaskStoragesAndLogCanBeRead(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + actor := client.Actor("apify/hello-world") + run, err := actor.Call(ctx, nil, apify.ActorStartOptions{Log: &apify.StreamedLogOptions{Disabled: true}}, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + runClient := client.Run(run.ID) + + if logText, ok, err := client.Log(run.ID).Get(ctx); err != nil || !ok || logText == "" { + t.Fatalf("Log(%s).Get: ok=%v err=%v len=%d", run.ID, ok, err, len(logText)) + } + + if ds, ok, err := runClient.Dataset().Get(ctx); err != nil || !ok || ds.ID != run.DefaultDatasetID { + t.Fatalf("run dataset Get: %+v ok=%v err=%v", ds, ok, err) + } + if _, err := runClient.Dataset().GetStatistics(ctx); err != nil { + t.Fatalf("run dataset GetStatistics: %v", err) + } + if store, ok, err := runClient.KeyValueStore().Get(ctx); err != nil || !ok || store.ID != run.DefaultKeyValueStoreID { + t.Fatalf("run key-value store Get: %+v ok=%v err=%v", store, ok, err) + } + if _, err := runClient.KeyValueStore().ListKeys(ctx, apify.ListKeysOptions{}); err != nil { + t.Fatalf("run key-value store ListKeys: %v", err) + } + if queue, ok, err := runClient.RequestQueue().Get(ctx); err != nil || !ok || queue.ID != run.DefaultRequestQueueID { + t.Fatalf("run request queue Get: %+v ok=%v err=%v", queue, ok, err) + } + if _, err := runClient.RequestQueue().ListHead(ctx, nil); err != nil { + t.Fatalf("run request queue ListHead: %v", err) + } + if _, err := runClient.RequestQueue().ListRequests(ctx, apify.ListRequestsOptions{}); err != nil { + t.Fatalf("run request queue ListRequests: %v", err) + } + + if _, err := actor.DefaultBuild(ctx, nil); err != nil { + t.Fatalf("DefaultBuild: %v", err) + } + if _, err := actor.Webhooks().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}); err != nil { + t.Fatalf("actor webhooks List: %v", err) + } + + task, err := client.Tasks().Create(ctx, taskDef(uniqueName("task-last-run"))) + if err != nil { + t.Fatalf("create task: %v", err) + } + defer func() { _ = client.Task(task.ID).Delete(ctx) }() + taskRun, err := client.Task(task.ID).Call(ctx, nil, apify.TaskStartOptions{}, ptr(int64(120))) + if err != nil { + t.Fatalf("call task: %v", err) + } + if _, err := client.Task(task.ID).Webhooks().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}); err != nil { + t.Fatalf("task webhooks List: %v", err) + } + last, ok, err := client.Task(task.ID).LastRun("SUCCEEDED").Get(ctx) + if err != nil || !ok || last.ID != taskRun.ID { + t.Fatalf("task LastRun: %+v ok=%v err=%v", last, ok, err) + } + if logText, ok, err := client.Task(task.ID).LastRun("SUCCEEDED").Log().Get(ctx); err != nil || !ok || logText == "" { + t.Fatalf("task last run log: ok=%v err=%v len=%d", ok, err, len(logText)) + } + // The storages of a filtered last run are those of the run the filter resolves to. + if ds, ok, err := client.Task(task.ID).LastRun("SUCCEEDED").Dataset().Get(ctx); err != nil || !ok || ds.ID != taskRun.DefaultDatasetID { + t.Fatalf("task last run dataset: %+v ok=%v err=%v", ds, ok, err) + } +} + +// TestModelURLFieldsAreNormalized checks, against the real API, that the URL fields of a run and +// of its storages come back in their normalized form: an absolute URL with a path. +func TestModelURLFieldsAreNormalized(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{Log: &apify.StreamedLogOptions{Disabled: true}}, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + got, ok, err := client.Run(run.ID).Get(ctx) + if err != nil || !ok { + t.Fatalf("get run: ok=%v err=%v", ok, err) + } + for name, value := range map[string]string{"consoleUrl": got.ConsoleURL, "containerUrl": got.ContainerURL} { + if value == "" { + continue // the API does not return the field for this run + } + if !strings.HasPrefix(value, "https://") || strings.Count(value, "/") < 3 { + t.Errorf("%s is not a normalized absolute URL: %q", name, value) + } + } + if got.ConsoleURL == "" { + t.Error("the API is expected to return the consoleUrl of a run") + } + + ds, ok, err := client.Dataset(run.DefaultDatasetID).Get(ctx) + if err != nil || !ok { + t.Fatalf("get dataset: ok=%v err=%v", ok, err) + } + if !strings.HasPrefix(ds.ConsoleURL, "https://") || ds.ItemsPublicURL == "" { + t.Errorf("unexpected dataset URLs: %q, %q", ds.ConsoleURL, ds.ItemsPublicURL) + } + store, ok, err := client.KeyValueStore(run.DefaultKeyValueStoreID).Get(ctx) + if err != nil || !ok { + t.Fatalf("get key-value store: ok=%v err=%v", ok, err) + } + if !strings.HasPrefix(store.ConsoleURL, "https://") || store.KeysPublicURL == "" || store.RecordsPublicURL == "" { + t.Errorf("unexpected key-value store URLs: %+v", store) + } +} + +// TestIterateRunDatasetItems pushes items into the default dataset of a finished run, then reads +// them back through RunClient.IterateDatasetItems, which must also catch the rows past a lagging +// itemCount once the run is finished. +func TestIterateRunDatasetItems(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + run, err := client.Actor("apify/hello-world").Call(ctx, nil, apify.ActorStartOptions{Log: &apify.StreamedLogOptions{Disabled: true}}, ptr(int64(120))) + if err != nil { + t.Fatalf("call hello-world: %v", err) + } + runClient := client.Run(run.ID) + const total = 7 + items := make([]map[string]any, total) + for i := range items { + items[i] = map[string]any{"n": i} + } + if err := runClient.Dataset().PushItems(ctx, items); err != nil { + t.Fatalf("push items: %v", err) + } + + // hello-world pushes an item of its own, which has no "n" and is left out below. + type item struct { + N *int `json:"n"` + } + it := apify.IterateRunDatasetItems[item](runClient, apify.RunIterateDatasetItemsOptions{ + ChunkSize: ptr(int64(3)), + PollIntervalSecs: ptr(int64(1)), + }) + var got []int + for { + next, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate run dataset items: %v", err) + } + if next == nil { + break + } + if next.N != nil { + got = append(got, *next.N) + } + } + if len(got) != total { + t.Fatalf("expected the %d pushed items, got %v", total, got) + } + for i, n := range got { + if n != i { + t.Fatalf("items out of order: %v", got) + } + } + + // A Limit caps the rows scanned. + limited := runClient.IterateDatasetItems(apify.RunIterateDatasetItemsOptions{ + Items: apify.DatasetListItemsOptions{Limit: ptr(int64(3))}, + }) + count := 0 + for { + next, err := limited.Next(ctx) + if err != nil { + t.Fatalf("iterate with offset and limit: %v", err) + } + if next == nil { + break + } + count++ + } + if count != 3 { + t.Fatalf("expected 3 items with limit 3, got %d", count) + } +} + +// TestRunListDeploymentFilter lists the Actor Standby runs of a pool through the account-wide +// and Actor-scoped run collections: the filter is accepted (the result may well be empty). +func TestRunListDeploymentFilter(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + filter := apify.RunListOptions{ + Status: []string{"READY", "RUNNING"}, + Deployment: []string{"SINGLE_TENANT", "MULTI_TENANT_FREE_USERS"}, + } + if _, err := client.Runs().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}, filter); err != nil { + t.Fatalf("Runs().List with a deployment filter: %v", err) + } + if _, err := client.Actor("apify/hello-world").Runs().List(ctx, apify.ListOptions{Limit: ptr(int64(5))}, filter); err != nil { + t.Fatalf("Actor.Runs().List with a deployment filter: %v", err) + } +} diff --git a/tests/iterate_test.go b/tests/iterate_test.go index ede6fec..a1e6c54 100644 --- a/tests/iterate_test.go +++ b/tests/iterate_test.go @@ -13,8 +13,8 @@ import ( // consistent, so a resource can be missing from the first listing right after creation; the // helper re-iterates a fresh iterator until it converges. const ( - iterFindAttempts = 8 - iterFindBackoff = time.Second + iterFindAttempts = 32 + iterFindBackoff = 3 * time.Second ) // iterateFindIDs repeatedly drains a fresh iterator (from makeIter) and checks that every id diff --git a/tests/store_test.go b/tests/store_test.go index d58fdf1..32ce870 100644 --- a/tests/store_test.go +++ b/tests/store_test.go @@ -44,3 +44,22 @@ func TestIterateStore(t *testing.T) { t.Fatalf("expected to iterate at least 12 store actors, got %d", count) } } + +// TestListStoreIncludeInputSchema asks the Store to add the compact input schema to every Actor: +// with it set, the API defaults the limit to 10 and accepts nothing above it. +func TestListStoreIncludeInputSchema(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + page, err := client.Store().List(ctx, apify.StoreListOptions{Limit: ptr(int64(3)), IncludeInputSchema: ptr(true)}) + if err != nil { + t.Fatalf("Store().List with includeInputSchema: %v", err) + } + if len(page.Items) == 0 { + t.Fatal("expected at least one Store Actor") + } + if len(page.Items) > 10 { + t.Fatalf("with includeInputSchema the API returns at most 10 Actors, got %d", len(page.Items)) + } +} diff --git a/tests/user_test.go b/tests/user_test.go index 26ce317..10b7815 100644 --- a/tests/user_test.go +++ b/tests/user_test.go @@ -1,6 +1,9 @@ package apify_test -import "testing" +import ( + "strings" + "testing" +) func TestGetOwnAccount(t *testing.T) { client := requireClient(t) @@ -66,3 +69,52 @@ func TestGetLimits(t *testing.T) { t.Fatal("expected a non-empty limits object") } } + +// TestGetLimitsAndUsageByUsername reads the limits and the monthly usage through the by-username +// client: the API serves them for your own account by ID or username as well as through "me". +func TestGetLimitsAndUsageByUsername(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + me, ok, err := client.Me().Get(ctx) + if err != nil || !ok || me.Username == "" { + t.Fatalf("Me().Get: ok=%v err=%v username=%q", ok, err, me.Username) + } + user := client.User(me.Username) + limits, err := user.Limits(ctx) + if err != nil { + t.Fatalf("User(%q).Limits: %v", me.Username, err) + } + if len(limits) == 0 { + t.Fatal("expected a non-empty limits object") + } + usage, err := user.MonthlyUsage(ctx) + if err != nil { + t.Fatalf("User(%q).MonthlyUsage: %v", me.Username, err) + } + if len(usage) == 0 { + t.Fatal("expected a non-empty monthly usage object") + } +} + +// TestUserProfileURLsAreNormalized checks that the profile of the own account is parsed and its +// URL fields, when set, are absolute URLs. +func TestUserProfileURLsAreNormalized(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + me, ok, err := client.Me().Get(ctx) + if err != nil || !ok { + t.Fatalf("Me().Get: ok=%v err=%v", ok, err) + } + if me.Profile == nil { + t.Fatal("expected the profile of the own account") + } + for name, value := range map[string]*string{"pictureUrl": me.Profile.PictureURL, "websiteUrl": me.Profile.WebsiteURL} { + if value != nil && !strings.Contains(*value, "://") { + t.Errorf("%s is not an absolute URL: %q", name, *value) + } + } +} diff --git a/timeout_retry_test.go b/timeout_retry_test.go new file mode 100644 index 0000000..1a234a3 --- /dev/null +++ b/timeout_retry_test.go @@ -0,0 +1,339 @@ +package apify + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +// slowBackend answers every request with a 200 and body after delay, unless the request's +// context ends first (then it fails with the context's error, as a timed-out transport does). It +// counts the requests it received and keeps the last URL. +type slowBackend struct { + delay time.Duration + body string + calls atomic.Int64 + url atomic.Value +} + +func (b *slowBackend) Do(req *http.Request) (*http.Response, error) { + b.calls.Add(1) + b.url.Store(req.URL.String()) + select { + case <-time.After(b.delay): + case <-req.Context().Done(): + return nil, req.Context().Err() + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader([]byte(b.body)))}, nil +} + +const ( + // slowResponseDelay is how long slowBackend takes to answer: longer than the tiny tier used in + // the tests below, so a request bound by that tier times out, but short enough to keep them fast. + slowResponseDelay = 400 * time.Millisecond + // tinyTier is the timeout tier of the test clients below. + tinyTier = 40 * time.Millisecond + // retriesAllowed is the number of retries of the test clients below, so a retried request + // shows up as extra calls. + retriesAllowed = 3 +) + +// noLogRedirection turns the log redirection of Call off, so a test counts only the requests of +// the call itself. +var noLogRedirection = &StreamedLogOptions{Disabled: true} + +func slowClient(backend HTTPBackend, options ...Option) *ApifyClient { + base := []Option{ + WithToken("t"), + WithHTTPBackend(backend), + WithMaxRetries(retriesAllowed), + WithMinDelayBetweenRetries(time.Millisecond), + WithTimeoutShort(tinyTier), + WithTimeoutMedium(tinyTier), + } + return NewClient(append(base, options...)...) +} + +// A request that is not idempotent must not be retried after it timed out: it may already have +// taken effect on the server. +func TestNonIdempotentRequestsAreNotRetriedAfterATimeout(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"Actor Start", func(c *ApifyClient) error { + _, err := c.Actor("a").Start(ctx, map[string]any{}, ActorStartOptions{}) + return err + }}, + {"Actor StartRaw", func(c *ApifyClient) error { + _, err := c.Actor("a").StartRaw(ctx, []byte("x"), ActorStartOptions{}) + return err + }}, + {"Actor Build", func(c *ApifyClient) error { + _, err := c.Actor("a").Build(ctx, "0.1", ActorBuildOptions{}) + return err + }}, + {"Task Start", func(c *ApifyClient) error { + _, err := c.Task("t").Start(ctx, nil, TaskStartOptions{}) + return err + }}, + {"Dataset PushItems", func(c *ApifyClient) error { + return c.Dataset("d").PushItems(ctx, map[string]any{"a": 1}) + }}, + {"KeyValueStore SetRecordRawWithOptions", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordRawWithOptions(ctx, "k", []byte("v"), "text/plain", SetRecordOptions{DoNotRetryTimeouts: true}) + }}, + {"KeyValueStore SetRecordJSONWithOptions", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordJSONWithOptions(ctx, "k", map[string]any{"a": 1}, SetRecordOptions{DoNotRetryTimeouts: true}) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: `{"data":{}}`} + // The record upload and the dataset push run on tiers the tiny medium one covers; the + // upload uses the long tier, so shrink it too. + if err := c.run(slowClient(backend, WithTimeoutLong(tinyTier))); err == nil { + t.Fatal("expected a timeout error") + } + if got := backend.calls.Load(); got != 1 { + t.Fatalf("a timed-out request must not be retried, got %d attempts", got) + } + }) + } +} + +// The reverse of the test above: an idempotent request is retried after a timeout, and so is a +// record upload that did not ask for DoNotRetryTimeouts. +func TestIdempotentRequestsAreRetriedAfterATimeout(t *testing.T) { + ctx := context.Background() + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"Actor Get", func(c *ApifyClient) error { + _, _, err := c.Actor("a").Get(ctx) + return err + }}, + {"KeyValueStore SetRecordRaw", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordRaw(ctx, "k", []byte("v"), "text/plain") + }}, + {"KeyValueStore SetRecordRawWithOptions", func(c *ApifyClient) error { + return c.KeyValueStore("s").SetRecordRawWithOptions(ctx, "k", []byte("v"), "text/plain", SetRecordOptions{}) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: `{"data":{}}`} + if err := c.run(slowClient(backend, WithTimeoutLong(tinyTier))); err == nil { + t.Fatal("expected a timeout error") + } + if got := backend.calls.Load(); got != retriesAllowed+1 { + t.Fatalf("expected %d attempts, got %d", retriesAllowed+1, got) + } + }) + } +} + +// A non-idempotent request still retries failures that are not timeouts (here a 500). +func TestNonIdempotentRequestsStillRetryServerErrors(t *testing.T) { + backend := &mockBackend{responses: constant(500, `{"error":{"type":"x","message":"m"}}`)} + client := testClient(backend, 2) + if err := client.Dataset("d").PushItems(context.Background(), map[string]any{"a": 1}); err == nil { + t.Fatal("expected an error") + } + if backend.calls != 3 { + t.Fatalf("expected 3 attempts, got %d", backend.calls) + } +} + +// Call has no client-side timeout of its own: a start slower than the medium tier must neither be +// aborted nor repeated (which would start a second, billed run). Start itself is bound by it. +func TestCallStartsWithoutATimeout(t *testing.T) { + run := `{"data":{"id":"run1","status":"SUCCEEDED"}}` + t.Run("Actor Call", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + got, err := slowClient(backend).Actor("a").Call(context.Background(), map[string]any{}, ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))) + if err != nil || got.ID != "run1" { + t.Fatalf("Call: %+v, %v", got, err) + } + if n := backend.calls.Load(); n != 2 { // the start and one poll + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Actor CallRaw", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Actor("a").CallRaw(context.Background(), []byte("x"), ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))); err != nil { + t.Fatalf("CallRaw: %v", err) + } + if n := backend.calls.Load(); n != 2 { + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Task Call", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Task("t").Call(context.Background(), nil, TaskStartOptions{}, Ptr(int64(0))); err != nil { + t.Fatalf("Task Call: %v", err) + } + if n := backend.calls.Load(); n != 2 { + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Start stays bound by the medium tier", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Actor("a").Start(context.Background(), nil, ActorStartOptions{}); err == nil { + t.Fatal("expected Start to time out") + } + }) + t.Run("ctx still bounds Call", func(t *testing.T) { + backend := &slowBackend{delay: time.Minute, body: run} + ctx, cancel := context.WithTimeout(context.Background(), 3*tinyTier) + defer cancel() + started := time.Now() + if _, err := slowClient(backend).Actor("a").Call(ctx, nil, ActorStartOptions{Log: noLogRedirection}, nil); err == nil { + t.Fatal("expected an error once ctx is done") + } + if time.Since(started) > 5*time.Second { + t.Fatal("Call did not stop at the ctx deadline") + } + }) +} + +// The calls the reference client puts on the long tier must not be cut off by a short tier: here +// the short tier (tinyTier) is shorter than the response time, and the calls still succeed. +func TestKeyValueStoreAndPublicURLCallsUseTheLongTier(t *testing.T) { + ctx := context.Background() + store := `{"data":{"id":"s","urlSigningSecretKey":"secret"}}` + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"KeyValueStore Update", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").Update(ctx, map[string]any{"name": "n"}) + return err + }}, + {"KeyValueStore RecordExists", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").RecordExists(ctx, "k") + return err + }}, + {"KeyValueStore GetRecordPublicURL", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").GetRecordPublicURL(ctx, "k") + return err + }}, + {"KeyValueStore CreateKeysPublicURL", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").CreateKeysPublicURL(ctx, nil) + return err + }}, + {"Dataset CreateItemsPublicURL", func(c *ApifyClient) error { + _, err := c.Dataset("d").CreateItemsPublicURL(ctx, DatasetListItemsOptions{}, nil) + return err + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: store} + if err := c.run(slowClient(backend)); err != nil { + t.Fatalf("a long-tier call must outlive the %v short and medium tiers: %v", tinyTier, err) + } + if got := backend.calls.Load(); got != 1 { + t.Fatalf("expected a single request, got %d", got) + } + }) + } +} + +func TestSetRecordRawWithOptionsSendsTheContentEncoding(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + value := []byte(strings.Repeat("x", 2*minCompressRequestBytes)) + + err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{ContentEncoding: "gzip"}) + if err != nil { + t.Fatalf("SetRecordRawWithOptions: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got != "gzip" { + t.Fatalf("expected Content-Encoding gzip, got %q", got) + } + if backend.lastBody != string(value) { + t.Fatal("a pre-compressed value must be uploaded as given, never compressed again") + } + + if err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{}); err != nil { + t.Fatalf("SetRecordRawWithOptions: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got == "" { + t.Fatal("without a ContentEncoding the client compresses a large value of its own accord") + } +} + +func TestPushItemsAcceptsOnlyObjectsAndArraysOfObjects(t *testing.T) { + type item struct { + A int `json:"a"` + } + valid := map[string]any{ + "map": map[string]any{"a": 1}, + "struct": item{A: 1}, + "struct pointer": &item{A: 1}, + "slice of structs": []item{{A: 1}, {A: 2}}, + "slice of maps": []map[string]any{{"a": 1}}, + "empty slice": []item{}, + "slice of any": []any{map[string]any{"a": 1}, item{A: 2}}, + "raw JSON object": rawJSON(`{"a":1}`), + "raw JSON array": rawJSON(`[{"a":1},{"a":2}]`), + "padded raw JSON obj": rawJSON(" \n{\"a\":1} "), + } + for name, items := range valid { + t.Run("accepts "+name, func(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err != nil { + t.Fatalf("PushItems rejected a valid shape: %v", err) + } + if backend.calls != 1 { + t.Fatalf("expected 1 request, got %d", backend.calls) + } + }) + } + + invalid := map[string]any{ + "string": "[1,2]", + "[]byte": []byte(`{"a":1}`), + "number": 1, + "bool": true, + "nil": nil, + "slice of strings": []string{"a", "b"}, + "slice of numbers": []int{1, 2}, + "mixed slice": []any{map[string]any{"a": 1}, "b"}, + "slice of slices": [][]int{{1}}, + "raw JSON string": rawJSON(`"a"`), + "raw JSON null element": rawJSON(`[{"a":1},null]`), + } + for name, items := range invalid { + t.Run("rejects "+name, func(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err == nil { + t.Fatal("PushItems accepted a shape the API rejects") + } + if backend.calls != 0 { + t.Fatalf("nothing may be sent for an invalid shape, got %d requests", backend.calls) + } + }) + } +} + +func TestPushItemsRejectsAnInvalidDatasetIDBeforeSending(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("").PushItems(context.Background(), map[string]any{"a": 1}); err == nil { + t.Fatal("expected an error for an empty dataset ID") + } + if backend.calls != 0 { + t.Fatalf("nothing may be sent for an invalid ID, got %d requests", backend.calls) + } +} + +func rawJSON(s string) json.RawMessage { return json.RawMessage(s) } diff --git a/url_normalization.go b/url_normalization.go new file mode 100644 index 0000000..5cbcff3 --- /dev/null +++ b/url_normalization.go @@ -0,0 +1,273 @@ +package apify + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "golang.org/x/net/idna" +) + +// The fields the OpenAPI specification declares as `format: uri` are returned in their +// normalized form, the same string for the same field in every Apify client (the JavaScript +// client returns the WHATWG URL serialization, and the Python one an equivalent): the scheme and +// host are lowercased, a non-ASCII host is converted to punycode, the default port is dropped, an +// empty path becomes "/", dot segments are resolved and characters that are unsafe in a URL are +// percent-encoded. + +// defaultPorts maps a special URL scheme to its default port, which normalization drops. +var defaultPorts = map[string]string{ + "http": "80", + "https": "443", + "ws": "80", + "wss": "443", + "ftp": "21", +} + +// urlSchemePattern matches the scheme at the start of an absolute URL. +var urlSchemePattern = regexp.MustCompile(`^([A-Za-z][A-Za-z0-9+.-]*):`) + +// maxPort is the largest valid TCP/UDP port number. +const maxPort = 65535 + +// Bytes that are percent-encoded in each URL component, in addition to every control character, +// space and non-ASCII byte (the WHATWG percent-encode sets). +const ( + unsafePathBytes = `"#<>?` + "`{}" + unsafeQueryBytes = `"#<>'` + unsafeFragmentBytes = `"<>` + "`" + unsafeUserinfoBytes = `"#<>?` + "`{}/:;=@[\\]^|" +) + +// idnaProfile converts an internationalized host name to its ASCII (punycode) form. The checks +// that only apply to registrable names (hyphen rules, bidi, joiners, label length) are off, +// because the host of a URL is not necessarily a registrable one. +var idnaProfile = idna.New( + idna.MapForLookup(), + idna.StrictDomainName(false), + idna.VerifyDNSLength(false), + idna.CheckHyphens(false), + idna.CheckJoiners(false), + idna.BidiRule(), +) + +// normalizeURL returns the normalized form of an absolute URL, or an error if raw is not one. +func normalizeURL(raw string) (string, error) { + rest := strings.TrimSpace(raw) + match := urlSchemePattern.FindStringSubmatch(rest) + if match == nil { + return "", fmt.Errorf("invalid URL %q: no scheme", raw) + } + scheme := strings.ToLower(match[1]) + rest = rest[len(match[0]):] + defaultPort, special := defaultPorts[scheme] + special = special || scheme == "file" + + var fragment string + hasFragment := false + if i := strings.IndexByte(rest, '#'); i >= 0 { + rest, fragment, hasFragment = rest[:i], rest[i+1:], true + } + var query string + hasQuery := false + if i := strings.IndexByte(rest, '?'); i >= 0 { + rest, query, hasQuery = rest[:i], rest[i+1:], true + } + if special { + rest = strings.ReplaceAll(rest, `\`, "/") + } + + var out strings.Builder + out.WriteString(scheme) + out.WriteByte(':') + path := rest + if authority, ok := strings.CutPrefix(rest, "//"); ok { + end := strings.IndexByte(authority, '/') + if end < 0 { + end = len(authority) + } + path = authority[end:] + host, err := normalizeAuthority(authority[:end], defaultPort) + if err != nil { + return "", fmt.Errorf("invalid URL %q: %w", raw, err) + } + out.WriteString("//") + out.WriteString(host) + } else if special { + return "", fmt.Errorf("invalid URL %q: %q URLs need a host", raw, scheme) + } + + if special && path == "" { + path = "/" + } + if special || strings.HasPrefix(path, "/") { + path = removeDotSegments(path) + } + out.WriteString(percentEncode(path, unsafePathBytes)) + if hasQuery { + unsafe := unsafeQueryBytes + if !special { + unsafe = `"#<>` + } + out.WriteByte('?') + out.WriteString(percentEncode(query, unsafe)) + } + if hasFragment { + out.WriteByte('#') + out.WriteString(percentEncode(fragment, unsafeFragmentBytes)) + } + return out.String(), nil +} + +// normalizeAuthority normalizes the "userinfo@host:port" part of a URL. +func normalizeAuthority(authority, defaultPort string) (string, error) { + var userinfo string + if i := strings.LastIndexByte(authority, '@'); i >= 0 { + userinfo, authority = authority[:i], authority[i+1:] + } + host, port := authority, "" + if strings.HasPrefix(authority, "[") { + end := strings.IndexByte(authority, ']') + if end < 0 { + return "", errors.New("unterminated IPv6 address") + } + host = strings.ToLower(authority[:end+1]) + port = strings.TrimPrefix(authority[end+1:], ":") + if tail := authority[end+1:]; tail != "" && !strings.HasPrefix(tail, ":") { + return "", errors.New("unexpected text after the IPv6 address") + } + } else { + if i := strings.LastIndexByte(authority, ':'); i >= 0 { + host, port = authority[:i], authority[i+1:] + } + if strings.ContainsFunc(host, isForbiddenHostRune) { + return "", fmt.Errorf("invalid host %q", host) + } + ascii, err := idnaProfile.ToASCII(strings.ToLower(host)) + if err != nil { + return "", fmt.Errorf("invalid host %q: %w", host, err) + } + host = ascii + } + if host == "" { + return "", errors.New("empty host") + } + if port != "" { + number, err := strconv.Atoi(port) + if err != nil || number < 0 || number > maxPort { + return "", fmt.Errorf("invalid port %q", port) + } + port = strconv.Itoa(number) + if port == defaultPort { + port = "" + } + } + + var out strings.Builder + if userinfo != "" { + username, password, hasPassword := strings.Cut(userinfo, ":") + out.WriteString(percentEncode(username, unsafeUserinfoBytes)) + if hasPassword { + out.WriteByte(':') + out.WriteString(percentEncode(password, unsafeUserinfoBytes)) + } + out.WriteByte('@') + } + out.WriteString(host) + if port != "" { + out.WriteByte(':') + out.WriteString(port) + } + return out.String(), nil +} + +// forbiddenHostBytes are the ASCII characters a host may not contain (besides control characters +// and the space). +const forbiddenHostBytes = "#%/<>?@[\\]^|" + +// isForbiddenHostRune reports whether r may not appear in a host. +func isForbiddenHostRune(r rune) bool { + return r <= ' ' || r == 0x7F || strings.ContainsRune(forbiddenHostBytes, r) +} + +// removeDotSegments resolves the "." and ".." segments of an absolute path. +func removeDotSegments(path string) string { + if path == "" { + return path + } + segments := strings.Split(path, "/") + kept := make([]string, 0, len(segments)) + for i, segment := range segments { + last := i == len(segments)-1 + switch strings.ToLower(segment) { + case ".", "%2e": + if last { + kept = append(kept, "") + } + case "..", ".%2e", "%2e.", "%2e%2e": + if len(kept) > 1 { + kept = kept[:len(kept)-1] + } + if last { + kept = append(kept, "") + } + default: + kept = append(kept, segment) + } + } + return strings.Join(kept, "/") +} + +// percentEncode percent-encodes every byte of s that is a control character, a space, a +// non-ASCII byte or listed in unsafe. An existing percent escape is kept as it is. +func percentEncode(s, unsafe string) string { + const upperHex = "0123456789ABCDEF" + var out strings.Builder + for i := 0; i < len(s); i++ { + b := s[i] + if b <= ' ' || b >= 0x7F || strings.IndexByte(unsafe, b) >= 0 { + out.WriteByte('%') + out.WriteByte(upperHex[b>>4]) + out.WriteByte(upperHex[b&0x0F]) + continue + } + out.WriteByte(b) + } + return out.String() +} + +// normalizeURLField normalizes the URL held in *field in place. An empty value (a missing or +// null field) is left as it is; any other value that is not an absolute URL is an error naming +// the field. +func normalizeURLField(name string, field *string) error { + if *field == "" { + return nil + } + normalized, err := normalizeURL(*field) + if err != nil { + return fmt.Errorf("field %q: %w", name, err) + } + *field = normalized + return nil +} + +// normalizeOptionalURLField is [normalizeURLField] for a nullable field held as a *string. +func normalizeOptionalURLField(name string, field **string) error { + if *field == nil { + return nil + } + return normalizeURLField(name, *field) +} + +// normalizeURLMap normalizes every value of a map of URLs in place. +func normalizeURLMap(name string, urls map[string]string) error { + for key, value := range urls { + if err := normalizeURLField(name+"."+key, &value); err != nil { + return err + } + urls[key] = value + } + return nil +} diff --git a/url_normalization_test.go b/url_normalization_test.go new file mode 100644 index 0000000..1b998e9 --- /dev/null +++ b/url_normalization_test.go @@ -0,0 +1,149 @@ +package apify + +import ( + "encoding/json" + "testing" +) + +func TestNormalizeURL(t *testing.T) { + cases := []struct{ raw, want string }{ + // The scheme and host are lowercased and the default port dropped. + {"HTTPS://Run-1.Example.COM:443", "https://run-1.example.com/"}, + {"http://example.com:80/a", "http://example.com/a"}, + {"https://example.com:8443/a", "https://example.com:8443/a"}, + // An empty path becomes "/", an existing path is kept. + {"https://example.com", "https://example.com/"}, + {"https://example.com/v2/datasets/abc/items?format=json", "https://example.com/v2/datasets/abc/items?format=json"}, + {"https://example.com?q=1", "https://example.com/?q=1"}, + // A non-ASCII host becomes punycode. + {"https://bücher.example/", "https://xn--bcher-kva.example/"}, + // Unsafe characters are percent-encoded; existing escapes are kept. + {"https://example.com/a b?q=1 2#f g", "https://example.com/a%20b?q=1%202#f%20g"}, + {"https://example.com/a%20b/é", "https://example.com/a%20b/%C3%A9"}, + {"https://example.com/a?x='\"'", "https://example.com/a?x=%27%22%27"}, + // Dot segments are resolved. + {"https://example.com/a/b/../c/./d", "https://example.com/a/c/d"}, + {"https://example.com/a/..", "https://example.com/"}, + // Userinfo and IPv6 hosts survive. + {"https://user:pa ss@Example.com/", "https://user:pa%20ss@example.com/"}, + {"http://[::1]:80/x", "http://[::1]/x"}, + // A URL that is not special keeps its path as it is. + {"mailto:Someone@Example.com", "mailto:Someone@Example.com"}, + // Surrounding whitespace is dropped, a backslash is a slash. + {" https://example.com\\a ", "https://example.com/a"}, + } + for _, c := range cases { + got, err := normalizeURL(c.raw) + if err != nil { + t.Errorf("normalizeURL(%q) failed: %v", c.raw, err) + continue + } + if got != c.want { + t.Errorf("normalizeURL(%q) = %q, want %q", c.raw, got, c.want) + } + again, err := normalizeURL(got) + if err != nil || again != got { + t.Errorf("normalizeURL is not idempotent for %q: got %q, %v", got, again, err) + } + } +} + +func TestNormalizeURLRejectsInvalidURLs(t *testing.T) { + for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "https:foo"} { + if got, err := normalizeURL(raw); err == nil { + t.Errorf("normalizeURL(%q) = %q, want an error", raw, got) + } + } +} + +func TestEveryURLFieldOfTheModelsIsNormalized(t *testing.T) { + const raw = "HTTP://Example.COM:80" + const want = "http://example.com/" + + var run ActorRun + mustUnmarshal(t, `{"id":"r","containerUrl":"`+raw+`","consoleUrl":"`+raw+`","output":{"results":"`+raw+`"}}`, &run) + if run.ContainerURL != want || run.ConsoleURL != want || run.Output["results"] != want { + t.Errorf("run URLs not normalized: %+v", run) + } + + var task Task + mustUnmarshal(t, `{"id":"t","standbyUrl":"`+raw+`"}`, &task) + if task.StandbyURL == nil || *task.StandbyURL != want { + t.Errorf("task standbyUrl not normalized: %v", task.StandbyURL) + } + var noStandby Task + mustUnmarshal(t, `{"id":"t","standbyUrl":null}`, &noStandby) + if noStandby.StandbyURL != nil { + t.Errorf("a null standbyUrl must stay nil, got %q", *noStandby.StandbyURL) + } + + var dataset Dataset + mustUnmarshal(t, `{"id":"d","consoleUrl":"`+raw+`","itemsPublicUrl":"`+raw+`"}`, &dataset) + if dataset.ConsoleURL != want || dataset.ItemsPublicURL != want { + t.Errorf("dataset URLs not normalized: %+v", dataset) + } + + var store KeyValueStore + mustUnmarshal(t, `{"id":"s","consoleUrl":"`+raw+`","keysPublicUrl":"`+raw+`","recordsPublicUrl":"`+raw+`"}`, &store) + if store.ConsoleURL != want || store.KeysPublicURL != want || store.RecordsPublicURL != want { + t.Errorf("store URLs not normalized: %+v", store) + } + + var key KeyValueStoreKey + mustUnmarshal(t, `{"key":"k","size":1,"recordPublicUrl":"`+raw+`"}`, &key) + if key.RecordPublicURL != want { + t.Errorf("key recordPublicUrl not normalized: %q", key.RecordPublicURL) + } + + var queue RequestQueue + mustUnmarshal(t, `{"id":"q","consoleUrl":"`+raw+`"}`, &queue) + if queue.ConsoleURL != want { + t.Errorf("queue consoleUrl not normalized: %q", queue.ConsoleURL) + } + + var webhook Webhook + mustUnmarshal(t, `{"id":"w","requestUrl":"`+raw+`"}`, &webhook) + if webhook.RequestURL != want { + t.Errorf("webhook requestUrl not normalized: %q", webhook.RequestURL) + } + var noURL Webhook + mustUnmarshal(t, `{"id":"w","requestUrl":null}`, &noURL) + if noURL.RequestURL != "" { + t.Errorf("a null requestUrl must stay empty, got %q", noURL.RequestURL) + } + + var dispatch WebhookDispatch + mustUnmarshal(t, `{"id":"x","webhookId":"w","webhook":{"_id":"w","actionType":"HTTP","isAdHoc":false,"requestUrl":"`+raw+`","slackMessage":{}}}`, &dispatch) + if dispatch.Webhook == nil || dispatch.Webhook.RequestURL != want || dispatch.Webhook.ID != "w" { + t.Errorf("dispatch webhook summary not parsed and normalized: %+v", dispatch.Webhook) + } + if _, ok := dispatch.Webhook.Extra["slackMessage"]; !ok { + t.Errorf("an unmodelled field of the summary must land in Extra: %v", dispatch.Webhook.Extra) + } + + var item ActorStoreListItem + mustUnmarshal(t, `{"id":"a","url":"`+raw+`","userPictureUrl":"`+raw+`"}`, &item) + if item.URL == nil || *item.URL != want || item.UserPictureURL == nil || *item.UserPictureURL != want { + t.Errorf("store item URLs not normalized: %+v", item) + } + + var user User + mustUnmarshal(t, `{"id":"u","username":"me","profile":{"name":"N","pictureUrl":"`+raw+`","websiteUrl":null}}`, &user) + if user.Profile == nil || user.Profile.PictureURL == nil || *user.Profile.PictureURL != want || user.Profile.WebsiteURL != nil { + t.Errorf("user profile URLs not normalized: %+v", user.Profile) + } +} + +func TestAnInvalidURLFieldFailsTheResponse(t *testing.T) { + var run ActorRun + if err := json.Unmarshal([]byte(`{"id":"r","consoleUrl":"not a url"}`), &run); err == nil { + t.Fatal("expected an error for an invalid URL field") + } +} + +func mustUnmarshal(t *testing.T, data string, v any) { + t.Helper() + if err := json.Unmarshal([]byte(data), v); err != nil { + t.Fatalf("unmarshal %s: %v", data, err) + } +} diff --git a/user.go b/user.go index 6f1d5e2..4b60eb4 100644 --- a/user.go +++ b/user.go @@ -3,37 +3,29 @@ package apify import ( "context" "encoding/json" - "errors" "net/http" ) // UserClient is a client for accessing user data (/v2/users/{userId} or /v2/users/me). // -// For the current user ("me"), it also exposes account usage and limits. Those endpoints -// only exist for "me" and return an error if called on another user's client. +// It also exposes account usage and limits. For another user than the current one, the API only +// serves them for your own account (by ID or username) or for the organization your API token +// belongs to, and rejects tokens with limited permissions. type UserClient struct { - ctx *resourceContext - isMe bool + ctx *resourceContext } func newUserClient(hc *httpClient, baseURL, id string) *UserClient { - return &UserClient{ - ctx: newSingleContext(hc, baseURL, "users", id), - isMe: id == meUserPlaceholder, - } + return &UserClient{ctx: newSingleContext(hc, baseURL, "users", id)} } -// errNotMe is returned by the me-only methods when called on another user's client. -var errNotMe = errors.New("this operation is only available for the current user (use Me())") - // Get fetches the user. For "me" it returns private account details; for other users it // returns the public profile. The bool reports whether the user exists. func (c *UserClient) Get(ctx context.Context) (User, bool, error) { return getResource[User](ctx, c.ctx, "", NewQueryParams()) } -// MonthlyUsage fetches the current account's monthly usage for the current month. Only -// available for "me". +// MonthlyUsage fetches the account's monthly usage for the current month. // // It returns the raw JSON usage report from the API (a JSON object with the account's usage // breakdown and totals for the period). @@ -49,11 +41,8 @@ type MonthlyUsageOptions struct { } // MonthlyUsageWithOptions is like [UserClient.MonthlyUsage] but can select another period -// with [MonthlyUsageOptions.Date]. Only available for "me". +// with [MonthlyUsageOptions.Date]. func (c *UserClient) MonthlyUsageWithOptions(ctx context.Context, options MonthlyUsageOptions) (json.RawMessage, error) { - if !c.isMe { - return nil, errNotMe - } params := NewQueryParams() if options.Date != "" { params.AddString("date", &options.Date) @@ -61,18 +50,15 @@ func (c *UserClient) MonthlyUsageWithOptions(ctx context.Context, options Monthl return getResourceRequired[json.RawMessage](ctx, c.ctx, "usage/monthly", params, c.ctx.shortTimeout()) } -// Limits fetches the current account's resource limits. Only available for "me". +// Limits fetches the account's resource limits. func (c *UserClient) Limits(ctx context.Context) (json.RawMessage, error) { - if !c.isMe { - return nil, errNotMe - } return getResourceRequired[json.RawMessage](ctx, c.ctx, "limits", NewQueryParams(), c.ctx.shortTimeout()) } -// UpdateLimits updates the current account's resource limits. Only available for "me". +// UpdateLimits updates the account's resource limits. func (c *UserClient) UpdateLimits(ctx context.Context, newLimits any) error { - if !c.isMe { - return errNotMe + if c.ctx.idErr != nil { + return c.ctx.idErr } data, err := json.Marshal(newLimits) if err != nil { diff --git a/version.go b/version.go index bb1dd5b..eeaa11e 100644 --- a/version.go +++ b/version.go @@ -4,10 +4,10 @@ package apify // // It follows Semantic Versioning (https://semver.org/). Changes to the public // interface (other than additive ones) are considered breaking changes. -const ClientVersion = "0.13.1" +const ClientVersion = "0.14.0" // APISpecVersion is the version of the Apify OpenAPI specification that this // client was generated and verified against. // // It corresponds to the `info.version` field of the Apify OpenAPI document. -const APISpecVersion = "v2-2026-10-26T112451Z" +const APISpecVersion = "v2-2026-10-29T100357Z" From 3402c95e5e39c4e6eaa72f299d0784dca726880d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 12:40:55 +0000 Subject: [PATCH 02/12] fix: address the first review of the v3 port --- CHANGELOG.md | 19 +- README.md | 46 +-- actor_version.go | 24 +- call_test.go | 63 ++++ dataset_push_test.go | 73 +++++ docs/actors.md | 2 +- docs/runs.md | 17 +- docs/storages.md | 9 +- docs/tasks.md | 2 +- examples/log_redirection/main.go | 5 +- go.mod | 9 +- go.sum | 4 - key_value_store.go | 40 +-- key_value_store_test.go | 31 ++ spec_sync_test.go => last_run_test.go | 70 ----- models_test.go | 31 ++ request_queue.go | 2 + run_collection.go | 5 +- run_collection_test.go | 35 +++ store_collection_test.go | 24 ++ streamed_log.go | 60 +++- streamed_log_test.go | 88 +++++- tests/actor_test.go | 4 +- tests/iterate_test.go | 2 +- timeout_retry_test.go | 191 ----------- timeout_tier_test.go | 49 +++ url_normalization.go | 435 +++++++++++++++++++++++--- url_normalization_test.go | 27 +- 28 files changed, 937 insertions(+), 430 deletions(-) create mode 100644 call_test.go create mode 100644 dataset_push_test.go create mode 100644 key_value_store_test.go rename spec_sync_test.go => last_run_test.go (55%) create mode 100644 models_test.go create mode 100644 run_collection_test.go create mode 100644 store_collection_test.go create mode 100644 timeout_tier_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fa20827..cde938a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,14 +28,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `KeyValueStoreKey.RecordPublicURL`, `RequestQueue.ConsoleURL`, `Task.StandbyURL`, `ActorStoreListItem.URL`/`UserPictureURL`, `User.Profile` (`UserProfile`) and `WebhookDispatch.Webhook` (`WebhookDispatchWebhookSummary`). The URL fields the API specification - declares (these, `ActorRun.ContainerURL` and `Webhook.RequestURL`) are returned normalized: the - scheme and host are lowercased, a non-ASCII host becomes punycode, a default port is dropped, an - empty path becomes `/`, dot segments are resolved and unsafe characters are percent-encoded. A - value that is not an absolute URL fails decoding the response. + declares (these, `ActorRun.ContainerURL` and `Webhook.RequestURL`) are returned normalized, close + to the WHATWG URL serialization: the scheme and host are lowercased, a non-ASCII host becomes + punycode, an IPv4 or IPv6 host is written canonically, a default port is dropped, an empty path + becomes `/`, dot segments are resolved and unsafe characters are percent-encoded. A value that + is not an absolute URL fails decoding the response. ### Changed - Bumped `APISpecVersion` to `v2-2026-10-29T100357Z` and `ClientVersion` to `0.14.0`. +- **Breaking:** `ActorVersionCollectionClient.List` takes no options and `Iterate` takes no + arguments, like the environment-variable collection: the versions endpoint documents no query + parameters and always returns every version in one response. Matches the reference client, + whose versions `list()` takes no arguments. +- The keys of the new typed fields (`consoleUrl`, `output`, `itemsPublicUrl`, `keysPublicUrl`, + `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, `url`, `userPictureUrl`, `profile` and a + dispatch's `webhook`) are no longer in the `Extra` map of their model. +- `ListRequestsOptions.ExclusiveStartID` is marked deprecated: the API deprecated the + `exclusiveStartId` parameter in favor of `cursor`. - `ActorClient.Call`/`CallRaw` and `TaskClient.Call` start the run without a client-side timeout, so a slow start is no longer aborted and repeated, which started a second run. - A request that timed out is no longer retried when it is not idempotent: `Start`, `StartRaw` @@ -45,7 +55,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The storages and the log of a run reached through `LastRun`/`LastRunWithOptions`, and their public URLs, carry its `status` and `origin` filters, so they address the run the filters resolve to instead of the newest run of any status. -- Added `golang.org/x/net` as a dependency, for the punycode conversion of URL hosts. - Documented that `GetRecord` returns the raw bytes without parsing them and that `SetRecordRaw` sends them as they are, whatever the content type. diff --git a/README.md b/README.md index 1f5d9bc..d582c0d 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,8 @@ request queues, tasks, schedules, webhooks, the store, users and logs). and request-queue iterators, live iteration of a running Actor's dataset items, dataset export, signed public URLs, request-queue locking, and more. - Forward-compatible models that keep unknown API fields in an `Extra` map. -- Two third-party dependencies: `github.com/andybalholm/brotli` (Brotli request-body - compression) and `golang.org/x/net/idna` (punycode conversion of hosts when normalizing URL - fields); everything else is the Go standard library. +- A single third-party dependency (`github.com/andybalholm/brotli`, used for Brotli + request-body compression); everything else is the Go standard library. ## Installation @@ -126,19 +125,20 @@ func main() { ### Timeouts Every request the client sends is assigned one of three timeout tiers, matching the kind of -operation its method performs — `short` for a metadata read/write, `medium` for a listing, -batch or trigger call, `long` for a download, upload or streaming call — except the polling -requests behind `WaitForFinish`/`Call`, and the start request of `Call`, which run with no -client-imposed timeout of their own (the time budget `waitSecs` bounds the overall wait, and the -`context.Context` the call itself). The duration of -each tier is configured on the client (see the table above); a timeout still grows with each -retry, capped at `WithTimeoutMax`. - -A single call that needs a different bound already has full control via the -`context.Context` every method takes: canceling it, or giving it an earlier deadline with -`context.WithTimeout`, overrides whichever tier the method would otherwise use (the client's -internal per-attempt deadline is always the *later* of the two to apply — i.e. -`context.WithTimeout` composes with whichever is sooner). +operation its method performs: `short` for a metadata read/write, `medium` for a listing, batch or +trigger call, `long` for a download, upload or streaming call. The duration of each tier is +configured on the client (see the table above); a timeout still grows with each retry, capped at +`WithTimeoutMax`. + +Two kinds of request have no client-side timeout of their own: the polling requests behind +`WaitForFinish` and `Call`, and the start request of `Call`. `waitSecs` bounds how long `Call` and +`WaitForFinish` wait for the run, and the `context.Context` bounds the whole call. + +Every method takes a `context.Context`, and its deadline or cancellation can only shorten a +request: the request ends when its tier's timeout or the context's deadline passes, whichever is +sooner. To give one call a shorter bound than its tier, pass a context created with +`context.WithTimeout`; a context cannot extend a tier, so raise the tier (`WithTimeoutShort`, +`WithTimeoutMedium`, `WithTimeoutLong`, `WithTimeoutMax`) for a call that needs longer. A request that timed out is retried like any other transient failure, with a longer timeout, except one that is not idempotent, where the timed-out attempt may already have taken effect: @@ -173,12 +173,14 @@ variable the JavaScript reference client reads); if it is set to a non-empty val The fields the API specification declares as URLs (`consoleUrl`, `containerUrl`, `itemsPublicUrl`, `keysPublicUrl`, `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, the webhook `requestUrl`, a run's `output` links, a Store Actor's `url` and `userPictureUrl`, and a user profile's -`pictureUrl` and `websiteUrl`) are returned in a normalized form, so every Apify client returns -the same string for the same field: the scheme and host are lowercased, a non-ASCII host becomes -punycode, a default port is dropped, an empty path becomes `/`, dot segments are resolved, and -unsafe characters are percent-encoded. Compare a field against a stored URL after normalizing -the stored one the same way (for example by reading it back from the API). A value that is not -an absolute URL makes decoding the response fail. +`pictureUrl` and `websiteUrl`) are returned in a normalized form, close to the WHATWG URL serialization that the JavaScript +client returns: the scheme and host are lowercased, a non-ASCII host becomes punycode, an IPv4 or +IPv6 host is written canonically, a default port is dropped, an empty path becomes `/`, dot +segments are resolved, and unsafe characters are percent-encoded. It covers the http(s), ws(s), +ftp and file URLs the API returns; it does not apply the full Unicode (UTS #46) mapping to a +non-ASCII host. Compare a field against a stored URL after normalizing the stored one the same +way (for example by reading it back from the API). A value that is not an absolute URL makes +decoding the response fail. ## Resource clients diff --git a/actor_version.go b/actor_version.go index bf050d8..c53344b 100644 --- a/actor_version.go +++ b/actor_version.go @@ -12,22 +12,18 @@ func newActorVersionCollectionClient(hc *httpClient, actorURL string) *ActorVers return &ActorVersionCollectionClient{ctx: newCollectionContext(hc, actorURL, "versions")} } -// List lists the Actor's versions. -func (c *ActorVersionCollectionClient) List(ctx context.Context, options ListOptions) (PaginationList[ActorVersion], error) { - params := NewQueryParams() - options.apply(params) - return listResource[ActorVersion](ctx, c.ctx, "", params) +// List lists the Actor's versions. The endpoint takes no pagination parameters (per the OpenAPI +// specification, it always returns every version in one response), so neither does this method. +func (c *ActorVersionCollectionClient) List(ctx context.Context) (PaginationList[ActorVersion], error) { + return listResource[ActorVersion](ctx, c.ctx, "", NewQueryParams()) } -// Iterate returns a lazy iterator over the Actor's versions. The versions endpoint ignores -// offset/limit/desc (per the OpenAPI specification, it always returns every version in one -// response), so options and chunkSize are accepted only for interface symmetry with the other -// collections' Iterate and have no effect; the iterator drains a single page. Mirrors the -// reference client's versions().list(), which likewise exposes no pagination options. -func (c *ActorVersionCollectionClient) Iterate(options ListOptions, chunkSize *int64) *ListIterator[ActorVersion] { - return newUnpaginatedIterator(func(ctx context.Context) (PaginationList[ActorVersion], error) { - return c.List(ctx, options) - }) +// Iterate returns a lazy iterator over the Actor's versions. The versions endpoint is not +// offset-paginated (it always returns every version in one response), so there is no +// Limit/chunkSize control; the iterator drains that one page. Mirrors the reference client's +// versions().list(), which likewise exposes no pagination options. +func (c *ActorVersionCollectionClient) Iterate() *ListIterator[ActorVersion] { + return newUnpaginatedIterator(c.List) } // Create creates a new Actor version. version is any JSON-serializable version definition. diff --git a/call_test.go b/call_test.go new file mode 100644 index 0000000..0662daa --- /dev/null +++ b/call_test.go @@ -0,0 +1,63 @@ +package apify + +import ( + "context" + "testing" + "time" +) + +// noLogRedirection turns the log redirection of Call off, so a test counts only the requests of +// the call itself. +var noLogRedirection = &StreamedLogOptions{Disabled: true} + +// Call has no client-side timeout of its own: a start slower than the medium tier must neither be +// aborted nor repeated (which would start a second, billed run). Start itself is bound by it. +func TestCallStartsWithoutATimeout(t *testing.T) { + run := `{"data":{"id":"run1","status":"SUCCEEDED"}}` + t.Run("Actor Call", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + got, err := slowClient(backend).Actor("a").Call(context.Background(), map[string]any{}, ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))) + if err != nil || got.ID != "run1" { + t.Fatalf("Call: %+v, %v", got, err) + } + if n := backend.calls.Load(); n != 2 { // the start and one poll + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Actor CallRaw", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Actor("a").CallRaw(context.Background(), []byte("x"), ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))); err != nil { + t.Fatalf("CallRaw: %v", err) + } + if n := backend.calls.Load(); n != 2 { + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Task Call", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Task("t").Call(context.Background(), nil, TaskStartOptions{}, Ptr(int64(0))); err != nil { + t.Fatalf("Task Call: %v", err) + } + if n := backend.calls.Load(); n != 2 { + t.Fatalf("expected the start once and one poll, got %d requests", n) + } + }) + t.Run("Start stays bound by the medium tier", func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: run} + if _, err := slowClient(backend).Actor("a").Start(context.Background(), nil, ActorStartOptions{}); err == nil { + t.Fatal("expected Start to time out") + } + }) + t.Run("ctx still bounds Call", func(t *testing.T) { + backend := &slowBackend{delay: time.Minute, body: run} + ctx, cancel := context.WithTimeout(context.Background(), 3*tinyTier) + defer cancel() + started := time.Now() + if _, err := slowClient(backend).Actor("a").Call(ctx, nil, ActorStartOptions{Log: noLogRedirection}, nil); err == nil { + t.Fatal("expected an error once ctx is done") + } + if time.Since(started) > 5*time.Second { + t.Fatal("Call did not stop at the ctx deadline") + } + }) +} diff --git a/dataset_push_test.go b/dataset_push_test.go new file mode 100644 index 0000000..51c2a66 --- /dev/null +++ b/dataset_push_test.go @@ -0,0 +1,73 @@ +package apify + +import ( + "context" + "encoding/json" + "testing" +) + +func TestPushItemsAcceptsOnlyObjectsAndArraysOfObjects(t *testing.T) { + type item struct { + A int `json:"a"` + } + valid := map[string]any{ + "map": map[string]any{"a": 1}, + "struct": item{A: 1}, + "struct pointer": &item{A: 1}, + "slice of structs": []item{{A: 1}, {A: 2}}, + "slice of maps": []map[string]any{{"a": 1}}, + "empty slice": []item{}, + "slice of any": []any{map[string]any{"a": 1}, item{A: 2}}, + "raw JSON object": rawJSON(`{"a":1}`), + "raw JSON array": rawJSON(`[{"a":1},{"a":2}]`), + "padded raw JSON obj": rawJSON(" \n{\"a\":1} "), + } + for name, items := range valid { + t.Run("accepts "+name, func(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err != nil { + t.Fatalf("PushItems rejected a valid shape: %v", err) + } + if backend.calls != 1 { + t.Fatalf("expected 1 request, got %d", backend.calls) + } + }) + } + + invalid := map[string]any{ + "string": "[1,2]", + "[]byte": []byte(`{"a":1}`), + "number": 1, + "bool": true, + "nil": nil, + "slice of strings": []string{"a", "b"}, + "slice of numbers": []int{1, 2}, + "mixed slice": []any{map[string]any{"a": 1}, "b"}, + "slice of slices": [][]int{{1}}, + "raw JSON string": rawJSON(`"a"`), + "raw JSON null element": rawJSON(`[{"a":1},null]`), + } + for name, items := range invalid { + t.Run("rejects "+name, func(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err == nil { + t.Fatal("PushItems accepted a shape the API rejects") + } + if backend.calls != 0 { + t.Fatalf("nothing may be sent for an invalid shape, got %d requests", backend.calls) + } + }) + } +} + +func TestPushItemsRejectsAnInvalidDatasetIDBeforeSending(t *testing.T) { + backend := &mockBackend{responses: okData()} + if err := testClient(backend, 0).Dataset("").PushItems(context.Background(), map[string]any{"a": 1}); err == nil { + t.Fatal("expected an error for an empty dataset ID") + } + if backend.calls != 0 { + t.Fatalf("nothing may be sent for an invalid ID, got %d requests", backend.calls) + } +} + +func rawJSON(s string) json.RawMessage { return json.RawMessage(s) } diff --git a/docs/actors.md b/docs/actors.md index d49d9ac..cee353d 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -198,7 +198,7 @@ run, err := client.Actor("apify/hello-world").Call(ctx, | Method | Description | | --- | --- | -| `Versions().List(ctx, ListOptions)` / `Versions().Iterate(ListOptions, chunkSize *int64)` / `Versions().Create(ctx, def)` | List/iterate/create versions. | +| `Versions().List(ctx)` / `Versions().Iterate()` / `Versions().Create(ctx, def)` | List/iterate/create versions. The endpoint returns every version in one response and takes no pagination parameters, so neither method takes options. | | `Version(n).Get/Update/Delete(ctx)` | Manage a single version. | | `Version(n).EnvVars().List(ctx)` / `.Iterate()` / `.Create(ctx, ActorEnvVar)` | List/iterate/create env vars. | | `Version(n).EnvVar(name).Get/Update/Delete(ctx)` | Manage a single env var. | diff --git a/docs/runs.md b/docs/runs.md index 17e07e3..ca06da9 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -20,7 +20,7 @@ with `client.Run(id)`, and an Actor's or task's runs with `client.Actor(id).Runs | `StartedBefore` | `*string` | Only runs started before this ISO-8601 timestamp. | | `Deployment` | `[]string` | Only Actor Standby runs served by these pools (`SINGLE_TENANT`, `MULTI_TENANT_FREE_USERS`, `MULTI_TENANT_PAYING_USERS`, `MULTI_TENANT_OWNER_PAID`); sent as a comma-separated list. The API requires `Status` to hold only non-terminal statuses (`READY`, `RUNNING`, `TIMING-OUT`, `ABORTING`) with it. | -The time filters apply only to Actor- and task-scoped collections, and `Deployment` only to the account-wide and Actor-scoped ones. +`StartedAfter`, `StartedBefore` and `Deployment` apply only to the account-wide (`client.Runs()`) and Actor-scoped (`client.Actor(id).Runs()`) collections, not to a task's. ## Single run @@ -42,7 +42,7 @@ The time filters apply only to Actor- and task-scoped collections, and `Deployme | `RequestQueue() *RequestQueueClient` | The run's default request queue. | | `Log() *LogClient` | The run's log. | | `GetStreamedLog(ctx) (io.ReadCloser, error)` | Live stream of the run's raw log, as bytes. | -| `NewStreamedLog(ctx, StreamedLogOptions) (*StreamedLog, error)` | Redirects the run's log message by message to a writer; see [Log redirection](#log-redirection). `(nil, nil)` for a run that does not exist (with the default prefix). | +| `NewStreamedLog(ctx, StreamedLogOptions) (*StreamedLog, bool, error)` | Redirects the run's log message by message to a writer; see [Log redirection](#log-redirection). The bool is `false` for a run that does not exist (with the default prefix). | | `IterateDatasetItems(RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[json.RawMessage]` | Iterates over the items of the run's default dataset while the run is still producing them; see [Live iteration of dataset items](#live-iteration-of-dataset-items). | On a client from `LastRun(...)`, `Dataset()`, `KeyValueStore()`, `RequestQueue()` and `Log()` carry @@ -120,8 +120,8 @@ run's metadata. The commonly used fields: | `DefaultKeyValueStoreID` | `string` | ID of the run's default key-value store. | | `DefaultRequestQueueID` | `string` | ID of the run's default request queue. | | `ContainerURL` | `string` | URL of the run's container (for live access), normalized. | -| `ConsoleURL` | `string` | URL of the run in the Apify Console, normalized. | -| `Output` | `map[string]string` | Output links of the run (name to normalized URL), generated from the output schema of the Actor build. `nil` if the build defines none. | +| `ConsoleURL` | `string` | URL of the run in the Apify Console, normalized. Empty on items from `List`/`Iterate`. | +| `Output` | `map[string]string` | Output links of the run (name to normalized URL), generated from the output schema of the Actor build. `nil` if the build defines none, and on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API (forward compatibility). | `ActorRun.IsTerminal()` reports whether a run has finished. Status message convenience: @@ -134,10 +134,13 @@ run's metadata. The commonly used fields: change where it goes or turn it off). To redirect the log of any run, create a `StreamedLog`: ```go -streamed, err := client.Run(runID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) +streamed, found, err := client.Run(runID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) if err != nil { log.Fatal(err) } +if !found { + log.Fatal("run not found") +} if err := streamed.Start(ctx); err != nil { log.Fatal(err) } @@ -160,7 +163,9 @@ The log is split into messages at the timestamps the platform puts in front of e message that spans several lines is written as one. `Start` begins the redirection in the background, and it also ends when the `ctx` it got is done. `Stop` gives the messages in flight up to one second, then aborts the stream; if the stream delivered nothing yet, it reads the whole log -in one request, so a short run that finished before the stream connected is not lost. A log that +in one request, so a short run that finished before the stream connected is not lost. The `ctx` +given to `Stop` bounds that wait and that read: when it is done first, `Stop` returns its error +and the rest of the log is not redirected. A second `Stop` while one is running returns an error. A log that has no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, until its first bytes arrive. `Stop` returns the error that ended the redirection early, if any; `Call` reports it on the redirection's output and goes on. diff --git a/docs/storages.md b/docs/storages.md index ef4d561..f45c124 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -36,8 +36,8 @@ The `Dataset` value returned by `Get`/`GetOrCreate`/`Update` and listed by `List | `CreatedAt` | `*time.Time` | When the dataset was created. | | `ModifiedAt` | `*time.Time` | When the dataset was last modified. | | `ItemCount` | `int64` | Number of items currently stored. | -| `ConsoleURL` | `string` | URL of the dataset in the Apify Console (normalized). | -| `ItemsPublicURL` | `string` | Public URL of the dataset's items (normalized); signed for a private dataset. | +| `ConsoleURL` | `string` | URL of the dataset in the Apify Console (normalized). Empty on items from `List`/`Iterate`. | +| `ItemsPublicURL` | `string` | Public URL of the dataset's items (normalized); signed for a private dataset. Empty on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | `DatasetListItemsOptions` controls filtering, projection, and pagination of the items @@ -192,6 +192,9 @@ The `KeyValueStore` value returned by `Get`/`GetOrCreate`/`Update` and listed by | `UserID` | `string` | ID of the user who owns the store. | | `CreatedAt` | `*time.Time` | When the store was created. | | `ModifiedAt` | `*time.Time` | When the store was last modified. | +| `ConsoleURL` | `string` | URL of the store in the Apify Console (normalized). Empty on items from `List`/`Iterate`, which do not carry it. | +| `KeysPublicURL` | `string` | Public URL listing the store's keys (normalized); signed for a private store. Empty on items from `List`/`Iterate`. | +| `RecordsPublicURL` | `string` | Public URL prefix of the store's records (normalized). Empty on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | Option structs (all fields optional): @@ -304,7 +307,7 @@ The `RequestQueue` value returned by `Get`/`GetOrCreate`/`Update` and listed by | `CreatedAt` | `*time.Time` | When the queue was created. | | `ModifiedAt` | `*time.Time` | When the queue was last modified. | | `TotalRequestCount` | `int64` | Total number of requests ever added. | -| `ConsoleURL` | `string` | URL of the queue in the Apify Console (normalized). | +| `ConsoleURL` | `string` | URL of the queue in the Apify Console (normalized). Empty on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | `RequestQueueRequest` is the request payload/record. `URL` is required; `ID` is assigned by the diff --git a/docs/tasks.md b/docs/tasks.md index c4f7f6b..ce63c70 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -25,7 +25,7 @@ A task is a pre-configured Actor run with stored input. Access the task collecti | `ModifiedAt` | `*time.Time` | When the task was last modified. | | `IsPublic` | `*bool` | Whether the task is published on its public landing page, derived from `PublicConfig.PublishedAt`; use `Publish`/`Unpublish` to change it. | | `PublicConfig` | `*TaskPublicConfig` | Public-facing display configuration of the landing page, set once the task has been configured for publishing. | -| `StandbyURL` | `*string` | URL of the task's Actor Standby endpoint, normalized. `nil` when the task has none. | +| `StandbyURL` | `*string` | URL of the task's Actor Standby endpoint, normalized. `nil` when the task has none, and on items from `List`/`Iterate`. | | `Extra` | `map[string]json.RawMessage` | Any other fields returned by the API. | ### `TaskPublicConfig` fields diff --git a/examples/log_redirection/main.go b/examples/log_redirection/main.go index 49a68bc..f24d93f 100644 --- a/examples/log_redirection/main.go +++ b/examples/log_redirection/main.go @@ -32,10 +32,13 @@ func main() { // To redirect the log of a run that was started elsewhere, create the redirection for it and // stop it once the run is done. Starting it for a finished run reads the whole log. - streamed, err := client.Run(run.ID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) + streamed, found, err := client.Run(run.ID).NewStreamedLog(ctx, apify.StreamedLogOptions{Output: os.Stdout}) if err != nil { log.Fatalf("create log redirection: %v", err) } + if !found { + log.Fatal("run not found") + } if err := streamed.Start(ctx); err != nil { log.Fatalf("start log redirection: %v", err) } diff --git a/go.mod b/go.mod index 91c6357..c9a8d11 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,7 @@ module github.com/apify/apify-client-go -go 1.23.0 +go 1.23 toolchain go1.26.10 -require ( - github.com/andybalholm/brotli v1.2.2 - golang.org/x/net v0.38.0 -) - -require golang.org/x/text v0.23.0 // indirect +require github.com/andybalholm/brotli v1.2.2 diff --git a/go.sum b/go.sum index 4f9b9e0..80d4b3a 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,3 @@ github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtn github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= diff --git a/key_value_store.go b/key_value_store.go index a0787aa..787aca2 100644 --- a/key_value_store.go +++ b/key_value_store.go @@ -270,13 +270,11 @@ func (c *KeyValueStoreClient) GetRecordStreamWithOptions(ctx context.Context, ke }, true, nil } -// SetRecordRaw stores a record with raw bytes and the given content type. +// SetRecordRaw stores a record with raw bytes and the given content type. The bytes are sent as +// they are, whatever the content type: the client neither parses nor converts them, so a value +// stored as application/json must already be valid JSON. func (c *KeyValueStoreClient) SetRecordRaw(ctx context.Context, key string, value []byte, contentType string) error { - safeKey, err := safePathSegment(key) - if err != nil { - return err - } - return putRaw(ctx, c.ctx, "records/"+safeKey, NewQueryParams(), value, contentType) + return c.setRecord(ctx, key, value, contentType, "", false, false) } // SetRecordRawWithEncoding is like [KeyValueStoreClient.SetRecordRaw], for a value that is @@ -287,11 +285,26 @@ func (c *KeyValueStoreClient) SetRecordRaw(ctx context.Context, key string, valu // contentEncoding == "") for a value that is not pre-compressed. Mirrors the reference client's // setRecord({ contentEncoding }). func (c *KeyValueStoreClient) SetRecordRawWithEncoding(ctx context.Context, key string, value []byte, contentType, contentEncoding string) error { + return c.setRecord(ctx, key, value, contentType, contentEncoding, true, false) +} + +// setRecord is the shared implementation of the SetRecord methods. preEncoded sends value exactly +// as given under contentEncoding (which may be empty), skipping the client's own request +// compression; doNotRetryTimeouts makes an upload attempt that timed out fail at once. +func (c *KeyValueStoreClient) setRecord(ctx context.Context, key string, value []byte, contentType, contentEncoding string, preEncoded, doNotRetryTimeouts bool) error { safeKey, err := safePathSegment(key) if err != nil { return err } - return putRawWithEncoding(ctx, c.ctx, "records/"+safeKey, NewQueryParams(), value, contentType, contentEncoding) + resource := c.ctx + if doNotRetryTimeouts { + resource = resource.withoutTimeoutRetry() + } + subPath := "records/" + safeKey + if preEncoded { + return putRawWithEncoding(ctx, resource, subPath, NewQueryParams(), value, contentType, contentEncoding) + } + return putRaw(ctx, resource, subPath, NewQueryParams(), value, contentType) } // SetRecordOptions configures [KeyValueStoreClient.SetRecordRawWithOptions] and @@ -311,18 +324,7 @@ type SetRecordOptions struct { // SetRecordRawWithOptions is like [KeyValueStoreClient.SetRecordRaw], with the per-call // [SetRecordOptions]. func (c *KeyValueStoreClient) SetRecordRawWithOptions(ctx context.Context, key string, value []byte, contentType string, options SetRecordOptions) error { - safeKey, err := safePathSegment(key) - if err != nil { - return err - } - resource := c.ctx - if options.DoNotRetryTimeouts { - resource = resource.withoutTimeoutRetry() - } - if options.ContentEncoding != "" { - return putRawWithEncoding(ctx, resource, "records/"+safeKey, NewQueryParams(), value, contentType, options.ContentEncoding) - } - return putRaw(ctx, resource, "records/"+safeKey, NewQueryParams(), value, contentType) + return c.setRecord(ctx, key, value, contentType, options.ContentEncoding, options.ContentEncoding != "", options.DoNotRetryTimeouts) } // SetRecordJSON stores a record holding the JSON serialization of value. diff --git a/key_value_store_test.go b/key_value_store_test.go new file mode 100644 index 0000000..cc3a3a4 --- /dev/null +++ b/key_value_store_test.go @@ -0,0 +1,31 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +func TestSetRecordRawWithOptionsSendsTheContentEncoding(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + value := []byte(strings.Repeat("x", 2*minCompressRequestBytes)) + + err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{ContentEncoding: "gzip"}) + if err != nil { + t.Fatalf("SetRecordRawWithOptions: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got != "gzip" { + t.Fatalf("expected Content-Encoding gzip, got %q", got) + } + if backend.lastBody != string(value) { + t.Fatal("a pre-compressed value must be uploaded as given, never compressed again") + } + + if err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{}); err != nil { + t.Fatalf("SetRecordRawWithOptions: %v", err) + } + if got := backend.lastHeaders.Get("Content-Encoding"); got == "" { + t.Fatal("without a ContentEncoding the client compresses a large value of its own accord") + } +} diff --git a/spec_sync_test.go b/last_run_test.go similarity index 55% rename from spec_sync_test.go rename to last_run_test.go index f124038..5fe8a45 100644 --- a/spec_sync_test.go +++ b/last_run_test.go @@ -2,55 +2,10 @@ package apify import ( "context" - "net/url" "strings" "testing" ) -func TestRunListsSendTheDeploymentFilter(t *testing.T) { - ctx := context.Background() - filter := RunListOptions{Status: []string{"RUNNING"}, Deployment: []string{"SINGLE_TENANT", "MULTI_TENANT_FREE_USERS"}} - - for name, list := range map[string]func(client *ApifyClient) error{ - "account runs": func(c *ApifyClient) error { _, err := c.Runs().List(ctx, ListOptions{}, filter); return err }, - "actor runs": func(c *ApifyClient) error { _, err := c.Actor("a").Runs().List(ctx, ListOptions{}, filter); return err }, - } { - backend := &mockBackend{responses: okData()} - if err := list(testClient(backend, 0)); err != nil { - t.Fatalf("%s: %v", name, err) - } - query, _ := url.ParseQuery(strings.SplitN(backend.lastURL, "?", 2)[1]) - if got := query.Get("deployment"); got != "SINGLE_TENANT,MULTI_TENANT_FREE_USERS" { - t.Errorf("%s: deployment = %q", name, got) - } - } - - backend := &mockBackend{responses: okData()} - if _, err := testClient(backend, 0).Runs().List(ctx, ListOptions{}, RunListOptions{}); err != nil { - t.Fatal(err) - } - if strings.Contains(backend.lastURL, "deployment") { - t.Errorf("deployment must be omitted when unset, got %q", backend.lastURL) - } -} - -func TestStoreListSendsIncludeInputSchema(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - if _, err := client.Store().List(context.Background(), StoreListOptions{IncludeInputSchema: Ptr(true)}); err != nil { - t.Fatal(err) - } - if !strings.Contains(backend.lastURL, "includeInputSchema=1") { - t.Errorf("expected includeInputSchema=1, got %q", backend.lastURL) - } - if _, err := client.Store().List(context.Background(), StoreListOptions{}); err != nil { - t.Fatal(err) - } - if strings.Contains(backend.lastURL, "includeInputSchema") { - t.Errorf("includeInputSchema must be omitted when unset, got %q", backend.lastURL) - } -} - // The storages and the log of a run reached through LastRun carry its status and origin filters, // so they address the storages and the log of the same run, not those of the newest run of any // status. @@ -142,28 +97,3 @@ func TestStoragesOfAnUnfilteredRunHaveNoFilterParams(t *testing.T) { t.Errorf("unexpected query string in %q", backend.lastURL) } } - -// The API returns a pending request with a null handledAt, and a schedule whose Console editor -// state was dropped with a null uiState: both parse, the null kept in Extra. -func TestNullFieldsOfRequestsAndSchedulesParse(t *testing.T) { - var request RequestQueueRequest - mustUnmarshal(t, `{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}`, &request) - if request.ID != "r1" || request.UniqueKey != "k" || string(request.Extra["handledAt"]) != "null" { - t.Errorf("unexpected request %+v", request) - } - if request.Method != "" || request.UserData != nil { - t.Errorf("the optional fields of a minimal request stay empty: %+v", request) - } - - var schedule Schedule - mustUnmarshal(t, `{"id":"s1","userId":"u","name":"n","cronExpression":"* * * * *","isEnabled":true,"uiState":null}`, &schedule) - if schedule.ID != "s1" || string(schedule.Extra["uiState"]) != "null" { - t.Errorf("unexpected schedule %+v", schedule) - } - - backend := &mockBackend{responses: constant(200, `{"data":{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}}`)} - got, present, err := testClient(backend, 0).RequestQueue("q").GetRequest(context.Background(), "r1") - if err != nil || !present || got.ID != "r1" { - t.Fatalf("GetRequest of a pending request: %+v, %v, %v", got, present, err) - } -} diff --git a/models_test.go b/models_test.go new file mode 100644 index 0000000..610107e --- /dev/null +++ b/models_test.go @@ -0,0 +1,31 @@ +package apify + +import ( + "context" + "testing" +) + +// The API returns a pending request with a null handledAt, and a schedule whose Console editor +// state was dropped with a null uiState: both parse, the null kept in Extra. +func TestNullFieldsOfRequestsAndSchedulesParse(t *testing.T) { + var request RequestQueueRequest + mustUnmarshal(t, `{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}`, &request) + if request.ID != "r1" || request.UniqueKey != "k" || string(request.Extra["handledAt"]) != "null" { + t.Errorf("unexpected request %+v", request) + } + if request.Method != "" || request.UserData != nil { + t.Errorf("the optional fields of a minimal request stay empty: %+v", request) + } + + var schedule Schedule + mustUnmarshal(t, `{"id":"s1","userId":"u","name":"n","cronExpression":"* * * * *","isEnabled":true,"uiState":null}`, &schedule) + if schedule.ID != "s1" || string(schedule.Extra["uiState"]) != "null" { + t.Errorf("unexpected schedule %+v", schedule) + } + + backend := &mockBackend{responses: constant(200, `{"data":{"id":"r1","url":"https://example.com/","uniqueKey":"k","handledAt":null}}`)} + got, present, err := testClient(backend, 0).RequestQueue("q").GetRequest(context.Background(), "r1") + if err != nil || !present || got.ID != "r1" { + t.Fatalf("GetRequest of a pending request: %+v, %v, %v", got, present, err) + } +} diff --git a/request_queue.go b/request_queue.go index 0ed6ead..4685575 100644 --- a/request_queue.go +++ b/request_queue.go @@ -14,6 +14,8 @@ type ListRequestsOptions struct { // Limit is the maximum number of requests to return. Limit *int64 // ExclusiveStartID lists requests after this ID. + // + // Deprecated: the API deprecated the exclusiveStartId parameter. Use Cursor. ExclusiveStartID *string // Cursor is an opaque pagination cursor (alternative to ExclusiveStartID). Cursor *string diff --git a/run_collection.go b/run_collection.go index 38a7fa9..8dc8ea9 100644 --- a/run_collection.go +++ b/run_collection.go @@ -3,9 +3,8 @@ package apify import "context" // RunListOptions adds run-specific filters on top of [ListOptions] for -// [RunCollectionClient.List]. The startedAfter/startedBefore filters are only honoured by -// the Actor-scoped and task-scoped run collections, and the deployment filter only by the -// account-wide and Actor-scoped ones. +// [RunCollectionClient.List]. The StartedAfter, StartedBefore and Deployment filters are only +// honoured by the account-wide and Actor-scoped run collections, not by a task's. type RunListOptions struct { // Status filters by one or more run statuses (e.g. "SUCCEEDED", "RUNNING"). Sent as a // comma-separated list, as the API accepts. diff --git a/run_collection_test.go b/run_collection_test.go new file mode 100644 index 0000000..cade32b --- /dev/null +++ b/run_collection_test.go @@ -0,0 +1,35 @@ +package apify + +import ( + "context" + "net/url" + "strings" + "testing" +) + +func TestRunListsSendTheDeploymentFilter(t *testing.T) { + ctx := context.Background() + filter := RunListOptions{Status: []string{"RUNNING"}, Deployment: []string{"SINGLE_TENANT", "MULTI_TENANT_FREE_USERS"}} + + for name, list := range map[string]func(client *ApifyClient) error{ + "account runs": func(c *ApifyClient) error { _, err := c.Runs().List(ctx, ListOptions{}, filter); return err }, + "actor runs": func(c *ApifyClient) error { _, err := c.Actor("a").Runs().List(ctx, ListOptions{}, filter); return err }, + } { + backend := &mockBackend{responses: okData()} + if err := list(testClient(backend, 0)); err != nil { + t.Fatalf("%s: %v", name, err) + } + query, _ := url.ParseQuery(strings.SplitN(backend.lastURL, "?", 2)[1]) + if got := query.Get("deployment"); got != "SINGLE_TENANT,MULTI_TENANT_FREE_USERS" { + t.Errorf("%s: deployment = %q", name, got) + } + } + + backend := &mockBackend{responses: okData()} + if _, err := testClient(backend, 0).Runs().List(ctx, ListOptions{}, RunListOptions{}); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "deployment") { + t.Errorf("deployment must be omitted when unset, got %q", backend.lastURL) + } +} diff --git a/store_collection_test.go b/store_collection_test.go new file mode 100644 index 0000000..0b9e469 --- /dev/null +++ b/store_collection_test.go @@ -0,0 +1,24 @@ +package apify + +import ( + "context" + "strings" + "testing" +) + +func TestStoreListSendsIncludeInputSchema(t *testing.T) { + backend := &mockBackend{responses: okData()} + client := testClient(backend, 0) + if _, err := client.Store().List(context.Background(), StoreListOptions{IncludeInputSchema: Ptr(true)}); err != nil { + t.Fatal(err) + } + if !strings.Contains(backend.lastURL, "includeInputSchema=1") { + t.Errorf("expected includeInputSchema=1, got %q", backend.lastURL) + } + if _, err := client.Store().List(context.Background(), StoreListOptions{}); err != nil { + t.Fatal(err) + } + if strings.Contains(backend.lastURL, "includeInputSchema") { + t.Errorf("includeInputSchema must be omitted when unset, got %q", backend.lastURL) + } +} diff --git a/streamed_log.go b/streamed_log.go index d11f769..478b55e 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -70,6 +70,7 @@ type StreamedLog struct { mu sync.Mutex // serializes writes to output and guards the fields below running bool stopping bool + stopCtx context.Context // the context of the Stop call, which bounds the final read cancelStream context.CancelFunc wake chan struct{} done chan error @@ -78,21 +79,24 @@ type StreamedLog struct { // NewStreamedLog creates the redirection of this run's log, which does nothing until // [StreamedLog.Start]. With the default prefix it fetches the run and its Actor to name them, and -// it returns (nil, nil) if the run does not exist. -func (c *RunClient) NewStreamedLog(ctx context.Context, options StreamedLogOptions) (*StreamedLog, error) { +// the bool reports whether the run exists: it is false (with a nil [StreamedLog]) for a run that +// does not. A non-nil [StreamedLogOptions.Prefix] skips the lookups, so the bool is then always +// true. +func (c *RunClient) NewStreamedLog(ctx context.Context, options StreamedLogOptions) (*StreamedLog, bool, error) { prefix := "" if options.Prefix != nil { prefix = *options.Prefix } else { run, present, err := getResourceWithTimeout[ActorRun](ctx, c.ctx, "", NewQueryParams(), c.ctx.longTimeout()) if err != nil || !present { - return nil, err + return nil, false, err } actorName := "" if run.ActID != "" { - actor, found, err := c.root.Actor(run.ActID).Get(ctx) + actorClient := c.root.Actor(run.ActID) + actor, found, err := getResourceWithTimeout[Actor](ctx, actorClient.ctx, "", NewQueryParams(), actorClient.ctx.longTimeout()) if err != nil { - return nil, err + return nil, false, err } if found { actorName = actor.Name @@ -115,7 +119,7 @@ func (c *RunClient) NewStreamedLog(ctx context.Context, options StreamedLogOptio prefix: prefix, startedAt: time.Now(), fromStart: options.FromStart == nil || *options.FromStart, - }, nil + }, true, nil } // Start begins the redirection in the background. The redirection also ends when ctx is done. @@ -129,6 +133,7 @@ func (s *StreamedLog) Start(ctx context.Context) error { streamCtx, cancel := context.WithCancel(ctx) s.running = true s.stopping = false + s.stopCtx = nil s.cancelStream = cancel s.wake = make(chan struct{}) s.done = make(chan error, 1) @@ -142,15 +147,22 @@ func (s *StreamedLog) Start(ctx context.Context) error { // Stop ends the redirection. It gives the messages already in flight up to one second, then // aborts the log stream; if the stream delivered nothing yet, it reads the whole log in one // request instead, so a short run that finished before the stream connected is not lost. It -// returns the error that made the redirection stop early, if any. ctx bounds the final read of -// the whole log. +// returns the error that made the redirection stop early, if any. ctx bounds the wait for it, +// the final read of the whole log included: when ctx is done first, Stop gives up on the rest of +// the log and returns the error of ctx. It returns an error if the redirection is not running or +// is already being stopped. func (s *StreamedLog) Stop(ctx context.Context) error { s.mu.Lock() if !s.running { s.mu.Unlock() return errors.New("StreamedLog: not started") } + if s.stopping { + s.mu.Unlock() + return errors.New("StreamedLog: already stopping") + } s.stopping = true + s.stopCtx = ctx close(s.wake) cancel, done := s.cancelStream, s.done s.mu.Unlock() @@ -165,6 +177,10 @@ func (s *StreamedLog) Stop(ctx context.Context) error { err = <-done } cancel() + // A read cut short by the context of this call reports a plain cancellation: name the cause. + if ctxErr := ctx.Err(); ctxErr != nil && (err == nil || errors.Is(err, context.Canceled)) { + err = ctxErr + } s.mu.Lock() s.running = false @@ -172,6 +188,20 @@ func (s *StreamedLog) Stop(ctx context.Context) error { return err } +// finalReadContext returns the context of the final read of the whole log: it ends with the +// context of the redirection and with the one of the Stop call. +func (s *StreamedLog) finalReadContext(parent context.Context) (context.Context, context.CancelFunc) { + s.mu.Lock() + stopCtx := s.stopCtx + s.mu.Unlock() + readCtx, cancel := context.WithCancel(parent) + stopWatching := context.AfterFunc(stopCtx, cancel) + return readCtx, func() { + stopWatching() + cancel() + } +} + func (s *StreamedLog) isStopping() bool { s.mu.Lock() defer s.mu.Unlock() @@ -197,8 +227,11 @@ func (s *StreamedLog) redirect(parent, streamCtx context.Context) error { for !gotBytes { if s.isStopping() { - // The stream's context ends with the grace period, which must not cut this read short. - body, _, err := s.log.GetBufferWithOptions(parent, options) + // The stream's context ends with the grace period, which must not cut this read + // short: it ends with the context of the caller's Stop instead. + readCtx, cancelRead := s.finalReadContext(parent) + body, _, err := s.log.GetBufferWithOptions(readCtx, options) + cancelRead() if err != nil { return err } @@ -324,14 +357,17 @@ func startLogRedirection(ctx context.Context, run *RunClient, options *StreamedL if settings.Disabled { return nil } - streamed, err := run.NewStreamedLog(ctx, settings) - if err == nil && streamed != nil { + streamed, found, err := run.NewStreamedLog(ctx, settings) + if err == nil && found { err = streamed.Start(ctx) } if err != nil { reportLogRedirectionError(settings.Output, err) return nil } + if !found { + return nil // the run is gone: there is no log to redirect + } return streamed } diff --git a/streamed_log_test.go b/streamed_log_test.go index 975ec5a..c8b7289 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -42,6 +42,8 @@ type logBackend struct { fullLogCalls int rawOnStream bool streamTailErr error // makes the stream fail with this error after its body + // blockFullLog makes the read of the whole log wait until its request is canceled. + blockFullLog bool } type failingReader struct { @@ -80,6 +82,12 @@ func (b *logBackend) Do(req *http.Request) (*http.Response, error) { return ok(reader) } b.fullLogCalls++ + if b.blockFullLog { + b.mu.Unlock() + <-req.Context().Done() + b.mu.Lock() + return nil, req.Context().Err() + } return ok(strings.NewReader(b.fullLog)) case strings.HasSuffix(path, "/actors/actor1"): return ok(strings.NewReader(`{"data":{"id":"actor1","name":"hello-world"}}`)) @@ -98,9 +106,9 @@ func TestStreamedLogRedirectsMessagesWithThePrefixOfTheActorAndTheRun(t *testing out := &syncBuffer{} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out}) - if err != nil || streamed == nil { - t.Fatalf("NewStreamedLog: %v, %v", streamed, err) + streamed, found, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out}) + if err != nil || !found { + t.Fatalf("NewStreamedLog: %v, %v, %v", streamed, found, err) } if err := streamed.Start(ctx); err != nil { t.Fatal(err) @@ -126,9 +134,9 @@ func TestStreamedLogPrefixOverrideSkipsTheLookups(t *testing.T) { backend := &logBackend{streamBodies: []string{sampleLog}} out := &syncBuffer{} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) - if err != nil || streamed == nil { - t.Fatalf("NewStreamedLog: %v, %v", streamed, err) + streamed, found, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil || !found { + t.Fatalf("NewStreamedLog: %v, %v, %v", streamed, found, err) } if err := streamed.Start(ctx); err != nil { t.Fatal(err) @@ -148,7 +156,7 @@ func TestStreamedLogDropsMessagesBeforeTheStartWhenNotFromStart(t *testing.T) { backend := &logBackend{streamBodies: []string{log}} out := &syncBuffer{} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(""), FromStart: Ptr(false)}) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(""), FromStart: Ptr(false)}) if err != nil { t.Fatal(err) } @@ -170,7 +178,7 @@ func TestStreamedLogReopensAnEmptyStream(t *testing.T) { backend := &logBackend{streamBodies: []string{"", "", sampleLog}} out := &syncBuffer{} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) if err != nil { t.Fatal(err) } @@ -192,7 +200,7 @@ func TestStreamedLogStoppedBeforeTheFirstBytesReadsTheWholeLog(t *testing.T) { backend := &logBackend{streamBodies: []string{""}, fullLog: sampleLog} out := &syncBuffer{} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) if err != nil { t.Fatal(err) } @@ -214,7 +222,7 @@ func TestStreamedLogFlushesALastLineWithoutAnEndOfLine(t *testing.T) { backend := &logBackend{streamBodies: []string{"2026-10-30T10:00:00.000Z one\n2026-10-30T10:00:01.000Z unterminated"}} out := &syncBuffer{} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(">")}) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr(">")}) if err != nil { t.Fatal(err) } @@ -233,7 +241,7 @@ func TestStreamedLogFlushesALastLineWithoutAnEndOfLine(t *testing.T) { func TestStreamedLogStartAndStopMustBePaired(t *testing.T) { backend := &logBackend{streamBodies: []string{sampleLog}} ctx := context.Background() - streamed, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) if err != nil { t.Fatal(err) } @@ -251,11 +259,61 @@ func TestStreamedLogStartAndStopMustBePaired(t *testing.T) { } } -func TestNewStreamedLogOfAMissingRunIsNil(t *testing.T) { +// Two goroutines may call Stop at once: one stops the redirection, the other gets an error +// instead of closing a channel twice. +func TestStreamedLogConcurrentStopDoesNotPanic(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, fullLog: sampleLog} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + const callers = 8 + results := make(chan error, callers) + for range callers { + go func() { results <- streamed.Stop(ctx) }() + } + succeeded := 0 + for range callers { + if <-results == nil { + succeeded++ + } + } + if succeeded != 1 { + t.Fatalf("exactly one Stop must succeed, %d did", succeeded) + } +} + +// The context of Stop bounds the final read of the whole log, not only the wait for the stream. +func TestStreamedLogStopContextBoundsTheReadOfTheWholeLog(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + stopCtx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + started := time.Now() + err = streamed.Stop(stopCtx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected the deadline error of the Stop context, got %v", err) + } + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Fatalf("Stop must return once its context is done, took %v", elapsed) + } +} + +func TestNewStreamedLogOfAMissingRunIsNotFound(t *testing.T) { backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} - streamed, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) - if err != nil || streamed != nil { - t.Fatalf("expected (nil, nil), got %v, %v", streamed, err) + streamed, found, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) + if err != nil || found || streamed != nil { + t.Fatalf("expected (nil, false, nil), got %v, %v, %v", streamed, found, err) } } diff --git a/tests/actor_test.go b/tests/actor_test.go index 9d31c69..3ba4ba0 100644 --- a/tests/actor_test.go +++ b/tests/actor_test.go @@ -97,7 +97,7 @@ func TestActorCRUDFlow(t *testing.T) { if _, err := actor.Builds().List(ctx, apify.ListOptions{}); err != nil { t.Fatalf("builds list: %v", err) } - if _, err := actor.Versions().List(ctx, apify.ListOptions{}); err != nil { + if _, err := actor.Versions().List(ctx); err != nil { t.Fatalf("versions list: %v", err) } } @@ -131,7 +131,7 @@ func TestActorVersionCRUDFlow(t *testing.T) { if _, ok, err := actor.Version("0.1").Get(ctx); err != nil || !ok { t.Fatalf("get version: ok=%v err=%v", ok, err) } - if _, err := actor.Versions().List(ctx, apify.ListOptions{}); err != nil { + if _, err := actor.Versions().List(ctx); err != nil { t.Fatalf("list versions: %v", err) } if _, err := actor.Version("0.1").Update(ctx, map[string]any{"buildTag": "beta", "sourceType": "SOURCE_FILES", "sourceFiles": []any{}}); err != nil { diff --git a/tests/iterate_test.go b/tests/iterate_test.go index a1e6c54..a24a93f 100644 --- a/tests/iterate_test.go +++ b/tests/iterate_test.go @@ -305,7 +305,7 @@ func TestIterateActorVersions(t *testing.T) { } iterateFindIDs(t, ctx, func() *apify.ListIterator[apify.ActorVersion] { - return actor.Versions().Iterate(smallPageDesc(), ptr(int64(1))) + return actor.Versions().Iterate() }, func(v *apify.ActorVersion) string { return v.VersionNumber }, []string{"0.0", "0.1"}, 100) } diff --git a/timeout_retry_test.go b/timeout_retry_test.go index 1a234a3..6a427d7 100644 --- a/timeout_retry_test.go +++ b/timeout_retry_test.go @@ -3,10 +3,8 @@ package apify import ( "bytes" "context" - "encoding/json" "io" "net/http" - "strings" "sync/atomic" "testing" "time" @@ -44,10 +42,6 @@ const ( retriesAllowed = 3 ) -// noLogRedirection turns the log redirection of Call off, so a test counts only the requests of -// the call itself. -var noLogRedirection = &StreamedLogOptions{Disabled: true} - func slowClient(backend HTTPBackend, options ...Option) *ApifyClient { base := []Option{ WithToken("t"), @@ -152,188 +146,3 @@ func TestNonIdempotentRequestsStillRetryServerErrors(t *testing.T) { t.Fatalf("expected 3 attempts, got %d", backend.calls) } } - -// Call has no client-side timeout of its own: a start slower than the medium tier must neither be -// aborted nor repeated (which would start a second, billed run). Start itself is bound by it. -func TestCallStartsWithoutATimeout(t *testing.T) { - run := `{"data":{"id":"run1","status":"SUCCEEDED"}}` - t.Run("Actor Call", func(t *testing.T) { - backend := &slowBackend{delay: slowResponseDelay, body: run} - got, err := slowClient(backend).Actor("a").Call(context.Background(), map[string]any{}, ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))) - if err != nil || got.ID != "run1" { - t.Fatalf("Call: %+v, %v", got, err) - } - if n := backend.calls.Load(); n != 2 { // the start and one poll - t.Fatalf("expected the start once and one poll, got %d requests", n) - } - }) - t.Run("Actor CallRaw", func(t *testing.T) { - backend := &slowBackend{delay: slowResponseDelay, body: run} - if _, err := slowClient(backend).Actor("a").CallRaw(context.Background(), []byte("x"), ActorStartOptions{Log: noLogRedirection}, Ptr(int64(0))); err != nil { - t.Fatalf("CallRaw: %v", err) - } - if n := backend.calls.Load(); n != 2 { - t.Fatalf("expected the start once and one poll, got %d requests", n) - } - }) - t.Run("Task Call", func(t *testing.T) { - backend := &slowBackend{delay: slowResponseDelay, body: run} - if _, err := slowClient(backend).Task("t").Call(context.Background(), nil, TaskStartOptions{}, Ptr(int64(0))); err != nil { - t.Fatalf("Task Call: %v", err) - } - if n := backend.calls.Load(); n != 2 { - t.Fatalf("expected the start once and one poll, got %d requests", n) - } - }) - t.Run("Start stays bound by the medium tier", func(t *testing.T) { - backend := &slowBackend{delay: slowResponseDelay, body: run} - if _, err := slowClient(backend).Actor("a").Start(context.Background(), nil, ActorStartOptions{}); err == nil { - t.Fatal("expected Start to time out") - } - }) - t.Run("ctx still bounds Call", func(t *testing.T) { - backend := &slowBackend{delay: time.Minute, body: run} - ctx, cancel := context.WithTimeout(context.Background(), 3*tinyTier) - defer cancel() - started := time.Now() - if _, err := slowClient(backend).Actor("a").Call(ctx, nil, ActorStartOptions{Log: noLogRedirection}, nil); err == nil { - t.Fatal("expected an error once ctx is done") - } - if time.Since(started) > 5*time.Second { - t.Fatal("Call did not stop at the ctx deadline") - } - }) -} - -// The calls the reference client puts on the long tier must not be cut off by a short tier: here -// the short tier (tinyTier) is shorter than the response time, and the calls still succeed. -func TestKeyValueStoreAndPublicURLCallsUseTheLongTier(t *testing.T) { - ctx := context.Background() - store := `{"data":{"id":"s","urlSigningSecretKey":"secret"}}` - cases := []struct { - name string - run func(client *ApifyClient) error - }{ - {"KeyValueStore Update", func(c *ApifyClient) error { - _, err := c.KeyValueStore("s").Update(ctx, map[string]any{"name": "n"}) - return err - }}, - {"KeyValueStore RecordExists", func(c *ApifyClient) error { - _, err := c.KeyValueStore("s").RecordExists(ctx, "k") - return err - }}, - {"KeyValueStore GetRecordPublicURL", func(c *ApifyClient) error { - _, err := c.KeyValueStore("s").GetRecordPublicURL(ctx, "k") - return err - }}, - {"KeyValueStore CreateKeysPublicURL", func(c *ApifyClient) error { - _, err := c.KeyValueStore("s").CreateKeysPublicURL(ctx, nil) - return err - }}, - {"Dataset CreateItemsPublicURL", func(c *ApifyClient) error { - _, err := c.Dataset("d").CreateItemsPublicURL(ctx, DatasetListItemsOptions{}, nil) - return err - }}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - backend := &slowBackend{delay: slowResponseDelay, body: store} - if err := c.run(slowClient(backend)); err != nil { - t.Fatalf("a long-tier call must outlive the %v short and medium tiers: %v", tinyTier, err) - } - if got := backend.calls.Load(); got != 1 { - t.Fatalf("expected a single request, got %d", got) - } - }) - } -} - -func TestSetRecordRawWithOptionsSendsTheContentEncoding(t *testing.T) { - backend := &mockBackend{responses: okData()} - client := testClient(backend, 0) - value := []byte(strings.Repeat("x", 2*minCompressRequestBytes)) - - err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{ContentEncoding: "gzip"}) - if err != nil { - t.Fatalf("SetRecordRawWithOptions: %v", err) - } - if got := backend.lastHeaders.Get("Content-Encoding"); got != "gzip" { - t.Fatalf("expected Content-Encoding gzip, got %q", got) - } - if backend.lastBody != string(value) { - t.Fatal("a pre-compressed value must be uploaded as given, never compressed again") - } - - if err := client.KeyValueStore("s").SetRecordRawWithOptions(context.Background(), "k", value, "text/plain", SetRecordOptions{}); err != nil { - t.Fatalf("SetRecordRawWithOptions: %v", err) - } - if got := backend.lastHeaders.Get("Content-Encoding"); got == "" { - t.Fatal("without a ContentEncoding the client compresses a large value of its own accord") - } -} - -func TestPushItemsAcceptsOnlyObjectsAndArraysOfObjects(t *testing.T) { - type item struct { - A int `json:"a"` - } - valid := map[string]any{ - "map": map[string]any{"a": 1}, - "struct": item{A: 1}, - "struct pointer": &item{A: 1}, - "slice of structs": []item{{A: 1}, {A: 2}}, - "slice of maps": []map[string]any{{"a": 1}}, - "empty slice": []item{}, - "slice of any": []any{map[string]any{"a": 1}, item{A: 2}}, - "raw JSON object": rawJSON(`{"a":1}`), - "raw JSON array": rawJSON(`[{"a":1},{"a":2}]`), - "padded raw JSON obj": rawJSON(" \n{\"a\":1} "), - } - for name, items := range valid { - t.Run("accepts "+name, func(t *testing.T) { - backend := &mockBackend{responses: okData()} - if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err != nil { - t.Fatalf("PushItems rejected a valid shape: %v", err) - } - if backend.calls != 1 { - t.Fatalf("expected 1 request, got %d", backend.calls) - } - }) - } - - invalid := map[string]any{ - "string": "[1,2]", - "[]byte": []byte(`{"a":1}`), - "number": 1, - "bool": true, - "nil": nil, - "slice of strings": []string{"a", "b"}, - "slice of numbers": []int{1, 2}, - "mixed slice": []any{map[string]any{"a": 1}, "b"}, - "slice of slices": [][]int{{1}}, - "raw JSON string": rawJSON(`"a"`), - "raw JSON null element": rawJSON(`[{"a":1},null]`), - } - for name, items := range invalid { - t.Run("rejects "+name, func(t *testing.T) { - backend := &mockBackend{responses: okData()} - if err := testClient(backend, 0).Dataset("d").PushItems(context.Background(), items); err == nil { - t.Fatal("PushItems accepted a shape the API rejects") - } - if backend.calls != 0 { - t.Fatalf("nothing may be sent for an invalid shape, got %d requests", backend.calls) - } - }) - } -} - -func TestPushItemsRejectsAnInvalidDatasetIDBeforeSending(t *testing.T) { - backend := &mockBackend{responses: okData()} - if err := testClient(backend, 0).Dataset("").PushItems(context.Background(), map[string]any{"a": 1}); err == nil { - t.Fatal("expected an error for an empty dataset ID") - } - if backend.calls != 0 { - t.Fatalf("nothing may be sent for an invalid ID, got %d requests", backend.calls) - } -} - -func rawJSON(s string) json.RawMessage { return json.RawMessage(s) } diff --git a/timeout_tier_test.go b/timeout_tier_test.go new file mode 100644 index 0000000..9fe4b6a --- /dev/null +++ b/timeout_tier_test.go @@ -0,0 +1,49 @@ +package apify + +import ( + "context" + "testing" +) + +// The calls the reference client puts on the long tier must not be cut off by a short tier: here +// the short tier (tinyTier) is shorter than the response time, and the calls still succeed. +func TestKeyValueStoreAndPublicURLCallsUseTheLongTier(t *testing.T) { + ctx := context.Background() + store := `{"data":{"id":"s","urlSigningSecretKey":"secret"}}` + cases := []struct { + name string + run func(client *ApifyClient) error + }{ + {"KeyValueStore Update", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").Update(ctx, map[string]any{"name": "n"}) + return err + }}, + {"KeyValueStore RecordExists", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").RecordExists(ctx, "k") + return err + }}, + {"KeyValueStore GetRecordPublicURL", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").GetRecordPublicURL(ctx, "k") + return err + }}, + {"KeyValueStore CreateKeysPublicURL", func(c *ApifyClient) error { + _, err := c.KeyValueStore("s").CreateKeysPublicURL(ctx, nil) + return err + }}, + {"Dataset CreateItemsPublicURL", func(c *ApifyClient) error { + _, err := c.Dataset("d").CreateItemsPublicURL(ctx, DatasetListItemsOptions{}, nil) + return err + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + backend := &slowBackend{delay: slowResponseDelay, body: store} + if err := c.run(slowClient(backend)); err != nil { + t.Fatalf("a long-tier call must outlive the %v short and medium tiers: %v", tinyTier, err) + } + if got := backend.calls.Load(); got != 1 { + t.Fatalf("expected a single request, got %d", got) + } + }) + } +} diff --git a/url_normalization.go b/url_normalization.go index 5cbcff3..e414d35 100644 --- a/url_normalization.go +++ b/url_normalization.go @@ -3,19 +3,21 @@ package apify import ( "errors" "fmt" + "net/netip" "regexp" "strconv" "strings" - - "golang.org/x/net/idna" + "unicode" + "unicode/utf8" ) -// The fields the OpenAPI specification declares as `format: uri` are returned in their -// normalized form, the same string for the same field in every Apify client (the JavaScript -// client returns the WHATWG URL serialization, and the Python one an equivalent): the scheme and -// host are lowercased, a non-ASCII host is converted to punycode, the default port is dropped, an -// empty path becomes "/", dot segments are resolved and characters that are unsafe in a URL are -// percent-encoded. +// The fields the OpenAPI specification declares as `format: uri` are returned in a normalized +// form, close to the WHATWG URL serialization that the JavaScript client returns: the scheme and +// host are lowercased, a non-ASCII host is converted to punycode, a numeric (IPv4) or IPv6 host is +// written in its canonical form, the default port is dropped, an empty path becomes "/", dot +// segments are resolved and characters that are unsafe in a URL are percent-encoded. It covers +// the http(s), ws(s), ftp and file URLs the API returns and keeps the rest as given; it does not +// apply the full Unicode (UTS #46) mapping to a non-ASCII host. // defaultPorts maps a special URL scheme to its default port, which normalization drops. var defaultPorts = map[string]string{ @@ -35,35 +37,46 @@ const maxPort = 65535 // Bytes that are percent-encoded in each URL component, in addition to every control character, // space and non-ASCII byte (the WHATWG percent-encode sets). const ( - unsafePathBytes = `"#<>?` + "`{}" - unsafeQueryBytes = `"#<>'` - unsafeFragmentBytes = `"<>` + "`" - unsafeUserinfoBytes = `"#<>?` + "`{}/:;=@[\\]^|" + unsafePathBytes = `"#<>?` + "`{}" + unsafeQueryBytes = `"#<>'` + unsafeOpaqueQueryByte = `"#<>` + unsafeFragmentBytes = `"<>` + "`" + unsafeUserinfoBytes = `"#<>?` + "`{}/:;=@[\\]^|" ) -// idnaProfile converts an internationalized host name to its ASCII (punycode) form. The checks -// that only apply to registrable names (hyphen rules, bidi, joiners, label length) are off, -// because the host of a URL is not necessarily a registrable one. -var idnaProfile = idna.New( - idna.MapForLookup(), - idna.StrictDomainName(false), - idna.VerifyDNSLength(false), - idna.CheckHyphens(false), - idna.CheckJoiners(false), - idna.BidiRule(), +// forbiddenHostBytes are the ASCII characters a host may not contain (besides control characters +// and the space). +const forbiddenHostBytes = "#%/:<>?@[\\]^|" + +// ipv4PartCount is the number of parts of a dotted IPv4 address, and ipv4PartBits the bits of +// each; ipv6PieceCount is the number of 16-bit pieces of an IPv6 address. +const ( + ipv4PartCount = 4 + ipv4PartBits = 8 + ipv6PieceCount = 8 + maxIPv4Part = 1<` + unsafe = unsafeOpaqueQueryByte } out.WriteByte('?') out.WriteString(percentEncode(query, unsafe)) @@ -121,8 +174,9 @@ func normalizeURL(raw string) (string, error) { return out.String(), nil } -// normalizeAuthority normalizes the "userinfo@host:port" part of a URL. -func normalizeAuthority(authority, defaultPort string) (string, error) { +// normalizeAuthority normalizes the "userinfo@host:port" part of a URL. With special false (a +// scheme that is not one of the special ones) the host is opaque and kept as given. +func normalizeAuthority(authority, defaultPort string, special bool) (string, error) { var userinfo string if i := strings.LastIndexByte(authority, '@'); i >= 0 { userinfo, authority = authority[:i], authority[i+1:] @@ -133,26 +187,23 @@ func normalizeAuthority(authority, defaultPort string) (string, error) { if end < 0 { return "", errors.New("unterminated IPv6 address") } - host = strings.ToLower(authority[:end+1]) - port = strings.TrimPrefix(authority[end+1:], ":") - if tail := authority[end+1:]; tail != "" && !strings.HasPrefix(tail, ":") { + tail := authority[end+1:] + if tail != "" && !strings.HasPrefix(tail, ":") { return "", errors.New("unexpected text after the IPv6 address") } + address, err := netip.ParseAddr(authority[1:end]) + if err != nil || !address.Is6() { + return "", fmt.Errorf("invalid IPv6 address %q", authority[1:end]) + } + host, port = serializeIPv6(address), strings.TrimPrefix(tail, ":") } else { if i := strings.LastIndexByte(authority, ':'); i >= 0 { host, port = authority[:i], authority[i+1:] } - if strings.ContainsFunc(host, isForbiddenHostRune) { - return "", fmt.Errorf("invalid host %q", host) + var err error + if host, err = normalizeHost(host, special); err != nil { + return "", err } - ascii, err := idnaProfile.ToASCII(strings.ToLower(host)) - if err != nil { - return "", fmt.Errorf("invalid host %q: %w", host, err) - } - host = ascii - } - if host == "" { - return "", errors.New("empty host") } if port != "" { number, err := strconv.Atoi(port) @@ -183,15 +234,64 @@ func normalizeAuthority(authority, defaultPort string) (string, error) { return out.String(), nil } -// forbiddenHostBytes are the ASCII characters a host may not contain (besides control characters -// and the space). -const forbiddenHostBytes = "#%/<>?@[\\]^|" +// normalizeHost normalizes a host name: percent-decoded, lowercased, converted to punycode, and +// an IPv4 address (in any of its numeric notations) written as four decimal parts. An empty host +// is an error for a special scheme. With special false the host is opaque and kept as given. +func normalizeHost(host string, special bool) (string, error) { + if !special { + return percentEncode(host, ""), nil + } + decoded, err := percentDecode(host) + if err != nil { + return "", fmt.Errorf("invalid host %q: %w", host, err) + } + if decoded == "" { + return "", errors.New("empty host") + } + if strings.ContainsFunc(decoded, isForbiddenHostRune) { + return "", fmt.Errorf("invalid host %q", host) + } + ascii := hostToASCII(strings.ToLower(decoded)) + if endsInANumber(ascii) { + return parseIPv4(ascii) + } + return ascii, nil +} // isForbiddenHostRune reports whether r may not appear in a host. func isForbiddenHostRune(r rune) bool { return r <= ' ' || r == 0x7F || strings.ContainsRune(forbiddenHostBytes, r) } +// percentDecode decodes the percent escapes of s, which must make valid UTF-8. +func percentDecode(s string) (string, error) { + if !strings.Contains(s, "%") { + return s, nil + } + decoded := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + if s[i] != '%' { + decoded = append(decoded, s[i]) + continue + } + if i+3 > len(s) { + decoded = append(decoded, s[i]) + continue + } + value, err := strconv.ParseUint(s[i+1:i+3], 16, 8) + if err != nil { + decoded = append(decoded, s[i]) + continue + } + decoded = append(decoded, byte(value)) + i += 2 + } + if !utf8.Valid(decoded) { + return "", errors.New("not valid UTF-8") + } + return string(decoded), nil +} + // removeDotSegments resolves the "." and ".." segments of an absolute path. func removeDotSegments(path string) string { if path == "" { @@ -238,6 +338,239 @@ func percentEncode(s, unsafe string) string { return out.String() } +// serializeIPv6 writes an IPv6 address as WHATWG does: lowercase hexadecimal pieces without +// leading zeros, the longest run of zero pieces (of at least two) compressed to "::", in brackets. +func serializeIPv6(address netip.Addr) string { + bytes := address.As16() + var pieces [ipv6PieceCount]uint16 + for i := range pieces { + pieces[i] = uint16(bytes[2*i])<<8 | uint16(bytes[2*i+1]) + } + compressStart, compressLength := -1, 0 + for i := 0; i < ipv6PieceCount; { + if pieces[i] != 0 { + i++ + continue + } + j := i + for j < ipv6PieceCount && pieces[j] == 0 { + j++ + } + if j-i > compressLength && j-i >= 2 { + compressStart, compressLength = i, j-i + } + i = j + } + var out strings.Builder + out.WriteByte('[') + for i := 0; i < ipv6PieceCount; i++ { + if i == compressStart { + if i == 0 { + out.WriteString("::") + } else { + out.WriteByte(':') + } + i += compressLength - 1 + continue + } + out.WriteString(strconv.FormatUint(uint64(pieces[i]), 16)) + if i != ipv6PieceCount-1 { + out.WriteByte(':') + } + } + out.WriteByte(']') + return out.String() +} + +// endsInANumber reports whether the last label of host is a number (decimal, 0x hexadecimal or +// 0 octal), which makes the whole host an IPv4 address. +func endsInANumber(host string) bool { + labels := strings.Split(host, ".") + if labels[len(labels)-1] == "" { + if len(labels) == 1 { + return false + } + labels = labels[:len(labels)-1] + } + last := labels[len(labels)-1] + if last != "" && strings.Trim(last, "0123456789") == "" { + return true + } + _, ok := parseIPv4Part(last) + return ok +} + +// parseIPv4Part parses one part of an IPv4 address in decimal, 0x hexadecimal or 0 octal. +func parseIPv4Part(part string) (uint64, bool) { + if part == "" { + return 0, false + } + base := 10 + digits := part + switch { + case len(part) >= 2 && (part[:2] == "0x" || part[:2] == "0X"): + base, digits = 16, part[2:] + case len(part) >= 2 && part[0] == '0': + base, digits = 8, part[1:] + } + if digits == "" { + return 0, true // "0x" is zero + } + value, err := strconv.ParseUint(digits, base, 64) + return value, err == nil +} + +// parseIPv4 parses an IPv4 host in any of its numeric notations and writes it as four decimal +// parts. A host that ends in a number but is not a valid address is an error. +func parseIPv4(host string) (string, error) { + parts := strings.Split(host, ".") + if parts[len(parts)-1] == "" && len(parts) > 1 { + parts = parts[:len(parts)-1] + } + if len(parts) > ipv4PartCount { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + numbers := make([]uint64, len(parts)) + for i, part := range parts { + value, ok := parseIPv4Part(part) + if !ok { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + numbers[i] = value + } + for _, number := range numbers[:len(numbers)-1] { + if number > maxIPv4Part { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + } + last := numbers[len(numbers)-1] + if last >= 1<<(ipv4PartBits*uint(ipv4PartCount-len(numbers)+1)) { + return "", fmt.Errorf("invalid IPv4 address %q", host) + } + address := last + for i, number := range numbers[:len(numbers)-1] { + address += number << (ipv4PartBits * uint(ipv4PartCount-1-i)) + } + octets := make([]string, ipv4PartCount) + for i := range octets { + octets[i] = strconv.FormatUint(address>>(ipv4PartBits*uint(ipv4PartCount-1-i))&maxIPv4Part, 10) + } + return strings.Join(octets, "."), nil +} + +// Punycode (RFC 3492) parameters. +const ( + punycodeBase = 36 + punycodeTMin = 1 + punycodeTMax = 26 + punycodeSkew = 38 + punycodeDamp = 700 + punycodeInitialBias = 72 + punycodeInitialN = 128 + punycodePrefix = "xn--" +) + +// hostToASCII converts every non-ASCII label of a lowercased host to its punycode form. +func hostToASCII(host string) string { + labels := strings.Split(host, ".") + for i, label := range labels { + if !isASCII(label) { + labels[i] = punycodePrefix + punycodeEncode(label) + } + } + return strings.Join(labels, ".") +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// punycodeEncode encodes a label as punycode (RFC 3492), without the "xn--" prefix. +func punycodeEncode(label string) string { + runes := []rune(label) + var out []byte + for _, r := range runes { + if r < punycodeInitialN { + out = append(out, byte(r)) + } + } + basicCount := len(out) + handled := basicCount + if basicCount > 0 { + out = append(out, '-') + } + n, delta, bias := rune(punycodeInitialN), 0, punycodeInitialBias + for handled < len(runes) { + next := rune(unicode.MaxRune + 1) + for _, r := range runes { + if r >= n && r < next { + next = r + } + } + delta += int(next-n) * (handled + 1) + n = next + for _, r := range runes { + if r < n { + delta++ + } + if r != n { + continue + } + q := delta + for k := punycodeBase; ; k += punycodeBase { + threshold := k - bias + if threshold < punycodeTMin { + threshold = punycodeTMin + } else if threshold > punycodeTMax { + threshold = punycodeTMax + } + if q < threshold { + break + } + out = append(out, punycodeDigit(threshold+(q-threshold)%(punycodeBase-threshold))) + q = (q - threshold) / (punycodeBase - threshold) + } + out = append(out, punycodeDigit(q)) + bias = adaptPunycodeBias(delta, handled+1, handled == basicCount) + delta = 0 + handled++ + } + delta++ + n++ + } + return string(out) +} + +// punycodeDigit maps 0..35 to "a".."z", "0".."9". +func punycodeDigit(d int) byte { + const letters = 26 + if d < letters { + return byte('a' + d) + } + return byte('0' + d - letters) +} + +// adaptPunycodeBias is the bias adaptation function of RFC 3492, section 6.1. +func adaptPunycodeBias(delta, points int, first bool) int { + if first { + delta /= punycodeDamp + } else { + delta /= 2 + } + delta += delta / points + k := 0 + for delta > ((punycodeBase-punycodeTMin)*punycodeTMax)/2 { + delta /= punycodeBase - punycodeTMin + k += punycodeBase + } + return k + (punycodeBase-punycodeTMin+1)*delta/(delta+punycodeSkew) +} + // normalizeURLField normalizes the URL held in *field in place. An empty value (a missing or // null field) is left as it is; any other value that is not an absolute URL is an error naming // the field. diff --git a/url_normalization_test.go b/url_normalization_test.go index 1b998e9..d1609c2 100644 --- a/url_normalization_test.go +++ b/url_normalization_test.go @@ -27,8 +27,31 @@ func TestNormalizeURL(t *testing.T) { // Userinfo and IPv6 hosts survive. {"https://user:pa ss@Example.com/", "https://user:pa%20ss@example.com/"}, {"http://[::1]:80/x", "http://[::1]/x"}, - // A URL that is not special keeps its path as it is. + // A URL that is not special keeps its host and path as they are. {"mailto:Someone@Example.com", "mailto:Someone@Example.com"}, + {"foo://Example.COM/A", "foo://Example.COM/A"}, + // A special URL has an authority, whatever the number of slashes after its scheme. + {"https:foo", "https://foo/"}, + {"https:///foo/a", "https://foo/a"}, + {"https:\\\\foo\\a", "https://foo/a"}, + // The host is percent-decoded, and numeric hosts are written canonically. + {"https://ex%41mple.com/", "https://example.com/"}, + {"http://0x7f.1/", "http://127.0.0.1/"}, + {"http://127.1/", "http://127.0.0.1/"}, + {"http://2130706433/", "http://127.0.0.1/"}, + {"http://[2001:db8:0:0:0:0:0:1]/", "http://[2001:db8::1]/"}, + {"http://[0:0:0:0:0:0:0:0]/", "http://[::]/"}, + {"http://[2001:db8:0:1:0:0:0:1]:8080/", "http://[2001:db8:0:1::1]:8080/"}, + // file URLs. + {"file:///tmp/a", "file:///tmp/a"}, + {"file://localhost/tmp/a", "file:///tmp/a"}, + {"file:tmp/a", "file:///tmp/a"}, + // Tabs and line breaks are removed. + {"https://exa\tmple.com/pa\nth", "https://example.com/path"}, + // More punycode. + {"https://m\u00fcnchen.example/", "https://xn--mnchen-3ya.example/"}, + {"https://\u65e5\u672c\u8a9e.jp/", "https://xn--wgv71a119e.jp/"}, + {"https://\u30c9\u30e1\u30a4\u30f3\u540d\u4f8b.example/", "https://xn--eckwd4c7cu47r2wf.example/"}, // Surrounding whitespace is dropped, a backslash is a slash. {" https://example.com\\a ", "https://example.com/a"}, } @@ -49,7 +72,7 @@ func TestNormalizeURL(t *testing.T) { } func TestNormalizeURLRejectsInvalidURLs(t *testing.T) { - for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "https:foo"} { + for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "http://1.2.3.4.5/", "http://256.1.1.1/", "http://foo.123/", "https://%ff.example/"} { if got, err := normalizeURL(raw); err == nil { t.Errorf("normalizeURL(%q) = %q, want an error", raw, got) } From 45f4a74f0acff792e9d2523dc23fe3a85aa836e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 12:52:41 +0000 Subject: [PATCH 03/12] fix: address the second review of the v3 port --- streamed_log.go | 22 +++++++++-- streamed_log_test.go | 70 ++++++++++++++++++++++++++++++++- url_normalization.go | 83 ++++++++++++++++++++++++++++++--------- url_normalization_test.go | 11 +++++- 4 files changed, 161 insertions(+), 25 deletions(-) diff --git a/streamed_log.go b/streamed_log.go index 478b55e..44085f6 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -178,7 +178,7 @@ func (s *StreamedLog) Stop(ctx context.Context) error { } cancel() // A read cut short by the context of this call reports a plain cancellation: name the cause. - if ctxErr := ctx.Err(); ctxErr != nil && (err == nil || errors.Is(err, context.Canceled)) { + if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, context.Canceled) { err = ctxErr } @@ -202,6 +202,18 @@ func (s *StreamedLog) finalReadContext(parent context.Context) (context.Context, } } +// logReadError wraps the failure of the final read of the whole log, which leaves the log +// unredirected. Unlike a stream that is cut on purpose, it is never swallowed. +type logReadError struct{ err error } + +func (e *logReadError) Error() string { return e.err.Error() } +func (e *logReadError) Unwrap() error { return e.err } + +// isContextError reports whether err is the cancellation or the deadline of a context. +func isContextError(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + func (s *StreamedLog) isStopping() bool { s.mu.Lock() defer s.mu.Unlock() @@ -213,7 +225,11 @@ func (s *StreamedLog) isStopping() bool { // It returns nil when the redirection was ended on purpose. func (s *StreamedLog) streamLog(parent, streamCtx context.Context) error { err := s.redirect(parent, streamCtx) - if err != nil && streamCtx.Err() != nil { + var lost *logReadError + if errors.As(err, &lost) { + return lost.err // the log was not redirected: the caller must hear of it, whatever the cause + } + if err != nil && streamCtx.Err() != nil && isContextError(err) { return nil // ended on purpose: by Stop, or because the caller's context is done } return err @@ -233,7 +249,7 @@ func (s *StreamedLog) redirect(parent, streamCtx context.Context) error { body, _, err := s.log.GetBufferWithOptions(readCtx, options) cancelRead() if err != nil { - return err + return &logReadError{err: err} } s.consume(body) break diff --git a/streamed_log_test.go b/streamed_log_test.go index c8b7289..e54c163 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -44,6 +44,10 @@ type logBackend struct { streamTailErr error // makes the stream fail with this error after its body // blockFullLog makes the read of the whole log wait until its request is canceled. blockFullLog bool + // fullLogFailure, when set, makes the read of the whole log fail with this status after + // fullLogDelay. + fullLogFailure int + fullLogDelay time.Duration } type failingReader struct { @@ -88,6 +92,12 @@ func (b *logBackend) Do(req *http.Request) (*http.Response, error) { b.mu.Lock() return nil, req.Context().Err() } + if b.fullLogFailure != 0 { + b.mu.Unlock() + time.Sleep(b.fullLogDelay) + b.mu.Lock() + return &http.Response{StatusCode: b.fullLogFailure, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"error":{"type":"forbidden","message":"no"}}`))}, nil + } return ok(strings.NewReader(b.fullLog)) case strings.HasSuffix(path, "/actors/actor1"): return ok(strings.NewReader(`{"data":{"id":"actor1","name":"hello-world"}}`)) @@ -309,6 +319,60 @@ func TestStreamedLogStopContextBoundsTheReadOfTheWholeLog(t *testing.T) { } } +// Stop with a context that is already done does not turn a log that was redirected completely +// into an error: nothing was cut short. +func TestStreamedLogStopWithADoneContextAfterACompleteLogIsNotAnError(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}} + out := &syncBuffer{} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + done, cancel := context.WithCancel(context.Background()) + cancel() + if err := streamed.Stop(done); err != nil { + t.Fatalf("a complete log is not an error, got %v", err) + } +} + +// A Call whose context ends while it waits does not report a failed redirection after a log that +// was redirected completely. +func TestCallWhoseContextEndsDoesNotReportAFalseRedirectionError(t *testing.T) { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, fullLog: sampleLog}, pollDelay: 300 * time.Millisecond} + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + _, err := testClient(backend, 0).Actor("actor1").Call(ctx, nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) + if err == nil { + t.Fatal("expected the call to end with the error of its context") + } + if strings.Contains(out.String(), "Log redirection stopped due to error") { + t.Fatalf("a complete log is not a redirection error: %q", out.String()) + } +} + +// A failed final read of the whole log is reported even when it fails after the grace period of +// Stop: the log is lost, and the caller must hear of it. +func TestStreamedLogStopReportsAFailedReadOfTheWholeLog(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, fullLogFailure: http.StatusForbidden, fullLogDelay: stopGrace + 500*time.Millisecond} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + err = streamed.Stop(context.Background()) + apiErr, ok := AsAPIError(err) + if !ok || apiErr.StatusCode != http.StatusForbidden { + t.Fatalf("expected the 403 of the final read, got %v", err) + } +} + func TestNewStreamedLogOfAMissingRunIsNotFound(t *testing.T) { backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} streamed, found, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) @@ -365,7 +429,11 @@ func (b *callLogBackend) Do(req *http.Request) (*http.Response, error) { return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"RUNNING"}}`))}, nil } if strings.Contains(req.URL.Path, "/actor-runs/") && !strings.HasSuffix(req.URL.Path, "/log") && req.URL.Query().Get("waitForFinish") != "" { - time.Sleep(b.pollDelay) + select { + case <-time.After(b.pollDelay): + case <-req.Context().Done(): + return nil, req.Context().Err() + } return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"data":{"id":"run1","actId":"actor1","status":"SUCCEEDED"}}`))}, nil } return b.logBackend.Do(req) diff --git a/url_normalization.go b/url_normalization.go index e414d35..7ab3baa 100644 --- a/url_normalization.go +++ b/url_normalization.go @@ -15,9 +15,9 @@ import ( // form, close to the WHATWG URL serialization that the JavaScript client returns: the scheme and // host are lowercased, a non-ASCII host is converted to punycode, a numeric (IPv4) or IPv6 host is // written in its canonical form, the default port is dropped, an empty path becomes "/", dot -// segments are resolved and characters that are unsafe in a URL are percent-encoded. It covers -// the http(s), ws(s), ftp and file URLs the API returns and keeps the rest as given; it does not -// apply the full Unicode (UTS #46) mapping to a non-ASCII host. +// segments are resolved and characters that are unsafe in a URL are percent-encoded. It is built +// for the http(s), ws(s), ftp and file URLs the API returns. The host of any other scheme is kept +// as given. It does not apply the full Unicode (UTS #46) mapping to a non-ASCII host. // defaultPorts maps a special URL scheme to its default port, which normalization drops. var defaultPorts = map[string]string{ @@ -106,7 +106,7 @@ func normalizeURL(raw string) (string, error) { end = len(authority) } path = authority[end:] - host, err := normalizeAuthority(authority[:end], defaultPorts[scheme], true) + host, err := normalizeAuthority(authority[:end], authorityRules{defaultPort: defaultPorts[scheme], special: true, allowPort: true}) if err != nil { return "", fmt.Errorf("invalid URL %q: %w", raw, err) } @@ -123,7 +123,7 @@ func normalizeURL(raw string) (string, error) { path = authority[end:] if authority[:end] != "" { var err error - if host, err = normalizeAuthority(authority[:end], "", true); err != nil { + if host, err = normalizeAuthority(authority[:end], authorityRules{special: true}); err != nil { return "", fmt.Errorf("invalid URL %q: %w", raw, err) } } @@ -143,7 +143,7 @@ func normalizeURL(raw string) (string, error) { end = len(authority) } path = authority[end:] - host, err := normalizeAuthority(authority[:end], "", false) + host, err := normalizeAuthority(authority[:end], authorityRules{allowPort: true}) if err != nil { return "", fmt.Errorf("invalid URL %q: %w", raw, err) } @@ -155,10 +155,13 @@ func normalizeURL(raw string) (string, error) { if special && path == "" { path = "/" } - if special { + opaquePath := !special && !strings.HasPrefix(path, "/") + if !opaquePath { path = removeDotSegments(path) + out.WriteString(percentEncode(path, unsafePathBytes)) + } else { + out.WriteString(percentEncodeControls(path)) } - out.WriteString(percentEncode(path, unsafePathBytes)) if hasQuery { unsafe := unsafeQueryBytes if !special { @@ -174,9 +177,19 @@ func normalizeURL(raw string) (string, error) { return out.String(), nil } -// normalizeAuthority normalizes the "userinfo@host:port" part of a URL. With special false (a -// scheme that is not one of the special ones) the host is opaque and kept as given. -func normalizeAuthority(authority, defaultPort string, special bool) (string, error) { +// authorityRules says how the authority of a URL is normalized. +type authorityRules struct { + // defaultPort is the port that is dropped as the default one of the scheme. + defaultPort string + // special is true for the special schemes, whose host is a domain, IPv4 or IPv6 address. The + // host of any other scheme is opaque and kept as given. + special bool + // allowPort is false for a scheme whose URLs have no port (file). + allowPort bool +} + +// normalizeAuthority normalizes the "userinfo@host:port" part of a URL. +func normalizeAuthority(authority string, rules authorityRules) (string, error) { var userinfo string if i := strings.LastIndexByte(authority, '@'); i >= 0 { userinfo, authority = authority[:i], authority[i+1:] @@ -192,7 +205,7 @@ func normalizeAuthority(authority, defaultPort string, special bool) (string, er return "", errors.New("unexpected text after the IPv6 address") } address, err := netip.ParseAddr(authority[1:end]) - if err != nil || !address.Is6() { + if err != nil || !address.Is6() || address.Zone() != "" { return "", fmt.Errorf("invalid IPv6 address %q", authority[1:end]) } host, port = serializeIPv6(address), strings.TrimPrefix(tail, ":") @@ -201,26 +214,26 @@ func normalizeAuthority(authority, defaultPort string, special bool) (string, er host, port = authority[:i], authority[i+1:] } var err error - if host, err = normalizeHost(host, special); err != nil { + if host, err = normalizeHost(host, rules.special); err != nil { return "", err } } if port != "" { - number, err := strconv.Atoi(port) - if err != nil || number < 0 || number > maxPort { + number, err := parsePort(port) + if err != nil || !rules.allowPort { return "", fmt.Errorf("invalid port %q", port) } port = strconv.Itoa(number) - if port == defaultPort { + if port == rules.defaultPort { port = "" } } var out strings.Builder - if userinfo != "" { - username, password, hasPassword := strings.Cut(userinfo, ":") + username, password, _ := strings.Cut(userinfo, ":") + if username != "" || password != "" { out.WriteString(percentEncode(username, unsafeUserinfoBytes)) - if hasPassword { + if password != "" { out.WriteByte(':') out.WriteString(percentEncode(password, unsafeUserinfoBytes)) } @@ -234,6 +247,20 @@ func normalizeAuthority(authority, defaultPort string, special bool) (string, er return out.String(), nil } +// parsePort parses a port: ASCII digits only, at most maxPort. +func parsePort(port string) (int, error) { + for i := 0; i < len(port); i++ { + if port[i] < '0' || port[i] > '9' { + return 0, errors.New("not a number") + } + } + number, err := strconv.Atoi(port) + if err != nil || number > maxPort { + return 0, errors.New("out of range") + } + return number, nil +} + // normalizeHost normalizes a host name: percent-decoded, lowercased, converted to punycode, and // an IPv4 address (in any of its numeric notations) written as four decimal parts. An empty host // is an error for a special scheme. With special false the host is opaque and kept as given. @@ -338,6 +365,24 @@ func percentEncode(s, unsafe string) string { return out.String() } +// percentEncodeControls percent-encodes only the control characters and the non-ASCII bytes of s, +// as is done for the opaque path of a URL such as mailto:. +func percentEncodeControls(s string) string { + const upperHex = "0123456789ABCDEF" + var out strings.Builder + for i := 0; i < len(s); i++ { + b := s[i] + if b < ' ' || b >= 0x7F { + out.WriteByte('%') + out.WriteByte(upperHex[b>>4]) + out.WriteByte(upperHex[b&0x0F]) + continue + } + out.WriteByte(b) + } + return out.String() +} + // serializeIPv6 writes an IPv6 address as WHATWG does: lowercase hexadecimal pieces without // leading zeros, the longest run of zero pieces (of at least two) compressed to "::", in brackets. func serializeIPv6(address netip.Addr) string { diff --git a/url_normalization_test.go b/url_normalization_test.go index d1609c2..ef6c9b7 100644 --- a/url_normalization_test.go +++ b/url_normalization_test.go @@ -27,9 +27,16 @@ func TestNormalizeURL(t *testing.T) { // Userinfo and IPv6 hosts survive. {"https://user:pa ss@Example.com/", "https://user:pa%20ss@example.com/"}, {"http://[::1]:80/x", "http://[::1]/x"}, - // A URL that is not special keeps its host and path as they are. + // The host of a URL that is not special is kept as given; its path is still resolved. {"mailto:Someone@Example.com", "mailto:Someone@Example.com"}, + {"mailto:a b", "mailto:a b"}, {"foo://Example.COM/A", "foo://Example.COM/A"}, + {"foo://host/a/../b", "foo://host/b"}, + {"foo:/a/./b", "foo:/a/b"}, + // An empty password, and an empty userinfo, are dropped. + {"https://u:@ex.com/", "https://u@ex.com/"}, + {"https://:@ex.com/", "https://ex.com/"}, + {"https://:p@ex.com/", "https://:p@ex.com/"}, // A special URL has an authority, whatever the number of slashes after its scheme. {"https:foo", "https://foo/"}, {"https:///foo/a", "https://foo/a"}, @@ -72,7 +79,7 @@ func TestNormalizeURL(t *testing.T) { } func TestNormalizeURLRejectsInvalidURLs(t *testing.T) { - for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "http://1.2.3.4.5/", "http://256.1.1.1/", "http://foo.123/", "https://%ff.example/"} { + for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "http://1.2.3.4.5/", "http://256.1.1.1/", "http://foo.123/", "https://%ff.example/", "http://ex.com:+80/", "http://ex.com:-0/", "http://[fe80::1%25eth0]/", "file://host:80/x"} { if got, err := normalizeURL(raw); err == nil { t.Errorf("normalizeURL(%q) = %q, want an error", raw, got) } From 47bf75e3b26ab05a11ab0ab95184e1eb321d541e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 13:04:48 +0000 Subject: [PATCH 04/12] fix: address the third review of the v3 port --- streamed_log.go | 18 +++++-- streamed_log_test.go | 108 ++++++++++++++++++++++++++++++++++++++ url_normalization.go | 26 +++++++-- url_normalization_test.go | 5 +- 4 files changed, 147 insertions(+), 10 deletions(-) diff --git a/streamed_log.go b/streamed_log.go index 44085f6..46da510 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -173,8 +173,16 @@ func (s *StreamedLog) Stop(ctx context.Context) error { select { case err = <-done: case <-ctx.Done(): - cancel() - err = <-done + select { + case err = <-done: // it finished as the context ended + default: + // The redirection is still running: cutting it off loses the rest of the log, which + // the caller must be able to tell from a complete one. + cancel() + if err = <-done; err == nil { + err = ctx.Err() + } + } } cancel() // A read cut short by the context of this call reports a plain cancellation: name the cause. @@ -226,8 +234,10 @@ func (s *StreamedLog) isStopping() bool { func (s *StreamedLog) streamLog(parent, streamCtx context.Context) error { err := s.redirect(parent, streamCtx) var lost *logReadError - if errors.As(err, &lost) { - return lost.err // the log was not redirected: the caller must hear of it, whatever the cause + if errors.As(err, &lost) && !(isContextError(lost.err) && parent.Err() != nil) { + // The log was not redirected: the caller must hear of it, unless the cause is the end of + // the caller's own context, which ends the redirection on purpose. + return lost.err } if err != nil && streamCtx.Err() != nil && isContextError(err) { return nil // ended on purpose: by Stop, or because the caller's context is done diff --git a/streamed_log_test.go b/streamed_log_test.go index e54c163..7bf9762 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -44,6 +44,8 @@ type logBackend struct { streamTailErr error // makes the stream fail with this error after its body // blockFullLog makes the read of the whole log wait until its request is canceled. blockFullLog bool + // holdStream keeps the live stream open after its body, until the request is canceled. + holdStream bool // fullLogFailure, when set, makes the read of the whole log fail with this status after // fullLogDelay. fullLogFailure int @@ -63,6 +65,31 @@ func (r *failingReader) Read(p []byte) (int, error) { return n, err } +// holdingReader serves data, then blocks until ctx is done, like an open log stream of a run that +// is still running but logs nothing. +type holdingReader struct { + data io.Reader + ctx context.Context +} + +func (r *holdingReader) Read(p []byte) (int, error) { + n, err := r.data.Read(p) + if errors.Is(err, io.EOF) && n == 0 { + <-r.ctx.Done() + return 0, r.ctx.Err() + } + if errors.Is(err, io.EOF) { + return n, nil + } + return n, err +} + +func (b *logBackend) fullLogReads() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.fullLogCalls +} + func (b *logBackend) Do(req *http.Request) (*http.Response, error) { b.mu.Lock() defer b.mu.Unlock() @@ -80,6 +107,9 @@ func (b *logBackend) Do(req *http.Request) (*http.Response, error) { body := b.streamBodies[min(b.streamCalls, len(b.streamBodies)-1)] b.streamCalls++ var reader io.Reader = strings.NewReader(body) + if b.holdStream { + reader = &holdingReader{data: reader, ctx: req.Context()} + } if b.streamTailErr != nil { reader = &failingReader{data: reader, err: b.streamTailErr} } @@ -373,6 +403,84 @@ func TestStreamedLogStopReportsAFailedReadOfTheWholeLog(t *testing.T) { } } +// Stop with a context that ends while the stream is still open and has delivered lines cuts the +// log off: it returns the error of its context, so a truncated log is not mistaken for a complete +// one. +func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} + out := &syncBuffer{} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := streamed.Stop(stopCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected the error of the Stop context for a log that was cut off, got %v", err) + } +} + +// Without a Stop context to cut it off, a live stream ends after the grace period: that is how a +// redirection ends normally, and it is not an error. +func TestStreamedLogStopOfALiveStreamAfterTheGracePeriodIsNotAnError(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} + out := &syncBuffer{} + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(context.Background(), StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + if err := streamed.Stop(context.Background()); err != nil { + t.Fatalf("a normal stop is not an error, got %v", err) + } +} + +// The context given to Start ends the redirection on purpose, so its end is not an error of Stop, +// whether it ends before Stop or during the final read of the whole log. +func TestStreamedLogStartContextEndingIsNotAnErrorOfStop(t *testing.T) { + t.Run("before Stop", func(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + parent, cancelParent := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + cancelParent() + time.Sleep(50 * time.Millisecond) + if err := streamed.Stop(context.Background()); err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) + t.Run("during the final read", func(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + parent, cancelParent := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + stopped := make(chan error, 1) + go func() { stopped <- streamed.Stop(context.Background()) }() + waitFor(t, func() bool { return backend.fullLogReads() == 1 }) + cancelParent() + if err := <-stopped; err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) +} + func TestNewStreamedLogOfAMissingRunIsNotFound(t *testing.T) { backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} streamed, found, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) diff --git a/url_normalization.go b/url_normalization.go index 7ab3baa..2ae9e2a 100644 --- a/url_normalization.go +++ b/url_normalization.go @@ -96,6 +96,7 @@ func normalizeURL(raw string) (string, error) { out.WriteString(scheme) out.WriteByte(':') path := rest + hasAuthority := special switch { case special && !isFile: // A special URL always has an authority, whatever number of slashes (even none) follows @@ -106,7 +107,7 @@ func normalizeURL(raw string) (string, error) { end = len(authority) } path = authority[end:] - host, err := normalizeAuthority(authority[:end], authorityRules{defaultPort: defaultPorts[scheme], special: true, allowPort: true}) + host, err := normalizeAuthority(authority[:end], authorityRules{defaultPort: defaultPorts[scheme], special: true, allowPort: true, allowUserinfo: true}) if err != nil { return "", fmt.Errorf("invalid URL %q: %w", raw, err) } @@ -138,12 +139,13 @@ func normalizeURL(raw string) (string, error) { } default: if authority, ok := strings.CutPrefix(rest, "//"); ok { + hasAuthority = true end := strings.IndexByte(authority, '/') if end < 0 { end = len(authority) } path = authority[end:] - host, err := normalizeAuthority(authority[:end], authorityRules{allowPort: true}) + host, err := normalizeAuthority(authority[:end], authorityRules{allowPort: true, allowUserinfo: true}) if err != nil { return "", fmt.Errorf("invalid URL %q: %w", raw, err) } @@ -158,6 +160,10 @@ func normalizeURL(raw string) (string, error) { opaquePath := !special && !strings.HasPrefix(path, "/") if !opaquePath { path = removeDotSegments(path) + if !hasAuthority && strings.HasPrefix(path, "//") { + // Without this marker the path would read as the start of an authority. + out.WriteString("/.") + } out.WriteString(percentEncode(path, unsafePathBytes)) } else { out.WriteString(percentEncodeControls(path)) @@ -184,17 +190,23 @@ type authorityRules struct { // special is true for the special schemes, whose host is a domain, IPv4 or IPv6 address. The // host of any other scheme is opaque and kept as given. special bool - // allowPort is false for a scheme whose URLs have no port (file). - allowPort bool + // allowPort is false for a scheme whose URLs have no port (file), and allowUserinfo for one + // whose URLs have no credentials (file). + allowPort bool + allowUserinfo bool } // normalizeAuthority normalizes the "userinfo@host:port" part of a URL. func normalizeAuthority(authority string, rules authorityRules) (string, error) { var userinfo string if i := strings.LastIndexByte(authority, '@'); i >= 0 { + if !rules.allowUserinfo { + return "", errors.New("credentials are not allowed") + } userinfo, authority = authority[:i], authority[i+1:] } host, port := authority, "" + hasPortSeparator := false if strings.HasPrefix(authority, "[") { end := strings.IndexByte(authority, ']') if end < 0 { @@ -209,15 +221,19 @@ func normalizeAuthority(authority string, rules authorityRules) (string, error) return "", fmt.Errorf("invalid IPv6 address %q", authority[1:end]) } host, port = serializeIPv6(address), strings.TrimPrefix(tail, ":") + hasPortSeparator = tail != "" } else { if i := strings.LastIndexByte(authority, ':'); i >= 0 { - host, port = authority[:i], authority[i+1:] + host, port, hasPortSeparator = authority[:i], authority[i+1:], true } var err error if host, err = normalizeHost(host, rules.special); err != nil { return "", err } } + if hasPortSeparator && !rules.allowPort { + return "", errors.New("a port is not allowed") + } if port != "" { number, err := parsePort(port) if err != nil || !rules.allowPort { diff --git a/url_normalization_test.go b/url_normalization_test.go index ef6c9b7..3ee5f5b 100644 --- a/url_normalization_test.go +++ b/url_normalization_test.go @@ -33,6 +33,9 @@ func TestNormalizeURL(t *testing.T) { {"foo://Example.COM/A", "foo://Example.COM/A"}, {"foo://host/a/../b", "foo://host/b"}, {"foo:/a/./b", "foo:/a/b"}, + {"foo:/.//p", "foo:/.//p"}, + {"foo:/a/..//p", "foo:/.//p"}, + {"foo://h/.//p", "foo://h//p"}, // An empty password, and an empty userinfo, are dropped. {"https://u:@ex.com/", "https://u@ex.com/"}, {"https://:@ex.com/", "https://ex.com/"}, @@ -79,7 +82,7 @@ func TestNormalizeURL(t *testing.T) { } func TestNormalizeURLRejectsInvalidURLs(t *testing.T) { - for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "http://1.2.3.4.5/", "http://256.1.1.1/", "http://foo.123/", "https://%ff.example/", "http://ex.com:+80/", "http://ex.com:-0/", "http://[fe80::1%25eth0]/", "file://host:80/x"} { + for _, raw := range []string{"not a url", "/relative/path", "https://", "http://exa mple.com/", "http://example.com:99999/", "http://example.com:abc/", "http://[::1/", "http://1.2.3.4.5/", "http://256.1.1.1/", "http://foo.123/", "https://%ff.example/", "http://ex.com:+80/", "http://ex.com:-0/", "http://[fe80::1%25eth0]/", "file://host:80/x", "file://host:/x", "file://u@h/x", "file://u:p@h/x"} { if got, err := normalizeURL(raw); err == nil { t.Errorf("normalizeURL(%q) = %q, want an error", raw, got) } From 9a39c2d11bd366eb44795d9fece4fabf6af270b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 13:05:14 +0000 Subject: [PATCH 05/12] chore: apply De Morgan's law in streamLog --- streamed_log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamed_log.go b/streamed_log.go index 46da510..9d88a09 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -234,7 +234,7 @@ func (s *StreamedLog) isStopping() bool { func (s *StreamedLog) streamLog(parent, streamCtx context.Context) error { err := s.redirect(parent, streamCtx) var lost *logReadError - if errors.As(err, &lost) && !(isContextError(lost.err) && parent.Err() != nil) { + if errors.As(err, &lost) && (!isContextError(lost.err) || parent.Err() == nil) { // The log was not redirected: the caller must hear of it, unless the cause is the end of // the caller's own context, which ends the redirection on purpose. return lost.err From 0d50caca8a55ff6f1296a4b2a968a49ae2612a03 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 13:16:40 +0000 Subject: [PATCH 06/12] fix: address the fourth review of the v3 port --- streamed_log.go | 11 +++++++---- streamed_log_test.go | 37 +++++++++++++++++++++++++++++++++++++ url_normalization.go | 2 +- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/streamed_log.go b/streamed_log.go index 9d88a09..ec7bb70 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -70,6 +70,7 @@ type StreamedLog struct { mu sync.Mutex // serializes writes to output and guards the fields below running bool stopping bool + startCtx context.Context // the context of the Start call, which ends the redirection stopCtx context.Context // the context of the Stop call, which bounds the final read cancelStream context.CancelFunc wake chan struct{} @@ -133,6 +134,7 @@ func (s *StreamedLog) Start(ctx context.Context) error { streamCtx, cancel := context.WithCancel(ctx) s.running = true s.stopping = false + s.startCtx = ctx s.stopCtx = nil s.cancelStream = cancel s.wake = make(chan struct{}) @@ -164,7 +166,7 @@ func (s *StreamedLog) Stop(ctx context.Context) error { s.stopping = true s.stopCtx = ctx close(s.wake) - cancel, done := s.cancelStream, s.done + cancel, done, startCtx := s.cancelStream, s.done, s.startCtx s.mu.Unlock() timer := time.AfterFunc(stopGrace, cancel) @@ -179,14 +181,15 @@ func (s *StreamedLog) Stop(ctx context.Context) error { // The redirection is still running: cutting it off loses the rest of the log, which // the caller must be able to tell from a complete one. cancel() - if err = <-done; err == nil { + if err = <-done; err == nil && startCtx.Err() == nil { err = ctx.Err() } } } cancel() - // A read cut short by the context of this call reports a plain cancellation: name the cause. - if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, context.Canceled) { + // A read cut short by the context of this call reports a plain cancellation: name the cause, + // unless the context of Start, which ends the redirection on purpose, ended as well. + if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, context.Canceled) && startCtx.Err() == nil { err = ctxErr } diff --git a/streamed_log_test.go b/streamed_log_test.go index 7bf9762..ee22312 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -481,6 +481,43 @@ func TestStreamedLogStartContextEndingIsNotAnErrorOfStop(t *testing.T) { }) } +// The end of the context given to Start ends the redirection on purpose: Stop with the same +// context, as Call does, reports no error for an open stream that is cut off by it. +func TestStreamedLogStopAfterTheStartContextEndedIsNotAnError(t *testing.T) { + backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} + out := &syncBuffer{} + parent, cancel := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + cancel() + if err := streamed.Stop(parent); err != nil { + t.Fatalf("the end of the Start context is not an error, got %v", err) + } +} + +// A Call whose context ends while the log stream is still open reports no redirection error. +func TestCallWhoseContextEndsWithAnOpenLogStreamReportsNoError(t *testing.T) { + for range 10 { + out := &syncBuffer{} + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, holdStream: true}, pollDelay: 300 * time.Millisecond} + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) + _, err := testClient(backend, 0).Actor("actor1").Call(ctx, nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) + cancel() + if err == nil { + t.Fatal("expected the call to end with the error of its context") + } + if strings.Contains(out.String(), "Log redirection stopped due to error") { + t.Fatalf("the end of the context of the call is not a redirection error: %q", out.String()) + } + } +} + func TestNewStreamedLogOfAMissingRunIsNotFound(t *testing.T) { backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} streamed, found, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) diff --git a/url_normalization.go b/url_normalization.go index 2ae9e2a..8c2c21a 100644 --- a/url_normalization.go +++ b/url_normalization.go @@ -236,7 +236,7 @@ func normalizeAuthority(authority string, rules authorityRules) (string, error) } if port != "" { number, err := parsePort(port) - if err != nil || !rules.allowPort { + if err != nil { return "", fmt.Errorf("invalid port %q", port) } port = strconv.Itoa(number) From 208d0a06f851f699a653e639e752b285050d531a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 13:33:59 +0000 Subject: [PATCH 07/12] fix: address the fifth review of the v3 port --- docs/runs.md | 4 +++- streamed_log.go | 10 +++++----- streamed_log_test.go | 40 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/docs/runs.md b/docs/runs.md index ca06da9..5edade2 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -165,7 +165,9 @@ background, and it also ends when the `ctx` it got is done. `Stop` gives the mes to one second, then aborts the stream; if the stream delivered nothing yet, it reads the whole log in one request, so a short run that finished before the stream connected is not lost. The `ctx` given to `Stop` bounds that wait and that read: when it is done first, `Stop` returns its error -and the rest of the log is not redirected. A second `Stop` while one is running returns an error. A log that +and the rest of the log is not redirected, unless the `ctx` given to `Start` is done too, which +ends the redirection on purpose and is not an error. A second `Stop` while one is running returns +an error. A log that has no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, until its first bytes arrive. `Stop` returns the error that ended the redirection early, if any; `Call` reports it on the redirection's output and goes on. diff --git a/streamed_log.go b/streamed_log.go index ec7bb70..57959ae 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -151,8 +151,9 @@ func (s *StreamedLog) Start(ctx context.Context) error { // request instead, so a short run that finished before the stream connected is not lost. It // returns the error that made the redirection stop early, if any. ctx bounds the wait for it, // the final read of the whole log included: when ctx is done first, Stop gives up on the rest of -// the log and returns the error of ctx. It returns an error if the redirection is not running or -// is already being stopped. +// the log and returns the error of ctx, unless the context given to Start is done too, which ends +// the redirection on purpose. It returns an error if the redirection is not running or is already +// being stopped. func (s *StreamedLog) Stop(ctx context.Context) error { s.mu.Lock() if !s.running { @@ -187,9 +188,8 @@ func (s *StreamedLog) Stop(ctx context.Context) error { } } cancel() - // A read cut short by the context of this call reports a plain cancellation: name the cause, - // unless the context of Start, which ends the redirection on purpose, ended as well. - if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, context.Canceled) && startCtx.Err() == nil { + // A read cut short by the context of this call reports a plain cancellation: name the cause. + if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, context.Canceled) { err = ctxErr } diff --git a/streamed_log_test.go b/streamed_log_test.go index ee22312..088efc0 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -501,14 +501,23 @@ func TestStreamedLogStopAfterTheStartContextEndedIsNotAnError(t *testing.T) { } } -// A Call whose context ends while the log stream is still open reports no redirection error. +// A Call whose context ends while the log stream is still open, and has delivered lines, reports +// no redirection error. It is a smoke test of the whole path: whether the redirection is still +// running when the call stops it depends on timing, so the deterministic guard of this behavior is +// TestStreamedLogStopAfterTheStartContextEndedIsNotAnError. func TestCallWhoseContextEndsWithAnOpenLogStreamReportsNoError(t *testing.T) { for range 10 { out := &syncBuffer{} - backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, holdStream: true}, pollDelay: 300 * time.Millisecond} - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) + backend := &callLogBackend{logBackend: logBackend{streamBodies: []string{sampleLog}, holdStream: true}, pollDelay: time.Minute} + ctx, cancel := context.WithCancel(context.Background()) + go func() { + defer cancel() + deadline := time.Now().Add(10 * time.Second) + for !strings.Contains(out.String(), "last message") && time.Now().Before(deadline) { + time.Sleep(2 * time.Millisecond) + } + }() _, err := testClient(backend, 0).Actor("actor1").Call(ctx, nil, ActorStartOptions{Log: &StreamedLogOptions{Output: out, Prefix: Ptr("")}}, Ptr(int64(0))) - cancel() if err == nil { t.Fatal("expected the call to end with the error of its context") } @@ -518,6 +527,29 @@ func TestCallWhoseContextEndsWithAnOpenLogStreamReportsNoError(t *testing.T) { } } +// Stop never returns a bare cancellation: the Start context ending just after the Stop context +// (as Call does) gives either the error of the Stop context or nil. +func TestStreamedLogStopNeverReturnsABareCancellation(t *testing.T) { + for range 40 { + backend := &logBackend{streamBodies: []string{""}, blockFullLog: true} + parent, cancelParent := context.WithCancel(context.Background()) + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(parent, StreamedLogOptions{Output: io.Discard, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(parent); err != nil { + t.Fatal(err) + } + stopCtx, cancelStop := context.WithTimeout(context.Background(), 20*time.Millisecond) + context.AfterFunc(stopCtx, cancelParent) + err = streamed.Stop(stopCtx) + cancelStop() + if err != nil && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected nil or the error of the Stop context, got %v", err) + } + } +} + func TestNewStreamedLogOfAMissingRunIsNotFound(t *testing.T) { backend := &mockBackend{responses: constant(404, `{"error":{"type":"record-not-found","message":"m"}}`)} streamed, found, err := testClient(backend, 0).Run("gone").NewStreamedLog(context.Background(), StreamedLogOptions{}) From ffc30bfc9480f85b3ecbc0f333b6aa69b4d69bde Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 13:42:39 +0000 Subject: [PATCH 08/12] docs: correct the Stop contract of StreamedLog when both contexts end --- docs/runs.md | 12 ++++++------ streamed_log.go | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/runs.md b/docs/runs.md index 5edade2..1e1e66e 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -165,12 +165,12 @@ background, and it also ends when the `ctx` it got is done. `Stop` gives the mes to one second, then aborts the stream; if the stream delivered nothing yet, it reads the whole log in one request, so a short run that finished before the stream connected is not lost. The `ctx` given to `Stop` bounds that wait and that read: when it is done first, `Stop` returns its error -and the rest of the log is not redirected, unless the `ctx` given to `Start` is done too, which -ends the redirection on purpose and is not an error. A second `Stop` while one is running returns -an error. A log that -has no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, -until its first bytes arrive. `Stop` returns the error that ended the redirection early, if any; -`Call` reports it on the redirection's output and goes on. +and the rest of the log is not redirected, unless the `ctx` given to `Start` ended first, which +ends the redirection on purpose and is not an error. When both end at about the same time, +either result is possible. A second `Stop` while one is running returns an error. A log that has +no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, until its +first bytes arrive. `Stop` returns the error that ended the redirection early, if any; `Call` +reports it on the redirection's output and goes on. ## Live iteration of dataset items diff --git a/streamed_log.go b/streamed_log.go index 57959ae..a21a1be 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -151,9 +151,9 @@ func (s *StreamedLog) Start(ctx context.Context) error { // request instead, so a short run that finished before the stream connected is not lost. It // returns the error that made the redirection stop early, if any. ctx bounds the wait for it, // the final read of the whole log included: when ctx is done first, Stop gives up on the rest of -// the log and returns the error of ctx, unless the context given to Start is done too, which ends -// the redirection on purpose. It returns an error if the redirection is not running or is already -// being stopped. +// the log and returns the error of ctx, unless the context given to Start ended first, which ends +// the redirection on purpose; when both end at about the same time, either result is possible. It +// returns an error if the redirection is not running or is already being stopped. func (s *StreamedLog) Stop(ctx context.Context) error { s.mu.Lock() if !s.running { From 848f3ba8d059a597005b783886ea895f21bd03af Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 14:07:22 +0000 Subject: [PATCH 09/12] fix: address the full review of the v3 port Stop keeps reading the log in flight, a multi-line message split across reads is kept whole, PaginateRequestsWithOptions adds Limit, Cursor and Filter to the request iterator, GetStreamedLog is deprecated in favor of NewStreamedLog, and the 0.14.0 changelog entries are shortened. --- CHANGELOG.md | 71 +++++++++-------------------- docs/misc.md | 8 ++-- docs/runs.md | 5 ++- docs/storages.md | 10 +++++ request_queue.go | 61 ++++++++++++++++++++----- run.go | 11 +++-- streamed_log.go | 45 ++++++++++--------- streamed_log_test.go | 90 +++++++++++++++++++++++++++++++++---- tests/request_queue_test.go | 69 ++++++++++++++++++++++++++++ 9 files changed, 269 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cde938a..171a1e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,63 +9,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `RunListOptions.Deployment` filters the account-wide and Actor-scoped run lists by the Actor - Standby pool that serves a run; `StoreListOptions.IncludeInputSchema` adds each Actor's compact - input schema to the Store listing. -- `UserClient.Limits`, `UpdateLimits` and `MonthlyUsage*` work on `client.User(id)` too (the API - serves them for your own account or your organization by ID or username); they no longer fail - on a client that is not `Me()`. -- `StreamedLog` (`RunClient.NewStreamedLog`, `Start`, `Stop`, `StreamedLogOptions`) redirects a - run's log message by message to a writer with a prefix. `ActorClient.Call` and `CallRaw` use it - by default, writing to `os.Stderr` with the prefix `"{Actor name} runId:{run ID} -> "`; - `ActorStartOptions.Log` changes the output or turns it off (`Disabled`). -- `RunClient.IterateDatasetItems` and `IterateRunDatasetItems[T]` iterate over the items of a - run's default dataset while the run is still producing them. -- `KeyValueStoreClient.SetRecordRawWithOptions` and `SetRecordJSONWithOptions` with - `SetRecordOptions` (`ContentEncoding`, `DoNotRetryTimeouts`). -- Typed, normalized URL fields on the models: `ActorRun.ConsoleURL`/`Output`, `Dataset.ConsoleURL`/ - `ItemsPublicURL`, `KeyValueStore.ConsoleURL`/`KeysPublicURL`/`RecordsPublicURL`, - `KeyValueStoreKey.RecordPublicURL`, `RequestQueue.ConsoleURL`, `Task.StandbyURL`, - `ActorStoreListItem.URL`/`UserPictureURL`, `User.Profile` (`UserProfile`) and - `WebhookDispatch.Webhook` (`WebhookDispatchWebhookSummary`). The URL fields the API specification - declares (these, `ActorRun.ContainerURL` and `Webhook.RequestURL`) are returned normalized, close - to the WHATWG URL serialization: the scheme and host are lowercased, a non-ASCII host becomes - punycode, an IPv4 or IPv6 host is written canonically, a default port is dropped, an empty path - becomes `/`, dot segments are resolved and unsafe characters are percent-encoded. A value that - is not an absolute URL fails decoding the response. +- `RunListOptions.Deployment` and `StoreListOptions.IncludeInputSchema` list parameters. +- `UserClient` limits and monthly usage methods work on `client.User(id)` too, not only `Me()`. +- `StreamedLog` (`RunClient.NewStreamedLog`) redirects a run's log to a writer; `ActorClient.Call` + and `CallRaw` use it by default, `ActorStartOptions.Log` configures or disables it. +- `RunClient.IterateDatasetItems` and `IterateRunDatasetItems[T]` iterate a run's dataset while it + is still being filled. +- `KeyValueStoreClient.SetRecordRawWithOptions` and `SetRecordJSONWithOptions`. +- `RequestQueueClient.PaginateRequestsWithOptions` iterates with `Limit`, `Cursor` and `Filter`. +- Typed URL fields on the models (`ConsoleURL`, `ItemsPublicURL`, `RecordPublicURL`, + `User.Profile`, `WebhookDispatch.Webhook` and others), returned normalized. ### Changed - Bumped `APISpecVersion` to `v2-2026-10-29T100357Z` and `ClientVersion` to `0.14.0`. -- **Breaking:** `ActorVersionCollectionClient.List` takes no options and `Iterate` takes no - arguments, like the environment-variable collection: the versions endpoint documents no query - parameters and always returns every version in one response. Matches the reference client, - whose versions `list()` takes no arguments. -- The keys of the new typed fields (`consoleUrl`, `output`, `itemsPublicUrl`, `keysPublicUrl`, - `recordsPublicUrl`, `recordPublicUrl`, `standbyUrl`, `url`, `userPictureUrl`, `profile` and a - dispatch's `webhook`) are no longer in the `Extra` map of their model. -- `ListRequestsOptions.ExclusiveStartID` is marked deprecated: the API deprecated the - `exclusiveStartId` parameter in favor of `cursor`. -- `ActorClient.Call`/`CallRaw` and `TaskClient.Call` start the run without a client-side timeout, - so a slow start is no longer aborted and repeated, which started a second run. -- A request that timed out is no longer retried when it is not idempotent: `Start`, `StartRaw` - and `Build` of an Actor, a task's `Start`, and `PushItems`. -- `KeyValueStoreClient.Update`, `RecordExists`, `GetRecordPublicURL`, `CreateKeysPublicURL` and - `DatasetClient.CreateItemsPublicURL*` use the "long" timeout tier, like the reference client. -- The storages and the log of a run reached through `LastRun`/`LastRunWithOptions`, and their - public URLs, carry its `status` and `origin` filters, so they address the run the filters - resolve to instead of the newest run of any status. -- Documented that `GetRecord` returns the raw bytes without parsing them and that `SetRecordRaw` - sends them as they are, whatever the content type. +- **Breaking:** `ActorVersionCollectionClient.List` and `Iterate` take no options. +- The typed URL fields are no longer in the `Extra` map of their model. +- `ListRequestsOptions.ExclusiveStartID` is deprecated in favor of `Cursor`. +- `RunClient.GetStreamedLog` is deprecated: it returns the raw stream, use `NewStreamedLog`. +- `Call` starts the run without a client-side timeout. +- Requests that are not idempotent (`Start`, `Build`, `PushItems`) are not retried after a timeout. +- Some `KeyValueStoreClient` and `DatasetClient` methods use the "long" timeout tier. +- Storages, log and public URLs of a `LastRun` carry its `status` and `origin` filters. +- Documented that `GetRecord` and `SetRecordRaw` handle raw bytes. ### Fixed -- `DatasetClient.PushItems` rejects, before sending, a value that is not a JSON object or an array - of objects, the only shapes the API accepts. A plain Go string, number, `nil` or a slice of - strings used to be sent and rejected by the API. -- `DatasetClient.ListItems`, `ActorClient.DefaultBuild` and `ValidateInputForBuild`, - `TaskClient.UpdateInput`, `UserClient.UpdateLimits` and `RunClient.Charge` now fail on an empty or - dot-segment ID like every other method, instead of sending a malformed request. +- `DatasetClient.PushItems` rejects values that are not a JSON object or an array of objects. +- Methods that lacked an ID check now fail on an empty or dot-segment ID. ## [0.13.1] - 2026-10-26 diff --git a/docs/misc.md b/docs/misc.md index c996193..5cc6a2b 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -117,15 +117,15 @@ fmt.Println(string(usage)) // raw JSON usage report for the current month | `Raw` | `*bool` | Return the unprocessed log content (no platform post-processing). | | `Download` | `*bool` | Set `Content-Disposition` so the log is served as a download. | -`client.Run(id).GetStreamedLog(ctx)` returns a raw live stream of the run's log directly. To -redirect the log message by message to a writer, with a prefix, see -[log redirection](runs.md#log-redirection). +`client.Run(id).GetStreamedLog(ctx)` is deprecated: despite the name it returns the raw live +stream, like `Log().StreamWithOptions` with `Raw` set. To redirect the log message by message to +a writer, with a prefix, use `NewStreamedLog`; see [log redirection](runs.md#log-redirection). ```go // Read the whole log. logText, ok, err := client.Run(runID).Log().Get(ctx) -// Or stream it live (log redirection). +// Or stream it live, as raw bytes. stream, err := client.Run(runID).Log().Stream(ctx) if err != nil { log.Fatal(err) diff --git a/docs/runs.md b/docs/runs.md index 1e1e66e..20fcdd7 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -41,7 +41,7 @@ with `client.Run(id)`, and an Actor's or task's runs with `client.Actor(id).Runs | `KeyValueStore() *KeyValueStoreClient` | The run's default key-value store. | | `RequestQueue() *RequestQueueClient` | The run's default request queue. | | `Log() *LogClient` | The run's log. | -| `GetStreamedLog(ctx) (io.ReadCloser, error)` | Live stream of the run's raw log, as bytes. | +| `GetStreamedLog(ctx) (io.ReadCloser, error)` | Deprecated: a raw live stream of the run's log, as bytes, despite the name. Use `NewStreamedLog` to redirect the log, or `Log().StreamWithOptions` for the raw stream. | | `NewStreamedLog(ctx, StreamedLogOptions) (*StreamedLog, bool, error)` | Redirects the run's log message by message to a writer; see [Log redirection](#log-redirection). The bool is `false` for a run that does not exist (with the default prefix). | | `IterateDatasetItems(RunIterateDatasetItemsOptions) *RunDatasetItemsIterator[json.RawMessage]` | Iterates over the items of the run's default dataset while the run is still producing them; see [Live iteration of dataset items](#live-iteration-of-dataset-items). | @@ -160,7 +160,8 @@ if err := streamed.Stop(ctx); err != nil { | `Disabled` | `bool` | Turns the redirection off in `ActorStartOptions.Log`. | The log is split into messages at the timestamps the platform puts in front of each of them, so a -message that spans several lines is written as one. `Start` begins the redirection in the +message that spans several lines is written as one, even when its lines arrive in separate +reads; the last message is therefore written when the next one starts or the log ends. `Start` begins the redirection in the background, and it also ends when the `ctx` it got is done. `Stop` gives the messages in flight up to one second, then aborts the stream; if the stream delivered nothing yet, it reads the whole log in one request, so a short run that finished before the stream connected is not lost. The `ctx` diff --git a/docs/storages.md b/docs/storages.md index f45c124..0366b38 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -289,6 +289,7 @@ Single queue: `client.RequestQueue(id)`: | `DeleteRequest(ctx, id) error` | Delete a request. | | `ListRequests(ctx, ListRequestsOptions) (json.RawMessage, error)` | Paginated list. Returns the raw API response (`{ "items": [...], "nextCursor": ... }`); unmarshal into your own type, or use `PaginateRequests` for typed iteration. | | `PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator` | Lazy iterator over all requests (yields `*RequestQueueRequest`). | +| `PaginateRequestsWithOptions(ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator` | The same with `Limit` (caps the total across pages), `Cursor` (where to start) and `Filter` (every page); `chunkSize` is the page size. | | `BatchAddRequests(ctx, []RequestQueueRequest, forefront) (BatchAddResult, error)` | Add many requests, auto-split into batches that respect the API's 25-request and (effective) 9 MiB payload-size limits. Fails before sending anything if any single request is too large for a batch of its own. Unlike the reference client, does not itself retry a batch's `UnprocessedRequests` — inspect the result and retry those explicitly if needed. | | `BatchDeleteRequests(ctx, requests any) (json.RawMessage, error)` | Delete many requests in one call. `requests` is the JSON-marshalable batch payload — a slice in which each element identifies one request by **either** its `id` **or** its `uniqueKey` (e.g. `[]map[string]string{{"uniqueKey": "..."}}` or a slice of `RequestQueueRequest`), matching the reference client's `batchDeleteRequests`. Returns the raw API response. | | `ListAndLockHead(ctx, lockSecs int64, limit *int64) (json.RawMessage, error)` | Fetch the head of the queue and lock the returned requests for `lockSecs` seconds. Returns the raw API response. | @@ -380,6 +381,15 @@ for { } fmt.Println(req.URL) } + +// Iterate over at most 50 pending requests, 25 per page. +pending := queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ + Limit: apify.Ptr(int64(50)), + Filter: []string{apify.RequestFilterPending}, +}, apify.Ptr(int64(25))) +for req, err := pending.Next(ctx); req != nil && err == nil; req, err = pending.Next(ctx) { + fmt.Println(req.URL) +} ``` `StorageListOptions` is shared by the three storage collections' `List` methods. All fields are diff --git a/request_queue.go b/request_queue.go index 4685575..c3f5d6d 100644 --- a/request_queue.go +++ b/request_queue.go @@ -382,20 +382,33 @@ func (c *RequestQueueClient) UnlockRequests(ctx context.Context) (json.RawMessag } // PaginateRequests returns a lazy iterator over all requests in the queue, fetching pages -// of up to pageLimit requests at a time (nil for the server default). +// of up to pageLimit requests at a time (nil for the server default). To cap the total, start +// from a cursor or filter the states, use [RequestQueueClient.PaginateRequestsWithOptions]. func (c *RequestQueueClient) PaginateRequests(pageLimit *int64) *RequestQueueRequestsIterator { - return &RequestQueueRequestsIterator{client: c, pageLimit: pageLimit} + return c.PaginateRequestsWithOptions(ListRequestsOptions{}, pageLimit) +} + +// PaginateRequestsWithOptions is [RequestQueueClient.PaginateRequests] with listing options, like +// the reference client's paginateRequests: Limit caps the number of requests yielded across all +// pages (nil or ptr(0) means all of them), Cursor (or the deprecated ExclusiveStartID) sets where +// to start, and Filter applies to every page. The page size is the separate chunkSize argument +// (nil for the server default). Invalid options are reported by the first call of Next. +func (c *RequestQueueClient) PaginateRequestsWithOptions(options ListRequestsOptions, chunkSize *int64) *RequestQueueRequestsIterator { + return &RequestQueueRequestsIterator{client: c, options: options, chunkSize: chunkSize} } // RequestQueueRequestsIterator lazily iterates over a request queue's requests, fetching one // page at a time via the cursor-based listing endpoint. type RequestQueueRequestsIterator struct { client *RequestQueueClient - pageLimit *int64 + options ListRequestsOptions + chunkSize *int64 buffer []RequestQueueRequest pos int nextCursor string + capped bool // whether a total cap (options.Limit) is in effect + remaining int64 // requests the cap still allows; only meaningful when capped started bool exhausted bool } @@ -421,13 +434,33 @@ func (it *RequestQueueRequestsIterator) Next(ctx context.Context) (*RequestQueue return &req, nil } -// fetchPage loads the next page of requests into the buffer. +// fetchPage loads the next page of requests into the buffer. The page size is the smaller of the +// remaining total cap and the chunk size, so the last page is never over-fetched. func (it *RequestQueueRequestsIterator) fetchPage(ctx context.Context) error { - params := NewQueryParams() - params.AddInt("limit", it.pageLimit) - if it.nextCursor != "" { - params.AddString("cursor", &it.nextCursor) + opts := it.options + if !it.started { + if err := opts.validate(); err != nil { + return err + } + if l := opts.Limit; l != nil && *l > 0 { + it.capped = true + it.remaining = *l + } + } else { + opts.ExclusiveStartID = nil + opts.Cursor = &it.nextCursor + } + var capLeft, chunk int64 + if it.capped { + capLeft = it.remaining } + if it.chunkSize != nil { + chunk = *it.chunkSize + } + opts.Limit = pageLimitPtr(minForLimitParam(capLeft, chunk)) + + params := NewQueryParams() + opts.apply(params) it.client.withClientKey(params) raw, err := getResourceRequired[requestsPage](ctx, it.client.ctx, "requests", params, it.client.ctx.mediumTimeout()) @@ -437,12 +470,18 @@ func (it *RequestQueueRequestsIterator) fetchPage(ctx context.Context) error { it.started = true it.buffer = raw.Items it.pos = 0 + it.nextCursor = "" if raw.NextCursor != nil { it.nextCursor = *raw.NextCursor - } else { - it.nextCursor = "" } - if len(raw.Items) == 0 && it.nextCursor == "" { + if it.capped { + it.remaining -= int64(len(it.buffer)) + if it.remaining < 0 { // a server that returned more than asked for + it.buffer = it.buffer[:int64(len(it.buffer))+it.remaining] + it.remaining = 0 + } + } + if it.nextCursor == "" || (it.capped && it.remaining == 0) { it.exhausted = true } return nil diff --git a/run.go b/run.go index 009fdb4..f587e61 100644 --- a/run.go +++ b/run.go @@ -270,11 +270,14 @@ func (c *RunClient) Log() *LogClient { return client } -// GetStreamedLog opens a live stream of this run's raw log, for convenient log redirection. +// GetStreamedLog opens a live stream of this run's raw log. The caller must close the returned +// reader. It is a convenience wrapper over Log().StreamWithOptions with raw=true. // -// It is a convenience wrapper over Log().StreamWithOptions with raw=true (matching the -// reference client's getStreamedLog, which streams raw log content). The caller must close -// the returned reader. +// Despite the name it is not the counterpart of the reference client's getStreamedLog(), which +// redirects the log message by message: that is [RunClient.NewStreamedLog]. +// +// Deprecated: use [RunClient.NewStreamedLog] to redirect the log, or Log().StreamWithOptions for +// the raw stream. func (c *RunClient) GetStreamedLog(ctx context.Context) (io.ReadCloser, error) { raw := true return c.Log().StreamWithOptions(ctx, LogOptions{Raw: &raw}) diff --git a/streamed_log.go b/streamed_log.go index a21a1be..5f3e0cb 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -285,6 +285,7 @@ func (s *StreamedLog) redirect(parent, streamCtx context.Context) error { gotBytes, readErr = s.consumeStream(stream) _ = stream.Close() if readErr != nil && streamCtx.Err() == nil { + s.flush() return readErr } if !gotBytes { @@ -300,8 +301,9 @@ func (s *StreamedLog) redirect(parent, streamCtx context.Context) error { return nil } -// consumeStream redirects the complete messages of stream and reports whether it delivered any -// byte. It stops at the end of the stream, an error, or when the redirection is being stopped. +// consumeStream redirects the messages of stream and reports whether it delivered any byte. It +// reads until the end of the stream or an error; Stop ends it by cancelling the stream after its +// grace period, so the messages in flight are not lost. func (s *StreamedLog) consumeStream(stream io.Reader) (bool, error) { chunk := make([]byte, logReadChunkBytes) gotBytes := false @@ -317,40 +319,41 @@ func (s *StreamedLog) consumeStream(stream io.Reader) (bool, error) { } return gotBytes, err } - if s.isStopping() { - return gotBytes, nil - } } } -// consume adds data to the buffer and redirects every complete message in it: everything before -// the last end of line. An incomplete last line stays in the buffer for the next chunk. +// consume adds data to the buffer and redirects every message that is known to be complete: the +// ones followed by the timestamp of the next message. The last message stays in the buffer, since +// its continuation lines (a stack trace, say) may arrive in a later read; [StreamedLog.flush] +// writes it when the log ends. func (s *StreamedLog) consume(data []byte) { s.buffer.Write(data) - buffered := s.buffer.Bytes() - lastEOL := bytes.LastIndexByte(buffered, '\n') - if lastEOL < 0 { + text := s.buffer.String() + markers := logMessageMarker.FindAllStringIndex(text, -1) + if len(markers) < 2 { return } - complete := string(buffered[:lastEOL]) - rest := append([]byte(nil), buffered[lastEOL:]...) + last := markers[len(markers)-1][0] + s.redirectMessages(text[:last]) s.buffer.Reset() - s.buffer.Write(rest) - s.redirectMessages(complete) + s.buffer.WriteString(text[last:]) } -// flush redirects whatever is left in the buffer when the log ends: an incomplete last message, -// or a last line without an end of line. +// flush redirects whatever is left in the buffer when the log ends: the last message, or a last +// line that has no timestamp. func (s *StreamedLog) flush() { - last := strings.TrimSpace(s.buffer.String()) + text := s.buffer.String() s.buffer.Reset() - if last != "" { - s.write(last) + switch { + case strings.TrimSpace(text) == "": + case logMessageMarker.MatchString(text): + s.redirectMessages(text) + default: + s.write(strings.TrimSpace(text)) } } -// redirectMessages splits the complete part of the log into messages at their timestamps and -// writes them. +// redirectMessages splits text into messages at their timestamps and writes them. func (s *StreamedLog) redirectMessages(text string) { markers := logMessageMarker.FindAllStringSubmatchIndex(text, -1) for i, marker := range markers { diff --git a/streamed_log_test.go b/streamed_log_test.go index 088efc0..662771d 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "net/http" "strings" @@ -41,7 +42,8 @@ type logBackend struct { streamCalls int fullLogCalls int rawOnStream bool - streamTailErr error // makes the stream fail with this error after its body + streamChunks []string // when set, the stream serves these pieces, one per read + streamTailErr error // makes the stream fail with this error after its body // blockFullLog makes the read of the whole log wait until its request is canceled. blockFullLog bool // holdStream keeps the live stream open after its body, until the request is canceled. @@ -84,6 +86,22 @@ func (r *holdingReader) Read(p []byte) (int, error) { return n, err } +// chunkReader serves each of its chunks in a read of its own, like a log stream whose data +// arrives in separate pieces. +type chunkReader struct{ chunks []string } + +func (r *chunkReader) Read(p []byte) (int, error) { + if len(r.chunks) == 0 { + return 0, io.EOF + } + n := copy(p, r.chunks[0]) + r.chunks[0] = r.chunks[0][n:] + if r.chunks[0] == "" { + r.chunks = r.chunks[1:] + } + return n, nil +} + func (b *logBackend) fullLogReads() int { b.mu.Lock() defer b.mu.Unlock() @@ -107,6 +125,9 @@ func (b *logBackend) Do(req *http.Request) (*http.Response, error) { body := b.streamBodies[min(b.streamCalls, len(b.streamBodies)-1)] b.streamCalls++ var reader io.Reader = strings.NewReader(body) + if b.streamChunks != nil { + reader = &chunkReader{chunks: append([]string(nil), b.streamChunks...)} + } if b.holdStream { reader = &holdingReader{data: reader, ctx: req.Context()} } @@ -153,7 +174,7 @@ func TestStreamedLogRedirectsMessagesWithThePrefixOfTheActorAndTheRun(t *testing if err := streamed.Start(ctx); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) if err := streamed.Stop(ctx); err != nil { t.Fatalf("Stop: %v", err) } @@ -181,7 +202,7 @@ func TestStreamedLogPrefixOverrideSkipsTheLookups(t *testing.T) { if err := streamed.Start(ctx); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) if err := streamed.Stop(ctx); err != nil { t.Fatal(err) } @@ -225,7 +246,7 @@ func TestStreamedLogReopensAnEmptyStream(t *testing.T) { if err := streamed.Start(ctx); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) if err := streamed.Stop(ctx); err != nil { t.Fatal(err) } @@ -361,7 +382,7 @@ func TestStreamedLogStopWithADoneContextAfterACompleteLogIsNotAnError(t *testing if err := streamed.Start(context.Background()); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) done, cancel := context.WithCancel(context.Background()) cancel() if err := streamed.Stop(done); err != nil { @@ -416,7 +437,7 @@ func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { if err := streamed.Start(context.Background()); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() if err := streamed.Stop(stopCtx); !errors.Is(err, context.DeadlineExceeded) { @@ -426,6 +447,57 @@ func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { // Without a Stop context to cut it off, a live stream ends after the grace period: that is how a // redirection ends normally, and it is not an error. +func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { + backend := &logBackend{streamBodies: []string{""}, streamChunks: []string{ + "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n", + " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n", + }} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "next") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + want := "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n" + if got := out.String(); got != want { + t.Fatalf("redirected log:\n%q\nwant:\n%q", got, want) + } +} + +// Stop promises the messages in flight up to a second: a stream that is still delivering when +// Stop is called is read to its end, not just to the end of the current read. +func TestStreamedLogStopKeepsReadingTheMessagesInFlight(t *testing.T) { + const lines = 20000 + var log strings.Builder + for i := 0; i < lines; i++ { + fmt.Fprintf(&log, "2026-10-30T10:00:00.000Z message %d\n", i) + } + backend := &logBackend{streamBodies: []string{log.String()}} + out := &syncBuffer{} + ctx := context.Background() + streamed, _, err := testClient(backend, 0).Run("run1").NewStreamedLog(ctx, StreamedLogOptions{Output: out, Prefix: Ptr("")}) + if err != nil { + t.Fatal(err) + } + if err := streamed.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return strings.Contains(out.String(), "message 0\n") }) + if err := streamed.Stop(ctx); err != nil { + t.Fatal(err) + } + if got := strings.Count(out.String(), "\n"); got != lines { + t.Fatalf("redirected %d of %d messages", got, lines) + } +} + func TestStreamedLogStopOfALiveStreamAfterTheGracePeriodIsNotAnError(t *testing.T) { backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} out := &syncBuffer{} @@ -436,7 +508,7 @@ func TestStreamedLogStopOfALiveStreamAfterTheGracePeriodIsNotAnError(t *testing. if err := streamed.Start(context.Background()); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) if err := streamed.Stop(context.Background()); err != nil { t.Fatalf("a normal stop is not an error, got %v", err) } @@ -494,7 +566,7 @@ func TestStreamedLogStopAfterTheStartContextEndedIsNotAnError(t *testing.T) { if err := streamed.Start(parent); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "last message") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "first line") }) cancel() if err := streamed.Stop(parent); err != nil { t.Fatalf("the end of the Start context is not an error, got %v", err) @@ -513,7 +585,7 @@ func TestCallWhoseContextEndsWithAnOpenLogStreamReportsNoError(t *testing.T) { go func() { defer cancel() deadline := time.Now().Add(10 * time.Second) - for !strings.Contains(out.String(), "last message") && time.Now().Before(deadline) { + for !strings.Contains(out.String(), "first line") && time.Now().Before(deadline) { time.Sleep(2 * time.Millisecond) } }() diff --git a/tests/request_queue_test.go b/tests/request_queue_test.go index 4565735..9b0dbfc 100644 --- a/tests/request_queue_test.go +++ b/tests/request_queue_test.go @@ -1,6 +1,7 @@ package apify_test import ( + "encoding/json" "fmt" "testing" @@ -131,6 +132,74 @@ func TestRequestQueuePaginateMultiplePages(t *testing.T) { } } +// TestRequestQueuePaginateRequestsWithOptions validates the total cap, the filter and the cursor +// of the iterator over the requests. +func TestRequestQueuePaginateRequestsWithOptions(t *testing.T) { + client := requireClient(t) + ctx, cancel := testContext(t) + defer cancel() + + rq, err := client.RequestQueues().GetOrCreate(ctx, uniqueName("rq-page-opts")) + if err != nil { + t.Fatalf("get-or-create: %v", err) + } + defer func() { _ = client.RequestQueue(rq.ID).Delete(ctx) }() + queue := client.RequestQueue(rq.ID) + for i := 0; i < 5; i++ { + url := fmt.Sprintf("https://example.com/%d", i) + if _, err := queue.AddRequest(ctx, apify.RequestQueueRequest{URL: url, UniqueKey: url}, false); err != nil { + t.Fatalf("add request %d: %v", i, err) + } + } + + drain := func(it *apify.RequestQueueRequestsIterator) []string { + t.Helper() + var urls []string + for { + req, err := it.Next(ctx) + if err != nil { + t.Fatalf("iterate: %v", err) + } + if req == nil { + return urls + } + urls = append(urls, req.URL) + } + } + + // Limit caps the total across pages (3 of 5 with 2 per page), and Filter applies to each page. + pending := drain(queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ + Limit: ptr(int64(3)), Filter: []string{apify.RequestFilterPending}, + }, ptr(int64(2)))) + if len(pending) != 3 { + t.Fatalf("expected 3 pending requests, got %v", pending) + } + if locked := drain(queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ + Filter: []string{apify.RequestFilterLocked}, + }, nil)); len(locked) != 0 { + t.Fatalf("expected no locked requests, got %v", locked) + } + + // Cursor starts the iteration after the first page. + raw, err := queue.ListRequests(ctx, apify.ListRequestsOptions{Limit: ptr(int64(2))}) + if err != nil { + t.Fatalf("list requests: %v", err) + } + var first struct { + NextCursor string `json:"nextCursor"` + } + if err := json.Unmarshal(raw, &first); err != nil || first.NextCursor == "" { + t.Fatalf("expected a next cursor in %s (%v)", raw, err) + } + if rest := drain(queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{Cursor: &first.NextCursor}, nil)); len(rest) != 3 { + t.Fatalf("expected 3 requests after the cursor, got %v", rest) + } + + if _, err := queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{Filter: []string{"bogus"}}, nil).Next(ctx); err == nil { + t.Fatal("expected an invalid filter to be rejected") + } +} + // TestRequestQueueBatchAddRequests exercises the chunked batch-add path with more than the // 25-per-call API limit, verifying all requests are accepted across chunks. func TestRequestQueueBatchAddRequests(t *testing.T) { From 023f53f09bc3907a7aec4249afb2ff2ca9097d3d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 14:23:55 +0000 Subject: [PATCH 10/12] fix: scan only new log data for message timestamps Address the re-review of the full-review fixes: a large message is no longer scanned again on every read, plus test, docs and iterator cleanups. --- docs/runs.md | 24 ++++++++++++------------ docs/storages.md | 9 ++++++++- request_queue.go | 2 +- streamed_log.go | 33 +++++++++++++++++++++++++-------- streamed_log_test.go | 43 ++++++++++++++++++++++++++++++++++++++----- 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/docs/runs.md b/docs/runs.md index 20fcdd7..16e63cb 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -160,18 +160,18 @@ if err := streamed.Stop(ctx); err != nil { | `Disabled` | `bool` | Turns the redirection off in `ActorStartOptions.Log`. | The log is split into messages at the timestamps the platform puts in front of each of them, so a -message that spans several lines is written as one, even when its lines arrive in separate -reads; the last message is therefore written when the next one starts or the log ends. `Start` begins the redirection in the -background, and it also ends when the `ctx` it got is done. `Stop` gives the messages in flight up -to one second, then aborts the stream; if the stream delivered nothing yet, it reads the whole log -in one request, so a short run that finished before the stream connected is not lost. The `ctx` -given to `Stop` bounds that wait and that read: when it is done first, `Stop` returns its error -and the rest of the log is not redirected, unless the `ctx` given to `Start` ended first, which -ends the redirection on purpose and is not an error. When both end at about the same time, -either result is possible. A second `Stop` while one is running returns an error. A log that has -no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, until its -first bytes arrive. `Stop` returns the error that ended the redirection early, if any; `Call` -reports it on the redirection's output and goes on. +message that spans several lines is written as one, even when its lines arrive in separate reads; +the last message is therefore written when the next one starts or the log ends. `Start` begins the +redirection in the background, and it also ends when the `ctx` it got is done. `Stop` gives the +messages in flight up to one second, then aborts the stream; if the stream delivered nothing yet, +it reads the whole log in one request, so a short run that finished before the stream connected is +not lost. The `ctx` given to `Stop` bounds that wait and that read: when it is done first, `Stop` +returns its error and the rest of the log is not redirected, unless the `ctx` given to `Start` +ended first, which ends the redirection on purpose and is not an error. When both end at about the +same time, either result is possible. A second `Stop` while one is running returns an error. A log +that has no bytes yet (the run has not logged anything) is reopened, pausing from 500ms up to 5s, +until its first bytes arrive. `Stop` returns the error that ended the redirection early, if any; +`Call` reports it on the redirection's output and goes on. ## Live iteration of dataset items diff --git a/docs/storages.md b/docs/storages.md index 0366b38..4d2c671 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -387,7 +387,14 @@ pending := queue.PaginateRequestsWithOptions(apify.ListRequestsOptions{ Limit: apify.Ptr(int64(50)), Filter: []string{apify.RequestFilterPending}, }, apify.Ptr(int64(25))) -for req, err := pending.Next(ctx); req != nil && err == nil; req, err = pending.Next(ctx) { +for { + req, err := pending.Next(ctx) + if err != nil { + log.Fatal(err) + } + if req == nil { + break + } fmt.Println(req.URL) } ``` diff --git a/request_queue.go b/request_queue.go index c3f5d6d..2ff316e 100644 --- a/request_queue.go +++ b/request_queue.go @@ -422,7 +422,7 @@ type requestsPage struct { // Next returns the next request, or (nil, nil) when the iterator is exhausted. func (it *RequestQueueRequestsIterator) Next(ctx context.Context) (*RequestQueueRequest, error) { for it.pos >= len(it.buffer) { - if it.exhausted || (it.started && it.nextCursor == "") { + if it.exhausted { return nil, nil } if err := it.fetchPage(ctx); err != nil { diff --git a/streamed_log.go b/streamed_log.go index 5f3e0cb..dea0fb7 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -26,6 +26,8 @@ const ( stopGrace = time.Second // logRedirectPrefixSeparator ends the default message prefix, which names the Actor and the run. logRedirectPrefixSeparator = " -> " + // logMarkerOverlapBytes is the length of a timestamp together with the end of line before it. + logMarkerOverlapBytes = 25 // logReadChunkBytes is the size of the buffer the log stream is read through. logReadChunkBytes = 32 * 1024 ) @@ -35,6 +37,10 @@ const ( // trace, say) is redirected as one. var logMessageMarker = regexp.MustCompile(`(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)`) +// logMessageStart matches the timestamp that starts a message in the middle of the buffer, with +// the end of line before it. +var logMessageStart = regexp.MustCompile(`\n\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z`) + // StreamedLogOptions configures [RunClient.NewStreamedLog] and, through // [ActorStartOptions.Log], the log redirection of [ActorClient.Call]. type StreamedLogOptions struct { @@ -76,6 +82,7 @@ type StreamedLog struct { wake chan struct{} done chan error buffer bytes.Buffer + scanned int // how much of the buffer was searched for message timestamps } // NewStreamedLog creates the redirection of this run's log, which does nothing until @@ -140,6 +147,7 @@ func (s *StreamedLog) Start(ctx context.Context) error { s.wake = make(chan struct{}) s.done = make(chan error, 1) s.buffer.Reset() + s.scanned = 0 go func() { s.done <- s.streamLog(ctx, streamCtx) }() @@ -325,18 +333,26 @@ func (s *StreamedLog) consumeStream(stream io.Reader) (bool, error) { // consume adds data to the buffer and redirects every message that is known to be complete: the // ones followed by the timestamp of the next message. The last message stays in the buffer, since // its continuation lines (a stack trace, say) may arrive in a later read; [StreamedLog.flush] -// writes it when the log ends. +// writes it when the log ends. Only the new data is searched for timestamps, so a large message +// is not scanned again on every read. func (s *StreamedLog) consume(data []byte) { + from := max(s.scanned-logMarkerOverlapBytes, 0) // a timestamp may be split across reads s.buffer.Write(data) - text := s.buffer.String() - markers := logMessageMarker.FindAllStringIndex(text, -1) - if len(markers) < 2 { + buffered := s.buffer.Bytes() + s.scanned = len(buffered) + found := logMessageStart.FindAllIndex(buffered[from:], -1) + if len(found) == 0 { return } - last := markers[len(markers)-1][0] - s.redirectMessages(text[:last]) - s.buffer.Reset() - s.buffer.WriteString(text[last:]) + // The newest timestamp starts the message that stays; everything before it is complete. + last := from + found[len(found)-1][0] + if last == 0 { + return + } + complete := string(buffered[:last]) + s.buffer.Next(last) + s.scanned = s.buffer.Len() + s.redirectMessages(complete) } // flush redirects whatever is left in the buffer when the log ends: the last message, or a last @@ -344,6 +360,7 @@ func (s *StreamedLog) consume(data []byte) { func (s *StreamedLog) flush() { text := s.buffer.String() s.buffer.Reset() + s.scanned = 0 switch { case strings.TrimSpace(text) == "": case logMessageMarker.MatchString(text): diff --git a/streamed_log_test.go b/streamed_log_test.go index 662771d..10aa2e3 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -445,12 +445,13 @@ func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { } } -// Without a Stop context to cut it off, a live stream ends after the grace period: that is how a -// redirection ends normally, and it is not an error. +// A message whose continuation lines arrive in a later read is redirected whole, and so is one +// whose timestamp is split across reads. func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { backend := &logBackend{streamBodies: []string{""}, streamChunks: []string{ "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n", - " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n", + " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:0", + "0:02.000Z split\n", }} out := &syncBuffer{} ctx := context.Background() @@ -461,11 +462,11 @@ func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { if err := streamed.Start(ctx); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "next") }) + waitFor(t, func() bool { return strings.Contains(out.String(), "split") }) if err := streamed.Stop(ctx); err != nil { t.Fatal(err) } - want := "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n" + want := "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:00:02.000Z split\n" if got := out.String(); got != want { t.Fatalf("redirected log:\n%q\nwant:\n%q", got, want) } @@ -498,6 +499,8 @@ func TestStreamedLogStopKeepsReadingTheMessagesInFlight(t *testing.T) { } } +// Without a Stop context to cut it off, a live stream ends after the grace period: that is how a +// redirection ends normally, and it is not an error. func TestStreamedLogStopOfALiveStreamAfterTheGracePeriodIsNotAnError(t *testing.T) { backend := &logBackend{streamBodies: []string{sampleLog}, holdStream: true} out := &syncBuffer{} @@ -698,3 +701,33 @@ func waitFor(t *testing.T, condition func() bool) { time.Sleep(5 * time.Millisecond) } } + +// One large message must not be scanned again on every read: Stop gives the log only a second. +// The bound is loose to allow the race detector (0.2s without it); scanning again on every read +// takes about 9s without it. +func TestStreamedLogRedirectsALargeMessageQuickly(t *testing.T) { + const lines = 4 << 20 / 8 + var log strings.Builder + log.WriteString("2026-10-30T10:00:00.000Z big\n") + for i := 0; i < lines; i++ { + log.WriteString("line 1\n") + } + log.WriteString("2026-10-30T10:00:01.000Z next\n") + out := &syncBuffer{} + streamed := &StreamedLog{output: out, fromStart: true} + body := []byte(log.String()) + + started := time.Now() + for len(body) > 0 { + n := min(logReadChunkBytes, len(body)) + streamed.consume(body[:n]) + body = body[n:] + } + streamed.flush() + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Fatalf("redirecting 4 MiB took %v", elapsed) + } + if got := strings.Count(out.String(), "\n"); got != lines+2 { + t.Fatalf("redirected %d lines, want %d", got, lines+2) + } +} From 1f51a19fc78f57f3556d8d0054dbc46c9b73d404 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 14:35:11 +0000 Subject: [PATCH 11/12] test: guard the timestamp overlap of the log redirection Hold the stream open in the split-timestamp test, derive the size of the large-message test from its line, and build both timestamp regexes from one pattern. --- streamed_log.go | 6 ++++-- streamed_log_test.go | 14 ++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/streamed_log.go b/streamed_log.go index dea0fb7..9730bb8 100644 --- a/streamed_log.go +++ b/streamed_log.go @@ -26,6 +26,8 @@ const ( stopGrace = time.Second // logRedirectPrefixSeparator ends the default message prefix, which names the Actor and the run. logRedirectPrefixSeparator = " -> " + // logTimestampPattern matches the ISO timestamp, 24 characters long, that starts a message. + logTimestampPattern = `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z` // logMarkerOverlapBytes is the length of a timestamp together with the end of line before it. logMarkerOverlapBytes = 25 // logReadChunkBytes is the size of the buffer the log stream is read through. @@ -35,11 +37,11 @@ const ( // logMessageMarker matches the ISO timestamp that starts every message of an Actor run log. The // log is split into messages at these markers, so a message that spans several lines (a stack // trace, say) is redirected as one. -var logMessageMarker = regexp.MustCompile(`(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)`) +var logMessageMarker = regexp.MustCompile(`(?:\n|^)(` + logTimestampPattern + `)`) // logMessageStart matches the timestamp that starts a message in the middle of the buffer, with // the end of line before it. -var logMessageStart = regexp.MustCompile(`\n\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z`) +var logMessageStart = regexp.MustCompile(`\n` + logTimestampPattern) // StreamedLogOptions configures [RunClient.NewStreamedLog] and, through // [ActorStartOptions.Log], the log redirection of [ActorClient.Call]. diff --git a/streamed_log_test.go b/streamed_log_test.go index 10aa2e3..73ecb0d 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -448,7 +448,7 @@ func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { // A message whose continuation lines arrive in a later read is redirected whole, and so is one // whose timestamp is split across reads. func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { - backend := &logBackend{streamBodies: []string{""}, streamChunks: []string{ + backend := &logBackend{streamBodies: []string{""}, holdStream: true, streamChunks: []string{ "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n", " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:0", "0:02.000Z split\n", @@ -462,7 +462,9 @@ func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { if err := streamed.Start(ctx); err != nil { t.Fatal(err) } - waitFor(t, func() bool { return strings.Contains(out.String(), "split") }) + // The stream stays open: the message before the split timestamp is redirected only if the + // timestamp is recognized across the two reads, not when the log ends. + waitFor(t, func() bool { return strings.Contains(out.String(), "next") }) if err := streamed.Stop(ctx); err != nil { t.Fatal(err) } @@ -703,14 +705,14 @@ func waitFor(t *testing.T, condition func() bool) { } // One large message must not be scanned again on every read: Stop gives the log only a second. -// The bound is loose to allow the race detector (0.2s without it); scanning again on every read -// takes about 9s without it. +// The bound is loose, to allow for the race detector. func TestStreamedLogRedirectsALargeMessageQuickly(t *testing.T) { - const lines = 4 << 20 / 8 + const line = "line 1\n" + const lines = 4 << 20 / len(line) var log strings.Builder log.WriteString("2026-10-30T10:00:00.000Z big\n") for i := 0; i < lines; i++ { - log.WriteString("line 1\n") + log.WriteString(line) } log.WriteString("2026-10-30T10:00:01.000Z next\n") out := &syncBuffer{} From 0b8e7c0e53767193cf819fa7eed7f4878fd7a0b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 14:41:46 +0000 Subject: [PATCH 12/12] test: split the timestamp at its last character --- streamed_log_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streamed_log_test.go b/streamed_log_test.go index 73ecb0d..851822f 100644 --- a/streamed_log_test.go +++ b/streamed_log_test.go @@ -450,8 +450,8 @@ func TestStreamedLogStopReportsALogCutOffByItsContext(t *testing.T) { func TestStreamedLogKeepsAMessageWhoseLinesArriveInSeparateReads(t *testing.T) { backend := &logBackend{streamBodies: []string{""}, holdStream: true, streamChunks: []string{ "2026-10-30T10:00:00.000Z Error: boom\n at first (a.js:1)\n", - " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:0", - "0:02.000Z split\n", + " at second (b.js:2)\n2026-10-30T10:00:01.000Z next\n2026-10-30T10:00:02.000", + "Z split\n", }} out := &syncBuffer{} ctx := context.Background()