From 3d5d2d406c72409e46e146acf8d80fd5f82ef30c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 15:06:36 +0000 Subject: [PATCH 01/11] feat!: port apify-client-js v3 changes and sync with Apify OpenAPI spec v2-2026-10-29T100357Z Adds timeout tiers with per-client overrides, configurable compressors and default headers, ambiguous-404 handling, path-segment validation, push_items shape check and the new spec query parameters. --- CHANGELOG.md | 42 ++ Cargo.toml | 5 +- README.md | 81 +++- docs/README.md | 54 ++- docs/actors.md | 2 +- docs/builds.md | 2 +- docs/misc.md | 42 +- docs/runs.md | 4 +- docs/schedules.md | 2 +- docs/storages.md | 4 +- docs/tasks.md | 2 +- examples/log_redirection.rs | 2 +- examples/raw_log.rs | 12 +- src/client.rs | 91 +++- src/clients/actor.rs | 48 ++- src/clients/actor_collection.rs | 7 +- src/clients/actor_env_var.rs | 9 +- src/clients/actor_env_var_collection.rs | 7 +- src/clients/actor_version.rs | 9 +- src/clients/actor_version_collection.rs | 7 +- src/clients/base.rs | 370 +++++++++------- src/clients/build.rs | 40 +- src/clients/build_collection.rs | 5 +- src/clients/dataset.rs | 168 +++++--- src/clients/dataset_collection.rs | 7 +- src/clients/key_value_store.rs | 68 +-- src/clients/key_value_store_collection.rs | 7 +- src/clients/log.rs | 52 ++- src/clients/request_queue.rs | 346 ++++++++------- src/clients/request_queue_collection.rs | 7 +- src/clients/run.rs | 74 +++- src/clients/run_collection.rs | 14 +- src/clients/schedule.rs | 27 +- src/clients/schedule_collection.rs | 7 +- src/clients/store_collection.rs | 12 +- src/clients/task.rs | 46 +- src/clients/task_collection.rs | 7 +- src/clients/user.rs | 88 ++-- src/clients/webhook.rs | 19 +- src/clients/webhook_collection.rs | 7 +- src/clients/webhook_dispatch.rs | 5 +- src/clients/webhook_dispatch_collection.rs | 5 +- src/common.rs | 46 +- src/http_client.rs | 391 ++++++----------- src/http_compressors.rs | 399 ++++++++++++++++++ src/lib.rs | 18 +- src/models.rs | 81 +++- src/timeouts.rs | 149 +++++++ src/version.rs | 2 +- tests/actor_run.rs | 80 +++- tests/common/mock_backend.rs | 120 ++++++ tests/common/mod.rs | 47 ++- tests/store.rs | 18 + tests/task.rs | 2 +- tests/unit_http.rs | 239 ++++++----- tests/unit_resources.rs | 465 +++++++++++++++++++++ tests/unit_timeouts.rs | 424 +++++++++++++++++++ tests/user.rs | 46 ++ 58 files changed, 3313 insertions(+), 1027 deletions(-) create mode 100644 src/http_compressors.rs create mode 100644 src/timeouts.rs create mode 100644 tests/common/mock_backend.rs create mode 100644 tests/unit_resources.rs create mode 100644 tests/unit_timeouts.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ab50197..ba9e39f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,48 @@ All notable changes to the Rust Apify API client are documented here. The format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.15.0] - 2026-10-29 + +Ports the `apify-client-js` v3 changes. + +### Added +- Timeout tiers: `Timeout` (`Short`, `Medium`, `Long`, `NoTimeout`, `Custom`), the builder options + `timeout_short`, `timeout_medium`, `timeout_long` and `timeout_max`, and `with_timeout` on every + resource client. +- `HttpCompressor` with `BrotliHttpCompressor` and `GzipHttpCompressor` (configurable quality), set + with `ApifyClientBuilder::http_compressor`. +- `ApifyClientBuilder::default_header`. +- `RunListOptions::deployment`, `StoreListOptions::include_input_schema` and + `LogOptions::download`. +- Offline tests for the timeout tiers, the 404 handling, id validation and batch add. + +### Changed +- Synced with Apify OpenAPI spec `v2-2026-10-29T100357Z` (`API_SPEC_VERSION`). +- **Breaking:** every method has a timeout tier (5s, 30s or 360s by default) instead of one 360s + timeout; `timeout` is now an alias of `timeout_max`. `wait_for_finish` has no timeout, and a + `waitForFinish` request gets the server-side wait added to its tier. +- **Breaking:** `HttpRequest::timeout` may be `timeouts::NO_TIMEOUT`, which a custom `HttpBackend` + must not abort. +- **Breaking:** `RetryConfig` no longer has a `timeout` field. +- **Breaking:** any `404` resolves to `None`, not only `record-not-found`, but only for a client that + names its resource by id. `run.dataset()`, `run.log()` and the other chained clients return it as + an error. +- **Breaking:** `get_statistics`, `get_input`, `schedule.get_log` and `get_openapi_definition` + return the `404` as an error and no longer wrap the result in `Option`. +- **Breaking:** `LogClient::stream` and `stream_with_options` return `Option`, `None` for a missing log. +- **Breaking:** `ActorStartOptions::timeout_secs` and `RunResurrectOptions::timeout_secs` are now + `run_timeout_secs`. +- **Breaking:** `ListRequestsOptions::exclusive_start_id` is removed; use `cursor`. +- **Breaking:** `LogOptions`, `RunListOptions` and `StoreListOptions` gained a public field. +- **Breaking:** an id, record key or request id that is empty or a dot segment fails with + `InvalidArgument`; ids are percent-encoded and every `/` of an id becomes `~`. +- **Breaking:** `dataset.push_items` rejects anything but an object or an array of objects. +- **Breaking:** `ActorRun::container_url` and `Webhook::request_url` are normalized URLs. +- `UserClient::limits`, `monthly_usage` and `update_limits` work for any user, not only `me`. +- The default gzip quality is 9. +- `batch_add_requests` serializes each request once and rejects an oversized one before sending. +- The `list` integration tests wait out replica lag for 87 seconds. + ## [0.14.1] - 2026-10-26 ### Changed diff --git a/Cargo.toml b/Cargo.toml index 551e8c5..fd97bd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apify-client" -version = "0.14.1" +version = "0.15.0" authors = ["Apify Technologies "] description = "An official, but experimental, AI-generated and AI-maintained Rust client for the Apify API (https://apify.com)." license = "Apache-2.0" @@ -21,6 +21,7 @@ thiserror = "2.0" async-trait = "0.1" futures-util = "0.3" tokio = { version = "1", features = ["time"] } +url = "2" base64 = "0.22" hmac = "0.12" sha2 = "0.10" @@ -33,7 +34,7 @@ rustls-webpki = { version = "0.103.16", optional = true } webpki-roots = { version = "1.0.10", optional = true } [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread", "time", "test-util"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time", "test-util", "net", "io-util"] } uuid = { version = "1", features = ["v4"] } [features] diff --git a/README.md b/README.md index 958808c..a922144 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ It provides a resource-oriented, async interface that mirrors the official ```toml [dependencies] -apify-client = "0.14" +apify-client = "0.15" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } serde_json = "1" # for the `serde_json::Value` responses used in the Quick start ``` @@ -50,7 +50,7 @@ project needs `serde_json`. Two more dependencies are needed only for specific f By default the client uses the system TLS (`native-tls`). To use rustls instead: ```toml -apify-client = { version = "0.14", default-features = false, features = ["rustls"] } +apify-client = { version = "0.15", default-features = false, features = ["rustls"] } ``` The `rustls` feature requires `rustls-webpki` 0.103.16 or newer and `webpki-roots` 1.0.10 or @@ -104,19 +104,75 @@ let client = ApifyClient::builder() .public_base_url("https://api.apify.com") // origin for public/shareable URLs; defaults to `base_url` .max_retries(8) // default: 8 .min_delay_between_retries(Duration::from_millis(500)) // default: 500ms - .timeout(Duration::from_secs(360)) // default: 360s + .timeout_short(Duration::from_secs(5)) // default: 5s, simple metadata reads and writes + .timeout_medium(Duration::from_secs(30)) // default: 30s, listing, batch and trigger calls + .timeout_long(Duration::from_secs(360)) // default: 360s, downloads, uploads and streaming + .timeout_max(Duration::from_secs(360)) // default: 360s, cap on a single attempt .user_agent_suffix("my-app/1.0") + .default_header("X-My-Header", "value") // sent with every request .request_compression(RequestCompression::Brotli) // default: Brotli; or RequestCompression::Gzip .build(); ``` +### Timeouts + +Every method picks the timeout tier that fits its request: `Timeout::Short` for simple metadata +reads and writes, `Timeout::Medium` for listing, batch and trigger calls, and `Timeout::Long` for +downloads, uploads and streaming. Polling that waits for a job to finish (`wait_for_finish`) has no +client-side timeout. The duration of each tier is set on the builder, and `timeout_max` caps the +timeout of a single attempt: a tier or an explicit timeout above it is lowered to it. The timeout +applies to each attempt separately and doubles with every retry, up to the cap. + +A resource client's `with_timeout` replaces the tier of every request that client sends, with a +tier, an exact duration, or `Timeout::NoTimeout` to let a request run for as long as it takes. +With `NoTimeout` the client never aborts the request, so a connection that stalls is not retried +either. Clients obtained from that client, such as `run.dataset()`, do not inherit the timeout. + +```rust,no_run +use std::time::Duration; +use apify_client::{ApifyClient, Timeout}; + +# async fn run() -> Result<(), Box> { +let client = ApifyClient::new("my-api-token"); + +// A dataset export that legitimately takes longer than the `Long` tier. +let items = client + .dataset("my-dataset-id") + .with_timeout(Timeout::Custom(Duration::from_secs(300))) + .list_items::(Default::default()) + .await?; +# let _ = items; +# Ok(()) +# } +``` + ### Request compression Request bodies of at least 1024 bytes are compressed once (before retries) and sent with the matching `Content-Encoding` header. The default is Brotli (`br`); select `RequestCompression::Gzip` (`gzip`) for environments or intermediaries that do not handle brotli. `RequestCompression` is re-exported at the crate root -(`use apify_client::RequestCompression;`). +(`use apify_client::RequestCompression;`). To tune the quality or use another algorithm, pass an +`HttpCompressor` to `ApifyClientBuilder::http_compressor`: the built-in +`BrotliHttpCompressor` (quality 0 to 11, default 6) and `GzipHttpCompressor` (quality 1 to 9, +default 9), or your own implementation of the trait. + +```rust,no_run +use std::sync::Arc; +use apify_client::{ApifyClient, BrotliHttpCompressor}; + +# fn run() -> Result<(), Box> { +let client = ApifyClient::builder() + .token("my-api-token") + .http_compressor(Arc::new(BrotliHttpCompressor::new(11)?)) + .build(); +# Ok(()) +# } +``` + +The compressor is skipped for bodies under 1024 bytes, for content types that already carry their +own compression (images, audio, video, archives, ...), and for a request that sets its own +`Content-Encoding`. ### User-Agent @@ -223,7 +279,7 @@ run with `cargo run --example `: - `run_and_last_run_storages` — run an Actor, then access the last run and its storages. - `iterate_store` — lazily iterate Actors in the Apify Store. - `log_redirection` — run a separate Actor and redirect its run log into your output live, with each line prefixed by the source Actor's name. -- `raw_log` — fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true) }`. +- `raw_log` — fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true), ..Default::default() }`. See [`docs/`](docs) for the full API documentation. @@ -255,8 +311,11 @@ All fallible methods return `Result`. API errors expose the error `type` (as the field `error_type`), `message`, `status_code` and request details. An Actor (or task/store) id may be a bare id, a `username/name` reference, or the equivalent -`username~name` form (the client encodes the first `/` as `~` on the wire, so the two are -interchangeable). The example below uses `"nonexistent~actor"` — a deliberately non-existent +`username~name` form (the client encodes every `/` as `~` on the wire, so the two are +interchangeable). An id is percent-encoded when it is placed in a URL path. An id that is empty or +a dot segment (`.` or `..`) cannot be placed in a path without addressing another endpoint, so +every request through a client created with it fails with `ApifyClientError::InvalidArgument`; +the same goes for a key-value store record key or a request queue request id. The example below uses `"nonexistent~actor"` — a deliberately non-existent reference — to show how a missing resource surfaces as `Ok(None)` rather than an error, while real calls elsewhere in this README use a live actor such as `"apify/hello-world"`: @@ -276,8 +335,12 @@ match client.actor("nonexistent~actor").get().await { # } ``` -`get`/`delete` operations resolve a missing resource (`404 record-not-found`) to -`Ok(None)` / a successful no-op, matching the reference clients. +`get`/`delete` operations resolve a missing resource (any `404`) to `Ok(None)` / a successful +no-op, matching the reference clients, but only on a client that names its resource by id. A +client chained from a parent without an id of its own, such as `run.dataset()` or `run.log()`, +returns the `404` as an error: the response cannot tell a missing run from a missing dataset. The +same goes for methods that address a fixed sub-path of a resource (`get_statistics`, `get_input`, +`get_log`, `get_openapi_definition`, `limits`, `monthly_usage`): they return the `404` as an error. `ApiError` also exposes `is_not_found()`, `is_rate_limited()`, `is_server_error()` and similar `is_*` predicates classifying it by HTTP status — the idiomatic equivalent of the reference diff --git a/docs/README.md b/docs/README.md index 864c218..e9bbf97 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,7 +31,7 @@ Add the crate and an async runtime: ```toml [dependencies] -apify-client = "0.14" +apify-client = "0.15" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } ``` @@ -140,9 +140,14 @@ Builder options: | `public_base_url(u)` | = `base_url` | Origin used when building public/shareable URLs. | | `max_retries(n)` | `8` | Maximum retries for `429`/`5xx`/network errors. | | `min_delay_between_retries(d)` | `500ms` | Base backoff delay (doubled each retry, with jitter). | -| `timeout(d)` | `360s` | Overall per-request timeout budget. | +| `timeout_short(d)` | `5s` | Duration of the `Timeout::Short` tier (simple metadata reads and writes). | +| `timeout_medium(d)` | `30s` | Duration of the `Timeout::Medium` tier (listing, batch and trigger calls). | +| `timeout_long(d)` | `360s` | Duration of the `Timeout::Long` tier (downloads, uploads and streaming). | +| `timeout_max(d)` | `360s` | Cap on the timeout of a single attempt. `timeout(d)` is the same option under its original name. | | `user_agent_suffix(s)` | none | Extra text appended to the `User-Agent` header. | +| `default_header(name, value)` | none | Header sent with every request. It wins over the built-in `User-Agent` and `Authorization` headers; a header a request sets itself wins over it. Names are compared case-insensitively. | | `request_compression(c)` | `Brotli` | Encoding for large request bodies: `RequestCompression::Brotli` (`br`) or `RequestCompression::Gzip` (`gzip`). | +| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11), `GzipHttpCompressor::new(quality)` (1 to 9), or your own implementation. | | `http_backend(b)` | reqwest | Replaceable HTTP transport (`apify_client::http_client::{HttpBackend, ReqwestBackend}`). | Request bodies of at least 1024 bytes are compressed once (before retries) with the selected @@ -167,6 +172,40 @@ reach those values the client maps Rust's native `std::env::consts::OS` spelling (`macos` → `darwin`, `windows` → `win32`, `solaris`/`illumos` → `sunos`); all other tokens (`linux`, `android`, `freebsd`, …) are already identical and pass through unchanged. +### Timeouts + +Every method uses a timeout tier: `Timeout::Short` for simple metadata reads and writes, +`Timeout::Medium` for listing, batch and trigger calls, and `Timeout::Long` for downloads, uploads +and streaming. `wait_for_finish` polls without a client-side timeout. A tier or an explicit +timeout above `timeout_max` is lowered to it, and the timeout of each attempt doubles with every +retry, up to the same cap. + +Every resource client has a `with_timeout(Timeout)` method that replaces the tier of all the +requests it sends with a tier, `Timeout::Custom(duration)`, or `Timeout::NoTimeout` (the client +never aborts the request, so a stalled connection is not retried either). Clients obtained from it, +such as `run.dataset()`, do not inherit the timeout. + +```rust,no_run +use std::time::Duration; +use apify_client::{ApifyClient, DownloadItemsFormat, Timeout}; + +# async fn run() -> Result<(), Box> { +let client = ApifyClient::builder() + .token("my-api-token") + .timeout_short(Duration::from_secs(10)) + .build(); + +// This export may take longer than the default `Long` tier of 6 minutes. +let csv = client + .dataset("my-dataset-id") + .with_timeout(Timeout::NoTimeout) + .download_items(DownloadItemsFormat::Csv, Default::default()) + .await?; +# let _ = csv; +# Ok(()) +# } +``` + ### Convenience methods Beyond the resource accessors, `ApifyClient` exposes one convenience method: @@ -251,7 +290,14 @@ Every fallible method returns `Result`. The variants are: - `Serde(..)` — (de)serialization failure. - `InvalidResponse(..)` / `InvalidArgument(..)` — unexpected response or bad argument. -`get`/`delete` map a missing resource to `Ok(None)` / a no-op. +`get`/`delete` map a missing resource (any `404`) to `Ok(None)` / a no-op, on a client that names +its resource by id. A client chained from a parent, such as `run.dataset()` or `run.log()`, and the +methods that address a fixed sub-path of a resource (`get_statistics`, `get_input`, `get_log`, +`get_openapi_definition`) return the `404` as an error, because the response cannot tell which +resource is missing. + +An id that is empty or a dot segment (`.` or `..`) cannot be placed in a URL path, so every request +through a client created with it fails with `InvalidArgument`. To inspect the API-level details of an error without matching every variant, use `ApifyClientError::as_api_error`, which returns `Some(&ApiError)` for the `Api` variant and @@ -283,4 +329,4 @@ Each example in [`../examples`](../examples) is runnable with | `run_and_last_run_storages` | Run an Actor, then read the last run's storages. | | `iterate_store` | Lazily iterate Store Actors. | | `log_redirection` | Redirect a separate Actor's run log into your output live, prefixing each line with the source Actor's name. | -| `raw_log` | Fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true) }`. | +| `raw_log` | Fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true), ..Default::default() }`. | diff --git a/docs/actors.md b/docs/actors.md index 5b382d3..8e5578a 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -43,7 +43,7 @@ All fields are optional. Used by both `start` and `call` here, and by the identi |---|---|---| | `build` | `Option` | Tag or number of the build to run (e.g. `latest`, `0.1.2`). | | `memory_mbytes` | `Option` | Memory in megabytes allocated for the run. | -| `timeout_secs` | `Option` | Timeout for the run in seconds (`0` means no timeout). | +| `run_timeout_secs` | `Option` | Timeout of the run itself in seconds (`0` means no timeout). The timeout of the API request is set with the client's `with_timeout`. | | `wait_for_finish` | `Option` | Maximum seconds to wait server-side for the run to finish (max 60). | | `max_items` | `Option` | Maximum number of dataset items to charge (pay-per-result Actors). | | `max_total_charge_usd` | `Option` | Maximum total cost of the run in USD, across all pricing models. | diff --git a/docs/builds.md b/docs/builds.md index 229eb09..a4bd025 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -18,7 +18,7 @@ collections are available via `actor.builds()`. | `abort()` | — | `Build` | Aborts the build. | | `delete()` | — | `()` | Deletes the build. | | `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. A build ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | -| `get_openapi_definition()` | — | `Option` | Fetches the OpenAPI definition generated for the build (raw JSON, endpoint `.../openapi.json`). | +| `get_openapi_definition()` | — | `serde_json::Value` | Fetches the OpenAPI definition generated for the build (raw JSON, endpoint `.../openapi.json`). A `404` is an error. | | `log()` | — | `LogClient` | Access the build's log. | The returned `Build` model's fields (`id`, `status`, `build_number`, …) are documented in diff --git a/docs/misc.md b/docs/misc.md index 7d71076..69ca467 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -71,15 +71,14 @@ and falls back to `name` (e.g. `actor.title.or(actor.name)`). | Method | Arguments | Returns | Description | |---|---|---|---| | `get()` | — | `Option` | Account details (private for `me`, public otherwise). | -| `monthly_usage()` | — | `Value` | Current account's monthly usage for the current billing cycle (`me` only). | -| `monthly_usage_for_date(date)` | `Option<&str>` | `Value` | Monthly usage for the billing cycle containing the `YYYY-MM-DD` `date`; `None` == current month (`me` only). | -| `limits()` | — | `Value` | Current account's limits (`me` only). | -| `update_limits(limits)` | `&impl Serialize` | `()` | Updates the account's limits (`me` only). | +| `monthly_usage()` | — | `Value` | The user's monthly usage for the current billing cycle. | +| `monthly_usage_for_date(date)` | `Option<&str>` | `Value` | Monthly usage for the billing cycle containing the `YYYY-MM-DD` `date`; `None` == current month. | +| `limits()` | — | `Value` | The user's account and usage limits. | +| `update_limits(limits)` | `&impl Serialize` | `()` | Updates the user's limits. | -The methods marked **(`me` only)** operate on the authenticated account and are only valid on the -`client.me()` client. Calling any of them on a specific-user client (`client.user(id)`) returns -`Err(ApifyClientError::InvalidArgument(..))` without making a network request; `get()` is the only -method that works for both `me` and other users. +`monthly_usage*`, `limits` and `update_limits` work for `client.me()` and for `client.user(id)` +alike. For a user other than the current one, the API only allows your own account and the +organization your API token belongs to. They return a `404` as an error. `get()` returns a `User` (from `apify_client::models`). Its fields — including the `user.id` and `user.username` the [`get_account`](../examples/get_account.rs) example reads: @@ -143,12 +142,19 @@ Also reachable via `run.log()` and `build.log()`. | `get_with_options(options)` | `LogOptions` | `Option` | As `get()`, with options (e.g. `raw`). | | `get_bytes()` | — | `Option>` | The entire log as raw bytes (e.g. if it is not valid UTF-8). | | `get_bytes_with_options(options)` | `LogOptions` | `Option>` | As `get_bytes()`, with options (e.g. `raw`). | -| `stream()` | — | `Result>>>` (async — `.await` it) | Streams log chunks live (log redirection). | -| `stream_with_options(options)` | `LogOptions` | `Result>>>` (async — `.await` it) | As `stream()`, with options (e.g. `raw`). | +| `stream()` | — | `Result>>>>` (async — `.await` it) | Streams log chunks live (log redirection); `None` if the log does not exist. | +| `stream_with_options(options)` | `LogOptions` | `Result>>>>` (async — `.await` it) | As `stream()`, with options (e.g. `raw`). | -`LogOptions` has a single field, `raw: Option`. When `Some(true)`, the API returns the -raw log content without server-side processing (e.g. without the per-line timestamps it adds by -default); leaving it `None` uses the default processed format. Fetch a run's raw log as text: +`get*` and `stream*` return `None` for a missing log only on a client that names its log by id +(`client.log(id)`). A client chained from a run or build (`run.log()`, `build.log()`) has no id of +its own, so it returns the `404` as an error: the response cannot tell a missing run from a missing +log. `run.get_streamed_log()` returns the stream directly and fails for a missing run. + +`LogOptions` has two fields, `raw` and `download`, both `Option`. When `raw` is `Some(true)`, +the API returns the raw log content without server-side processing (e.g. without the per-line +timestamps it adds by default); leaving it `None` uses the default processed format. `download` +asks a web browser to download the log as a file rather than open it in a tab. Fetch a run's raw +log as text: ```rust,no_run use apify_client::{ApifyClient, LogOptions}; @@ -157,7 +163,7 @@ use apify_client::{ApifyClient, LogOptions}; let raw_log = client .run(run_id) .log() - .get_with_options(LogOptions { raw: Some(true) }) + .get_with_options(LogOptions { raw: Some(true), ..Default::default() }) .await?; if let Some(text) = raw_log { print!("{text}"); @@ -166,11 +172,13 @@ if let Some(text) = raw_log { # } ``` -The streaming variant takes the same options — `client.run(run_id).log().stream_with_options(LogOptions { raw: Some(true) }).await?` +The streaming variant takes the same options — `client.run(run_id).log().stream_with_options(LogOptions { raw: Some(true), ..Default::default() }).await?` yields the raw log chunks (this is what log redirection uses). A gap of more than 6 minutes between chunks (the run legitimately logging nothing for that long) ends the stream with an `ApifyClientError::Timeout`; this is an inactivity timeout, not a cap on the stream's total -duration, so an actively (if slowly) logging run is never cut off. +duration, so an actively (if slowly) logging run is never cut off. Set the window with the +`Timeout::Long` tier (`ApifyClientBuilder::timeout_long`) or the log client's `with_timeout`, where +`Timeout::NoTimeout` disables it. Consuming `stream()` requires the `futures_util::StreamExt` trait (from the `futures-util` crate) in scope to call `.next()` on the returned stream. Add it to your `Cargo.toml`: @@ -187,7 +195,7 @@ use apify_client::ApifyClient; use futures_util::StreamExt; # async fn run(client: ApifyClient, run_id: &str) -> Result<(), Box> { -let mut stream = client.run(run_id).log().stream().await?; +let mut stream = client.run(run_id).get_streamed_log().await?; while let Some(chunk) = stream.next().await { let chunk = chunk?; print!("{}", String::from_utf8_lossy(&chunk)); diff --git a/docs/runs.md b/docs/runs.md index c0361e6..489a8d7 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -25,7 +25,7 @@ collections are available via `actor.runs()` and `task.runs()`. | `charge(options)` | `RunChargeOptions` | `()` | Charges a pay-per-event run (always sends an idempotency key). | | `wait_for_finish(wait_secs)` | `Option` | `ActorRun` | Polls until the run is terminal. `None` waits indefinitely; `Some(n)` bounds the wait and may return a still-running (non-terminal) run if `n` elapses first. A run ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | | `dataset()` / `key_value_store()` / `request_queue()` / `log()` | — | resource client | Access the run's default storages and log. | -| `get_streamed_log()` | — | `Result>>>` (async — `.await` it) | Convenience for `log().stream()` — streams the run's log chunks live (log redirection). | +| `get_streamed_log()` | — | `Result>>>` (async — `.await` it) | Streams the run's log chunks live (log redirection). A run that does not exist is an error. | | `get_streamed_log_with_options(options)` | `LogOptions` | `Result>>>` (async — `.await` it) | As `get_streamed_log()`, forwarding `LogOptions` (e.g. `raw`) to the log stream. | `get_streamed_log()` / `get_streamed_log_with_options()` are `async` and yield the stream inside a @@ -35,7 +35,7 @@ collections are available via `actor.runs()` and `task.runs()`. `get_streamed_log_with_options` exactly this way. See [Logs](misc.md#logs--clientlogbuild_or_run_id) for a full streaming snippet. -`RunResurrectOptions`: `build`, `memory_mbytes`, `timeout_secs`, `max_items`, `max_total_charge_usd`, `restart_on_error` (all optional). +`RunResurrectOptions`: `build`, `memory_mbytes`, `run_timeout_secs`, `max_items`, `max_total_charge_usd`, `restart_on_error` (all optional). `RunMetamorphOptions`: `build`, `content_type` (both optional; `content_type` defaults to `application/json`). diff --git a/docs/schedules.md b/docs/schedules.md index 326eda3..2734634 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -17,7 +17,7 @@ Obtained via `client.schedules()` (collection) and `client.schedule(id)` (single | `get()` | — | `Option` | Fetches the schedule. | | `update(fields)` | `&impl Serialize` | `Schedule` | Updates the schedule. | | `delete()` | — | `()` | Deletes the schedule. | -| `get_log()` | — | `Option` | Fetches the schedule's invocation log. | +| `get_log()` | — | `String` | Fetches the schedule's invocation log. A `404` is an error. | ## The `Schedule` model diff --git a/docs/storages.md b/docs/storages.md index 53999ff..44113c3 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -63,7 +63,7 @@ let scratch = client.datasets().get_or_create(None).await?; | `list_items::(options)` | `DatasetListItemsOptions` | `PaginationList` | Reads items (pagination via response headers). | | `iterate_items::(options)` | `DatasetListItemsOptions` | `ListIterator` | Lazily iterates all items across pages (auto-pagination). | | `push_items(items)` | `&impl Serialize` | `()` | Appends items (object or array). | -| `get_statistics()` | — | `Option` | Field statistics. | +| `get_statistics()` | — | `Value` | Field statistics. A `404` is an error. | | `download_items(format, options)` | `DownloadItemsFormat`, `DatasetDownloadOptions` | `Vec` | Export items as JSON/JSONL/CSV/XLSX/XML/RSS/HTML. | | `download_items_stream(format, options)` | `DownloadItemsFormat`, `DatasetDownloadOptions` | `Result>>>` (async — `.await` it) | Like `download_items`, but streams the export instead of buffering it, for constant-memory processing of a large export. | | `create_items_public_url(options, expires)` | `DatasetListItemsOptions`, `Option` | `String` | Shareable (HMAC-signed for private) items URL, served as `json`. | @@ -221,7 +221,7 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: | `list_and_lock_head(lock_secs, limit)` | `i64`, `Option` | `LockedRequestQueueHead` | Locks head requests. | | `batch_add_requests(requests, options)` | `&[RequestQueueRequest]`, `BatchAddRequestsOptions` | `BatchRequestsOperationResult` | Batch add, with automatic chunking, bounded parallelism, and retries for rate-limited requests. | | `batch_delete_requests(requests)` | `&[impl Serialize]` | `BatchRequestsOperationResult` | Batch delete (max 25 requests per call; larger inputs are rejected client-side). | -| `list_requests(options)` | `ListRequestsOptions { limit, exclusive_start_id, cursor, filter }` | `RequestQueueRequestsPage` | List requests (cursor/filter pagination). | +| `list_requests(options)` | `ListRequestsOptions { limit, cursor, filter }` | `RequestQueueRequestsPage` | List requests (cursor/filter pagination). | | `paginate_requests(page_limit)` | `Option` | `RequestQueueRequestsIterator` | Lazy request iterator. | | `prolong_request_lock(id, lock_secs, forefront)` | `&str`, `i64`, `bool` | `RequestLockInfo` | Extend a lock. | | `delete_request_lock(id, forefront)` | `&str`, `bool` | `()` | Release a lock. | diff --git a/docs/tasks.md b/docs/tasks.md index 3088742..f86cef3 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -21,7 +21,7 @@ Obtained via `client.tasks()` (collection) and `client.task(id)` (single). | `delete()` | — | `()` | Deletes the task. | | `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run. See [`ActorStartOptions`](actors.md#actorstartoptions) for the full field list. | | `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits. Same [`ActorStartOptions`](actors.md#actorstartoptions) as `start`. | -| `get_input()` / `update_input(input)` | — / `&impl Serialize` | `Option` / `Value` | The task's saved input. | +| `get_input()` / `update_input(input)` | — / `&impl Serialize` | `Value` / `Value` | The task's saved input. `get_input` returns a `404` as an error. | | `last_run(status)` | `Option<&str>` | `RunClient` | The task's last run, optionally filtered by status. See [Actor runs](runs.md) for the accepted `status` values. | | `last_run_with_options(options)` | `LastRunOptions { status, origin }` | `RunClient` | The task's last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | | `runs()` | — | `RunCollectionClient` | The task's runs. | diff --git a/examples/log_redirection.rs b/examples/log_redirection.rs index 66ebc2a..78201c5 100644 --- a/examples/log_redirection.rs +++ b/examples/log_redirection.rs @@ -36,7 +36,7 @@ async fn main() -> Result<(), Box> { // `"{name} -> "` convention the JS reference uses for redirected Actor logs). The log is // streamed as raw byte chunks, so we buffer until a newline to emit whole, prefixed lines. let prefix = format!("{source_actor} -> "); - let mut stream = client.run(&run.id).log().stream().await?; + let mut stream = client.run(&run.id).get_streamed_log().await?; let mut buf = String::new(); while let Some(chunk) = stream.next().await { let chunk = chunk?; diff --git a/examples/raw_log.rs b/examples/raw_log.rs index ebb9b53..c6d651b 100644 --- a/examples/raw_log.rs +++ b/examples/raw_log.rs @@ -1,7 +1,7 @@ //! Fetch an Actor run's log in its raw (unprocessed) form. //! //! By default the API adds a per-line timestamp prefix to log lines. Passing -//! `LogOptions { raw: Some(true) }` requests the unprocessed log instead — the same raw form +//! `LogOptions { raw: Some(true), ..Default::default() }` requests the unprocessed log instead — the same raw form //! the JS reference's log redirection consumes. This example exercises both the buffered //! (`get_with_options`) and the streaming (`stream_with_options`) raw-log paths end-to-end so //! the `Test examples` CI step covers the new `raw` parameter against the live API. @@ -27,7 +27,10 @@ async fn main() -> Result<(), Box> { let raw_log = client .run(&run.id) .log() - .get_with_options(LogOptions { raw: Some(true) }) + .get_with_options(LogOptions { + raw: Some(true), + ..Default::default() + }) .await?; match raw_log { Some(log) => println!("Fetched raw log ({} bytes)", log.len()), @@ -37,7 +40,10 @@ async fn main() -> Result<(), Box> { // Streaming raw log: stream the same unprocessed log chunk by chunk. let mut stream = client .run(&run.id) - .get_streamed_log_with_options(LogOptions { raw: Some(true) }) + .get_streamed_log_with_options(LogOptions { + raw: Some(true), + ..Default::default() + }) .await?; let mut streamed_bytes = 0usize; while let Some(chunk) = stream.next().await { diff --git a/src/client.rs b/src/client.rs index 87fe490..cd55c9b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -27,9 +27,9 @@ use crate::clients::webhook_collection::WebhookCollectionClient; use crate::clients::webhook_dispatch::WebhookDispatchClient; use crate::clients::webhook_dispatch_collection::WebhookDispatchCollectionClient; use crate::common::build_user_agent; -use crate::http_client::{ - HttpBackend, HttpClient, RequestCompression, ReqwestBackend, RetryConfig, -}; +use crate::http_client::{HttpBackend, HttpClient, HttpClientConfig, ReqwestBackend, RetryConfig}; +use crate::http_compressors::{HttpCompressor, RequestCompression}; +use crate::timeouts::TimeoutTiers; /// Default base URL of the Apify API (without the `/v2` suffix). const DEFAULT_BASE_URL: &str = "https://api.apify.com"; @@ -40,8 +40,6 @@ const API_VERSION_PATH: &str = "/v2"; const DEFAULT_MAX_RETRIES: u32 = 8; /// Default minimum delay between retries. const DEFAULT_MIN_DELAY_BETWEEN_RETRIES: Duration = Duration::from_millis(500); -/// Default overall per-request timeout (6 minutes), matching the reference clients. -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(360); /// Placeholder used to address the current user (`/users/me`). pub(crate) const ME_USER_PLACEHOLDER: &str = "me"; @@ -62,10 +60,11 @@ pub struct ApifyClientBuilder { public_base_url: Option, max_retries: u32, min_delay_between_retries: Duration, - timeout: Duration, + tiers: TimeoutTiers, user_agent_suffix: Option, http_backend: Option>, - request_compression: RequestCompression, + compressor: Arc, + default_headers: Vec<(String, String)>, } impl Default for ApifyClientBuilder { @@ -76,10 +75,11 @@ impl Default for ApifyClientBuilder { public_base_url: None, max_retries: DEFAULT_MAX_RETRIES, min_delay_between_retries: DEFAULT_MIN_DELAY_BETWEEN_RETRIES, - timeout: DEFAULT_TIMEOUT, + tiers: TimeoutTiers::default(), user_agent_suffix: None, http_backend: None, - request_compression: RequestCompression::default(), + compressor: RequestCompression::default().into_compressor(), + default_headers: Vec::new(), } } } @@ -122,27 +122,75 @@ impl ApifyClientBuilder { self } - /// Sets the overall per-request timeout (default `360s`). - pub fn timeout(mut self, timeout: Duration) -> Self { - self.timeout = timeout; + /// Sets the duration of the [`Timeout::Short`](crate::Timeout::Short) tier, used by simple + /// metadata reads and writes (default `5s`). + pub fn timeout_short(mut self, timeout: Duration) -> Self { + self.tiers.short = timeout; self } + /// Sets the duration of the [`Timeout::Medium`](crate::Timeout::Medium) tier, used by + /// listing, batch and trigger operations (default `30s`). + pub fn timeout_medium(mut self, timeout: Duration) -> Self { + self.tiers.medium = timeout; + self + } + + /// Sets the duration of the [`Timeout::Long`](crate::Timeout::Long) tier, used by downloads, + /// uploads and streaming (default `360s`). + pub fn timeout_long(mut self, timeout: Duration) -> Self { + self.tiers.long = timeout; + self + } + + /// Caps the timeout of a single request attempt (default `360s`). A tier or an explicit + /// timeout above the cap is lowered to it. The timeout of each attempt doubles with every + /// retry, up to this cap. + pub fn timeout_max(mut self, timeout: Duration) -> Self { + self.tiers.max = timeout; + self + } + + /// Caps the timeout of a single request attempt (default `360s`). Equivalent to + /// [`timeout_max`](Self::timeout_max), kept under its original name. + pub fn timeout(self, timeout: Duration) -> Self { + self.timeout_max(timeout) + } + /// Appends a custom suffix to the `User-Agent` header. pub fn user_agent_suffix(mut self, suffix: impl Into) -> Self { self.user_agent_suffix = Some(suffix.into()); self } - /// Selects the algorithm used to compress large request bodies (default - /// [`RequestCompression::Brotli`]). + /// Selects the built-in algorithm used to compress large request bodies (default + /// [`RequestCompression::Brotli`]), at its default quality. /// /// The client compresses request bodies of at least 1024 bytes; this chooses whether that /// uses brotli (`Content-Encoding: br`) or gzip (`Content-Encoding: gzip`). Brotli is /// preferred (matching the reference client); select [`RequestCompression::Gzip`] for - /// environments or intermediaries that do not handle brotli. + /// environments or intermediaries that do not handle brotli. To tune the quality or use + /// another algorithm, pass an [`HttpCompressor`] to [`http_compressor`](Self::http_compressor). pub fn request_compression(mut self, compression: RequestCompression) -> Self { - self.request_compression = compression; + self.compressor = compression.into_compressor(); + self + } + + /// Replaces the request-body compressor with a custom [`HttpCompressor`], for example a + /// [`BrotliHttpCompressor`](crate::BrotliHttpCompressor) with a non-default quality. + pub fn http_compressor(mut self, compressor: Arc) -> Self { + self.compressor = compressor; + self + } + + /// Adds a header sent with every request. It wins over the built-in `User-Agent` and + /// `Authorization` headers, and a header a request sets itself wins over it. Header names are + /// compared case-insensitively, and adding the same name again replaces the earlier value. + pub fn default_header(mut self, name: impl Into, value: impl Into) -> Self { + let name = name.into(); + self.default_headers + .retain(|(existing, _)| !existing.eq_ignore_ascii_case(&name)); + self.default_headers.push((name, value.into())); self } @@ -165,16 +213,17 @@ impl ApifyClientBuilder { let retry = RetryConfig { max_retries: self.max_retries, min_delay_between_retries: self.min_delay_between_retries, - timeout: self.timeout, }; - let http = HttpClient::new( + let http = HttpClient::new(HttpClientConfig { backend, - self.token, + token: self.token, user_agent, retry, - self.request_compression, - ); + tiers: self.tiers, + compressor: self.compressor, + default_headers: self.default_headers, + }); let base_url = to_api_base_url(&self.base_url); let public_base_url = diff --git a/src/clients/actor.rs b/src/clients/actor.rs index f177255..ed4d074 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -18,6 +18,7 @@ use crate::common::{parse_data_envelope, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::{Actor, ActorRun, Build}; +use crate::timeouts::Timeout; /// How long to keep retrying a run start that the API rejects for lack of resources (see /// [`ActorStartOptions::wait_for_resources`]). @@ -38,8 +39,9 @@ pub struct ActorStartOptions { pub build: Option, /// Memory in megabytes allocated for the run. pub memory_mbytes: Option, - /// Timeout for the run in seconds (`0` means no timeout). - pub timeout_secs: Option, + /// Timeout of the run itself in seconds (`0` means no timeout). Not the timeout of the API + /// request, which is set with the client's `with_timeout`. + pub run_timeout_secs: Option, /// Maximum seconds to wait server-side for the run to finish (max 60). pub wait_for_finish: Option, /// Maximum number of dataset items to charge (pay-per-result Actors). @@ -77,7 +79,7 @@ impl ActorStartOptions { params .add_str("build", self.build.clone()) .add_int("memory", self.memory_mbytes) - .add_int("timeout", self.timeout_secs) + .add_int("timeout", self.run_timeout_secs) .add_int("waitForFinish", self.wait_for_finish) .add_int("maxItems", self.max_items) .add_float("maxTotalChargeUsd", self.max_total_charge_usd) @@ -120,6 +122,8 @@ pub struct ActorClient { id: String, } +crate::clients::base::impl_with_timeout!(ActorClient); + impl ActorClient { pub(crate) fn new(root: ApifyClient, http: HttpClient, base_url: &str, id: &str) -> Self { Self { @@ -132,17 +136,17 @@ impl ActorClient { /// Fetches the Actor object, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the Actor with the given fields and returns the updated object. pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the Actor. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Starts the Actor and returns immediately with the created run. @@ -195,6 +199,9 @@ impl ActorClient { .content_type .clone() .unwrap_or_else(|| default_content_type.to_string()); + let timeout = self + .ctx + .timeout_with_wait_hold(Timeout::Medium, options.wait_for_finish); crate::clients::base::retry_while_resource_limited(options.wait_for_resources, || { post_with_body( &self.ctx, @@ -202,6 +209,7 @@ impl ActorClient { ¶ms, body.clone(), &content_type, + timeout, ) }) .await @@ -250,10 +258,21 @@ impl ActorClient { params .add_str("version", Some(version_number.to_string())) .add_bool("betaPackages", options.beta_packages) - .add_str("tag", options.tag) + .add_str("tag", options.tag.clone()) .add_bool("useCache", options.use_cache) .add_int("waitForFinish", options.wait_for_finish); - post_with_body(&self.ctx, Some("builds"), ¶ms, None, "application/json").await + let timeout = self + .ctx + .timeout_with_wait_hold(Timeout::Medium, options.wait_for_finish); + post_with_body( + &self.ctx, + Some("builds"), + ¶ms, + None, + "application/json", + timeout, + ) + .await } /// Resolves the Actor's default build and returns a client for it. @@ -267,16 +286,12 @@ impl ActorClient { let mut params = QueryParams::new(); params.add_int("waitForFinish", wait_for_finish); let url = params.apply_to_url(&self.ctx.url(Some("builds/default"))); + let timeout = self + .ctx + .timeout_with_wait_hold(Timeout::Short, wait_for_finish); let response = self .ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + .send(crate::http_client::HttpMethod::Get, url, timeout) .await?; let build: Build = parse_data_envelope(&response.body)?; Ok(BuildClient::new( @@ -316,6 +331,7 @@ impl ActorClient { ¶ms, Some(body), Some("application/json"), + self.ctx.timeout(Timeout::Short), ) .await } diff --git a/src/clients/actor_collection.rs b/src/clients/actor_collection.rs index 12c3fc7..8d61766 100644 --- a/src/clients/actor_collection.rs +++ b/src/clients/actor_collection.rs @@ -8,6 +8,7 @@ use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Actor; +use crate::timeouts::Timeout; /// Options for listing Actors (`GET /v2/actors`). #[derive(Debug, Default, Clone)] @@ -30,6 +31,8 @@ pub struct ActorCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ActorCollectionClient); + impl ActorCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -51,7 +54,7 @@ impl ActorCollectionClient { .add_bool("desc", options.desc) .add_bool("my", options.my) .add_str("sortBy", options.sort_by); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all Actors matching `options`, fetching pages on demand. @@ -71,6 +74,6 @@ impl ActorCollectionClient { /// /// `actor` is any JSON-serializable Actor definition (at minimum a `name`). pub async fn create(&self, actor: &T) -> ApifyClientResult { - create_resource(&self.ctx, &QueryParams::new(), actor).await + create_resource(&self.ctx, &QueryParams::new(), actor, Timeout::Medium).await } } diff --git a/src/clients/actor_env_var.rs b/src/clients/actor_env_var.rs index 6062a21..68a8646 100644 --- a/src/clients/actor_env_var.rs +++ b/src/clients/actor_env_var.rs @@ -5,6 +5,7 @@ use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorEnvVar; +use crate::timeouts::Timeout; /// Client for a specific environment variable of an Actor version. #[derive(Debug, Clone)] @@ -12,6 +13,8 @@ pub struct ActorEnvVarClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ActorEnvVarClient); + impl ActorEnvVarClient { pub(crate) fn new(http: HttpClient, base_url: &str, name: &str) -> Self { Self { @@ -21,16 +24,16 @@ impl ActorEnvVarClient { /// Fetches the environment variable, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the environment variable. pub async fn update(&self, env_var: &ActorEnvVar) -> ApifyClientResult { - update_resource(&self.ctx, None, env_var).await + update_resource(&self.ctx, None, env_var, Timeout::Short).await } /// Deletes the environment variable. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } } diff --git a/src/clients/actor_env_var_collection.rs b/src/clients/actor_env_var_collection.rs index 7048b4a..5dc2042 100644 --- a/src/clients/actor_env_var_collection.rs +++ b/src/clients/actor_env_var_collection.rs @@ -6,6 +6,7 @@ use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorEnvVar; +use crate::timeouts::Timeout; /// Client for listing and creating environment variables of an Actor version. #[derive(Debug, Clone)] @@ -13,6 +14,8 @@ pub struct ActorEnvVarCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ActorEnvVarCollectionClient); + impl ActorEnvVarCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -22,7 +25,7 @@ impl ActorEnvVarCollectionClient { /// Lists the environment variables of the Actor version. pub async fn list(&self) -> ApifyClientResult> { - list_resource(&self.ctx, None, &QueryParams::new()).await + list_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Lazily iterates over the Actor version's environment variables. @@ -36,6 +39,6 @@ impl ActorEnvVarCollectionClient { /// Creates a new environment variable. pub async fn create(&self, env_var: &ActorEnvVar) -> ApifyClientResult { - create_resource(&self.ctx, &QueryParams::new(), env_var).await + create_resource(&self.ctx, &QueryParams::new(), env_var, Timeout::Short).await } } diff --git a/src/clients/actor_version.rs b/src/clients/actor_version.rs index c96a9b3..80a9ad8 100644 --- a/src/clients/actor_version.rs +++ b/src/clients/actor_version.rs @@ -9,6 +9,7 @@ use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorVersion; +use crate::timeouts::Timeout; /// Client for a specific Actor version. #[derive(Debug, Clone)] @@ -16,6 +17,8 @@ pub struct ActorVersionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ActorVersionClient); + impl ActorVersionClient { pub(crate) fn new(http: HttpClient, base_url: &str, version_number: &str) -> Self { Self { @@ -25,17 +28,17 @@ impl ActorVersionClient { /// Fetches the version, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the version with the given fields. pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the version. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Returns a client for a specific environment variable of this version. diff --git a/src/clients/actor_version_collection.rs b/src/clients/actor_version_collection.rs index b32535b..7326154 100644 --- a/src/clients/actor_version_collection.rs +++ b/src/clients/actor_version_collection.rs @@ -8,6 +8,7 @@ use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorVersion; +use crate::timeouts::Timeout; /// Client for listing and creating Actor versions. #[derive(Debug, Clone)] @@ -15,6 +16,8 @@ pub struct ActorVersionCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ActorVersionCollectionClient); + impl ActorVersionCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -25,7 +28,7 @@ impl ActorVersionCollectionClient { /// Lists the Actor's versions. The endpoint takes no query parameters and returns every /// version in a single page. pub async fn list(&self) -> ApifyClientResult> { - list_resource(&self.ctx, None, &QueryParams::new()).await + list_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Lazily iterates over the Actor's versions. @@ -39,6 +42,6 @@ impl ActorVersionCollectionClient { /// Creates a new Actor version. pub async fn create(&self, version: &T) -> ApifyClientResult { - create_resource(&self.ctx, &QueryParams::new(), version).await + create_resource(&self.ctx, &QueryParams::new(), version, Timeout::Short).await } } diff --git a/src/clients/base.rs b/src/clients/base.rs index 371635e..56bbd76 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -5,16 +5,18 @@ //! and wait-for-finish primitives once, so that every resource client stays small and //! consistent (the DRY principle). +use std::collections::HashMap; use std::time::Duration; use serde::de::DeserializeOwned; use serde::Serialize; use crate::common::{ - catch_not_found, parse_data_envelope, to_safe_id, PaginationList, QueryParams, + catch_not_found, parse_data_envelope, to_path_segment, to_safe_id, PaginationList, QueryParams, }; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{HttpClient, HttpMethod, HttpRequest, HttpResponse}; +use crate::timeouts::Timeout; /// How long to wait between polls while waiting for a run/build to finish. const WAIT_FOR_FINISH_POLL_INTERVAL: Duration = Duration::from_millis(250); @@ -28,15 +30,9 @@ const WAIT_FOR_FINISH_REQUEST_SECS: i64 = 60; /// Without this bound, a run/build ID that was simply never valid (e.g. a typo, or the resource /// was deleted) would poll indefinitely when `wait_secs` is `None`. const NOT_FOUND_GRACE_PERIOD: Duration = Duration::from_secs(3); - -/// Default per-request timeout used when an endpoint does not specify its own (6 minutes). -/// This is the single source of truth for the base request timeout; clients that issue raw -/// requests reuse it instead of repeating the literal. -pub(crate) const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(360); -/// Short timeout for fast, idempotent metadata operations (mirrors the JS `SMALL_TIMEOUT`). -pub(crate) const SMALL_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); -/// Medium timeout for operations that may take a little longer (mirrors JS `MEDIUM_TIMEOUT`). -pub(crate) const MEDIUM_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// Path segment standing in for an ID that failed validation. The URL is never requested: +/// the [`HttpClient`] of such a context is poisoned and fails every call first. +const INVALID_ID_PLACEHOLDER: &str = "invalid-id"; /// The resolved context for a resource client: its base URL and the shared HTTP client. #[derive(Debug, Clone)] @@ -50,26 +46,45 @@ pub(crate) struct ResourceContext { pub(crate) api_origin: String, /// Origin used to build public, shareable URLs (defaults to `api_origin`). pub(crate) public_origin: String, + /// Whether the client names its resource by ID. A client chained from a parent, such as + /// `run.dataset()`, has none, so a `404` from its path cannot tell a missing parent from a + /// missing sub-resource (see [`catch_not_found_for`]). + pub(crate) has_id: bool, + /// Timeout set with the client's `with_timeout`, replacing the tier a method defaults to. + pub(crate) timeout: Option, } impl ResourceContext { /// Creates a context for a collection endpoint: `{base}/{resource_path}`. pub(crate) fn collection(http: HttpClient, base_url: &str, resource_path: &str) -> Self { - Self::new(http, format!("{base_url}/{resource_path}"), base_url) + Self::new(http, format!("{base_url}/{resource_path}"), base_url, false) } /// Creates a context for a single resource: `{base}/{resource_path}/{safe_id}`. + /// + /// An ID that is empty or a dot segment cannot be placed in a URL path without changing which + /// endpoint it addresses. Such a context keeps working as a handle, but every request through + /// it, or through a client chained from it, fails with + /// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument). pub(crate) fn single(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self { + let (http, segment) = match to_path_segment(&to_safe_id(id)) { + Ok(segment) => (http, segment), + Err(err) => ( + http.poisoned(err.to_string()), + INVALID_ID_PLACEHOLDER.to_string(), + ), + }; Self::new( http, - format!("{base_url}/{resource_path}/{}", to_safe_id(id)), + format!("{base_url}/{resource_path}/{segment}"), base_url, + true, ) } /// Internal constructor that derives the API origin from `base_url`. The public origin /// defaults to the API origin and can be overridden with [`with_public_origin`]. - fn new(http: HttpClient, url: String, base_url: &str) -> Self { + fn new(http: HttpClient, url: String, base_url: &str, has_id: bool) -> Self { let api_origin = origin_of(base_url); Self { http, @@ -77,6 +92,8 @@ impl ResourceContext { base_params: QueryParams::new(), public_origin: api_origin.clone(), api_origin, + has_id, + timeout: None, } } @@ -113,8 +130,87 @@ impl ResourceContext { merged.extend(params); merged } + + /// The timeout of a request that defaults to `tier`: the client's own override if it has one, + /// resolved to the duration of the first attempt. + pub(crate) fn timeout(&self, tier: Timeout) -> Duration { + self.http.resolve_timeout(self.timeout.unwrap_or(tier)) + } + + /// Like [`timeout`](Self::timeout), for a request that asks the API to hold its response for + /// `wait_for_finish_secs`: the default tier gets the hold added, while a timeout set with + /// `with_timeout` is used as given. + pub(crate) fn timeout_with_wait_hold( + &self, + tier: Timeout, + wait_for_finish_secs: Option, + ) -> Duration { + match self.timeout { + Some(timeout) => self.http.resolve_timeout(timeout), + None => self + .http + .resolve_timeout_with_wait_hold(tier, wait_for_finish_secs), + } + } + + /// Sends a request with no body or extra headers. + pub(crate) async fn send( + &self, + method: HttpMethod, + url: String, + timeout: Duration, + ) -> ApifyClientResult { + self.send_with(method, url, HashMap::new(), None, timeout) + .await + } + + /// Sends a request through the shared [`HttpClient`]. + pub(crate) async fn send_with( + &self, + method: HttpMethod, + url: String, + headers: HashMap, + body: Option>, + timeout: Duration, + ) -> ApifyClientResult { + self.http + .call(HttpRequest { + method, + url, + headers, + body, + timeout, + }) + .await + } + + /// The URL of `sub_path` with the inherited and per-call query parameters applied. + fn url_with_params(&self, sub_path: Option<&str>, params: &QueryParams) -> String { + self.merged_params(params).apply_to_url(&self.url(sub_path)) + } } +/// Generates the public `with_timeout` method of a resource client holding a [`ResourceContext`] +/// in its `ctx` field. +macro_rules! impl_with_timeout { + ($client:ident) => { + impl $client { + /// Sets the timeout of every request this client sends, replacing the tier each + /// method defaults to (see [`Timeout`](crate::Timeout)). + /// + /// The timeout applies to each attempt of a request separately and doubles with every + /// retry, up to the client's `timeout_max`. A longer timeout is lowered to that cap. + /// Clients obtained from this one, such as `run.dataset()`, do not inherit it. + #[must_use] + pub fn with_timeout(mut self, timeout: $crate::Timeout) -> Self { + self.ctx.timeout = Some(timeout); + self + } + } + }; +} +pub(crate) use impl_with_timeout; + /// Extracts the origin (`scheme://host[:port]`) from a URL, dropping any path. fn origin_of(url: &str) -> String { match url.split_once("://") { @@ -135,14 +231,36 @@ impl QueryParams { } } -/// A `GET` that unwraps the `data` envelope and maps `404` to `None`. +/// Header map for a request with a JSON body. +fn json_headers() -> HashMap { + HashMap::from([("Content-Type".to_string(), "application/json".to_string())]) +} + +/// Swallows a `404` and returns `Ok(None)` only when the client names its resource by ID. A +/// chained client without one, such as `run.dataset()`, requests a path where a `404` can mean +/// either the parent or the default sub-resource is missing, and the response cannot tell the two +/// apart, so the error propagates. +pub(crate) fn catch_not_found_for( + ctx: &ResourceContext, + result: ApifyClientResult, +) -> ApifyClientResult> { + if ctx.has_id { + catch_not_found(result) + } else { + result.map(Some) + } +} + +/// A `GET` that unwraps the `data` envelope and maps `404` to `None` (see +/// [`catch_not_found_for`]). pub(crate) async fn get_resource( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, + tier: Timeout, ) -> ApifyClientResult> { - let result = get_resource_required(ctx, sub_path, params).await; - catch_not_found(result) + let result = get_resource_required(ctx, sub_path, params, tier).await; + catch_not_found_for(ctx, result) } /// A `GET` that unwraps the `data` envelope and propagates errors (including `404`). @@ -150,18 +268,10 @@ pub(crate) async fn get_resource_required( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, + tier: Timeout, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let url = ctx.url_with_params(sub_path, params); + let response = ctx.send(HttpMethod::Get, url, ctx.timeout(tier)).await?; parse_data_envelope(&response.body) } @@ -170,45 +280,30 @@ pub(crate) async fn update_resource( ctx: &ResourceContext, sub_path: Option<&str>, body: &B, + tier: Timeout, ) -> ApifyClientResult { - let url = ctx - .merged_params(&QueryParams::new()) - .apply_to_url(&ctx.url(sub_path)); - let body_bytes = serde_json::to_vec(body)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); + let url = ctx.url_with_params(sub_path, &QueryParams::new()); let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Put, + .send_with( + HttpMethod::Put, url, - headers, - body: Some(body_bytes), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) + json_headers(), + Some(serde_json::to_vec(body)?), + ctx.timeout(tier), + ) .await?; parse_data_envelope(&response.body) } -/// A `DELETE` that maps `404` to a successful no-op. +/// A `DELETE` that maps `404` to a successful no-op (see [`catch_not_found_for`]). pub(crate) async fn delete_resource( ctx: &ResourceContext, sub_path: Option<&str>, + tier: Timeout, ) -> ApifyClientResult<()> { - let url = ctx - .merged_params(&QueryParams::new()) - .apply_to_url(&ctx.url(sub_path)); - let result = ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await; - catch_not_found(result.map(|_| ()))?; + let url = ctx.url_with_params(sub_path, &QueryParams::new()); + let result = ctx.send(HttpMethod::Delete, url, ctx.timeout(tier)).await; + catch_not_found_for(ctx, result)?; Ok(()) } @@ -217,8 +312,9 @@ pub(crate) async fn list_resource( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, + tier: Timeout, ) -> ApifyClientResult> { - get_resource_required(ctx, sub_path, params).await + get_resource_required(ctx, sub_path, params, tier).await } /// A `POST` with a JSON body that unwraps the `data` envelope (used for `create`). @@ -226,20 +322,17 @@ pub(crate) async fn create_resource( ctx: &ResourceContext, params: &QueryParams, body: &B, + tier: Timeout, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(None)); - let body_bytes = serde_json::to_vec(body)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); + let url = ctx.url_with_params(None, params); let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Post, + .send_with( + HttpMethod::Post, url, - headers, - body: Some(body_bytes), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) + json_headers(), + Some(serde_json::to_vec(body)?), + ctx.timeout(tier), + ) .await?; parse_data_envelope(&response.body) } @@ -249,20 +342,12 @@ pub(crate) async fn create_resource( pub(crate) async fn get_or_create_named( ctx: &ResourceContext, name: Option<&str>, + tier: Timeout, ) -> ApifyClientResult { let mut params = QueryParams::new(); params.add_str("name", name.map(|s| s.to_string())); let url = params.apply_to_url(&ctx.url(None)); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Post, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let response = ctx.send(HttpMethod::Post, url, ctx.timeout(tier)).await?; parse_data_envelope(&response.body) } @@ -275,8 +360,9 @@ pub(crate) async fn post_action( params: &QueryParams, body: Option>, content_type: Option<&str>, + timeout: Duration, ) -> ApifyClientResult { - let response = post_send(ctx, sub_path, params, body, content_type).await?; + let response = post_send(ctx, sub_path, params, body, content_type, timeout).await?; parse_data_envelope(&response.body) } @@ -290,8 +376,9 @@ pub(crate) async fn post_action_raw( params: &QueryParams, body: Option>, content_type: Option<&str>, + timeout: Duration, ) -> ApifyClientResult { - let response = post_send(ctx, sub_path, params, body, content_type).await?; + let response = post_send(ctx, sub_path, params, body, content_type, timeout).await?; Ok(serde_json::from_slice(&response.body)?) } @@ -303,42 +390,49 @@ async fn post_send( params: &QueryParams, body: Option>, content_type: Option<&str>, -) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let mut headers = std::collections::HashMap::new(); + timeout: Duration, +) -> ApifyClientResult { + let url = ctx.url_with_params(sub_path, params); + let mut headers = HashMap::new(); if let Some(ct) = content_type { headers.insert("Content-Type".to_string(), ct.to_string()); } - ctx.http - .call(HttpRequest { - method: HttpMethod::Post, - url, - headers, - body, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) + ctx.send_with(HttpMethod::Post, url, headers, body, timeout) .await } -/// A `GET` returning the raw response body bytes (no `data` envelope). Maps `404`/`HEAD` -/// not-found to `None`. Used for logs and key-value-store record values. +/// A `GET` returning the raw response body bytes (no `data` envelope), with a `404` mapped to +/// `None` whatever the client's path. Used for key-value-store record values. pub(crate) async fn get_raw( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, -) -> ApifyClientResult> { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let result = ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await; - catch_not_found(result) + tier: Timeout, +) -> ApifyClientResult> { + catch_not_found(get_raw_required(ctx, sub_path, params, tier).await) +} + +/// Like [`get_raw`], but a `404` maps to `None` only when the client names its resource by ID +/// (see [`catch_not_found_for`]). Used for logs. +pub(crate) async fn get_raw_for_resource( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + tier: Timeout, +) -> ApifyClientResult> { + catch_not_found_for(ctx, get_raw_required(ctx, sub_path, params, tier).await) +} + +/// A `GET` returning the raw response body bytes (no `data` envelope), propagating errors +/// (including `404`). +pub(crate) async fn get_raw_required( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + tier: Timeout, +) -> ApifyClientResult { + let url = ctx.url_with_params(sub_path, params); + ctx.send(HttpMethod::Get, url, ctx.timeout(tier)).await } /// A `HEAD` request returning whether the resource exists (`true` on 2xx, `false` on 404). @@ -346,18 +440,10 @@ pub(crate) async fn head_exists( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, + tier: Timeout, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let result = ctx - .http - .call(HttpRequest { - method: HttpMethod::Head, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await; + let url = ctx.url_with_params(sub_path, params); + let result = ctx.send(HttpMethod::Head, url, ctx.timeout(tier)).await; Ok(catch_not_found(result)?.is_some()) } @@ -368,8 +454,9 @@ pub(crate) async fn put_raw( params: &QueryParams, body: Vec, content_type: &str, + tier: Timeout, ) -> ApifyClientResult<()> { - put_raw_with_content_encoding(ctx, sub_path, params, body, content_type, None).await + put_raw_with_content_encoding(ctx, sub_path, params, body, content_type, None, tier).await } /// Like [`put_raw`], but can also set a `Content-Encoding` header (used for KVS record uploads @@ -383,21 +470,15 @@ pub(crate) async fn put_raw_with_content_encoding( body: Vec, content_type: &str, content_encoding: Option<&str>, + tier: Timeout, ) -> ApifyClientResult<()> { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let mut headers = std::collections::HashMap::new(); + let url = ctx.url_with_params(sub_path, params); + let mut headers = HashMap::new(); headers.insert("Content-Type".to_string(), content_type.to_string()); if let Some(encoding) = content_encoding { headers.insert("Content-Encoding".to_string(), encoding.to_string()); } - ctx.http - .call(HttpRequest { - method: HttpMethod::Put, - url, - headers, - body: Some(body), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) + ctx.send_with(HttpMethod::Put, url, headers, Some(body), ctx.timeout(tier)) .await?; Ok(()) } @@ -412,6 +493,7 @@ pub(crate) async fn put_raw_with_content_encoding( pub(crate) async fn get_byte_stream( ctx: &ResourceContext, url: String, + tier: Timeout, ) -> ApifyClientResult>>> { ctx.http .call_streaming(HttpRequest { @@ -419,7 +501,7 @@ pub(crate) async fn get_byte_stream( url, headers: Default::default(), body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, + timeout: ctx.timeout(tier), }) .await } @@ -432,8 +514,9 @@ pub(crate) async fn post_with_body( params: &QueryParams, body: Option>, content_type: &str, + timeout: Duration, ) -> ApifyClientResult { - post_action(ctx, sub_path, params, body, Some(content_type)).await + post_action(ctx, sub_path, params, body, Some(content_type), timeout).await } /// A `DELETE` with a JSON body (used for batch request deletion). @@ -442,20 +525,17 @@ pub(crate) async fn delete_with_body( sub_path: Option<&str>, params: &QueryParams, body: &B, + tier: Timeout, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let body_bytes = serde_json::to_vec(body)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); + let url = ctx.url_with_params(sub_path, params); let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, + .send_with( + HttpMethod::Delete, url, - headers, - body: Some(body_bytes), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) + json_headers(), + Some(serde_json::to_vec(body)?), + ctx.timeout(tier), + ) .await?; parse_data_envelope(&response.body) } @@ -465,6 +545,10 @@ pub(crate) async fn delete_with_body( /// /// `is_terminal` extracts whether the fetched resource is finished. Returns the latest /// resource (finished, or still running if the wait budget was exhausted). +/// +/// Each poll has no client-side timeout unless the client was given one with `with_timeout`: the +/// API holds a poll for up to a minute, and a poll that is aborted and retried loses nothing but +/// time, yet an arbitrary tier would cut the hold short. pub(crate) async fn wait_for_finish( ctx: &ResourceContext, wait_secs: Option, @@ -488,7 +572,7 @@ where // Budget exhausted: do one final immediate fetch and return it. let mut params = QueryParams::new(); params.add_int("waitForFinish", Some(0)); - return get_resource_required(ctx, None, ¶ms).await; + return get_resource_required(ctx, None, ¶ms, Timeout::NoTimeout).await; } let remaining = (budget - elapsed).as_secs() as i64; remaining.min(WAIT_FOR_FINISH_REQUEST_SECS) @@ -499,7 +583,7 @@ where let mut params = QueryParams::new(); params.add_int("waitForFinish", Some(remaining_request_secs)); - match get_resource_required::(ctx, None, ¶ms).await { + match get_resource_required::(ctx, None, ¶ms, Timeout::NoTimeout).await { Ok(resource) => { not_found_since = None; if is_terminal(&resource) { diff --git a/src/clients/build.rs b/src/clients/build.rs index fd887df..de0532a 100644 --- a/src/clients/build.rs +++ b/src/clients/build.rs @@ -8,6 +8,7 @@ use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Build; +use crate::timeouts::Timeout; /// Client for a specific Actor build. #[derive(Debug, Clone)] @@ -15,6 +16,8 @@ pub struct BuildClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(BuildClient); + impl BuildClient { pub(crate) fn new(http: HttpClient, base_url: &str, id: &str) -> Self { Self { @@ -24,17 +27,25 @@ impl BuildClient { /// Fetches the build object, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Aborts the build. pub async fn abort(&self) -> ApifyClientResult { - post_action(&self.ctx, Some("abort"), &QueryParams::new(), None, None).await + post_action( + &self.ctx, + Some("abort"), + &QueryParams::new(), + None, + None, + self.ctx.timeout(Timeout::Short), + ) + .await } /// Deletes the build. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Waits (by client-side polling) for the build to reach a terminal state. @@ -48,18 +59,21 @@ impl BuildClient { wait_for_finish(&self.ctx, wait_secs, |b: &Build| b.is_terminal()).await } - /// Retrieves the OpenAPI definition generated for this build, if available. + /// Retrieves the OpenAPI definition generated for this build. /// /// Corresponds to `GET /v2/actor-builds/{buildId}/openapi.json`. The response is a raw - /// OpenAPI document (not wrapped in a `data` envelope), returned as JSON. - pub async fn get_openapi_definition(&self) -> ApifyClientResult> { - let response = - crate::clients::base::get_raw(&self.ctx, Some("openapi.json"), &QueryParams::new()) - .await?; - match response { - Some(r) => Ok(Some(serde_json::from_slice(&r.body)?)), - None => Ok(None), - } + /// OpenAPI document (not wrapped in a `data` envelope), returned as JSON. A build that does + /// not exist, or has no OpenAPI definition, is a [`NotFound`](crate::ApiError::is_not_found) + /// error: the response cannot tell the two apart. + pub async fn get_openapi_definition(&self) -> ApifyClientResult { + let response = crate::clients::base::get_raw_required( + &self.ctx, + Some("openapi.json"), + &QueryParams::new(), + Timeout::Medium, + ) + .await?; + Ok(serde_json::from_slice(&response.body)?) } /// Returns a client for the build's log. diff --git a/src/clients/build_collection.rs b/src/clients/build_collection.rs index e458ade..602f6ed 100644 --- a/src/clients/build_collection.rs +++ b/src/clients/build_collection.rs @@ -6,6 +6,7 @@ use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Build; +use crate::timeouts::Timeout; /// Client for listing Actor builds. #[derive(Debug, Clone)] @@ -13,6 +14,8 @@ pub struct BuildCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(BuildCollectionClient); + impl BuildCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -34,7 +37,7 @@ impl BuildCollectionClient { .add_int("offset", options.offset) .add_int("limit", options.limit) .add_bool("desc", options.desc); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all builds matching `options`, fetching pages on demand. diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 65423ab..43bf899 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -1,17 +1,90 @@ //! Client for a single dataset (`/v2/datasets/{datasetId}` and run-nested variants). -use serde::de::DeserializeOwned; -use serde::Serialize; +use serde::de::{DeserializeOwned, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; use crate::clients::base::{ - delete_resource, get_byte_stream, get_resource, update_resource, ResourceContext, + delete_resource, get_byte_stream, get_resource, get_resource_required, update_resource, + ResourceContext, }; use crate::clients::pagination::ListIterator; -use crate::common::{parse_data_envelope, sign_storage_content, PaginationList, QueryParams}; -use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::common::{sign_storage_content, PaginationList, QueryParams}; +use crate::error::{ApifyClientError, ApifyClientResult}; +use crate::http_client::{HttpClient, HttpMethod}; use crate::models::Dataset; +use crate::timeouts::Timeout; + +/// Checks that a serialized `push_items` body is a JSON object or an array of JSON objects, +/// without building a value out of it. +fn check_items_shape(body: &[u8]) -> ApifyClientResult<()> { + serde_json::from_slice::(body) + .map(|_| ()) + .map_err(|err| { + ApifyClientError::InvalidArgument(format!( + "push_items takes a JSON object or an array of JSON objects: {err}" + )) + }) +} + +/// Marker deserialized from a JSON object or an array of JSON objects, and nothing else. +struct ItemsShape; + +impl<'de> Deserialize<'de> for ItemsShape { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(ItemsShapeVisitor) + } +} + +struct ItemsShapeVisitor; + +impl<'de> Visitor<'de> for ItemsShapeVisitor { + type Value = ItemsShape; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a JSON object or an array of JSON objects") + } + + fn visit_map>(self, map: A) -> Result { + skip_object(map)?; + Ok(ItemsShape) + } + + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + Ok(ItemsShape) + } +} + +/// Marker deserialized from a JSON object and nothing else. +struct ObjectShape; + +impl<'de> Deserialize<'de> for ObjectShape { + fn deserialize>(deserializer: D) -> Result { + struct ObjectVisitor; + + impl<'de> Visitor<'de> for ObjectVisitor { + type Value = ObjectShape; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a JSON object") + } + + fn visit_map>(self, map: A) -> Result { + skip_object(map)?; + Ok(ObjectShape) + } + } + + deserializer.deserialize_map(ObjectVisitor) + } +} + +/// Reads the entries of a JSON object without keeping them. +fn skip_object<'de, A: MapAccess<'de>>(mut map: A) -> Result<(), A::Error> { + while map.next_entry::()?.is_some() {} + Ok(()) +} /// Options for listing or downloading dataset items. /// @@ -151,6 +224,8 @@ pub struct DatasetClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(DatasetClient); + impl DatasetClient { pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self { Self { @@ -173,17 +248,17 @@ impl DatasetClient { /// Fetches the dataset metadata, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the dataset metadata (e.g. `name`, `title`). pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the dataset. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Lists items from the dataset. @@ -200,14 +275,7 @@ impl DatasetClient { let url = params.apply_to_url(&self.ctx.url(Some("items"))); let response = self .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + .send(HttpMethod::Get, url, self.ctx.timeout(Timeout::Long)) .await?; let items: Vec = serde_json::from_slice(&response.body)?; @@ -300,14 +368,7 @@ impl DatasetClient { let url = params.apply_to_url(&self.ctx.url(Some("items"))); let response = self .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + .send(HttpMethod::Get, url, self.ctx.timeout(Timeout::Long)) .await?; Ok(response.body) } @@ -317,9 +378,10 @@ impl DatasetClient { /// /// Returns a `Stream` of byte chunks; drive it with `futures_util::StreamExt::next()` (or /// pipe it through an `AsyncRead` adapter) to process a large export in constant memory. A - /// gap of more than 6 minutes between chunks ends the stream with - /// [`ApifyClientError::Timeout`](crate::error::ApifyClientError::Timeout) — an inactivity - /// timeout, not a cap on the export's total duration. + /// gap between chunks longer than the [`Timeout::Long`] tier (6 minutes by default) ends the + /// stream with [`ApifyClientError::Timeout`] — an + /// inactivity timeout, not a cap on the export's total duration. Use + /// [`with_timeout`](Self::with_timeout) to change it. /// /// Goes through the configured `HttpBackend`'s streaming path instead of the buffered one, /// to keep the connection open — like [`crate::clients::log::LogClient::stream`], a failed @@ -334,14 +396,18 @@ impl DatasetClient { params.add_str("format", Some(format.as_str())); options.apply(&mut params); let url = params.apply_to_url(&self.ctx.url(Some("items"))); - get_byte_stream(&self.ctx, url).await + get_byte_stream(&self.ctx, url, Timeout::Long).await } /// Pushes one or more items to the dataset. /// - /// `items` must serialize to a JSON object or an array of objects. + /// `items` must serialize to a JSON object or an array of objects, the only shapes the + /// endpoint accepts. Anything else, such as a plain string or an array of strings, is + /// rejected with [`ApifyClientError::InvalidArgument`] + /// before a request is sent. pub async fn push_items(&self, items: &T) -> ApifyClientResult<()> { let body = serde_json::to_vec(items)?; + check_items_shape(&body)?; let url = self.ctx.url(Some("items")); let mut headers = std::collections::HashMap::new(); headers.insert( @@ -349,14 +415,13 @@ impl DatasetClient { "application/json; charset=utf-8".to_string(), ); self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Post, + .send_with( + HttpMethod::Post, url, headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + Some(body), + self.ctx.timeout(Timeout::Medium), + ) .await?; Ok(()) } @@ -405,23 +470,18 @@ impl DatasetClient { Ok(params.apply_to_url(&self.ctx.public_url(Some("items")))) } - /// Returns statistical information about the dataset, or `None` if unavailable. - pub async fn get_statistics(&self) -> ApifyClientResult> { - let result: ApifyClientResult = async { - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url: self.ctx.url(Some("statistics")), - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - parse_data_envelope(&response.body) - } - .await; - crate::common::catch_not_found(result) + /// Returns statistical information about the dataset. + /// + /// A dataset that does not exist is a [`NotFound`](crate::ApiError::is_not_found) error. The + /// response cannot tell a missing dataset from a missing statistics resource, and a client + /// chained from a run, such as `run.dataset()`, cannot tell a missing run either. + pub async fn get_statistics(&self) -> ApifyClientResult { + get_resource_required( + &self.ctx, + Some("statistics"), + &QueryParams::new(), + Timeout::Short, + ) + .await } } diff --git a/src/clients/dataset_collection.rs b/src/clients/dataset_collection.rs index a2399e3..f274663 100644 --- a/src/clients/dataset_collection.rs +++ b/src/clients/dataset_collection.rs @@ -6,6 +6,7 @@ use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Dataset; +use crate::timeouts::Timeout; /// Client for listing datasets and getting-or-creating a dataset by name. #[derive(Debug, Clone)] @@ -13,6 +14,8 @@ pub struct DatasetCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(DatasetCollectionClient); + impl DatasetCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -27,7 +30,7 @@ impl DatasetCollectionClient { ) -> ApifyClientResult> { let mut params = QueryParams::new(); options.apply(&mut params); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all datasets matching `options`, fetching pages on demand. @@ -44,6 +47,6 @@ impl DatasetCollectionClient { /// /// Passing `None` for `name` creates an unnamed dataset. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { - get_or_create_named(&self.ctx, name).await + get_or_create_named(&self.ctx, name, Timeout::Short).await } } diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 65c9619..6519490 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -8,12 +8,18 @@ use crate::clients::base::{ delete_resource, get_raw, get_resource, get_resource_required, head_exists, put_raw, put_raw_with_content_encoding, update_resource, ResourceContext, }; -use crate::common::{ - create_hmac_signature, encode_path_segment, sign_storage_content, QueryParams, -}; +use crate::common::{create_hmac_signature, sign_storage_content, to_path_segment, QueryParams}; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{HttpClient, HttpMethod}; use crate::models::{KeyValueStore, KeyValueStoreKey, KeyValueStoreKeysPage, KeyValueStoreRecord}; +use crate::timeouts::Timeout; + +/// The path of the record with the given key, relative to the store. A key that is empty or a dot +/// segment cannot be placed in the path, so it is rejected with +/// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument). +fn record_path(key: &str) -> ApifyClientResult { + Ok(format!("records/{}", to_path_segment(key)?)) +} /// Options for listing keys in a key-value store. #[derive(Debug, Default, Clone)] @@ -52,6 +58,8 @@ pub struct KeyValueStoreClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(KeyValueStoreClient); + impl KeyValueStoreClient { pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self { Self { @@ -74,17 +82,17 @@ impl KeyValueStoreClient { /// Fetches the store metadata, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the store metadata (e.g. `name`, `title`). pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the store. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Lists the keys in the store (key-based pagination). @@ -99,7 +107,7 @@ impl KeyValueStoreClient { .add_str("prefix", options.prefix) .add_str("collection", options.collection) .add_str("signature", options.signature); - get_resource_required(&self.ctx, Some("keys"), ¶ms).await + get_resource_required(&self.ctx, Some("keys"), ¶ms, Timeout::Medium).await } /// Lazily iterates over every key in the store, fetching pages on demand. @@ -137,14 +145,19 @@ impl KeyValueStoreClient { pub async fn record_exists(&self, key: &str) -> ApifyClientResult { head_exists( &self.ctx, - Some(&format!("records/{}", encode_path_segment(key))), + Some(&record_path(key)?), &QueryParams::new(), + Timeout::Short, ) .await } /// Gets a record's raw value (and content type), or `None` if it does not exist. /// + /// The value is returned as the bytes the API sent: unlike the reference client's + /// `getRecord`, it is never parsed according to its content type, so a malformed JSON body + /// is not an error here. Use [`KeyValueStoreRecord::as_text`] or `serde_json` to decode it. + /// /// Like the reference client's `getRecord`, this sends `attachment=true`. Use /// [`get_record_with_options`](Self::get_record_with_options) to override the attachment /// behaviour or to pass a URL-signing `signature` for a private store. @@ -168,12 +181,7 @@ impl KeyValueStoreClient { params .add_bool("attachment", Some(options.attachment.unwrap_or(true))) .add_str("signature", options.signature); - let response = get_raw( - &self.ctx, - Some(&format!("records/{}", encode_path_segment(key))), - ¶ms, - ) - .await?; + let response = get_raw(&self.ctx, Some(&record_path(key)?), ¶ms, Timeout::Long).await?; Ok(response.map(|r| { let content_type = r.header("content-type").map(|s| s.to_string()); KeyValueStoreRecord { @@ -185,6 +193,10 @@ impl KeyValueStoreClient { } /// Stores a record with raw bytes and an explicit content type. + /// + /// The bytes are sent as they are, whatever the content type: under `application/json` they + /// have to be valid JSON already. To serialize a value, use + /// [`set_record_json`](Self::set_record_json). pub async fn set_record_raw( &self, key: &str, @@ -193,10 +205,11 @@ impl KeyValueStoreClient { ) -> ApifyClientResult<()> { put_raw( &self.ctx, - Some(&format!("records/{}", encode_path_segment(key))), + Some(&record_path(key)?), &QueryParams::new(), value, content_type, + Timeout::Long, ) .await } @@ -222,11 +235,12 @@ impl KeyValueStoreClient { ) -> ApifyClientResult<()> { put_raw_with_content_encoding( &self.ctx, - Some(&format!("records/{}", encode_path_segment(key))), + Some(&record_path(key)?), &QueryParams::new(), value, content_type, Some(content_encoding), + Timeout::Long, ) .await } @@ -249,6 +263,7 @@ impl KeyValueStoreClient { /// `signature` over the record key so the URL works without an API token. The URL is /// built from the configured public base URL. pub async fn get_record_public_url(&self, key: &str) -> ApifyClientResult { + let path = record_path(key)?; let mut params = QueryParams::new(); if let Some(store) = self.get().await? { if let Some(secret) = store @@ -259,11 +274,7 @@ impl KeyValueStoreClient { params.add_str("signature", Some(create_hmac_signature(secret, key))); } } - Ok(params.apply_to_url( - &self - .ctx - .public_url(Some(&format!("records/{}", encode_path_segment(key)))), - )) + Ok(params.apply_to_url(&self.ctx.public_url(Some(&path)))) } /// Builds a public URL for listing this store's keys. @@ -291,18 +302,9 @@ impl KeyValueStoreClient { /// Deletes the record with the given key. pub async fn delete_record(&self, key: &str) -> ApifyClientResult<()> { - let url = self - .ctx - .url(Some(&format!("records/{}", encode_path_segment(key)))); + let url = self.ctx.url(Some(&record_path(key)?)); self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + .send(HttpMethod::Delete, url, self.ctx.timeout(Timeout::Short)) .await?; Ok(()) } diff --git a/src/clients/key_value_store_collection.rs b/src/clients/key_value_store_collection.rs index 65a0803..715e7af 100644 --- a/src/clients/key_value_store_collection.rs +++ b/src/clients/key_value_store_collection.rs @@ -6,6 +6,7 @@ use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::KeyValueStore; +use crate::timeouts::Timeout; /// Client for listing key-value stores and getting-or-creating one by name. #[derive(Debug, Clone)] @@ -13,6 +14,8 @@ pub struct KeyValueStoreCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(KeyValueStoreCollectionClient); + impl KeyValueStoreCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -28,7 +31,7 @@ impl KeyValueStoreCollectionClient { ) -> ApifyClientResult> { let mut params = QueryParams::new(); options.apply(&mut params); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all key-value stores matching `options`, fetching pages on demand. @@ -43,6 +46,6 @@ impl KeyValueStoreCollectionClient { /// Gets the store with the given `name`, creating it if it does not exist. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { - get_or_create_named(&self.ctx, name).await + get_or_create_named(&self.ctx, name, Timeout::Short).await } } diff --git a/src/clients/log.rs b/src/clients/log.rs index ab7bde3..17eb999 100644 --- a/src/clients/log.rs +++ b/src/clients/log.rs @@ -6,15 +6,18 @@ use futures_util::Stream; -use crate::clients::base::{get_byte_stream, get_raw, ResourceContext}; +use crate::clients::base::{ + catch_not_found_for, get_byte_stream, get_raw_for_resource, ResourceContext, +}; use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; +use crate::timeouts::Timeout; /// Options for retrieving or streaming a log ([`LogClient::get_with_options`] / /// [`LogClient::stream_with_options`]). /// -/// Covers the spec's optional `raw` query parameter on the log endpoints +/// Covers the spec's optional `raw` and `download` query parameters on the log endpoints /// (`GET /v2/logs/{buildOrRunId}`, `GET /v2/actor-runs/{runId}/log`, and the last-run log /// variants), matching the reference client's `LogOptions`. #[derive(Debug, Default, Clone)] @@ -22,6 +25,9 @@ pub struct LogOptions { /// If `true`, return the raw log content without any server-side processing (e.g. without /// the per-line timestamps the API adds by default). Defaults to `false` when unset. pub raw: Option, + /// If `true`, the API asks a web browser to download the log as a file rather than open it in + /// a tab (it adds a `Content-Disposition: attachment` header). Defaults to `false` when unset. + pub download: Option, } /// Client for an Actor run or build log. @@ -30,6 +36,8 @@ pub struct LogClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(LogClient); + impl LogClient { pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self { Self { @@ -44,7 +52,9 @@ impl LogClient { } } - /// Fetches the entire log as a string, or `None` if it does not exist. + /// Fetches the entire log as a string, or `None` if it does not exist. A log client chained + /// from a run or build, such as `run.log()`, has no ID of its own, so it returns the + /// `404` as an error: the response cannot tell a missing run from a missing log. /// /// Uses the default (processed) log format. To request the raw log, use /// [`LogClient::get_with_options`]. To get the log as raw bytes instead of a `String` @@ -79,29 +89,36 @@ impl LogClient { options: LogOptions, ) -> ApifyClientResult>> { let mut params = QueryParams::new(); - params.add_bool("raw", options.raw); - let response = get_raw(&self.ctx, None, ¶ms).await?; + params + .add_bool("raw", options.raw) + .add_bool("download", options.download); + let response = get_raw_for_resource(&self.ctx, None, ¶ms, Timeout::Long).await?; Ok(response.map(|r| r.body)) } - /// Opens a streaming connection to the log, yielding chunks of bytes as they arrive. + /// Opens a streaming connection to the log, yielding chunks of bytes as they arrive, or + /// `None` if the log does not exist. /// /// This powers real-time log redirection: callers can forward each chunk to their own /// logger/stdout while a run is still in progress. The stream completes when the log ends - /// (i.e. the run finishes) — as long as a new chunk arrives at least every 6 minutes; a run - /// that legitimately logs nothing for that long errors with + /// (i.e. the run finishes) — as long as a new chunk arrives at least every 6 minutes (the + /// [`Timeout::Long`] tier, see [`with_timeout`](Self::with_timeout)); a run that legitimately + /// logs nothing for that long errors with /// [`ApifyClientError::Timeout`](crate::error::ApifyClientError::Timeout) instead. /// + /// A log client chained from a run or build, such as `run.log()`, has no ID of its own, so it + /// returns the `404` as an error: the response cannot tell a missing run from a missing log. + /// /// Uses the default (processed) log format. To stream the raw log, use /// [`LogClient::stream_with_options`]. pub async fn stream( &self, - ) -> ApifyClientResult>>> { + ) -> ApifyClientResult>>>> { self.stream_with_options(LogOptions::default()).await } /// Opens a streaming connection to the log applying the given [`LogOptions`], yielding - /// chunks of bytes as they arrive. + /// chunks of bytes as they arrive, or `None` if the log does not exist. /// /// Like [`LogClient::stream`], but lets the caller request the raw log via /// [`LogOptions::raw`] (as the reference client's log redirection does, which streams @@ -109,14 +126,25 @@ impl LogClient { pub async fn stream_with_options( &self, options: LogOptions, + ) -> ApifyClientResult>>>> { + catch_not_found_for(&self.ctx, self.stream_required(options).await) + } + + /// Like [`stream_with_options`](Self::stream_with_options), but a `404` is an error whatever + /// the client's path. Used where the caller already knows the run exists. + pub(crate) async fn stream_required( + &self, + options: LogOptions, ) -> ApifyClientResult>>> { // Goes through `get_byte_stream` (the configured `HttpBackend`'s streaming path) instead // of `HttpClient::call`, whose buffered backend fully reads a response before returning // it. The retry policy does not apply to an open stream. let mut params = QueryParams::new(); params.push_raw("stream".to_string(), "1".to_string()); - params.add_bool("raw", options.raw); + params + .add_bool("raw", options.raw) + .add_bool("download", options.download); let url = params.apply_to_url(&self.ctx.url(None)); - get_byte_stream(&self.ctx, url).await + get_byte_stream(&self.ctx, url, Timeout::Long).await } } diff --git a/src/clients/request_queue.rs b/src/clients/request_queue.rs index f365da0..6e88d34 100644 --- a/src/clients/request_queue.rs +++ b/src/clients/request_queue.rs @@ -10,14 +10,15 @@ use crate::clients::base::{ delete_resource, delete_with_body, get_resource, get_resource_required, post_action, post_with_body, update_resource, ResourceContext, }; -use crate::common::{encode_path_segment, QueryParams}; +use crate::common::{catch_not_found, to_path_segment, QueryParams}; use crate::error::{ApifyClientError, ApifyClientResult}; -use crate::http_client::{sleep_public, HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{sleep_public, HttpClient, HttpMethod}; use crate::models::{ BatchRequestsOperationResult, LockedRequestQueueHead, RequestLockInfo, RequestQueue, RequestQueueHead, RequestQueueOperationInfo, RequestQueueRequest, RequestQueueRequestsPage, UnlockRequestsResult, UnprocessedRequest, }; +use crate::timeouts::Timeout; /// Maximum number of requests the API accepts in a single `requests/batch` call. Larger /// `batch_add_requests` inputs are split into chunks of at most this size (matching the @@ -28,6 +29,8 @@ const MAX_REQUESTS_PER_BATCH_OPERATION: usize = 25; /// `MAX_PAYLOAD_SIZE_BYTES` (9 MiB) that the reference client chunks `batch_add_requests` calls /// against, on top of the per-call request-count limit. const MAX_PAYLOAD_SIZE_BYTES: usize = 9_437_184; +/// Bytes of the `[` and `]` around a JSON array. +const ARRAY_BRACKETS_BYTES: usize = 2; /// Fraction of [`MAX_PAYLOAD_SIZE_BYTES`] held back as a safety margin (0.01%), matching the /// reference client's `SAFETY_BUFFER_PERCENT`. const SAFETY_BUFFER_PERCENT: f64 = 0.0001; @@ -43,6 +46,13 @@ const DEFAULT_MAX_UNPROCESSED_REQUESTS_RETRIES: u32 = 3; /// `DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS`. const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES: Duration = Duration::from_millis(500); +/// The path of the request with the given ID, relative to the queue. An ID that is empty or a dot +/// segment cannot be placed in the path, so it is rejected with +/// [`ApifyClientError::InvalidArgument`]. +fn request_path(id: &str) -> ApifyClientResult { + Ok(format!("requests/{}", to_path_segment(id)?)) +} + /// Options for [`RequestQueueClient::batch_add_requests`]. /// /// Mirrors the reference client's retrying, chunked, parallel `batchAddRequests`: large inputs @@ -66,23 +76,54 @@ pub struct BatchAddRequestsOptions { pub min_delay_between_unprocessed_requests_retries: Option, } -/// Returns the key used to correlate a request across the batch-add retry loop: its explicit -/// `unique_key` if set, otherwise its `url` — matching the API's own fallback (a request added -/// without a `unique_key` is deduplicated by its raw `url`). -fn dedup_key(request: &RequestQueueRequest) -> &str { - request.unique_key.as_deref().unwrap_or(&request.url) +/// A request of a batch add with its JSON serialized once, up front. The same string measures the +/// request against the payload limit and is joined into the batch body that is sent, so a request +/// is never serialized twice, whichever chunk or retry it ends up in. +struct SerializedRequest { + /// Key correlating the request across the retry loop: its explicit `unique_key` if set, + /// otherwise its `url` — matching the API's own fallback (a request added without a + /// `unique_key` is deduplicated by its raw `url`). + dedup_key: String, + url: String, + method: Option, + json: String, +} + +impl SerializedRequest { + fn new(request: &RequestQueueRequest) -> ApifyClientResult { + Ok(Self { + dedup_key: request + .unique_key + .as_deref() + .unwrap_or(&request.url) + .to_string(), + url: request.url.clone(), + method: request.method.clone(), + json: serde_json::to_string(request)?, + }) + } + + /// This request as the API reports it back when it could not process it. + fn to_unprocessed(&self) -> UnprocessedRequest { + UnprocessedRequest { + unique_key: self.dedup_key.clone(), + url: self.url.clone(), + method: self.method.clone(), + } + } } -/// Returns the JSON-serialized byte length of `value`. -fn json_byte_len(value: &T) -> ApifyClientResult { - Ok(serde_json::to_vec(value)?.len()) +/// Joins already-serialized requests into the JSON array a batch add sends. +fn batch_body(requests: &[SerializedRequest]) -> Vec { + let json: Vec<&str> = requests.iter().map(|r| r.json.as_str()).collect(); + format!("[{}]", json.join(",")).into_bytes() } -/// Slices `requests` down to a byte-limited prefix, mirroring the reference client's -/// `sliceArrayByByteLength`: if the whole slice already fits under `max_bytes` it is returned -/// unchanged; otherwise items are accumulated one at a time until the next one would exceed the -/// budget. `start_index` is only used to name the offending item in the error message, so it -/// should be the slice's absolute position within the caller's full input. +/// Returns how many of the leading `requests` fit in one batch under `max_bytes`, counting the +/// brackets and commas of the JSON array: all of them if the whole slice already fits, otherwise +/// items are accumulated one at a time until the next one would exceed the budget. `start_index` +/// is only used to name the offending item in the error message, so it should be the slice's +/// absolute position within the caller's full input. /// /// The first item is always included regardless of size (once its own size has been checked /// against `max_bytes`), guaranteeing a non-empty result for a non-empty input — unlike the @@ -91,18 +132,21 @@ fn json_byte_len(value: &T) -> ApifyClientResult { /// /// Returns [`ApifyClientError::InvalidArgument`] if a single request's JSON exceeds `max_bytes` /// on its own (mirroring the reference client's thrown error). -fn slice_requests_by_byte_length( - requests: &[RequestQueueRequest], +fn count_requests_fitting_byte_length( + requests: &[SerializedRequest], max_bytes: usize, start_index: usize, -) -> ApifyClientResult> { - if json_byte_len(requests)? < max_bytes { - return Ok(requests.to_vec()); - } - let mut out = Vec::new(); - let mut byte_length = 2usize; // 2 bytes for the empty array `[]`. +) -> ApifyClientResult { + let whole_bytes = ARRAY_BRACKETS_BYTES + + requests.iter().map(|r| r.json.len()).sum::() + + requests.len().saturating_sub(1); + if whole_bytes < max_bytes { + return Ok(requests.len()); + } + let mut taken = 0; + let mut byte_length = ARRAY_BRACKETS_BYTES; for (offset, request) in requests.iter().enumerate() { - let item_bytes = json_byte_len(request)?; + let item_bytes = request.json.len(); if item_bytes > max_bytes { return Err(ApifyClientError::InvalidArgument(format!( "RequestQueueClient::batch_add_requests: the request at index {} exceeds the \ @@ -110,31 +154,35 @@ fn slice_requests_by_byte_length( start_index + offset ))); } - if !out.is_empty() && byte_length + item_bytes >= max_bytes { + let added_bytes = item_bytes + usize::from(taken > 0); + if taken > 0 && byte_length + added_bytes >= max_bytes { break; } - byte_length += item_bytes; - out.push(request.clone()); + byte_length += added_bytes; + taken += 1; } - Ok(out) + Ok(taken) } /// Splits `requests` into chunks that each satisfy both the per-call count limit /// ([`MAX_REQUESTS_PER_BATCH_OPERATION`]) and the payload byte-size limit /// (`max_bytes`), mirroring the reference client's chunking loop in `batchAddRequests`. fn chunk_requests_for_batch_add( - requests: &[RequestQueueRequest], + requests: Vec, max_bytes: usize, -) -> ApifyClientResult>> { +) -> ApifyClientResult>> { let mut chunks = Vec::new(); - let mut i = 0; - while i < requests.len() { - let group_end = (i + MAX_REQUESTS_PER_BATCH_OPERATION).min(requests.len()); - let chunk = slice_requests_by_byte_length(&requests[i..group_end], max_bytes, i)?; - // `slice_requests_by_byte_length` always returns at least one item for a non-empty + let mut start = 0; + let mut remaining = requests; + while !remaining.is_empty() { + let group_len = MAX_REQUESTS_PER_BATCH_OPERATION.min(remaining.len()); + // `count_requests_fitting_byte_length` always returns at least one item for a non-empty // input, so this advances on every iteration. - i += chunk.len(); - chunks.push(chunk); + let take = count_requests_fitting_byte_length(&remaining[..group_len], max_bytes, start)?; + let rest = remaining.split_off(take); + start += take; + chunks.push(remaining); + remaining = rest; } Ok(chunks) } @@ -146,8 +194,6 @@ fn chunk_requests_for_batch_add( pub struct ListRequestsOptions { /// Maximum number of requests to return. pub limit: Option, - /// Start listing after this request ID (exclusive). - pub exclusive_start_id: Option, /// Opaque pagination cursor returned by a previous call. pub cursor: Option, /// Restrict the returned requests to the given states. The spec defines this as an array of @@ -163,6 +209,8 @@ pub struct RequestQueueClient { client_key: Option, } +crate::clients::base::impl_with_timeout!(RequestQueueClient); + impl RequestQueueClient { pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self { Self { @@ -193,24 +241,24 @@ impl RequestQueueClient { /// Fetches the queue metadata, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the queue metadata (e.g. `name`, `title`). pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the queue. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Lists requests from the head of the queue (without locking them). pub async fn list_head(&self, limit: Option) -> ApifyClientResult { let mut params = self.base_params(); params.add_int("limit", limit); - get_resource_required(&self.ctx, Some("head"), ¶ms).await + get_resource_required(&self.ctx, Some("head"), ¶ms, Timeout::Short).await } /// Adds a single request to the queue. If `forefront` is true, adds it to the front. @@ -228,18 +276,24 @@ impl RequestQueueClient { ¶ms, Some(body), "application/json", + self.ctx.timeout(Timeout::Short), ) .await } /// Gets a request by ID, or `None` if it does not exist. pub async fn get_request(&self, id: &str) -> ApifyClientResult> { - get_resource( - &self.ctx, - Some(&format!("requests/{}", encode_path_segment(id))), - &self.base_params(), + // Unlike the other `get`s, a missing request is `None` even on a queue chained from a run: + // the request ID is part of the path, so a `404` means that request does not exist. + catch_not_found( + get_resource_required( + &self.ctx, + Some(&request_path(id)?), + &self.base_params(), + Timeout::Short, + ) + .await, ) - .await } /// Updates a request (which must include its `id`). @@ -255,24 +309,19 @@ impl RequestQueueClient { })?; let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}", encode_path_segment(&id)))), - ); + let url = params.apply_to_url(&self.ctx.url(Some(&request_path(&id)?))); let body = serde_json::to_vec(request)?; let mut headers = std::collections::HashMap::new(); headers.insert("Content-Type".to_string(), "application/json".to_string()); let response = self .ctx - .http - .call(HttpRequest { - method: HttpMethod::Put, + .send_with( + HttpMethod::Put, url, headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + Some(body), + self.ctx.timeout(Timeout::Medium), + ) .await?; crate::common::parse_data_envelope(&response.body) } @@ -280,20 +329,9 @@ impl RequestQueueClient { /// Deletes a request by ID. pub async fn delete_request(&self, id: &str) -> ApifyClientResult<()> { let params = self.base_params(); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}", encode_path_segment(id)))), - ); + let url = params.apply_to_url(&self.ctx.url(Some(&request_path(id)?))); self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + .send(HttpMethod::Delete, url, self.ctx.timeout(Timeout::Short)) .await?; Ok(()) } @@ -308,7 +346,15 @@ impl RequestQueueClient { params .add_int("lockSecs", Some(lock_secs)) .add_int("limit", limit); - post_action(&self.ctx, Some("head/lock"), ¶ms, None, None).await + post_action( + &self.ctx, + Some("head/lock"), + ¶ms, + None, + None, + self.ctx.timeout(Timeout::Medium), + ) + .await } /// Adds multiple requests to the queue in a single logical operation. @@ -348,7 +394,13 @@ impl RequestQueueClient { .max(1); let payload_limit_bytes = MAX_PAYLOAD_SIZE_BYTES - (MAX_PAYLOAD_SIZE_BYTES as f64 * SAFETY_BUFFER_PERCENT).ceil() as usize; - let chunks = chunk_requests_for_batch_add(requests, payload_limit_bytes)?; + // Each request is serialized exactly once, here, before anything is sent: a request too + // large for the payload limit is rejected before any batch is in flight. + let serialized = requests + .iter() + .map(SerializedRequest::new) + .collect::>>()?; + let chunks = chunk_requests_for_batch_add(serialized, payload_limit_bytes)?; let merged = stream::iter(chunks) .map(|chunk| { @@ -383,7 +435,7 @@ impl RequestQueueClient { /// so a single bad chunk cannot fail the whole (possibly-parallel) `batch_add_requests` call. async fn batch_add_requests_chunk_with_retries( &self, - chunk: Vec, + chunk: Vec, options: BatchAddRequestsOptions, ) -> BatchRequestsOperationResult { let max_retries = options @@ -407,7 +459,7 @@ impl RequestQueueClient { .iter() .filter_map(|p| p.unique_key.as_deref()) .collect(); - remaining.retain(|r| !processed_keys.contains(dedup_key(r))); + remaining.retain(|r| !processed_keys.contains(r.dedup_key.as_str())); processed.extend(result.processed_requests); if remaining.is_empty() { return BatchRequestsOperationResult { @@ -430,11 +482,7 @@ impl RequestQueueClient { processed_requests: processed, unprocessed_requests: remaining .iter() - .map(|r| UnprocessedRequest { - unique_key: dedup_key(r).to_string(), - url: r.url.clone(), - method: r.method.clone(), - }) + .map(SerializedRequest::to_unprocessed) .collect(), }; } @@ -453,11 +501,7 @@ impl RequestQueueClient { processed_requests: processed, unprocessed_requests: remaining .iter() - .map(|r| UnprocessedRequest { - unique_key: dedup_key(r).to_string(), - url: r.url.clone(), - method: r.method.clone(), - }) + .map(SerializedRequest::to_unprocessed) .collect(), } } @@ -466,18 +510,18 @@ impl RequestQueueClient { /// requests, and within the byte-size budget already enforced by the caller). async fn batch_add_requests_raw( &self, - requests: &[RequestQueueRequest], + requests: &[SerializedRequest], forefront: bool, ) -> ApifyClientResult { let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let body = serde_json::to_vec(requests)?; post_with_body( &self.ctx, Some("requests/batch"), ¶ms, - Some(body), + Some(batch_body(requests)), "application/json", + self.ctx.timeout(Timeout::Medium), ) .await } @@ -503,14 +547,15 @@ impl RequestQueueClient { Some("requests/batch"), &self.base_params(), &requests, + Timeout::Short, ) .await } /// Lists requests in the queue. /// - /// Supports pagination via `limit`/`exclusive_start_id` and the spec's `cursor`/`filter` - /// parameters (see [`ListRequestsOptions`]). + /// Supports pagination via `limit` and `cursor`, and the spec's `filter` parameter (see + /// [`ListRequestsOptions`]). pub async fn list_requests( &self, options: ListRequestsOptions, @@ -518,10 +563,9 @@ impl RequestQueueClient { let mut params = self.base_params(); params .add_int("limit", options.limit) - .add_str("exclusiveStartId", options.exclusive_start_id) .add_str("cursor", options.cursor) .add_csv("filter", options.filter.as_deref()); - get_resource_required(&self.ctx, Some("requests"), ¶ms).await + get_resource_required(&self.ctx, Some("requests"), ¶ms, Timeout::Medium).await } /// Prolongs the lock on a request for another `lock_secs` seconds. @@ -538,21 +582,10 @@ impl RequestQueueClient { params .add_int("lockSecs", Some(lock_secs)) .add_bool("forefront", Some(forefront)); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}/lock", encode_path_segment(id)))), - ); + let url = params.apply_to_url(&self.ctx.url(Some(&format!("{}/lock", request_path(id)?)))); let response = self .ctx - .http - .call(HttpRequest { - method: HttpMethod::Put, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::MEDIUM_REQUEST_TIMEOUT, - }) + .send(HttpMethod::Put, url, self.ctx.timeout(Timeout::Medium)) .await?; crate::common::parse_data_envelope(&response.body) } @@ -563,20 +596,9 @@ impl RequestQueueClient { pub async fn delete_request_lock(&self, id: &str, forefront: bool) -> ApifyClientResult<()> { let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}/lock", encode_path_segment(id)))), - ); + let url = params.apply_to_url(&self.ctx.url(Some(&format!("{}/lock", request_path(id)?)))); self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::SMALL_REQUEST_TIMEOUT, - }) + .send(HttpMethod::Delete, url, self.ctx.timeout(Timeout::Short)) .await?; Ok(()) } @@ -584,9 +606,8 @@ impl RequestQueueClient { /// Lazily paginates over all requests in the queue, fetching pages on demand. /// /// Returns a [`RequestQueueRequestsIterator`]; call its `next()` to get one request at a - /// time. Pagination uses the API's opaque `nextCursor` token: the first page may be - /// anchored with `exclusiveStartId`, but every subsequent page is fetched with `cursor` - /// (matching the JS reference). `cursor` and `exclusiveStartId` are mutually exclusive. + /// time. Pagination uses the API's opaque `nextCursor` token: every page after the first is + /// fetched with `cursor` (matching the JS reference). pub fn paginate_requests(&self, page_limit: Option) -> RequestQueueRequestsIterator { RequestQueueRequestsIterator { client: self.clone(), @@ -605,6 +626,7 @@ impl RequestQueueClient { &self.base_params(), None, None, + self.ctx.timeout(Timeout::Long), ) .await } @@ -634,10 +656,8 @@ impl RequestQueueRequestsIterator { return Ok(None); } - // The first page may be anchored by exclusiveStartId; every later page is fetched - // with the opaque `cursor` token (mutually exclusive with exclusiveStartId), matching - // the JS reference. Here we only ever paginate from the queue head, so the first page - // uses neither and subsequent pages use `cursor`. + // Every page after the first is fetched with the opaque `cursor` token, matching the JS + // reference. Here we only ever paginate from the queue head, so the first page has none. let page = self .client .list_requests(ListRequestsOptions { @@ -666,8 +686,8 @@ impl RequestQueueRequestsIterator { #[cfg(test)] mod batch_add_tests { use super::{ - chunk_requests_for_batch_add, dedup_key, slice_requests_by_byte_length, - MAX_REQUESTS_PER_BATCH_OPERATION, + batch_body, chunk_requests_for_batch_add, count_requests_fitting_byte_length, + SerializedRequest, MAX_REQUESTS_PER_BATCH_OPERATION, }; use crate::models::RequestQueueRequest; @@ -682,25 +702,50 @@ mod batch_add_tests { } } + fn serialized(requests: &[RequestQueueRequest]) -> Vec { + requests + .iter() + .map(|r| SerializedRequest::new(r).unwrap()) + .collect() + } + /// A request without an explicit `unique_key` is correlated by `url`, matching the API's own /// deduplication fallback. #[test] fn dedup_key_falls_back_to_url() { - let with_key = request("https://example.com", Some("k1")); - assert_eq!(dedup_key(&with_key), "k1"); + let with_key = SerializedRequest::new(&request("https://example.com", Some("k1"))).unwrap(); + assert_eq!(with_key.dedup_key, "k1"); - let without_key = request("https://example.com/no-key", None); - assert_eq!(dedup_key(&without_key), "https://example.com/no-key"); + let without_key = + SerializedRequest::new(&request("https://example.com/no-key", None)).unwrap(); + assert_eq!(without_key.dedup_key, "https://example.com/no-key"); } - /// A slice that already fits under the byte budget is returned unchanged. + /// The batch body is the JSON array of the strings serialized up front, so it parses back to + /// the original requests. + #[test] + fn batch_body_is_the_json_array_of_the_serialized_requests() { + let requests = vec![ + request("https://example.com/1", Some("a")), + request("https://example.com/2", None), + ]; + let body = batch_body(&serialized(&requests)); + let parsed: Vec = serde_json::from_slice(&body).unwrap(); + assert_eq!(parsed.len(), 2); + assert_eq!(parsed[0].unique_key.as_deref(), Some("a")); + assert_eq!(parsed[1].url, "https://example.com/2"); + assert_eq!(body, serde_json::to_vec(&requests).unwrap()); + } + + /// A slice that already fits under the byte budget is taken whole. #[test] fn byte_slice_returns_everything_when_under_budget() { let requests: Vec<_> = (0..5) .map(|i| request(&format!("https://example.com/{i}"), None)) .collect(); - let sliced = slice_requests_by_byte_length(&requests, 1_000_000, 0).unwrap(); - assert_eq!(sliced.len(), 5); + let taken = + count_requests_fitting_byte_length(&serialized(&requests), 1_000_000, 0).unwrap(); + assert_eq!(taken, 5); } /// When the whole slice exceeds the byte budget, only a byte-limited prefix is taken — but @@ -712,21 +757,37 @@ mod batch_add_tests { .collect(); // Each request serializes to roughly 30 bytes; a budget of 50 fits one comfortably but // never two. - let sliced = slice_requests_by_byte_length(&requests, 50, 0).unwrap(); + let taken = count_requests_fitting_byte_length(&serialized(&requests), 50, 0).unwrap(); assert_eq!( - sliced.len(), - 1, + taken, 1, "budget of 50 bytes should admit exactly one ~30-byte request" ); } + /// The brackets and commas of the JSON array count against the budget, so the batch body that + /// is actually sent never reaches the limit. + #[test] + fn byte_slice_counts_brackets_and_commas() { + let requests: Vec<_> = (0..4) + .map(|i| request(&format!("https://example.com/{i}"), None)) + .collect(); + let entries = serialized(&requests); + let item_bytes = entries[0].json.len(); + // Three items need 2 brackets + 2 commas on top of their own bytes, which leaves a + // body of exactly `limit` bytes: not below it, so only two items are taken. + let limit = 3 * item_bytes + 4; + let taken = count_requests_fitting_byte_length(&entries, limit, 0).unwrap(); + assert_eq!(taken, 2); + assert!(batch_body(&entries[..taken]).len() < limit); + } + /// A single request whose own JSON exceeds the byte budget is a hard error, not a silently /// dropped item — the caller could never send it in any chunk. #[test] fn byte_slice_errors_on_oversized_single_request() { let huge_url = format!("https://example.com/{}", "x".repeat(1000)); let requests = vec![request(&huge_url, None)]; - let err = slice_requests_by_byte_length(&requests, 100, 3).unwrap_err(); + let err = count_requests_fitting_byte_length(&serialized(&requests), 100, 3).unwrap_err(); let message = err.to_string(); assert!( message.contains("index 3"), @@ -741,7 +802,7 @@ mod batch_add_tests { let requests: Vec<_> = (0..(MAX_REQUESTS_PER_BATCH_OPERATION * 2 + 3)) .map(|i| request(&format!("https://example.com/{i}"), None)) .collect(); - let chunks = chunk_requests_for_batch_add(&requests, 10_000_000).unwrap(); + let chunks = chunk_requests_for_batch_add(serialized(&requests), 10_000_000).unwrap(); let sizes: Vec = chunks.iter().map(Vec::len).collect(); assert_eq!( sizes, @@ -763,7 +824,7 @@ mod batch_add_tests { .map(|i| request(&format!("https://example.com/{i}"), None)) .collect(); // ~30 bytes/request; a 100-byte budget forces multiple chunks well under the 25-item cap. - let chunks = chunk_requests_for_batch_add(&requests, 100).unwrap(); + let chunks = chunk_requests_for_batch_add(serialized(&requests), 100).unwrap(); assert!( chunks.len() > 1, "a tight byte budget must force more than one chunk, got {}", @@ -775,5 +836,8 @@ mod batch_add_tests { requests.len(), "every request must end up in exactly one chunk" ); + for chunk in &chunks { + assert!(batch_body(chunk).len() < 100 || chunk.len() == 1); + } } } diff --git a/src/clients/request_queue_collection.rs b/src/clients/request_queue_collection.rs index 360faa5..c1dc0d5 100644 --- a/src/clients/request_queue_collection.rs +++ b/src/clients/request_queue_collection.rs @@ -6,6 +6,7 @@ use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::RequestQueue; +use crate::timeouts::Timeout; /// Client for listing request queues and getting-or-creating one by name. #[derive(Debug, Clone)] @@ -13,6 +14,8 @@ pub struct RequestQueueCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(RequestQueueCollectionClient); + impl RequestQueueCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -28,7 +31,7 @@ impl RequestQueueCollectionClient { ) -> ApifyClientResult> { let mut params = QueryParams::new(); options.apply(&mut params); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all request queues matching `options`, fetching pages on demand. @@ -43,6 +46,6 @@ impl RequestQueueCollectionClient { /// Gets the queue with the given `name`, creating it if it does not exist. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { - get_or_create_named(&self.ctx, name).await + get_or_create_named(&self.ctx, name, Timeout::Short).await } } diff --git a/src/clients/run.rs b/src/clients/run.rs index 55e09df..9242a0d 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -14,6 +14,7 @@ use crate::common::{to_safe_id, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorRun; +use crate::timeouts::Timeout; /// Header the API uses to deduplicate charge requests (matching the reference client). const CHARGE_IDEMPOTENCY_HEADER: &str = "idempotency-key"; @@ -43,8 +44,9 @@ pub struct RunResurrectOptions { pub build: Option, /// Memory in megabytes. pub memory_mbytes: Option, - /// Timeout in seconds. - pub timeout_secs: Option, + /// Timeout of the run itself in seconds. Not the timeout of the API request, which is set with + /// the client's `with_timeout`. + pub run_timeout_secs: Option, /// Maximum number of dataset items to charge (pay-per-result Actors). pub max_items: Option, /// Maximum total cost of the run in USD, across all pricing models. @@ -94,6 +96,8 @@ pub struct RunClient { id: String, } +crate::clients::base::impl_with_timeout!(RunClient); + impl RunClient { pub(crate) fn new( _root: crate::client::ApifyClient, @@ -118,17 +122,17 @@ impl RunClient { /// Fetches the run object, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the run (e.g. its status message) and returns the updated object. pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the run. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Aborts the run. `gracefully` is optional, matching the reference client's optional @@ -138,7 +142,15 @@ impl RunClient { pub async fn abort(&self, gracefully: Option) -> ApifyClientResult { let mut params = QueryParams::new(); params.add_bool("gracefully", gracefully); - post_action(&self.ctx, Some("abort"), ¶ms, None, None).await + post_action( + &self.ctx, + Some("abort"), + ¶ms, + None, + None, + self.ctx.timeout(Timeout::Medium), + ) + .await } /// Transforms the run into a run of another Actor (metamorph). @@ -199,12 +211,28 @@ impl RunClient { .as_deref() .unwrap_or(default_content_type) .to_string(); - post_with_body(&self.ctx, Some("metamorph"), ¶ms, body, &content_type).await + post_with_body( + &self.ctx, + Some("metamorph"), + ¶ms, + body, + &content_type, + self.ctx.timeout(Timeout::Medium), + ) + .await } /// Reboots the run (restarts its container, preserving the run ID and storages). pub async fn reboot(&self) -> ApifyClientResult { - post_action(&self.ctx, Some("reboot"), &QueryParams::new(), None, None).await + post_action( + &self.ctx, + Some("reboot"), + &QueryParams::new(), + None, + None, + self.ctx.timeout(Timeout::Medium), + ) + .await } /// Resurrects a finished run, starting it again with (optionally overridden) settings. @@ -219,11 +247,19 @@ impl RunClient { params .add_str("build", options.build) .add_int("memory", options.memory_mbytes) - .add_int("timeout", options.timeout_secs) + .add_int("timeout", options.run_timeout_secs) .add_int("maxItems", options.max_items) .add_float("maxTotalChargeUsd", options.max_total_charge_usd) .add_bool("restartOnError", options.restart_on_error); - post_action(&self.ctx, Some("resurrect"), ¶ms, None, None).await + post_action( + &self.ctx, + Some("resurrect"), + ¶ms, + None, + None, + self.ctx.timeout(Timeout::Medium), + ) + .await } /// Charges the run for a pay-per-event run, recording occurrences of a named event. @@ -251,14 +287,13 @@ impl RunClient { // A successful `HttpClient::call` already guarantees a 2xx status; the (empty) body // is intentionally ignored rather than parsed as a `data` envelope. self.ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Post, + .send_with( + crate::http_client::HttpMethod::Post, url, headers, - body: Some(body_bytes), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + Some(body_bytes), + self.ctx.timeout(Timeout::Short), + ) .await?; Ok(()) } @@ -316,11 +351,12 @@ impl RunClient { /// /// Convenience equivalent to `run.log().stream()` (mirrors the reference client's /// `getStreamedLog`): yields log chunks as they arrive, so callers can forward them to - /// their own logger/stdout while the run is in progress. + /// their own logger/stdout while the run is in progress. A run that does not exist is a + /// [`NotFound`](crate::ApiError::is_not_found) error. pub async fn get_streamed_log( &self, ) -> ApifyClientResult>>> { - self.log().stream().await + self.log().stream_required(Default::default()).await } /// Opens a live stream of the run's log for redirection, applying the given @@ -335,6 +371,6 @@ impl RunClient { &self, options: crate::clients::log::LogOptions, ) -> ApifyClientResult>>> { - self.log().stream_with_options(options).await + self.log().stream_required(options).await } } diff --git a/src/clients/run_collection.rs b/src/clients/run_collection.rs index fccc0ec..a7129ee 100644 --- a/src/clients/run_collection.rs +++ b/src/clients/run_collection.rs @@ -6,6 +6,7 @@ use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorRun; +use crate::timeouts::Timeout; /// Filtering options for [`RunCollectionClient::list`], covering the spec query parameters of /// `GET /v2/actor-runs` and `GET /v2/actors/{actorId}/runs`. @@ -24,6 +25,12 @@ pub struct RunListOptions { pub started_after: Option, /// Only return runs started at or before this ISO 8601 timestamp. (Actor-scoped lists only.) pub started_before: Option, + /// Only return Actor Standby runs served by one of these deployment pools + /// (`SINGLE_TENANT`, `MULTI_TENANT_FREE_USERS`, `MULTI_TENANT_PAYING_USERS` or + /// `MULTI_TENANT_OWNER_PAID`); sent as a comma-separated list. Requires `status` to contain + /// only non-terminal statuses (`READY`, `RUNNING`, `TIMING-OUT` or `ABORTING`). Empty means no + /// deployment filter. (Account and Actor scoped lists only.) + pub deployment: Vec, } /// Client for listing Actor runs. @@ -32,6 +39,8 @@ pub struct RunCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(RunCollectionClient); + impl RunCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str) -> Self { Self { @@ -75,8 +84,9 @@ impl RunCollectionClient { .add_bool("desc", options.desc) .add_csv("status", Some(&filter.status)) .add_str("startedAfter", filter.started_after) - .add_str("startedBefore", filter.started_before); - list_resource(&self.ctx, None, ¶ms).await + .add_str("startedBefore", filter.started_before) + .add_csv("deployment", Some(&filter.deployment)); + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all runs matching `options`/`filter`, fetching pages on demand. diff --git a/src/clients/schedule.rs b/src/clients/schedule.rs index b03a057..36388a0 100644 --- a/src/clients/schedule.rs +++ b/src/clients/schedule.rs @@ -3,12 +3,13 @@ use serde::Serialize; use crate::clients::base::{ - delete_resource, get_raw, get_resource, update_resource, ResourceContext, + delete_resource, get_raw_required, get_resource, update_resource, ResourceContext, }; use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Schedule; +use crate::timeouts::Timeout; /// Client for a specific schedule. #[derive(Debug, Clone)] @@ -16,6 +17,8 @@ pub struct ScheduleClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ScheduleClient); + impl ScheduleClient { pub(crate) fn new(http: HttpClient, base_url: &str, id: &str) -> Self { Self { @@ -25,25 +28,27 @@ impl ScheduleClient { /// Fetches the schedule, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the schedule with the given fields. pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the schedule. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } - /// Fetches the schedule's invocation log as text, or `None` if not available. - pub async fn get_log(&self) -> ApifyClientResult> { - let response = get_raw(&self.ctx, Some("log"), &QueryParams::new()).await?; - match response { - Some(r) => Ok(Some(String::from_utf8_lossy(&r.body).into_owned())), - None => Ok(None), - } + /// Fetches the schedule's invocation log as text. + /// + /// A schedule that does not exist is a [`NotFound`](crate::ApiError::is_not_found) error: + /// the log is a fixed sub-path of the schedule, so a `404` cannot be told apart from a + /// missing log. + pub async fn get_log(&self) -> ApifyClientResult { + let response = + get_raw_required(&self.ctx, Some("log"), &QueryParams::new(), Timeout::Medium).await?; + Ok(String::from_utf8_lossy(&response.body).into_owned()) } } diff --git a/src/clients/schedule_collection.rs b/src/clients/schedule_collection.rs index 722c12a..23e67bd 100644 --- a/src/clients/schedule_collection.rs +++ b/src/clients/schedule_collection.rs @@ -8,6 +8,7 @@ use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Schedule; +use crate::timeouts::Timeout; /// Client for listing and creating schedules. #[derive(Debug, Clone)] @@ -15,6 +16,8 @@ pub struct ScheduleCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(ScheduleCollectionClient); + impl ScheduleCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -29,7 +32,7 @@ impl ScheduleCollectionClient { .add_int("offset", options.offset) .add_int("limit", options.limit) .add_bool("desc", options.desc); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all schedules matching `options`, fetching pages on demand. @@ -44,6 +47,6 @@ impl ScheduleCollectionClient { /// Creates a new schedule from the given definition. pub async fn create(&self, schedule: &T) -> ApifyClientResult { - create_resource(&self.ctx, &QueryParams::new(), schedule).await + create_resource(&self.ctx, &QueryParams::new(), schedule, Timeout::Short).await } } diff --git a/src/clients/store_collection.rs b/src/clients/store_collection.rs index c62236c..13d3d9f 100644 --- a/src/clients/store_collection.rs +++ b/src/clients/store_collection.rs @@ -6,6 +6,7 @@ use crate::common::{PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorStoreListItem; +use crate::timeouts::Timeout; /// A lazy, page-fetching iterator over Apify Store Actors. /// @@ -40,6 +41,10 @@ pub struct StoreListOptions { pub allows_agentic_users: Option, /// Response format requested from the API (`full` or `agent`, the API default is `full`). pub response_format: Option, + /// If `true`, each Actor in the results includes an `inputSchema` field with a compact version + /// of the input schema of its default build (property types and required properties only). + /// With this option, the API defaults `limit` to `10` and does not accept a larger one. + pub include_input_schema: Option, } /// Client for the Apify Store. @@ -48,6 +53,8 @@ pub struct StoreCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(StoreCollectionClient); + impl StoreCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -64,7 +71,7 @@ impl StoreCollectionClient { options: StoreListOptions, ) -> ApifyClientResult> { let params = self.build_params(&options); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates all Store Actors matching `options`, fetching pages on demand. @@ -92,7 +99,8 @@ impl StoreCollectionClient { .add_str("pricingModel", options.pricing_model.clone()) .add_bool("includeUnrunnableActors", options.include_unrunnable_actors) .add_bool("allowsAgenticUsers", options.allows_agentic_users) - .add_str("responseFormat", options.response_format.clone()); + .add_str("responseFormat", options.response_format.clone()) + .add_bool("includeInputSchema", options.include_input_schema); params } } diff --git a/src/clients/task.rs b/src/clients/task.rs index 99a38ca..dfa0c84 100644 --- a/src/clients/task.rs +++ b/src/clients/task.rs @@ -15,6 +15,7 @@ use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::{ActorRun, Task}; +use crate::timeouts::Timeout; /// Client for a specific Actor task. #[derive(Debug, Clone)] @@ -23,6 +24,8 @@ pub struct TaskClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(TaskClient); + impl TaskClient { pub(crate) fn new(root: ApifyClient, http: HttpClient, base_url: &str, id: &str) -> Self { Self { @@ -33,17 +36,17 @@ impl TaskClient { /// Fetches the task object, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the task with the given fields. pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the task. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Publishes the task on its public landing page, by setting `isPublic: true` through @@ -84,6 +87,9 @@ impl TaskClient { Some(value) => Some(serde_json::to_vec(value)?), None => None, }; + let timeout = self + .ctx + .timeout_with_wait_hold(Timeout::Medium, options.wait_for_finish); crate::clients::base::retry_while_resource_limited(options.wait_for_resources, || { post_with_body( &self.ctx, @@ -91,6 +97,7 @@ impl TaskClient { ¶ms, body.clone(), "application/json", + timeout, ) }) .await @@ -116,14 +123,20 @@ impl TaskClient { self.root.run(run.id).wait_for_finish(wait_secs).await } - /// Fetches the task's saved input, or `None` if not set. - pub async fn get_input(&self) -> ApifyClientResult> { - let response = - crate::clients::base::get_raw(&self.ctx, Some("input"), &QueryParams::new()).await?; - match response { - Some(r) => Ok(Some(serde_json::from_slice(&r.body)?)), - None => Ok(None), - } + /// Fetches the task's saved input. + /// + /// A task that does not exist is a [`NotFound`](crate::ApiError::is_not_found) error: the + /// input is a fixed sub-path of the task, so a `404` cannot be told apart from a missing + /// input. + pub async fn get_input(&self) -> ApifyClientResult { + let response = crate::clients::base::get_raw_required( + &self.ctx, + Some("input"), + &QueryParams::new(), + Timeout::Short, + ) + .await?; + Ok(serde_json::from_slice(&response.body)?) } /// Updates the task's saved input. @@ -134,14 +147,13 @@ impl TaskClient { headers.insert("Content-Type".to_string(), "application/json".to_string()); let response = self .ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Put, + .send_with( + crate::http_client::HttpMethod::Put, url, headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + Some(body), + self.ctx.timeout(Timeout::Short), + ) .await?; Ok(serde_json::from_slice(&response.body)?) } diff --git a/src/clients/task_collection.rs b/src/clients/task_collection.rs index 796b460..b00f0ca 100644 --- a/src/clients/task_collection.rs +++ b/src/clients/task_collection.rs @@ -8,6 +8,7 @@ use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Task; +use crate::timeouts::Timeout; /// Client for listing and creating Actor tasks. #[derive(Debug, Clone)] @@ -15,6 +16,8 @@ pub struct TaskCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(TaskCollectionClient); + impl TaskCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -29,7 +32,7 @@ impl TaskCollectionClient { .add_int("offset", options.offset) .add_int("limit", options.limit) .add_bool("desc", options.desc); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all tasks matching `options`, fetching pages on demand. @@ -44,6 +47,6 @@ impl TaskCollectionClient { /// Creates a new task from the given definition. pub async fn create(&self, task: &T) -> ApifyClientResult { - create_resource(&self.ctx, &QueryParams::new(), task).await + create_resource(&self.ctx, &QueryParams::new(), task, Timeout::Medium).await } } diff --git a/src/clients/user.rs b/src/clients/user.rs index a7e2075..fcf4a64 100644 --- a/src/clients/user.rs +++ b/src/clients/user.rs @@ -6,8 +6,9 @@ use serde_json::Value; use crate::clients::base::{get_resource, get_resource_required, ResourceContext}; use crate::common::QueryParams; use crate::error::ApifyClientResult; -use crate::http_client::HttpClient; +use crate::http_client::{HttpClient, HttpMethod}; use crate::models::User; +use crate::timeouts::Timeout; /// Client for a specific user (or the current user via [`ApifyClient::me`]). /// @@ -15,89 +16,76 @@ use crate::models::User; #[derive(Debug, Clone)] pub struct UserClient { ctx: ResourceContext, - is_me: bool, } +crate::clients::base::impl_with_timeout!(UserClient); + impl UserClient { pub(crate) fn new(http: HttpClient, base_url: &str, id: &str) -> Self { Self { ctx: ResourceContext::single(http, base_url, "users", id), - is_me: id == crate::client::ME_USER_PLACEHOLDER, } } - /// Fetches the user account information. + /// Fetches the user account information, or `None` if the user does not exist. /// /// For the current user (`me`) this returns private account details; for other users /// it returns the public profile. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } - /// Returns the current user's monthly usage for the current month. Only valid for the `me` - /// client. To fetch usage for a specific month, use [`UserClient::monthly_usage_for_date`]. + /// Returns the user's monthly usage for the current month. To fetch usage for a specific + /// month, use [`UserClient::monthly_usage_for_date`]. + /// + /// For a user other than the current one, only your own account and the organization your API + /// token belongs to can be accessed. pub async fn monthly_usage(&self) -> ApifyClientResult { - self.monthly_usage_for_date_named(None, "monthly_usage") - .await + self.monthly_usage_for_date(None).await } - /// Returns the current user's monthly usage, optionally for the month containing `date`. + /// Returns the user's monthly usage, optionally for the month containing `date`. /// /// `date` is an optional `YYYY-MM-DD` string selecting the month to report (the spec's - /// optional `date` query parameter on `GET /v2/users/me/usage/monthly`); passing `None` - /// returns the current month, which is equivalent to [`UserClient::monthly_usage`]. Only - /// valid for the `me` client. + /// optional `date` query parameter on `GET /v2/users/{userId}/usage/monthly`); passing `None` + /// returns the current month, which is equivalent to [`UserClient::monthly_usage`]. pub async fn monthly_usage_for_date(&self, date: Option<&str>) -> ApifyClientResult { - self.monthly_usage_for_date_named(date, "monthly_usage_for_date") - .await - } - - /// Shared implementation for [`UserClient::monthly_usage`] and - /// [`UserClient::monthly_usage_for_date`]. `method` is the caller's own public method name, - /// so the `me`-only guard error names the method the caller actually invoked. - async fn monthly_usage_for_date_named( - &self, - date: Option<&str>, - method: &str, - ) -> ApifyClientResult { - self.require_me(method)?; let mut params = QueryParams::new(); params.add_str("date", date); - get_resource_required(&self.ctx, Some("usage/monthly"), ¶ms).await + get_resource_required(&self.ctx, Some("usage/monthly"), ¶ms, Timeout::Short).await } - /// Returns the current user's account and usage limits. Only valid for the `me` client. + /// Returns the user's account and usage limits. + /// + /// For a user other than the current one, only your own account and the organization your API + /// token belongs to can be accessed. pub async fn limits(&self) -> ApifyClientResult { - self.require_me("limits")?; - get_resource_required(&self.ctx, Some("limits"), &QueryParams::new()).await + get_resource_required( + &self.ctx, + Some("limits"), + &QueryParams::new(), + Timeout::Short, + ) + .await } - /// Updates the current user's limits. Only valid for the `me` client. + /// Updates the user's limits. + /// + /// For a user other than the current one, only your own account and the organization your API + /// token belongs to can be accessed. pub async fn update_limits(&self, new_limits: &T) -> ApifyClientResult<()> { - self.require_me("update_limits")?; - let body = serde_json::to_vec(new_limits)?; - let url = self.ctx.url(Some("limits")); let mut headers = std::collections::HashMap::new(); headers.insert("Content-Type".to_string(), "application/json".to_string()); + // The endpoint's response body carries nothing, so it is ignored rather than parsed. self.ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Put, - url, + .send_with( + HttpMethod::Put, + self.ctx.url(Some("limits")), headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) + Some(serde_json::to_vec(new_limits)?), + self.ctx.timeout(Timeout::Short), + ) .await?; Ok(()) } - - fn require_me(&self, method: &str) -> ApifyClientResult<()> { - if !self.is_me { - return Err(crate::error::ApifyClientError::InvalidArgument(format!( - "`{method}` is only available for the current user (use ApifyClient::me())" - ))); - } - Ok(()) - } } diff --git a/src/clients/webhook.rs b/src/clients/webhook.rs index 1e090d5..86005eb 100644 --- a/src/clients/webhook.rs +++ b/src/clients/webhook.rs @@ -10,6 +10,7 @@ use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::{Webhook, WebhookDispatch}; +use crate::timeouts::Timeout; /// Client for a specific webhook. #[derive(Debug, Clone)] @@ -17,6 +18,8 @@ pub struct WebhookClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(WebhookClient); + impl WebhookClient { pub(crate) fn new(http: HttpClient, base_url: &str, id: &str) -> Self { Self { @@ -26,22 +29,30 @@ impl WebhookClient { /// Fetches the webhook, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } /// Updates the webhook with the given fields. pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource(&self.ctx, None, new_fields, Timeout::Short).await } /// Deletes the webhook. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource(&self.ctx, None, Timeout::Short).await } /// Tests the webhook by dispatching it immediately, returning the dispatch. pub async fn test(&self) -> ApifyClientResult { - post_action(&self.ctx, Some("test"), &QueryParams::new(), None, None).await + post_action( + &self.ctx, + Some("test"), + &QueryParams::new(), + None, + None, + self.ctx.timeout(Timeout::Medium), + ) + .await } /// Returns a client for this webhook's dispatch collection. diff --git a/src/clients/webhook_collection.rs b/src/clients/webhook_collection.rs index b4e4362..2bb8fc0 100644 --- a/src/clients/webhook_collection.rs +++ b/src/clients/webhook_collection.rs @@ -8,6 +8,7 @@ use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Webhook; +use crate::timeouts::Timeout; /// Client for listing and creating webhooks. #[derive(Debug, Clone)] @@ -15,6 +16,8 @@ pub struct WebhookCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(WebhookCollectionClient); + impl WebhookCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -36,7 +39,7 @@ impl WebhookCollectionClient { .add_int("offset", options.offset) .add_int("limit", options.limit) .add_bool("desc", options.desc); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all webhooks matching `options`, fetching pages on demand. @@ -51,6 +54,6 @@ impl WebhookCollectionClient { /// Creates a new webhook from the given definition. pub async fn create(&self, webhook: &T) -> ApifyClientResult { - create_resource(&self.ctx, &QueryParams::new(), webhook).await + create_resource(&self.ctx, &QueryParams::new(), webhook, Timeout::Short).await } } diff --git a/src/clients/webhook_dispatch.rs b/src/clients/webhook_dispatch.rs index 1b55fe6..e2a2fcc 100644 --- a/src/clients/webhook_dispatch.rs +++ b/src/clients/webhook_dispatch.rs @@ -5,6 +5,7 @@ use crate::common::QueryParams; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::WebhookDispatch; +use crate::timeouts::Timeout; /// Client for a specific webhook dispatch. #[derive(Debug, Clone)] @@ -12,6 +13,8 @@ pub struct WebhookDispatchClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(WebhookDispatchClient); + impl WebhookDispatchClient { pub(crate) fn new(http: HttpClient, base_url: &str, id: &str) -> Self { Self { @@ -21,6 +24,6 @@ impl WebhookDispatchClient { /// Fetches the webhook dispatch, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Short).await } } diff --git a/src/clients/webhook_dispatch_collection.rs b/src/clients/webhook_dispatch_collection.rs index 2ac21ab..85a6cf0 100644 --- a/src/clients/webhook_dispatch_collection.rs +++ b/src/clients/webhook_dispatch_collection.rs @@ -6,6 +6,7 @@ use crate::common::{ListOptions, PaginationList, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::WebhookDispatch; +use crate::timeouts::Timeout; /// Client for listing webhook dispatches. #[derive(Debug, Clone)] @@ -13,6 +14,8 @@ pub struct WebhookDispatchCollectionClient { ctx: ResourceContext, } +crate::clients::base::impl_with_timeout!(WebhookDispatchCollectionClient); + impl WebhookDispatchCollectionClient { pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { @@ -37,7 +40,7 @@ impl WebhookDispatchCollectionClient { .add_int("offset", options.offset) .add_int("limit", options.limit) .add_bool("desc", options.desc); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, ¶ms, Timeout::Medium).await } /// Lazily iterates over all webhook dispatches matching `options`, fetching pages on demand. diff --git a/src/common.rs b/src/common.rs index 8f92da7..eda213c 100644 --- a/src/common.rs +++ b/src/common.rs @@ -4,13 +4,11 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::error::ApifyClientResult; +use crate::error::{ApifyClientError, ApifyClientResult}; use crate::version::CLIENT_VERSION; /// Status code returned when a resource is not found. const NOT_FOUND_STATUS_CODE: u16 = 404; -const RECORD_NOT_FOUND_TYPE: &str = "record-not-found"; -const RECORD_OR_TOKEN_NOT_FOUND_TYPE: &str = "record-or-token-not-found"; /// Most Apify endpoints wrap their payload in a top-level `data` property. /// This envelope unwraps `{ "data": ... }` into the inner type. @@ -28,23 +26,13 @@ pub(crate) fn parse_data_envelope(body: &[u8]) -> ApifyClie /// Translates a "not found" API error into `Ok(None)`, re-raising any other error. /// /// This mirrors `catchNotFoundOrThrow` in the reference clients: a `get`/`delete` on a -/// missing resource resolves to `None` rather than raising. +/// missing resource resolves to `None` rather than raising, whatever the error `type` of the +/// `404` response. pub(crate) fn catch_not_found(result: ApifyClientResult) -> ApifyClientResult> { match result { Ok(value) => Ok(Some(value)), - Err(err) => { - if let Some(api_error) = err.as_api_error() { - let is_not_found_status = api_error.status_code == NOT_FOUND_STATUS_CODE; - let is_not_found_type = matches!( - api_error.error_type.as_deref(), - Some(RECORD_NOT_FOUND_TYPE) | Some(RECORD_OR_TOKEN_NOT_FOUND_TYPE) - ) || api_error.http_method.as_deref() == Some("HEAD"); - if is_not_found_status && is_not_found_type { - return Ok(None); - } - } - Err(err) - } + Err(err) if err.status_code() == Some(NOT_FOUND_STATUS_CODE) => Ok(None), + Err(err) => Err(err), } } @@ -155,12 +143,32 @@ fn url_encode(input: &str) -> String { /// Encodes everything except the RFC 3986 "unreserved" characters. Notably `/`, `?`, `#`, /// space and any non-ASCII bytes are escaped — unlike a raw `format!(".../{key}")`, which /// would leave them intact and produce a malformed or wrong-endpoint URL. +/// +/// This does not reject a segment that is empty or a dot segment (`.` or `..`), which a URL +/// parser resolves to another endpoint even when percent-encoded; the client methods that take +/// such a value validate it with `to_path_segment` instead. pub fn encode_path_segment(input: &str) -> String { // Path segments and query components share the same unreserved set here, so reuse the // query encoder. (`url_encode` already escapes `/`, `?`, `#`, space and non-ASCII.) url_encode(input) } +/// Validates and percent-encodes a caller-supplied URL path segment so it cannot restructure +/// the request path. +/// +/// An empty value is rejected because it would address the collection instead of a resource, and +/// so is `.` or `..`: URL parsers resolve dot segments after percent-decoding, so +/// `records/%2E%2E` collapses to the parent endpoint just like `records/..`. The error is +/// [`ApifyClientError::InvalidArgument`]. +pub(crate) fn to_path_segment(input: &str) -> ApifyClientResult { + if input.is_empty() || input == "." || input == ".." { + return Err(ApifyClientError::InvalidArgument(format!( + "URL path segment must be non-empty and must not be a dot segment, got `{input}`" + ))); + } + Ok(encode_path_segment(input)) +} + /// Standard offset/limit pagination options shared by most list endpoints. #[derive(Debug, Default, Clone)] pub struct ListOptions { @@ -293,9 +301,9 @@ fn map_os_to_node_platform(os: &str) -> &str { } /// Encodes a resource id so it is safe to embed in a URL path. Apify uses the -/// `username~resourcename` form, so the first `/` of an id is replaced with `~`. +/// `username~resourcename` form, so every `/` of an id is replaced with `~`. pub fn to_safe_id(id: &str) -> String { - id.replacen('/', "~", 1) + id.replace('/', "~") } /// Version tag embedded in storage-content signatures. Matches the upstream library default diff --git a/src/http_client.rs b/src/http_client.rs index 1055643..ab42709 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -18,6 +18,8 @@ use std::time::Duration; use async_trait::async_trait; use crate::error::{ApiError, ApiErrorBody, ApifyClientError, ApifyClientResult}; +use crate::http_compressors::{maybe_compress, HttpCompressor}; +use crate::timeouts::{Timeout, TimeoutTiers, NO_TIMEOUT}; /// HTTP status code returned by the API when the per-resource rate limit is exceeded. const RATE_LIMIT_EXCEEDED_STATUS_CODE: u16 = 429; @@ -29,102 +31,6 @@ const MAX_SUCCESS_STATUS_CODE: u16 = 300; /// Matches the reference client's `async-retry` default factor of 2. const BACKOFF_FACTOR: u32 = 2; -/// Request bodies at least this many bytes are compressed before sending. Smaller bodies are -/// left uncompressed because the CPU cost outweighs the transfer savings. Matches the reference -/// client's `MIN_COMPRESS_BYTES` threshold. -const MIN_COMPRESS_BYTES: usize = 1024; -/// `Content-Encoding` value used for brotli-compressed request bodies. -const CONTENT_ENCODING_BROTLI: &str = "br"; -/// `Content-Encoding` value used for gzip-compressed request bodies. -const CONTENT_ENCODING_GZIP: &str = "gzip"; -/// Brotli quality level (0–11). The reference client compresses request bodies at quality 6, -/// which balances ratio against CPU cost; we mirror that. -const BROTLI_QUALITY: u32 = 6; -/// Brotli sliding-window size (log2), 22 is the library default (a 4 MiB window). -const BROTLI_WINDOW_SIZE: u32 = 22; -/// Internal buffer size for the brotli encoder. -const BROTLI_BUFFER_SIZE: usize = 4096; -/// Gzip compression level (0–9). Matches the reference client, which gzips using `node:zlib`'s -/// default level (6). -const GZIP_COMPRESSION_LEVEL: u32 = 6; - -/// Media-type prefixes whose payloads already carry their own compression, so running them -/// through brotli/gzip burns CPU and memory for a result that is usually no smaller — and the -/// request also keeps the intended `Content-Type`, rather than becoming an unreadable -/// `Content-Encoding: br` blob the destination may not expect for these formats. -/// Matches the reference client's `ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES`. -const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES: [&str; 3] = ["audio/", "image/", "video/"]; - -/// Exact media types (outside the prefixes above) whose payloads already carry their own -/// compression. Matches the reference client's `ALREADY_COMPRESSED_MEDIA_TYPES`. -const ALREADY_COMPRESSED_MEDIA_TYPES: [&str; 19] = [ - "application/epub+zip", - "application/gzip", - "application/java-archive", - "application/vnd.android.package-archive", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.rar", - "application/x-7z-compressed", - "application/x-bzip", - "application/x-bzip2", - "application/x-gzip", - "application/x-rar-compressed", - "application/x-xz", - "application/x-zip-compressed", - "application/zip", - "application/zstd", - "font/woff", - "font/woff2", -]; - -/// Uncompressed media types that sit under an already-compressed prefix above, so compressing -/// them still pays off. Matches the reference client's `COMPRESSIBLE_MEDIA_TYPES`. -const COMPRESSIBLE_MEDIA_TYPES: [&str; 16] = [ - "audio/aiff", - "audio/basic", - "audio/l16", - "audio/l24", - "audio/midi", - "audio/vnd.wave", - "audio/wav", - "audio/wave", - "audio/x-aiff", - "audio/x-wav", - "image/bmp", - "image/tiff", - "image/vnd.adobe.photoshop", - "image/vnd.microsoft.icon", - "image/x-icon", - "image/x-ms-bmp", -]; - -/// Structured-syntax suffixes that mark a media type as text even under an already-compressed -/// prefix (e.g. `image/svg+xml`). Matches the reference client's -/// `COMPRESSIBLE_MEDIA_TYPE_SUFFIXES`. -const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES: [&str; 2] = ["+json", "+xml"]; - -/// Algorithm used to compress large request bodies before they are sent. -/// -/// The Apify API accepts both brotli (`br`) and gzip (`gzip`) request bodies. The reference JS -/// client picks between them automatically (brotli when available, gzip otherwise); this client -/// exposes the choice explicitly via -/// [`ApifyClientBuilder::request_compression`](crate::ApifyClientBuilder::request_compression), -/// because Rust's brotli support is always compiled in and would leave no runtime path to gzip. -/// [`Brotli`](RequestCompression::Brotli) is the default (best ratio). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -#[non_exhaustive] -pub enum RequestCompression { - /// Brotli (`Content-Encoding: br`). The default: best compression ratio, and the encoding the - /// reference client prefers. - #[default] - Brotli, - /// Gzip (`Content-Encoding: gzip`). Choose this for environments or intermediaries that do not - /// handle brotli. - Gzip, -} - /// HTTP method of a request. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HttpMethod { @@ -167,7 +73,9 @@ pub struct HttpRequest { pub headers: HashMap, /// Raw request body bytes (already serialized). pub body: Option>, - /// Per-request timeout. The backend should abort the request after this duration. + /// Per-request timeout. The backend should abort the request after this duration, unless it + /// is [`NO_TIMEOUT`]: such a request may run for as long as it + /// takes and the backend must not impose a timeout of its own. pub timeout: Duration, } @@ -285,10 +193,10 @@ impl HttpBackend for ReqwestBackend { async fn send(&self, request: HttpRequest) -> ApifyClientResult { let method = to_reqwest_method(request.method); - let mut builder = self - .client - .request(method, &request.url) - .timeout(request.timeout); + let mut builder = self.client.request(method, &request.url); + if request.timeout != NO_TIMEOUT { + builder = builder.timeout(request.timeout); + } for (key, value) in &request.headers { builder = builder.header(key, value); @@ -355,15 +263,26 @@ impl HttpBackend for ReqwestBackend { } } -/// Configuration for the retry/timeout behaviour of the [`HttpClient`]. +/// Configuration for the retry behaviour of the [`HttpClient`]. #[derive(Debug, Clone)] pub struct RetryConfig { /// Maximum number of *retries* (i.e. the request is attempted up to `max_retries + 1` times). pub max_retries: u32, /// Minimum delay between retries; doubled on each subsequent retry (exponential backoff). pub min_delay_between_retries: Duration, - /// Overall per-request timeout budget. Each attempt's timeout grows but is capped here. - pub timeout: Duration, +} + +/// Everything an [`HttpClient`] is built from. +pub(crate) struct HttpClientConfig { + pub(crate) backend: Arc, + pub(crate) token: Option, + pub(crate) user_agent: String, + pub(crate) retry: RetryConfig, + pub(crate) tiers: TimeoutTiers, + pub(crate) compressor: Arc, + /// Extra headers sent with every request. They win over the built-in `User-Agent` and + /// `Authorization` headers, and a request's own headers win over them. + pub(crate) default_headers: Vec<(String, String)>, } /// The orchestrating HTTP client shared by every resource client. @@ -377,48 +296,102 @@ pub struct HttpClient { token: Option, user_agent: String, retry: RetryConfig, - compression: RequestCompression, + tiers: TimeoutTiers, + compressor: Arc, + default_headers: Vec<(String, String)>, + /// Set when the resource client holding this handle was addressed with an invalid ID. Every + /// call then fails with [`ApifyClientError::InvalidArgument`] before reaching the backend. + /// The handle is cloned down to every client chained from that resource, so the failure + /// follows them. + poison: Option>, } impl HttpClient { - pub(crate) fn new( - backend: Arc, - token: Option, - user_agent: String, - retry: RetryConfig, - compression: RequestCompression, - ) -> Self { + pub(crate) fn new(config: HttpClientConfig) -> Self { Self { - backend, - token, - user_agent, - retry, - compression, + backend: config.backend, + token: config.token, + user_agent: config.user_agent, + retry: config.retry, + tiers: config.tiers, + compressor: config.compressor, + default_headers: config.default_headers, + poison: None, + } + } + + /// Returns a handle that fails every call with [`ApifyClientError::InvalidArgument`] carrying + /// `message`. + pub(crate) fn poisoned(mut self, message: String) -> Self { + self.poison = Some(Arc::from(message)); + self + } + + fn fail_if_poisoned(&self) -> ApifyClientResult<()> { + match &self.poison { + Some(message) => Err(ApifyClientError::InvalidArgument(message.to_string())), + None => Ok(()), + } + } + + /// Resolves a [`Timeout`] to the duration of the first attempt of a request, capped at the + /// client's `timeout_max`. + pub(crate) fn resolve_timeout(&self, timeout: Timeout) -> Duration { + self.tiers.resolve(timeout) + } + + /// Resolves the timeout of a request that asks the API to hold its response for + /// `wait_for_finish_secs` (see [`TimeoutTiers::resolve_with_wait_hold`]). + pub(crate) fn resolve_timeout_with_wait_hold( + &self, + tier: Timeout, + wait_for_finish_secs: Option, + ) -> Duration { + self.tiers.resolve( + self.tiers + .resolve_with_wait_hold(tier, wait_for_finish_secs), + ) + } + + /// Adds the headers every request carries: the built-in `User-Agent` and `Authorization`, + /// overridden by the client's default headers, overridden by the request's own headers. + /// Header names are compared case-insensitively. + fn apply_headers(&self, request: &mut HttpRequest) { + let mut merged: HashMap = HashMap::new(); + merged.insert("User-Agent".to_string(), self.user_agent.clone()); + if let Some(token) = &self.token { + merged.insert("Authorization".to_string(), format!("Bearer {token}")); + } + for (name, value) in &self.default_headers { + set_header(&mut merged, name, value); + } + for (name, value) in std::mem::take(&mut request.headers) { + set_header(&mut merged, &name, &value); } + request.headers = merged; } /// Sends `request` with authentication, the user-agent header, and the retry policy /// applied. Returns the first successful response, or the final error. pub async fn call(&self, mut request: HttpRequest) -> ApifyClientResult { - // Inject auth + user-agent headers shared by every endpoint. - request - .headers - .insert("User-Agent".to_string(), self.user_agent.clone()); - if let Some(token) = &self.token { - request - .headers - .insert("Authorization".to_string(), format!("Bearer {token}")); - } + self.fail_if_poisoned()?; + self.apply_headers(&mut request); // Compress the request body once (not per attempt) when it is large enough, mirroring the // reference client. The API accepts both brotli- and gzip-encoded request bodies. - maybe_compress_request(&mut request, self.compression); + if let Some(body) = request.body.take() { + request.body = Some(maybe_compress( + self.compressor.as_ref(), + &mut request.headers, + body, + )?); + } let method_str = request.method.as_str().to_string(); let path = extract_path(&request.url); - // The caller-supplied `request.timeout` is the per-endpoint base; it grows with each - // attempt up to the client's overall timeout budget. + // The caller-supplied `request.timeout` is the base of the request's tier; it grows with + // each attempt up to the client's `timeout_max`. let base_timeout = request.timeout; let mut delay = self.retry.min_delay_between_retries; // `saturating_add` so an extreme `max_retries` can't overflow the attempt count. @@ -457,19 +430,23 @@ impl HttpClient { // can never exceed the budget the whole request is allowed. sleep(randomized_delay(delay)).await; // `saturating_mul` mirrors the saturating arithmetic in `attempt_timeout`. - delay = delay.saturating_mul(BACKOFF_FACTOR).min(self.retry.timeout); + delay = delay.saturating_mul(BACKOFF_FACTOR).min(self.tiers.max); attempt += 1; } } - /// Per-attempt timeout: `min(overall_timeout, base * 2^(attempt-1))`. + /// Per-attempt timeout: `min(timeout_max, base * 2^(attempt-1))`. /// - /// The first attempt uses the per-endpoint `base` timeout; each retry doubles it so a + /// The first attempt uses the `base` timeout of the request; each retry doubles it so a /// slow-but-progressing connection gets more time, while never exceeding the client's - /// overall timeout budget. Mirrors the reference clients. + /// `timeout_max`. A request without a timeout ([`NO_TIMEOUT`]) stays without one. Mirrors the + /// reference clients. fn attempt_timeout(&self, base: Duration, attempt: u32) -> Duration { + if base == NO_TIMEOUT { + return NO_TIMEOUT; + } let scaled = base.saturating_mul(2u32.saturating_pow(attempt.saturating_sub(1))); - scaled.min(self.retry.timeout) + scaled.min(self.tiers.max) } pub(crate) fn user_agent(&self) -> &str { @@ -493,28 +470,22 @@ impl HttpClient { &self, mut request: HttpRequest, ) -> ApifyClientResult>>> { - request - .headers - .insert("User-Agent".to_string(), self.user_agent.clone()); - if let Some(token) = &self.token { - request - .headers - .insert("Authorization".to_string(), format!("Bearer {token}")); - } + self.fail_if_poisoned()?; + self.apply_headers(&mut request); let method_str = request.method.as_str().to_string(); let path = extract_path(&request.url); - // The caller-supplied `request.timeout` (the same per-endpoint base every buffered call - // uses — e.g. `DEFAULT_REQUEST_TIMEOUT`, the reference clients' "long" tier) is reused - // here as both the response-start wait and the per-chunk inactivity window: one - // configured duration, not a separate, larger one for inactivity. The JS reference client - // documents the same single-duration behavior ("it only limits inactivity"), though it - // gets there differently — via its HTTP library's socket-level idle timeout, not an - // explicit per-chunk reset like `with_inactivity_timeout` below — so this is a - // same-outcome, not same-mechanism, match. It carries the same tradeoff either way: a - // stream legitimately silent for the whole window (e.g. a long-running Actor logging - // nothing for `DEFAULT_REQUEST_TIMEOUT`) times out; the reference's escape hatch for that - // is an explicit `'noTimeout'` request option, which this crate does not yet expose. + // The caller-supplied `request.timeout` (the same tier-resolved base every buffered call + // uses — the `Long` tier for the streaming endpoints) is reused here as both the + // response-start wait and the per-chunk inactivity window: one configured duration, not + // a separate, larger one for inactivity. The JS reference client documents the same + // single-duration behavior ("it only limits inactivity"), though it gets there + // differently — via its HTTP library's socket-level idle timeout, not an explicit + // per-chunk reset like `with_inactivity_timeout` below — so this is a same-outcome, + // not same-mechanism, match. It carries the same tradeoff either way: a stream + // legitimately silent for the whole window (e.g. a long-running Actor logging nothing + // for the `Long` tier) times out; the escape hatch for that is `Timeout::NoTimeout`, + // set with a resource client's `with_timeout`. let budget = request.timeout; let response = tokio::time::timeout(budget, self.backend.send_streaming(request)) @@ -594,115 +565,11 @@ async fn collect_with_timeout( buf } -/// Compresses `request.body` in place when it is present, at least [`MIN_COMPRESS_BYTES`] long, -/// and no `Content-Encoding` is already set, adding the matching `Content-Encoding` header. -/// -/// The algorithm is chosen by `compression` (defaulting to brotli). The size threshold and the -/// "compress once, before retries" behaviour mirror the reference client. -fn maybe_compress_request(request: &mut HttpRequest, compression: RequestCompression) { - let Some(body) = request.body.as_ref() else { - return; - }; - if body.len() < MIN_COMPRESS_BYTES { - return; - } - // Respect a caller-provided `Content-Encoding` (case-insensitive): the body is then assumed to - // already be encoded, so re-compressing it would corrupt it. - let already_encoded = request - .headers - .keys() - .any(|k| k.eq_ignore_ascii_case("Content-Encoding")); - if already_encoded { - return; - } - // Skip media types that already carry their own compression (images, audio, video, - // archives): compressing them again burns CPU for little to no size reduction. This is - // mainly relevant to a raw-bytes Actor input (e.g. `ActorClient::start_raw` with - // `content_type: Some("application/zip")`); JSON request bodies always pass this check. - let content_type = request - .headers - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("Content-Type")) - .map(|(_, v)| v.as_str()); - if !is_compressible_content_type(content_type) { - return; - } - - let (encoding, compressed) = match compression { - RequestCompression::Brotli => (CONTENT_ENCODING_BROTLI, brotli_compress(body)), - RequestCompression::Gzip => (CONTENT_ENCODING_GZIP, gzip_compress(body)), - }; - request - .headers - .insert("Content-Encoding".to_string(), encoding.to_string()); - request.body = Some(compressed); -} - -/// Decides whether a request body with the given content type is worth compressing. -/// -/// Images, audio, video and archives already carry their own compression: running them through -/// brotli or gzip burns CPU, holds a second full copy of the body in memory, and usually produces -/// output no smaller than the input (sometimes larger). Formats that are raw despite such a media -/// type, e.g. `image/bmp` or `audio/wav`, are still compressed. A body with no content type is -/// assumed to be compressible. Matches the reference client's `isCompressibleContentType`. -fn is_compressible_content_type(content_type: Option<&str>) -> bool { - let Some(content_type) = content_type else { - return true; - }; - // `Content-Type` is case-insensitive and may carry parameters, e.g. `text/plain; charset=utf-8`. - let media_type = content_type - .split(';') - .next() - .unwrap_or(content_type) - .trim() - .to_ascii_lowercase(); - - if COMPRESSIBLE_MEDIA_TYPES.contains(&media_type.as_str()) { - return true; - } - if COMPRESSIBLE_MEDIA_TYPE_SUFFIXES - .iter() - .any(|suffix| media_type.ends_with(suffix)) - { - return true; - } - if ALREADY_COMPRESSED_MEDIA_TYPES.contains(&media_type.as_str()) { - return false; - } - !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES - .iter() - .any(|prefix| media_type.starts_with(prefix)) -} - -/// Brotli-compresses `data`. Writing to an in-memory `Vec` is infallible, so this cannot fail. -fn brotli_compress(data: &[u8]) -> Vec { - use std::io::Write; - - let mut writer = brotli::CompressorWriter::new( - Vec::new(), - BROTLI_BUFFER_SIZE, - BROTLI_QUALITY, - BROTLI_WINDOW_SIZE, - ); - writer - .write_all(data) - .expect("writing to an in-memory Vec never fails"); - writer.into_inner() -} - -/// Gzip-compresses `data`. Writing to and finishing an in-memory `Vec` is infallible, so this -/// cannot fail. -fn gzip_compress(data: &[u8]) -> Vec { - use flate2::{write::GzEncoder, Compression}; - use std::io::Write; - - let mut encoder = GzEncoder::new(Vec::new(), Compression::new(GZIP_COMPRESSION_LEVEL)); - encoder - .write_all(data) - .expect("writing to an in-memory Vec never fails"); - encoder - .finish() - .expect("finishing an in-memory Vec never fails") +/// Sets header `name` to `value`, replacing any existing header of the same name regardless of +/// its casing. +fn set_header(headers: &mut HashMap, name: &str, value: &str) { + headers.retain(|existing, _| !existing.eq_ignore_ascii_case(name)); + headers.insert(name.to_string(), value.to_string()); } /// Returns the path + query portion of a URL, for error reporting. diff --git a/src/http_compressors.rs b/src/http_compressors.rs new file mode 100644 index 0000000..bfccc22 --- /dev/null +++ b/src/http_compressors.rs @@ -0,0 +1,399 @@ +//! Request-body compression. +//! +//! The client compresses request bodies of at least 1024 bytes whose content type +//! is not already compressed and that carry no `Content-Encoding` of their own. The algorithm is an +//! [`HttpCompressor`]: the built-in [`BrotliHttpCompressor`] (the default) and +//! [`GzipHttpCompressor`], or a custom implementation passed to +//! [`ApifyClientBuilder::http_compressor`](crate::ApifyClientBuilder::http_compressor). + +use std::collections::HashMap; +use std::fmt::Debug; +use std::io::Write; + +use crate::error::{ApifyClientError, ApifyClientResult}; + +/// Request bodies at least this many bytes are compressed before sending. Smaller bodies are +/// left uncompressed because the CPU cost outweighs the transfer savings. Matches the reference +/// client's `MIN_COMPRESS_BYTES` threshold. +pub(crate) const MIN_COMPRESS_BYTES: usize = 1024; +/// `Content-Encoding` value used for brotli-compressed request bodies. +const CONTENT_ENCODING_BROTLI: &str = "br"; +/// `Content-Encoding` value used for gzip-compressed request bodies. +const CONTENT_ENCODING_GZIP: &str = "gzip"; +/// Lowest brotli quality, the fastest with the least compression. +const BROTLI_MIN_QUALITY: u32 = 0; +/// Highest brotli quality, the slowest with the best compression. +const BROTLI_MAX_QUALITY: u32 = 11; +/// Brotli quality used by [`BrotliHttpCompressor::default`]. Quality 6 balances the compression +/// ratio against CPU cost, matching the reference client. +const BROTLI_DEFAULT_QUALITY: u32 = 6; +/// Brotli sliding-window size (log2), 22 is the library default (a 4 MiB window). +const BROTLI_WINDOW_SIZE: u32 = 22; +/// Internal buffer size for the brotli encoder. +const BROTLI_BUFFER_SIZE: usize = 4096; +/// Lowest gzip quality, the fastest with the least compression. +const GZIP_MIN_QUALITY: u32 = 1; +/// Highest gzip quality, the slowest with the best compression. +const GZIP_MAX_QUALITY: u32 = 9; +/// Gzip quality used by [`GzipHttpCompressor::default`], matching the reference client. +const GZIP_DEFAULT_QUALITY: u32 = 9; + +/// Media-type prefixes whose payloads already carry their own compression, so running them +/// through brotli/gzip burns CPU and memory for a result that is usually no smaller — and the +/// request also keeps the intended `Content-Type`, rather than becoming an unreadable +/// `Content-Encoding: br` blob the destination may not expect for these formats. +/// Matches the reference client's `ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES`. +const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES: [&str; 3] = ["audio/", "image/", "video/"]; + +/// Exact media types (outside the prefixes above) whose payloads already carry their own +/// compression. Matches the reference client's `ALREADY_COMPRESSED_MEDIA_TYPES`. +const ALREADY_COMPRESSED_MEDIA_TYPES: [&str; 19] = [ + "application/epub+zip", + "application/gzip", + "application/java-archive", + "application/vnd.android.package-archive", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.rar", + "application/x-7z-compressed", + "application/x-bzip", + "application/x-bzip2", + "application/x-gzip", + "application/x-rar-compressed", + "application/x-xz", + "application/x-zip-compressed", + "application/zip", + "application/zstd", + "font/woff", + "font/woff2", +]; + +/// Uncompressed media types that sit under an already-compressed prefix above, so compressing +/// them still pays off. Matches the reference client's `COMPRESSIBLE_MEDIA_TYPES`. +const COMPRESSIBLE_MEDIA_TYPES: [&str; 16] = [ + "audio/aiff", + "audio/basic", + "audio/l16", + "audio/l24", + "audio/midi", + "audio/vnd.wave", + "audio/wav", + "audio/wave", + "audio/x-aiff", + "audio/x-wav", + "image/bmp", + "image/tiff", + "image/vnd.adobe.photoshop", + "image/vnd.microsoft.icon", + "image/x-icon", + "image/x-ms-bmp", +]; + +/// Structured-syntax suffixes that mark a media type as text even under an already-compressed +/// prefix (e.g. `image/svg+xml`). Matches the reference client's +/// `COMPRESSIBLE_MEDIA_TYPE_SUFFIXES`. +const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES: [&str; 2] = ["+json", "+xml"]; + +/// Strategy for compressing request bodies. +/// +/// Implement this trait to plug in a custom compressor. [`content_encoding`](Self::content_encoding) +/// is sent in the `Content-Encoding` header and [`compress`](Self::compress) produces the body. The +/// client calls it only for bodies that are large enough to benefit from compression, whose content +/// type does not already carry its own compression, and which the caller did not send with a +/// `Content-Encoding` of their own. +pub trait HttpCompressor: Send + Sync + Debug { + /// Value sent in the `Content-Encoding` header, for example `gzip` or `br`. + fn content_encoding(&self) -> &str; + + /// Compresses a request body and returns the compressed bytes. + fn compress(&self, data: &[u8]) -> ApifyClientResult>; +} + +/// Brotli compressor (`Content-Encoding: br`), the client's default. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BrotliHttpCompressor { + quality: u32, +} + +impl BrotliHttpCompressor { + /// Creates a brotli compressor with the given quality, from 0 (fastest, least compression) to + /// 11 (slowest, best compression). Returns + /// [`ApifyClientError::InvalidArgument`] when `quality` is out of range. + pub fn new(quality: u32) -> ApifyClientResult { + check_quality("brotli", quality, BROTLI_MIN_QUALITY, BROTLI_MAX_QUALITY)?; + Ok(Self { quality }) + } +} + +impl Default for BrotliHttpCompressor { + /// Brotli at quality 6. + fn default() -> Self { + Self { + quality: BROTLI_DEFAULT_QUALITY, + } + } +} + +impl HttpCompressor for BrotliHttpCompressor { + fn content_encoding(&self) -> &str { + CONTENT_ENCODING_BROTLI + } + + fn compress(&self, data: &[u8]) -> ApifyClientResult> { + let mut writer = brotli::CompressorWriter::new( + Vec::new(), + BROTLI_BUFFER_SIZE, + self.quality, + BROTLI_WINDOW_SIZE, + ); + write_all(&mut writer, data)?; + Ok(writer.into_inner()) + } +} + +/// Gzip compressor (`Content-Encoding: gzip`). Choose it for environments or intermediaries that +/// do not handle brotli. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GzipHttpCompressor { + quality: u32, +} + +impl GzipHttpCompressor { + /// Creates a gzip compressor with the given quality, from 1 (fastest, least compression) to 9 + /// (slowest, best compression). Returns [`ApifyClientError::InvalidArgument`] when `quality` + /// is out of range. + pub fn new(quality: u32) -> ApifyClientResult { + check_quality("gzip", quality, GZIP_MIN_QUALITY, GZIP_MAX_QUALITY)?; + Ok(Self { quality }) + } +} + +impl Default for GzipHttpCompressor { + /// Gzip at quality 9. + fn default() -> Self { + Self { + quality: GZIP_DEFAULT_QUALITY, + } + } +} + +impl HttpCompressor for GzipHttpCompressor { + fn content_encoding(&self) -> &str { + CONTENT_ENCODING_GZIP + } + + fn compress(&self, data: &[u8]) -> ApifyClientResult> { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(self.quality)); + write_all(&mut encoder, data)?; + encoder.finish().map_err(compression_failed) + } +} + +/// Built-in compression algorithm selected by name, at its default quality. A shorthand for +/// [`BrotliHttpCompressor::default`] and [`GzipHttpCompressor::default`]; use the compressor +/// structs directly to tune the quality, or implement [`HttpCompressor`] for another algorithm. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum RequestCompression { + /// Brotli (`Content-Encoding: br`). The default: best compression ratio, and the encoding the + /// reference client prefers. + #[default] + Brotli, + /// Gzip (`Content-Encoding: gzip`). Choose this for environments or intermediaries that do not + /// handle brotli. + Gzip, +} + +impl RequestCompression { + /// The built-in compressor this algorithm name stands for. + pub(crate) fn into_compressor(self) -> std::sync::Arc { + match self { + RequestCompression::Brotli => std::sync::Arc::new(BrotliHttpCompressor::default()), + RequestCompression::Gzip => std::sync::Arc::new(GzipHttpCompressor::default()), + } + } +} + +fn check_quality(algorithm: &str, quality: u32, min: u32, max: u32) -> ApifyClientResult<()> { + if (min..=max).contains(&quality) { + Ok(()) + } else { + Err(ApifyClientError::InvalidArgument(format!( + "{algorithm} quality must be between {min} and {max}, got {quality}" + ))) + } +} + +fn write_all(writer: &mut impl Write, data: &[u8]) -> ApifyClientResult<()> { + writer.write_all(data).map_err(compression_failed) +} + +fn compression_failed(err: std::io::Error) -> ApifyClientError { + ApifyClientError::InvalidArgument(format!("compressing the request body failed: {err}")) +} + +/// Compresses `body` with `compressor` and adds the matching `Content-Encoding` header, when the +/// body is large enough, no `Content-Encoding` is already set, and its content type is not +/// already compressed. Returns the (possibly unchanged) body. +/// +/// The size threshold and the "compress once, before retries" behaviour mirror the reference +/// client. +pub(crate) fn maybe_compress( + compressor: &dyn HttpCompressor, + headers: &mut HashMap, + body: Vec, +) -> ApifyClientResult> { + if body.len() < MIN_COMPRESS_BYTES { + return Ok(body); + } + // Respect a caller-provided `Content-Encoding` (case-insensitive): the body is then assumed to + // already be encoded, so re-compressing it would corrupt it. + let already_encoded = headers + .keys() + .any(|k| k.eq_ignore_ascii_case("Content-Encoding")); + if already_encoded { + return Ok(body); + } + // Skip media types that already carry their own compression (images, audio, video, + // archives): compressing them again burns CPU for little to no size reduction. This is + // mainly relevant to a raw-bytes Actor input (e.g. `ActorClient::start_raw` with + // `content_type: Some("application/zip")`); JSON request bodies always pass this check. + let content_type = headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("Content-Type")) + .map(|(_, v)| v.as_str()); + if !is_compressible_content_type(content_type) { + return Ok(body); + } + + let compressed = compressor.compress(&body)?; + headers.insert( + "Content-Encoding".to_string(), + compressor.content_encoding().to_string(), + ); + Ok(compressed) +} + +/// Decides whether a request body with the given content type is worth compressing. +/// +/// Images, audio, video and archives already carry their own compression: running them through +/// brotli or gzip burns CPU, holds a second full copy of the body in memory, and usually produces +/// output no smaller than the input (sometimes larger). Formats that are raw despite such a media +/// type, e.g. `image/bmp` or `audio/wav`, are still compressed. A body with no content type is +/// assumed to be compressible. Matches the reference client's `isCompressibleContentType`. +fn is_compressible_content_type(content_type: Option<&str>) -> bool { + let Some(content_type) = content_type else { + return true; + }; + // `Content-Type` is case-insensitive and may carry parameters, e.g. `text/plain; charset=utf-8`. + let media_type = content_type + .split(';') + .next() + .unwrap_or(content_type) + .trim() + .to_ascii_lowercase(); + + if COMPRESSIBLE_MEDIA_TYPES.contains(&media_type.as_str()) { + return true; + } + if COMPRESSIBLE_MEDIA_TYPE_SUFFIXES + .iter() + .any(|suffix| media_type.ends_with(suffix)) + { + return true; + } + if ALREADY_COMPRESSED_MEDIA_TYPES.contains(&media_type.as_str()) { + return false; + } + !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES + .iter() + .any(|prefix| media_type.starts_with(prefix)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn headers(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn quality_outside_the_algorithm_range_is_rejected() { + assert!(BrotliHttpCompressor::new(11).is_ok()); + assert!(BrotliHttpCompressor::new(12).is_err()); + assert!(GzipHttpCompressor::new(1).is_ok()); + assert!(GzipHttpCompressor::new(0).is_err()); + assert!(GzipHttpCompressor::new(10).is_err()); + } + + #[test] + fn built_in_compressors_report_their_content_encoding() { + assert_eq!(BrotliHttpCompressor::default().content_encoding(), "br"); + assert_eq!(GzipHttpCompressor::default().content_encoding(), "gzip"); + } + + #[test] + fn small_bodies_are_left_uncompressed() { + let mut headers = headers(&[("Content-Type", "application/json")]); + let body = maybe_compress( + &GzipHttpCompressor::default(), + &mut headers, + vec![b'a'; MIN_COMPRESS_BYTES - 1], + ) + .unwrap(); + assert_eq!(body.len(), MIN_COMPRESS_BYTES - 1); + assert!(!headers.contains_key("Content-Encoding")); + } + + #[test] + fn large_json_bodies_are_compressed_with_the_configured_compressor() { + let mut headers = headers(&[("Content-Type", "application/json")]); + let body = maybe_compress( + &GzipHttpCompressor::default(), + &mut headers, + vec![b'a'; MIN_COMPRESS_BYTES * 4], + ) + .unwrap(); + assert!(body.len() < MIN_COMPRESS_BYTES); + assert_eq!(headers["Content-Encoding"], "gzip"); + } + + #[test] + fn caller_set_content_encoding_is_respected_case_insensitively() { + let mut headers = headers(&[("content-encoding", "gzip")]); + let original = vec![b'a'; MIN_COMPRESS_BYTES * 2]; + let body = maybe_compress( + &BrotliHttpCompressor::default(), + &mut headers, + original.clone(), + ) + .unwrap(); + assert_eq!(body, original); + assert_eq!(headers.len(), 1); + } + + #[test] + fn already_compressed_content_types_are_skipped() { + for (content_type, compressible) in [ + ("image/png", false), + ("application/zip", false), + ("image/bmp", true), + ("image/svg+xml", true), + ("application/octet-stream", true), + ("Application/JSON; charset=utf-8", true), + ] { + assert_eq!( + is_compressible_content_type(Some(content_type)), + compressible, + "{content_type}" + ); + } + assert!(is_compressible_content_type(None)); + } +} diff --git a/src/lib.rs b/src/lib.rs index 529d8eb..aa7a07a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,11 @@ //! [`http_client::ReqwestBackend`]. Swap it via //! [`ApifyClientBuilder::http_backend`]. //! - **Cross-cutting behaviour** (auth, `User-Agent`, retries with exponential backoff, -//! timeouts) lives in [`http_client::HttpClient`] and is applied to every request. +//! timeouts, request compression) lives in [`http_client::HttpClient`] and is applied to every +//! request. +//! - **Timeouts**: every method uses a [`Timeout`] tier (`Short`, `Medium` or `Long`) whose +//! duration is configured on the [`ApifyClientBuilder`]. A resource client's `with_timeout` +//! overrides the tier of all its requests. //! //! ## Cancellation //! @@ -49,7 +53,10 @@ //! nothing until polled, so dropping one (e.g. via `tokio::select!`, or letting a //! `tokio::time::timeout` elapse) stops the underlying request without any API for it. This //! client therefore exposes no separate cancellation parameter — dropping the future returned -//! by any method is the idiomatic equivalent. +//! by any method is the idiomatic equivalent: the request in flight ends and no retry or further +//! poll follows. The built-in waits, such as [`RunClient::wait_for_finish`], are futures too. +//! +//! [`RunClient::wait_for_finish`]: crate::clients::run::RunClient::wait_for_finish #![warn(missing_docs)] @@ -58,13 +65,18 @@ pub mod clients; pub mod common; pub mod error; pub mod http_client; +pub mod http_compressors; pub mod models; +pub mod timeouts; mod version; pub use client::{ApifyClient, ApifyClientBuilder}; pub use common::{ListOptions, PaginationList, QueryParams, StorageListOptions}; pub use error::{ApiError, ApifyClientError, ApifyClientResult}; -pub use http_client::RequestCompression; +pub use http_compressors::{ + BrotliHttpCompressor, GzipHttpCompressor, HttpCompressor, RequestCompression, +}; +pub use timeouts::Timeout; pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; // Re-export the most commonly used option/parameter types for ergonomic access. diff --git a/src/models.rs b/src/models.rs index 7bc9e50..0730216 100644 --- a/src/models.rs +++ b/src/models.rs @@ -9,12 +9,33 @@ use std::collections::HashMap; use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; /// Convenience alias for the catch-all map of unmodelled JSON fields. pub type Extra = HashMap; +/// Deserializes an optional URL field the OpenAPI specification declares as `format: uri`, into +/// the WHATWG URL serialization of the value the API sent, so every Apify client returns the same +/// string for the same field. +/// +/// Normalization adds a trailing slash to an empty path, lowercases and punycodes the host, drops +/// a default port, and percent-encodes unsafe characters. A value that is not a valid absolute URL +/// fails the deserialization of the whole response. +fn deserialize_normalized_url<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value: Option = Option::deserialize(deserializer)?; + value + .map(|raw| { + url::Url::parse(&raw) + .map(String::from) + .map_err(|err| serde::de::Error::custom(format!("invalid URL `{raw}`: {err}"))) + }) + .transpose() +} + /// An Actor on the Apify platform. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -89,8 +110,8 @@ pub struct ActorRun { /// Default request queue ID associated with the run. #[serde(default)] pub default_request_queue_id: Option, - /// URL of the run's container, if running. - #[serde(default)] + /// URL of the run's container, if running. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] pub container_url: Option, /// Any other fields returned by the API. #[serde(flatten)] @@ -569,8 +590,8 @@ pub struct Webhook { /// ID of the owner. #[serde(default)] pub user_id: Option, - /// The URL that receives the webhook POST request. - #[serde(default)] + /// The URL that receives the webhook POST request. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] pub request_url: Option, /// Event types that trigger this webhook. #[serde(default)] @@ -658,3 +679,53 @@ pub struct ActorEnvVar { #[serde(flatten)] pub extra: Extra, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn url_fields_are_normalized_to_the_whatwg_serialization() { + let run: ActorRun = serde_json::from_value(serde_json::json!({ + "id": "run", + "containerUrl": "HTTPS://Run-1.Example.COM:443" + })) + .unwrap(); + assert_eq!( + run.container_url.as_deref(), + Some("https://run-1.example.com/") + ); + + let webhook: Webhook = serde_json::from_value(serde_json::json!({ + "id": "hook", + "requestUrl": "https://example.com/a b?q=1" + })) + .unwrap(); + assert_eq!( + webhook.request_url.as_deref(), + Some("https://example.com/a%20b?q=1") + ); + } + + #[test] + fn null_and_missing_url_fields_stay_none() { + let run: ActorRun = serde_json::from_value(serde_json::json!({ + "id": "run", + "containerUrl": null + })) + .unwrap(); + assert_eq!(run.container_url, None); + + let webhook: Webhook = serde_json::from_value(serde_json::json!({ "id": "hook" })).unwrap(); + assert_eq!(webhook.request_url, None); + } + + #[test] + fn an_invalid_url_fails_the_deserialization() { + let result: Result = serde_json::from_value(serde_json::json!({ + "id": "hook", + "requestUrl": "not a url" + })); + assert!(result.is_err()); + } +} diff --git a/src/timeouts.rs b/src/timeouts.rs new file mode 100644 index 0000000..16d44ca --- /dev/null +++ b/src/timeouts.rs @@ -0,0 +1,149 @@ +//! Request timeout tiers. +//! +//! Every client method picks the tier that fits the expected duration of its request: a simple +//! metadata read or write is [`Timeout::Short`], listing, batch and trigger operations are +//! [`Timeout::Medium`], and downloads, uploads and streaming are [`Timeout::Long`]. Polling that +//! waits for a job to finish uses [`Timeout::NoTimeout`]. The duration of each tier is set on +//! [`ApifyClientBuilder`](crate::ApifyClientBuilder), and a single resource client can override the +//! tier of all its requests with its `with_timeout` method. + +use std::time::Duration; + +/// Default duration of the [`Timeout::Short`] tier. +pub const DEFAULT_TIMEOUT_SHORT: Duration = Duration::from_secs(5); +/// Default duration of the [`Timeout::Medium`] tier. +pub const DEFAULT_TIMEOUT_MEDIUM: Duration = Duration::from_secs(30); +/// Default duration of the [`Timeout::Long`] tier. +pub const DEFAULT_TIMEOUT_LONG: Duration = Duration::from_secs(360); +/// Default cap on the timeout of a single request attempt. +pub const DEFAULT_TIMEOUT_MAX: Duration = Duration::from_secs(360); + +/// The value [`HttpRequest::timeout`](crate::http_client::HttpRequest::timeout) carries for a +/// request that must run for as long as it takes ([`Timeout::NoTimeout`]). An +/// [`HttpBackend`](crate::http_client::HttpBackend) must not abort such a request on its own. +pub const NO_TIMEOUT: Duration = Duration::MAX; + +/// How long the API holds a `waitForFinish` response, however long the parameter asks for. +pub(crate) const MAX_WAIT_FOR_FINISH_HOLD: Duration = Duration::from_secs(60); + +/// Timeout of a single API request. +/// +/// A tier name picks the duration configured for that tier on +/// [`ApifyClientBuilder`](crate::ApifyClientBuilder), [`Timeout::Custom`] is an exact duration, +/// and [`Timeout::NoTimeout`] lets the request run for as long as it takes. A tier or an explicit +/// duration above the client's `timeout_max` is capped at it. The timeout applies to each attempt +/// separately and doubles with every retry, up to the same cap. +/// +/// With [`Timeout::NoTimeout`] the client never aborts the request, so a connection that stalls +/// is not retried either. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Timeout { + /// Simple metadata reads and writes (default 5 seconds). + Short, + /// Listing, batch and trigger operations (default 30 seconds). + Medium, + /// Downloads, uploads and streaming (default 360 seconds). + Long, + /// No client-side timeout. + NoTimeout, + /// An exact duration, capped at the client's `timeout_max`. + Custom(Duration), +} + +/// The configured duration of every timeout tier plus the cap on a single attempt. +#[derive(Debug, Clone, Copy)] +pub(crate) struct TimeoutTiers { + pub(crate) short: Duration, + pub(crate) medium: Duration, + pub(crate) long: Duration, + pub(crate) max: Duration, +} + +impl Default for TimeoutTiers { + fn default() -> Self { + Self { + short: DEFAULT_TIMEOUT_SHORT, + medium: DEFAULT_TIMEOUT_MEDIUM, + long: DEFAULT_TIMEOUT_LONG, + max: DEFAULT_TIMEOUT_MAX, + } + } +} + +impl TimeoutTiers { + /// Resolves `timeout` to the duration of the first attempt of a request, capped at `max`. + /// [`Timeout::NoTimeout`] resolves to [`NO_TIMEOUT`]. + pub(crate) fn resolve(&self, timeout: Timeout) -> Duration { + let requested = match timeout { + Timeout::Short => self.short, + Timeout::Medium => self.medium, + Timeout::Long => self.long, + Timeout::NoTimeout => return NO_TIMEOUT, + Timeout::Custom(duration) => duration, + }; + requested.min(self.max) + } + + /// Resolves a request that asks the API to hold its response with `waitForFinish`: the hold + /// the caller asked for (at most [`MAX_WAIT_FOR_FINISH_HOLD`]) plus the round trip the + /// `tier` allows, so the client does not abort a request the API is still holding. An + /// aborted `start` or `build` would be retried, creating a second resource. + pub(crate) fn resolve_with_wait_hold( + &self, + tier: Timeout, + wait_for_finish_secs: Option, + ) -> Timeout { + let hold = wait_for_finish_secs + .filter(|&secs| secs > 0) + .map(|secs| Duration::from_secs(secs as u64).min(MAX_WAIT_FOR_FINISH_HOLD)); + match hold { + Some(hold) => Timeout::Custom(self.resolve(tier).saturating_add(hold)), + None => tier, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tiers_resolve_to_their_configured_duration_capped_at_max() { + let tiers = TimeoutTiers { + short: Duration::from_secs(1), + medium: Duration::from_secs(2), + long: Duration::from_secs(100), + max: Duration::from_secs(10), + }; + assert_eq!(tiers.resolve(Timeout::Short), Duration::from_secs(1)); + assert_eq!(tiers.resolve(Timeout::Medium), Duration::from_secs(2)); + assert_eq!(tiers.resolve(Timeout::Long), Duration::from_secs(10)); + assert_eq!( + tiers.resolve(Timeout::Custom(Duration::from_secs(50))), + Duration::from_secs(10) + ); + assert_eq!(tiers.resolve(Timeout::NoTimeout), NO_TIMEOUT); + } + + #[test] + fn wait_hold_is_added_to_the_tier_and_capped_at_one_minute() { + let tiers = TimeoutTiers::default(); + assert_eq!( + tiers.resolve_with_wait_hold(Timeout::Short, None), + Timeout::Short + ); + assert_eq!( + tiers.resolve_with_wait_hold(Timeout::Short, Some(0)), + Timeout::Short + ); + assert_eq!( + tiers.resolve_with_wait_hold(Timeout::Short, Some(10)), + Timeout::Custom(Duration::from_secs(15)) + ); + assert_eq!( + tiers.resolve_with_wait_hold(Timeout::Medium, Some(999)), + Timeout::Custom(Duration::from_secs(90)) + ); + } +} diff --git a/src/version.rs b/src/version.rs index 50ea1cb..5a0adda 100644 --- a/src/version.rs +++ b/src/version.rs @@ -10,4 +10,4 @@ pub const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// and verified against. /// /// This corresponds to the `info.version` field of the Apify OpenAPI document. -pub const API_SPEC_VERSION: &str = "v2-2026-10-26T112451Z"; +pub const API_SPEC_VERSION: &str = "v2-2026-10-29T100357Z"; diff --git a/tests/actor_run.rs b/tests/actor_run.rs index 2151490..281ba70 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -68,7 +68,10 @@ async fn run_actor_and_read_outputs() { let raw_log = client .run(&run.id) .log() - .get_with_options(apify_client::LogOptions { raw: Some(true) }) + .get_with_options(apify_client::LogOptions { + raw: Some(true), + ..Default::default() + }) .await .expect("get raw run log"); assert!(raw_log.is_some(), "finished run should have a raw log"); @@ -202,3 +205,78 @@ async fn log_stream_of_missing_run_carries_structured_api_error() { assert_eq!(api_error.status_code, 404); assert!(api_error.is_not_found()); } + +/// A chained client cannot tell a missing run from a missing storage or log, so a `404` through it +/// is an error, while a client that names its resource by id resolves it to `None`. +#[tokio::test(flavor = "multi_thread")] +async fn not_found_through_a_chained_client_is_an_error() { + let client = require_client!(); + let run = client.run("nonexistent-run-id-xyz"); + + let err = run + .dataset() + .get() + .await + .expect_err("a chained client returns the 404"); + assert!(err.as_api_error().expect("an API error").is_not_found()); + let err = run + .log() + .get() + .await + .expect_err("a chained log client returns the 404"); + assert!(err.as_api_error().expect("an API error").is_not_found()); + + assert!(client + .log("nonexistent-run-id-xyz") + .get() + .await + .expect("a log addressed by id resolves a 404") + .is_none()); + assert!(client + .log("nonexistent-run-id-xyz") + .stream() + .await + .expect("a log stream addressed by id resolves a 404") + .is_none()); + assert!(client + .dataset("nonexistent-dataset-id-xyz") + .get() + .await + .expect("a dataset addressed by id resolves a 404") + .is_none()); +} + +/// An id that cannot be placed in a URL path fails without a request. +#[tokio::test(flavor = "multi_thread")] +async fn an_invalid_id_is_rejected() { + let client = require_client!(); + let err = client + .run("..") + .get() + .await + .expect_err("a dot segment is not a run id"); + assert!( + matches!(err, apify_client::ApifyClientError::InvalidArgument(_)), + "{err:?}" + ); +} + +/// The `deployment` filter is accepted by the run listing, together with the non-terminal status +/// it requires. +#[tokio::test(flavor = "multi_thread")] +async fn list_runs_filtered_by_deployment() { + let client = require_client!(); + let page = client + .runs() + .list( + Default::default(), + apify_client::RunListOptions { + status: vec!["RUNNING".to_string()], + deployment: vec!["SINGLE_TENANT".to_string()], + ..Default::default() + }, + ) + .await + .expect("listing runs by deployment should succeed"); + assert!(page.total >= 0); +} diff --git a/tests/common/mock_backend.rs b/tests/common/mock_backend.rs new file mode 100644 index 0000000..a80a916 --- /dev/null +++ b/tests/common/mock_backend.rs @@ -0,0 +1,120 @@ +//! A scripted [`HttpBackend`] for the offline tests: it returns a queued sequence of responses and +//! records what the client sent, so the tests need no network access or `APIFY_TOKEN`. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use apify_client::http_client::{HttpBackend, HttpRequest, HttpResponse}; +use apify_client::{ApifyClient, ApifyClientError}; +use async_trait::async_trait; + +/// A scripted backend that returns a queued sequence of responses and counts calls. +#[derive(Debug)] +pub struct MockBackend { + responses: Mutex>, + calls: AtomicUsize, + last_url: Mutex>, + /// Every request URL in call order, so tests can assert per-page pagination wiring. + urls: Mutex>, + last_headers: Mutex>, + last_body: Mutex>>, + /// The `timeout` of every request in call order, so tests can assert the timeout tiers. + timeouts: Mutex>, +} + +#[derive(Debug, Clone)] +pub enum MockOutcome { + Status(u16, Vec), + NetworkError, +} + +impl MockBackend { + pub fn new(responses: Vec) -> Arc { + Arc::new(Self { + responses: Mutex::new(responses), + calls: AtomicUsize::new(0), + last_url: Mutex::new(None), + urls: Mutex::new(Vec::new()), + last_headers: Mutex::new(std::collections::HashMap::new()), + last_body: Mutex::new(None), + timeouts: Mutex::new(Vec::new()), + }) + } + + /// The timeout of every request in call order. + pub fn timeouts(&self) -> Vec { + self.timeouts.lock().unwrap().clone() + } + + /// The timeout of the last request. + pub fn last_timeout(&self) -> Duration { + *self.timeouts().last().expect("a request was sent") + } + + pub fn call_count(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } + + pub fn last_url(&self) -> Option { + self.last_url.lock().unwrap().clone() + } + + /// All request URLs in call order. + pub fn urls(&self) -> Vec { + self.urls.lock().unwrap().clone() + } + + /// Case-insensitive lookup of the last request's header value. + pub fn last_header(&self, name: &str) -> Option { + let headers = self.last_headers.lock().unwrap(); + headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.clone()) + } + + pub fn last_body(&self) -> Option> { + self.last_body.lock().unwrap().clone() + } +} + +#[async_trait] +impl HttpBackend for MockBackend { + async fn send(&self, request: HttpRequest) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + *self.last_url.lock().unwrap() = Some(request.url.clone()); + self.urls.lock().unwrap().push(request.url.clone()); + *self.last_headers.lock().unwrap() = request.headers.clone(); + *self.last_body.lock().unwrap() = request.body.clone(); + self.timeouts.lock().unwrap().push(request.timeout); + let mut queue = self.responses.lock().unwrap(); + let outcome = if queue.len() > 1 { + queue.remove(0) + } else { + // Repeat the last scripted outcome for any further calls. + queue + .first() + .cloned() + .expect("at least one scripted response") + }; + match outcome { + MockOutcome::Status(status, body) => Ok(HttpResponse { + status, + headers: Default::default(), + body, + }), + MockOutcome::NetworkError => Err(ApifyClientError::Http("boom".to_string())), + } + } +} + +pub fn client_with(backend: Arc, max_retries: u32) -> ApifyClient { + ApifyClient::builder() + .token("test-token") + .max_retries(max_retries) + // Keep backoff tiny so tests run fast. + .min_delay_between_retries(Duration::from_millis(1)) + .http_backend(backend) + .build() +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 8145a90..269217d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -10,15 +10,18 @@ #![allow(dead_code)] -use apify_client::ApifyClient; +pub mod mock_backend; + +use apify_client::{ApifyClient, ApifyClientBuilder}; /// The API URL the tests target, mirroring the integration-test contract. const DEFAULT_API_URL: &str = "https://api.apify.com/v2"; -/// Builds an [`ApifyClient`] configured from the environment. +/// Builds an [`ApifyClientBuilder`] configured from the environment, for tests that customize the +/// client further. /// /// Returns `None` (so the caller can skip) if `APIFY_TOKEN` is not set. -pub fn make_client() -> Option { +pub fn make_client_builder() -> Option { let token = std::env::var("APIFY_TOKEN") .ok() .filter(|t| !t.is_empty())?; @@ -26,12 +29,14 @@ pub fn make_client() -> Option { .ok() .filter(|u| !u.is_empty()); let base_url = resolve_base_url(api_url.as_deref()); - Some( - ApifyClient::builder() - .token(token) - .base_url(base_url) - .build(), - ) + Some(ApifyClient::builder().token(token).base_url(base_url)) +} + +/// Builds an [`ApifyClient`] configured from the environment. +/// +/// Returns `None` (so the caller can skip) if `APIFY_TOKEN` is not set. +pub fn make_client() -> Option { + make_client_builder().map(ApifyClientBuilder::build) } /// Resolves the client `base_url` from an optional `APIFY_API_URL`. @@ -144,17 +149,17 @@ where /// Number of times [`iter_contains_eventually`] rebuilds the iterator and re-scans while waiting /// for a just-created resource to become visible in its collection LIST endpoint. -pub const ITER_RETRY_ATTEMPTS: usize = 16; +pub const ITER_RETRY_ATTEMPTS: usize = 30; /// Delay between the attempts made by [`iter_contains_eventually`]. -pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(1000); +pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_secs(3); -/// Like [`iter_contains`], but tolerant of eventual consistency in collection LIST endpoints. +/// Like [`iter_contains`], but tolerant of replica lag in collection LIST endpoints. /// -/// A resource created through a write endpoint is not always immediately reflected in its -/// collection's LIST response — the write and the list index converge asynchronously on the -/// server. A create-then-iterate test that scans the collection exactly once therefore races that -/// convergence and flakes when the just-created entity has not yet propagated. +/// The list endpoints read from database secondaries that may trail the primary by up to 90 +/// seconds, so a resource created through a write endpoint is not always immediately reflected in +/// its collection's LIST response. A create-then-iterate test that scans the collection exactly +/// once therefore races that lag and flakes when the just-created entity has not yet propagated. /// /// This helper rebuilds a fresh iterator via `make_iter` and re-scans it with [`iter_contains`] up /// to [`ITER_RETRY_ATTEMPTS`] times, sleeping [`ITER_RETRY_BACKOFF`] between attempts, returning @@ -162,13 +167,9 @@ pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_mi /// attempt and returns immediately with no sleeping — so it is a no-op in the common /// already-consistent case and only pays the backoff on the rare lagging run. /// -/// Budget: `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` = ~15s of retrying before giving up. -/// This is deliberately larger than a "couple of seconds": the only Apify propagation lag actually -/// measured in this suite is the dataset-items count settling at ~10s, and the collection LIST -/// index convergence time is not independently measured, so a ~2s budget could let the flake recur -/// at a lower (harder-to-diagnose) frequency. ~15s gives real headroom above the ~10s observation -/// while still failing fast enough on a genuinely-missing entity (a true bug). The cost lands only -/// on lagging or genuinely-failing runs; a consistent account never sleeps. +/// Budget: `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` = 87s of retrying before giving up, +/// which covers the 90-second staleness window. The cost lands only on lagging or +/// genuinely-failing runs; a consistent account never sleeps. pub async fn iter_contains_eventually(mut make_iter: Mk, mut pred: F) -> bool where Mk: FnMut() -> apify_client::ListIterator, diff --git a/tests/store.rs b/tests/store.rs index 26e507d..549fa0e 100644 --- a/tests/store.rs +++ b/tests/store.rs @@ -40,3 +40,21 @@ async fn iterate_store() { } assert!(seen > 0, "store iteration should yield at least one actor"); } + +/// Simple GET: `include_input_schema` is accepted by the Store endpoint and adds the compact input +/// schema to the listed Actors (the API then caps `limit` at 10). +#[tokio::test(flavor = "multi_thread")] +async fn list_store_with_input_schema() { + let client = require_client!(); + let page = client + .store() + .list(StoreListOptions { + limit: Some(3), + include_input_schema: Some(true), + ..Default::default() + }) + .await + .expect("listing the store with input schemas should succeed"); + assert!(!page.items.is_empty(), "the Store lists Actors"); + assert!(page.items.len() <= 3); +} diff --git a/tests/task.rs b/tests/task.rs index 75cfa6a..7ae9a60 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -281,7 +281,7 @@ async fn task_crud_flow() { .await .expect("update input"); let input = task_client.get_input().await.expect("get input"); - assert!(input.is_some()); + assert_eq!(input["message"], "updated"); // Update (rename + set description). let renamed = common::unique_name("task-renamed"); diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 49faf28..363d27f 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -3,113 +3,18 @@ //! These use a mock [`HttpBackend`] and require no network access or `APIFY_TOKEN`, so the //! most bug-prone logic is always exercised in CI. -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; +mod common; + +use std::sync::Arc; use std::time::Duration; -use apify_client::http_client::{HttpBackend, HttpRequest, HttpResponse}; use apify_client::{ ActorStartOptions, ApifyClient, ApifyClientError, LastRunOptions, RequestCompression, WaitForResources, }; -use async_trait::async_trait; +use common::mock_backend::{client_with, MockBackend, MockOutcome}; use futures_util::StreamExt; - -/// A scripted backend that returns a queued sequence of responses and counts calls. -#[derive(Debug)] -struct MockBackend { - responses: Mutex>, - calls: AtomicUsize, - last_url: Mutex>, - /// Every request URL in call order, so tests can assert per-page pagination wiring. - urls: Mutex>, - last_headers: Mutex>, - last_body: Mutex>>, -} - -#[derive(Debug, Clone)] -enum MockOutcome { - Status(u16, Vec), - NetworkError, -} - -impl MockBackend { - fn new(responses: Vec) -> Arc { - Arc::new(Self { - responses: Mutex::new(responses), - calls: AtomicUsize::new(0), - last_url: Mutex::new(None), - urls: Mutex::new(Vec::new()), - last_headers: Mutex::new(std::collections::HashMap::new()), - last_body: Mutex::new(None), - }) - } - - fn call_count(&self) -> usize { - self.calls.load(Ordering::SeqCst) - } - - fn last_url(&self) -> Option { - self.last_url.lock().unwrap().clone() - } - - /// All request URLs in call order. - fn urls(&self) -> Vec { - self.urls.lock().unwrap().clone() - } - - /// Case-insensitive lookup of the last request's header value. - fn last_header(&self, name: &str) -> Option { - let headers = self.last_headers.lock().unwrap(); - headers - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case(name)) - .map(|(_, v)| v.clone()) - } - - fn last_body(&self) -> Option> { - self.last_body.lock().unwrap().clone() - } -} - -#[async_trait] -impl HttpBackend for MockBackend { - async fn send(&self, request: HttpRequest) -> Result { - self.calls.fetch_add(1, Ordering::SeqCst); - *self.last_url.lock().unwrap() = Some(request.url.clone()); - self.urls.lock().unwrap().push(request.url.clone()); - *self.last_headers.lock().unwrap() = request.headers.clone(); - *self.last_body.lock().unwrap() = request.body.clone(); - let mut queue = self.responses.lock().unwrap(); - let outcome = if queue.len() > 1 { - queue.remove(0) - } else { - // Repeat the last scripted outcome for any further calls. - queue - .first() - .cloned() - .expect("at least one scripted response") - }; - match outcome { - MockOutcome::Status(status, body) => Ok(HttpResponse { - status, - headers: Default::default(), - body, - }), - MockOutcome::NetworkError => Err(ApifyClientError::Http("boom".to_string())), - } - } -} - -fn client_with(backend: Arc, max_retries: u32) -> ApifyClient { - ApifyClient::builder() - .token("test-token") - .max_retries(max_retries) - // Keep backoff tiny so tests run fast. - .min_delay_between_retries(Duration::from_millis(1)) - .http_backend(backend) - .build() -} +use serde_json::json; /// Builds a client with an explicit request-compression algorithm and no retries. fn client_with_compression( @@ -332,7 +237,7 @@ async fn api_error_status_predicates_classify_by_status_code() { let client = client_with(backend, 0); client .me() - .get() + .limits() .await .expect_err("should fail") .as_api_error() @@ -506,7 +411,10 @@ async fn log_get_sends_raw_query_param() { client .run("some-run-id") .log() - .get_with_options(apify_client::LogOptions { raw: Some(true) }) + .get_with_options(apify_client::LogOptions { + raw: Some(true), + ..Default::default() + }) .await .expect("ok"); let url = backend.last_url().expect("a request was sent"); @@ -538,7 +446,8 @@ async fn log_stream_goes_through_the_configured_backend() { .log() .stream() .await - .expect("stream should open"); + .expect("stream should open") + .expect("the log should exist"); let mut collected = Vec::new(); while let Some(chunk) = stream.next().await { collected.extend(chunk.expect("chunk")); @@ -1556,3 +1465,127 @@ async fn call_wait_for_resources_does_not_count_toward_wait_secs() { .expect("the resource-limit retry must not eat into the wait_for_finish budget"); assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); } + +/// A request of a batch add, identified by its URL. +fn queue_request(url: &str) -> apify_client::models::RequestQueueRequest { + serde_json::from_value(json!({ "url": url })).expect("a valid request") +} + +/// The batch body is the JSON array of the requests, each serialized once, with the explicit JSON +/// content type. +#[tokio::test] +async fn batch_add_requests_sends_the_requests_as_one_json_array() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 200, + json!({ "data": { + "processedRequests": [ + { "uniqueKey": "https://example.com/1", "requestId": "1" }, + { "uniqueKey": "https://example.com/2", "requestId": "2" } + ], + "unprocessedRequests": [] + } }) + .to_string() + .into_bytes(), + )]); + let client = client_with(backend.clone(), 0); + let requests = vec![ + queue_request("https://example.com/1"), + queue_request("https://example.com/2"), + ]; + + let result = client + .request_queue("q") + .batch_add_requests(&requests, Default::default()) + .await + .expect("batch add"); + + assert_eq!(result.processed_requests.len(), 2); + assert_eq!(backend.call_count(), 1); + assert_eq!( + backend.last_body().unwrap(), + serde_json::to_vec(&requests).unwrap() + ); + assert_eq!( + backend.last_header("Content-Type").as_deref(), + Some("application/json") + ); +} + +/// Only the requests the API reports as unprocessed are sent again. +#[tokio::test] +async fn batch_add_requests_retries_only_the_unprocessed_requests() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + json!({ "data": { + "processedRequests": [{ "uniqueKey": "https://example.com/1", "requestId": "1" }], + "unprocessedRequests": [{ "uniqueKey": "https://example.com/2", "url": "https://example.com/2" }] + } }) + .to_string() + .into_bytes(), + ), + MockOutcome::Status( + 200, + json!({ "data": { + "processedRequests": [{ "uniqueKey": "https://example.com/2", "requestId": "2" }], + "unprocessedRequests": [] + } }) + .to_string() + .into_bytes(), + ), + ]); + let client = client_with(backend.clone(), 0); + let requests = vec![ + queue_request("https://example.com/1"), + queue_request("https://example.com/2"), + ]; + + let result = client + .request_queue("q") + .batch_add_requests( + &requests, + apify_client::BatchAddRequestsOptions { + min_delay_between_unprocessed_requests_retries: Some(Duration::from_millis(1)), + ..Default::default() + }, + ) + .await + .expect("batch add"); + + assert_eq!(result.processed_requests.len(), 2); + assert!(result.unprocessed_requests.is_empty()); + assert_eq!(backend.call_count(), 2); + assert_eq!( + backend.last_body().unwrap(), + serde_json::to_vec(&requests[1..]).unwrap(), + "the retry carries only the request that was not processed" + ); +} + +/// A request that cannot fit in any batch is rejected before a single batch is sent, not when its +/// batch comes up with earlier batches already in flight. +#[tokio::test] +async fn batch_add_requests_rejects_an_oversized_request_before_sending_anything() { + const OVERSIZED_URL_BYTES: usize = 10 * 1024 * 1024; + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"{}".to_vec())]); + let client = client_with(backend.clone(), 0); + let requests = vec![ + queue_request("https://example.com/small"), + queue_request(&format!( + "https://example.com/{}", + "x".repeat(OVERSIZED_URL_BYTES) + )), + ]; + + let err = client + .request_queue("q") + .batch_add_requests(&requests, Default::default()) + .await + .expect_err("an oversized request is rejected"); + + assert!( + matches!(err, ApifyClientError::InvalidArgument(_)), + "{err:?}" + ); + assert_eq!(backend.call_count(), 0, "nothing was sent"); +} diff --git a/tests/unit_resources.rs b/tests/unit_resources.rs new file mode 100644 index 0000000..cf358b2 --- /dev/null +++ b/tests/unit_resources.rs @@ -0,0 +1,465 @@ +//! Offline unit tests for how resource clients address resources and report a missing one: the +//! URL path of resource ids and keys, the clients chained from a parent, and the spec query +//! parameters of the list endpoints. +//! +//! These use a mock [`HttpBackend`](apify_client::http_client::HttpBackend) and require no network +//! access or `APIFY_TOKEN`. + +mod common; + +use apify_client::{ + ApifyClientError, ListRequestsOptions, LogOptions, RunListOptions, StoreListOptions, +}; +use common::mock_backend::{client_with, MockBackend, MockOutcome}; +use futures_util::StreamExt; +use serde_json::{json, Value}; + +fn not_found() -> MockOutcome { + MockOutcome::Status( + 404, + br#"{"error":{"type":"record-not-found","message":"missing"}}"#.to_vec(), + ) +} + +fn ok_data(data: Value) -> MockOutcome { + MockOutcome::Status(200, json!({ "data": data }).to_string().into_bytes()) +} + +fn is_not_found(result: Result) -> bool { + result + .err() + .and_then(|err| err.as_api_error().map(|api| api.is_not_found())) + .unwrap_or(false) +} + +fn is_invalid_argument(result: Result) -> bool { + matches!(result.err(), Some(ApifyClientError::InvalidArgument(_))) +} + +// ----- 404 handling ----------------------------------------------------------------------- + +/// A `404` resolves to `None` whatever its error type, for a client that names its resource. +#[tokio::test] +async fn any_not_found_resolves_to_none_for_a_client_with_an_id() { + let backend = MockBackend::new(vec![MockOutcome::Status(404, b"{}".to_vec())]); + let client = client_with(backend.clone(), 0); + assert!(client.actor("a").get().await.unwrap().is_none()); + assert!(client.dataset("d").get().await.unwrap().is_none()); + assert!(client.me().get().await.unwrap().is_none()); + client + .dataset("d") + .delete() + .await + .expect("idempotent delete"); +} + +/// A chained client without an id cannot tell a missing run from a missing dataset, so its `404` +/// is an error. +#[tokio::test] +async fn not_found_is_an_error_for_a_client_chained_from_a_run() { + let backend = MockBackend::new(vec![not_found()]); + let client = client_with(backend, 0); + let run = client.run("r"); + assert!(is_not_found(run.dataset().get().await)); + assert!(is_not_found(run.dataset().delete().await)); + assert!(is_not_found(run.key_value_store().get().await)); + assert!(is_not_found(run.key_value_store().delete().await)); + assert!(is_not_found(run.request_queue().get().await)); + assert!(is_not_found(run.request_queue().delete().await)); + assert!(is_not_found(run.log().get().await)); + assert!(is_not_found(client.build("b").log().get().await)); +} + +/// A record or a request is addressed by its own key, so a `404` is that record being missing, +/// even on a store or queue chained from a run. +#[tokio::test] +async fn a_missing_record_or_request_is_none_even_on_a_chained_client() { + let backend = MockBackend::new(vec![not_found()]); + let client = client_with(backend, 0); + let run = client.run("r"); + assert!(run + .key_value_store() + .get_record("key") + .await + .unwrap() + .is_none()); + assert!(!run.key_value_store().record_exists("key").await.unwrap()); + assert!(run + .request_queue() + .get_request("id") + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn a_log_is_none_for_a_client_with_an_id_and_an_error_when_chained() { + let backend = MockBackend::new(vec![not_found()]); + let client = client_with(backend, 0); + + assert!(client.log("r").get().await.unwrap().is_none()); + assert!(client.log("r").get_bytes().await.unwrap().is_none()); + assert!(client.log("r").stream().await.unwrap().is_none()); + + assert!(is_not_found(client.run("r").log().get_bytes().await)); + assert!(is_not_found( + client.run("r").log().stream().await.map(|_| ()) + )); + assert!(is_not_found( + client.run("r").get_streamed_log().await.map(|_| ()) + )); +} + +#[tokio::test] +async fn a_log_stream_is_yielded_for_an_existing_log() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"line".to_vec())]); + let client = client_with(backend, 0); + let mut stream = client + .log("r") + .stream() + .await + .unwrap() + .expect("the log exists"); + let chunk = stream.next().await.expect("a chunk").unwrap(); + assert_eq!(chunk, b"line"); +} + +/// The fixed sub-paths of a resource cannot tell a missing resource from a missing sub-resource, +/// so a `404` is an error even for a client with an id. +#[tokio::test] +async fn fixed_sub_paths_return_not_found_as_an_error() { + let backend = MockBackend::new(vec![not_found()]); + let client = client_with(backend, 0); + assert!(is_not_found(client.dataset("d").get_statistics().await)); + assert!(is_not_found(client.task("t").get_input().await)); + assert!(is_not_found(client.schedule("s").get_log().await)); + assert!(is_not_found( + client.build("b").get_openapi_definition().await + )); + assert!(is_not_found(client.me().limits().await)); + assert!(is_not_found(client.me().monthly_usage().await)); +} + +// ----- resource ids and keys -------------------------------------------------------------- + +/// An id that cannot be placed in a URL path fails every request without sending one. +#[tokio::test] +async fn an_empty_or_dot_segment_id_fails_without_a_request() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "x" }))]); + let client = client_with(backend.clone(), 0); + for id in ["", ".", ".."] { + assert!(is_invalid_argument(client.actor(id).get().await), "{id:?}"); + assert!(is_invalid_argument(client.dataset(id).get().await)); + assert!(is_invalid_argument(client.run(id).get().await)); + assert!(is_invalid_argument(client.user(id).get().await)); + } + assert_eq!(backend.call_count(), 0); +} + +/// The failure follows a client chained from a client with an invalid id. +#[tokio::test] +async fn a_client_chained_from_an_invalid_id_fails_too() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "x", "items": [] }))]); + let client = client_with(backend.clone(), 0); + let actor = client.actor(".."); + assert!(is_invalid_argument( + actor + .runs() + .list(Default::default(), Default::default()) + .await + )); + assert!(is_invalid_argument(actor.version("0.1").get().await)); + assert!(is_invalid_argument( + actor.version("0.1").env_var("A").get().await + )); + assert!(is_invalid_argument(client.run("..").dataset().get().await)); + assert!(is_invalid_argument(client.run("..").log().get().await)); + assert!(is_invalid_argument(client.build("..").log().get().await)); + assert!(is_invalid_argument( + client.run("..").log().stream().await.map(|_| ()) + )); + assert_eq!(backend.call_count(), 0); +} + +#[tokio::test] +async fn an_empty_version_or_env_var_name_is_rejected() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "x" }))]); + let client = client_with(backend.clone(), 0); + assert!(is_invalid_argument( + client.actor("a").version("").get().await + )); + assert!(is_invalid_argument( + client.actor("a").version("0.1").env_var("").get().await + )); + assert_eq!(backend.call_count(), 0); +} + +/// Every `/` of an id is the `~` separator of the `username~name` form. +#[tokio::test] +async fn every_slash_of_an_id_becomes_a_tilde() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "x" }))]); + let client = client_with(backend.clone(), 0); + client.actor("user/name/extra").get().await.unwrap(); + assert!(backend + .last_url() + .unwrap() + .ends_with("/v2/actors/user~name~extra")); + assert_eq!(apify_client::common::to_safe_id("a/b/c"), "a~b~c"); +} + +/// An id is percent-encoded, so it cannot add a path segment or a query string. +#[tokio::test] +async fn an_id_is_percent_encoded_in_the_path() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "x" }))]); + let client = client_with(backend.clone(), 0); + client.dataset("a b?x=1#y").get().await.unwrap(); + assert!(backend + .last_url() + .unwrap() + .ends_with("/v2/datasets/a%20b%3Fx%3D1%23y")); +} + +#[tokio::test] +async fn a_record_key_is_validated_and_percent_encoded() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, Vec::new())]); + let client = client_with(backend.clone(), 0); + let store = client.key_value_store("s"); + + for key in ["", ".", ".."] { + assert!(is_invalid_argument(store.get_record(key).await), "{key:?}"); + assert!(is_invalid_argument(store.record_exists(key).await)); + assert!(is_invalid_argument(store.delete_record(key).await)); + assert!(is_invalid_argument( + store.set_record_raw(key, Vec::new(), "text/plain").await + )); + assert!(is_invalid_argument(store.get_record_public_url(key).await)); + } + assert_eq!(backend.call_count(), 0); + + store.get_record("../a b").await.unwrap(); + assert!(backend.last_url().unwrap().contains("/records/..%2Fa%20b?")); +} + +#[tokio::test] +async fn a_request_id_is_validated_and_percent_encoded() { + let backend = MockBackend::new(vec![ok_data( + json!({ "id": "x", "url": "https://example.com" }), + )]); + let client = client_with(backend.clone(), 0); + let queue = client.request_queue("q"); + + for id in ["", ".", ".."] { + assert!(is_invalid_argument(queue.get_request(id).await), "{id:?}"); + assert!(is_invalid_argument(queue.delete_request(id).await)); + assert!(is_invalid_argument( + queue.prolong_request_lock(id, 60, false).await + )); + assert!(is_invalid_argument( + queue.delete_request_lock(id, false).await + )); + } + assert_eq!(backend.call_count(), 0); + + queue.get_request("a/b").await.unwrap(); + assert!(backend.last_url().unwrap().contains("/requests/a%2Fb")); +} + +// ----- query parameters of the spec ------------------------------------------------------- + +#[tokio::test] +async fn run_list_filters_by_standby_deployment() { + let backend = MockBackend::new(vec![ok_data(json!({ "items": [] }))]); + let client = client_with(backend.clone(), 0); + client + .runs() + .list( + Default::default(), + RunListOptions { + status: vec!["RUNNING".to_string()], + deployment: vec![ + "SINGLE_TENANT".to_string(), + "MULTI_TENANT_OWNER_PAID".to_string(), + ], + ..Default::default() + }, + ) + .await + .unwrap(); + let url = backend.last_url().unwrap(); + assert!( + url.contains("deployment=SINGLE_TENANT%2CMULTI_TENANT_OWNER_PAID"), + "{url}" + ); + + client + .runs() + .list(Default::default(), Default::default()) + .await + .unwrap(); + assert!(!backend.last_url().unwrap().contains("deployment")); +} + +#[tokio::test] +async fn store_list_can_include_the_input_schema() { + let backend = MockBackend::new(vec![ok_data(json!({ "items": [] }))]); + let client = client_with(backend.clone(), 0); + client + .store() + .list(StoreListOptions { + include_input_schema: Some(true), + ..Default::default() + }) + .await + .unwrap(); + assert!(backend.last_url().unwrap().contains("includeInputSchema=1")); +} + +#[tokio::test] +async fn log_download_is_sent_when_set() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"log".to_vec())]); + let client = client_with(backend.clone(), 0); + client + .log("r") + .get_with_options(LogOptions { + download: Some(true), + ..Default::default() + }) + .await + .unwrap(); + assert!(backend.last_url().unwrap().contains("download=1")); + client.log("r").get().await.unwrap(); + assert!(!backend.last_url().unwrap().contains("download")); +} + +#[tokio::test] +async fn request_list_has_no_exclusive_start_id() { + let backend = MockBackend::new(vec![ok_data(json!({ "items": [] }))]); + let client = client_with(backend.clone(), 0); + client + .request_queue("q") + .list_requests(ListRequestsOptions { + cursor: Some("c".to_string()), + ..Default::default() + }) + .await + .unwrap(); + let url = backend.last_url().unwrap(); + assert!(url.contains("cursor=c")); + assert!(!url.contains("exclusiveStartId")); +} + +/// The by-id user endpoints are not limited to the current user. +#[tokio::test] +async fn limits_and_usage_work_for_a_user_other_than_me() { + let backend = MockBackend::new(vec![ok_data(json!({ "monthlyUsageCycle": {} }))]); + let client = client_with(backend.clone(), 0); + let user = client.user("someone"); + + user.limits().await.unwrap(); + assert!(backend + .last_url() + .unwrap() + .contains("/v2/users/someone/limits")); + user.monthly_usage().await.unwrap(); + assert!(backend + .last_url() + .unwrap() + .contains("/v2/users/someone/usage/monthly")); + user.update_limits(&json!({ "dataRetentionDays": 7 })) + .await + .unwrap(); + assert!(backend + .last_url() + .unwrap() + .contains("/v2/users/someone/limits")); + assert_eq!( + backend.last_body().unwrap(), + br#"{"dataRetentionDays":7}"#.to_vec() + ); +} + +// ----- dataset items and stored requests -------------------------------------------------- + +#[tokio::test] +async fn push_items_accepts_an_object_or_an_array_of_objects() { + let backend = MockBackend::new(vec![MockOutcome::Status(201, Vec::new())]); + let client = client_with(backend.clone(), 0); + let dataset = client.dataset("d"); + + dataset.push_items(&json!({ "a": 1 })).await.unwrap(); + dataset + .push_items(&json!([{ "a": 1 }, { "b": { "c": [1, 2] } }])) + .await + .unwrap(); + dataset.push_items(&Vec::::new()).await.unwrap(); + assert_eq!(backend.call_count(), 3); + assert_eq!( + backend.last_body().unwrap(), + b"[]".to_vec(), + "the items are sent as serialized" + ); +} + +/// The endpoint only accepts an object or an array of objects, so anything else fails at once +/// instead of at request time. +#[tokio::test] +async fn push_items_rejects_any_other_shape_without_a_request() { + let backend = MockBackend::new(vec![MockOutcome::Status(201, Vec::new())]); + let client = client_with(backend.clone(), 0); + let dataset = client.dataset("d"); + + assert!(is_invalid_argument(dataset.push_items(&"text").await)); + assert!(is_invalid_argument( + dataset.push_items(&vec!["a", "b"]).await + )); + assert!(is_invalid_argument(dataset.push_items(&7).await)); + assert!(is_invalid_argument(dataset.push_items(&Value::Null).await)); + assert!(is_invalid_argument( + dataset.push_items(&json!([{ "a": 1 }, 5])).await + )); + assert!(is_invalid_argument( + dataset.push_items(&json!([[{ "a": 1 }]])).await + )); + assert_eq!(backend.call_count(), 0); +} + +/// A request that was added but not handled yet comes back with a `null` `handledAt`, and with +/// only the fields the API requires. +#[tokio::test] +async fn a_pending_request_with_a_null_handled_at_deserializes() { + let backend = MockBackend::new(vec![ok_data(json!({ + "id": "r1", + "uniqueKey": "https://example.com", + "url": "https://example.com", + "handledAt": null + }))]); + let client = client_with(backend, 0); + let request = client + .request_queue("q") + .get_request("r1") + .await + .unwrap() + .expect("the request exists"); + assert_eq!(request.id.as_deref(), Some("r1")); + assert_eq!(request.method, None); + assert_eq!(request.extra.get("handledAt"), Some(&Value::Null)); +} + +/// The API returns a `null` `uiState` once an update changed the schedule's `cronExpression` or +/// `timezone`, although the spec types it as an object. +#[tokio::test] +async fn a_schedule_with_a_null_ui_state_deserializes() { + let backend = MockBackend::new(vec![ok_data(json!({ + "id": "s1", + "cronExpression": "0 * * * *", + "uiState": null + }))]); + let client = client_with(backend, 0); + let schedule = client + .schedule("s1") + .get() + .await + .unwrap() + .expect("the schedule exists"); + assert_eq!(schedule.extra.get("uiState"), Some(&Value::Null)); +} diff --git a/tests/unit_timeouts.rs b/tests/unit_timeouts.rs new file mode 100644 index 0000000..f37045e --- /dev/null +++ b/tests/unit_timeouts.rs @@ -0,0 +1,424 @@ +//! Offline unit tests for the timeout tiers, the default headers and the request-body compressors. +//! +//! These use a mock [`HttpBackend`](apify_client::http_client::HttpBackend) and require no network +//! access or `APIFY_TOKEN`. + +mod common; + +use std::sync::Arc; +use std::time::Duration; + +use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, ReqwestBackend}; +use apify_client::timeouts::NO_TIMEOUT; +use apify_client::{ + ActorStartOptions, ApifyClient, ApifyClientError, BrotliHttpCompressor, GzipHttpCompressor, + HttpCompressor, Timeout, +}; +use common::mock_backend::{client_with, MockBackend, MockOutcome}; +use serde_json::{json, Value}; + +/// Minimum request-body size that gets compressed. +const COMPRESSIBLE_BODY_BYTES: usize = 4096; + +fn ok_data(data: Value) -> MockOutcome { + MockOutcome::Status(200, json!({ "data": data }).to_string().into_bytes()) +} + +/// A builder preconfigured with the mock backend, the way `client_with` does it. +fn builder_with(backend: Arc) -> apify_client::ApifyClientBuilder { + ApifyClient::builder() + .token("test-token") + .max_retries(0) + .min_delay_between_retries(Duration::from_millis(1)) + .http_backend(backend) +} + +#[tokio::test] +async fn simple_metadata_reads_use_the_short_tier() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = client_with(backend.clone(), 0); + client.actor("a").get().await.expect("get actor"); + assert_eq!(backend.last_timeout(), Duration::from_secs(5)); +} + +#[tokio::test] +async fn listing_uses_the_medium_tier() { + let backend = MockBackend::new(vec![ok_data(json!({ "items": [] }))]); + let client = client_with(backend.clone(), 0); + client + .actors() + .list(Default::default()) + .await + .expect("list actors"); + assert_eq!(backend.last_timeout(), Duration::from_secs(30)); +} + +#[tokio::test] +async fn dataset_items_use_the_long_tier() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"[]".to_vec())]); + let client = client_with(backend.clone(), 0); + client + .dataset("d") + .list_items::(Default::default()) + .await + .expect("list items"); + assert_eq!(backend.last_timeout(), Duration::from_secs(360)); +} + +#[tokio::test] +async fn builder_sets_the_duration_of_each_tier() { + let backend = MockBackend::new(vec![ + ok_data(json!({ "id": "a" })), + ok_data(json!({ "items": [] })), + MockOutcome::Status(200, b"[]".to_vec()), + ]); + let client = builder_with(backend.clone()) + .timeout_short(Duration::from_secs(1)) + .timeout_medium(Duration::from_secs(2)) + .timeout_long(Duration::from_secs(3)) + .build(); + client.actor("a").get().await.unwrap(); + client.actors().list(Default::default()).await.unwrap(); + client + .dataset("d") + .list_items::(Default::default()) + .await + .unwrap(); + assert_eq!( + backend.timeouts(), + vec![ + Duration::from_secs(1), + Duration::from_secs(2), + Duration::from_secs(3) + ] + ); +} + +#[tokio::test] +async fn timeout_max_caps_a_tier() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = builder_with(backend.clone()) + .timeout_max(Duration::from_secs(2)) + .build(); + client.actor("a").get().await.unwrap(); + assert_eq!(backend.last_timeout(), Duration::from_secs(2)); +} + +#[tokio::test] +async fn the_original_timeout_option_is_the_cap_on_an_attempt() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = builder_with(backend.clone()) + .timeout(Duration::from_secs(3)) + .build(); + client.actor("a").get().await.unwrap(); + assert_eq!(backend.last_timeout(), Duration::from_secs(3)); +} + +#[tokio::test] +async fn with_timeout_replaces_the_default_tier() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .with_timeout(Timeout::Long) + .get() + .await + .unwrap(); + client + .actor("a") + .with_timeout(Timeout::Custom(Duration::from_secs(7))) + .get() + .await + .unwrap(); + assert_eq!( + backend.timeouts(), + vec![Duration::from_secs(360), Duration::from_secs(7)] + ); +} + +#[tokio::test] +async fn an_explicit_timeout_above_the_cap_is_lowered_to_it() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .with_timeout(Timeout::Custom(Duration::from_secs(10_000))) + .get() + .await + .unwrap(); + assert_eq!(backend.last_timeout(), Duration::from_secs(360)); +} + +#[tokio::test] +async fn no_timeout_is_passed_to_the_backend_as_such() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .with_timeout(Timeout::NoTimeout) + .get() + .await + .unwrap(); + assert_eq!(backend.last_timeout(), NO_TIMEOUT); +} + +#[tokio::test] +async fn the_timeout_doubles_with_every_retry_up_to_the_cap() { + let backend = MockBackend::new(vec![MockOutcome::Status(500, b"{}".to_vec())]); + let client = builder_with(backend.clone()) + .max_retries(5) + .timeout_max(Duration::from_secs(30)) + .build(); + client.actor("a").get().await.expect_err("always fails"); + let seconds: Vec = backend.timeouts().iter().map(Duration::as_secs).collect(); + assert_eq!(seconds, vec![5, 10, 20, 30, 30, 30]); +} + +#[tokio::test] +async fn a_request_without_a_timeout_never_gets_one_on_retry() { + let backend = MockBackend::new(vec![MockOutcome::Status(500, b"{}".to_vec())]); + let client = builder_with(backend.clone()).max_retries(2).build(); + client + .actor("a") + .with_timeout(Timeout::NoTimeout) + .get() + .await + .expect_err("always fails"); + assert_eq!(backend.timeouts(), vec![NO_TIMEOUT; 3]); +} + +#[tokio::test] +async fn starting_a_run_adds_the_server_side_wait_to_the_tier() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "r" }))]); + let client = client_with(backend.clone(), 0); + let actor = client.actor("a"); + for wait_for_finish in [None, Some(10), Some(999)] { + actor + .start::( + None, + ActorStartOptions { + wait_for_finish, + ..Default::default() + }, + ) + .await + .unwrap(); + } + // The API holds a `waitForFinish` response for at most a minute, so the client allows the + // hold the caller asked for (capped there) on top of the `Medium` tier. + assert_eq!( + backend.timeouts(), + vec![ + Duration::from_secs(30), + Duration::from_secs(40), + Duration::from_secs(90) + ] + ); +} + +#[tokio::test] +async fn an_explicit_timeout_is_used_as_given_with_a_server_side_wait() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "r" }))]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .with_timeout(Timeout::Custom(Duration::from_secs(7))) + .start::( + None, + ActorStartOptions { + wait_for_finish: Some(30), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(backend.last_timeout(), Duration::from_secs(7)); +} + +#[tokio::test(start_paused = true)] +async fn polling_for_a_run_to_finish_has_no_client_side_timeout() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "r", "status": "SUCCEEDED" }))]); + let client = client_with(backend.clone(), 0); + client.run("r").wait_for_finish(None).await.unwrap(); + assert_eq!(backend.last_timeout(), NO_TIMEOUT); +} + +/// The default backend must not apply a timeout of its own to a request without one. +#[tokio::test] +async fn reqwest_backend_sends_a_request_without_a_timeout() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = [0u8; 1024]; + let _ = socket.read(&mut buffer).await.unwrap(); + socket + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}") + .await + .unwrap(); + }); + + let response = ReqwestBackend::new() + .send(HttpRequest { + method: HttpMethod::Get, + url: format!("http://{address}/"), + headers: Default::default(), + body: None, + timeout: NO_TIMEOUT, + }) + .await + .expect("a request without a timeout is sent"); + assert_eq!(response.status, 200); +} + +/// The default backend aborts a request that outlives its timeout. +#[tokio::test] +async fn reqwest_backend_aborts_a_request_that_outlives_its_timeout() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + // Accept the connection, but never answer it. + let _silent_server = tokio::spawn(async move { + let (_socket, _) = listener.accept().await.unwrap(); + tokio::time::sleep(Duration::from_secs(30)).await; + }); + + let error = ReqwestBackend::new() + .send(HttpRequest { + method: HttpMethod::Get, + url: format!("http://{address}/"), + headers: Default::default(), + body: None, + timeout: Duration::from_millis(50), + }) + .await + .expect_err("the request must time out"); + assert!(matches!(error, ApifyClientError::Timeout), "{error:?}"); +} + +#[tokio::test] +async fn default_headers_win_over_the_built_in_ones_and_are_case_insensitive() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = builder_with(backend.clone()) + .default_header("X-Custom", "first") + .default_header("x-custom", "second") + .default_header("user-agent", "my-agent/1.0") + .build(); + client.actor("a").get().await.unwrap(); + assert_eq!(backend.last_header("X-Custom").as_deref(), Some("second")); + assert_eq!( + backend.last_header("User-Agent").as_deref(), + Some("my-agent/1.0") + ); + assert_eq!( + backend.last_header("Authorization").as_deref(), + Some("Bearer test-token"), + "an unrelated built-in header is kept" + ); +} + +#[tokio::test] +async fn a_request_header_wins_over_a_default_header() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, Vec::new())]); + let client = builder_with(backend.clone()) + .default_header("content-type", "text/csv") + .build(); + client + .key_value_store("s") + .set_record_raw("k", b"x".to_vec(), "text/plain") + .await + .unwrap(); + assert_eq!( + backend.last_header("Content-Type").as_deref(), + Some("text/plain") + ); +} + +/// A compressor that leaves the body as it is, to see the custom compressor reach the wire. +#[derive(Debug)] +struct IdentityCompressor; + +impl HttpCompressor for IdentityCompressor { + fn content_encoding(&self) -> &str { + "identity" + } + + fn compress(&self, data: &[u8]) -> Result, ApifyClientError> { + Ok(data.to_vec()) + } +} + +/// Uploads a compressible record through a client using `compressor`, and returns the backend that +/// saw the request. +async fn upload_with(compressor: Arc) -> Arc { + let backend = MockBackend::new(vec![MockOutcome::Status(200, Vec::new())]); + let client = builder_with(backend.clone()) + .http_compressor(compressor) + .build(); + client + .key_value_store("s") + .set_record_raw("k", vec![b'a'; COMPRESSIBLE_BODY_BYTES], "text/plain") + .await + .unwrap(); + backend +} + +#[tokio::test] +async fn a_custom_compressor_sets_the_content_encoding_and_the_body() { + let backend = upload_with(Arc::new(IdentityCompressor)).await; + assert_eq!( + backend.last_header("Content-Encoding").as_deref(), + Some("identity") + ); + assert_eq!( + backend.last_body().unwrap(), + vec![b'a'; COMPRESSIBLE_BODY_BYTES] + ); +} + +#[tokio::test] +async fn brotli_quality_is_configurable() { + let compressor = BrotliHttpCompressor::new(11).expect("11 is a valid brotli quality"); + let backend = upload_with(Arc::new(compressor)).await; + assert_eq!( + backend.last_header("Content-Encoding").as_deref(), + Some("br") + ); + let sent = backend.last_body().unwrap(); + let mut decompressed = Vec::new(); + let mut decoder = brotli::DecompressorWriter::new(&mut decompressed, 4096); + std::io::Write::write_all(&mut decoder, &sent).expect("decode"); + drop(decoder); + assert_eq!(decompressed, vec![b'a'; COMPRESSIBLE_BODY_BYTES]); +} + +#[tokio::test] +async fn gzip_quality_is_configurable() { + let compressor = GzipHttpCompressor::new(1).expect("1 is a valid gzip quality"); + let backend = upload_with(Arc::new(compressor)).await; + assert_eq!( + backend.last_header("Content-Encoding").as_deref(), + Some("gzip") + ); + let sent = backend.last_body().unwrap(); + let mut decompressed = Vec::new(); + std::io::Read::read_to_end( + &mut flate2::read::GzDecoder::new(&sent[..]), + &mut decompressed, + ) + .expect("decode"); + assert_eq!(decompressed, vec![b'a'; COMPRESSIBLE_BODY_BYTES]); +} + +#[test] +fn a_quality_outside_the_range_of_the_algorithm_is_rejected() { + assert!(matches!( + BrotliHttpCompressor::new(12), + Err(ApifyClientError::InvalidArgument(_)) + )); + assert!(matches!( + GzipHttpCompressor::new(0), + Err(ApifyClientError::InvalidArgument(_)) + )); +} diff --git a/tests/user.rs b/tests/user.rs index f209579..b7d2dfb 100644 --- a/tests/user.rs +++ b/tests/user.rs @@ -87,3 +87,49 @@ async fn get_limits() { let limits = client.me().limits().await.expect("get limits"); assert!(limits.is_object(), "limits should be a JSON object"); } + +/// Simple GET: the by-id limits and monthly usage endpoints answer for the caller's own account, +/// not only for `me`. +#[tokio::test(flavor = "multi_thread")] +async fn get_limits_and_monthly_usage_by_user_id() { + let client = require_client!(); + let user = client + .me() + .get() + .await + .expect("get current user") + .expect("current user should exist"); + + let by_id = client.user(&user.id); + let limits = by_id.limits().await.expect("get limits by user id"); + assert!(limits.is_object(), "limits should be a JSON object"); + let usage = by_id + .monthly_usage() + .await + .expect("get monthly usage by user id"); + assert!(usage.is_object(), "monthly usage should be a JSON object"); +} + +/// A resource client with an explicit timeout aborts a request that outlives it. With no retries +/// the first attempt is the only one, and no real request completes within a millisecond. +#[tokio::test(flavor = "multi_thread")] +async fn with_timeout_aborts_a_request_that_outlives_it() { + let Some(builder) = common::make_client_builder() else { + eprintln!("Skipping: APIFY_TOKEN is not set"); + return; + }; + let client = builder.max_retries(0).build(); + + let error = client + .me() + .with_timeout(apify_client::Timeout::Custom( + std::time::Duration::from_millis(1), + )) + .get() + .await + .expect_err("no response arrives within a millisecond"); + assert!( + matches!(error, apify_client::ApifyClientError::Timeout), + "{error:?}" + ); +} From 39b287b144777ad2c8cec2c9b5a8709aceff534b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 15:26:51 +0000 Subject: [PATCH 02/11] fix: address review of the v3 port Adds statistics, do-not-retry-on-timeout for non-idempotent calls, the request queue timeout cap, typed normalized URL fields and the remaining reference tiers; keeps the deprecated exclusiveStartId and the reference gzip quality. --- CHANGELOG.md | 35 ++++-- README.md | 51 +++++++- docs/README.md | 22 +++- docs/misc.md | 9 +- docs/runs.md | 6 +- docs/storages.md | 3 +- examples/log_redirection.rs | 6 +- examples/raw_log.rs | 13 +- src/client.rs | 8 ++ src/clients/base.rs | 135 +++++++++++++++------ src/clients/dataset.rs | 17 ++- src/clients/key_value_store.rs | 104 ++++++++++++---- src/clients/request_queue.rs | 23 +++- src/clients/run.rs | 27 +++-- src/common.rs | 5 +- src/error.rs | 29 +++-- src/http_client.rs | 82 +++++++++---- src/http_compressors.rs | 55 ++++++--- src/lib.rs | 2 + src/models.rs | 167 +++++++++++++++++++++++++- src/statistics.rs | 83 +++++++++++++ tests/common/mock_backend.rs | 3 + tests/common/mod.rs | 4 +- tests/unit_resources.rs | 24 +++- tests/unit_timeouts.rs | 210 ++++++++++++++++++++++++++++++++- 25 files changed, 946 insertions(+), 177 deletions(-) create mode 100644 src/statistics.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ba9e39f..0d5bc5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,20 @@ Ports the `apify-client-js` v3 changes. ### Added - Timeout tiers: `Timeout` (`Short`, `Medium`, `Long`, `NoTimeout`, `Custom`), the builder options - `timeout_short`, `timeout_medium`, `timeout_long` and `timeout_max`, and `with_timeout` on every - resource client. + `timeout_short`, `timeout_medium`, `timeout_long` and `timeout_max`, `with_timeout` on every + resource client and `RequestQueueClient::with_timeout_cap`. - `HttpCompressor` with `BrotliHttpCompressor` and `GzipHttpCompressor` (configurable quality), set with `ApifyClientBuilder::http_compressor`. - `ApifyClientBuilder::default_header`. -- `RunListOptions::deployment`, `StoreListOptions::include_input_schema` and - `LogOptions::download`. -- Offline tests for the timeout tiers, the 404 handling, id validation and batch add. +- `ApifyClient::stats`: the `Statistics` of the calls, requests and `429` responses. +- `HttpClient::call_with_options` and `CallOptions::do_not_retry_timeouts`; + `KeyValueStoreClient::set_record_raw_with_options`, `set_record_json_with_options` and + `SetRecordOptions`. +- `RunListOptions::deployment`, `StoreListOptions::include_input_schema` and `LogOptions::download`. +- Typed, normalized URL fields: `ActorRun::console_url` and `output`, `Task::standby_url`, + `Dataset::console_url` and `items_public_url`, `KeyValueStore::console_url`, `keys_public_url` and + `records_public_url`, `KeyValueStoreKey::record_public_url`, `RequestQueue::console_url`, + `ActorStoreListItem::url` and `user_picture_url`, `User::profile`, `WebhookDispatch::webhook`. ### Changed - Synced with Apify OpenAPI spec `v2-2026-10-29T100357Z` (`API_SPEC_VERSION`). @@ -25,26 +31,29 @@ Ports the `apify-client-js` v3 changes. timeout; `timeout` is now an alias of `timeout_max`. `wait_for_finish` has no timeout, and a `waitForFinish` request gets the server-side wait added to its tier. - **Breaking:** `HttpRequest::timeout` may be `timeouts::NO_TIMEOUT`, which a custom `HttpBackend` - must not abort. -- **Breaking:** `RetryConfig` no longer has a `timeout` field. + must not abort. `RetryConfig` no longer has a `timeout` field. - **Breaking:** any `404` resolves to `None`, not only `record-not-found`, but only for a client that names its resource by id. `run.dataset()`, `run.log()` and the other chained clients return it as an error. - **Breaking:** `get_statistics`, `get_input`, `schedule.get_log` and `get_openapi_definition` return the `404` as an error and no longer wrap the result in `Option`. -- **Breaking:** `LogClient::stream` and `stream_with_options` return `Option`, `None` for a missing log. +- **Breaking:** `LogClient::stream`, `stream_with_options`, `RunClient::get_streamed_log` and + `get_streamed_log_with_options` return an `Option`: `None` for a missing log or run. - **Breaking:** `ActorStartOptions::timeout_secs` and `RunResurrectOptions::timeout_secs` are now `run_timeout_secs`. -- **Breaking:** `ListRequestsOptions::exclusive_start_id` is removed; use `cursor`. +- **Breaking:** `ListRequestsOptions::exclusive_start_id` is deprecated in favor of `cursor`. - **Breaking:** `LogOptions`, `RunListOptions` and `StoreListOptions` gained a public field. - **Breaking:** an id, record key or request id that is empty or a dot segment fails with `InvalidArgument`; ids are percent-encoded and every `/` of an id becomes `~`. -- **Breaking:** `dataset.push_items` rejects anything but an object or an array of objects. -- **Breaking:** `ActorRun::container_url` and `Webhook::request_url` are normalized URLs. +- **Breaking:** `dataset.push_items` rejects anything but an object or an array of objects, and does + not retry a transport timeout. +- **Breaking:** the URL fields listed above are normalized, and no longer in `extra`. +- **Breaking:** `RequestCompression` moved to `http_compressors` (still re-exported at the crate + root). - `UserClient::limits`, `monthly_usage` and `update_limits` work for any user, not only `me`. -- The default gzip quality is 9. - `batch_add_requests` serializes each request once and rejects an oversized one before sending. -- The `list` integration tests wait out replica lag for 87 seconds. +- Key-value store `update`, `record_exists` and the public URL methods, and + `DatasetClient::create_items_public_url`, use the `Long` tier. ## [0.14.1] - 2026-10-26 diff --git a/README.md b/README.md index a922144..56444f6 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,13 @@ applies to each attempt separately and doubles with every retry, up to the cap. A resource client's `with_timeout` replaces the tier of every request that client sends, with a tier, an exact duration, or `Timeout::NoTimeout` to let a request run for as long as it takes. With `NoTimeout` the client never aborts the request, so a connection that stalls is not retried -either. Clients obtained from that client, such as `run.dataset()`, do not inherit the timeout. +either. A zero duration is rejected: every request then fails with `InvalidArgument`. Clients +obtained from that client, such as `run.dataset()`, do not inherit the timeout. A request queue +client also has `with_timeout_cap(duration)`, which only lowers the tiers longer than `duration`. + +A call that is not idempotent is not retried after a transport timeout, because the request may +have reached the API: `dataset.push_items` never retries one, and the `set_record_*_with_options` +methods of a key-value store do not either when `do_not_retry_timeouts` is set. ```rust,no_run use std::time::Duration; @@ -155,7 +161,7 @@ matching `Content-Encoding` header. The default is Brotli (`br`); select (`use apify_client::RequestCompression;`). To tune the quality or use another algorithm, pass an `HttpCompressor` to `ApifyClientBuilder::http_compressor`: the built-in `BrotliHttpCompressor` (quality 0 to 11, default 6) and `GzipHttpCompressor` (quality 1 to 9, -default 9), or your own implementation of the trait. +default 6), or your own implementation of the trait. ```rust,no_run use std::sync::Arc; @@ -174,6 +180,12 @@ The compressor is skipped for bodies under 1024 bytes, for content types that al own compression (images, audio, video, archives, ...), and for a request that sets its own `Content-Encoding`. +### Statistics + +`client.stats()` counts the calls made through the client and every resource client obtained from +it: `calls()` (each call counts once), `requests()` (every attempt counts) and +`rate_limit_errors()` (the `429` responses, indexed by attempt). + ### User-Agent Every request carries a `User-Agent` of the form @@ -362,8 +374,39 @@ built-in `ReqwestBackend` with a `reqwest::Client` you pre-configure (custom pro pool, …), as shown below; to route requests through an entirely different HTTP stack, implement `HttpBackend` on your own type instead — just `async fn send`; `send_streaming` (used by `log().stream()` and `download_items_stream()`) has a default implementation that buffers the -whole response via `send`, so it is optional unless you want true incremental streaming. The -snippet below constructs a +whole response via `send`, so it is optional unless you want true incremental streaming. A request +whose `timeout` is `timeouts::NO_TIMEOUT` (see [Timeouts](#timeouts)) may run for as long as it +takes, so a backend must check for it before it applies a timeout of its own: + +```rust,no_run +use apify_client::http_client::{HttpBackend, HttpRequest, HttpResponse}; +use apify_client::timeouts::NO_TIMEOUT; +use apify_client::ApifyClientError; +use async_trait::async_trait; + +#[derive(Debug)] +struct MyBackend; + +#[async_trait] +impl HttpBackend for MyBackend { + async fn send(&self, request: HttpRequest) -> Result { + let send = async { + // Send the request with your own HTTP stack. + let response: Result = todo!(); + response + }; + if request.timeout == NO_TIMEOUT { + send.await + } else { + tokio::time::timeout(request.timeout, send) + .await + .map_err(|_| ApifyClientError::Timeout)? + } + } +} +``` + +The snippet below constructs a `reqwest::Client` directly, so it requires adding `reqwest = "0.12"` (matching this crate's `reqwest` version) as your own dependency — see [Installation](#installation): diff --git a/docs/README.md b/docs/README.md index e9bbf97..fd5d8af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -147,8 +147,8 @@ Builder options: | `user_agent_suffix(s)` | none | Extra text appended to the `User-Agent` header. | | `default_header(name, value)` | none | Header sent with every request. It wins over the built-in `User-Agent` and `Authorization` headers; a header a request sets itself wins over it. Names are compared case-insensitively. | | `request_compression(c)` | `Brotli` | Encoding for large request bodies: `RequestCompression::Brotli` (`br`) or `RequestCompression::Gzip` (`gzip`). | -| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11), `GzipHttpCompressor::new(quality)` (1 to 9), or your own implementation. | -| `http_backend(b)` | reqwest | Replaceable HTTP transport (`apify_client::http_client::{HttpBackend, ReqwestBackend}`). | +| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11, default 6), `GzipHttpCompressor::new(quality)` (1 to 9, default 6), or your own implementation. | +| `http_backend(b)` | reqwest | Replaceable HTTP transport (`apify_client::http_client::{HttpBackend, ReqwestBackend}`). A request whose `timeout` is `timeouts::NO_TIMEOUT` may run for as long as it takes, so a backend must check for it before it applies a timeout of its own. | Request bodies of at least 1024 bytes are compressed once (before retries) with the selected `request_compression` algorithm and sent with the matching `Content-Encoding` header. Brotli is @@ -182,8 +182,18 @@ retry, up to the same cap. Every resource client has a `with_timeout(Timeout)` method that replaces the tier of all the requests it sends with a tier, `Timeout::Custom(duration)`, or `Timeout::NoTimeout` (the client -never aborts the request, so a stalled connection is not retried either). Clients obtained from it, -such as `run.dataset()`, do not inherit the timeout. +never aborts the request, so a stalled connection is not retried either). A zero duration is +rejected: every request then fails with `InvalidArgument`. Clients obtained from it, such as +`run.dataset()`, do not inherit the timeout. A request queue client also has +`with_timeout_cap(duration)`, which only lowers the tiers longer than `duration`. + +`dataset.push_items` is not retried after a transport timeout, because the request may have +reached the API and a retry could store the items twice. The same goes for the key-value store +`set_record_*_with_options` methods when `do_not_retry_timeouts` is set. + +`client.stats()` returns the `Statistics` of the client and of every resource client obtained from +it: `calls()` (each call counts once), `requests()` (every attempt counts) and `rate_limit_errors()` +(the `429` responses, indexed by attempt). ```rust,no_run use std::time::Duration; @@ -293,8 +303,8 @@ Every fallible method returns `Result`. The variants are: `get`/`delete` map a missing resource (any `404`) to `Ok(None)` / a no-op, on a client that names its resource by id. A client chained from a parent, such as `run.dataset()` or `run.log()`, and the methods that address a fixed sub-path of a resource (`get_statistics`, `get_input`, `get_log`, -`get_openapi_definition`) return the `404` as an error, because the response cannot tell which -resource is missing. +`get_openapi_definition`, `limits`, `monthly_usage*`, webhook `test`) return the `404` as an error, +because the response cannot tell which resource is missing. An id that is empty or a dot segment (`.` or `..`) cannot be placed in a URL path, so every request through a client created with it fails with `InvalidArgument`. diff --git a/docs/misc.md b/docs/misc.md index 69ca467..73da7de 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -148,7 +148,8 @@ Also reachable via `run.log()` and `build.log()`. `get*` and `stream*` return `None` for a missing log only on a client that names its log by id (`client.log(id)`). A client chained from a run or build (`run.log()`, `build.log()`) has no id of its own, so it returns the `404` as an error: the response cannot tell a missing run from a missing -log. `run.get_streamed_log()` returns the stream directly and fails for a missing run. +log. `run.get_streamed_log()` returns `None` for a missing run, because the run is the resource +that client names. `LogOptions` has two fields, `raw` and `download`, both `Option`. When `raw` is `Some(true)`, the API returns the raw log content without server-side processing (e.g. without the per-line @@ -195,7 +196,11 @@ use apify_client::ApifyClient; use futures_util::StreamExt; # async fn run(client: ApifyClient, run_id: &str) -> Result<(), Box> { -let mut stream = client.run(run_id).get_streamed_log().await?; +let mut stream = client + .run(run_id) + .get_streamed_log() + .await? + .ok_or("the run does not exist")?; while let Some(chunk) = stream.next().await { let chunk = chunk?; print!("{}", String::from_utf8_lossy(&chunk)); diff --git a/docs/runs.md b/docs/runs.md index 489a8d7..6ffe19d 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -25,11 +25,11 @@ collections are available via `actor.runs()` and `task.runs()`. | `charge(options)` | `RunChargeOptions` | `()` | Charges a pay-per-event run (always sends an idempotency key). | | `wait_for_finish(wait_secs)` | `Option` | `ActorRun` | Polls until the run is terminal. `None` waits indefinitely; `Some(n)` bounds the wait and may return a still-running (non-terminal) run if `n` elapses first. A run ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | | `dataset()` / `key_value_store()` / `request_queue()` / `log()` | — | resource client | Access the run's default storages and log. | -| `get_streamed_log()` | — | `Result>>>` (async — `.await` it) | Streams the run's log chunks live (log redirection). A run that does not exist is an error. | -| `get_streamed_log_with_options(options)` | `LogOptions` | `Result>>>` (async — `.await` it) | As `get_streamed_log()`, forwarding `LogOptions` (e.g. `raw`) to the log stream. | +| `get_streamed_log()` | — | `Result>>>>` (async — `.await` it) | Streams the run's log chunks live (log redirection); `None` if the run does not exist. | +| `get_streamed_log_with_options(options)` | `LogOptions` | `Result>>>>` (async — `.await` it) | As `get_streamed_log()`, forwarding `LogOptions` (e.g. `raw`) to the log stream. | `get_streamed_log()` / `get_streamed_log_with_options()` are `async` and yield the stream inside a -`Result`, so `.await?` them to obtain it. Polling the returned stream with `.next()` requires the +`Result` and an `Option`, so `.await?` them and handle the `None` of a missing run to obtain it. Polling the returned stream with `.next()` requires the `futures_util::StreamExt` trait (from the `futures-util` crate — add `futures-util = "0.3"` to your `Cargo.toml`) in scope; the [`raw_log`](../examples/raw_log.rs) example drives `get_streamed_log_with_options` exactly this way. See diff --git a/docs/storages.md b/docs/storages.md index 44113c3..3ee6cb9 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -156,6 +156,7 @@ println!("streamed {total_bytes} bytes of JSONL"); | `set_record_raw(key, bytes, content_type)` | `&str`, `Vec`, `&str` | `()` | Stores a raw record. | | `set_record_raw_with_content_encoding(key, bytes, content_type, content_encoding)` | `&str`, `Vec`, `&str`, `&str` | `()` | Like `set_record_raw`, for a value that is already compressed (`gzip`, `br`, `deflate`): sends `content_encoding` as the `Content-Encoding` header instead of compressing the bytes again. | | `set_record_json(key, value)` | `&str`, `&impl Serialize` | `()` | Stores a JSON record. | +| `set_record_raw_with_options(key, bytes, content_type, options)` / `set_record_json_with_options(key, value, options)` | as above plus `SetRecordOptions { content_encoding, do_not_retry_timeouts }` | `()` | Like the methods above; `do_not_retry_timeouts` gives up on the first transport timeout instead of retrying it. `content_encoding` is for raw bytes only. | | `delete_record(key)` | `&str` | `()` | Deletes a record. | | `get_record_with_options(key, options)` | `&str`, `GetRecordOptions { attachment, signature }` | `Option` | Reads a record with explicit attachment/signature options. | | `get_record_public_url(key)` | `&str` | `String` | Shareable (HMAC-signed for private) record URL. | @@ -221,7 +222,7 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: | `list_and_lock_head(lock_secs, limit)` | `i64`, `Option` | `LockedRequestQueueHead` | Locks head requests. | | `batch_add_requests(requests, options)` | `&[RequestQueueRequest]`, `BatchAddRequestsOptions` | `BatchRequestsOperationResult` | Batch add, with automatic chunking, bounded parallelism, and retries for rate-limited requests. | | `batch_delete_requests(requests)` | `&[impl Serialize]` | `BatchRequestsOperationResult` | Batch delete (max 25 requests per call; larger inputs are rejected client-side). | -| `list_requests(options)` | `ListRequestsOptions { limit, cursor, filter }` | `RequestQueueRequestsPage` | List requests (cursor/filter pagination). | +| `list_requests(options)` | `ListRequestsOptions { limit, cursor, filter }` (and the deprecated `exclusive_start_id`) | `RequestQueueRequestsPage` | List requests (cursor/filter pagination). | | `paginate_requests(page_limit)` | `Option` | `RequestQueueRequestsIterator` | Lazy request iterator. | | `prolong_request_lock(id, lock_secs, forefront)` | `&str`, `i64`, `bool` | `RequestLockInfo` | Extend a lock. | | `delete_request_lock(id, forefront)` | `&str`, `bool` | `()` | Release a lock. | diff --git a/examples/log_redirection.rs b/examples/log_redirection.rs index 78201c5..2cac0b9 100644 --- a/examples/log_redirection.rs +++ b/examples/log_redirection.rs @@ -36,7 +36,11 @@ async fn main() -> Result<(), Box> { // `"{name} -> "` convention the JS reference uses for redirected Actor logs). The log is // streamed as raw byte chunks, so we buffer until a newline to emit whole, prefixed lines. let prefix = format!("{source_actor} -> "); - let mut stream = client.run(&run.id).get_streamed_log().await?; + let mut stream = client + .run(&run.id) + .get_streamed_log() + .await? + .ok_or("the run does not exist")?; let mut buf = String::new(); while let Some(chunk) = stream.next().await { let chunk = chunk?; diff --git a/examples/raw_log.rs b/examples/raw_log.rs index c6d651b..83b6cc8 100644 --- a/examples/raw_log.rs +++ b/examples/raw_log.rs @@ -1,10 +1,9 @@ //! Fetch an Actor run's log in its raw (unprocessed) form. //! //! By default the API adds a per-line timestamp prefix to log lines. Passing -//! `LogOptions { raw: Some(true), ..Default::default() }` requests the unprocessed log instead — the same raw form -//! the JS reference's log redirection consumes. This example exercises both the buffered -//! (`get_with_options`) and the streaming (`stream_with_options`) raw-log paths end-to-end so -//! the `Test examples` CI step covers the new `raw` parameter against the live API. +//! `LogOptions { raw: Some(true), ..Default::default() }` requests the unprocessed log instead. +//! The example fetches it as a whole with `get_with_options`, and streams it with +//! `get_streamed_log_with_options`. //! //! Run with: `APIFY_TOKEN=... cargo run --example raw_log` @@ -25,8 +24,7 @@ async fn main() -> Result<(), Box> { // Buffered raw log: fetch the whole log without the server-added per-line timestamps. let raw_log = client - .run(&run.id) - .log() + .log(&run.id) .get_with_options(LogOptions { raw: Some(true), ..Default::default() @@ -44,7 +42,8 @@ async fn main() -> Result<(), Box> { raw: Some(true), ..Default::default() }) - .await?; + .await? + .ok_or("the run does not exist")?; let mut streamed_bytes = 0usize; while let Some(chunk) = stream.next().await { streamed_bytes += chunk?.len(); diff --git a/src/client.rs b/src/client.rs index cd55c9b..fd7eabc 100644 --- a/src/client.rs +++ b/src/client.rs @@ -223,6 +223,7 @@ impl ApifyClientBuilder { tiers: self.tiers, compressor: self.compressor, default_headers: self.default_headers, + stats: Arc::default(), }); let base_url = to_api_base_url(&self.base_url); @@ -266,6 +267,13 @@ impl ApifyClient { self.http.clone() } + /// Statistics of the API calls made through this client and every resource client obtained + /// from it: the number of calls, the number of requests (every attempt counts) and the number + /// of `429` rate-limit errors by attempt. + pub fn stats(&self) -> &crate::Statistics { + self.http.stats() + } + /// The `User-Agent` header value this client sends. pub fn user_agent(&self) -> &str { self.http.user_agent() diff --git a/src/clients/base.rs b/src/clients/base.rs index 56bbd76..c6c29a0 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -15,7 +15,7 @@ use crate::common::{ catch_not_found, parse_data_envelope, to_path_segment, to_safe_id, PaginationList, QueryParams, }; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest, HttpResponse}; +use crate::http_client::{CallOptions, HttpClient, HttpMethod, HttpRequest, HttpResponse}; use crate::timeouts::Timeout; /// How long to wait between polls while waiting for a run/build to finish. @@ -34,6 +34,10 @@ const NOT_FOUND_GRACE_PERIOD: Duration = Duration::from_secs(3); /// the [`HttpClient`] of such a context is poisoned and fails every call first. const INVALID_ID_PLACEHOLDER: &str = "invalid-id"; +/// Why a zero timeout is refused. +const ZERO_TIMEOUT_MESSAGE: &str = + "a timeout must be longer than zero; use Timeout::NoTimeout to run without one"; + /// The resolved context for a resource client: its base URL and the shared HTTP client. #[derive(Debug, Clone)] pub(crate) struct ResourceContext { @@ -52,6 +56,8 @@ pub(crate) struct ResourceContext { pub(crate) has_id: bool, /// Timeout set with the client's `with_timeout`, replacing the tier a method defaults to. pub(crate) timeout: Option, + /// Cap set with the client's `with_timeout_cap`: a tier longer than it is lowered to it. + pub(crate) timeout_cap: Option, } impl ResourceContext { @@ -94,9 +100,23 @@ impl ResourceContext { api_origin, has_id, timeout: None, + timeout_cap: None, } } + /// Replaces the tier of every request with `timeout`. A zero duration cannot be told apart + /// from an immediate failure of every attempt (`Timeout::NoTimeout` is the way to run without + /// a timeout), so it makes every request through this context fail with + /// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument), + /// like an invalid id does. + pub(crate) fn with_timeout(mut self, timeout: Timeout) -> Self { + if timeout == Timeout::Custom(Duration::ZERO) { + self.http = self.http.poisoned(ZERO_TIMEOUT_MESSAGE.to_string()); + } + self.timeout = Some(timeout); + self + } + /// Overrides the origin used when building public URLs. pub(crate) fn with_public_origin(mut self, public_origin: &str) -> Self { self.public_origin = origin_of(public_origin); @@ -134,7 +154,25 @@ impl ResourceContext { /// The timeout of a request that defaults to `tier`: the client's own override if it has one, /// resolved to the duration of the first attempt. pub(crate) fn timeout(&self, tier: Timeout) -> Duration { - self.http.resolve_timeout(self.timeout.unwrap_or(tier)) + match self.timeout { + Some(timeout) => self.http.resolve_timeout(timeout), + None => self.cap(self.http.resolve_timeout(tier)), + } + } + + /// Lowers the tier of every request to `cap` where it is longer. A zero cap is rejected like + /// a zero timeout (see [`with_timeout`](Self::with_timeout)). + pub(crate) fn with_timeout_cap(mut self, cap: Duration) -> Self { + if cap.is_zero() { + self.http = self.http.poisoned(ZERO_TIMEOUT_MESSAGE.to_string()); + } + self.timeout_cap = Some(cap); + self + } + + /// Lowers `timeout` to the cap of this context, if it has one. + fn cap(&self, timeout: Duration) -> Duration { + self.timeout_cap.map_or(timeout, |cap| timeout.min(cap)) } /// Like [`timeout`](Self::timeout), for a request that asks the API to hold its response for @@ -147,9 +185,10 @@ impl ResourceContext { ) -> Duration { match self.timeout { Some(timeout) => self.http.resolve_timeout(timeout), - None => self - .http - .resolve_timeout_with_wait_hold(tier, wait_for_finish_secs), + None => self.cap( + self.http + .resolve_timeout_with_wait_hold(tier, wait_for_finish_secs), + ), } } @@ -172,15 +211,32 @@ impl ResourceContext { headers: HashMap, body: Option>, timeout: Duration, + ) -> ApifyClientResult { + self.send_with_options(method, url, headers, body, timeout, CallOptions::default()) + .await + } + + /// Like [`send_with`](Self::send_with), with per-call [`CallOptions`]. + pub(crate) async fn send_with_options( + &self, + method: HttpMethod, + url: String, + headers: HashMap, + body: Option>, + timeout: Duration, + options: CallOptions, ) -> ApifyClientResult { self.http - .call(HttpRequest { - method, - url, - headers, - body, - timeout, - }) + .call_with_options( + HttpRequest { + method, + url, + headers, + body, + timeout, + }, + options, + ) .await } @@ -199,11 +255,13 @@ macro_rules! impl_with_timeout { /// method defaults to (see [`Timeout`](crate::Timeout)). /// /// The timeout applies to each attempt of a request separately and doubles with every - /// retry, up to the client's `timeout_max`. A longer timeout is lowered to that cap. + /// retry, up to the client's `timeout_max`. A longer timeout is lowered to that cap. A + /// zero timeout is rejected: every request then fails with `InvalidArgument`, and + /// [`Timeout::NoTimeout`](crate::Timeout::NoTimeout) is the way to run without one. /// Clients obtained from this one, such as `run.dataset()`, do not inherit it. #[must_use] pub fn with_timeout(mut self, timeout: $crate::Timeout) -> Self { - self.ctx.timeout = Some(timeout); + self.ctx = self.ctx.with_timeout(timeout); self } } @@ -448,41 +506,44 @@ pub(crate) async fn head_exists( } /// A `PUT` with raw bytes and a content type (used for KVS record uploads). +/// +/// `content_encoding`, when set, is sent as the `Content-Encoding` header for an +/// already-compressed value; it makes [`HttpClient::call`]'s automatic request compression skip +/// the body (it already refuses to double-encode a request that carries a `Content-Encoding`), so +/// the bytes reach the API exactly as given. `options` are the per-call [`CallOptions`]. pub(crate) async fn put_raw( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, - body: Vec, - content_type: &str, - tier: Timeout, -) -> ApifyClientResult<()> { - put_raw_with_content_encoding(ctx, sub_path, params, body, content_type, None, tier).await -} - -/// Like [`put_raw`], but can also set a `Content-Encoding` header (used for KVS record uploads -/// of an already-compressed value). Setting it makes [`HttpClient::call`]'s automatic request -/// compression skip the body (it already refuses to double-encode a request that carries a -/// `Content-Encoding`), so the bytes reach the API exactly as given. -pub(crate) async fn put_raw_with_content_encoding( - ctx: &ResourceContext, - sub_path: Option<&str>, - params: &QueryParams, - body: Vec, - content_type: &str, - content_encoding: Option<&str>, + upload: RawUpload<'_>, + options: CallOptions, tier: Timeout, ) -> ApifyClientResult<()> { let url = ctx.url_with_params(sub_path, params); let mut headers = HashMap::new(); - headers.insert("Content-Type".to_string(), content_type.to_string()); - if let Some(encoding) = content_encoding { + headers.insert("Content-Type".to_string(), upload.content_type.to_string()); + if let Some(encoding) = upload.content_encoding { headers.insert("Content-Encoding".to_string(), encoding.to_string()); } - ctx.send_with(HttpMethod::Put, url, headers, Some(body), ctx.timeout(tier)) - .await?; + ctx.send_with_options( + HttpMethod::Put, + url, + headers, + Some(upload.body), + ctx.timeout(tier), + options, + ) + .await?; Ok(()) } +/// The payload of a [`put_raw`] upload. +pub(crate) struct RawUpload<'a> { + pub(crate) body: Vec, + pub(crate) content_type: &'a str, + pub(crate) content_encoding: Option<&'a str>, +} + /// Opens a streaming `GET` connection to `url` (already fully built, including query string), /// for a caller that wants to process a large body (a log or a dataset export) incrementally /// instead of waiting for [`HttpClient::call`] to buffer the whole response. Delegates to @@ -597,7 +658,7 @@ where } } Err(err) => { - let is_not_found = err.as_api_error().is_some_and(|e| e.status_code == 404); + let is_not_found = err.as_api_error().is_some_and(|e| e.is_not_found()); if !is_not_found { return Err(err); } diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 43bf899..2b9a09a 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -11,7 +11,7 @@ use crate::clients::base::{ use crate::clients::pagination::ListIterator; use crate::common::{sign_storage_content, PaginationList, QueryParams}; use crate::error::{ApifyClientError, ApifyClientResult}; -use crate::http_client::{HttpClient, HttpMethod}; +use crate::http_client::{CallOptions, HttpClient, HttpMethod}; use crate::models::Dataset; use crate::timeouts::Timeout; @@ -405,6 +405,9 @@ impl DatasetClient { /// endpoint accepts. Anything else, such as a plain string or an array of strings, is /// rejected with [`ApifyClientError::InvalidArgument`] /// before a request is sent. + /// + /// A push that times out is not retried, because the request may have reached the API and a + /// retry could store the items twice; the other errors are retried as usual. pub async fn push_items(&self, items: &T) -> ApifyClientResult<()> { let body = serde_json::to_vec(items)?; check_items_shape(&body)?; @@ -414,13 +417,18 @@ impl DatasetClient { "Content-Type".to_string(), "application/json; charset=utf-8".to_string(), ); + // A push that timed out may have reached the API, so retrying it could store the items + // twice: the first transport timeout is final. self.ctx - .send_with( + .send_with_options( HttpMethod::Post, url, headers, Some(body), self.ctx.timeout(Timeout::Medium), + CallOptions { + do_not_retry_timeouts: true, + }, ) .await?; Ok(()) @@ -457,7 +465,10 @@ impl DatasetClient { options.apply(&mut params); params.add_str("format", format.map(|f| f.as_str().to_string())); - if let Some(dataset) = self.get().await? { + // Like the reference client, fetches the dataset with the `Long` tier of this method. + let dataset: Option = + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Long).await?; + if let Some(dataset) = dataset { if let Some(secret) = dataset .extra .get("urlSigningSecretKey") diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 6519490..fe359c4 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -6,17 +6,17 @@ use serde::Serialize; use crate::clients::base::{ delete_resource, get_raw, get_resource, get_resource_required, head_exists, put_raw, - put_raw_with_content_encoding, update_resource, ResourceContext, + update_resource, RawUpload, ResourceContext, }; use crate::common::{create_hmac_signature, sign_storage_content, to_path_segment, QueryParams}; -use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod}; +use crate::error::{ApifyClientError, ApifyClientResult}; +use crate::http_client::{CallOptions, HttpClient, HttpMethod}; use crate::models::{KeyValueStore, KeyValueStoreKey, KeyValueStoreKeysPage, KeyValueStoreRecord}; use crate::timeouts::Timeout; /// The path of the record with the given key, relative to the store. A key that is empty or a dot /// segment cannot be placed in the path, so it is rejected with -/// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument). +/// [`ApifyClientError::InvalidArgument`]. fn record_path(key: &str) -> ApifyClientResult { Ok(format!("records/{}", to_path_segment(key)?)) } @@ -52,6 +52,20 @@ pub struct GetRecordOptions { pub signature: Option, } +/// Options of [`KeyValueStoreClient::set_record_raw_with_options`] and +/// [`KeyValueStoreClient::set_record_json_with_options`]. +#[derive(Debug, Default, Clone)] +pub struct SetRecordOptions { + /// `Content-Encoding` of a value that is already compressed (`gzip`, `br`, `deflate` or + /// `identity`). The client then sends the bytes as they are, instead of compressing them + /// again. + pub content_encoding: Option, + /// Give up on the first transport timeout instead of retrying it. A retry of an upload that + /// timed out is safe for a record, which the next upload overwrites, but it can take as long + /// as the first attempt: set this to fail fast on a slow connection. + pub do_not_retry_timeouts: bool, +} + /// Client for a specific key-value store. #[derive(Debug, Clone)] pub struct KeyValueStoreClient { @@ -87,7 +101,7 @@ impl KeyValueStoreClient { /// Updates the store metadata (e.g. `name`, `title`). pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields, Timeout::Short).await + update_resource(&self.ctx, None, new_fields, Timeout::Long).await } /// Deletes the store. @@ -147,7 +161,7 @@ impl KeyValueStoreClient { &self.ctx, Some(&record_path(key)?), &QueryParams::new(), - Timeout::Short, + Timeout::Long, ) .await } @@ -203,15 +217,8 @@ impl KeyValueStoreClient { value: Vec, content_type: &str, ) -> ApifyClientResult<()> { - put_raw( - &self.ctx, - Some(&record_path(key)?), - &QueryParams::new(), - value, - content_type, - Timeout::Long, - ) - .await + self.set_record_raw_with_options(key, value, content_type, SetRecordOptions::default()) + .await } /// Stores a record with raw, already-compressed bytes, sending `content_encoding` as the @@ -233,13 +240,38 @@ impl KeyValueStoreClient { content_type: &str, content_encoding: &str, ) -> ApifyClientResult<()> { - put_raw_with_content_encoding( + self.set_record_raw_with_options( + key, + value, + content_type, + SetRecordOptions { + content_encoding: Some(content_encoding.to_string()), + ..Default::default() + }, + ) + .await + } + + /// Like [`set_record_raw`](Self::set_record_raw), with [`SetRecordOptions`]. + pub async fn set_record_raw_with_options( + &self, + key: &str, + value: Vec, + content_type: &str, + options: SetRecordOptions, + ) -> ApifyClientResult<()> { + put_raw( &self.ctx, Some(&record_path(key)?), &QueryParams::new(), - value, - content_type, - Some(content_encoding), + RawUpload { + body: value, + content_type, + content_encoding: options.content_encoding.as_deref(), + }, + CallOptions { + do_not_retry_timeouts: options.do_not_retry_timeouts, + }, Timeout::Long, ) .await @@ -251,11 +283,39 @@ impl KeyValueStoreClient { key: &str, value: &T, ) -> ApifyClientResult<()> { + self.set_record_json_with_options(key, value, SetRecordOptions::default()) + .await + } + + /// Like [`set_record_json`](Self::set_record_json), with [`SetRecordOptions`]. A + /// `content_encoding` is rejected with + /// [`ApifyClientError::InvalidArgument`]: the + /// client serializes the value itself, so it can never be pre-compressed. + pub async fn set_record_json_with_options( + &self, + key: &str, + value: &T, + options: SetRecordOptions, + ) -> ApifyClientResult<()> { + if options.content_encoding.is_some() { + return Err(ApifyClientError::InvalidArgument( + "set_record_json_with_options cannot send a Content-Encoding: the JSON value is \ + serialized by the client, so it is never pre-compressed; use \ + set_record_raw_with_options for pre-compressed bytes" + .to_string(), + )); + } let bytes = serde_json::to_vec(value)?; - self.set_record_raw(key, bytes, "application/json; charset=utf-8") + self.set_record_raw_with_options(key, bytes, "application/json; charset=utf-8", options) .await } + /// Fetches the store to read its URL-signing secret key, with the `Long` tier of the public URL + /// methods. + async fn get_for_public_url(&self) -> ApifyClientResult> { + get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Long).await + } + /// Builds a public URL for reading the record with the given key. /// /// Mirrors the reference client's `getRecordPublicUrl`: it fetches the store, and if the @@ -265,7 +325,7 @@ impl KeyValueStoreClient { pub async fn get_record_public_url(&self, key: &str) -> ApifyClientResult { let path = record_path(key)?; let mut params = QueryParams::new(); - if let Some(store) = self.get().await? { + if let Some(store) = self.get_for_public_url().await? { if let Some(secret) = store .extra .get("urlSigningSecretKey") @@ -287,7 +347,7 @@ impl KeyValueStoreClient { expires_in_secs: Option, ) -> ApifyClientResult { let mut params = QueryParams::new(); - if let Some(store) = self.get().await? { + if let Some(store) = self.get_for_public_url().await? { if let Some(secret) = store .extra .get("urlSigningSecretKey") diff --git a/src/clients/request_queue.rs b/src/clients/request_queue.rs index 6e88d34..b29bbbb 100644 --- a/src/clients/request_queue.rs +++ b/src/clients/request_queue.rs @@ -194,6 +194,11 @@ fn chunk_requests_for_batch_add( pub struct ListRequestsOptions { /// Maximum number of requests to return. pub limit: Option, + /// Start listing after this request ID (exclusive). + /// + /// Deprecated by the API: use `cursor`, which cannot be combined with it. + #[deprecated(note = "the API deprecated `exclusiveStartId`: use `cursor`")] + pub exclusive_start_id: Option, /// Opaque pagination cursor returned by a previous call. pub cursor: Option, /// Restrict the returned requests to the given states. The spec defines this as an array of @@ -227,6 +232,17 @@ impl RequestQueueClient { } } + /// Caps the timeout of every request this queue client sends: a tier longer than `cap` is + /// lowered to it, a shorter one is kept. Unlike [`with_timeout`](Self::with_timeout), which + /// replaces the tier, this never makes a request wait longer than it would by default. A + /// timeout set with `with_timeout` wins over the cap. A zero cap is rejected: every request + /// then fails with `InvalidArgument`. + #[must_use] + pub fn with_timeout_cap(mut self, cap: Duration) -> Self { + self.ctx = self.ctx.with_timeout_cap(cap); + self + } + /// Sets the `clientKey` used to identify this client across requests (for locking). pub fn with_client_key(mut self, client_key: impl Into) -> Self { self.client_key = Some(client_key.into()); @@ -555,14 +571,17 @@ impl RequestQueueClient { /// Lists requests in the queue. /// /// Supports pagination via `limit` and `cursor`, and the spec's `filter` parameter (see - /// [`ListRequestsOptions`]). + /// [`ListRequestsOptions`]). The deprecated `exclusive_start_id` is still sent when set. pub async fn list_requests( &self, options: ListRequestsOptions, ) -> ApifyClientResult { let mut params = self.base_params(); + params.add_int("limit", options.limit); + // The field is deprecated for callers; the spec still documents the parameter. + #[allow(deprecated)] + params.add_str("exclusiveStartId", options.exclusive_start_id); params - .add_int("limit", options.limit) .add_str("cursor", options.cursor) .add_csv("filter", options.filter.as_deref()); get_resource_required(&self.ctx, Some("requests"), ¶ms, Timeout::Medium).await diff --git a/src/clients/run.rs b/src/clients/run.rs index 9242a0d..7b32a92 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -10,7 +10,7 @@ use crate::clients::dataset::DatasetClient; use crate::clients::key_value_store::KeyValueStoreClient; use crate::clients::log::LogClient; use crate::clients::request_queue::RequestQueueClient; -use crate::common::{to_safe_id, QueryParams}; +use crate::common::{catch_not_found, to_safe_id, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorRun; @@ -347,30 +347,33 @@ impl RunClient { LogClient::nested(self.ctx.http.clone(), &self.ctx.url(None), "log") } - /// Opens a live stream of the run's log for redirection. + /// Opens a live stream of the run's log for redirection, or `None` if the run does not exist. /// /// Convenience equivalent to `run.log().stream()` (mirrors the reference client's /// `getStreamedLog`): yields log chunks as they arrive, so callers can forward them to - /// their own logger/stdout while the run is in progress. A run that does not exist is a - /// [`NotFound`](crate::ApiError::is_not_found) error. + /// their own logger/stdout while the run is in progress. Unlike `run.log().stream()`, whose + /// chained client cannot tell a missing run from a missing log, a missing run is `None` here, + /// because the run is the resource this client names. pub async fn get_streamed_log( &self, - ) -> ApifyClientResult>>> { - self.log().stream_required(Default::default()).await + ) -> ApifyClientResult>>>> + { + self.get_streamed_log_with_options(Default::default()).await } /// Opens a live stream of the run's log for redirection, applying the given /// [`LogOptions`](crate::LogOptions) (e.g. [`LogOptions::raw`](crate::LogOptions::raw) to /// stream the unprocessed log, which is the form the JS reference's log redirection consumes - /// internally). + /// internally). `None` if the run does not exist. /// - /// This is a Rust-specific convenience that simply forwards `LogOptions` to - /// [`LogClient::stream_with_options`]; it is not a 1:1 mirror of the JS `getStreamedLog` - /// method's signature (which takes redirect options and returns a `StreamedLog` object). + /// This is a Rust-specific convenience that simply forwards `LogOptions` to the log stream; + /// it is not a 1:1 mirror of the JS `getStreamedLog` method's signature (which takes redirect + /// options and returns a `StreamedLog` object). pub async fn get_streamed_log_with_options( &self, options: crate::clients::log::LogOptions, - ) -> ApifyClientResult>>> { - self.log().stream_required(options).await + ) -> ApifyClientResult>>>> + { + catch_not_found(self.log().stream_required(options).await) } } diff --git a/src/common.rs b/src/common.rs index eda213c..120bd91 100644 --- a/src/common.rs +++ b/src/common.rs @@ -4,12 +4,9 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::error::{ApifyClientError, ApifyClientResult}; +use crate::error::{ApifyClientError, ApifyClientResult, NOT_FOUND_STATUS_CODE}; use crate::version::CLIENT_VERSION; -/// Status code returned when a resource is not found. -const NOT_FOUND_STATUS_CODE: u16 = 404; - /// Most Apify endpoints wrap their payload in a top-level `data` property. /// This envelope unwraps `{ "data": ... }` into the inner type. #[derive(Debug, Deserialize)] diff --git a/src/error.rs b/src/error.rs index 3ba015c..e958697 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,6 +2,21 @@ use serde::Deserialize; +/// HTTP status code of a request that failed validation. +pub(crate) const BAD_REQUEST_STATUS_CODE: u16 = 400; +/// HTTP status code of a missing or invalid token. +pub(crate) const UNAUTHORIZED_STATUS_CODE: u16 = 401; +/// HTTP status code of a token that lacks permission for the operation. +pub(crate) const FORBIDDEN_STATUS_CODE: u16 = 403; +/// HTTP status code of a resource that is not found. +pub(crate) const NOT_FOUND_STATUS_CODE: u16 = 404; +/// HTTP status code of a request that conflicts with the state of the resource. +pub(crate) const CONFLICT_STATUS_CODE: u16 = 409; +/// HTTP status code of an exceeded per-resource rate limit. +pub(crate) const RATE_LIMIT_EXCEEDED_STATUS_CODE: u16 = 429; +/// First HTTP status code of a server error. +pub(crate) const MIN_SERVER_ERROR_STATUS_CODE: u16 = 500; + /// The result type used throughout this crate. pub type ApifyClientResult = Result; @@ -66,17 +81,17 @@ impl ApiError { /// since Rust has no exception hierarchy to mirror, these `is_*` predicates are the /// idiomatic equivalent classification on the one [`ApiError`] type. pub fn is_invalid_request(&self) -> bool { - self.status_code == 400 + self.status_code == BAD_REQUEST_STATUS_CODE } /// `true` for HTTP 401 Unauthorized: the token is missing or invalid. pub fn is_unauthorized(&self) -> bool { - self.status_code == 401 + self.status_code == UNAUTHORIZED_STATUS_CODE } /// `true` for HTTP 403 Forbidden: the token lacks permission for the operation. pub fn is_forbidden(&self) -> bool { - self.status_code == 403 + self.status_code == FORBIDDEN_STATUS_CODE } /// `true` for HTTP 404 Not Found. @@ -86,26 +101,26 @@ impl ApiError { /// that return other errors directly, e.g. [`crate::clients::actor::ActorClient::start`] /// with a nonexistent Actor ID. pub fn is_not_found(&self) -> bool { - self.status_code == 404 + self.status_code == NOT_FOUND_STATUS_CODE } /// `true` for HTTP 409 Conflict. pub fn is_conflict(&self) -> bool { - self.status_code == 409 + self.status_code == CONFLICT_STATUS_CODE } /// `true` for HTTP 429 Too Many Requests. The client already retries these internally (see /// [`crate::ApifyClientBuilder::max_retries`]), so this surfaces only once retries are /// exhausted. pub fn is_rate_limited(&self) -> bool { - self.status_code == 429 + self.status_code == RATE_LIMIT_EXCEEDED_STATUS_CODE } /// `true` for an HTTP 5xx status. Like [`is_rate_limited`](Self::is_rate_limited), the /// client already retries these internally, so this surfaces only once retries are /// exhausted. pub fn is_server_error(&self) -> bool { - self.status_code >= 500 + self.status_code >= MIN_SERVER_ERROR_STATUS_CODE } } diff --git a/src/http_client.rs b/src/http_client.rs index ab42709..4ffc396 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -17,14 +17,14 @@ use std::time::Duration; use async_trait::async_trait; -use crate::error::{ApiError, ApiErrorBody, ApifyClientError, ApifyClientResult}; +use crate::error::{ + ApiError, ApiErrorBody, ApifyClientError, ApifyClientResult, MIN_SERVER_ERROR_STATUS_CODE, + RATE_LIMIT_EXCEEDED_STATUS_CODE, +}; use crate::http_compressors::{maybe_compress, HttpCompressor}; +use crate::statistics::Statistics; use crate::timeouts::{Timeout, TimeoutTiers, NO_TIMEOUT}; -/// HTTP status code returned by the API when the per-resource rate limit is exceeded. -const RATE_LIMIT_EXCEEDED_STATUS_CODE: u16 = 429; -/// Statuses `>= 500` are considered retryable internal server errors. -const MIN_SERVER_ERROR_STATUS_CODE: u16 = 500; /// Responses with status `< 300` are treated as success. const MAX_SUCCESS_STATUS_CODE: u16 = 300; /// Multiplier applied to the inter-retry delay after each attempt (exponential backoff). @@ -283,6 +283,17 @@ pub(crate) struct HttpClientConfig { /// Extra headers sent with every request. They win over the built-in `User-Agent` and /// `Authorization` headers, and a request's own headers win over them. pub(crate) default_headers: Vec<(String, String)>, + pub(crate) stats: Arc, +} + +/// Options of a single [`HttpClient::call_with_options`]. +#[derive(Debug, Clone, Copy, Default)] +pub struct CallOptions { + /// Give up on the first transport timeout instead of retrying it. For a call that is not + /// idempotent, such as pushing dataset items, whose request may have reached the API before + /// the client gave up on it: a retry could then store the same data twice. Other errors are + /// retried as usual. + pub do_not_retry_timeouts: bool, } /// The orchestrating HTTP client shared by every resource client. @@ -299,6 +310,7 @@ pub struct HttpClient { tiers: TimeoutTiers, compressor: Arc, default_headers: Vec<(String, String)>, + stats: Arc, /// Set when the resource client holding this handle was addressed with an invalid ID. Every /// call then fails with [`ApifyClientError::InvalidArgument`] before reaching the backend. /// The handle is cloned down to every client chained from that resource, so the failure @@ -316,6 +328,7 @@ impl HttpClient { tiers: config.tiers, compressor: config.compressor, default_headers: config.default_headers, + stats: config.stats, poison: None, } } @@ -371,10 +384,26 @@ impl HttpClient { request.headers = merged; } + /// The statistics of the calls made through this client. + pub(crate) fn stats(&self) -> &Statistics { + &self.stats + } + /// Sends `request` with authentication, the user-agent header, and the retry policy /// applied. Returns the first successful response, or the final error. - pub async fn call(&self, mut request: HttpRequest) -> ApifyClientResult { + pub async fn call(&self, request: HttpRequest) -> ApifyClientResult { + self.call_with_options(request, CallOptions::default()) + .await + } + + /// Like [`call`](Self::call), with the given per-call [`CallOptions`]. + pub async fn call_with_options( + &self, + mut request: HttpRequest, + options: CallOptions, + ) -> ApifyClientResult { self.fail_if_poisoned()?; + self.stats.add_call(); self.apply_headers(&mut request); // Compress the request body once (not per attempt) when it is large enough, mirroring the @@ -399,21 +428,27 @@ impl HttpClient { let mut attempt = 1; loop { - // Grow per-attempt timeout with each attempt, capped at the overall budget. + // Grow the timeout of the attempt with each attempt, up to `timeout_max`. let mut attempt_request = request.clone(); attempt_request.timeout = self.attempt_timeout(base_timeout, attempt); + self.stats.add_request(); let outcome = match self.backend.send(attempt_request).await { Ok(response) => { if response.status < MAX_SUCCESS_STATUS_CODE { return Ok(response); } + if response.status == RATE_LIMIT_EXCEEDED_STATUS_CODE { + self.stats.add_rate_limit_error(attempt); + } let api_error = build_api_error(&response, attempt, &method_str, &path); let retryable = is_status_retryable(response.status); (ApifyClientError::from(api_error), retryable) } Err(err) => { - let retryable = is_error_retryable(&err); + let retryable = is_error_retryable(&err) + && !(options.do_not_retry_timeouts + && matches!(err, ApifyClientError::Timeout)); (err, retryable) } }; @@ -426,8 +461,7 @@ impl HttpClient { // Sleep with randomized exponential backoff before the next attempt. The backoff // doubles each retry (matching the reference client, which uses `async-retry` with - // a factor of 2) and is capped at the overall request timeout so a single backoff - // can never exceed the budget the whole request is allowed. + // a factor of 2) and is capped at `timeout_max`. sleep(randomized_delay(delay)).await; // `saturating_mul` mirrors the saturating arithmetic in `attempt_timeout`. delay = delay.saturating_mul(BACKOFF_FACTOR).min(self.tiers.max); @@ -462,30 +496,25 @@ impl HttpClient { /// On a non-success status, reads the body (bounded by the same inactivity timeout as a /// successful body below) and returns the same structured [`ApiError`] [`call`](Self::call) /// would. On success, returns the body as a stream whose `.next()` must resolve within the - /// client's overall timeout budget each time — bounding silence (a stalled connection) - /// without capping the stream's total duration, so a long-but-active stream (e.g. live log - /// redirection) is never cut off. That same budget also bounds the initial wait for the - /// response to start, mirroring the reference clients' streaming `timeoutSecs`. + /// request's resolved timeout each time — bounding silence (a stalled connection) without + /// capping the stream's total duration, so a long-but-active stream (e.g. live log + /// redirection) is never cut off. The same timeout also bounds the initial wait for the + /// response to start. pub(crate) async fn call_streaming( &self, mut request: HttpRequest, ) -> ApifyClientResult>>> { self.fail_if_poisoned()?; + self.stats.add_call(); + self.stats.add_request(); self.apply_headers(&mut request); let method_str = request.method.as_str().to_string(); let path = extract_path(&request.url); - // The caller-supplied `request.timeout` (the same tier-resolved base every buffered call - // uses — the `Long` tier for the streaming endpoints) is reused here as both the - // response-start wait and the per-chunk inactivity window: one configured duration, not - // a separate, larger one for inactivity. The JS reference client documents the same - // single-duration behavior ("it only limits inactivity"), though it gets there - // differently — via its HTTP library's socket-level idle timeout, not an explicit - // per-chunk reset like `with_inactivity_timeout` below — so this is a same-outcome, - // not same-mechanism, match. It carries the same tradeoff either way: a stream - // legitimately silent for the whole window (e.g. a long-running Actor logging nothing - // for the `Long` tier) times out; the escape hatch for that is `Timeout::NoTimeout`, - // set with a resource client's `with_timeout`. + // The request timeout is both the response-start wait and the per-chunk inactivity + // window, so a stream that stays silent for that long (e.g. a run that logs nothing for + // the `Long` tier) times out. `Timeout::NoTimeout`, set with a resource client's + // `with_timeout`, turns that off. let budget = request.timeout; let response = tokio::time::timeout(budget, self.backend.send_streaming(request)) @@ -502,6 +531,9 @@ impl HttpClient { // body that fails to read (e.g. the connection drops mid-response) still surfaces the // status code, just without a parsed message. let body = collect_with_timeout(response.body, budget).await; + if response.status == RATE_LIMIT_EXCEEDED_STATUS_CODE { + self.stats.add_rate_limit_error(1); + } let http_response = HttpResponse { status: response.status, headers: response.headers, diff --git a/src/http_compressors.rs b/src/http_compressors.rs index bfccc22..d669d12 100644 --- a/src/http_compressors.rs +++ b/src/http_compressors.rs @@ -35,8 +35,10 @@ const BROTLI_BUFFER_SIZE: usize = 4096; const GZIP_MIN_QUALITY: u32 = 1; /// Highest gzip quality, the slowest with the best compression. const GZIP_MAX_QUALITY: u32 = 9; -/// Gzip quality used by [`GzipHttpCompressor::default`], matching the reference client. -const GZIP_DEFAULT_QUALITY: u32 = 9; +/// Gzip quality used by [`GzipHttpCompressor::default`]: zlib's own default, which the reference +/// client uses too. The levels above it roughly double the CPU cost for about a percent fewer +/// bytes. +const GZIP_DEFAULT_QUALITY: u32 = 6; /// Media-type prefixes whose payloads already carry their own compression, so running them /// through brotli/gzip burns CPU and memory for a result that is usually no smaller — and the @@ -106,7 +108,9 @@ pub trait HttpCompressor: Send + Sync + Debug { /// Value sent in the `Content-Encoding` header, for example `gzip` or `br`. fn content_encoding(&self) -> &str; - /// Compresses a request body and returns the compressed bytes. + /// Compresses a request body and returns the compressed bytes. The built-in compressors work + /// in memory and cannot fail; an implementation that can returns its failure as the error of + /// the request. fn compress(&self, data: &[u8]) -> ApifyClientResult>; } @@ -147,7 +151,9 @@ impl HttpCompressor for BrotliHttpCompressor { self.quality, BROTLI_WINDOW_SIZE, ); - write_all(&mut writer, data)?; + writer + .write_all(data) + .expect("writing to an in-memory Vec never fails"); Ok(writer.into_inner()) } } @@ -170,7 +176,7 @@ impl GzipHttpCompressor { } impl Default for GzipHttpCompressor { - /// Gzip at quality 9. + /// Gzip at quality 6, zlib's own default. fn default() -> Self { Self { quality: GZIP_DEFAULT_QUALITY, @@ -186,8 +192,12 @@ impl HttpCompressor for GzipHttpCompressor { fn compress(&self, data: &[u8]) -> ApifyClientResult> { let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(self.quality)); - write_all(&mut encoder, data)?; - encoder.finish().map_err(compression_failed) + encoder + .write_all(data) + .expect("writing to an in-memory Vec never fails"); + Ok(encoder + .finish() + .expect("finishing an in-memory Vec never fails")) } } @@ -226,14 +236,6 @@ fn check_quality(algorithm: &str, quality: u32, min: u32, max: u32) -> ApifyClie } } -fn write_all(writer: &mut impl Write, data: &[u8]) -> ApifyClientResult<()> { - writer.write_all(data).map_err(compression_failed) -} - -fn compression_failed(err: std::io::Error) -> ApifyClientError { - ApifyClientError::InvalidArgument(format!("compressing the request body failed: {err}")) -} - /// Compresses `body` with `compressor` and adds the matching `Content-Encoding` header, when the /// body is large enough, no `Content-Encoding` is already set, and its content type is not /// already compressed. Returns the (possibly unchanged) body. @@ -332,6 +334,29 @@ mod tests { assert!(GzipHttpCompressor::new(10).is_err()); } + #[test] + fn default_qualities_are_the_zlib_and_brotli_defaults_of_the_reference_client() { + let data: Vec = (0..4096u32).map(|i| (i % 251) as u8).collect(); + + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(6)); + gzip.write_all(&data).unwrap(); + assert_eq!( + GzipHttpCompressor::default().compress(&data).unwrap(), + gzip.finish().unwrap() + ); + assert_eq!( + GzipHttpCompressor::default().compress(&data).unwrap(), + GzipHttpCompressor::new(6).unwrap().compress(&data).unwrap() + ); + assert_eq!( + BrotliHttpCompressor::default().compress(&data).unwrap(), + BrotliHttpCompressor::new(6) + .unwrap() + .compress(&data) + .unwrap() + ); + } + #[test] fn built_in_compressors_report_their_content_encoding() { assert_eq!(BrotliHttpCompressor::default().content_encoding(), "br"); diff --git a/src/lib.rs b/src/lib.rs index aa7a07a..f0aaed4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,6 +67,7 @@ pub mod error; pub mod http_client; pub mod http_compressors; pub mod models; +pub mod statistics; pub mod timeouts; mod version; @@ -76,6 +77,7 @@ pub use error::{ApiError, ApifyClientError, ApifyClientResult}; pub use http_compressors::{ BrotliHttpCompressor, GzipHttpCompressor, HttpCompressor, RequestCompression, }; +pub use statistics::Statistics; pub use timeouts::Timeout; pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; diff --git a/src/models.rs b/src/models.rs index 0730216..bacc8e8 100644 --- a/src/models.rs +++ b/src/models.rs @@ -27,15 +27,33 @@ where D: Deserializer<'de>, { let value: Option = Option::deserialize(deserializer)?; + value.map(normalize_url).transpose() +} + +/// Like [`deserialize_normalized_url`], for a map whose values are URLs. +fn deserialize_normalized_url_map<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let value: Option> = Option::deserialize(deserializer)?; value - .map(|raw| { - url::Url::parse(&raw) - .map(String::from) - .map_err(|err| serde::de::Error::custom(format!("invalid URL `{raw}`: {err}"))) + .map(|links| { + links + .into_iter() + .map(|(name, raw)| Ok((name, normalize_url(raw)?))) + .collect() }) .transpose() } +fn normalize_url(raw: String) -> Result { + url::Url::parse(&raw) + .map(String::from) + .map_err(|err| E::custom(format!("invalid URL `{raw}`: {err}"))) +} + /// An Actor on the Apify platform. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -113,6 +131,13 @@ pub struct ActorRun { /// URL of the run's container, if running. Normalized to its WHATWG URL serialization. #[serde(default, deserialize_with = "deserialize_normalized_url")] pub container_url: Option, + /// URL of the run in Apify Console. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub console_url: Option, + /// Output links of the run, generated from the output schema of the Actor build (present only + /// if the build defines one). Each link is normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url_map")] + pub output: Option>, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -212,6 +237,9 @@ pub struct Task { /// The task's public landing page display configuration, or `None` if never configured. #[serde(default)] pub public_config: Option, + /// URL of the task's Actor Standby endpoint, or `None` if it has none. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub standby_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -270,6 +298,12 @@ pub struct Dataset { /// Total number of items in the dataset. #[serde(default)] pub item_count: Option, + /// URL of the dataset in Apify Console. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub console_url: Option, + /// Public URL of the dataset items. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub items_public_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -293,6 +327,15 @@ pub struct KeyValueStore { /// When the store was last modified. #[serde(default)] pub modified_at: Option>, + /// URL of the store in Apify Console. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub console_url: Option, + /// Public URL of the store's key listing. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub keys_public_url: Option, + /// Public URL prefix of the store's records. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub records_public_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -307,6 +350,9 @@ pub struct KeyValueStoreKey { /// Size of the record value in bytes. #[serde(default)] pub size: Option, + /// Public link to access this record directly. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub record_public_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -377,6 +423,9 @@ pub struct RequestQueue { /// Total number of requests ever added. #[serde(default)] pub total_request_count: Option, + /// URL of the queue in Apify Console. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub console_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -610,6 +659,23 @@ pub struct WebhookDispatch { /// ID of the webhook that produced this dispatch. #[serde(default)] pub webhook_id: Option, + /// A summary of the webhook as it was when the dispatch was made, or `None` if the webhook has + /// since been deleted. + #[serde(default)] + pub webhook: Option, + /// Any other fields returned by the API. + #[serde(flatten)] + pub extra: Extra, +} + +/// The webhook of a [`WebhookDispatch`], as it was when the dispatch was made. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhookDispatchWebhookSummary { + /// The URL that received the webhook POST request. Normalized to its WHATWG URL + /// serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub request_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -624,11 +690,32 @@ pub struct User { /// Username. #[serde(default)] pub username: Option, + /// The user's public profile. + #[serde(default)] + pub profile: Option, /// Any other fields returned by the API (public or private depending on the call). #[serde(flatten)] pub extra: Extra, } +/// The public profile of a [`User`]. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UserProfile { + /// The user's display name. + #[serde(default)] + pub name: Option, + /// URL of the user's picture. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub picture_url: Option, + /// URL of the user's website. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub website_url: Option, + /// Any other fields returned by the API. + #[serde(flatten)] + pub extra: Extra, +} + /// A single Actor entry as returned by the Apify Store listing. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -644,6 +731,12 @@ pub struct ActorStoreListItem { /// Human-readable title. #[serde(default)] pub title: Option, + /// URL of the Actor's page. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub url: Option, + /// URL of the owner's picture. Normalized to its WHATWG URL serialization. + #[serde(default, deserialize_with = "deserialize_normalized_url")] + pub user_picture_url: Option, /// Any other fields returned by the API. #[serde(flatten)] pub extra: Extra, @@ -707,6 +800,72 @@ mod tests { ); } + #[test] + fn every_url_field_of_the_models_is_normalized() { + let url = "HTTP://Example.COM:80"; + let normalized = "http://example.com/"; + + let run: ActorRun = serde_json::from_value(serde_json::json!({ + "id": "r", "consoleUrl": url, "output": { "results": url } + })) + .unwrap(); + assert_eq!(run.console_url.as_deref(), Some(normalized)); + assert_eq!(run.output.unwrap()["results"], normalized); + + let task: Task = + serde_json::from_value(serde_json::json!({ "id": "t", "standbyUrl": url })).unwrap(); + assert_eq!(task.standby_url.as_deref(), Some(normalized)); + + let dataset: Dataset = serde_json::from_value(serde_json::json!({ + "id": "d", "consoleUrl": url, "itemsPublicUrl": url + })) + .unwrap(); + assert_eq!(dataset.console_url.as_deref(), Some(normalized)); + assert_eq!(dataset.items_public_url.as_deref(), Some(normalized)); + + let store: KeyValueStore = serde_json::from_value(serde_json::json!({ + "id": "s", "consoleUrl": url, "keysPublicUrl": url, "recordsPublicUrl": url + })) + .unwrap(); + assert_eq!(store.console_url.as_deref(), Some(normalized)); + assert_eq!(store.keys_public_url.as_deref(), Some(normalized)); + assert_eq!(store.records_public_url.as_deref(), Some(normalized)); + + let key: KeyValueStoreKey = + serde_json::from_value(serde_json::json!({ "key": "k", "recordPublicUrl": url })) + .unwrap(); + assert_eq!(key.record_public_url.as_deref(), Some(normalized)); + + let queue: RequestQueue = + serde_json::from_value(serde_json::json!({ "id": "q", "consoleUrl": url })).unwrap(); + assert_eq!(queue.console_url.as_deref(), Some(normalized)); + + let actor: ActorStoreListItem = serde_json::from_value(serde_json::json!({ + "id": "a", "url": url, "userPictureUrl": url + })) + .unwrap(); + assert_eq!(actor.url.as_deref(), Some(normalized)); + assert_eq!(actor.user_picture_url.as_deref(), Some(normalized)); + + let user: User = serde_json::from_value(serde_json::json!({ + "id": "u", "profile": { "pictureUrl": url, "websiteUrl": url, "bio": "hi" } + })) + .unwrap(); + let profile = user.profile.unwrap(); + assert_eq!(profile.picture_url.as_deref(), Some(normalized)); + assert_eq!(profile.website_url.as_deref(), Some(normalized)); + assert_eq!(profile.extra["bio"], "hi"); + + let dispatch: WebhookDispatch = serde_json::from_value(serde_json::json!({ + "id": "w", "webhook": { "requestUrl": url } + })) + .unwrap(); + assert_eq!( + dispatch.webhook.unwrap().request_url.as_deref(), + Some(normalized) + ); + } + #[test] fn null_and_missing_url_fields_stay_none() { let run: ActorRun = serde_json::from_value(serde_json::json!({ diff --git a/src/statistics.rs b/src/statistics.rs new file mode 100644 index 0000000..93df60d --- /dev/null +++ b/src/statistics.rs @@ -0,0 +1,83 @@ +//! Statistics of the API calls made through a client. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +/// Statistics of the API calls made through an [`ApifyClient`](crate::ApifyClient), read with +/// [`ApifyClient::stats`](crate::ApifyClient::stats). +/// +/// The counters are shared by every clone of the client and by every resource client obtained +/// from it. +#[derive(Debug, Default)] +pub struct Statistics { + calls: AtomicU64, + requests: AtomicU64, + rate_limit_errors: Mutex>, +} + +impl Statistics { + /// Number of client function calls: each call counts once, however many attempts it took. + pub fn calls(&self) -> u64 { + self.calls.load(Ordering::Relaxed) + } + + /// Number of API requests: every attempt of every call counts. + pub fn requests(&self) -> u64 { + self.requests.load(Ordering::Relaxed) + } + + /// Number of times the API returned a `429` rate-limit error, by attempt: an error on the first + /// attempt is counted at index 0, an error on the first retry at index 1, and so on. + pub fn rate_limit_errors(&self) -> Vec { + self.lock_rate_limit_errors().clone() + } + + pub(crate) fn add_call(&self) { + self.calls.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn add_request(&self) { + self.requests.fetch_add(1, Ordering::Relaxed); + } + + /// Counts a `429` returned on the 1-based `attempt`. + pub(crate) fn add_rate_limit_error(&self, attempt: u32) { + let index = attempt.saturating_sub(1) as usize; + let mut errors = self.lock_rate_limit_errors(); + if errors.len() <= index { + errors.resize(index + 1, 0); + } + errors[index] += 1; + } + + /// The counters hold plain numbers, so a panic while one was held cannot leave them in an + /// inconsistent state and the poison is ignored. + fn lock_rate_limit_errors(&self) -> std::sync::MutexGuard<'_, Vec> { + self.rate_limit_errors + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_limit_errors_are_counted_by_attempt() { + let stats = Statistics::default(); + stats.add_rate_limit_error(1); + stats.add_rate_limit_error(3); + stats.add_rate_limit_error(3); + assert_eq!(stats.rate_limit_errors(), vec![1, 0, 2]); + } + + #[test] + fn calls_and_requests_are_counted_separately() { + let stats = Statistics::default(); + stats.add_call(); + stats.add_request(); + stats.add_request(); + assert_eq!((stats.calls(), stats.requests()), (1, 2)); + } +} diff --git a/tests/common/mock_backend.rs b/tests/common/mock_backend.rs index a80a916..a55049b 100644 --- a/tests/common/mock_backend.rs +++ b/tests/common/mock_backend.rs @@ -27,6 +27,8 @@ pub struct MockBackend { pub enum MockOutcome { Status(u16, Vec), NetworkError, + /// The transport gave up on the request because it outlived its timeout. + Timeout, } impl MockBackend { @@ -105,6 +107,7 @@ impl HttpBackend for MockBackend { body, }), MockOutcome::NetworkError => Err(ApifyClientError::Http("boom".to_string())), + MockOutcome::Timeout => Err(ApifyClientError::Timeout), } } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 269217d..f4bc744 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -149,7 +149,7 @@ where /// Number of times [`iter_contains_eventually`] rebuilds the iterator and re-scans while waiting /// for a just-created resource to become visible in its collection LIST endpoint. -pub const ITER_RETRY_ATTEMPTS: usize = 30; +pub const ITER_RETRY_ATTEMPTS: usize = 32; /// Delay between the attempts made by [`iter_contains_eventually`]. pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_secs(3); @@ -167,7 +167,7 @@ pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_se /// attempt and returns immediately with no sleeping — so it is a no-op in the common /// already-consistent case and only pays the backoff on the rare lagging run. /// -/// Budget: `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` = 87s of retrying before giving up, +/// Budget: `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` = 93s of retrying before giving up, /// which covers the 90-second staleness window. The cost lands only on lagging or /// genuinely-failing runs; a consistent account never sleeps. pub async fn iter_contains_eventually(mut make_iter: Mk, mut pred: F) -> bool diff --git a/tests/unit_resources.rs b/tests/unit_resources.rs index cf358b2..ed36528 100644 --- a/tests/unit_resources.rs +++ b/tests/unit_resources.rs @@ -105,9 +105,7 @@ async fn a_log_is_none_for_a_client_with_an_id_and_an_error_when_chained() { assert!(is_not_found( client.run("r").log().stream().await.map(|_| ()) )); - assert!(is_not_found( - client.run("r").get_streamed_log().await.map(|_| ()) - )); + assert!(client.run("r").get_streamed_log().await.unwrap().is_none()); } #[tokio::test] @@ -332,11 +330,13 @@ async fn log_download_is_sent_when_set() { } #[tokio::test] -async fn request_list_has_no_exclusive_start_id() { +#[allow(deprecated)] +async fn request_list_sends_the_deprecated_exclusive_start_id_only_when_set() { let backend = MockBackend::new(vec![ok_data(json!({ "items": [] }))]); let client = client_with(backend.clone(), 0); - client - .request_queue("q") + let queue = client.request_queue("q"); + + queue .list_requests(ListRequestsOptions { cursor: Some("c".to_string()), ..Default::default() @@ -346,6 +346,18 @@ async fn request_list_has_no_exclusive_start_id() { let url = backend.last_url().unwrap(); assert!(url.contains("cursor=c")); assert!(!url.contains("exclusiveStartId")); + + queue + .list_requests(ListRequestsOptions { + exclusive_start_id: Some("request-id".to_string()), + ..Default::default() + }) + .await + .unwrap(); + assert!(backend + .last_url() + .unwrap() + .contains("exclusiveStartId=request-id")); } /// The by-id user endpoints are not limited to the current user. diff --git a/tests/unit_timeouts.rs b/tests/unit_timeouts.rs index f37045e..2fcb86f 100644 --- a/tests/unit_timeouts.rs +++ b/tests/unit_timeouts.rs @@ -1,4 +1,5 @@ -//! Offline unit tests for the timeout tiers, the default headers and the request-body compressors. +//! Offline unit tests for the timeout tiers, the retry of timeouts, the default headers, the +//! request-body compressors and the statistics. //! //! These use a mock [`HttpBackend`](apify_client::http_client::HttpBackend) and require no network //! access or `APIFY_TOKEN`. @@ -8,6 +9,7 @@ mod common; use std::sync::Arc; use std::time::Duration; +use apify_client::clients::key_value_store::SetRecordOptions; use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, ReqwestBackend}; use apify_client::timeouts::NO_TIMEOUT; use apify_client::{ @@ -422,3 +424,209 @@ fn a_quality_outside_the_range_of_the_algorithm_is_rejected() { Err(ApifyClientError::InvalidArgument(_)) )); } + +#[tokio::test] +async fn a_zero_timeout_is_rejected_without_a_request() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "a" }))]); + let client = client_with(backend.clone(), 0); + let error = client + .actor("a") + .with_timeout(Timeout::Custom(Duration::ZERO)) + .get() + .await + .expect_err("a zero timeout fails every attempt at once"); + assert!( + matches!(error, ApifyClientError::InvalidArgument(_)), + "{error:?}" + ); + assert_eq!(backend.call_count(), 0); + + let error = client + .request_queue("q") + .with_timeout_cap(Duration::ZERO) + .get() + .await + .expect_err("a zero cap fails every attempt at once"); + assert!(matches!(error, ApifyClientError::InvalidArgument(_))); +} + +/// A queue client with a cap lowers a longer tier to it and keeps a shorter one, while a timeout +/// set with `with_timeout` is used as given. +#[tokio::test] +async fn a_request_queue_timeout_cap_only_lowers_a_tier() { + let backend = MockBackend::new(vec![ok_data( + json!({ "id": "q", "items": [], "unlockedCount": 0 }), + )]); + let client = client_with(backend.clone(), 0); + let queue = client + .request_queue("q") + .with_timeout_cap(Duration::from_secs(10)); + + queue.get().await.unwrap(); // Short: 5s stays. + queue.list_requests(Default::default()).await.unwrap(); // Medium: 30s is lowered to 10s. + queue.unlock_requests().await.unwrap(); // Long: 360s is lowered to 10s. + queue + .clone() + .with_timeout(Timeout::Custom(Duration::from_secs(20))) + .get() + .await + .unwrap(); + + assert_eq!( + backend.timeouts(), + vec![ + Duration::from_secs(5), + Duration::from_secs(10), + Duration::from_secs(10), + Duration::from_secs(20) + ] + ); +} + +/// A push that timed out may have reached the API, so it is not sent again; other errors are +/// retried as usual. +#[tokio::test] +async fn push_items_does_not_retry_a_timeout() { + let backend = MockBackend::new(vec![MockOutcome::Timeout]); + let client = client_with(backend.clone(), 3); + let error = client + .dataset("d") + .push_items(&json!({ "a": 1 })) + .await + .expect_err("the push times out"); + assert!(matches!(error, ApifyClientError::Timeout), "{error:?}"); + assert_eq!(backend.call_count(), 1, "a timed-out push is not retried"); + + let backend = MockBackend::new(vec![MockOutcome::Status(500, b"{}".to_vec())]); + let client = client_with(backend.clone(), 3); + client + .dataset("d") + .push_items(&json!({ "a": 1 })) + .await + .expect_err("the push fails"); + assert_eq!(backend.call_count(), 4, "a 500 is retried as usual"); +} + +#[tokio::test] +async fn other_calls_retry_a_timeout() { + let backend = MockBackend::new(vec![MockOutcome::Timeout]); + let client = client_with(backend.clone(), 2); + client.actor("a").get().await.expect_err("always times out"); + assert_eq!(backend.call_count(), 3); +} + +#[tokio::test] +async fn set_record_can_give_up_on_the_first_timeout() { + let store_of = + |backend: &Arc| client_with(backend.clone(), 2).key_value_store("s"); + + let backend = MockBackend::new(vec![MockOutcome::Timeout]); + let error = store_of(&backend) + .set_record_raw("k", b"x".to_vec(), "text/plain") + .await + .expect_err("times out"); + assert!(matches!(error, ApifyClientError::Timeout)); + assert_eq!(backend.call_count(), 3, "retried by default"); + + let backend = MockBackend::new(vec![MockOutcome::Timeout]); + let options = SetRecordOptions { + do_not_retry_timeouts: true, + ..Default::default() + }; + store_of(&backend) + .set_record_raw_with_options("k", b"x".to_vec(), "text/plain", options.clone()) + .await + .expect_err("times out"); + assert_eq!(backend.call_count(), 1); + + let backend = MockBackend::new(vec![MockOutcome::Timeout]); + store_of(&backend) + .set_record_json_with_options("k", &json!({ "a": 1 }), options) + .await + .expect_err("times out"); + assert_eq!(backend.call_count(), 1); +} + +#[tokio::test] +async fn set_record_with_options_sends_the_content_encoding() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, Vec::new())]); + let client = client_with(backend.clone(), 0); + let store = client.key_value_store("s"); + store + .set_record_raw_with_options( + "k", + b"already compressed".to_vec(), + "text/plain", + SetRecordOptions { + content_encoding: Some("gzip".to_string()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!( + backend.last_header("Content-Encoding").as_deref(), + Some("gzip") + ); + + let error = store + .set_record_json_with_options( + "k", + &json!({}), + SetRecordOptions { + content_encoding: Some("gzip".to_string()), + ..Default::default() + }, + ) + .await + .expect_err("a JSON value is never pre-compressed"); + assert!(matches!(error, ApifyClientError::InvalidArgument(_))); +} + +#[tokio::test] +async fn statistics_count_calls_requests_and_rate_limit_errors_by_attempt() { + let backend = MockBackend::new(vec![ + MockOutcome::Status(429, b"{}".to_vec()), + MockOutcome::Status(429, b"{}".to_vec()), + ok_data(json!({ "id": "a" })), + ]); + let client = client_with(backend.clone(), 5); + assert_eq!(client.stats().calls(), 0); + + client.actor("a").get().await.unwrap(); + client.actor("a").get().await.unwrap(); + + let stats = client.stats(); + assert_eq!(stats.calls(), 2); + assert_eq!(stats.requests(), 4, "three attempts, then one"); + assert_eq!(stats.rate_limit_errors(), vec![1, 1]); + assert_eq!( + client.clone().stats().calls(), + 2, + "a clone shares the counters" + ); +} + +/// Methods that read or write whole stores and datasets, or build a public URL from one, use the +/// `Long` tier, like the reference client. +#[tokio::test] +async fn key_value_store_writes_and_public_urls_use_the_long_tier() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 200, + json!({ "data": { "id": "s" } }).to_string().into_bytes(), + )]); + let client = client_with(backend.clone(), 0); + let store = client.key_value_store("s"); + + store.update(&json!({ "title": "t" })).await.unwrap(); + store.record_exists("k").await.unwrap(); + store.get_record_public_url("k").await.unwrap(); + store.create_keys_public_url(None).await.unwrap(); + client + .dataset("d") + .create_items_public_url(Default::default(), None) + .await + .unwrap(); + + assert_eq!(backend.timeouts(), vec![Duration::from_secs(360); 5]); +} From da48e64fb167b64ee9ca1d8d4ef32cfff63543fd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 15:42:10 +0000 Subject: [PATCH 03/11] fix: address the second review of the v3 port Adds RunClient::iterate_dataset_items, carries the last_run filters into chained clients, keeps a zero timeout local to its client and rejects zero timeout tiers. --- CHANGELOG.md | 13 +- docs/misc.md | 5 +- docs/runs.md | 5 +- docs/storages.md | 5 + docs/tasks.md | 1 + docs/webhooks.md | 3 +- src/client.rs | 18 +- src/clients/base.rs | 57 ++++--- src/clients/build.rs | 7 +- src/clients/dataset.rs | 20 ++- src/clients/key_value_store.rs | 14 +- src/clients/log.rs | 12 +- src/clients/mod.rs | 1 + src/clients/request_queue.rs | 22 ++- src/clients/run.rs | 86 +++++++++- src/clients/run_dataset_items.rs | 278 +++++++++++++++++++++++++++++++ src/http_client.rs | 8 +- src/lib.rs | 1 + tests/actor_run.rs | 35 ++++ tests/unit_resources.rs | 83 ++++++++- tests/unit_run_dataset_items.rs | 272 ++++++++++++++++++++++++++++++ tests/unit_timeouts.rs | 65 ++++++++ 22 files changed, 952 insertions(+), 59 deletions(-) create mode 100644 src/clients/run_dataset_items.rs create mode 100644 tests/unit_run_dataset_items.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d5bc5a..f0f7c51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,12 @@ Ports the `apify-client-js` v3 changes. with `ApifyClientBuilder::http_compressor`. - `ApifyClientBuilder::default_header`. - `ApifyClient::stats`: the `Statistics` of the calls, requests and `429` responses. -- `HttpClient::call_with_options` and `CallOptions::do_not_retry_timeouts`; - `KeyValueStoreClient::set_record_raw_with_options`, `set_record_json_with_options` and - `SetRecordOptions`. +- `KeyValueStoreClient::set_record_raw_with_options`, `set_record_json_with_options` and + `SetRecordOptions`, with `do_not_retry_timeouts`. - `RunListOptions::deployment`, `StoreListOptions::include_input_schema` and `LogOptions::download`. +- `RunClient::iterate_dataset_items`, `RunDatasetItemsIterator`: iterate the items of a run's + default dataset while the run is still producing them, like the reference client's + `iterateDatasetItems`. - Typed, normalized URL fields: `ActorRun::console_url` and `output`, `Task::standby_url`, `Dataset::console_url` and `items_public_url`, `KeyValueStore::console_url`, `keys_public_url` and `records_public_url`, `KeyValueStoreKey::record_public_url`, `RequestQueue::console_url`, @@ -41,7 +43,6 @@ Ports the `apify-client-js` v3 changes. `get_streamed_log_with_options` return an `Option`: `None` for a missing log or run. - **Breaking:** `ActorStartOptions::timeout_secs` and `RunResurrectOptions::timeout_secs` are now `run_timeout_secs`. -- **Breaking:** `ListRequestsOptions::exclusive_start_id` is deprecated in favor of `cursor`. - **Breaking:** `LogOptions`, `RunListOptions` and `StoreListOptions` gained a public field. - **Breaking:** an id, record key or request id that is empty or a dot segment fails with `InvalidArgument`; ids are percent-encoded and every `/` of an id becomes `~`. @@ -50,6 +51,10 @@ Ports the `apify-client-js` v3 changes. - **Breaking:** the URL fields listed above are normalized, and no longer in `extra`. - **Breaking:** `RequestCompression` moved to `http_compressors` (still re-exported at the crate root). +- `ListRequestsOptions::exclusive_start_id` is deprecated in favor of `cursor`. +- The storages and the log chained from a `last_run()` with a `status` or `origin` filter now + belong to the filtered run. +- A zero `timeout_*` option or `with_timeout` fails the requests with `InvalidArgument`. - `UserClient::limits`, `monthly_usage` and `update_limits` work for any user, not only `me`. - `batch_add_requests` serializes each request once and rejects an oversized one before sending. - Key-value store `update`, `record_exists` and the public URL methods, and diff --git a/docs/misc.md b/docs/misc.md index 73da7de..8daf29c 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -59,9 +59,11 @@ and the iterator. Its fields: | `name` | `Option` | Technical name of the Actor. | | `username` | `Option` | Username of the Actor's owner. | | `title` | `Option` | Human-readable title. | +| `url` | `Option` | URL of the Actor's page. | +| `user_picture_url` | `Option` | URL of the owner's picture. | | `extra` | `Extra` | Any other fields returned by the API. | -`name`, `username` and `title` are optional, so a display routine typically prefers `title` +`url` and `user_picture_url` are normalized to their WHATWG URL serialization. `name`, `username` and `title` are optional, so a display routine typically prefers `title` and falls back to `name` (e.g. `actor.title.or(actor.name)`). ## Users — `client.me()` / `client.user(id)` @@ -87,6 +89,7 @@ organization your API token belongs to. They return a `404` as an error. |---|---|---| | `id` | `String` | Unique user ID (always present). | | `username` | `Option` | Username. | +| `profile` | `Option` | The public profile: `name`, `picture_url` and `website_url` (both normalized to their WHATWG URL serialization) and the other fields in `extra`. | | `extra` | `Extra` | Any other fields returned by the API (more fields are present for `me` than for a public `user(id)`). | ```rust,no_run diff --git a/docs/runs.md b/docs/runs.md index 6ffe19d..406572b 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -15,6 +15,7 @@ collections are available via `actor.runs()` and `task.runs()`. | Method | Arguments | Returns | Description | |---|---|---|---| | `get()` | — | `Option` | Fetches the run. | +| `iterate_dataset_items::(options)` | `DatasetListItemsOptions` | `RunDatasetItemsIterator` | Lazily iterates the items of the run's default dataset while the run is still producing them, until it finishes. `with_chunk_size(n)` sets the rows per request (default 1000) and `with_poll_interval(d)` the wait for the run between polls (default 5s). `offset` and `limit` of the options bound the rows that are read; `desc` is not supported. A `last_run()` client sticks to the run its first request resolves to. | | `update(fields)` | `&impl Serialize` | `ActorRun` | Updates the run (e.g. status message). | | `delete()` | — | `()` | Deletes the run. | | `abort(gracefully)` | `Option` | `ActorRun` | Aborts the run. `None` omits the param (server default, immediate); `Some(true)`/`Some(false)` abort gracefully/immediately. | @@ -80,7 +81,9 @@ Leave a field as `None` to omit it. | `default_dataset_id` | `Option` | Default dataset ID — pass to `client.dataset(..)` to read results. | | `default_key_value_store_id` | `Option` | Default key-value store ID for the run. | | `default_request_queue_id` | `Option` | Default request queue ID for the run. | -| `container_url` | `Option` | URL of the run's container, while running. | +| `container_url` | `Option` | URL of the run's container, while running. Normalized to its WHATWG URL serialization, like every URL field below. | +| `console_url` | `Option` | URL of the run in Apify Console. | +| `output` | `Option>` | Output links of the run, generated from the output schema of the Actor build (present only if the build defines one). | | `extra` | `Extra` | Any other fields returned by the API. | The three `default_*_id` fields are `Option` because they are only populated once the diff --git a/docs/storages.md b/docs/storages.md index 3ee6cb9..7de9da1 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -16,6 +16,10 @@ core; the `.id` field is what the examples read to build a per-storage client | `created_at` | `Option>` | all three | When the storage was created. | | `modified_at` | `Option>` | all three | When the storage was last modified. | | `item_count` | `Option` | `Dataset` only | Total number of items in the dataset. | +| `console_url` | `Option` | all three | URL of the storage in Apify Console. | +| `items_public_url` | `Option` | `Dataset` only | Public URL of the dataset items. | +| `keys_public_url` | `Option` | `KeyValueStore` only | Public URL of the store's key listing. | +| `records_public_url` | `Option` | `KeyValueStore` only | Public URL prefix of the store's records. | | `total_request_count` | `Option` | `RequestQueue` only | Total number of requests ever added. | | `extra` | `Extra` | all three | Any other fields returned by the API. | @@ -199,6 +203,7 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: |---|---|---| | `key` | `String` | The record key (always present). | | `size` | `Option` | Size of the record value in bytes, if reported by the API. | +| `record_public_url` | `Option` | Public link to access this record directly. | | `extra` | `Extra` | Any other fields returned by the API. | ## Request queues — `client.request_queues()` / `client.request_queue(id)` diff --git a/docs/tasks.md b/docs/tasks.md index f86cef3..99758e6 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -45,6 +45,7 @@ are preserved in `extra`. | `modified_at` | `Option>` | When the task was last modified. | | `is_public` | `Option` | Whether the task is published on its public landing page. Derived from `public_config.published_at`; set it via `publish()`/`unpublish()` or `update()`. | | `public_config` | `Option` | The task's public landing page display configuration, or `None` if never configured. | +| `standby_url` | `Option` | URL of the task's Actor Standby endpoint, or `None` if it has none. Normalized to its WHATWG URL serialization. | | `extra` | `Extra` | Any other fields returned by the API. | ## The `TaskPublicConfig` model diff --git a/docs/webhooks.md b/docs/webhooks.md index 13b8ec3..f9df644 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -39,7 +39,7 @@ Unknown fields returned by the API are preserved in `extra`. |---|---|---| | `id` | `String` | Unique webhook ID (always present); used to build a `client.webhook(&webhook.id)` client. | | `user_id` | `Option` | ID of the user who owns the webhook. | -| `request_url` | `Option` | The URL that receives the webhook POST request. | +| `request_url` | `Option` | The URL that receives the webhook POST request, normalized to its WHATWG URL serialization. | | `event_types` | `Vec` | Event types that trigger this webhook (e.g. `ACTOR.RUN.SUCCEEDED`). | | `extra` | `Extra` | Any other fields returned by the API (e.g. `condition`, `isAdHoc`, `payloadTemplate`). | @@ -53,4 +53,5 @@ returned by the API are preserved in `extra`. |---|---|---| | `id` | `String` | Unique dispatch ID (always present). | | `webhook_id` | `Option` | ID of the webhook that produced this dispatch. | +| `webhook` | `Option` | The webhook as it was when the dispatch was made: `request_url` (normalized) and the other fields in `extra`. | | `extra` | `Extra` | Any other fields returned by the API (e.g. `status`, `attempts`, `eventType`). | diff --git a/src/client.rs b/src/client.rs index fd7eabc..19f2ecd 100644 --- a/src/client.rs +++ b/src/client.rs @@ -123,7 +123,8 @@ impl ApifyClientBuilder { } /// Sets the duration of the [`Timeout::Short`](crate::Timeout::Short) tier, used by simple - /// metadata reads and writes (default `5s`). + /// metadata reads and writes (default `5s`). Like the other timeout options, it must be longer + /// than zero: a client built with a zero one fails every request with `InvalidArgument`. pub fn timeout_short(mut self, timeout: Duration) -> Self { self.tiers.short = timeout; self @@ -215,6 +216,15 @@ impl ApifyClientBuilder { min_delay_between_retries: self.min_delay_between_retries, }; + let zero_tier = [ + ("timeout_short", self.tiers.short), + ("timeout_medium", self.tiers.medium), + ("timeout_long", self.tiers.long), + ("timeout_max", self.tiers.max), + ] + .into_iter() + .find(|(_, duration)| duration.is_zero()); + let http = HttpClient::new(HttpClientConfig { backend, token: self.token, @@ -225,6 +235,12 @@ impl ApifyClientBuilder { default_headers: self.default_headers, stats: Arc::default(), }); + // A zero timeout fails every attempt at once, and `Timeout::NoTimeout` is the way to run + // without one, so the client fails every request with `InvalidArgument` instead. + let http = match zero_tier { + Some((option, _)) => http.poisoned(format!("{option} must be longer than zero")), + None => http, + }; let base_url = to_api_base_url(&self.base_url); let public_base_url = diff --git a/src/clients/base.rs b/src/clients/base.rs index c6c29a0..8f67ba6 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -14,7 +14,7 @@ use serde::Serialize; use crate::common::{ catch_not_found, parse_data_envelope, to_path_segment, to_safe_id, PaginationList, QueryParams, }; -use crate::error::ApifyClientResult; +use crate::error::{ApifyClientError, ApifyClientResult}; use crate::http_client::{CallOptions, HttpClient, HttpMethod, HttpRequest, HttpResponse}; use crate::timeouts::Timeout; @@ -35,8 +35,8 @@ const NOT_FOUND_GRACE_PERIOD: Duration = Duration::from_secs(3); const INVALID_ID_PLACEHOLDER: &str = "invalid-id"; /// Why a zero timeout is refused. -const ZERO_TIMEOUT_MESSAGE: &str = - "a timeout must be longer than zero; use Timeout::NoTimeout to run without one"; +const ZERO_TIMEOUT_MESSAGE: &str = "a timeout and a timeout cap must be longer than zero; use \ + Timeout::NoTimeout to run without a timeout"; /// The resolved context for a resource client: its base URL and the shared HTTP client. #[derive(Debug, Clone)] @@ -104,19 +104,38 @@ impl ResourceContext { } } - /// Replaces the tier of every request with `timeout`. A zero duration cannot be told apart - /// from an immediate failure of every attempt (`Timeout::NoTimeout` is the way to run without - /// a timeout), so it makes every request through this context fail with - /// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument), - /// like an invalid id does. + /// Replaces the tier of every request with `timeout`. A zero duration is refused when a request + /// is sent (see [`check_request_allowed`](Self::check_request_allowed)). pub(crate) fn with_timeout(mut self, timeout: Timeout) -> Self { - if timeout == Timeout::Custom(Duration::ZERO) { - self.http = self.http.poisoned(ZERO_TIMEOUT_MESSAGE.to_string()); - } self.timeout = Some(timeout); self } + /// Fails with [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument) + /// when a request cannot be sent: the client was addressed with an invalid id (which follows + /// every client chained from it), or was given a zero timeout or cap. A zero timeout makes + /// every attempt fail at once and can never grow by doubling, and `Timeout::NoTimeout` is the + /// way to run without one. The timeout and cap of this client alone are checked, so a zero one + /// is neither inherited by chained clients nor sticky once `with_timeout` replaces it. + fn check_request_allowed(&self) -> ApifyClientResult<()> { + self.http.fail_if_poisoned()?; + let zero_timeout = self.timeout == Some(Timeout::Custom(Duration::ZERO)) + || self.timeout_cap == Some(Duration::ZERO); + if zero_timeout { + return Err(ApifyClientError::InvalidArgument( + ZERO_TIMEOUT_MESSAGE.to_string(), + )); + } + Ok(()) + } + + /// Gives this context the query parameters of the client it is chained from, so that e.g. the + /// dataset of `actor.last_run(Some("SUCCEEDED"))` is the dataset of the last *succeeded* run. + pub(crate) fn with_base_params(mut self, base_params: &QueryParams) -> Self { + self.base_params = base_params.clone(); + self + } + /// Overrides the origin used when building public URLs. pub(crate) fn with_public_origin(mut self, public_origin: &str) -> Self { self.public_origin = origin_of(public_origin); @@ -160,12 +179,9 @@ impl ResourceContext { } } - /// Lowers the tier of every request to `cap` where it is longer. A zero cap is rejected like - /// a zero timeout (see [`with_timeout`](Self::with_timeout)). + /// Lowers the tier of every request to `cap` where it is longer. A zero cap is refused like a + /// zero timeout. pub(crate) fn with_timeout_cap(mut self, cap: Duration) -> Self { - if cap.is_zero() { - self.http = self.http.poisoned(ZERO_TIMEOUT_MESSAGE.to_string()); - } self.timeout_cap = Some(cap); self } @@ -226,6 +242,7 @@ impl ResourceContext { timeout: Duration, options: CallOptions, ) -> ApifyClientResult { + self.check_request_allowed()?; self.http .call_with_options( HttpRequest { @@ -240,8 +257,9 @@ impl ResourceContext { .await } - /// The URL of `sub_path` with the inherited and per-call query parameters applied. - fn url_with_params(&self, sub_path: Option<&str>, params: &QueryParams) -> String { + /// The URL of `sub_path` with the inherited (see [`with_base_params`](Self::with_base_params)) + /// and per-call query parameters applied. + pub(crate) fn url_with_params(&self, sub_path: Option<&str>, params: &QueryParams) -> String { self.merged_params(params).apply_to_url(&self.url(sub_path)) } } @@ -510,7 +528,7 @@ pub(crate) async fn head_exists( /// `content_encoding`, when set, is sent as the `Content-Encoding` header for an /// already-compressed value; it makes [`HttpClient::call`]'s automatic request compression skip /// the body (it already refuses to double-encode a request that carries a `Content-Encoding`), so -/// the bytes reach the API exactly as given. `options` are the per-call [`CallOptions`]. +/// the bytes reach the API exactly as given. pub(crate) async fn put_raw( ctx: &ResourceContext, sub_path: Option<&str>, @@ -556,6 +574,7 @@ pub(crate) async fn get_byte_stream( url: String, tier: Timeout, ) -> ApifyClientResult>>> { + ctx.check_request_allowed()?; ctx.http .call_streaming(HttpRequest { method: HttpMethod::Get, diff --git a/src/clients/build.rs b/src/clients/build.rs index de0532a..bf81d76 100644 --- a/src/clients/build.rs +++ b/src/clients/build.rs @@ -78,6 +78,11 @@ impl BuildClient { /// Returns a client for the build's log. pub fn log(&self) -> LogClient { - LogClient::nested(self.ctx.http.clone(), &self.ctx.url(None), "log") + LogClient::nested( + self.ctx.http.clone(), + &self.ctx.url(None), + "log", + &QueryParams::new(), + ) } } diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 2b9a09a..294989b 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -234,9 +234,15 @@ impl DatasetClient { } /// Creates a dataset client for a run's default dataset (no ID; nested path only). - pub(crate) fn nested(http: HttpClient, base_url: &str, sub_path: &str) -> Self { + pub(crate) fn nested( + http: HttpClient, + base_url: &str, + sub_path: &str, + base_params: &QueryParams, + ) -> Self { Self { - ctx: ResourceContext::collection(http, base_url, sub_path), + ctx: ResourceContext::collection(http, base_url, sub_path) + .with_base_params(base_params), } } @@ -272,7 +278,7 @@ impl DatasetClient { ) -> ApifyClientResult> { let mut params = QueryParams::new(); options.apply(&mut params); - let url = params.apply_to_url(&self.ctx.url(Some("items"))); + let url = self.ctx.url_with_params(Some("items"), ¶ms); let response = self .ctx .send(HttpMethod::Get, url, self.ctx.timeout(Timeout::Long)) @@ -365,7 +371,7 @@ impl DatasetClient { let mut params = QueryParams::new(); params.add_str("format", Some(format.as_str())); options.apply(&mut params); - let url = params.apply_to_url(&self.ctx.url(Some("items"))); + let url = self.ctx.url_with_params(Some("items"), ¶ms); let response = self .ctx .send(HttpMethod::Get, url, self.ctx.timeout(Timeout::Long)) @@ -395,7 +401,7 @@ impl DatasetClient { let mut params = QueryParams::new(); params.add_str("format", Some(format.as_str())); options.apply(&mut params); - let url = params.apply_to_url(&self.ctx.url(Some("items"))); + let url = self.ctx.url_with_params(Some("items"), ¶ms); get_byte_stream(&self.ctx, url, Timeout::Long).await } @@ -411,14 +417,12 @@ impl DatasetClient { pub async fn push_items(&self, items: &T) -> ApifyClientResult<()> { let body = serde_json::to_vec(items)?; check_items_shape(&body)?; - let url = self.ctx.url(Some("items")); + let url = self.ctx.url_with_params(Some("items"), &QueryParams::new()); let mut headers = std::collections::HashMap::new(); headers.insert( "Content-Type".to_string(), "application/json; charset=utf-8".to_string(), ); - // A push that timed out may have reached the API, so retrying it could store the items - // twice: the first transport timeout is final. self.ctx .send_with_options( HttpMethod::Post, diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index fe359c4..4aa51ff 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -82,9 +82,15 @@ impl KeyValueStoreClient { } /// Creates a KVS client for a run's default store (nested path, no ID). - pub(crate) fn nested(http: HttpClient, base_url: &str, sub_path: &str) -> Self { + pub(crate) fn nested( + http: HttpClient, + base_url: &str, + sub_path: &str, + base_params: &QueryParams, + ) -> Self { Self { - ctx: ResourceContext::collection(http, base_url, sub_path), + ctx: ResourceContext::collection(http, base_url, sub_path) + .with_base_params(base_params), } } @@ -362,7 +368,9 @@ impl KeyValueStoreClient { /// Deletes the record with the given key. pub async fn delete_record(&self, key: &str) -> ApifyClientResult<()> { - let url = self.ctx.url(Some(&record_path(key)?)); + let url = self + .ctx + .url_with_params(Some(&record_path(key)?), &QueryParams::new()); self.ctx .send(HttpMethod::Delete, url, self.ctx.timeout(Timeout::Short)) .await?; diff --git a/src/clients/log.rs b/src/clients/log.rs index 17eb999..6901ea1 100644 --- a/src/clients/log.rs +++ b/src/clients/log.rs @@ -46,9 +46,15 @@ impl LogClient { } /// Creates a log client nested under a run or build (path `.../log`). - pub(crate) fn nested(http: HttpClient, base_url: &str, sub_path: &str) -> Self { + pub(crate) fn nested( + http: HttpClient, + base_url: &str, + sub_path: &str, + base_params: &QueryParams, + ) -> Self { Self { - ctx: ResourceContext::collection(http, base_url, sub_path), + ctx: ResourceContext::collection(http, base_url, sub_path) + .with_base_params(base_params), } } @@ -144,7 +150,7 @@ impl LogClient { params .add_bool("raw", options.raw) .add_bool("download", options.download); - let url = params.apply_to_url(&self.ctx.url(None)); + let url = self.ctx.url_with_params(None, ¶ms); get_byte_stream(&self.ctx, url, Timeout::Long).await } } diff --git a/src/clients/mod.rs b/src/clients/mod.rs index 91308a1..afe4bcd 100644 --- a/src/clients/mod.rs +++ b/src/clients/mod.rs @@ -24,6 +24,7 @@ pub mod request_queue; pub mod request_queue_collection; pub mod run; pub mod run_collection; +pub mod run_dataset_items; pub mod schedule; pub mod schedule_collection; pub mod store_collection; diff --git a/src/clients/request_queue.rs b/src/clients/request_queue.rs index b29bbbb..8585326 100644 --- a/src/clients/request_queue.rs +++ b/src/clients/request_queue.rs @@ -225,9 +225,15 @@ impl RequestQueueClient { } /// Creates an RQ client for a run's default queue (nested path, no ID). - pub(crate) fn nested(http: HttpClient, base_url: &str, sub_path: &str) -> Self { + pub(crate) fn nested( + http: HttpClient, + base_url: &str, + sub_path: &str, + base_params: &QueryParams, + ) -> Self { Self { - ctx: ResourceContext::collection(http, base_url, sub_path), + ctx: ResourceContext::collection(http, base_url, sub_path) + .with_base_params(base_params), client_key: None, } } @@ -325,7 +331,7 @@ impl RequestQueueClient { })?; let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let url = params.apply_to_url(&self.ctx.url(Some(&request_path(&id)?))); + let url = self.ctx.url_with_params(Some(&request_path(&id)?), ¶ms); let body = serde_json::to_vec(request)?; let mut headers = std::collections::HashMap::new(); headers.insert("Content-Type".to_string(), "application/json".to_string()); @@ -345,7 +351,7 @@ impl RequestQueueClient { /// Deletes a request by ID. pub async fn delete_request(&self, id: &str) -> ApifyClientResult<()> { let params = self.base_params(); - let url = params.apply_to_url(&self.ctx.url(Some(&request_path(id)?))); + let url = self.ctx.url_with_params(Some(&request_path(id)?), ¶ms); self.ctx .send(HttpMethod::Delete, url, self.ctx.timeout(Timeout::Short)) .await?; @@ -601,7 +607,9 @@ impl RequestQueueClient { params .add_int("lockSecs", Some(lock_secs)) .add_bool("forefront", Some(forefront)); - let url = params.apply_to_url(&self.ctx.url(Some(&format!("{}/lock", request_path(id)?)))); + let url = self + .ctx + .url_with_params(Some(&format!("{}/lock", request_path(id)?)), ¶ms); let response = self .ctx .send(HttpMethod::Put, url, self.ctx.timeout(Timeout::Medium)) @@ -615,7 +623,9 @@ impl RequestQueueClient { pub async fn delete_request_lock(&self, id: &str, forefront: bool) -> ApifyClientResult<()> { let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let url = params.apply_to_url(&self.ctx.url(Some(&format!("{}/lock", request_path(id)?)))); + let url = self + .ctx + .url_with_params(Some(&format!("{}/lock", request_path(id)?)), ¶ms); self.ctx .send(HttpMethod::Delete, url, self.ctx.timeout(Timeout::Short)) .await?; diff --git a/src/clients/run.rs b/src/clients/run.rs index 7b32a92..bbec0a4 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -6,10 +6,11 @@ use crate::clients::base::{ delete_resource, get_resource, post_action, post_with_body, update_resource, wait_for_finish, ResourceContext, }; -use crate::clients::dataset::DatasetClient; +use crate::clients::dataset::{DatasetClient, DatasetListItemsOptions}; use crate::clients::key_value_store::KeyValueStoreClient; use crate::clients::log::LogClient; use crate::clients::request_queue::RequestQueueClient; +use crate::clients::run_dataset_items::RunDatasetItemsIterator; use crate::common::{catch_not_found, to_safe_id, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; @@ -91,6 +92,7 @@ pub struct RunChargeOptions { /// run's default dataset, key-value store, request queue and log. #[derive(Debug, Clone)] pub struct RunClient { + root: crate::client::ApifyClient, ctx: ResourceContext, /// The run ID, retained so `charge` can build a per-run idempotency key. id: String, @@ -100,18 +102,37 @@ crate::clients::base::impl_with_timeout!(RunClient); impl RunClient { pub(crate) fn new( - _root: crate::client::ApifyClient, + root: crate::client::ApifyClient, http: HttpClient, base_url: &str, resource_path: &str, id: &str, ) -> Self { Self { + root, ctx: ResourceContext::single(http, base_url, resource_path, id), id: id.to_string(), } } + /// The run ID this client addresses; `last` for the client of a `last_run()`. + pub(crate) fn id(&self) -> &str { + &self.id + } + + /// The timeout set with `with_timeout`, if any. + pub(crate) fn timeout(&self) -> Option { + self.ctx.timeout + } + + /// A client for the run with the given ID, with the timeout of this client. + pub(crate) fn pinned_to(&self, run_id: &str) -> RunClient { + let mut client = self.root.run(run_id); + client.ctx.timeout = self.ctx.timeout; + client.ctx.timeout_cap = self.ctx.timeout_cap; + client + } + /// Adds a base query parameter to this run client, inherited by its requests (used by /// `actor.last_run` / `task.last_run` to thread the `status` and `origin` filters). pub(crate) fn set_base_param(&mut self, key: &str, value: &str) { @@ -280,7 +301,9 @@ impl RunClient { body["requestId"] = serde_json::Value::String(request_id); } let body_bytes = serde_json::to_vec(&body)?; - let url = self.ctx.url(Some("charge")); + let url = self + .ctx + .url_with_params(Some("charge"), &QueryParams::new()); let mut headers = std::collections::HashMap::new(); headers.insert("Content-Type".to_string(), "application/json".to_string()); headers.insert(CHARGE_IDEMPOTENCY_HEADER.to_string(), idempotency_key); @@ -325,7 +348,12 @@ impl RunClient { /// Returns a client for the run's default dataset. pub fn dataset(&self) -> DatasetClient { - DatasetClient::nested(self.ctx.http.clone(), &self.ctx.url(None), "dataset") + DatasetClient::nested( + self.ctx.http.clone(), + &self.ctx.url(None), + "dataset", + &self.ctx.base_params, + ) } /// Returns a client for the run's default key-value store. @@ -334,17 +362,63 @@ impl RunClient { self.ctx.http.clone(), &self.ctx.url(None), "key-value-store", + &self.ctx.base_params, ) } /// Returns a client for the run's default request queue. pub fn request_queue(&self) -> RequestQueueClient { - RequestQueueClient::nested(self.ctx.http.clone(), &self.ctx.url(None), "request-queue") + RequestQueueClient::nested( + self.ctx.http.clone(), + &self.ctx.url(None), + "request-queue", + &self.ctx.base_params, + ) } /// Returns a client for the run's log. pub fn log(&self) -> LogClient { - LogClient::nested(self.ctx.http.clone(), &self.ctx.url(None), "log") + LogClient::nested( + self.ctx.http.clone(), + &self.ctx.url(None), + "log", + &self.ctx.base_params, + ) + } + + /// Iterates over the items of the run's default dataset while the run is still producing them. + /// + /// Returns a [`RunDatasetItemsIterator`]; call its `next()` to get one item at a time. Unlike + /// [`DatasetClient::iterate_items`], which stops at the end of the dataset as it is, this keeps + /// polling until the run finishes, so a consumer can process the items of a long-running Actor + /// as they appear. See [`RunDatasetItemsIterator`] for how the rows are read, and its + /// `with_chunk_size` and `with_poll_interval` for the page size and the polling interval. + /// + /// `options` filters and projects the items exactly like + /// [`DatasetClient::list_items`]: `offset` is the number of dataset rows to skip and `limit` + /// the maximum number of rows to scan. `desc` is not supported, since the end of a dataset that + /// is still growing is not known: the rows are always read oldest first. + /// + /// # Example + /// ```no_run + /// use apify_client::ApifyClient; + /// + /// # async fn run() -> Result<(), Box> { + /// let client = ApifyClient::new("my-api-token"); + /// let mut items = client + /// .run("my-run-id") + /// .iterate_dataset_items::(Default::default()); + /// while let Some(item) = items.next().await? { + /// println!("{item}"); + /// } + /// # Ok(()) + /// # } + /// ``` + pub fn iterate_dataset_items( + &self, + options: DatasetListItemsOptions, + ) -> RunDatasetItemsIterator { + RunDatasetItemsIterator::new(self.clone(), options) } /// Opens a live stream of the run's log for redirection, or `None` if the run does not exist. diff --git a/src/clients/run_dataset_items.rs b/src/clients/run_dataset_items.rs new file mode 100644 index 0000000..8542b7f --- /dev/null +++ b/src/clients/run_dataset_items.rs @@ -0,0 +1,278 @@ +//! Iteration over the default dataset of a run that is still producing items. + +use std::collections::VecDeque; +use std::time::Duration; + +use serde::de::DeserializeOwned; + +use crate::clients::dataset::{DatasetClient, DatasetListItemsOptions}; +use crate::clients::run::RunClient; +use crate::error::ApifyClientResult; +use crate::models::ActorRun; +use crate::timeouts::Timeout; + +/// Maximum number of dataset rows requested per API call unless +/// [`with_chunk_size`](RunDatasetItemsIterator::with_chunk_size) says otherwise. +const DEFAULT_CHUNK_SIZE: i64 = 1000; +/// The shortest wait between polls, in seconds: the API holds a poll for whole seconds, and a +/// poll that does not wait would repeat at once. +const MIN_POLL_SECS: u64 = 1; +/// How long to wait for the run to finish between polls unless +/// [`with_poll_interval`](RunDatasetItemsIterator::with_poll_interval) says otherwise. +const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(5); + +/// Where [`RunDatasetItemsIterator`] is in its walk over the dataset. +enum Phase { + /// Nothing is fetched yet: the run and its dataset still have to be resolved. + Start, + /// The run may still be producing items: rows are read up to the dataset's `itemCount`, then + /// the iterator waits for the run to make progress. + Live, + /// The run is finished, but `itemCount` lags behind the pushed items: rows are read a page at + /// a time until none are left. + Tail, + /// Every item was yielded. + Done, +} + +/// One round of the [`Phase::Live`] walk: the rows to read before the next poll. +struct Round { + /// The dataset's `itemCount` when the round started, capped at the end of the requested range. + item_count: i64, + /// Whether the run had reached a terminal status when the round started. + run_finished: bool, +} + +/// A lazy iterator over the items of a run's default dataset that keeps going while the run is +/// still producing them. Created by +/// [`RunClient::iterate_dataset_items`](crate::clients::run::RunClient::iterate_dataset_items). +/// +/// While the run has not finished, each round yields the rows below the dataset's `itemCount` and +/// then waits up to the poll interval for the run to finish, so the last rows are read as soon as +/// it does. Each page is requested with a `limit` that ends at `itemCount`, so it covers exactly +/// the rows it asks for, whatever the filters or `unwind` do to the items. `itemCount` lags a few +/// seconds behind the pushed items, so once the run reaches a terminal status, the rows past it +/// are read a page at a time until none are left, and the iterator ends. +/// +/// `offset` of the [`DatasetListItemsOptions`] is the number of rows to skip, and `limit` the +/// maximum number of dataset rows to scan: fewer items are yielded when filters drop some, more +/// when `unwind` splits a row into several. On a `last_run()` client, the iterator sticks to the +/// run that its first request resolves to. +/// +/// Every request uses the [`Timeout::Long`] tier, unless the run client was given another with +/// `with_timeout`. +pub struct RunDatasetItemsIterator { + run_client: RunClient, + options: DatasetListItemsOptions, + chunk_size: i64, + poll_interval: Duration, + /// The run client and dataset client the walk sticks to, resolved by the first request. + resolved: Option<(RunClient, DatasetClient)>, + run: Option, + /// Index of the next dataset row to read. + position: i64, + /// Index of the row the requested range ends before, if it is bounded. + end: Option, + round: Option, + phase: Phase, + buffer: VecDeque, +} + +impl RunDatasetItemsIterator { + pub(crate) fn new(run_client: RunClient, options: DatasetListItemsOptions) -> Self { + let position = options.offset.unwrap_or(0); + let end = options + .limit + .filter(|&limit| limit > 0) + .map(|limit| position + limit); + Self { + run_client, + options, + chunk_size: DEFAULT_CHUNK_SIZE, + poll_interval: DEFAULT_POLL_INTERVAL, + resolved: None, + run: None, + position, + end, + round: None, + phase: Phase::Start, + buffer: VecDeque::new(), + } + } + + /// Sets the maximum number of dataset rows requested per API call (default 1000). A + /// non-positive value keeps the default. + #[must_use] + pub fn with_chunk_size(mut self, chunk_size: i64) -> Self { + if chunk_size > 0 { + self.chunk_size = chunk_size; + } + self + } + + /// Sets how long to wait for the run to finish between polls (default 5 seconds). The API + /// holds a poll for whole seconds, so the interval is rounded down to them, to at least one. + #[must_use] + pub fn with_poll_interval(mut self, poll_interval: Duration) -> Self { + self.poll_interval = poll_interval; + self + } + + /// Returns the next item, or `None` when the run is finished and every item was read. Fetches + /// more rows, or waits for the run to make progress, when the local buffer is empty. + pub async fn next(&mut self) -> ApifyClientResult> { + loop { + if let Some(item) = self.buffer.pop_front() { + return Ok(Some(item)); + } + match self.phase { + Phase::Start => self.resolve().await?, + Phase::Live => self.read_live_rows().await?, + Phase::Tail => self.read_tail_rows().await?, + Phase::Done => return Ok(None), + } + } + } + + /// Eagerly drains the iterator into a single `Vec`, waiting for the run to finish. + pub async fn collect_all(mut self) -> ApifyClientResult> { + let mut items = Vec::new(); + while let Some(item) = self.next().await? { + items.push(item); + } + Ok(items) + } + + /// Fetches the run, and pins the walk to it and to its default dataset. + async fn resolve(&mut self) -> ApifyClientResult<()> { + let timeout = self.run_client.timeout().unwrap_or(Timeout::Long); + let run_client = self.run_client.clone().with_timeout(timeout); + let run = run_client.get().await?; + // A `last_run()` client resolves `runs/last` on each request, so a newer run would swap + // the dataset mid-iteration. + let run_client = match &run { + Some(run) if run.id != self.run_client.id() => { + self.run_client.pinned_to(&run.id).with_timeout(timeout) + } + _ => run_client, + }; + let dataset = run_client.dataset().with_timeout(timeout); + self.run = run; + self.resolved = Some((run_client, dataset)); + self.phase = Phase::Live; + Ok(()) + } + + fn clients(&self) -> &(RunClient, DatasetClient) { + self.resolved + .as_ref() + .expect("the run is resolved before any row is read") + } + + /// One step of the [`Phase::Live`] walk: starts a round, reads a page of it, or ends it. + async fn read_live_rows(&mut self) -> ApifyClientResult<()> { + let item_count = match &self.round { + Some(round) => round.item_count, + None => { + let run_finished = self.run.as_ref().is_none_or(ActorRun::is_terminal); + let dataset = self.clients().1.get().await?; + let mut item_count = dataset.and_then(|dataset| dataset.item_count).unwrap_or(0); + if let Some(end) = self.end { + item_count = item_count.min(end); + } + self.round = Some(Round { + item_count, + run_finished, + }); + item_count + } + }; + + if self.position < item_count { + let page_limit = self.chunk_size.min(item_count - self.position); + let page = self.list_page(self.position, page_limit).await?; + self.buffer.extend(page.items); + self.position += page_limit; + return Ok(()); + } + + // The round is read: the run is polled for more, unless this was the last round. + let run_finished = self.round.take().is_some_and(|round| round.run_finished); + if self.end.is_some_and(|end| self.position >= end) { + self.phase = Phase::Done; + } else if run_finished { + self.phase = Phase::Tail; + } else { + let wait_secs = + i64::try_from(self.poll_interval.as_secs().max(MIN_POLL_SECS)).unwrap_or(i64::MAX); + self.run = Some(self.clients().0.wait_for_finish(Some(wait_secs)).await?); + } + Ok(()) + } + + /// One step of the [`Phase::Tail`] walk: reads a page past `itemCount`. + async fn read_tail_rows(&mut self) -> ApifyClientResult<()> { + let page_limit = match self.end { + Some(end) => self.chunk_size.min(end - self.position), + None => self.chunk_size, + }; + let page = self.list_page(self.position, page_limit).await?; + // Only an empty page marks the end, as filters can shorten a full one. A page that + // `clean`, `skip_empty` or `unwind` emptied past a lagging `itemCount` reports no scanned + // rows either, so a plain read checks. + let is_empty = page.items.is_empty() && page.scanned.unwrap_or(0) == 0; + self.buffer.extend(page.items); + if is_empty && !self.filters_can_hide_rows() { + self.phase = Phase::Done; + return Ok(()); + } + if is_empty && !self.rows_are_left().await? { + self.phase = Phase::Done; + return Ok(()); + } + self.position += page_limit; + if self.end.is_some_and(|end| self.position >= end) { + self.phase = Phase::Done; + } + Ok(()) + } + + /// Whether a server-side filter can empty a page that still has rows behind it. + fn filters_can_hide_rows(&self) -> bool { + self.options.clean == Some(true) + || self.options.skip_empty == Some(true) + || self + .options + .unwind + .as_ref() + .is_some_and(|fields| !fields.is_empty()) + } + + /// Whether a plain, unfiltered read finds a row at the current position. + async fn rows_are_left(&self) -> ApifyClientResult { + let probe = DatasetListItemsOptions { + offset: Some(self.position), + limit: Some(1), + ..Default::default() + }; + let page = self + .clients() + .1 + .list_items::(probe) + .await?; + Ok(!page.items.is_empty()) + } + + async fn list_page( + &self, + offset: i64, + limit: i64, + ) -> ApifyClientResult> { + let options = DatasetListItemsOptions { + offset: Some(offset), + limit: Some(limit), + ..self.options.clone() + }; + self.clients().1.list_items::(options).await + } +} diff --git a/src/http_client.rs b/src/http_client.rs index 4ffc396..8fb2417 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -288,12 +288,12 @@ pub(crate) struct HttpClientConfig { /// Options of a single [`HttpClient::call_with_options`]. #[derive(Debug, Clone, Copy, Default)] -pub struct CallOptions { +pub(crate) struct CallOptions { /// Give up on the first transport timeout instead of retrying it. For a call that is not /// idempotent, such as pushing dataset items, whose request may have reached the API before /// the client gave up on it: a retry could then store the same data twice. Other errors are /// retried as usual. - pub do_not_retry_timeouts: bool, + pub(crate) do_not_retry_timeouts: bool, } /// The orchestrating HTTP client shared by every resource client. @@ -340,7 +340,7 @@ impl HttpClient { self } - fn fail_if_poisoned(&self) -> ApifyClientResult<()> { + pub(crate) fn fail_if_poisoned(&self) -> ApifyClientResult<()> { match &self.poison { Some(message) => Err(ApifyClientError::InvalidArgument(message.to_string())), None => Ok(()), @@ -397,7 +397,7 @@ impl HttpClient { } /// Like [`call`](Self::call), with the given per-call [`CallOptions`]. - pub async fn call_with_options( + pub(crate) async fn call_with_options( &self, mut request: HttpRequest, options: CallOptions, diff --git a/src/lib.rs b/src/lib.rs index f0aaed4..e43ac18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,6 +95,7 @@ pub use clients::run::{ LastRunOptions, RunChargeOptions, RunMetamorphOptions, RunResurrectOptions, }; pub use clients::run_collection::RunListOptions; +pub use clients::run_dataset_items::RunDatasetItemsIterator; pub use clients::store_collection::{StoreActorIterator, StoreListOptions}; // Compile-test the code snippets in the README and the external `docs/` pages so every diff --git a/tests/actor_run.rs b/tests/actor_run.rs index 281ba70..aa8b4b6 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -280,3 +280,38 @@ async fn list_runs_filtered_by_deployment() { .expect("listing runs by deployment should succeed"); assert!(page.total >= 0); } + +/// Iterating the default dataset of a run reads the items pushed to it, including those past the +/// dataset's `itemCount`, which lags a few seconds behind the pushed items. +#[tokio::test(flavor = "multi_thread")] +async fn iterate_run_dataset_items_reads_the_items_of_a_finished_run() { + let client = require_client!(); + let run = client + .actor("apify/hello-world") + .call::(None, Default::default(), Some(120)) + .await + .expect("call hello-world actor"); + let dataset_id = run + .default_dataset_id + .clone() + .expect("a run has a default dataset"); + client + .dataset(&dataset_id) + .push_items(&serde_json::json!([{ "n": 1 }, { "n": 2 }, { "n": 3 }])) + .await + .expect("push items to the run's dataset"); + + let items = client + .run(&run.id) + .iterate_dataset_items::(Default::default()) + .with_chunk_size(2) + .collect_all() + .await + .expect("iterate the run's dataset items"); + + // The Actor may have written items of its own before the three that were pushed. + assert!(items.len() >= 3, "{items:?}"); + let pushed = &items[items.len() - 3..]; + assert_eq!(pushed[0]["n"], 1); + assert_eq!(pushed[2]["n"], 3); +} diff --git a/tests/unit_resources.rs b/tests/unit_resources.rs index ed36528..ddcbff7 100644 --- a/tests/unit_resources.rs +++ b/tests/unit_resources.rs @@ -8,7 +8,8 @@ mod common; use apify_client::{ - ApifyClientError, ListRequestsOptions, LogOptions, RunListOptions, StoreListOptions, + ApifyClientError, LastRunOptions, ListRequestsOptions, LogOptions, RunListOptions, + StoreListOptions, }; use common::mock_backend::{client_with, MockBackend, MockOutcome}; use futures_util::StreamExt; @@ -475,3 +476,83 @@ async fn a_schedule_with_a_null_ui_state_deserializes() { .expect("the schedule exists"); assert_eq!(schedule.extra.get("uiState"), Some(&Value::Null)); } + +// ----- clients chained from the last run -------------------------------------------------- + +/// The `status` and `origin` filters of `last_run` select the run, so the log and the storages of +/// that run have to carry them, or they would belong to the newest run of any status. +#[tokio::test] +async fn storages_and_log_chained_from_a_filtered_last_run_carry_its_filters() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"[]".to_vec())]); + let client = client_with(backend.clone(), 0); + let last_run = client.actor("a").last_run_with_options(LastRunOptions { + status: Some("SUCCEEDED".to_string()), + origin: Some("API".to_string()), + }); + let carries_filters = |url: String, path: &str| { + assert!(url.contains(path), "{url}"); + assert!(url.contains("status=SUCCEEDED"), "{url}"); + assert!(url.contains("origin=API"), "{url}"); + }; + + last_run.log().get().await.unwrap(); + carries_filters(backend.last_url().unwrap(), "/runs/last/log"); + last_run.log().stream().await.unwrap(); + carries_filters(backend.last_url().unwrap(), "/runs/last/log"); + last_run + .dataset() + .list_items::(Default::default()) + .await + .unwrap(); + carries_filters(backend.last_url().unwrap(), "/runs/last/dataset/items"); + last_run + .dataset() + .push_items(&json!({ "a": 1 })) + .await + .unwrap(); + carries_filters(backend.last_url().unwrap(), "/runs/last/dataset/items"); + last_run + .key_value_store() + .get_record("OUTPUT") + .await + .unwrap(); + carries_filters( + backend.last_url().unwrap(), + "/runs/last/key-value-store/records/OUTPUT", + ); + last_run + .key_value_store() + .delete_record("OUTPUT") + .await + .unwrap(); + carries_filters( + backend.last_url().unwrap(), + "/runs/last/key-value-store/records/OUTPUT", + ); + last_run + .request_queue() + .delete_request("request-id") + .await + .unwrap(); + carries_filters( + backend.last_url().unwrap(), + "/runs/last/request-queue/requests/request-id", + ); +} + +/// The tasks' `last_run` filters apply the same way. +#[tokio::test] +async fn the_log_chained_from_a_task_last_run_carries_its_filters() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"log".to_vec())]); + let client = client_with(backend.clone(), 0); + client + .task("t") + .last_run(Some("FAILED")) + .log() + .get() + .await + .unwrap(); + let url = backend.last_url().unwrap(); + assert!(url.contains("/actor-tasks/t/runs/last/log"), "{url}"); + assert!(url.contains("status=FAILED"), "{url}"); +} diff --git a/tests/unit_run_dataset_items.rs b/tests/unit_run_dataset_items.rs new file mode 100644 index 0000000..224dcca --- /dev/null +++ b/tests/unit_run_dataset_items.rs @@ -0,0 +1,272 @@ +//! Offline unit tests for `RunClient::iterate_dataset_items`, which reads the items of a run's +//! default dataset while the run is still producing them. +//! +//! These use a mock [`HttpBackend`](apify_client::http_client::HttpBackend) that answers the +//! requests in order, and require no network access or `APIFY_TOKEN`. + +mod common; + +use apify_client::clients::dataset::DatasetListItemsOptions; +use common::mock_backend::{client_with, MockBackend, MockOutcome}; +use serde_json::{json, Value}; + +fn run(status: &str) -> MockOutcome { + MockOutcome::Status( + 200, + json!({ "data": { "id": "r", "status": status } }) + .to_string() + .into_bytes(), + ) +} + +fn dataset(item_count: i64) -> MockOutcome { + MockOutcome::Status( + 200, + json!({ "data": { "id": "d", "itemCount": item_count } }) + .to_string() + .into_bytes(), + ) +} + +fn items(items: Value) -> MockOutcome { + MockOutcome::Status(200, items.to_string().into_bytes()) +} + +/// The path and the query of every request, without the origin. +fn requests(backend: &MockBackend) -> Vec { + backend + .urls() + .iter() + .map(|url| { + url.split_once("/v2/") + .map(|(_, rest)| rest) + .unwrap() + .to_string() + }) + .collect() +} + +#[tokio::test] +async fn items_are_read_while_the_run_produces_them_and_after_it_finishes() { + let backend = MockBackend::new(vec![ + run("RUNNING"), + dataset(2), + items(json!([{ "n": 1 }, { "n": 2 }])), + run("SUCCEEDED"), // The poll that ends the wait. + dataset(3), + items(json!([{ "n": 3 }])), + items(json!([])), // The tail past a lagging `itemCount`. + ]); + let client = client_with(backend.clone(), 0); + + let collected: Vec = client + .run("r") + .iterate_dataset_items::(Default::default()) + .collect_all() + .await + .unwrap(); + + assert_eq!( + collected, + vec![json!({ "n": 1 }), json!({ "n": 2 }), json!({ "n": 3 })] + ); + let requests = requests(&backend); + assert_eq!(requests.len(), 7, "{requests:?}"); + assert_eq!(requests[0], "actor-runs/r"); + assert_eq!(requests[1], "actor-runs/r/dataset"); + assert_eq!(requests[2], "actor-runs/r/dataset/items?offset=0&limit=2"); + assert!( + requests[3].starts_with("actor-runs/r?waitForFinish="), + "{}", + requests[3] + ); + assert_eq!(requests[4], "actor-runs/r/dataset"); + assert_eq!(requests[5], "actor-runs/r/dataset/items?offset=2&limit=1"); + assert_eq!( + requests[6], + "actor-runs/r/dataset/items?offset=3&limit=1000" + ); +} + +#[tokio::test] +async fn items_are_yielded_one_at_a_time_as_pages_arrive() { + let backend = MockBackend::new(vec![ + run("SUCCEEDED"), + dataset(3), + items(json!([{ "n": 1 }, { "n": 2 }])), + items(json!([{ "n": 3 }])), + items(json!([])), + ]); + let client = client_with(backend.clone(), 0); + let mut iterator = client + .run("r") + .iterate_dataset_items::(Default::default()) + .with_chunk_size(2); + + assert_eq!(iterator.next().await.unwrap(), Some(json!({ "n": 1 }))); + assert_eq!(backend.call_count(), 3, "only the first page was read"); + assert_eq!(iterator.next().await.unwrap(), Some(json!({ "n": 2 }))); + assert_eq!(backend.call_count(), 3, "the page is buffered"); + assert_eq!(iterator.next().await.unwrap(), Some(json!({ "n": 3 }))); + assert_eq!(iterator.next().await.unwrap(), None); + assert_eq!( + iterator.next().await.unwrap(), + None, + "the iterator stays ended" + ); + + let requests = requests(&backend); + assert_eq!(requests[2], "actor-runs/r/dataset/items?offset=0&limit=2"); + assert_eq!(requests[3], "actor-runs/r/dataset/items?offset=2&limit=1"); +} + +/// `offset` and `limit` bound the rows that are read, however many items the dataset has. +#[tokio::test] +async fn the_range_of_rows_is_bounded_by_offset_and_limit() { + let backend = MockBackend::new(vec![ + run("SUCCEEDED"), + dataset(10), + items(json!([{ "n": 2 }, { "n": 3 }])), + ]); + let client = client_with(backend.clone(), 0); + + let collected: Vec = client + .run("r") + .iterate_dataset_items::(DatasetListItemsOptions { + offset: Some(1), + limit: Some(2), + ..Default::default() + }) + .collect_all() + .await + .unwrap(); + + assert_eq!(collected.len(), 2); + let requests = requests(&backend); + assert_eq!( + requests.len(), + 3, + "the range is complete, so nothing past it is read" + ); + assert_eq!(requests[2], "actor-runs/r/dataset/items?offset=1&limit=2"); +} + +/// A filter can empty a page past a lagging `itemCount` while rows are left behind it, so an +/// empty page ends the iteration only once a plain read confirms nothing is left. +#[tokio::test] +async fn a_filtered_empty_tail_page_is_checked_with_a_plain_read() { + let backend = MockBackend::new(vec![ + run("SUCCEEDED"), + dataset(0), + items(json!([])), // The tail page the filter emptied... + items(json!([{ "hidden": 1 }])), // ...while the plain read finds a row behind it. + items(json!([{ "n": 1 }])), // The next tail page has an item. + items(json!([])), + items(json!([])), // The plain read finds nothing: this is the end. + ]); + let client = client_with(backend.clone(), 0); + + let collected: Vec = client + .run("r") + .iterate_dataset_items::(DatasetListItemsOptions { + clean: Some(true), + ..Default::default() + }) + .with_chunk_size(5) + .collect_all() + .await + .unwrap(); + + assert_eq!(collected, vec![json!({ "n": 1 })]); + let requests = requests(&backend); + assert_eq!( + requests[2], + "actor-runs/r/dataset/items?offset=0&limit=5&clean=1" + ); + assert_eq!( + requests[3], "actor-runs/r/dataset/items?offset=0&limit=1", + "the plain read has no filter" + ); + assert_eq!( + requests[4], + "actor-runs/r/dataset/items?offset=5&limit=5&clean=1" + ); +} + +/// The iterator sticks to the run that its first request resolves to, instead of asking for the +/// last run again. +#[tokio::test] +async fn a_last_run_client_sticks_to_the_run_it_resolved() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + json!({ "data": { "id": "resolved-run", "status": "SUCCEEDED" } }) + .to_string() + .into_bytes(), + ), + dataset(1), + items(json!([{ "n": 1 }])), + items(json!([])), + ]); + let client = client_with(backend.clone(), 0); + + let collected: Vec = client + .actor("a") + .last_run(Some("SUCCEEDED")) + .iterate_dataset_items::(Default::default()) + .collect_all() + .await + .unwrap(); + + assert_eq!(collected.len(), 1); + let requests = requests(&backend); + assert_eq!(requests[0], "actors/a/runs/last?status=SUCCEEDED"); + for request in &requests[1..] { + assert!( + request.starts_with("actor-runs/resolved-run/dataset"), + "{request}" + ); + } +} + +/// The requests use the `Long` tier unless the run client has a timeout of its own. +#[tokio::test] +async fn every_request_uses_the_long_tier_unless_overridden() { + let script = || { + vec![ + run("SUCCEEDED"), + dataset(1), + items(json!([{ "n": 1 }])), + items(json!([])), + ] + }; + + let backend = MockBackend::new(script()); + let client = client_with(backend.clone(), 0); + client + .run("r") + .iterate_dataset_items::(Default::default()) + .collect_all() + .await + .unwrap(); + assert!(backend + .timeouts() + .iter() + .all(|timeout| timeout.as_secs() == 360)); + + let backend = MockBackend::new(script()); + let client = client_with(backend.clone(), 0); + client + .run("r") + .with_timeout(apify_client::Timeout::Custom( + std::time::Duration::from_secs(9), + )) + .iterate_dataset_items::(Default::default()) + .collect_all() + .await + .unwrap(); + assert!(backend + .timeouts() + .iter() + .all(|timeout| timeout.as_secs() == 9)); +} diff --git a/tests/unit_timeouts.rs b/tests/unit_timeouts.rs index 2fcb86f..020d367 100644 --- a/tests/unit_timeouts.rs +++ b/tests/unit_timeouts.rs @@ -630,3 +630,68 @@ async fn key_value_store_writes_and_public_urls_use_the_long_tier() { assert_eq!(backend.timeouts(), vec![Duration::from_secs(360); 5]); } + +/// A zero timeout belongs to the client it was set on: a client chained from it does not inherit +/// it, and a later `with_timeout` replaces it. +#[tokio::test] +async fn a_zero_timeout_is_neither_inherited_nor_sticky() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "d" }))]); + let client = client_with(backend.clone(), 0); + let run = client + .run("r") + .with_timeout(Timeout::Custom(Duration::ZERO)); + + assert!(matches!( + run.get().await, + Err(ApifyClientError::InvalidArgument(_)) + )); + run.dataset() + .get() + .await + .expect("a chained client has its own timeout"); + assert_eq!(backend.call_count(), 1); + + run.with_timeout(Timeout::Short) + .get() + .await + .expect("a later timeout replaces the zero one"); + assert_eq!(backend.call_count(), 2); +} + +/// An invalid id is reported as such, not hidden by a zero timeout set on the same client. +#[tokio::test] +async fn an_invalid_id_keeps_its_own_error_next_to_a_zero_timeout() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "x" }))]); + let client = client_with(backend, 0); + let error = client + .actor("..") + .with_timeout(Timeout::Custom(Duration::ZERO)) + .get() + .await + .expect_err("the id is invalid"); + assert!(error.to_string().contains("dot segment"), "{error}"); +} + +#[tokio::test] +async fn a_zero_tier_or_cap_fails_every_request() { + for builder in [ + builder_with(MockBackend::new(vec![ok_data(json!({ "id": "a" }))])) + .timeout_short(Duration::ZERO), + builder_with(MockBackend::new(vec![ok_data(json!({ "id": "a" }))])) + .timeout_medium(Duration::ZERO), + builder_with(MockBackend::new(vec![ok_data(json!({ "id": "a" }))])) + .timeout_long(Duration::ZERO), + builder_with(MockBackend::new(vec![ok_data(json!({ "id": "a" }))])) + .timeout_max(Duration::ZERO), + builder_with(MockBackend::new(vec![ok_data(json!({ "id": "a" }))])).timeout(Duration::ZERO), + ] { + let client = builder.build(); + let error = client.actor("a").get().await.expect_err("a zero timeout"); + assert!( + matches!(error, ApifyClientError::InvalidArgument(_)), + "{error:?}" + ); + // The failure follows the clients chained from it. + assert!(client.run("r").dataset().get().await.is_err()); + } +} From 3abf95102741b6483bb210650df05799966e7b45 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 15:50:07 +0000 Subject: [PATCH 04/11] fix: address the third review of the v3 port Rejects desc in iterate_dataset_items, keeps the last_run filters in public storage URLs and documents the zero timeout rule of the builder. --- README.md | 4 +++- docs/README.md | 6 ++++-- docs/runs.md | 2 +- src/client.rs | 12 ++++++++---- src/clients/base.rs | 10 ++++++++++ src/clients/dataset.rs | 2 +- src/clients/key_value_store.rs | 4 ++-- src/clients/run.rs | 13 ++++++------- src/clients/run_dataset_items.rs | 10 +++++++++- tests/unit_resources.rs | 32 ++++++++++++++++++++++++++++++++ tests/unit_run_dataset_items.rs | 24 ++++++++++++++++++++++++ 11 files changed, 100 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 56444f6..816773a 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,9 @@ applies to each attempt separately and doubles with every retry, up to the cap. A resource client's `with_timeout` replaces the tier of every request that client sends, with a tier, an exact duration, or `Timeout::NoTimeout` to let a request run for as long as it takes. With `NoTimeout` the client never aborts the request, so a connection that stalls is not retried -either. A zero duration is rejected: every request then fails with `InvalidArgument`. Clients +either. A zero duration is rejected: every request then fails with `InvalidArgument`, and so does +every request of a client built with a zero `timeout_short`, `timeout_medium`, `timeout_long` or +`timeout_max`. Clients obtained from that client, such as `run.dataset()`, do not inherit the timeout. A request queue client also has `with_timeout_cap(duration)`, which only lowers the tiers longer than `duration`. diff --git a/docs/README.md b/docs/README.md index fd5d8af..fa4b312 100644 --- a/docs/README.md +++ b/docs/README.md @@ -140,7 +140,7 @@ Builder options: | `public_base_url(u)` | = `base_url` | Origin used when building public/shareable URLs. | | `max_retries(n)` | `8` | Maximum retries for `429`/`5xx`/network errors. | | `min_delay_between_retries(d)` | `500ms` | Base backoff delay (doubled each retry, with jitter). | -| `timeout_short(d)` | `5s` | Duration of the `Timeout::Short` tier (simple metadata reads and writes). | +| `timeout_short(d)` | `5s` | Duration of the `Timeout::Short` tier (simple metadata reads and writes). Like the other timeout options, a zero duration makes every request of the client fail with `InvalidArgument`. | | `timeout_medium(d)` | `30s` | Duration of the `Timeout::Medium` tier (listing, batch and trigger calls). | | `timeout_long(d)` | `360s` | Duration of the `Timeout::Long` tier (downloads, uploads and streaming). | | `timeout_max(d)` | `360s` | Cap on the timeout of a single attempt. `timeout(d)` is the same option under its original name. | @@ -183,7 +183,9 @@ retry, up to the same cap. Every resource client has a `with_timeout(Timeout)` method that replaces the tier of all the requests it sends with a tier, `Timeout::Custom(duration)`, or `Timeout::NoTimeout` (the client never aborts the request, so a stalled connection is not retried either). A zero duration is -rejected: every request then fails with `InvalidArgument`. Clients obtained from it, such as +rejected: every request then fails with `InvalidArgument`, and so does every request of a client +built with a zero `timeout_short`, `timeout_medium`, `timeout_long` or `timeout_max`. Clients +obtained from it, such as `run.dataset()`, do not inherit the timeout. A request queue client also has `with_timeout_cap(duration)`, which only lowers the tiers longer than `duration`. diff --git a/docs/runs.md b/docs/runs.md index 406572b..6d7e03c 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -15,7 +15,7 @@ collections are available via `actor.runs()` and `task.runs()`. | Method | Arguments | Returns | Description | |---|---|---|---| | `get()` | — | `Option` | Fetches the run. | -| `iterate_dataset_items::(options)` | `DatasetListItemsOptions` | `RunDatasetItemsIterator` | Lazily iterates the items of the run's default dataset while the run is still producing them, until it finishes. `with_chunk_size(n)` sets the rows per request (default 1000) and `with_poll_interval(d)` the wait for the run between polls (default 5s). `offset` and `limit` of the options bound the rows that are read; `desc` is not supported. A `last_run()` client sticks to the run its first request resolves to. | +| `iterate_dataset_items::(options)` | `DatasetListItemsOptions` | `RunDatasetItemsIterator` | Lazily iterates the items of the run's default dataset while the run is still producing them, until it finishes. `with_chunk_size(n)` sets the rows per request (default 1000) and `with_poll_interval(d)` the wait for the run between polls (default 5s). `offset` and `limit` of the options bound the rows that are read; `desc: Some(true)` is rejected with `InvalidArgument`. A `last_run()` client sticks to the run its first request resolves to. | | `update(fields)` | `&impl Serialize` | `ActorRun` | Updates the run (e.g. status message). | | `delete()` | — | `()` | Deletes the run. | | `abort(gracefully)` | `Option` | `ActorRun` | Aborts the run. `None` omits the param (server default, immediate); `Some(true)`/`Some(false)` abort gracefully/immediately. | diff --git a/src/client.rs b/src/client.rs index 19f2ecd..8e55598 100644 --- a/src/client.rs +++ b/src/client.rs @@ -131,14 +131,16 @@ impl ApifyClientBuilder { } /// Sets the duration of the [`Timeout::Medium`](crate::Timeout::Medium) tier, used by - /// listing, batch and trigger operations (default `30s`). + /// listing, batch and trigger operations (default `30s`). A zero duration is rejected: a client + /// built with one fails every request with `InvalidArgument`. pub fn timeout_medium(mut self, timeout: Duration) -> Self { self.tiers.medium = timeout; self } /// Sets the duration of the [`Timeout::Long`](crate::Timeout::Long) tier, used by downloads, - /// uploads and streaming (default `360s`). + /// uploads and streaming (default `360s`). A zero duration is rejected: a client built with one + /// fails every request with `InvalidArgument`. pub fn timeout_long(mut self, timeout: Duration) -> Self { self.tiers.long = timeout; self @@ -146,14 +148,16 @@ impl ApifyClientBuilder { /// Caps the timeout of a single request attempt (default `360s`). A tier or an explicit /// timeout above the cap is lowered to it. The timeout of each attempt doubles with every - /// retry, up to this cap. + /// retry, up to this cap. A zero duration is rejected: a client built with one fails every + /// request with `InvalidArgument`. pub fn timeout_max(mut self, timeout: Duration) -> Self { self.tiers.max = timeout; self } /// Caps the timeout of a single request attempt (default `360s`). Equivalent to - /// [`timeout_max`](Self::timeout_max), kept under its original name. + /// [`timeout_max`](Self::timeout_max), kept under its original name, and a zero duration is + /// rejected the same way. pub fn timeout(self, timeout: Duration) -> Self { self.timeout_max(timeout) } diff --git a/src/clients/base.rs b/src/clients/base.rs index 8f67ba6..8002c80 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -163,6 +163,16 @@ impl ResourceContext { } } + /// Like [`url_with_params`](Self::url_with_params), for the public (shareable) form of the URL. + pub(crate) fn public_url_with_params( + &self, + sub_path: Option<&str>, + params: &QueryParams, + ) -> String { + self.merged_params(params) + .apply_to_url(&self.public_url(sub_path)) + } + /// Merges the inherited base params with per-call params into a final query string. pub(crate) fn merged_params(&self, params: &QueryParams) -> QueryParams { let mut merged = self.base_params.clone(); diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 294989b..a5f04ec 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -482,7 +482,7 @@ impl DatasetClient { params.add_str("signature", Some(signature)); } } - Ok(params.apply_to_url(&self.ctx.public_url(Some("items")))) + Ok(self.ctx.public_url_with_params(Some("items"), ¶ms)) } /// Returns statistical information about the dataset. diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 4aa51ff..eef583d 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -340,7 +340,7 @@ impl KeyValueStoreClient { params.add_str("signature", Some(create_hmac_signature(secret, key))); } } - Ok(params.apply_to_url(&self.ctx.public_url(Some(&path)))) + Ok(self.ctx.public_url_with_params(Some(&path), ¶ms)) } /// Builds a public URL for listing this store's keys. @@ -363,7 +363,7 @@ impl KeyValueStoreClient { params.add_str("signature", Some(signature)); } } - Ok(params.apply_to_url(&self.ctx.public_url(Some("keys")))) + Ok(self.ctx.public_url_with_params(Some("keys"), ¶ms)) } /// Deletes the record with the given key. diff --git a/src/clients/run.rs b/src/clients/run.rs index bbec0a4..ae22736 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -125,12 +125,9 @@ impl RunClient { self.ctx.timeout } - /// A client for the run with the given ID, with the timeout of this client. + /// A client for the run with the given ID. pub(crate) fn pinned_to(&self, run_id: &str) -> RunClient { - let mut client = self.root.run(run_id); - client.ctx.timeout = self.ctx.timeout; - client.ctx.timeout_cap = self.ctx.timeout_cap; - client + self.root.run(run_id) } /// Adds a base query parameter to this run client, inherited by its requests (used by @@ -396,8 +393,10 @@ impl RunClient { /// /// `options` filters and projects the items exactly like /// [`DatasetClient::list_items`]: `offset` is the number of dataset rows to skip and `limit` - /// the maximum number of rows to scan. `desc` is not supported, since the end of a dataset that - /// is still growing is not known: the rows are always read oldest first. + /// the maximum number of rows to scan. `desc: Some(true)` is rejected with + /// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument), since + /// the end of a dataset that is still growing is not known: the rows are always read oldest + /// first. /// /// # Example /// ```no_run diff --git a/src/clients/run_dataset_items.rs b/src/clients/run_dataset_items.rs index 8542b7f..3df857b 100644 --- a/src/clients/run_dataset_items.rs +++ b/src/clients/run_dataset_items.rs @@ -7,7 +7,7 @@ use serde::de::DeserializeOwned; use crate::clients::dataset::{DatasetClient, DatasetListItemsOptions}; use crate::clients::run::RunClient; -use crate::error::ApifyClientResult; +use crate::error::{ApifyClientError, ApifyClientResult}; use crate::models::ActorRun; use crate::timeouts::Timeout; @@ -145,6 +145,14 @@ impl RunDatasetItemsIterator { /// Fetches the run, and pins the walk to it and to its default dataset. async fn resolve(&mut self) -> ApifyClientResult<()> { + // The rows are counted from the oldest, since the end of a dataset that is still growing is + // not known, and `desc` would count them from the newest. + if self.options.desc == Some(true) { + return Err(ApifyClientError::InvalidArgument( + "iterate_dataset_items reads the rows oldest first and does not support `desc`" + .to_string(), + )); + } let timeout = self.run_client.timeout().unwrap_or(Timeout::Long); let run_client = self.run_client.clone().with_timeout(timeout); let run = run_client.get().await?; diff --git a/tests/unit_resources.rs b/tests/unit_resources.rs index ddcbff7..a0bf55f 100644 --- a/tests/unit_resources.rs +++ b/tests/unit_resources.rs @@ -556,3 +556,35 @@ async fn the_log_chained_from_a_task_last_run_carries_its_filters() { assert!(url.contains("/actor-tasks/t/runs/last/log"), "{url}"); assert!(url.contains("status=FAILED"), "{url}"); } + +/// The public URLs of a storage chained from a filtered `last_run` name the same run as the +/// storage they were signed for. +#[tokio::test] +async fn public_urls_of_storages_chained_from_a_filtered_last_run_carry_its_filters() { + let backend = MockBackend::new(vec![ok_data(json!({ "id": "s" }))]); + let client = client_with(backend, 0); + let last_run = client.actor("a").last_run(Some("SUCCEEDED")); + + let urls = [ + last_run + .dataset() + .create_items_public_url(Default::default(), None) + .await + .unwrap(), + last_run + .key_value_store() + .get_record_public_url("OUTPUT") + .await + .unwrap(), + last_run + .key_value_store() + .create_keys_public_url(None) + .await + .unwrap(), + ]; + + for url in urls { + assert!(url.contains("/runs/last/"), "{url}"); + assert!(url.contains("status=SUCCEEDED"), "{url}"); + } +} diff --git a/tests/unit_run_dataset_items.rs b/tests/unit_run_dataset_items.rs index 224dcca..8de730a 100644 --- a/tests/unit_run_dataset_items.rs +++ b/tests/unit_run_dataset_items.rs @@ -270,3 +270,27 @@ async fn every_request_uses_the_long_tier_unless_overridden() { .iter() .all(|timeout| timeout.as_secs() == 9)); } + +/// The rows are counted from the oldest, so `desc` would mix up the offsets of a dataset that is +/// still growing. +#[tokio::test] +async fn descending_order_is_rejected_without_a_request() { + let backend = MockBackend::new(vec![run("SUCCEEDED")]); + let client = client_with(backend.clone(), 0); + + let error = client + .run("r") + .iterate_dataset_items::(DatasetListItemsOptions { + desc: Some(true), + ..Default::default() + }) + .next() + .await + .expect_err("`desc` is not supported"); + + assert!( + matches!(error, apify_client::ApifyClientError::InvalidArgument(_)), + "{error:?}" + ); + assert_eq!(backend.call_count(), 0); +} From 135a7a17d029729a65c2fcf2e6d2632a8f8cf92e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 07:40:35 +0000 Subject: [PATCH 05/11] fix: address the full review of the v3 port Start and poll `call` without a client-side timeout, do not retry timed-out run and build starts, add log redirection (StreamedLog, LogSink, ActorCallOptions::log), bound polls of a finite wait, add TaskStartOptions, make internals crate-private, and add the missing live tests. --- .github/workflows/rust-integration-tests.yml | 15 +- .github/workflows/rust-publish.yml | 87 +--- CHANGELOG.md | 31 +- Cargo.toml | 4 +- README.md | 36 +- docs/README.md | 4 +- docs/actors.md | 38 +- docs/misc.md | 8 +- docs/runs.md | 48 +- docs/tasks.md | 4 +- examples/create_build_run_actor.rs | 8 +- examples/log_redirection.rs | 57 +- examples/raw_log.rs | 16 +- examples/run_and_last_run_storages.rs | 8 +- examples/run_store_actor.rs | 8 +- src/client.rs | 8 +- src/clients/actor.rs | 165 +++++- src/clients/base.rs | 99 +++- src/clients/mod.rs | 1 + src/clients/request_queue.rs | 5 +- src/clients/run.rs | 77 ++- src/clients/streamed_log.rs | 491 ++++++++++++++++++ src/clients/task.rs | 42 +- src/common.rs | 22 +- src/http_client.rs | 23 +- src/lib.rs | 13 +- src/timeouts.rs | 3 + tests/actor.rs | 43 ++ tests/actor_run.rs | 106 +++- tests/task.rs | 50 ++ tests/unit_call.rs | 517 +++++++++++++++++++ tests/unit_http.rs | 17 +- tests/unit_resources.rs | 8 +- 33 files changed, 1766 insertions(+), 296 deletions(-) create mode 100644 src/clients/streamed_log.rs create mode 100644 tests/unit_call.rs diff --git a/.github/workflows/rust-integration-tests.yml b/.github/workflows/rust-integration-tests.yml index 171840f..6cae9c8 100644 --- a/.github/workflows/rust-integration-tests.yml +++ b/.github/workflows/rust-integration-tests.yml @@ -77,24 +77,13 @@ jobs: env: # The integration-test token is stored as a repository secret. APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} - # Limit parallelism to be gentle on the shared test account. The documentation example - # programs in `tests/examples.rs` (test names prefixed `example_`) are exercised by the - # standalone `Test examples` step below, so they are skipped here to keep the two - # concerns separate. + # `example_*` tests run in the `Test examples` step below. run: cargo test --lib --tests --verbose -- --skip example_ --test-threads=4 - # Standalone CI step that verifies the documentation examples actually work. It runs the - # example programs from `examples/` end-to-end against the live API (via the `example_*` - # smoke tests in `tests/examples.rs`, each of which executes `cargo run --example `) - # and runs the in-documentation code snippets as doctests (`cargo test --doc`, which - # compiles every fenced `rust` block in the README and the `docs/` pages and runs the - # runnable ones). Both are required by the documentation requirements: each documentation - # example has a CI test that actually runs the code. + # Runs the example programs and the documentation doctests, see "Testing" in README.md. - name: Test examples env: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} - # Match the integration-test thread cap so the example programs and any doctests that hit - # the live API stay gentle on the shared account. run: | cargo test --test examples --verbose -- --test-threads=4 cargo test --doc --verbose -- --test-threads=4 diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index d28a801..ab3b420 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -1,19 +1,6 @@ name: Publish Rust client to crates.io -# Language-specific publish workflow for the Rust client. It is triggered manually only -# (workflow_dispatch) so a maintainer deliberately decides when a release is cut. Publishing -# to the crates.io registry is the language-specific distribution standard for Rust. -# -# In addition to publishing the crate, this workflow tags the released commit and creates a -# matching GitHub release. The release tag is derived from the single source of truth (the -# `version` field in Cargo.toml, which is also what `CLIENT_VERSION` reads via CARGO_PKG_VERSION) -# so the published crate version, the git tag, and the in-code client version can never disagree. -# -# Authentication to crates.io uses Trusted Publishing (OIDC) via `rust-lang/crates-io-auth-action`, -# the registry's recommended mechanism — no long-lived crates.io API token is stored as a secret. -# A Trusted Publisher must be configured for the crate on crates.io (this repository + this -# workflow file). The only repository secret still used is the built-in GITHUB_TOKEN, for pushing -# the tag and creating the GitHub release. +# Manual release from master; see "Releasing" in README.md. on: workflow_dispatch: inputs: @@ -22,16 +9,12 @@ on: type: boolean default: false -# Never allow two publish runs to race; a half-finished publish to a registry or a duplicate -# release tag is hard to undo. +# Two publish runs must never race. concurrency: group: rust-publish cancel-in-progress: false -# The default GITHUB_TOKEN needs write access to push the release tag and create the GitHub -# release. `id-token: write` lets the job mint a short-lived OpenID Connect token that -# `rust-lang/crates-io-auth-action` exchanges for an ephemeral crates.io publish token -# (Trusted Publishing) — so no long-lived crates.io registry secret is stored in the repo. +# `contents: write` for the tag and release, `id-token: write` for crates.io Trusted Publishing. permissions: contents: write id-token: write @@ -43,12 +26,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - # Full history so the tag points at the real commit and tag lookups work. + # Full history for the tag lookups. fetch-depth: 0 - # A published crate version and its release tag are immutable, so a release must only ever - # be cut from master. Refuse to run from any other ref, even when dispatched manually, so a - # maintainer cannot accidentally publish an unmerged feature-branch commit as a version. + # A released version is immutable, so only master may be released. - name: Require master branch run: | if [ "${GITHUB_REF}" != "refs/heads/master" ]; then @@ -70,7 +51,7 @@ jobs: target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} - # Gate the release on the same quality bar as CI so a broken build can never be published. + # Same quality gate as CI. - name: Check formatting run: cargo fmt --all -- --check @@ -80,10 +61,7 @@ jobs: - name: Build run: cargo build --verbose - # Derive the release tag from the single source of truth (Cargo.toml `version`, which is also - # what CLIENT_VERSION exposes via CARGO_PKG_VERSION). Keeping the tag in lock step with the - # crate version means consumers who read the constant and consumers who pull a tag always see - # the same version. + # The tag comes from the `version` in Cargo.toml, which `CLIENT_VERSION` reads as well. - name: Resolve version from Cargo.toml id: version run: | @@ -101,8 +79,7 @@ jobs: echo "tag=v${version}" >> "$GITHUB_OUTPUT" echo "Resolved release tag: v${version}" - # Fail early (before any publish or tag creation) if this version was already released, so a - # publish run can never silently no-op or clobber an existing release. + # Fail before publishing when this version was already released. - name: Ensure tag does not already exist env: TAG: ${{ steps.version.outputs.tag }} @@ -116,31 +93,13 @@ jobs: exit 1 fi - # Always verify packaging works (this also runs as part of `cargo publish`, but doing it - # explicitly surfaces packaging problems before any registry interaction or tag creation). - # No registry credentials are needed for `--dry-run`: it only packages and verifies the - # crate locally without contacting the registry for authentication. + # Surfaces packaging problems before any tag or registry interaction. - name: Verify package (dry run) run: cargo publish --dry-run --verbose - # Ordering rationale (deliberate): the git tag and GitHub release are created and pushed - # BEFORE `cargo publish`. Unlike Go — where pushing the tag *is* the publish, so there is one - # atomic step — Rust has two irreversible-ish actions (push a tag, publish to crates.io) that - # cannot be made a single atomic transaction. We tag first because: - # * The git tag is the canonical, immutable "this commit is version X" marker; deriving it - # and the GitHub release from the source-of-truth version up front guarantees they always - # agree with each other and with the crate version. - # * A failure AFTER tagging but BEFORE publish is cleanly recoverable: delete the just-pushed - # tag and release, then re-run. The "Ensure tag does not already exist" guard makes that - # re-run safe and refuses to clobber. - # * A failure AFTER publish (the genuinely unrecoverable step, since a crates.io version can - # never be re-published) then leaves the tag and release already in place and consistent, - # rather than a published crate with no tag/release. Recovery is just re-running the two - # git/release steps, never the publish. - # If `cargo publish` fails here, recover by deleting the tag/release and bumping the version, - # or by manually completing the publish for the existing tag. - - # Tag the released commit so the crate version, git tag, and GitHub release all line up. + # The tag and release come before `cargo publish`, see "Releasing" in README.md. + + # Tag the released commit. - name: Create and push release tag if: ${{ github.event.inputs.dry_run != 'true' }} env: @@ -151,16 +110,13 @@ jobs: git tag -a "${TAG}" -m "Release ${TAG}" git push origin "${TAG}" - # Extract the matching CHANGELOG section so the GitHub release page shows the actual changes - # for this version rather than a static pointer. Falls back to a one-liner if the section is - # missing, so a forgotten CHANGELOG entry never blocks the release. + # The release notes are the CHANGELOG section of this version. - name: Build release notes from CHANGELOG if: ${{ github.event.inputs.dry_run != 'true' }} env: VERSION: ${{ steps.version.outputs.version }} run: | - # Match the heading as a literal string (index) so the version's dots and brackets are - # not interpreted as a regex/character class. + # `index` matches the heading literally, not as a regex. notes=$(awk -v ver="## [${VERSION}]" ' index($0, ver)==1 {capture=1; next} capture && /^## / {exit} @@ -175,12 +131,7 @@ jobs: echo "__EOF__" } >> "$GITHUB_ENV" - # Create a GitHub release for the tag. Uses the default GITHUB_TOKEN from repository secrets; - # no personal access token is needed. - # Idempotent: create the release, or update it if one already exists for the tag. This keeps a - # re-run safe in the post-tag/pre-publish recovery path (see the ordering rationale above): if - # a maintainer re-pushes the tag after a partial failure, the release step will not hard-fail - # on an already-existing release. + # Creates the release, or updates it when a re-run finds one. - name: Create GitHub release if: ${{ github.event.inputs.dry_run != 'true' }} env: @@ -197,20 +148,14 @@ jobs: --notes "${RELEASE_NOTES}" fi - # Trusted Publishing: exchange the GitHub Actions OIDC identity for a short-lived crates.io - # token instead of storing a long-lived registry secret. The action sets a `token` output - # and automatically revokes it in its post step when the job finishes. This requires a - # matching Trusted Publisher to be configured for the crate on crates.io (organization, - # repository and workflow file name). + # Trusted Publishing: a short-lived crates.io token from the OIDC identity. - name: Authenticate to crates.io (Trusted Publishing) id: auth if: ${{ github.event.inputs.dry_run != 'true' }} uses: rust-lang/crates-io-auth-action@v1 - name: Publish to crates.io - # Skip the actual publish when the run was dispatched as a dry run. if: ${{ github.event.inputs.dry_run != 'true' }} env: - # Short-lived token minted by Trusted Publishing (no stored registry secret). CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} run: cargo publish --verbose diff --git a/CHANGELOG.md b/CHANGELOG.md index f0f7c51..a29d5f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ Ports the `apify-client-js` v3 changes. `records_public_url`, `KeyValueStoreKey::record_public_url`, `RequestQueue::console_url`, `ActorStoreListItem::url` and `user_picture_url`, `User::profile`, `WebhookDispatch::webhook`. +- Log redirection like the reference client's `StreamedLog`: `StreamedLog` with `start` and `stop`, + `StreamedLogOptions`, and the `LogSink` trait (`StdoutLogSink`, or any `Fn(&str)` closure) behind + `LogRedirect`. `ActorClient::call` and `call_raw` redirect the log of the run to the standard + output by default, with the `ActorCallOptions::log` option to change or disable it. +- `ActorCallOptions` and `TaskStartOptions`. + ### Changed - Synced with Apify OpenAPI spec `v2-2026-10-29T100357Z` (`API_SPEC_VERSION`). - **Breaking:** every method has a timeout tier (5s, 30s or 360s by default) instead of one 360s @@ -39,8 +45,27 @@ Ports the `apify-client-js` v3 changes. an error. - **Breaking:** `get_statistics`, `get_input`, `schedule.get_log` and `get_openapi_definition` return the `404` as an error and no longer wrap the result in `Option`. -- **Breaking:** `LogClient::stream`, `stream_with_options`, `RunClient::get_streamed_log` and - `get_streamed_log_with_options` return an `Option`: `None` for a missing log or run. +- **Breaking:** `LogClient::stream` and `stream_with_options` return an `Option`: `None` for a + missing log. +- **Breaking:** `RunClient::get_streamed_log` takes `StreamedLogOptions` and returns a + `StreamedLog` (`None` for a missing run), and `get_streamed_log_with_options` is removed. For the + raw chunks of the log, use `run.log().stream_with_options(..)`. +- **Breaking:** `ActorClient::call` and `call_raw` take `ActorCallOptions` (the start options, the + wait budget `wait_secs` and the `log` option) instead of `ActorStartOptions` and `wait_secs`. + `TaskClient::start` and `call` take `TaskStartOptions`, which has no `content_type` and no + `force_permission_level`, because `POST /actor-tasks/{id}/runs` does not document them. +- **Breaking:** `call` starts the run, and `TaskClient::call` too, without a client-side timeout + unless the client has `with_timeout`, which then applies to the polls too. A start that outlives + a tier is no longer aborted and retried into a second run. `start`, `start_raw`, `build` and + `TaskClient::start` do not retry a transport timeout either. +- **Breaking:** the internals `QueryParams`, `common::{encode_path_segment, build_user_agent, + to_safe_id, create_hmac_signature, sign_storage_content}`, `http_client::HttpClient` and + `http_client::RetryConfig` are no longer public. +- `wait_for_finish(Some(n))` gives each poll the `Medium` tier plus the hold it asks for, so a + stalled connection cannot hang it. +- `WaitForResources::ForSecs` with an infinite or huge value waits without a limit, and `NaN` + fails with `InvalidArgument`, instead of panicking. +- `RequestQueueClient::get_request` no longer sends `clientKey`. - **Breaking:** `ActorStartOptions::timeout_secs` and `RunResurrectOptions::timeout_secs` are now `run_timeout_secs`. - **Breaking:** `LogOptions`, `RunListOptions` and `StoreListOptions` gained a public field. @@ -57,8 +82,6 @@ Ports the `apify-client-js` v3 changes. - A zero `timeout_*` option or `with_timeout` fails the requests with `InvalidArgument`. - `UserClient::limits`, `monthly_usage` and `update_limits` work for any user, not only `me`. - `batch_add_requests` serializes each request once and rejects an oversized one before sending. -- Key-value store `update`, `record_exists` and the public URL methods, and - `DatasetClient::create_items_public_url`, use the `Long` tier. ## [0.14.1] - 2026-10-26 diff --git a/Cargo.toml b/Cargo.toml index fd97bd7..ba9a545 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "apify-client" version = "0.15.0" authors = ["Apify Technologies "] -description = "An official, but experimental, AI-generated and AI-maintained Rust client for the Apify API (https://apify.com)." +description = "Rust client for the Apify API (https://apify.com)." license = "Apache-2.0" edition = "2021" repository = "https://github.com/apify/apify-client-rust" @@ -20,7 +20,7 @@ chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0" async-trait = "0.1" futures-util = "0.3" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1", features = ["time", "rt", "sync", "macros"] } url = "2" base64 = "0.22" hmac = "0.12" diff --git a/README.md b/README.md index 816773a..2312e4e 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,7 @@ It provides a resource-oriented, async interface that mirrors the official - Transparent authentication, retries with exponential backoff, and timeouts. - Resource clients for Actors, runs, builds, tasks, datasets, key-value stores, request queues, schedules, webhooks, the Apify Store, users and logs. -- Convenience helpers: run/wait, log streaming (redirection; needs the `futures-util` crate — - see [Installation](#installation)), lazy Store iteration, and `set_status_message` for +- Convenience helpers: run/wait, log redirection, lazy Store iteration, and `set_status_message` for updating the current Actor run's status (see [`docs/README.md`](docs/README.md#convenience-methods)). - A replaceable HTTP transport for testing or custom runtimes. @@ -30,12 +29,11 @@ serde_json = "1" # for the `serde_json::Value` responses used in the Qu The Quick start below reads dynamically-typed records with `serde_json::Value`, so a fresh project needs `serde_json`. Two more dependencies are needed only for specific features: -- `futures-util = "0.3"` — to consume any of the log streams (`LogClient::stream()` / - `stream_with_options()` and the run shortcuts `RunClient::get_streamed_log()` / - `get_streamed_log_with_options()`); it provides the `StreamExt` trait used to poll the returned - stream. Both the [`log_redirection`](examples/log_redirection.rs) and - [`raw_log`](examples/raw_log.rs) examples import `futures_util::StreamExt` for this. See - [`docs/misc.md`](docs/misc.md#logs--clientlogbuild_or_run_id). +- `futures-util = "0.3"` — to consume the raw log streams (`LogClient::stream()` / + `stream_with_options()`); it provides the `StreamExt` trait used to poll the returned stream. + The [`raw_log`](examples/raw_log.rs) example imports `futures_util::StreamExt` for this. See + [`docs/misc.md`](docs/misc.md#logs--clientlogbuild_or_run_id). Log redirection + (`ActorClient::call`, `RunClient::get_streamed_log`) needs no extra dependency. - `chrono = "0.4"` — only if you construct or read timestamp values yourself. Model timestamp fields (e.g. `Actor::created_at`, `ActorRun::started_at`) are typed as `Option>` and `chrono` is **not** re-exported, so snippets that call `chrono::Utc::now()` (e.g. the @@ -69,7 +67,7 @@ async fn main() -> Result<(), Box> { // Start an Actor and wait for it to finish. let run = client .actor("apify/hello-world") - .call::(None, Default::default(), None) + .call::(None, Default::default()) .await?; println!("Run finished with status {:?}", run.status); @@ -215,6 +213,10 @@ Every request carries a `User-Agent` of the form | `client.me()` / `client.user(id)` | single | User accounts | | `client.log(build_or_run_id)` | single | Build/run logs | +`client.api_base_url()` returns the API base URL the client targets, including `/v2` and any +`base_url` override. The iterators returned by `iterate(..)` (and by `RunClient::iterate_dataset_items`) +yield one item per `next().await?`; `collect_all().await?` drains the remaining items into a `Vec`. + ## Versioning ```rust @@ -281,6 +283,15 @@ step, replacing the `git push origin :refs/tags/...` command above rather than a If `cargo publish` itself already succeeded, that version is permanent on crates.io; bump the `version` in `Cargo.toml` for the next release instead of re-running. +## Testing + +The integration tests in `tests/` run against the live API and need `APIFY_TOKEN`; the `unit_*` +tests run offline against a scripted HTTP backend. The +[`Rust integration tests`](.github/workflows/rust-integration-tests.yml) workflow runs them on pull +requests, with a limited thread count to be gentle on the shared test account. Its `Test examples` +step runs the programs in `examples/` through the `example_*` tests in `tests/examples.rs`, and the +code snippets in this README and `docs/` as doctests (`cargo test --doc`). + ## Examples Runnable examples live in [`examples/`](examples). Each requires `APIFY_TOKEN` and can be @@ -292,7 +303,7 @@ run with `cargo run --example `: - `create_build_run_actor` — create an Actor, build it, run it, fetch the run log. - `run_and_last_run_storages` — run an Actor, then access the last run and its storages. - `iterate_store` — lazily iterate Actors in the Apify Store. -- `log_redirection` — run a separate Actor and redirect its run log into your output live, with each line prefixed by the source Actor's name. +- `log_redirection` — run a separate Actor and redirect its run log into your output live, with each message prefixed by the source Actor's name, and to a custom sink. - `raw_log` — fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true), ..Default::default() }`. See [`docs/`](docs) for the full API documentation. @@ -356,8 +367,9 @@ returns the `404` as an error: the response cannot tell a missing run from a mis same goes for methods that address a fixed sub-path of a resource (`get_statistics`, `get_input`, `get_log`, `get_openapi_definition`, `limits`, `monthly_usage`): they return the `404` as an error. -`ApiError` also exposes `is_not_found()`, `is_rate_limited()`, `is_server_error()` and similar -`is_*` predicates classifying it by HTTP status — the idiomatic equivalent of the reference +`ApiError` also exposes the `is_invalid_request()` (400), `is_unauthorized()` (401), +`is_forbidden()` (403), `is_not_found()` (404), `is_conflict()` (409), `is_rate_limited()` (429) and +`is_server_error()` (5xx) predicates classifying it by HTTP status — the idiomatic equivalent of the reference clients' `NotFoundError`/`RateLimitError`/... error subclasses, since Rust has no exception hierarchy to mirror them with. diff --git a/docs/README.md b/docs/README.md index fa4b312..8f21e1a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,13 +71,13 @@ root, so you can import them directly from `apify_client` — you never need the `apify_client::clients::::` path. The complete set of re-exported option/parameter types is: -- Actors: `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions` +- Actors: `ActorStartOptions`, `ActorCallOptions`, `TaskStartOptions`, `ActorBuildOptions`, `ActorListOptions` - Runs: `RunListOptions`, `RunResurrectOptions`, `RunMetamorphOptions`, `RunChargeOptions`, `LastRunOptions` - Datasets: `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat` - Key-value stores: `ListKeysOptions`, `GetRecordOptions` - Request queues: `ListRequestsOptions` - Store: `StoreListOptions` -- Logs: `LogOptions` +- Logs: `LogOptions`, `LogRedirect`, `LogSink`, `StreamedLog`, `StreamedLogOptions` - Shared: `ListOptions`, `StorageListOptions` - Client configuration: `RequestCompression` diff --git a/docs/actors.md b/docs/actors.md index 8e5578a..f301dd6 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -20,8 +20,8 @@ be an Actor ID or a `username~name` (or `username/name`) reference. | `delete()` | — | `()` | Deletes the Actor. | | `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run, returns immediately. | | `start_raw(input, options)` | `&[u8]`, `ActorStartOptions` | `ActorRun` | Like `start`, but sends `input` as a raw request body (no JSON serialization) — for a non-JSON input such as a ZIP archive, paired with `options.content_type`. | -| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits for it to finish. | -| `call_raw(input, options, wait_secs)` | `&[u8]`, `ActorStartOptions`, `Option` | `ActorRun` | `start_raw` followed by `wait_for_finish`. | +| `call(input, options)` | `Option<&impl Serialize>`, `ActorCallOptions` | `ActorRun` | Starts a run and waits for it to finish, redirecting its log to the standard output unless `options.log` says otherwise. | +| `call_raw(input, options)` | `&[u8]`, `ActorCallOptions` | `ActorRun` | `start_raw` followed by `wait_for_finish`, like `call`. | | `build(version, options)` | `&str`, `ActorBuildOptions` | `Build` | Builds a version of the Actor. | | `default_build(wait_for_finish)` | `Option` | `BuildClient` | Resolves the Actor's default build, optionally waiting up to `wait_for_finish` seconds. | | `validate_input(input)` | `&impl Serialize` | `serde_json::Value` | Validates input against the default build's schema. | @@ -35,9 +35,11 @@ be an Actor ID or a `username~name` (or `username/name`) reference. ### `ActorStartOptions` -All fields are optional. Used by both `start` and `call` here, and by the identical `start` / -`call` methods on [tasks](tasks.md) (for `call`, `wait_for_finish` is server-side; the -`wait_secs` argument controls client-side polling). +All fields are optional. Used by `start` and `start_raw`, and as `ActorCallOptions::start` by +`call` and `call_raw` (there, `wait_for_finish` is server-side; `ActorCallOptions::wait_secs` +controls client-side polling). [Tasks](tasks.md) take `TaskStartOptions`, which has the same +fields except `content_type` and `force_permission_level`, because `POST /actor-tasks/{id}/runs` +documents neither. | Field | Type | Description | |---|---|---| @@ -53,7 +55,20 @@ All fields are optional. Used by both `start` and `call` here, and by the identi | `webhooks` | `Option>` | Ad-hoc webhooks to attach to this run. Encoded as base64 JSON in the `webhooks` query parameter, matching the reference clients. | | `wait_for_resources` | `Option` | Retry the start while the account lacks the memory or a concurrent-run slot for it (`ApiError::error_type` of `actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`). See below. | -The `wait_secs` argument of `call` (and of `wait_for_finish` on runs/builds) controls the +### `ActorCallOptions` + +| Field | Type | Description | +|---|---|---| +| `start` | `ActorStartOptions` | The options of the run start. | +| `wait_secs` | `Option` | The client-side polling budget, see below. | +| `log` | `LogRedirect` | Where to redirect the run's log while waiting: `LogRedirect::Default` (the default) prints it to the standard output prefixed with `"{Actor name} runId:{run id} -> "`, `LogRedirect::Disabled` turns it off, and `LogRedirect::to_sink(..)` sends it to a [`LogSink`](runs.md#log-redirection) of your own. | + +`call` starts the run without a client-side timeout, because a start that is aborted and retried +could create a second run. A timeout set with the client's `with_timeout` replaces that for the +start and applies to every poll as well. A start that times out is never retried, for `start`, +`start_raw` and `build` too. + +`wait_secs` (and the `wait_secs` argument of `wait_for_finish` on runs/builds) controls the client-side polling budget: - `None` polls indefinitely until the run reaches a terminal state. @@ -75,6 +90,7 @@ Set `ActorStartOptions::wait_for_resources` to retry the start every 10 seconds - `WaitForResources::Forever` retries until the run starts. - `WaitForResources::ForSecs(secs)` stops retrying after that many seconds and returns the last + error. An infinite or huge value is the same as `Forever`, and `NaN` is an `InvalidArgument` error. Any other error is returned right away. A run that asks for more memory than the account's whole @@ -82,7 +98,7 @@ memory limit is rejected with `actor-memory-limit-exceeded` too and never starts retries it forever. In `call`, time spent retrying the start does not count toward `wait_secs`. ```rust,no_run -use apify_client::{ApifyClient, ActorStartOptions, WaitForResources}; +use apify_client::{ActorCallOptions, ActorStartOptions, ApifyClient, WaitForResources}; # async fn run() -> Result<(), Box> { let client = ApifyClient::new("my-api-token"); @@ -92,11 +108,13 @@ let run = client .actor("username/actor-name") .call::( None, - ActorStartOptions { - wait_for_resources: Some(WaitForResources::Forever), + ActorCallOptions { + start: ActorStartOptions { + wait_for_resources: Some(WaitForResources::Forever), + ..Default::default() + }, ..Default::default() }, - None, ) .await?; # Ok(()) diff --git a/docs/misc.md b/docs/misc.md index 8daf29c..3f282c1 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -151,7 +151,7 @@ Also reachable via `run.log()` and `build.log()`. `get*` and `stream*` return `None` for a missing log only on a client that names its log by id (`client.log(id)`). A client chained from a run or build (`run.log()`, `build.log()`) has no id of its own, so it returns the `404` as an error: the response cannot tell a missing run from a missing -log. `run.get_streamed_log()` returns `None` for a missing run, because the run is the resource +log. `run.get_streamed_log(..)` returns `None` for a missing run, because the run is the resource that client names. `LogOptions` has two fields, `raw` and `download`, both `Option`. When `raw` is `Some(true)`, @@ -201,7 +201,8 @@ use futures_util::StreamExt; # async fn run(client: ApifyClient, run_id: &str) -> Result<(), Box> { let mut stream = client .run(run_id) - .get_streamed_log() + .log() + .stream() .await? .ok_or("the run does not exist")?; while let Some(chunk) = stream.next().await { @@ -211,3 +212,6 @@ while let Some(chunk) = stream.next().await { # Ok(()) # } ``` + +[`RunClient::get_streamed_log`](runs.md#log-redirection) does the line splitting and prefixing for +you. diff --git a/docs/runs.md b/docs/runs.md index 6d7e03c..ceef725 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -26,15 +26,45 @@ collections are available via `actor.runs()` and `task.runs()`. | `charge(options)` | `RunChargeOptions` | `()` | Charges a pay-per-event run (always sends an idempotency key). | | `wait_for_finish(wait_secs)` | `Option` | `ActorRun` | Polls until the run is terminal. `None` waits indefinitely; `Some(n)` bounds the wait and may return a still-running (non-terminal) run if `n` elapses first. A run ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | | `dataset()` / `key_value_store()` / `request_queue()` / `log()` | — | resource client | Access the run's default storages and log. | -| `get_streamed_log()` | — | `Result>>>>` (async — `.await` it) | Streams the run's log chunks live (log redirection); `None` if the run does not exist. | -| `get_streamed_log_with_options(options)` | `LogOptions` | `Result>>>>` (async — `.await` it) | As `get_streamed_log()`, forwarding `LogOptions` (e.g. `raw`) to the log stream. | +| `get_streamed_log(options)` | `StreamedLogOptions { to_log, from_start }` | `Result>` (async — `.await` it) | The [redirection](#log-redirection) of the run's log; `None` if the run does not exist or `to_log` is `LogRedirect::Disabled`. | -`get_streamed_log()` / `get_streamed_log_with_options()` are `async` and yield the stream inside a -`Result` and an `Option`, so `.await?` them and handle the `None` of a missing run to obtain it. Polling the returned stream with `.next()` requires the -`futures_util::StreamExt` trait (from the `futures-util` crate — add `futures-util = "0.3"` to your -`Cargo.toml`) in scope; the [`raw_log`](../examples/raw_log.rs) example drives -`get_streamed_log_with_options` exactly this way. See -[Logs](misc.md#logs--clientlogbuild_or_run_id) for a full streaming snippet. +### Log redirection + +`get_streamed_log` returns a `StreamedLog`: `start()` spawns a task that follows the run's log, and +`stop()` ends it (waiting up to a second for the lines in flight, then reading the rest of the +log). The log is split into messages at the timestamps its lines begin with, so a stack trace +stays in one piece, and each message goes to a `LogSink`: + +- `LogRedirect::Default` prints to the standard output, prefixing each message with + `"{Actor name} runId:{run id} -> "`. It costs a request for the run and one for its Actor. +- `LogRedirect::to_sink(sink)` sends the messages, without a prefix, to a `LogSink`: a trait with + one method, implemented by every `Fn(&str) + Send + Sync + 'static` closure, so it can forward to + a logging framework. +- `LogRedirect::Disabled` redirects nothing. + +`from_start: false` drops the messages stamped before the call. A run that has not logged yet is +followed by reopening its empty log stream until the first bytes arrive. `ActorClient::call` does +all of this for the run it waits for. To read the log as raw chunks instead, use +`run.log().stream_with_options(..)`; the [`raw_log`](../examples/raw_log.rs) example does. + +```rust,no_run +use apify_client::{ApifyClient, LogRedirect, StreamedLogOptions}; + +# async fn run(client: ApifyClient, run_id: &str) -> Result<(), Box> { +let run = client.run(run_id); +let mut streamed_log = run + .get_streamed_log(StreamedLogOptions { + to_log: LogRedirect::to_sink(|message: &str| println!("run -> {message}")), + from_start: true, + }) + .await? + .ok_or("the run does not exist")?; +streamed_log.start()?; +run.wait_for_finish(None).await?; +streamed_log.stop().await?; +# Ok(()) +# } +``` `RunResurrectOptions`: `build`, `memory_mbytes`, `run_timeout_secs`, `max_items`, `max_total_charge_usd`, `restart_on_error` (all optional). @@ -97,7 +127,7 @@ use apify_client::ApifyClient; let client = ApifyClient::new(std::env::var("APIFY_TOKEN")?); let run = client .actor("apify/hello-world") - .call::(None, Default::default(), None) + .call::(None, Default::default()) .await?; println!("run {} finished with status {:?}", run.id, run.status); diff --git a/docs/tasks.md b/docs/tasks.md index 99758e6..15f78a2 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -19,8 +19,8 @@ Obtained via `client.tasks()` (collection) and `client.task(id)` (single). | `publish()` | — | `Task` | Publishes the task on its public landing page (sets `isPublic: true`). Requires the task's Actor to be public and the task to have `publicConfig` set up. An Actor may have at most 10 published tasks, an account at most 100 (contact Apify support to raise either limit). | | `unpublish()` | — | `Task` | Unpublishes the task from its public landing page (sets `isPublic: false`), preserving `publicConfig`. | | `delete()` | — | `()` | Deletes the task. | -| `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run. See [`ActorStartOptions`](actors.md#actorstartoptions) for the full field list. | -| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits. Same [`ActorStartOptions`](actors.md#actorstartoptions) as `start`. | +| `start(input, options)` | `Option<&impl Serialize>`, `TaskStartOptions` | `ActorRun` | Starts a run. `TaskStartOptions` has the fields of [`ActorStartOptions`](actors.md#actorstartoptions) except `content_type` and `force_permission_level`, which the endpoint does not document. | +| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `TaskStartOptions`, `Option` | `ActorRun` | Starts a run and waits. Like [`ActorClient::call`](actors.md#actorcalloptions), it starts without a client-side timeout, but it does not redirect the log. | | `get_input()` / `update_input(input)` | — / `&impl Serialize` | `Value` / `Value` | The task's saved input. `get_input` returns a `404` as an error. | | `last_run(status)` | `Option<&str>` | `RunClient` | The task's last run, optionally filtered by status. See [Actor runs](runs.md) for the accepted `status` values. | | `last_run_with_options(options)` | `LastRunOptions { status, origin }` | `RunClient` | The task's last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | diff --git a/examples/create_build_run_actor.rs b/examples/create_build_run_actor.rs index 27b00b6..e89efe4 100644 --- a/examples/create_build_run_actor.rs +++ b/examples/create_build_run_actor.rs @@ -44,7 +44,13 @@ async fn main() -> Result<(), Box> { // Run the Actor and wait for it to finish. let run = actor_client - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await?; println!("Run {} status: {:?}", run.id, run.status); diff --git a/examples/log_redirection.rs b/examples/log_redirection.rs index 2cac0b9..3a02e09 100644 --- a/examples/log_redirection.rs +++ b/examples/log_redirection.rs @@ -2,17 +2,15 @@ //! //! "Log redirection" is the convenience of taking a *separate* Actor run's log and piping it, //! live, into the calling program's own output as it is produced. This mirrors the JS reference -//! client's `StreamedLog` / `LoggerActorRedirect`, which redirects a run's streamed log into -//! another `Log` sink and prefixes every line with `"{name} -> "` so the redirected output is -//! attributable to its source Actor. +//! client's `StreamedLog`, which prefixes every message with `"{Actor name} runId:{run id} -> "` +//! so the redirected output is attributable to its source Actor. //! -//! Here we start the `apify/hello-world` Actor (a different Actor from this program), then -//! redirect its run log into our stdout in real time, prefixing each line with the Actor's name. +//! The first part lets `ActorClient::call` do it, which redirects the log to the standard output +//! by default. The second part starts the run itself and redirects its log to a custom sink. //! //! Run with: `APIFY_TOKEN=... cargo run --example log_redirection` -use apify_client::ApifyClient; -use futures_util::StreamExt; +use apify_client::{ActorCallOptions, ApifyClient, LogRedirect, StreamedLogOptions}; #[tokio::main] async fn main() -> Result<(), Box> { @@ -22,38 +20,31 @@ async fn main() -> Result<(), Box> { // The (separate) Actor whose log we will redirect into this program's output. let source_actor = "apify/hello-world"; - // Start the Actor without waiting, so its log can be redirected while the run is in progress. + // `call` redirects the log of the run to the standard output while it waits. let run = client .actor(source_actor) - .start::(None, Default::default()) + .call::(None, ActorCallOptions::default()) .await?; - println!( - "Started run {} of {source_actor} — redirecting its log:", - run.id - ); + println!("--- run {} finished ---", run.id); - // Redirect the run's log into our stdout, line by line, with a source prefix (the same - // `"{name} -> "` convention the JS reference uses for redirected Actor logs). The log is - // streamed as raw byte chunks, so we buffer until a newline to emit whole, prefixed lines. - let prefix = format!("{source_actor} -> "); - let mut stream = client - .run(&run.id) - .get_streamed_log() + // Start the run without waiting, and redirect its log to a sink of our own. + let run = client + .actor(source_actor) + .start::(None, Default::default()) + .await?; + let run_client = client.run(&run.id); + let mut streamed_log = run_client + .get_streamed_log(StreamedLogOptions { + to_log: LogRedirect::to_sink(move |message: &str| { + println!("[{source_actor}] {message}") + }), + from_start: true, + }) .await? .ok_or("the run does not exist")?; - let mut buf = String::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - buf.push_str(&String::from_utf8_lossy(&chunk)); - while let Some(newline) = buf.find('\n') { - let line: String = buf.drain(..=newline).collect(); - print!("{prefix}{line}"); - } - } - // Flush any trailing partial line that was not newline-terminated. - if !buf.is_empty() { - println!("{prefix}{buf}"); - } + streamed_log.start()?; + run_client.wait_for_finish(Some(120)).await?; + streamed_log.stop().await?; println!("--- log redirection ended (run finished) ---"); Ok(()) diff --git a/examples/raw_log.rs b/examples/raw_log.rs index 83b6cc8..ffb7623 100644 --- a/examples/raw_log.rs +++ b/examples/raw_log.rs @@ -3,7 +3,7 @@ //! By default the API adds a per-line timestamp prefix to log lines. Passing //! `LogOptions { raw: Some(true), ..Default::default() }` requests the unprocessed log instead. //! The example fetches it as a whole with `get_with_options`, and streams it with -//! `get_streamed_log_with_options`. +//! the log client's `stream_with_options`. //! //! Run with: `APIFY_TOKEN=... cargo run --example raw_log` @@ -18,7 +18,14 @@ async fn main() -> Result<(), Box> { // Run a short public Actor and wait for it to finish so it has a complete log. let run = client .actor("apify/hello-world") - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + log: apify_client::LogRedirect::Disabled, + ..Default::default() + }, + ) .await?; println!("Run {} finished with status {:?}", run.id, run.status); @@ -38,12 +45,13 @@ async fn main() -> Result<(), Box> { // Streaming raw log: stream the same unprocessed log chunk by chunk. let mut stream = client .run(&run.id) - .get_streamed_log_with_options(LogOptions { + .log() + .stream_with_options(LogOptions { raw: Some(true), ..Default::default() }) .await? - .ok_or("the run does not exist")?; + .ok_or("the log does not exist")?; let mut streamed_bytes = 0usize; while let Some(chunk) = stream.next().await { streamed_bytes += chunk?.len(); diff --git a/examples/run_and_last_run_storages.rs b/examples/run_and_last_run_storages.rs index 5eb55c0..f57f253 100644 --- a/examples/run_and_last_run_storages.rs +++ b/examples/run_and_last_run_storages.rs @@ -13,7 +13,13 @@ async fn main() -> Result<(), Box> { // Start a run and wait for it to finish. let run = client .actor("apify/hello-world") - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await?; println!("Run {} finished: {:?}", run.id, run.status); diff --git a/examples/run_store_actor.rs b/examples/run_store_actor.rs index f0fdcf7..0975fe7 100644 --- a/examples/run_store_actor.rs +++ b/examples/run_store_actor.rs @@ -41,7 +41,13 @@ async fn main() -> Result<(), Box> { // Run the discovered Actor and wait up to 2 minutes for it to finish. let run = client .actor(&actor_id) - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await?; println!("Run {} finished with status {:?}", run.id, run.status); diff --git a/src/client.rs b/src/client.rs index 8e55598..0f2ce85 100644 --- a/src/client.rs +++ b/src/client.rs @@ -496,10 +496,16 @@ fn to_api_base_url(url: &str) -> String { } } +/// What separates the scheme of a URL from its authority. +const SCHEME_SEPARATOR: &str = "://"; + /// Splits a URL into its authority (`scheme://host[:port]`, or the whole string if there is no /// path) and its path (starting with `/`, or empty if there is none). fn split_authority(url: &str) -> (&str, &str) { - let search_from = url.find("://").map(|i| i + 3).unwrap_or(0); + let search_from = url + .find(SCHEME_SEPARATOR) + .map(|i| i + SCHEME_SEPARATOR.len()) + .unwrap_or(0); match url[search_from..].find('/') { Some(rel_idx) => url.split_at(search_from + rel_idx), None => (url, ""), diff --git a/src/clients/actor.rs b/src/clients/actor.rs index ed4d074..b03865e 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -7,12 +7,13 @@ use crate::client::ApifyClient; use crate::clients::actor_version::ActorVersionClient; use crate::clients::actor_version_collection::ActorVersionCollectionClient; use crate::clients::base::{ - delete_resource, get_resource, post_with_body, update_resource, ResourceContext, + delete_resource, get_resource, post_creating, update_resource, ResourceContext, }; use crate::clients::build::BuildClient; use crate::clients::build_collection::BuildCollectionClient; use crate::clients::run::{LastRunOptions, RunClient}; use crate::clients::run_collection::RunCollectionClient; +use crate::clients::streamed_log::{LogRedirect, StreamedLogOptions}; use crate::clients::webhook_collection::WebhookCollectionClient; use crate::common::{parse_data_envelope, QueryParams}; use crate::error::ApifyClientResult; @@ -27,7 +28,9 @@ pub enum WaitForResources { /// Retry until the run starts, how ever long that takes. Forever, /// Retry for up to this many seconds (clamped to `0` if negative), then return the last - /// error. `0` makes a single attempt and returns its error without retrying. + /// error. `0` makes a single attempt and returns its error without retrying. An infinite or + /// too large value is the same as [`Forever`](Self::Forever), and `NaN` is rejected with + /// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument). ForSecs(f64), } @@ -97,6 +100,71 @@ impl ActorStartOptions { } } +/// Options of [`ActorClient::call`] and [`ActorClient::call_raw`]. +#[derive(Debug, Default, Clone)] +pub struct ActorCallOptions { + /// The options of the run start. + pub start: ActorStartOptions, + /// The wait budget, in seconds: + /// - `None` polls indefinitely until the run reaches a terminal state. + /// - `Some(n)` bounds the wait to roughly `n` seconds; if the run has not finished by then, + /// the **last fetched (still non-terminal) run is returned** rather than an error. Check + /// `status` / `is_terminal()` on the result. + /// + /// Time spent retrying under [`ActorStartOptions::wait_for_resources`] does not count toward + /// it, since that budget only starts once the run has actually started. + pub wait_secs: Option, + /// Where to redirect the log of the run while it is waited for. Redirects to the standard + /// output by default; [`LogRedirect::Disabled`] turns that off. + pub log: LogRedirect, +} + +/// Options shared by [`TaskClient::start`](crate::clients::task::TaskClient::start) and +/// [`TaskClient::call`](crate::clients::task::TaskClient::call): the +/// [`ActorStartOptions`] the `POST /actor-tasks/{id}/runs` endpoint accepts. A task takes its +/// input as JSON, and its permission level is that of its Actor, so there is no `content_type` +/// or `force_permission_level`. +#[derive(Debug, Default, Clone)] +pub struct TaskStartOptions { + /// Tag or number of the build to run (e.g. `latest`, `0.1.2`). + pub build: Option, + /// Memory in megabytes allocated for the run. + pub memory_mbytes: Option, + /// Timeout of the run itself in seconds (`0` means no timeout). Not the timeout of the API + /// request, which is set with the client's `with_timeout`. + pub run_timeout_secs: Option, + /// Maximum seconds to wait server-side for the run to finish (max 60). + pub wait_for_finish: Option, + /// Maximum number of dataset items to charge (pay-per-result Actors). + pub max_items: Option, + /// Maximum total cost of the run in USD, across all pricing models. + pub max_total_charge_usd: Option, + /// Whether to restart the run if it fails. + pub restart_on_error: Option, + /// Ad-hoc webhooks to attach to this run, see [`ActorStartOptions::webhooks`]. + pub webhooks: Option>, + /// Retry the start while the account lacks the resources for the run, see + /// [`ActorStartOptions::wait_for_resources`]. + pub wait_for_resources: Option, +} + +impl From for ActorStartOptions { + fn from(options: TaskStartOptions) -> Self { + Self { + build: options.build, + memory_mbytes: options.memory_mbytes, + run_timeout_secs: options.run_timeout_secs, + wait_for_finish: options.wait_for_finish, + max_items: options.max_items, + max_total_charge_usd: options.max_total_charge_usd, + restart_on_error: options.restart_on_error, + webhooks: options.webhooks, + wait_for_resources: options.wait_for_resources, + ..Self::default() + } + } +} + /// Options for building an Actor. #[derive(Debug, Default, Clone)] pub struct ActorBuildOptions { @@ -164,7 +232,7 @@ impl ActorClient { Some(value) => Some(serde_json::to_vec(value)?), None => None, }; - self.start_with_body(body, "application/json", options) + self.start_with_body(body, "application/json", options, Timeout::Medium) .await } @@ -180,18 +248,25 @@ impl ActorClient { input: &[u8], options: ActorStartOptions, ) -> ApifyClientResult { - self.start_with_body(Some(input.to_vec()), "application/octet-stream", options) - .await + self.start_with_body( + Some(input.to_vec()), + "application/octet-stream", + options, + Timeout::Medium, + ) + .await } /// Shared implementation of [`start`](Self::start) and [`start_raw`](Self::start_raw): /// applies the run-start options as query parameters and posts the given body, falling back - /// to `default_content_type` only when the caller did not set `options.content_type`. + /// to `default_content_type` only when the caller did not set `options.content_type`. `tier` + /// is the timeout of the request unless the client was given one with `with_timeout`. async fn start_with_body( &self, body: Option>, default_content_type: &str, options: ActorStartOptions, + tier: Timeout, ) -> ApifyClientResult { let mut params = QueryParams::new(); options.apply(&mut params); @@ -201,9 +276,9 @@ impl ActorClient { .unwrap_or_else(|| default_content_type.to_string()); let timeout = self .ctx - .timeout_with_wait_hold(Timeout::Medium, options.wait_for_finish); + .timeout_with_wait_hold(tier, options.wait_for_finish); crate::clients::base::retry_while_resource_limited(options.wait_for_resources, || { - post_with_body( + post_creating( &self.ctx, Some("runs"), ¶ms, @@ -215,25 +290,27 @@ impl ActorClient { .await } - /// Starts the Actor and waits (client-side polling) for it to finish. + /// Starts the Actor and waits (client-side polling) for it to finish, redirecting the log of + /// the run while it waits. /// - /// `wait_secs` controls the wait budget: - /// - `None` polls indefinitely until the run reaches a terminal state. - /// - `Some(n)` bounds the wait to roughly `n` seconds; if the run has not finished by - /// then, the **last fetched (still non-terminal) run is returned** rather than an - /// error. Check `status` / `is_terminal()` on the result when using `Some`. + /// See [`ActorCallOptions`] for the wait budget and the log redirection. The log goes to the + /// standard output unless `options.log` says otherwise. A redirection that cannot be set up + /// does not fail the call, and neither does one that ends early. /// - /// Time spent retrying under [`ActorStartOptions::wait_for_resources`] does not count - /// toward `wait_secs`, since that budget only starts once the run has actually started. + /// Like the reference client, the start request has no client-side timeout, because an + /// aborted start would be retried and could create a second run; set a timeout with the + /// client's `with_timeout` to change that, and it applies to the polls as well. Without one, + /// a poll is bounded as [`RunClient::wait_for_finish`] describes. pub async fn call( &self, input: Option<&T>, - options: ActorStartOptions, - wait_secs: Option, + options: ActorCallOptions, ) -> ApifyClientResult { - let run = self.start(input, options).await?; - // Use the root client's run client so polling targets the canonical run route. - self.root.run(run.id).wait_for_finish(wait_secs).await + let body = match input { + Some(value) => Some(serde_json::to_vec(value)?), + None => None, + }; + self.call_with_body(body, "application/json", options).await } /// Starts the Actor with a raw request body ([`start_raw`](Self::start_raw)) and waits for @@ -241,11 +318,47 @@ impl ActorClient { pub async fn call_raw( &self, input: &[u8], - options: ActorStartOptions, - wait_secs: Option, + options: ActorCallOptions, ) -> ApifyClientResult { - let run = self.start_raw(input, options).await?; - self.root.run(run.id).wait_for_finish(wait_secs).await + self.call_with_body(Some(input.to_vec()), "application/octet-stream", options) + .await + } + + async fn call_with_body( + &self, + body: Option>, + default_content_type: &str, + options: ActorCallOptions, + ) -> ApifyClientResult { + let ActorCallOptions { + start, + wait_secs, + log, + } = options; + let run = self + .start_with_body(body, default_content_type, start, Timeout::NoTimeout) + .await?; + // Use the root client's run client so polling targets the canonical run route. + let mut run_client = self.root.run(run.id); + if let Some(timeout) = self.ctx.timeout { + run_client = run_client.with_timeout(timeout); + } + let mut streamed_log = run_client + .get_streamed_log(StreamedLogOptions { + to_log: log, + from_start: true, + }) + .await + .ok() + .flatten(); + if let Some(streamed_log) = streamed_log.as_mut() { + let _ = streamed_log.start(); + } + let finished = run_client.wait_for_finish(wait_secs).await; + if let Some(streamed_log) = streamed_log.as_mut() { + let _ = streamed_log.stop().await; + } + finished } /// Builds the given version of the Actor and returns the created build. @@ -264,7 +377,7 @@ impl ActorClient { let timeout = self .ctx .timeout_with_wait_hold(Timeout::Medium, options.wait_for_finish); - post_with_body( + post_creating( &self.ctx, Some("builds"), ¶ms, diff --git a/src/clients/base.rs b/src/clients/base.rs index 8002c80..98005bf 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -448,7 +448,43 @@ pub(crate) async fn post_action( content_type: Option<&str>, timeout: Duration, ) -> ApifyClientResult { - let response = post_send(ctx, sub_path, params, body, content_type, timeout).await?; + let response = post_send( + ctx, + sub_path, + params, + body, + content_type, + timeout, + CallOptions::default(), + ) + .await?; + parse_data_envelope(&response.body) +} + +/// Like [`post_with_body`], for a `POST` that creates something (a run or a build). A request +/// that timed out may have reached the API, so it is not retried: a retry could create a second +/// paid resource. +pub(crate) async fn post_creating( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Option>, + content_type: &str, + timeout: Duration, +) -> ApifyClientResult { + let options = CallOptions { + do_not_retry_timeouts: true, + }; + let response = post_send( + ctx, + sub_path, + params, + body, + Some(content_type), + timeout, + options, + ) + .await?; parse_data_envelope(&response.body) } @@ -464,7 +500,16 @@ pub(crate) async fn post_action_raw( content_type: Option<&str>, timeout: Duration, ) -> ApifyClientResult { - let response = post_send(ctx, sub_path, params, body, content_type, timeout).await?; + let response = post_send( + ctx, + sub_path, + params, + body, + content_type, + timeout, + CallOptions::default(), + ) + .await?; Ok(serde_json::from_slice(&response.body)?) } @@ -477,13 +522,14 @@ async fn post_send( body: Option>, content_type: Option<&str>, timeout: Duration, + options: CallOptions, ) -> ApifyClientResult { let url = ctx.url_with_params(sub_path, params); let mut headers = HashMap::new(); if let Some(ct) = content_type { headers.insert("Content-Type".to_string(), ct.to_string()); } - ctx.send_with(HttpMethod::Post, url, headers, body, timeout) + ctx.send_with_options(HttpMethod::Post, url, headers, body, timeout, options) .await } @@ -536,7 +582,7 @@ pub(crate) async fn head_exists( /// A `PUT` with raw bytes and a content type (used for KVS record uploads). /// /// `content_encoding`, when set, is sent as the `Content-Encoding` header for an -/// already-compressed value; it makes [`HttpClient::call`]'s automatic request compression skip +/// already-compressed value; it makes [`HttpClient::call_with_options`]'s automatic request compression skip /// the body (it already refuses to double-encode a request that carries a `Content-Encoding`), so /// the bytes reach the API exactly as given. pub(crate) async fn put_raw( @@ -574,7 +620,7 @@ pub(crate) struct RawUpload<'a> { /// Opens a streaming `GET` connection to `url` (already fully built, including query string), /// for a caller that wants to process a large body (a log or a dataset export) incrementally -/// instead of waiting for [`HttpClient::call`] to buffer the whole response. Delegates to +/// instead of waiting for [`HttpClient::call_with_options`] to buffer the whole response. Delegates to /// [`HttpClient::call_streaming`], which — unlike a one-off raw `reqwest` call — goes through /// the configured [`HttpBackend`](crate::http_client::HttpBackend), so a custom backend (or a /// `ReqwestBackend` wrapping a caller-provided, proxy/TLS-configured `reqwest::Client`) is @@ -636,9 +682,10 @@ pub(crate) async fn delete_with_body( /// `is_terminal` extracts whether the fetched resource is finished. Returns the latest /// resource (finished, or still running if the wait budget was exhausted). /// -/// Each poll has no client-side timeout unless the client was given one with `with_timeout`: the -/// API holds a poll for up to a minute, and a poll that is aborted and retried loses nothing but -/// time, yet an arbitrary tier would cut the hold short. +/// With `wait_secs: None`, a poll has no client-side timeout unless the client was given one with +/// `with_timeout`, like the reference client. With `Some`, a poll is allowed the `Medium` tier plus +/// the hold it asks the API for, so a stalled connection cannot outlast the wait budget by more +/// than that. A timeout set with `with_timeout` always wins. pub(crate) async fn wait_for_finish( ctx: &ResourceContext, wait_secs: Option, @@ -660,9 +707,7 @@ where let elapsed = start.elapsed(); if elapsed >= budget { // Budget exhausted: do one final immediate fetch and return it. - let mut params = QueryParams::new(); - params.add_int("waitForFinish", Some(0)); - return get_resource_required(ctx, None, ¶ms, Timeout::NoTimeout).await; + return poll(ctx, wait_secs, 0).await; } let remaining = (budget - elapsed).as_secs() as i64; remaining.min(WAIT_FOR_FINISH_REQUEST_SECS) @@ -670,10 +715,7 @@ where None => WAIT_FOR_FINISH_REQUEST_SECS, }; - let mut params = QueryParams::new(); - params.add_int("waitForFinish", Some(remaining_request_secs)); - - match get_resource_required::(ctx, None, ¶ms, Timeout::NoTimeout).await { + match poll::(ctx, wait_secs, remaining_request_secs).await { Ok(resource) => { not_found_since = None; if is_terminal(&resource) { @@ -707,6 +749,23 @@ where } } +/// One `waitForFinish` poll that asks the API to hold its response for `hold_secs`. +async fn poll( + ctx: &ResourceContext, + wait_secs: Option, + hold_secs: i64, +) -> ApifyClientResult { + let mut params = QueryParams::new(); + params.add_int("waitForFinish", Some(hold_secs)); + let timeout = match wait_secs { + Some(_) => ctx.timeout_with_wait_hold(Timeout::Medium, Some(hold_secs)), + None => ctx.timeout(Timeout::NoTimeout), + }; + let url = ctx.url_with_params(None, ¶ms); + let response = ctx.send(HttpMethod::Get, url, timeout).await?; + parse_data_envelope(&response.body) +} + /// Cooldown between retries of a run start the API rejected for lack of resources, matching the /// reference clients. const WAIT_FOR_RESOURCES_COOLDOWN: Duration = Duration::from_secs(10); @@ -738,9 +797,15 @@ where }; let deadline = match wait { WaitForResources::Forever => None, - WaitForResources::ForSecs(secs) => { - Some(tokio::time::Instant::now() + Duration::from_secs_f64(secs.max(0.0))) + WaitForResources::ForSecs(secs) if secs.is_nan() => { + return Err(ApifyClientError::InvalidArgument( + "WaitForResources::ForSecs must be a number of seconds, not NaN".to_string(), + )); } + // A wait too long to represent never ends in practice, so it is a wait without a limit. + WaitForResources::ForSecs(secs) => Duration::try_from_secs_f64(secs.max(0.0)) + .ok() + .and_then(|wait| tokio::time::Instant::now().checked_add(wait)), }; loop { diff --git a/src/clients/mod.rs b/src/clients/mod.rs index afe4bcd..35918d3 100644 --- a/src/clients/mod.rs +++ b/src/clients/mod.rs @@ -28,6 +28,7 @@ pub mod run_dataset_items; pub mod schedule; pub mod schedule_collection; pub mod store_collection; +pub mod streamed_log; pub mod task; pub mod task_collection; pub mod user; diff --git a/src/clients/request_queue.rs b/src/clients/request_queue.rs index 8585326..36645eb 100644 --- a/src/clients/request_queue.rs +++ b/src/clients/request_queue.rs @@ -306,12 +306,13 @@ impl RequestQueueClient { /// Gets a request by ID, or `None` if it does not exist. pub async fn get_request(&self, id: &str) -> ApifyClientResult> { // Unlike the other `get`s, a missing request is `None` even on a queue chained from a run: - // the request ID is part of the path, so a `404` means that request does not exist. + // the request ID is part of the path, so a `404` means that request does not exist. The + // endpoint takes no `clientKey`, so none is sent. catch_not_found( get_resource_required( &self.ctx, Some(&request_path(id)?), - &self.base_params(), + &QueryParams::new(), Timeout::Short, ) .await, diff --git a/src/clients/run.rs b/src/clients/run.rs index ae22736..2b21323 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -11,14 +11,20 @@ use crate::clients::key_value_store::KeyValueStoreClient; use crate::clients::log::LogClient; use crate::clients::request_queue::RequestQueueClient; use crate::clients::run_dataset_items::RunDatasetItemsIterator; -use crate::common::{catch_not_found, to_safe_id, QueryParams}; +use crate::clients::streamed_log::{ + LogRedirect, LogSink, StdoutLogSink, StreamedLog, StreamedLogOptions, +}; +use crate::common::{to_safe_id, QueryParams}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::ActorRun; use crate::timeouts::Timeout; +use std::sync::Arc; /// Header the API uses to deduplicate charge requests (matching the reference client). const CHARGE_IDEMPOTENCY_HEADER: &str = "idempotency-key"; +/// Nanoseconds in a millisecond: the sub-millisecond part of a timestamp is below it. +const NANOS_PER_MILLI: u32 = 1_000_000; /// Options for fetching an Actor's or task's last run /// ([`crate::clients::actor::ActorClient::last_run_with_options`] / @@ -328,7 +334,7 @@ impl RunClient { .unwrap_or_default(); let millis = now.as_millis(); // Sub-millisecond nanos give a cheap, non-crypto "random" suffix (cf. JS Math.random()). - let random_suffix = now.subsec_nanos() % 1_000_000; + let random_suffix = now.subsec_nanos() % NANOS_PER_MILLI; format!("{}-{event_name}-{millis}-{random_suffix}", self.id) } @@ -420,33 +426,50 @@ impl RunClient { RunDatasetItemsIterator::new(self.clone(), options) } - /// Opens a live stream of the run's log for redirection, or `None` if the run does not exist. + /// Returns the redirection of the run's log, or `None` if the run does not exist or + /// `options.to_log` is [`LogRedirect::Disabled`]. Call [`StreamedLog::start`] to begin + /// redirecting and [`StreamedLog::stop`] to end it. Mirrors the reference client's + /// `getStreamedLog`. /// - /// Convenience equivalent to `run.log().stream()` (mirrors the reference client's - /// `getStreamedLog`): yields log chunks as they arrive, so callers can forward them to - /// their own logger/stdout while the run is in progress. Unlike `run.log().stream()`, whose - /// chained client cannot tell a missing run from a missing log, a missing run is `None` here, - /// because the run is the resource this client names. + /// With [`LogRedirect::Default`], the messages go to the standard output prefixed with + /// `"{actor name} runId:{run id} -> "`, which costs a request for the run and one for its + /// Actor. With [`LogRedirect::Sink`], they go to the sink as they are, and nothing is + /// requested. To read the log as raw chunks instead, use + /// [`LogClient::stream_with_options`](crate::clients::log::LogClient::stream_with_options) of + /// [`RunClient::log`]. pub async fn get_streamed_log( &self, - ) -> ApifyClientResult>>>> - { - self.get_streamed_log_with_options(Default::default()).await - } - - /// Opens a live stream of the run's log for redirection, applying the given - /// [`LogOptions`](crate::LogOptions) (e.g. [`LogOptions::raw`](crate::LogOptions::raw) to - /// stream the unprocessed log, which is the form the JS reference's log redirection consumes - /// internally). `None` if the run does not exist. - /// - /// This is a Rust-specific convenience that simply forwards `LogOptions` to the log stream; - /// it is not a 1:1 mirror of the JS `getStreamedLog` method's signature (which takes redirect - /// options and returns a `StreamedLog` object). - pub async fn get_streamed_log_with_options( - &self, - options: crate::clients::log::LogOptions, - ) -> ApifyClientResult>>>> - { - catch_not_found(self.log().stream_required(options).await) + options: StreamedLogOptions, + ) -> ApifyClientResult> { + let StreamedLogOptions { to_log, from_start } = options; + let (sink, prefix): (Arc, String) = match to_log { + LogRedirect::Disabled => return Ok(None), + LogRedirect::Sink(sink) => (sink, String::new()), + LogRedirect::Default => { + let Some(run) = + get_resource::(&self.ctx, None, &QueryParams::new(), Timeout::Long) + .await? + else { + return Ok(None); + }; + let actor_name = match run.act_id.as_deref().filter(|id| !id.is_empty()) { + Some(actor_id) => self + .root + .actor(actor_id) + .with_timeout(Timeout::Long) + .get() + .await? + .and_then(|actor| actor.name), + None => None, + }; + let name = [actor_name, Some(format!("runId:{}", run.id))] + .into_iter() + .flatten() + .collect::>() + .join(" "); + (Arc::new(StdoutLogSink), format!("{name} -> ")) + } + }; + Ok(Some(StreamedLog::new(self.log(), sink, prefix, from_start))) } } diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs new file mode 100644 index 0000000..d122bf5 --- /dev/null +++ b/src/clients/streamed_log.rs @@ -0,0 +1,491 @@ +//! Log redirection: forwarding the live log of a run to a [`LogSink`]. +//! +//! [`StreamedLog`] is the equivalent of the reference client's `StreamedLog`. It follows the log +//! of a run in a background task, splits it into messages at the timestamps the Actor's log lines +//! start with, and hands every message, prefixed, to a [`LogSink`]. [`ActorClient::call`] uses it +//! to redirect the log of the run it waits for. +//! +//! [`ActorClient::call`]: crate::clients::actor::ActorClient::call + +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use futures_util::StreamExt; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use crate::clients::log::{LogClient, LogOptions}; +use crate::error::{ApifyClientError, ApifyClientResult}; + +/// How long to wait before reopening the log stream of a run that has not logged anything yet. +/// The API serves such a log as an empty stream that ends at once. The pause doubles up to +/// [`EMPTY_LOG_STREAM_MAX_RETRY`]. +const EMPTY_LOG_STREAM_RETRY: Duration = Duration::from_millis(500); +/// The longest pause between two reopenings of an empty log stream. +const EMPTY_LOG_STREAM_MAX_RETRY: Duration = Duration::from_secs(5); +/// How long [`StreamedLog::stop`] lets the lines already in flight arrive before it cuts the +/// log stream. +const STOP_GRACE: Duration = Duration::from_secs(1); +/// The length of the ISO 8601 timestamp (`2024-01-31T12:00:00.000Z`) every log message starts +/// with. +const TIMESTAMP_LEN: usize = 24; +/// The shape of that timestamp: `d` is a digit, any other byte stands for itself. +const TIMESTAMP_SHAPE: &[u8; TIMESTAMP_LEN] = b"dddd-dd-ddTdd:dd:dd.dddZ"; + +/// A destination of redirected log messages, such as the terminal or a logging framework. +/// +/// Every message arrives complete (a multi-line message, such as a stack trace, in one piece) +/// and with the prefix of the redirection already added. A closure taking `&str` is a sink too: +/// +/// ``` +/// use apify_client::clients::streamed_log::{LogRedirect, LogSink}; +/// +/// let to_stderr = LogRedirect::to_sink(|line: &str| eprintln!("{line}")); +/// # let _ = to_stderr; +/// ``` +pub trait LogSink: Send + Sync + 'static { + /// Receives one message of the redirected log. + fn line(&self, message: &str); +} + +impl LogSink for F +where + F: Fn(&str) + Send + Sync + 'static, +{ + fn line(&self, message: &str) { + self(message) + } +} + +/// The default [`LogSink`]: prints every message to the standard output. +#[derive(Debug, Clone, Copy, Default)] +pub struct StdoutLogSink; + +impl LogSink for StdoutLogSink { + fn line(&self, message: &str) { + println!("{message}"); + } +} + +/// Where the log of a run goes: the `log` option of [`ActorClient::call`] and the `to_log` option +/// of [`RunClient::get_streamed_log`]. +/// +/// [`ActorClient::call`]: crate::clients::actor::ActorClient::call +/// [`RunClient::get_streamed_log`]: crate::clients::run::RunClient::get_streamed_log +#[derive(Clone, Default)] +pub enum LogRedirect { + /// Print the log to the standard output, with every message prefixed with + /// `"{actor name} runId:{run id} -> "`. Costs two extra requests, for the run and its Actor. + #[default] + Default, + /// Do not redirect the log. + Disabled, + /// Send the log to this sink, without a prefix. + Sink(Arc), +} + +impl LogRedirect { + /// Redirects the log to `sink`. + pub fn to_sink(sink: impl LogSink) -> Self { + Self::Sink(Arc::new(sink)) + } +} + +impl fmt::Debug for LogRedirect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Default => f.write_str("Default"), + Self::Disabled => f.write_str("Disabled"), + Self::Sink(_) => f.write_str("Sink(..)"), + } + } +} + +/// Options of [`RunClient::get_streamed_log`](crate::clients::run::RunClient::get_streamed_log). +#[derive(Debug, Clone)] +pub struct StreamedLogOptions { + /// Where to redirect the log. [`LogRedirect::Disabled`] makes `get_streamed_log` return + /// `None`. + pub to_log: LogRedirect, + /// Redirect the log from its beginning. When `false`, messages stamped before the call are + /// dropped. Defaults to `true`. + pub from_start: bool, +} + +impl Default for StreamedLogOptions { + fn default() -> Self { + Self { + to_log: LogRedirect::Default, + from_start: true, + } + } +} + +/// What the redirection task is asked to do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Control { + /// Keep redirecting. + Run, + /// Finish the lines in flight, then read the rest of the log and end. + Stop, + /// The grace period of a stop is over: cut the log stream. + Abort, +} + +struct Running { + control: watch::Sender, + task: JoinHandle>, +} + +/// The redirection of one run's log to a [`LogSink`], created by +/// [`RunClient::get_streamed_log`](crate::clients::run::RunClient::get_streamed_log). +/// +/// [`start`](Self::start) spawns a task that follows the log, and [`stop`](Self::stop) ends it. +/// Dropping a started `StreamedLog` cancels the task without reading the rest of the log. +/// +/// Compared with the reference client, a line that continues a message in a later chunk is +/// emitted as a message of its own, instead of being dropped. +pub struct StreamedLog { + log: LogClient, + sink: Arc, + prefix: String, + relevancy_limit: Option>, + running: Option, +} + +impl fmt::Debug for StreamedLog { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StreamedLog") + .field("prefix", &self.prefix) + .field("started", &self.running.is_some()) + .finish() + } +} + +impl StreamedLog { + pub(crate) fn new( + log: LogClient, + sink: Arc, + prefix: String, + from_start: bool, + ) -> Self { + Self { + log, + sink, + prefix, + relevancy_limit: (!from_start).then(Utc::now), + running: None, + } + } + + /// Starts redirecting the log, in a task spawned on the current Tokio runtime. Fails with + /// [`ApifyClientError::InvalidArgument`] when the redirection is already started. + pub fn start(&mut self) -> ApifyClientResult<()> { + if self.running.is_some() { + return Err(ApifyClientError::InvalidArgument( + "the log redirection is already started".to_string(), + )); + } + let (control, receiver) = watch::channel(Control::Run); + let task = tokio::spawn(redirect( + self.log.clone(), + self.sink.clone(), + self.prefix.clone(), + self.relevancy_limit, + receiver, + )); + self.running = Some(Running { control, task }); + Ok(()) + } + + /// Stops redirecting the log. Waits up to one second for the lines in flight, then cuts the + /// log stream. If the stream has delivered nothing yet, reads the whole log in one request + /// instead. Fails with [`ApifyClientError::InvalidArgument`] when the redirection is not + /// started, and with the error that ended the redirection early, if there was one. + pub async fn stop(&mut self) -> ApifyClientResult<()> { + let Some(Running { control, mut task }) = self.running.take() else { + return Err(ApifyClientError::InvalidArgument( + "the log redirection is not started".to_string(), + )); + }; + control.send_replace(Control::Stop); + let outcome = match tokio::time::timeout(STOP_GRACE, &mut task).await { + Ok(outcome) => outcome, + Err(_) => { + control.send_replace(Control::Abort); + task.await + } + }; + match outcome { + Ok(None) | Err(_) => Ok(()), + Ok(Some(error)) => Err(error), + } + } +} + +impl Drop for StreamedLog { + fn drop(&mut self) { + if let Some(running) = &self.running { + running.task.abort(); + } + } +} + +fn is_stopping(control: &watch::Receiver) -> bool { + *control.borrow() != Control::Run +} + +/// Resolves once the grace period of a stop is over (or the [`StreamedLog`] is gone). +async fn aborted(mut control: watch::Receiver) { + let _ = control.wait_for(|c| *c == Control::Abort).await; +} + +/// Resolves once a stop is requested (or the [`StreamedLog`] is gone). +async fn stopped(mut control: watch::Receiver) { + let _ = control.wait_for(|c| *c != Control::Run).await; +} + +/// The redirection task. Returns the error that ended it early. +async fn redirect( + log: LogClient, + sink: Arc, + prefix: String, + relevancy_limit: Option>, + control: watch::Receiver, +) -> Option { + let mut messages = MessageSplitter::new(relevancy_limit); + let emit = |message: &str| sink.line(&format!("{prefix}{message}")); + let raw = LogOptions { + raw: Some(true), + download: None, + }; + let mut retry = EMPTY_LOG_STREAM_RETRY; + // The API serves the log of a run that has not logged anything yet as an empty stream that + // ends at once, so reopen it until the first bytes arrive. Once stopped, read the whole log + // in one request instead. + loop { + if is_stopping(&control) { + match log.get_bytes_with_options(raw.clone()).await { + Ok(content) => messages.feed(&content.unwrap_or_default(), &emit), + Err(error) => return Some(error), + } + break; + } + let opened = tokio::select! { + opened = log.stream_with_options(raw.clone()) => opened, + () = aborted(control.clone()) => continue, + }; + let stream = match opened { + Ok(Some(stream)) => stream, + Ok(None) => return None, + Err(_) if is_stopping(&control) => continue, + Err(error) => return Some(error), + }; + if is_stopping(&control) { + continue; + } + futures_util::pin_mut!(stream); + let mut received_any = false; + loop { + let item = tokio::select! { + item = stream.next() => item, + () = aborted(control.clone()) => break, + }; + match item { + Some(Ok(chunk)) => { + received_any |= !chunk.is_empty(); + messages.feed(&chunk, &emit); + if is_stopping(&control) { + break; + } + } + Some(Err(_)) if is_stopping(&control) => break, + Some(Err(error)) => { + messages.finish(&emit); + return Some(error); + } + None => break, + } + } + if received_any { + break; + } + tokio::select! { + () = tokio::time::sleep(retry) => {} + () = stopped(control.clone()) => {} + } + retry = (retry * 2).min(EMPTY_LOG_STREAM_MAX_RETRY); + } + // Whatever is left may be incomplete, or the last line of the log without a line break. + messages.finish(&emit); + None +} + +/// Splits the bytes of a raw log into messages. A message starts with a line that begins with an +/// ISO 8601 timestamp and runs up to the next such line, so a stack trace stays in one piece. +struct MessageSplitter { + /// The incomplete last line of the bytes fed so far, led by the line break before it. + remainder: Vec, + /// Messages stamped before this are dropped. + relevancy_limit: Option>, + /// Whether the message in progress at the end of the previous batch was relevant, so the + /// lines that continue it are too. + relevant: bool, +} + +impl MessageSplitter { + fn new(relevancy_limit: Option>) -> Self { + Self { + remainder: Vec::new(), + relevancy_limit, + relevant: relevancy_limit.is_none(), + } + } + + /// Emits every message in the complete lines of `chunk` (and the lines held back before it), + /// and holds back the incomplete last line. + fn feed(&mut self, chunk: &[u8], emit: &dyn Fn(&str)) { + self.remainder.extend_from_slice(chunk); + let Some(last_line_break) = self.remainder.iter().rposition(|&b| b == b'\n') else { + return; + }; + let complete: Vec = self.remainder.drain(..last_line_break).collect(); + self.emit_messages(&String::from_utf8_lossy(&complete), emit); + } + + /// Emits the held-back last line as well, when the log is over. + fn finish(&mut self, emit: &dyn Fn(&str)) { + self.remainder.push(b'\n'); + self.feed(&[], emit); + } + + fn emit_messages(&mut self, text: &str, emit: &dyn Fn(&str)) { + let mut message = String::new(); + for line in text.split('\n') { + if starts_with_timestamp(line) { + self.emit_message(&message, emit); + message.clear(); + self.relevant = self.is_relevant(&line[..TIMESTAMP_LEN]); + } + if !message.is_empty() { + message.push('\n'); + } + message.push_str(line); + } + self.emit_message(&message, emit); + } + + fn emit_message(&self, message: &str, emit: &dyn Fn(&str)) { + let message = message.trim(); + if self.relevant && !message.is_empty() { + emit(message); + } + } + + fn is_relevant(&self, timestamp: &str) -> bool { + match ( + self.relevancy_limit, + DateTime::parse_from_rfc3339(timestamp), + ) { + (Some(limit), Ok(time)) => time >= limit, + _ => true, + } + } +} + +/// Whether `line` begins with an ISO 8601 timestamp, such as `2024-01-31T12:00:00.000Z`. +fn starts_with_timestamp(line: &str) -> bool { + let bytes = line.as_bytes(); + bytes.len() >= TIMESTAMP_LEN + && bytes + .iter() + .zip(TIMESTAMP_SHAPE) + .all(|(&byte, &shape)| match shape { + b'd' => byte.is_ascii_digit(), + _ => byte == shape, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + fn split(limit: Option>, chunks: &[&str], finish: bool) -> Vec { + let out = RefCell::new(Vec::new()); + let emit = |m: &str| out.borrow_mut().push(m.to_string()); + let mut splitter = MessageSplitter::new(limit); + for chunk in chunks { + splitter.feed(chunk.as_bytes(), &emit); + } + if finish { + splitter.finish(&emit); + } + out.into_inner() + } + + #[test] + fn multi_line_messages_stay_together_and_the_last_line_waits_for_its_line_break() { + let log = "2024-01-01T00:00:00.000Z first\n at trace\n2024-01-01T00:00:01.000Z second\n2024-01-01T00:00:02.000Z third"; + assert_eq!( + split(None, &[log], false), + vec![ + "2024-01-01T00:00:00.000Z first\n at trace".to_string(), + "2024-01-01T00:00:01.000Z second".to_string(), + ] + ); + assert_eq!( + split(None, &[log], true), + vec![ + "2024-01-01T00:00:00.000Z first\n at trace".to_string(), + "2024-01-01T00:00:01.000Z second".to_string(), + "2024-01-01T00:00:02.000Z third".to_string(), + ] + ); + } + + #[test] + fn a_line_split_across_chunks_is_emitted_whole() { + let out = split( + None, + &[ + "2024-01-01T00:00:00.000Z hel", + "lo\n2024-01-01T00:00:01.000Z x\n", + ], + true, + ); + assert_eq!( + out, + vec![ + "2024-01-01T00:00:00.000Z hello".to_string(), + "2024-01-01T00:00:01.000Z x".to_string() + ] + ); + } + + #[test] + fn messages_stamped_before_the_limit_are_dropped() { + let limit = DateTime::parse_from_rfc3339("2024-01-01T00:00:01.000Z") + .unwrap() + .with_timezone(&Utc); + let log = "2024-01-01T00:00:00.000Z old\n old trace\n2024-01-01T00:00:01.000Z new\n"; + assert_eq!( + split(Some(limit), &[log], true), + vec!["2024-01-01T00:00:01.000Z new".to_string()] + ); + } + + #[test] + fn lines_without_a_timestamp_continue_the_previous_message() { + let out = split(None, &["2024-01-01T00:00:00.000Z a\n", "continued\n"], true); + assert_eq!( + out, + vec![ + "2024-01-01T00:00:00.000Z a".to_string(), + "continued".to_string() + ] + ); + } +} diff --git a/src/clients/task.rs b/src/clients/task.rs index dfa0c84..c0df615 100644 --- a/src/clients/task.rs +++ b/src/clients/task.rs @@ -4,9 +4,9 @@ use serde::Serialize; use serde_json::Value; use crate::client::ApifyClient; -use crate::clients::actor::ActorStartOptions; +use crate::clients::actor::{ActorStartOptions, TaskStartOptions}; use crate::clients::base::{ - delete_resource, get_resource, post_with_body, update_resource, ResourceContext, + delete_resource, get_resource, post_creating, update_resource, ResourceContext, }; use crate::clients::run::{LastRunOptions, RunClient}; use crate::clients::run_collection::RunCollectionClient; @@ -74,13 +74,25 @@ impl TaskClient { /// Starts the task and returns immediately with the created run. /// /// `input` overrides the task's saved input (or `None` to use the saved input). Set - /// [`ActorStartOptions::wait_for_resources`] to retry the start while the account lacks the + /// [`TaskStartOptions::wait_for_resources`] to retry the start while the account lacks the /// memory or a concurrent-run slot for it. pub async fn start( &self, input: Option<&T>, - options: ActorStartOptions, + options: TaskStartOptions, ) -> ApifyClientResult { + self.start_with_tier(input, options, Timeout::Medium).await + } + + /// Starts the task with `tier` as the timeout of the request (unless the client was given + /// one with `with_timeout`). + async fn start_with_tier( + &self, + input: Option<&T>, + options: TaskStartOptions, + tier: Timeout, + ) -> ApifyClientResult { + let options = ActorStartOptions::from(options); let mut params = QueryParams::new(); options.apply(&mut params); let body = match input { @@ -89,9 +101,9 @@ impl TaskClient { }; let timeout = self .ctx - .timeout_with_wait_hold(Timeout::Medium, options.wait_for_finish); + .timeout_with_wait_hold(tier, options.wait_for_finish); crate::clients::base::retry_while_resource_limited(options.wait_for_resources, || { - post_with_body( + post_creating( &self.ctx, Some("runs"), ¶ms, @@ -111,16 +123,26 @@ impl TaskClient { /// then, the **last fetched (still non-terminal) run is returned** rather than an /// error. Check `status` / `is_terminal()` on the result when using `Some`. /// - /// Time spent retrying under [`ActorStartOptions::wait_for_resources`] does not count + /// Time spent retrying under [`TaskStartOptions::wait_for_resources`] does not count /// toward `wait_secs`, since that budget only starts once the run has actually started. + /// + /// Like [`ActorClient::call`](crate::clients::actor::ActorClient::call), the start request + /// has no client-side timeout unless the client was given one with `with_timeout`, which + /// then applies to the polls as well. The log of the run is not redirected. pub async fn call( &self, input: Option<&T>, - options: ActorStartOptions, + options: TaskStartOptions, wait_secs: Option, ) -> ApifyClientResult { - let run = self.start(input, options).await?; - self.root.run(run.id).wait_for_finish(wait_secs).await + let run = self + .start_with_tier(input, options, Timeout::NoTimeout) + .await?; + let mut run_client = self.root.run(run.id); + if let Some(timeout) = self.ctx.timeout { + run_client = run_client.with_timeout(timeout); + } + run_client.wait_for_finish(wait_secs).await } /// Fetches the task's saved input. diff --git a/src/common.rs b/src/common.rs index 120bd91..d3a1f62 100644 --- a/src/common.rs +++ b/src/common.rs @@ -36,7 +36,7 @@ pub(crate) fn catch_not_found(result: ApifyClientResult) -> ApifyClientRes /// A mutable collection of query parameters that serializes booleans as `0`/`1` and /// omits `None` values, matching the Apify API conventions. #[derive(Debug, Default, Clone)] -pub struct QueryParams { +pub(crate) struct QueryParams { pairs: Vec<(String, String)>, } @@ -89,11 +89,6 @@ impl QueryParams { self } - /// Returns `true` if no parameters were added. - pub fn is_empty(&self) -> bool { - self.pairs.is_empty() - } - /// Internal: read access to the raw pairs (used when merging parent params). pub(crate) fn pairs_ref(&self) -> &[(String, String)] { &self.pairs @@ -144,7 +139,7 @@ fn url_encode(input: &str) -> String { /// This does not reject a segment that is empty or a dot segment (`.` or `..`), which a URL /// parser resolves to another endpoint even when percent-encoded; the client methods that take /// such a value validate it with `to_path_segment` instead. -pub fn encode_path_segment(input: &str) -> String { +pub(crate) fn encode_path_segment(input: &str) -> String { // Path segments and query components share the same unreserved set here, so reuse the // query encoder. (`url_encode` already escapes `/`, `?`, `#`, space and non-ASCII.) url_encode(input) @@ -249,7 +244,7 @@ fn env_var_set(name: &str) -> bool { /// Builds the `User-Agent` header value mandated by the client requirements: /// `ApifyClient/{version} ({os}; {language version}); isAtHome/{isAtHome}`. -pub fn build_user_agent(suffix: Option<&str>) -> String { +pub(crate) fn build_user_agent(suffix: Option<&str>) -> String { // The OS token must be byte-for-byte identical across all Apify clients, which means it must // equal the reference JS client's `os.platform()` value. `std::env::consts::OS` uses Rust's // own spellings (`macos`, `windows`, `solaris`, `illumos`), so map those to the Node @@ -299,7 +294,7 @@ fn map_os_to_node_platform(os: &str) -> &str { /// Encodes a resource id so it is safe to embed in a URL path. Apify uses the /// `username~resourcename` form, so every `/` of an id is replaced with `~`. -pub fn to_safe_id(id: &str) -> String { +pub(crate) fn to_safe_id(id: &str) -> String { id.replace('/', "~") } @@ -318,7 +313,7 @@ const BASE62_ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJK /// The algorithm is: `HMAC-SHA256(secret_key, message)` as lowercase hex, take the first 30 /// hex characters, interpret them as a big integer, then base62-encode (alphabet `0-9a-zA-Z`). /// Used to sign key-value-store record keys for public access. -pub fn create_hmac_signature(secret_key: &str, message: &str) -> String { +pub(crate) fn create_hmac_signature(secret_key: &str, message: &str) -> String { use hmac::{Hmac, Mac}; use sha2::Sha256; @@ -353,6 +348,9 @@ fn to_base62(mut value: u128) -> String { String::from_utf8(digits).expect("base62 alphabet is valid ASCII") } +/// Milliseconds in a second. +const MILLIS_PER_SECOND: i64 = 1000; + /// Builds a storage-content signature for a resource's public URL, byte-for-byte compatible /// with the platform's `@apify/utilities` `createStorageContentSignature`. /// @@ -360,7 +358,7 @@ fn to_base62(mut value: u128) -> String { /// is the absolute expiry in milliseconds, or `0` for a non-expiring URL) with /// [`create_hmac_signature`], then returns the base64url (no padding) encoding of /// `"{version}.{expiresAtMillis}.{hmac}"`. Used for dataset-items and key-list public URLs. -pub fn sign_storage_content( +pub(crate) fn sign_storage_content( secret_key: &str, resource_id: &str, expires_in_secs: Option, @@ -368,7 +366,7 @@ pub fn sign_storage_content( use base64::Engine; let expires_at_millis = match expires_in_secs { - Some(secs) => chrono::Utc::now().timestamp_millis() + secs * 1000, + Some(secs) => chrono::Utc::now().timestamp_millis() + secs * MILLIS_PER_SECOND, None => 0, }; let version = STORAGE_CONTENT_SIGNATURE_VERSION; diff --git a/src/http_client.rs b/src/http_client.rs index 8fb2417..9f50888 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -6,7 +6,7 @@ //! backend (e.g. for testing or for a different runtime) can be plugged in via //! [`ApifyClientBuilder::http_backend`](crate::ApifyClientBuilder::http_backend). //! -//! [`HttpClient`] wraps a backend and adds the cross-cutting concerns shared by every +//! The client wraps a backend in an internal `HttpClient` that adds the cross-cutting concerns shared by every //! endpoint: authentication, the `User-Agent` header, query-parameter serialization, //! timeouts and retries with exponential backoff (mirroring the JavaScript and Python //! reference clients). @@ -62,7 +62,7 @@ impl HttpMethod { /// A fully-resolved HTTP request, ready to be sent by an [`HttpBackend`]. /// /// All cross-cutting concerns (auth header, user agent, retry policy) are applied by -/// [`HttpClient`] before the request reaches the backend. +/// the client before the request reaches the backend. #[derive(Debug, Clone)] pub struct HttpRequest { /// The HTTP method. @@ -117,7 +117,7 @@ pub struct StreamingHttpResponse { /// /// Implementors are responsible only for sending a single request and returning the /// raw response. Retries, authentication and serialization are handled by -/// [`HttpClient`], so a backend only needs to perform one network round-trip. +/// the client, so a backend only needs to perform one network round-trip. #[async_trait] pub trait HttpBackend: Send + Sync + std::fmt::Debug { /// Sends a single HTTP request and returns the response. @@ -265,7 +265,7 @@ impl HttpBackend for ReqwestBackend { /// Configuration for the retry behaviour of the [`HttpClient`]. #[derive(Debug, Clone)] -pub struct RetryConfig { +pub(crate) struct RetryConfig { /// Maximum number of *retries* (i.e. the request is attempted up to `max_retries + 1` times). pub max_retries: u32, /// Minimum delay between retries; doubled on each subsequent retry (exponential backoff). @@ -302,7 +302,7 @@ pub(crate) struct CallOptions { /// It is cheap to clone (everything is reference-counted) so each resource client can /// hold its own handle. #[derive(Debug, Clone)] -pub struct HttpClient { +pub(crate) struct HttpClient { backend: Arc, token: Option, user_agent: String, @@ -390,13 +390,8 @@ impl HttpClient { } /// Sends `request` with authentication, the user-agent header, and the retry policy - /// applied. Returns the first successful response, or the final error. - pub async fn call(&self, request: HttpRequest) -> ApifyClientResult { - self.call_with_options(request, CallOptions::default()) - .await - } - - /// Like [`call`](Self::call), with the given per-call [`CallOptions`]. + /// applied, as `options` adjust it. Returns the first successful response, or the final + /// error. pub(crate) async fn call_with_options( &self, mut request: HttpRequest, @@ -488,13 +483,13 @@ impl HttpClient { } /// Sends `request` through the configured [`HttpBackend`]'s [`HttpBackend::send_streaming`], - /// with authentication and the user-agent header applied like [`call`](Self::call), but with + /// with authentication and the user-agent header applied like [`call_with_options`](Self::call_with_options), but with /// no retries and no request compression: matches the reference clients, which do not retry /// an open stream (a log redirection or a large export already in flight can't be safely /// replayed from the start). /// /// On a non-success status, reads the body (bounded by the same inactivity timeout as a - /// successful body below) and returns the same structured [`ApiError`] [`call`](Self::call) + /// successful body below) and returns the same structured [`ApiError`] [`call_with_options`](Self::call_with_options) /// would. On success, returns the body as a stream whose `.next()` must resolve within the /// request's resolved timeout each time — bounding silence (a stalled connection) without /// capping the stream's total duration, so a long-but-active stream (e.g. live log diff --git a/src/lib.rs b/src/lib.rs index e43ac18..51f2ef3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,7 @@ //! // Start an Actor and wait for it to finish. //! let run = client //! .actor("apify/hello-world") -//! .call::(None, Default::default(), None) +//! .call::(None, Default::default()) //! .await?; //! //! // Read items from the run's default dataset. @@ -40,7 +40,7 @@ //! [`http_client::ReqwestBackend`]. Swap it via //! [`ApifyClientBuilder::http_backend`]. //! - **Cross-cutting behaviour** (auth, `User-Agent`, retries with exponential backoff, -//! timeouts, request compression) lives in [`http_client::HttpClient`] and is applied to every +//! timeouts, request compression) lives in the client's internal HTTP layer and is applied to every //! request. //! - **Timeouts**: every method uses a [`Timeout`] tier (`Short`, `Medium` or `Long`) whose //! duration is configured on the [`ApifyClientBuilder`]. A resource client's `with_timeout` @@ -72,7 +72,7 @@ pub mod timeouts; mod version; pub use client::{ApifyClient, ApifyClientBuilder}; -pub use common::{ListOptions, PaginationList, QueryParams, StorageListOptions}; +pub use common::{ListOptions, PaginationList, StorageListOptions}; pub use error::{ApiError, ApifyClientError, ApifyClientResult}; pub use http_compressors::{ BrotliHttpCompressor, GzipHttpCompressor, HttpCompressor, RequestCompression, @@ -82,7 +82,9 @@ pub use timeouts::Timeout; pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; // Re-export the most commonly used option/parameter types for ergonomic access. -pub use clients::actor::{ActorBuildOptions, ActorStartOptions, WaitForResources}; +pub use clients::actor::{ + ActorBuildOptions, ActorCallOptions, ActorStartOptions, TaskStartOptions, WaitForResources, +}; pub use clients::actor_collection::ActorListOptions; pub use clients::dataset::{DatasetDownloadOptions, DatasetListItemsOptions, DownloadItemsFormat}; pub use clients::key_value_store::{GetRecordOptions, KeyValueStoreKeysIterator, ListKeysOptions}; @@ -97,6 +99,9 @@ pub use clients::run::{ pub use clients::run_collection::RunListOptions; pub use clients::run_dataset_items::RunDatasetItemsIterator; pub use clients::store_collection::{StoreActorIterator, StoreListOptions}; +pub use clients::streamed_log::{ + LogRedirect, LogSink, StdoutLogSink, StreamedLog, StreamedLogOptions, +}; // Compile-test the code snippets in the README and the external `docs/` pages so every // in-documentation code snippet stays valid and runnable. Pulling each Markdown file in as diff --git a/src/timeouts.rs b/src/timeouts.rs index 16d44ca..8b824d8 100644 --- a/src/timeouts.rs +++ b/src/timeouts.rs @@ -97,6 +97,9 @@ impl TimeoutTiers { let hold = wait_for_finish_secs .filter(|&secs| secs > 0) .map(|secs| Duration::from_secs(secs as u64).min(MAX_WAIT_FOR_FINISH_HOLD)); + if tier == Timeout::NoTimeout { + return tier; + } match hold { Some(hold) => Timeout::Custom(self.resolve(tier).saturating_add(hold)), None => tier, diff --git a/tests/actor.rs b/tests/actor.rs index d2abd23..8512a23 100644 --- a/tests/actor.rs +++ b/tests/actor.rs @@ -405,3 +405,46 @@ async fn actor_env_var_crud_flow() { "env var should be gone after delete" ); } + +/// Simple GET: resolve the default build of an Actor and fetch it. +#[tokio::test(flavor = "multi_thread")] +async fn get_default_build() { + let client = require_client!(); + let build = client + .actor("apify/hello-world") + .default_build(None) + .await + .expect("resolve the default build"); + let fetched = build + .get() + .await + .expect("get the default build") + .expect("the default build should exist"); + assert!(!fetched.id.is_empty()); +} + +/// Simple GET: listing the webhooks of an Actor. +#[tokio::test(flavor = "multi_thread")] +async fn list_actor_webhooks() { + let client = require_client!(); + let name = actor_name("actor-webhooks"); + let actor = client + .actors() + .create(&actor_definition(&name)) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&id).delete().await; + }); + + let page = client + .actor(&actor.id) + .webhooks() + .list(Default::default()) + .await + .expect("listing the webhooks of an Actor should succeed"); + assert!(page.total >= 0); +} diff --git a/tests/actor_run.rs b/tests/actor_run.rs index aa8b4b6..16bc791 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -26,7 +26,13 @@ async fn run_actor_and_read_outputs() { // Start + wait for the public hello-world Actor. let run = client .actor("apify/hello-world") - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await .expect("call hello-world actor"); @@ -107,7 +113,13 @@ async fn iterate_runs() { // Ensure at least one run exists on the account by calling the public hello-world Actor. let run = client .actor("apify/hello-world") - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await .expect("call hello-world actor"); @@ -142,7 +154,13 @@ async fn last_run_access() { // Ensure there is at least one run by calling the actor. client .actor("apify/hello-world") - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await .expect("call hello-world actor"); @@ -166,11 +184,14 @@ async fn call_raw_sends_raw_bytes_input() { .actor("apify/hello-world") .call_raw( b"{}", - apify_client::ActorStartOptions { - content_type: Some("application/json".to_string()), + apify_client::ActorCallOptions { + start: apify_client::ActorStartOptions { + content_type: Some("application/json".to_string()), + ..Default::default() + }, + wait_secs: Some(120), ..Default::default() }, - Some(120), ) .await .expect("call_raw hello-world actor"); @@ -288,7 +309,13 @@ async fn iterate_run_dataset_items_reads_the_items_of_a_finished_run() { let client = require_client!(); let run = client .actor("apify/hello-world") - .call::(None, Default::default(), Some(120)) + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + ..Default::default() + }, + ) .await .expect("call hello-world actor"); let dataset_id = run @@ -315,3 +342,68 @@ async fn iterate_run_dataset_items_reads_the_items_of_a_finished_run() { assert_eq!(pushed[0]["n"], 1); assert_eq!(pushed[2]["n"], 3); } + +/// Simple GETs of the storages of a finished run, and of the run's log through the top-level +/// `/logs/{id}` endpoint. +#[tokio::test(flavor = "multi_thread")] +async fn run_scoped_storages_and_log_can_be_read() { + let client = require_client!(); + let run = client + .actor("apify/hello-world") + .call::( + None, + apify_client::ActorCallOptions { + wait_secs: Some(120), + log: apify_client::LogRedirect::Disabled, + ..Default::default() + }, + ) + .await + .expect("call hello-world actor"); + let run_client = client.run(&run.id); + + let log = client.log(&run.id).get().await.expect("get the log by id"); + assert!(log.is_some(), "a finished run should have a log"); + + let store = run_client + .key_value_store() + .get() + .await + .expect("get the run's key-value store"); + assert!(store.is_some()); + let keys = run_client + .key_value_store() + .list_keys(Default::default()) + .await + .expect("list the keys of the run's key-value store"); + assert!(keys.items.iter().any(|key| key.key == "OUTPUT")); + + let queue = run_client + .request_queue() + .get() + .await + .expect("get the run's request queue"); + assert!(queue.is_some()); + run_client + .request_queue() + .list_head(Some(10)) + .await + .expect("list the head of the run's request queue"); + run_client + .request_queue() + .list_requests(Default::default()) + .await + .expect("list the requests of the run's request queue"); + + let dataset = run_client + .dataset() + .get() + .await + .expect("get the run's dataset"); + assert!(dataset.is_some()); + run_client + .dataset() + .get_statistics() + .await + .expect("get the statistics of the run's dataset"); +} diff --git a/tests/task.rs b/tests/task.rs index 7ae9a60..d02f826 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -307,3 +307,53 @@ async fn task_crud_flow() { task_client.delete().await.expect("delete task"); assert!(task_client.get().await.expect("get after delete").is_none()); } + +/// Simple GETs: the webhooks of a task, and its last run with that run's log. +#[tokio::test(flavor = "multi_thread")] +async fn task_webhooks_and_last_run() { + let client = require_client!(); + let name = common::unique_name("task-last-run"); + let task = client + .tasks() + .create(&task_definition(&name)) + .await + .expect("create task"); + + let cleanup_client = client.clone(); + let id = task.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.task(&id).delete().await; + }); + + let webhooks = client + .task(&task.id) + .webhooks() + .list(Default::default()) + .await + .expect("listing the webhooks of a task should succeed"); + assert!(webhooks.total >= 0); + + let run = client + .task(&task.id) + .call::(None, Default::default(), Some(120)) + .await + .expect("call the task"); + + let last = client + .task(&task.id) + .last_run(Some("SUCCEEDED")) + .get() + .await + .expect("get the last run of the task") + .expect("the task should have a last run"); + assert_eq!(last.id, run.id); + + let log = client + .task(&task.id) + .last_run(None) + .log() + .get() + .await + .expect("get the log of the last run of the task"); + assert!(log.is_some(), "the last run should have a log"); +} diff --git a/tests/unit_call.rs b/tests/unit_call.rs new file mode 100644 index 0000000..2f5bb83 --- /dev/null +++ b/tests/unit_call.rs @@ -0,0 +1,517 @@ +//! Offline unit tests for `call`, the retry of non-idempotent starts, the polling timeouts, the +//! `WaitForResources` limits and the log redirection. +//! +//! These use scripted [`HttpBackend`]s and require no network access or `APIFY_TOKEN`. + +mod common; + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use apify_client::clients::actor::WaitForResources; +use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, HttpResponse}; +use apify_client::timeouts::NO_TIMEOUT; +use apify_client::{ + ActorBuildOptions, ActorCallOptions, ActorStartOptions, ApifyClient, ApifyClientError, + LogRedirect, StreamedLogOptions, TaskStartOptions, Timeout, +}; +use async_trait::async_trait; +use common::mock_backend::{client_with, MockBackend, MockOutcome}; +use serde_json::{json, Value}; + +const RUN_READY: &str = r#"{"data":{"id":"r1","actId":"a1","status":"READY"}}"#; +const RUN_DONE: &str = r#"{"data":{"id":"r1","actId":"a1","status":"SUCCEEDED"}}"#; + +fn ok(body: &str) -> MockOutcome { + MockOutcome::Status(200, body.as_bytes().to_vec()) +} + +fn call_options(wait_secs: Option) -> ActorCallOptions { + ActorCallOptions { + wait_secs, + log: LogRedirect::Disabled, + ..Default::default() + } +} + +type Route = dyn Fn(&HttpRequest, usize) -> Result + Send + Sync; + +/// A backend that answers by request, and records every request. +struct RouteBackend { + requests: Mutex>, + route: Box, +} + +impl std::fmt::Debug for RouteBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("RouteBackend") + } +} + +impl RouteBackend { + fn new( + route: impl Fn(&HttpRequest, usize) -> Result + + Send + + Sync + + 'static, + ) -> Arc { + Arc::new(Self { + requests: Mutex::new(Vec::new()), + route: Box::new(route), + }) + } + + fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + + fn count(&self, method: HttpMethod, url_part: &str) -> usize { + self.requests() + .iter() + .filter(|r| r.method == method && r.url.contains(url_part)) + .count() + } +} + +#[async_trait] +impl HttpBackend for RouteBackend { + async fn send(&self, request: HttpRequest) -> Result { + let index = { + let mut requests = self.requests.lock().unwrap(); + requests.push(request.clone()); + requests.len() - 1 + }; + (self.route)(&request, index) + } +} + +fn response(status: u16, body: &str) -> Result { + Ok(HttpResponse { + status, + headers: Default::default(), + body: body.as_bytes().to_vec(), + }) +} + +fn client_on(backend: Arc, max_retries: u32) -> ApifyClient { + ApifyClient::builder() + .token("test-token") + .max_retries(max_retries) + .min_delay_between_retries(Duration::from_millis(1)) + .timeout_medium(Duration::from_secs(1)) + .http_backend(backend) + .build() +} + +/// A start the API holds for longer than the `Medium` tier must not be aborted and retried by +/// `call`: that would POST a second run. The start has no timeout, so the POST happens once. +#[tokio::test] +async fn call_does_not_abort_and_retry_a_slow_start() { + let backend = RouteBackend::new(|request, _| { + if request.method == HttpMethod::Post { + // The API holds the start for longer than the tier the client would give it. + if request.timeout < Duration::from_millis(1500) { + return Err(ApifyClientError::Timeout); + } + return response(201, RUN_READY); + } + response(200, RUN_DONE) + }); + let client = client_on(backend.clone(), 3); + + let run = client + .actor("a") + .call::(None, call_options(None)) + .await + .unwrap(); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + assert_eq!(backend.count(HttpMethod::Post, "/runs"), 1); + + let run = client + .task("t") + .call::(None, TaskStartOptions::default(), None) + .await + .unwrap(); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + assert_eq!(backend.count(HttpMethod::Post, "/runs"), 2); +} + +#[tokio::test] +async fn call_starts_without_a_timeout_and_polls_as_wait_secs_asks() { + let backend = MockBackend::new(vec![ok(RUN_READY), ok(RUN_DONE)]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .call::(None, call_options(None)) + .await + .unwrap(); + assert_eq!(backend.timeouts(), vec![NO_TIMEOUT, NO_TIMEOUT]); + + // A bounded wait gives each poll the `Medium` tier plus the hold it asks the API for, so a + // stalled connection cannot outlast the budget. + let backend = MockBackend::new(vec![ok(RUN_READY), ok(RUN_DONE)]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .call_raw(b"{}", call_options(Some(10))) + .await + .unwrap(); + let timeouts = backend.timeouts(); + assert_eq!(timeouts[0], NO_TIMEOUT); + assert!( + timeouts[1] > Duration::from_secs(30) && timeouts[1] <= Duration::from_secs(40), + "{timeouts:?}" + ); +} + +/// A timeout set on the client applies to the start and to every poll. +#[tokio::test] +async fn call_applies_the_timeout_of_the_client_to_the_start_and_the_polls() { + let backend = MockBackend::new(vec![ + ok(RUN_READY), + ok(RUN_DONE), + ok(RUN_READY), + ok(RUN_DONE), + ]); + let client = client_with(backend.clone(), 0); + let seven = Timeout::Custom(Duration::from_secs(7)); + client + .actor("a") + .with_timeout(seven) + .call::(None, call_options(None)) + .await + .unwrap(); + client + .task("t") + .with_timeout(seven) + .call::(None, TaskStartOptions::default(), Some(5)) + .await + .unwrap(); + assert_eq!(backend.timeouts(), vec![Duration::from_secs(7); 4]); +} + +/// A start or a build that timed out may have reached the API, so it is not retried. +#[tokio::test] +async fn a_start_or_build_that_times_out_is_not_retried() { + let backend = MockBackend::new(vec![MockOutcome::Timeout]); + let client = client_with(backend.clone(), 3); + let wait = ActorStartOptions { + wait_for_finish: Some(2), + ..Default::default() + }; + + let result = client.actor("a").start::(None, wait.clone()).await; + assert!(matches!(result, Err(ApifyClientError::Timeout))); + assert_eq!(backend.call_count(), 1); + + let result = client.actor("a").start_raw(b"{}", wait).await; + assert!(matches!(result, Err(ApifyClientError::Timeout))); + assert_eq!(backend.call_count(), 2); + + let result = client + .actor("a") + .build( + "0.1", + ActorBuildOptions { + wait_for_finish: Some(2), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(ApifyClientError::Timeout))); + assert_eq!(backend.call_count(), 3); + + let result = client + .task("t") + .start::(None, TaskStartOptions::default()) + .await; + assert!(matches!(result, Err(ApifyClientError::Timeout))); + assert_eq!(backend.call_count(), 4); + + // Other errors are still retried. + let backend = MockBackend::new(vec![MockOutcome::NetworkError]); + let client = client_with(backend.clone(), 2); + assert!(client + .actor("a") + .start::(None, Default::default()) + .await + .is_err()); + assert_eq!(backend.call_count(), 3); +} + +/// A task start takes neither `contentType` nor `forcePermissionLevel`. +#[tokio::test] +async fn a_task_start_sends_only_the_parameters_the_endpoint_documents() { + let backend = MockBackend::new(vec![ok(RUN_READY)]); + let client = client_with(backend.clone(), 0); + client + .task("t") + .start::( + Some(&json!({})), + TaskStartOptions { + build: Some("beta".to_string()), + memory_mbytes: Some(512), + ..Default::default() + }, + ) + .await + .unwrap(); + let url = backend.last_url().unwrap(); + assert!( + url.contains("build=beta") && url.contains("memory=512"), + "{url}" + ); + assert!(!url.contains("forcePermissionLevel"), "{url}"); +} + +#[tokio::test] +async fn get_request_sends_no_client_key() { + let backend = MockBackend::new(vec![ok(r#"{"data":{"id":"x","url":"u"}}"#)]); + let client = client_with(backend.clone(), 0); + client + .request_queue("q") + .with_client_key("key") + .get_request("x") + .await + .unwrap(); + assert!(!backend.last_url().unwrap().contains("clientKey")); +} + +#[tokio::test(start_paused = true)] +async fn wait_for_resources_accepts_an_unbounded_wait_and_rejects_nan() { + let limit = MockOutcome::Status( + 402, + br#"{"error":{"type":"actor-memory-limit-exceeded","message":"m"}}"#.to_vec(), + ); + for secs in [f64::INFINITY, 1e30, f64::MAX] { + let backend = MockBackend::new(vec![limit.clone(), ok(RUN_READY)]); + let client = client_with(backend.clone(), 0); + client + .actor("a") + .start::( + None, + ActorStartOptions { + wait_for_resources: Some(WaitForResources::ForSecs(secs)), + ..Default::default() + }, + ) + .await + .unwrap_or_else(|e| panic!("{secs}: {e}")); + assert_eq!(backend.call_count(), 2); + } + + let backend = MockBackend::new(vec![ok(RUN_READY)]); + let client = client_with(backend.clone(), 0); + let result = client + .actor("a") + .start::( + None, + ActorStartOptions { + wait_for_resources: Some(WaitForResources::ForSecs(f64::NAN)), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(ApifyClientError::InvalidArgument(_)))); + assert_eq!(backend.call_count(), 0); +} + +const LOG: &str = "2024-01-01T00:00:00.000Z first\n trace\n2024-01-01T00:00:01.000Z second\n"; + +/// Serves a run, its Actor and its log. The first `empty_streams` log streams are empty, like the +/// log of a run that has not logged yet. +fn log_backend(empty_streams: usize) -> Arc { + let streams = Arc::new(Mutex::new(0usize)); + RouteBackend::new(move |request, _| { + let url = &request.url; + if url.contains("/log") { + if url.contains("stream=1") { + let mut seen = streams.lock().unwrap(); + *seen += 1; + return response(200, if *seen <= empty_streams { "" } else { LOG }); + } + return response(200, LOG); + } + if url.contains("/actors/a1") { + return response(200, r#"{"data":{"id":"a1","name":"my-actor"}}"#); + } + if request.method == HttpMethod::Post { + return response(201, RUN_READY); + } + response(200, RUN_DONE) + }) +} + +fn collecting_sink() -> (Arc>>, LogRedirect) { + let messages = Arc::new(Mutex::new(Vec::new())); + let sink = messages.clone(); + let redirect = + LogRedirect::to_sink(move |line: &str| sink.lock().unwrap().push(line.to_string())); + (messages, redirect) +} + +#[tokio::test] +async fn a_streamed_log_redirects_the_messages_of_the_log_to_the_sink() { + let backend = log_backend(0); + let client = client_on(backend.clone(), 0); + let (messages, to_log) = collecting_sink(); + let mut streamed = client + .run("r1") + .get_streamed_log(StreamedLogOptions { + to_log, + from_start: true, + }) + .await + .unwrap() + .expect("a streamed log"); + streamed.start().unwrap(); + assert!(streamed.start().is_err(), "already started"); + tokio::time::sleep(Duration::from_millis(100)).await; + streamed.stop().await.unwrap(); + assert!(streamed.stop().await.is_err(), "not started"); + + assert_eq!( + *messages.lock().unwrap(), + vec![ + "2024-01-01T00:00:00.000Z first\n trace".to_string(), + "2024-01-01T00:00:01.000Z second".to_string() + ] + ); + // A custom sink needs no names, so neither the run nor its Actor is requested. + assert_eq!(backend.count(HttpMethod::Get, "/actors/"), 0); + assert!(backend + .requests() + .iter() + .any(|r| r.url.contains("/actor-runs/r1/log?stream=1&raw=1"))); +} + +/// The default redirection names the messages after the Actor and the run. +#[tokio::test] +async fn the_default_redirection_prefixes_the_messages_with_the_actor_and_the_run() { + let client = client_on(log_backend(0), 0); + let streamed = client + .run("r1") + .get_streamed_log(Default::default()) + .await + .unwrap() + .expect("a streamed log"); + assert!( + format!("{streamed:?}").contains("my-actor runId:r1 -> "), + "{streamed:?}" + ); + + let disabled = client + .run("r1") + .get_streamed_log(StreamedLogOptions { + to_log: LogRedirect::Disabled, + from_start: true, + }) + .await + .unwrap(); + assert!(disabled.is_none()); +} + +/// The log of a run that has not logged yet is an empty stream, which is reopened until the first +/// bytes arrive. +#[tokio::test(start_paused = true)] +async fn an_empty_log_stream_is_reopened() { + let backend = log_backend(2); + let client = client_on(backend.clone(), 0); + let (messages, to_log) = collecting_sink(); + let mut streamed = client + .run("r1") + .get_streamed_log(StreamedLogOptions { + to_log, + from_start: true, + }) + .await + .unwrap() + .unwrap(); + streamed.start().unwrap(); + tokio::time::sleep(Duration::from_secs(10)).await; + streamed.stop().await.unwrap(); + + let streams = backend.count(HttpMethod::Get, "stream=1"); + assert_eq!(streams, 3); + assert_eq!(messages.lock().unwrap().len(), 2); +} + +/// Stopping before the stream delivered anything reads the whole log in one request. +#[tokio::test(start_paused = true)] +async fn stopping_before_the_first_bytes_reads_the_whole_log() { + let backend = log_backend(usize::MAX); + let client = client_on(backend.clone(), 0); + let (messages, to_log) = collecting_sink(); + let mut streamed = client + .run("r1") + .get_streamed_log(StreamedLogOptions { + to_log, + from_start: true, + }) + .await + .unwrap() + .unwrap(); + streamed.start().unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(messages.lock().unwrap().is_empty()); + streamed.stop().await.unwrap(); + assert_eq!(messages.lock().unwrap().len(), 2); + assert!(backend + .requests() + .iter() + .any(|r| r.url.contains("/log?raw=1") && !r.url.contains("stream=1"))); +} + +#[tokio::test] +async fn from_start_false_drops_the_messages_stamped_before_the_call() { + let client = client_on(log_backend(0), 0); + let (messages, to_log) = collecting_sink(); + let mut streamed = client + .run("r1") + .get_streamed_log(StreamedLogOptions { + to_log, + from_start: false, + }) + .await + .unwrap() + .unwrap(); + streamed.start().unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + streamed.stop().await.unwrap(); + assert!(messages.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn call_redirects_the_log_of_the_run_to_the_sink() { + let backend = log_backend(0); + let client = client_on(backend.clone(), 0); + let (messages, log) = collecting_sink(); + let run = client + .actor("a1") + .call::( + None, + ActorCallOptions { + log, + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + assert_eq!(messages.lock().unwrap().len(), 2); +} + +#[tokio::test] +async fn a_streamed_log_of_a_missing_run_is_none() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 404, + br#"{"error":{"type":"record-not-found","message":"m"}}"#.to_vec(), + )]); + let client = client_with(backend, 0); + assert!(client + .run("r1") + .get_streamed_log(Default::default()) + .await + .unwrap() + .is_none()); +} diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 363d27f..a45f2a5 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use std::time::Duration; use apify_client::{ - ActorStartOptions, ApifyClient, ApifyClientError, LastRunOptions, RequestCompression, - WaitForResources, + ActorCallOptions, ActorStartOptions, ApifyClient, ApifyClientError, LastRunOptions, + LogRedirect, RequestCompression, TaskStartOptions, WaitForResources, }; use common::mock_backend::{client_with, MockBackend, MockOutcome}; use futures_util::StreamExt; @@ -1422,7 +1422,7 @@ async fn task_wait_for_resources_retries_until_success() { .task("some-task") .start::( None, - ActorStartOptions { + TaskStartOptions { wait_for_resources: Some(WaitForResources::Forever), ..Default::default() }, @@ -1455,11 +1455,14 @@ async fn call_wait_for_resources_does_not_count_toward_wait_secs() { .actor("some-actor") .call::( None, - ActorStartOptions { - wait_for_resources: Some(WaitForResources::Forever), - ..Default::default() + ActorCallOptions { + start: ActorStartOptions { + wait_for_resources: Some(WaitForResources::Forever), + ..Default::default() + }, + wait_secs: Some(1), + log: LogRedirect::Disabled, }, - Some(1), ) .await .expect("the resource-limit retry must not eat into the wait_for_finish budget"); diff --git a/tests/unit_resources.rs b/tests/unit_resources.rs index a0bf55f..5135531 100644 --- a/tests/unit_resources.rs +++ b/tests/unit_resources.rs @@ -106,7 +106,12 @@ async fn a_log_is_none_for_a_client_with_an_id_and_an_error_when_chained() { assert!(is_not_found( client.run("r").log().stream().await.map(|_| ()) )); - assert!(client.run("r").get_streamed_log().await.unwrap().is_none()); + assert!(client + .run("r") + .get_streamed_log(Default::default()) + .await + .unwrap() + .is_none()); } #[tokio::test] @@ -203,7 +208,6 @@ async fn every_slash_of_an_id_becomes_a_tilde() { .last_url() .unwrap() .ends_with("/v2/actors/user~name~extra")); - assert_eq!(apify_client::common::to_safe_id("a/b/c"), "a~b~c"); } /// An id is percent-encoded, so it cannot add a path segment or a query string. From 809e9777c24c28451415b2338e5227097c4aae13 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 07:50:59 +0000 Subject: [PATCH 06/11] fix: address the sixth review of the v3 port --- CHANGELOG.md | 8 ++-- README.md | 3 +- docs/README.md | 3 +- docs/builds.md | 2 +- docs/runs.md | 4 +- src/clients/actor.rs | 26 +++++++++---- src/clients/build.rs | 4 ++ src/clients/run.rs | 10 ++++- tests/unit_call.rs | 91 +++++++++++++++++++++++++++++++++++++++++++- 9 files changed, 132 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29d5f1..ae04200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,10 +36,10 @@ Ports the `apify-client-js` v3 changes. ### Changed - Synced with Apify OpenAPI spec `v2-2026-10-29T100357Z` (`API_SPEC_VERSION`). - **Breaking:** every method has a timeout tier (5s, 30s or 360s by default) instead of one 360s - timeout; `timeout` is now an alias of `timeout_max`. `wait_for_finish` has no timeout, and a + timeout; `timeout` is now an alias of `timeout_max`. `wait_for_finish(None)` has no timeout, and a `waitForFinish` request gets the server-side wait added to its tier. - **Breaking:** `HttpRequest::timeout` may be `timeouts::NO_TIMEOUT`, which a custom `HttpBackend` - must not abort. `RetryConfig` no longer has a `timeout` field. + must not abort. - **Breaking:** any `404` resolves to `None`, not only `record-not-found`, but only for a client that names its resource by id. `run.dataset()`, `run.log()` and the other chained clients return it as an error. @@ -48,7 +48,7 @@ Ports the `apify-client-js` v3 changes. - **Breaking:** `LogClient::stream` and `stream_with_options` return an `Option`: `None` for a missing log. - **Breaking:** `RunClient::get_streamed_log` takes `StreamedLogOptions` and returns a - `StreamedLog` (`None` for a missing run), and `get_streamed_log_with_options` is removed. For the + `StreamedLog` (`None` for a missing run when redirecting by default), and `get_streamed_log_with_options` is removed. For the raw chunks of the log, use `run.log().stream_with_options(..)`. - **Breaking:** `ActorClient::call` and `call_raw` take `ActorCallOptions` (the start options, the wait budget `wait_secs` and the `log` option) instead of `ActorStartOptions` and `wait_secs`. @@ -60,7 +60,7 @@ Ports the `apify-client-js` v3 changes. `TaskClient::start` do not retry a transport timeout either. - **Breaking:** the internals `QueryParams`, `common::{encode_path_segment, build_user_agent, to_safe_id, create_hmac_signature, sign_storage_content}`, `http_client::HttpClient` and - `http_client::RetryConfig` are no longer public. + `http_client::RetryConfig` (which also lost its `timeout` field) are no longer public. - `wait_for_finish(Some(n))` gives each poll the `Medium` tier plus the hold it asks for, so a stalled connection cannot hang it. - `WaitForResources::ForSecs` with an infinite or huge value waits without a limit, and `NaN` diff --git a/README.md b/README.md index 2312e4e..b4175eb 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,8 @@ let client = ApifyClient::builder() Every method picks the timeout tier that fits its request: `Timeout::Short` for simple metadata reads and writes, `Timeout::Medium` for listing, batch and trigger calls, and `Timeout::Long` for downloads, uploads and streaming. Polling that waits for a job to finish (`wait_for_finish`) has no -client-side timeout. The duration of each tier is set on the builder, and `timeout_max` caps the +client-side timeout when it waits indefinitely (`None`); a bounded wait (`Some(n)`) gives each poll +the `Medium` tier plus the server-side hold it asks for. The duration of each tier is set on the builder, and `timeout_max` caps the timeout of a single attempt: a tier or an explicit timeout above it is lowered to it. The timeout applies to each attempt separately and doubles with every retry, up to the cap. diff --git a/docs/README.md b/docs/README.md index 8f21e1a..3f18674 100644 --- a/docs/README.md +++ b/docs/README.md @@ -176,7 +176,8 @@ reach those values the client maps Rust's native `std::env::consts::OS` spelling Every method uses a timeout tier: `Timeout::Short` for simple metadata reads and writes, `Timeout::Medium` for listing, batch and trigger calls, and `Timeout::Long` for downloads, uploads -and streaming. `wait_for_finish` polls without a client-side timeout. A tier or an explicit +and streaming. `wait_for_finish(None)` polls without a client-side timeout, and `wait_for_finish(Some(n))` gives +each poll the `Medium` tier plus the server-side hold it asks for. A tier or an explicit timeout above `timeout_max` is lowered to it, and the timeout of each attempt doubles with every retry, up to the same cap. diff --git a/docs/builds.md b/docs/builds.md index a4bd025..0e8c004 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -17,7 +17,7 @@ collections are available via `actor.builds()`. | `get()` | — | `Option` | Fetches the build. | | `abort()` | — | `Build` | Aborts the build. | | `delete()` | — | `()` | Deletes the build. | -| `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. A build ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | +| `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. `None` waits indefinitely, without a client-side timeout on the polls; `Some(n)` bounds the wait and gives each poll the `Medium` tier plus the hold it asks the API for. A build ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | | `get_openapi_definition()` | — | `serde_json::Value` | Fetches the OpenAPI definition generated for the build (raw JSON, endpoint `.../openapi.json`). A `404` is an error. | | `log()` | — | `LogClient` | Access the build's log. | diff --git a/docs/runs.md b/docs/runs.md index ceef725..5546672 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -24,9 +24,9 @@ collections are available via `actor.runs()` and `task.runs()`. | `reboot()` | — | `ActorRun` | Reboots the run's container. | | `resurrect(options)` | `RunResurrectOptions` | `ActorRun` | Resurrects a run in a terminal state (`SUCCEEDED`, `FAILED`, `ABORTED`, `TIMED-OUT`). To continue a run that reached its cost limit, pass a higher `max_total_charge_usd`; the limit covers the whole run, including the cost before the resurrection. | | `charge(options)` | `RunChargeOptions` | `()` | Charges a pay-per-event run (always sends an idempotency key). | -| `wait_for_finish(wait_secs)` | `Option` | `ActorRun` | Polls until the run is terminal. `None` waits indefinitely; `Some(n)` bounds the wait and may return a still-running (non-terminal) run if `n` elapses first. A run ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | +| `wait_for_finish(wait_secs)` | `Option` | `ActorRun` | Polls until the run is terminal. `None` waits indefinitely, without a client-side timeout on the polls; `Some(n)` bounds the wait, gives each poll the `Medium` tier plus the hold it asks the API for, and may return a still-running (non-terminal) run if `n` elapses first. A timeout set with `with_timeout` replaces both. A run ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | | `dataset()` / `key_value_store()` / `request_queue()` / `log()` | — | resource client | Access the run's default storages and log. | -| `get_streamed_log(options)` | `StreamedLogOptions { to_log, from_start }` | `Result>` (async — `.await` it) | The [redirection](#log-redirection) of the run's log; `None` if the run does not exist or `to_log` is `LogRedirect::Disabled`. | +| `get_streamed_log(options)` | `StreamedLogOptions { to_log, from_start }` | `Result>` (async — `.await` it) | The [redirection](#log-redirection) of the run's log; `None` if `to_log` is `LogRedirect::Disabled`, or, for `LogRedirect::Default`, if the run does not exist. A custom sink makes no request, so a missing run shows up as the error of `stop()`. | ### Log redirection diff --git a/src/clients/actor.rs b/src/clients/actor.rs index b03865e..ce67648 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -165,6 +165,11 @@ impl From for ActorStartOptions { } } +/// Reports a log redirection that failed, without failing the call that waits for the run. +fn warn_log_redirection(error: &crate::error::ApifyClientError) { + eprintln!("Log redirection stopped due to error: {error}"); +} + /// Options for building an Actor. #[derive(Debug, Default, Clone)] pub struct ActorBuildOptions { @@ -294,13 +299,14 @@ impl ActorClient { /// the run while it waits. /// /// See [`ActorCallOptions`] for the wait budget and the log redirection. The log goes to the - /// standard output unless `options.log` says otherwise. A redirection that cannot be set up - /// does not fail the call, and neither does one that ends early. + /// standard output unless `options.log` says otherwise. A redirection that cannot be set up, + /// or that ends early on an error, does not fail the call; a warning goes to the standard + /// error instead, like the reference client's log. /// /// Like the reference client, the start request has no client-side timeout, because an /// aborted start would be retried and could create a second run; set a timeout with the /// client's `with_timeout` to change that, and it applies to the polls as well. Without one, - /// a poll is bounded as [`RunClient::wait_for_finish`] describes. + /// the polls are bounded by `wait_secs` as [`RunClient::wait_for_finish`] describes. pub async fn call( &self, input: Option<&T>, @@ -349,14 +355,20 @@ impl ActorClient { from_start: true, }) .await - .ok() - .flatten(); + .unwrap_or_else(|error| { + warn_log_redirection(&error); + None + }); if let Some(streamed_log) = streamed_log.as_mut() { - let _ = streamed_log.start(); + if let Err(error) = streamed_log.start() { + warn_log_redirection(&error); + } } let finished = run_client.wait_for_finish(wait_secs).await; if let Some(streamed_log) = streamed_log.as_mut() { - let _ = streamed_log.stop().await; + if let Err(error) = streamed_log.stop().await { + warn_log_redirection(&error); + } } finished } diff --git a/src/clients/build.rs b/src/clients/build.rs index bf81d76..a3958e9 100644 --- a/src/clients/build.rs +++ b/src/clients/build.rs @@ -55,6 +55,10 @@ impl BuildClient { /// - `Some(n)` bounds the wait to roughly `n` seconds; if the build has not finished by /// then, the **last fetched (still non-terminal) build is returned** rather than an /// error. Check `status` / `is_terminal()` on the result when using `Some`. + /// + /// With `None`, the polls have no client-side timeout. With `Some(n)`, each poll is allowed the + /// `Medium` tier plus the hold (up to a minute) it asks the API for, so a stalled connection + /// cannot hang the wait. A timeout set with `with_timeout` is used as given instead. pub async fn wait_for_finish(&self, wait_secs: Option) -> ApifyClientResult { wait_for_finish(&self.ctx, wait_secs, |b: &Build| b.is_terminal()).await } diff --git a/src/clients/run.rs b/src/clients/run.rs index 2b21323..e5cff26 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -345,6 +345,10 @@ impl RunClient { /// - `Some(n)` bounds the wait to roughly `n` seconds; if the run has not finished by /// then, the **last fetched (still non-terminal) run is returned** rather than an /// error. Check `status` / `is_terminal()` on the result when using `Some`. + /// + /// With `None`, the polls have no client-side timeout. With `Some(n)`, each poll is allowed the + /// `Medium` tier plus the hold (up to a minute) it asks the API for, so a stalled connection + /// cannot hang the wait. A timeout set with `with_timeout` is used as given instead. pub async fn wait_for_finish(&self, wait_secs: Option) -> ApifyClientResult { wait_for_finish(&self.ctx, wait_secs, |r: &ActorRun| r.is_terminal()).await } @@ -426,8 +430,10 @@ impl RunClient { RunDatasetItemsIterator::new(self.clone(), options) } - /// Returns the redirection of the run's log, or `None` if the run does not exist or - /// `options.to_log` is [`LogRedirect::Disabled`]. Call [`StreamedLog::start`] to begin + /// Returns the redirection of the run's log, or `None` if `options.to_log` is + /// [`LogRedirect::Disabled`] or, for [`LogRedirect::Default`], the run does not exist. With a + /// [`LogRedirect::Sink`] nothing is requested, so a missing run shows up as the error of + /// [`StreamedLog::stop`]. Call [`StreamedLog::start`] to begin /// redirecting and [`StreamedLog::stop`] to end it. Mirrors the reference client's /// `getStreamedLog`. /// diff --git a/tests/unit_call.rs b/tests/unit_call.rs index 2f5bb83..4769176 100644 --- a/tests/unit_call.rs +++ b/tests/unit_call.rs @@ -9,7 +9,9 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use apify_client::clients::actor::WaitForResources; -use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, HttpResponse}; +use apify_client::http_client::{ + HttpBackend, HttpMethod, HttpRequest, HttpResponse, StreamingHttpResponse, +}; use apify_client::timeouts::NO_TIMEOUT; use apify_client::{ ActorBuildOptions, ActorCallOptions, ActorStartOptions, ApifyClient, ApifyClientError, @@ -40,6 +42,8 @@ type Route = dyn Fn(&HttpRequest, usize) -> Result>, route: Box, + /// Makes every log stream deliver its first line and then fail. + failing_log_stream: bool, } impl std::fmt::Debug for RouteBackend { @@ -58,6 +62,20 @@ impl RouteBackend { Arc::new(Self { requests: Mutex::new(Vec::new()), route: Box::new(route), + failing_log_stream: false, + }) + } + + fn with_failing_log_stream( + route: impl Fn(&HttpRequest, usize) -> Result + + Send + + Sync + + 'static, + ) -> Arc { + Arc::new(Self { + requests: Mutex::new(Vec::new()), + route: Box::new(route), + failing_log_stream: true, }) } @@ -83,6 +101,30 @@ impl HttpBackend for RouteBackend { }; (self.route)(&request, index) } + + async fn send_streaming( + &self, + request: HttpRequest, + ) -> Result { + if !(self.failing_log_stream && request.url.contains("stream=1")) { + let response = self.send(request).await?; + return Ok(StreamingHttpResponse { + status: response.status, + headers: response.headers, + body: Box::pin(futures_util::stream::once(async move { Ok(response.body) })), + }); + } + self.requests.lock().unwrap().push(request); + let chunks: Vec, ApifyClientError>> = vec![ + Ok(b"2024-01-01T00:00:00.000Z first\n".to_vec()), + Err(ApifyClientError::Http("the stream broke".to_string())), + ]; + Ok(StreamingHttpResponse { + status: 200, + headers: Default::default(), + body: Box::pin(futures_util::stream::iter(chunks)), + }) + } } fn response(status: u16, body: &str) -> Result { @@ -515,3 +557,50 @@ async fn a_streamed_log_of_a_missing_run_is_none() { .unwrap() .is_none()); } + +fn run_route(request: &HttpRequest, _: usize) -> Result { + if request.method == HttpMethod::Post { + return response(201, RUN_READY); + } + response(200, RUN_DONE) +} + +/// A redirection that ends on a stream error reports it from `stop()`, after it passed on the +/// messages it had. `call` does not fail because of it. +#[tokio::test] +async fn a_failing_log_stream_is_reported_by_stop_and_does_not_fail_call() { + let backend = RouteBackend::with_failing_log_stream(run_route); + let client = client_on(backend.clone(), 0); + let (messages, to_log) = collecting_sink(); + let mut streamed = client + .run("r1") + .get_streamed_log(StreamedLogOptions { + to_log, + from_start: true, + }) + .await + .unwrap() + .unwrap(); + streamed.start().unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + let error = streamed.stop().await.expect_err("the stream broke"); + assert!(matches!(error, ApifyClientError::Http(_)), "{error:?}"); + assert_eq!( + *messages.lock().unwrap(), + vec!["2024-01-01T00:00:00.000Z first".to_string()] + ); + + let (_, log) = collecting_sink(); + let run = client + .actor("a1") + .call::( + None, + ActorCallOptions { + log, + ..Default::default() + }, + ) + .await + .expect("a failing redirection does not fail the call"); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); +} From d440771c6cce99819a603591426f7cb964a9a091 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 07:57:47 +0000 Subject: [PATCH 07/11] fix: address the seventh review of the v3 port --- CHANGELOG.md | 13 ++++---- README.md | 9 +++--- docs/README.md | 8 ++--- docs/actors.md | 4 +-- docs/builds.md | 2 +- docs/runs.md | 3 +- src/clients/actor.rs | 24 +++++++-------- src/clients/run.rs | 5 ++-- src/clients/streamed_log.rs | 9 ++++++ tests/unit_call.rs | 60 +++++++++++++++++++++++++++++-------- 10 files changed, 91 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae04200..bf9176e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,14 +30,16 @@ Ports the `apify-client-js` v3 changes. - Log redirection like the reference client's `StreamedLog`: `StreamedLog` with `start` and `stop`, `StreamedLogOptions`, and the `LogSink` trait (`StdoutLogSink`, or any `Fn(&str)` closure) behind `LogRedirect`. `ActorClient::call` and `call_raw` redirect the log of the run to the standard - output by default, with the `ActorCallOptions::log` option to change or disable it. + output by default, with the `ActorCallOptions::log` option to change or disable it. A redirection + that ends early on an error does not fail the call: a `Log redirection stopped due to error: ..` + message goes to the sink. - `ActorCallOptions` and `TaskStartOptions`. ### Changed - Synced with Apify OpenAPI spec `v2-2026-10-29T100357Z` (`API_SPEC_VERSION`). - **Breaking:** every method has a timeout tier (5s, 30s or 360s by default) instead of one 360s - timeout; `timeout` is now an alias of `timeout_max`. `wait_for_finish(None)` has no timeout, and a - `waitForFinish` request gets the server-side wait added to its tier. + timeout; `timeout` is now an alias of `timeout_max`. `wait_for_finish(None)` has no timeout, and + a `waitForFinish` request gets the server-side wait added to its tier. - **Breaking:** `HttpRequest::timeout` may be `timeouts::NO_TIMEOUT`, which a custom `HttpBackend` must not abort. - **Breaking:** any `404` resolves to `None`, not only `record-not-found`, but only for a client that @@ -48,8 +50,9 @@ Ports the `apify-client-js` v3 changes. - **Breaking:** `LogClient::stream` and `stream_with_options` return an `Option`: `None` for a missing log. - **Breaking:** `RunClient::get_streamed_log` takes `StreamedLogOptions` and returns a - `StreamedLog` (`None` for a missing run when redirecting by default), and `get_streamed_log_with_options` is removed. For the - raw chunks of the log, use `run.log().stream_with_options(..)`. + `StreamedLog` (`None` for a missing run when redirecting by default), and + `get_streamed_log_with_options` is removed. For the raw chunks of the log, use + `run.log().stream_with_options(..)`. - **Breaking:** `ActorClient::call` and `call_raw` take `ActorCallOptions` (the start options, the wait budget `wait_secs` and the `log` option) instead of `ActorStartOptions` and `wait_secs`. `TaskClient::start` and `call` take `TaskStartOptions`, which has no `content_type` and no diff --git a/README.md b/README.md index b4175eb..78a146b 100644 --- a/README.md +++ b/README.md @@ -117,10 +117,11 @@ let client = ApifyClient::builder() Every method picks the timeout tier that fits its request: `Timeout::Short` for simple metadata reads and writes, `Timeout::Medium` for listing, batch and trigger calls, and `Timeout::Long` for downloads, uploads and streaming. Polling that waits for a job to finish (`wait_for_finish`) has no -client-side timeout when it waits indefinitely (`None`); a bounded wait (`Some(n)`) gives each poll -the `Medium` tier plus the server-side hold it asks for. The duration of each tier is set on the builder, and `timeout_max` caps the -timeout of a single attempt: a tier or an explicit timeout above it is lowered to it. The timeout -applies to each attempt separately and doubles with every retry, up to the cap. +client-side timeout when it waits indefinitely (`None`); a bounded wait (`Some(n)`) gives each +poll the `Medium` tier plus the server-side hold it asks for. The duration of each tier is set on +the builder, and `timeout_max` caps the timeout of a single attempt: a tier or an explicit timeout +above it is lowered to it. The timeout applies to each attempt separately and doubles with every +retry, up to the cap. A resource client's `with_timeout` replaces the tier of every request that client sends, with a tier, an exact duration, or `Timeout::NoTimeout` to let a request run for as long as it takes. diff --git a/docs/README.md b/docs/README.md index 3f18674..fe3aabf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -176,10 +176,10 @@ reach those values the client maps Rust's native `std::env::consts::OS` spelling Every method uses a timeout tier: `Timeout::Short` for simple metadata reads and writes, `Timeout::Medium` for listing, batch and trigger calls, and `Timeout::Long` for downloads, uploads -and streaming. `wait_for_finish(None)` polls without a client-side timeout, and `wait_for_finish(Some(n))` gives -each poll the `Medium` tier plus the server-side hold it asks for. A tier or an explicit -timeout above `timeout_max` is lowered to it, and the timeout of each attempt doubles with every -retry, up to the same cap. +and streaming. `wait_for_finish(None)` polls without a client-side timeout, and +`wait_for_finish(Some(n))` gives each poll the `Medium` tier plus the server-side hold it asks for. +A tier or an explicit timeout above `timeout_max` is lowered to it, and the timeout of each +attempt doubles with every retry, up to the same cap. Every resource client has a `with_timeout(Timeout)` method that replaces the tier of all the requests it sends with a tier, `Timeout::Custom(duration)`, or `Timeout::NoTimeout` (the client diff --git a/docs/actors.md b/docs/actors.md index f301dd6..4dbc2ab 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -20,7 +20,7 @@ be an Actor ID or a `username~name` (or `username/name`) reference. | `delete()` | — | `()` | Deletes the Actor. | | `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run, returns immediately. | | `start_raw(input, options)` | `&[u8]`, `ActorStartOptions` | `ActorRun` | Like `start`, but sends `input` as a raw request body (no JSON serialization) — for a non-JSON input such as a ZIP archive, paired with `options.content_type`. | -| `call(input, options)` | `Option<&impl Serialize>`, `ActorCallOptions` | `ActorRun` | Starts a run and waits for it to finish, redirecting its log to the standard output unless `options.log` says otherwise. | +| `call(input, options)` | `Option<&impl Serialize>`, `ActorCallOptions` | `ActorRun` | Starts a run and waits for it to finish, redirecting its log to the standard output unless `options.log` says otherwise. A redirection that fails does not fail the call: the error goes to the log sink as a `Log redirection stopped due to error: ..` message. | | `call_raw(input, options)` | `&[u8]`, `ActorCallOptions` | `ActorRun` | `start_raw` followed by `wait_for_finish`, like `call`. | | `build(version, options)` | `&str`, `ActorBuildOptions` | `Build` | Builds a version of the Actor. | | `default_build(wait_for_finish)` | `Option` | `BuildClient` | Resolves the Actor's default build, optionally waiting up to `wait_for_finish` seconds. | @@ -61,7 +61,7 @@ documents neither. |---|---|---| | `start` | `ActorStartOptions` | The options of the run start. | | `wait_secs` | `Option` | The client-side polling budget, see below. | -| `log` | `LogRedirect` | Where to redirect the run's log while waiting: `LogRedirect::Default` (the default) prints it to the standard output prefixed with `"{Actor name} runId:{run id} -> "`, `LogRedirect::Disabled` turns it off, and `LogRedirect::to_sink(..)` sends it to a [`LogSink`](runs.md#log-redirection) of your own. | +| `log` | `LogRedirect` | Where to redirect the run's log while waiting: `LogRedirect::Default` (the default) prints it to the standard output prefixed with `"{Actor name} runId:{run id} -> "`, `LogRedirect::Disabled` turns it off, and `LogRedirect::to_sink(..)` sends it to a [`LogSink`](runs.md#log-redirection) of your own. A redirection that ends early on an error does not fail the call; a `Log redirection stopped due to error: ..` message goes to the sink. | `call` starts the run without a client-side timeout, because a start that is aborted and retried could create a second run. A timeout set with the client's `with_timeout` replaces that for the diff --git a/docs/builds.md b/docs/builds.md index 0e8c004..d1e9f43 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -17,7 +17,7 @@ collections are available via `actor.builds()`. | `get()` | — | `Option` | Fetches the build. | | `abort()` | — | `Build` | Aborts the build. | | `delete()` | — | `()` | Deletes the build. | -| `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. `None` waits indefinitely, without a client-side timeout on the polls; `Some(n)` bounds the wait and gives each poll the `Medium` tier plus the hold it asks the API for. A build ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | +| `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. `None` waits indefinitely, without a client-side timeout on the polls; `Some(n)` bounds the wait and gives each poll the `Medium` tier plus the hold it asks the API for. A timeout set with `with_timeout` replaces both. A build ID that keeps 404ing past a 3-second grace period (reset by any successful fetch) returns the `404` as an error instead of polling forever. | | `get_openapi_definition()` | — | `serde_json::Value` | Fetches the OpenAPI definition generated for the build (raw JSON, endpoint `.../openapi.json`). A `404` is an error. | | `log()` | — | `LogClient` | Access the build's log. | diff --git a/docs/runs.md b/docs/runs.md index 5546672..06a0268 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -44,7 +44,8 @@ stays in one piece, and each message goes to a `LogSink`: `from_start: false` drops the messages stamped before the call. A run that has not logged yet is followed by reopening its empty log stream until the first bytes arrive. `ActorClient::call` does -all of this for the run it waits for. To read the log as raw chunks instead, use +all of this for the run it waits for; when the redirection ends early on an error, it passes a +`Log redirection stopped due to error: ..` message to the sink instead of failing the call. To read the log as raw chunks instead, use `run.log().stream_with_options(..)`; the [`raw_log`](../examples/raw_log.rs) example does. ```rust,no_run diff --git a/src/clients/actor.rs b/src/clients/actor.rs index ce67648..bdea54b 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -115,7 +115,9 @@ pub struct ActorCallOptions { /// it, since that budget only starts once the run has actually started. pub wait_secs: Option, /// Where to redirect the log of the run while it is waited for. Redirects to the standard - /// output by default; [`LogRedirect::Disabled`] turns that off. + /// output by default; [`LogRedirect::Disabled`] turns that off. A redirection that ends early + /// on an error does not fail the call: a `Log redirection stopped due to error: ..` message + /// goes to the sink. pub log: LogRedirect, } @@ -165,11 +167,6 @@ impl From for ActorStartOptions { } } -/// Reports a log redirection that failed, without failing the call that waits for the run. -fn warn_log_redirection(error: &crate::error::ApifyClientError) { - eprintln!("Log redirection stopped due to error: {error}"); -} - /// Options for building an Actor. #[derive(Debug, Default, Clone)] pub struct ActorBuildOptions { @@ -299,14 +296,15 @@ impl ActorClient { /// the run while it waits. /// /// See [`ActorCallOptions`] for the wait budget and the log redirection. The log goes to the - /// standard output unless `options.log` says otherwise. A redirection that cannot be set up, - /// or that ends early on an error, does not fail the call; a warning goes to the standard - /// error instead, like the reference client's log. + /// standard output unless `options.log` says otherwise. A redirection that ends early on an + /// error does not fail the call; a `Log redirection stopped due to error: ..` message goes + /// to the log sink instead, like the reference client's warning. One that cannot be set up is + /// reported on the standard error. /// /// Like the reference client, the start request has no client-side timeout, because an /// aborted start would be retried and could create a second run; set a timeout with the /// client's `with_timeout` to change that, and it applies to the polls as well. Without one, - /// the polls are bounded by `wait_secs` as [`RunClient::wait_for_finish`] describes. + /// the polls are timed as [`RunClient::wait_for_finish`] describes. pub async fn call( &self, input: Option<&T>, @@ -356,18 +354,18 @@ impl ActorClient { }) .await .unwrap_or_else(|error| { - warn_log_redirection(&error); + eprintln!("Log redirection could not start: {error}"); None }); if let Some(streamed_log) = streamed_log.as_mut() { if let Err(error) = streamed_log.start() { - warn_log_redirection(&error); + streamed_log.warn(&error); } } let finished = run_client.wait_for_finish(wait_secs).await; if let Some(streamed_log) = streamed_log.as_mut() { if let Err(error) = streamed_log.stop().await { - warn_log_redirection(&error); + streamed_log.warn(&error); } } finished diff --git a/src/clients/run.rs b/src/clients/run.rs index e5cff26..4479b57 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -433,9 +433,8 @@ impl RunClient { /// Returns the redirection of the run's log, or `None` if `options.to_log` is /// [`LogRedirect::Disabled`] or, for [`LogRedirect::Default`], the run does not exist. With a /// [`LogRedirect::Sink`] nothing is requested, so a missing run shows up as the error of - /// [`StreamedLog::stop`]. Call [`StreamedLog::start`] to begin - /// redirecting and [`StreamedLog::stop`] to end it. Mirrors the reference client's - /// `getStreamedLog`. + /// [`StreamedLog::stop`]. Call [`StreamedLog::start`] to begin redirecting and + /// [`StreamedLog::stop`] to end it. Mirrors the reference client's `getStreamedLog`. /// /// With [`LogRedirect::Default`], the messages go to the standard output prefixed with /// `"{actor name} runId:{run id} -> "`, which costs a request for the run and one for its diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs index d122bf5..c8a28ba 100644 --- a/src/clients/streamed_log.rs +++ b/src/clients/streamed_log.rs @@ -180,6 +180,15 @@ impl StreamedLog { } } + /// Reports a failure of the redirection to the sink of the log, so the log does not just stop + /// without a trace. + pub(crate) fn warn(&self, error: &ApifyClientError) { + self.sink.line(&format!( + "{}Log redirection stopped due to error: {error}", + self.prefix + )); + } + /// Starts redirecting the log, in a task spawned on the current Tokio runtime. Fails with /// [`ApifyClientError::InvalidArgument`] when the redirection is already started. pub fn start(&mut self) -> ApifyClientResult<()> { diff --git a/tests/unit_call.rs b/tests/unit_call.rs index 4769176..b7b102b 100644 --- a/tests/unit_call.rs +++ b/tests/unit_call.rs @@ -5,6 +5,7 @@ mod common; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -42,8 +43,8 @@ type Route = dyn Fn(&HttpRequest, usize) -> Result>, route: Box, - /// Makes every log stream deliver its first line and then fail. - failing_log_stream: bool, + /// Makes every log stream deliver its first line and then fail, and raises the flag. + failing_log_stream: Option>, } impl std::fmt::Debug for RouteBackend { @@ -62,11 +63,12 @@ impl RouteBackend { Arc::new(Self { requests: Mutex::new(Vec::new()), route: Box::new(route), - failing_log_stream: false, + failing_log_stream: None, }) } fn with_failing_log_stream( + stream_failed: Arc, route: impl Fn(&HttpRequest, usize) -> Result + Send + Sync @@ -75,7 +77,7 @@ impl RouteBackend { Arc::new(Self { requests: Mutex::new(Vec::new()), route: Box::new(route), - failing_log_stream: true, + failing_log_stream: Some(stream_failed), }) } @@ -106,15 +108,20 @@ impl HttpBackend for RouteBackend { &self, request: HttpRequest, ) -> Result { - if !(self.failing_log_stream && request.url.contains("stream=1")) { + let failing = self + .failing_log_stream + .as_ref() + .filter(|_| request.url.contains("stream=1")); + let Some(stream_failed) = failing else { let response = self.send(request).await?; return Ok(StreamingHttpResponse { status: response.status, headers: response.headers, body: Box::pin(futures_util::stream::once(async move { Ok(response.body) })), }); - } + }; self.requests.lock().unwrap().push(request); + stream_failed.store(true, Ordering::SeqCst); let chunks: Vec, ApifyClientError>> = vec![ Ok(b"2024-01-01T00:00:00.000Z first\n".to_vec()), Err(ApifyClientError::Http("the stream broke".to_string())), @@ -558,18 +565,31 @@ async fn a_streamed_log_of_a_missing_run_is_none() { .is_none()); } -fn run_route(request: &HttpRequest, _: usize) -> Result { - if request.method == HttpMethod::Post { - return response(201, RUN_READY); +/// A run that stays unfinished until the log stream was opened (and has failed). +fn run_finishing_after( + stream_failed: Arc, +) -> impl Fn(&HttpRequest, usize) -> Result { + move |request, _| { + if request.method == HttpMethod::Post { + return response(201, RUN_READY); + } + if stream_failed.load(Ordering::SeqCst) { + response(200, RUN_DONE) + } else { + response(200, RUN_READY) + } } - response(200, RUN_DONE) } /// A redirection that ends on a stream error reports it from `stop()`, after it passed on the -/// messages it had. `call` does not fail because of it. +/// messages it had. `call` does not fail because of it, and tells the sink. #[tokio::test] async fn a_failing_log_stream_is_reported_by_stop_and_does_not_fail_call() { - let backend = RouteBackend::with_failing_log_stream(run_route); + let stream_failed = Arc::new(AtomicBool::new(false)); + let backend = RouteBackend::with_failing_log_stream( + stream_failed.clone(), + run_finishing_after(stream_failed.clone()), + ); let client = client_on(backend.clone(), 0); let (messages, to_log) = collecting_sink(); let mut streamed = client @@ -590,7 +610,9 @@ async fn a_failing_log_stream_is_reported_by_stop_and_does_not_fail_call() { vec!["2024-01-01T00:00:00.000Z first".to_string()] ); - let (_, log) = collecting_sink(); + stream_failed.store(false, Ordering::SeqCst); + let streams_before = backend.count(HttpMethod::Get, "stream=1"); + let (messages, log) = collecting_sink(); let run = client .actor("a1") .call::( @@ -603,4 +625,16 @@ async fn a_failing_log_stream_is_reported_by_stop_and_does_not_fail_call() { .await .expect("a failing redirection does not fail the call"); assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + assert_eq!( + backend.count(HttpMethod::Get, "stream=1"), + streams_before + 1 + ); + assert_eq!( + *messages.lock().unwrap(), + vec![ + "2024-01-01T00:00:00.000Z first".to_string(), + "Log redirection stopped due to error: HTTP transport error: the stream broke" + .to_string() + ] + ); } From ff97ee9d00461d0ba91e68016d2d74ce6cc2de16 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:05:20 +0000 Subject: [PATCH 08/11] fix: address the eighth review of the v3 port --- docs/actors.md | 2 +- docs/runs.md | 5 +++-- src/clients/actor.rs | 9 ++++----- src/clients/streamed_log.rs | 10 ++++------ tests/unit_call.rs | 15 +++++++++------ 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/actors.md b/docs/actors.md index 4dbc2ab..242ca74 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -20,7 +20,7 @@ be an Actor ID or a `username~name` (or `username/name`) reference. | `delete()` | — | `()` | Deletes the Actor. | | `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run, returns immediately. | | `start_raw(input, options)` | `&[u8]`, `ActorStartOptions` | `ActorRun` | Like `start`, but sends `input` as a raw request body (no JSON serialization) — for a non-JSON input such as a ZIP archive, paired with `options.content_type`. | -| `call(input, options)` | `Option<&impl Serialize>`, `ActorCallOptions` | `ActorRun` | Starts a run and waits for it to finish, redirecting its log to the standard output unless `options.log` says otherwise. A redirection that fails does not fail the call: the error goes to the log sink as a `Log redirection stopped due to error: ..` message. | +| `call(input, options)` | `Option<&impl Serialize>`, `ActorCallOptions` | `ActorRun` | Starts a run and waits for it to finish, redirecting its log to the standard output unless `options.log` says otherwise. A redirection that ends early on an error does not fail the call; the error goes to the log sink as a `Log redirection stopped due to error: ..` message. | | `call_raw(input, options)` | `&[u8]`, `ActorCallOptions` | `ActorRun` | `start_raw` followed by `wait_for_finish`, like `call`. | | `build(version, options)` | `&str`, `ActorBuildOptions` | `Build` | Builds a version of the Actor. | | `default_build(wait_for_finish)` | `Option` | `BuildClient` | Resolves the Actor's default build, optionally waiting up to `wait_for_finish` seconds. | diff --git a/docs/runs.md b/docs/runs.md index 06a0268..7bb5a0a 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -45,8 +45,9 @@ stays in one piece, and each message goes to a `LogSink`: `from_start: false` drops the messages stamped before the call. A run that has not logged yet is followed by reopening its empty log stream until the first bytes arrive. `ActorClient::call` does all of this for the run it waits for; when the redirection ends early on an error, it passes a -`Log redirection stopped due to error: ..` message to the sink instead of failing the call. To read the log as raw chunks instead, use -`run.log().stream_with_options(..)`; the [`raw_log`](../examples/raw_log.rs) example does. +`Log redirection stopped due to error: ..` message to the sink instead of failing the call. To +read the log as raw chunks instead, use `run.log().stream_with_options(..)`; the +[`raw_log`](../examples/raw_log.rs) example does. ```rust,no_run use apify_client::{ApifyClient, LogRedirect, StreamedLogOptions}; diff --git a/src/clients/actor.rs b/src/clients/actor.rs index bdea54b..cec5408 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -298,8 +298,8 @@ impl ActorClient { /// See [`ActorCallOptions`] for the wait budget and the log redirection. The log goes to the /// standard output unless `options.log` says otherwise. A redirection that ends early on an /// error does not fail the call; a `Log redirection stopped due to error: ..` message goes - /// to the log sink instead, like the reference client's warning. One that cannot be set up is - /// reported on the standard error. + /// to the log sink instead (without the prefix). One that cannot be set up is reported on the + /// standard error. /// /// Like the reference client, the start request has no client-side timeout, because an /// aborted start would be retried and could create a second run; set a timeout with the @@ -358,9 +358,8 @@ impl ActorClient { None }); if let Some(streamed_log) = streamed_log.as_mut() { - if let Err(error) = streamed_log.start() { - streamed_log.warn(&error); - } + // A redirection that was just created is not started yet, so this cannot fail. + let _ = streamed_log.start(); } let finished = run_client.wait_for_finish(wait_secs).await; if let Some(streamed_log) = streamed_log.as_mut() { diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs index c8a28ba..a619b36 100644 --- a/src/clients/streamed_log.rs +++ b/src/clients/streamed_log.rs @@ -180,13 +180,11 @@ impl StreamedLog { } } - /// Reports a failure of the redirection to the sink of the log, so the log does not just stop - /// without a trace. + /// Reports a failure of the redirection to the sink of the log, without the prefix, so the log + /// does not just stop without a trace. pub(crate) fn warn(&self, error: &ApifyClientError) { - self.sink.line(&format!( - "{}Log redirection stopped due to error: {error}", - self.prefix - )); + self.sink + .line(&format!("Log redirection stopped due to error: {error}")); } /// Starts redirecting the log, in a task spawned on the current Tokio runtime. Fails with diff --git a/tests/unit_call.rs b/tests/unit_call.rs index b7b102b..297fba7 100644 --- a/tests/unit_call.rs +++ b/tests/unit_call.rs @@ -613,17 +613,20 @@ async fn a_failing_log_stream_is_reported_by_stop_and_does_not_fail_call() { stream_failed.store(false, Ordering::SeqCst); let streams_before = backend.count(HttpMethod::Get, "stream=1"); let (messages, log) = collecting_sink(); - let run = client - .actor("a1") - .call::( + // Bounded, so that a call that never opens the log stream fails here instead of polling on. + let run = tokio::time::timeout( + Duration::from_secs(10), + client.actor("a1").call::( None, ActorCallOptions { log, ..Default::default() }, - ) - .await - .expect("a failing redirection does not fail the call"); + ), + ) + .await + .expect("the call finishes once the log stream has failed") + .expect("a failing redirection does not fail the call"); assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); assert_eq!( backend.count(HttpMethod::Get, "stream=1"), From b687cf5aff335536bcfd817b8b4f87761e08875e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:11:18 +0000 Subject: [PATCH 09/11] fix: address the ninth review of the v3 port --- src/clients/actor.rs | 2 +- src/clients/streamed_log.rs | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/clients/actor.rs b/src/clients/actor.rs index cec5408..c4f64d2 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -298,7 +298,7 @@ impl ActorClient { /// See [`ActorCallOptions`] for the wait budget and the log redirection. The log goes to the /// standard output unless `options.log` says otherwise. A redirection that ends early on an /// error does not fail the call; a `Log redirection stopped due to error: ..` message goes - /// to the log sink instead (without the prefix). One that cannot be set up is reported on the + /// to the log sink instead (without the `{actor} runId:{id} -> ` prefix of the log). One that cannot be set up is reported on the /// standard error. /// /// Like the reference client, the start request has no client-side timeout, because an diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs index a619b36..1e89206 100644 --- a/src/clients/streamed_log.rs +++ b/src/clients/streamed_log.rs @@ -36,8 +36,10 @@ const TIMESTAMP_SHAPE: &[u8; TIMESTAMP_LEN] = b"dddd-dd-ddTdd:dd:dd.dddZ"; /// A destination of redirected log messages, such as the terminal or a logging framework. /// -/// Every message arrives complete (a multi-line message, such as a stack trace, in one piece) -/// and with the prefix of the redirection already added. A closure taking `&str` is a sink too: +/// Every message of the log arrives complete (a multi-line message, such as a stack trace, in one +/// piece) and with the prefix of the redirection already added. The one message that does not +/// carry the prefix is the `Log redirection stopped due to error: ..` notice of a redirection that +/// ended early. A closure taking `&str` is a sink too: /// /// ``` /// use apify_client::clients::streamed_log::{LogRedirect, LogSink}; @@ -433,6 +435,23 @@ mod tests { out.into_inner() } + #[test] + fn the_notice_of_a_failed_redirection_carries_no_prefix() { + let lines = Arc::new(std::sync::Mutex::new(Vec::new())); + let sink = lines.clone(); + let streamed = StreamedLog::new( + crate::ApifyClient::new("token").log("run"), + Arc::new(move |line: &str| sink.lock().unwrap().push(line.to_string())), + "actor runId:run -> ".to_string(), + true, + ); + streamed.warn(&ApifyClientError::Http("it broke".to_string())); + assert_eq!( + *lines.lock().unwrap(), + vec!["Log redirection stopped due to error: HTTP transport error: it broke"] + ); + } + #[test] fn multi_line_messages_stay_together_and_the_last_line_waits_for_its_line_break() { let log = "2024-01-01T00:00:00.000Z first\n at trace\n2024-01-01T00:00:01.000Z second\n2024-01-01T00:00:02.000Z third"; From c2c5cf1fff3ced5dfaf8ade2e692884334f6006f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:15:26 +0000 Subject: [PATCH 10/11] docs: address the tenth review of the v3 port --- src/clients/actor.rs | 4 ++-- src/clients/streamed_log.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/clients/actor.rs b/src/clients/actor.rs index c4f64d2..cdcb273 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -298,8 +298,8 @@ impl ActorClient { /// See [`ActorCallOptions`] for the wait budget and the log redirection. The log goes to the /// standard output unless `options.log` says otherwise. A redirection that ends early on an /// error does not fail the call; a `Log redirection stopped due to error: ..` message goes - /// to the log sink instead (without the `{actor} runId:{id} -> ` prefix of the log). One that cannot be set up is reported on the - /// standard error. + /// to the log sink instead (without the `"{actor name} runId:{run id} -> "` prefix of the + /// default redirection). One that cannot be set up is reported on the standard error. /// /// Like the reference client, the start request has no client-side timeout, because an /// aborted start would be retried and could create a second run; set a timeout with the diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs index 1e89206..a15fab6 100644 --- a/src/clients/streamed_log.rs +++ b/src/clients/streamed_log.rs @@ -37,9 +37,9 @@ const TIMESTAMP_SHAPE: &[u8; TIMESTAMP_LEN] = b"dddd-dd-ddTdd:dd:dd.dddZ"; /// A destination of redirected log messages, such as the terminal or a logging framework. /// /// Every message of the log arrives complete (a multi-line message, such as a stack trace, in one -/// piece) and with the prefix of the redirection already added. The one message that does not -/// carry the prefix is the `Log redirection stopped due to error: ..` notice of a redirection that -/// ended early. A closure taking `&str` is a sink too: +/// piece). With [`LogRedirect::Default`] the prefix is added to it, while a sink of your own gets +/// the message as it is. The `Log redirection stopped due to error: ..` notice of a redirection +/// that ended early never carries a prefix. A closure taking `&str` is a sink too: /// /// ``` /// use apify_client::clients::streamed_log::{LogRedirect, LogSink}; From b28662d041706b96d237f9d8a75e5e27ca2687ec Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:18:17 +0000 Subject: [PATCH 11/11] docs: fix the module doc of the log redirection --- src/clients/streamed_log.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs index a15fab6..d787ea4 100644 --- a/src/clients/streamed_log.rs +++ b/src/clients/streamed_log.rs @@ -2,8 +2,8 @@ //! //! [`StreamedLog`] is the equivalent of the reference client's `StreamedLog`. It follows the log //! of a run in a background task, splits it into messages at the timestamps the Actor's log lines -//! start with, and hands every message, prefixed, to a [`LogSink`]. [`ActorClient::call`] uses it -//! to redirect the log of the run it waits for. +//! start with, and hands every message to a [`LogSink`]. [`ActorClient::call`] uses it to redirect +//! the log of the run it waits for. //! //! [`ActorClient::call`]: crate::clients::actor::ActorClient::call