From 15002609a6c6202845d3abd09474a83d5b1c9388 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:11:01 +0000 Subject: [PATCH 1/6] feat: add zstd request compression and parent_request_id for Server Actor requests Syncs with Apify OpenAPI spec v2-2026-11-09T143926Z. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- CHANGELOG.md | 15 +++++++ Cargo.toml | 3 +- README.md | 10 +++-- docs/README.md | 4 +- docs/actors.md | 1 + docs/storages.md | 2 +- src/client.rs | 9 ++-- src/clients/actor_server.rs | 11 +++++ src/clients/key_value_store.rs | 6 +-- src/http_client.rs | 3 +- src/http_compressors.rs | 81 ++++++++++++++++++++++++++++++++-- src/lib.rs | 1 + src/version.rs | 2 +- tests/key_value_store.rs | 43 ++++++++++++++++++ tests/unit_http.rs | 26 +++++++++++ tests/unit_server.rs | 25 +++++++++++ tests/unit_timeouts.rs | 21 ++++++++- 17 files changed, 239 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7388112..744bceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to the Rust Apify API client are documented here. The format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.17.0] - 2026-11-10 + +### Added +- `ZstdHttpCompressor` (level 1 to 19, default 3) and `RequestCompression::Zstd`, which send request + bodies with `Content-Encoding: zstd`. +- `ActorServerRequestOptions::parent_request_id`, sent as the `X-Actor-Request-ID` header so a + Server Actor that calls another one passes on the pricing tier of the request it handles. + +### Changed +- Synced with Apify OpenAPI spec `v2-2026-11-09T143926Z` (`API_SPEC_VERSION`), which lists `zstd` + among the `Content-Encoding` values and describes `actorPath` on Actor versions. +- **Breaking:** `ActorServerRequestOptions` has the new field `parent_request_id`, so a struct + literal without `..Default::default()` no longer compiles. +- Removed comments and documentation that repeated the code or described how a change was made. + ## [0.16.0] - 2026-11-06 Ports the `apify-client-js` Server Actor support and the `count` of dataset item pages. diff --git a/Cargo.toml b/Cargo.toml index 7a65a6e..0aef697 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apify-client" -version = "0.16.0" +version = "0.17.0" authors = ["Apify Technologies "] description = "Rust client for the Apify API (https://apify.com)." license = "Apache-2.0" @@ -27,6 +27,7 @@ hmac = "0.12" sha2 = "0.10" brotli = "7" flate2 = "1" +zstd = "0.13" # Minimum-version floors for the TLS trust stack behind the `rustls` feature. The crates are not # used directly: `reqwest` pulls them in, and the floor here makes Cargo resolve at least the # releases that no longer trust the compromised intermediate CA certificates. diff --git a/README.md b/README.md index 4688687..0d419d0 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ let client = ApifyClient::builder() .timeout_max(Duration::from_secs(360)) // default: 360s, cap on a single attempt .user_agent_suffix("my-app/1.0") .default_header("X-My-Header", "value") // sent with every request - .request_compression(RequestCompression::Brotli) // default: Brotli; or RequestCompression::Gzip + .request_compression(RequestCompression::Brotli) // default: Brotli; or Gzip, Zstd .build(); ``` @@ -158,12 +158,14 @@ let items = client Request bodies of at least 1024 bytes are compressed once (before retries) and sent with the matching `Content-Encoding` header. The default is Brotli (`br`); select -`RequestCompression::Gzip` (`gzip`) for environments or intermediaries that do not handle brotli. +`RequestCompression::Gzip` (`gzip`) for environments or intermediaries that do not handle brotli, +or `RequestCompression::Zstd` (`zstd`) for fast compression at a good ratio. `RequestCompression` is re-exported at the crate root (`use apify_client::RequestCompression;`). To tune the quality or use another algorithm, pass an `HttpCompressor` to `ApifyClientBuilder::http_compressor`: the built-in -`BrotliHttpCompressor` (quality 0 to 11, default 6) and `GzipHttpCompressor` (quality 1 to 9, -default 6), or your own implementation of the trait. +`BrotliHttpCompressor` (quality 0 to 11, default 6), `GzipHttpCompressor` (quality 1 to 9, +default 6) and `ZstdHttpCompressor` (level 1 to 19, default 3), or your own implementation of the +trait. ```rust,no_run use std::sync::Arc; diff --git a/docs/README.md b/docs/README.md index 0d591b2..2970db2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -147,8 +147,8 @@ Builder options: | `timeout_max(d)` | `360s` | Cap on the timeout of a single attempt. `timeout(d)` is the same option under its original name. | | `user_agent_suffix(s)` | none | Extra text appended to the `User-Agent` header. | | `default_header(name, value)` | none | Header sent with every request. It wins over the built-in `User-Agent` and `Authorization` headers; a header a request sets itself wins over it. Names are compared case-insensitively. | -| `request_compression(c)` | `Brotli` | Encoding for large request bodies: `RequestCompression::Brotli` (`br`) or `RequestCompression::Gzip` (`gzip`). | -| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11, default 6), `GzipHttpCompressor::new(quality)` (1 to 9, default 6), or your own implementation. | +| `request_compression(c)` | `Brotli` | Encoding for large request bodies: `RequestCompression::Brotli` (`br`), `Gzip` (`gzip`) or `Zstd` (`zstd`). | +| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11, default 6), `GzipHttpCompressor::new(quality)` (1 to 9, default 6), `ZstdHttpCompressor::new(level)` (1 to 19, default 3), or your own implementation. | | `http_backend(b)` | reqwest | Replaceable HTTP transport (`apify_client::http_client::{HttpBackend, ReqwestBackend}`). A request whose `timeout` is `timeouts::NO_TIMEOUT` may run for as long as it takes, so a backend must check for it before it applies a timeout of its own. A request whose `follow_redirects` is `false`, such as one to an Actor's server, carries a token that must not follow a redirect to another host, so a backend must return a redirect response as it is. | Request bodies of at least 1024 bytes are compressed once (before retries) with the selected diff --git a/docs/actors.md b/docs/actors.md index 2a1b981..f9c3566 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -257,6 +257,7 @@ the old name. | `params` | `Vec<(String, String)>` | Query parameters of the request, in order. A key can repeat. | | `headers` | `HashMap` | Additional headers of the request. They reach the Actor's server as they are, `Authorization` included. | | `body` | `Option` | `Json(Value)` (sent with `Content-Type: application/json` unless you set a content type), `Text(String)` or `Bytes(Vec)` (both sent as they are). `ActorServerBody::json(&value)` serializes a value. | +| `parent_request_id` | `Option` | The `X-Actor-Request-ID` header of the request your Server Actor is handling, sent on as the same header (replacing one in `headers`) so the called Actor charges at that request's pricing tier. Use a client created with the run's `APIFY_TOKEN`. | | `compression` | `Option` | `Algorithm(RequestCompression)` or `Custom(Arc)`. The body is sent uncompressed when it's `None`, since the Actor's server may not decode a compressed body. Set it only if the server decodes the resulting `Content-Encoding`. | ```rust,no_run diff --git a/docs/storages.md b/docs/storages.md index d9029fb..3bfba85 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -158,7 +158,7 @@ println!("streamed {total_bytes} bytes of JSONL"); | `record_exists(key)` | `&str` | `bool` | Whether a record exists (HEAD). | | `get_record(key)` | `&str` | `Option` | Reads a record's raw value. | | `set_record_raw(key, bytes, content_type)` | `&str`, `Vec`, `&str` | `()` | Stores a raw record. | -| `set_record_raw_with_content_encoding(key, bytes, content_type, content_encoding)` | `&str`, `Vec`, `&str`, `&str` | `()` | Like `set_record_raw`, for a value that is already compressed (`gzip`, `br`, `deflate`): sends `content_encoding` as the `Content-Encoding` header instead of compressing the bytes again. | +| `set_record_raw_with_content_encoding(key, bytes, content_type, content_encoding)` | `&str`, `Vec`, `&str`, `&str` | `()` | Like `set_record_raw`, for a value that is already compressed (`gzip`, `br`, `zstd`, `deflate`): sends `content_encoding` as the `Content-Encoding` header instead of compressing the bytes again. | | `set_record_json(key, value)` | `&str`, `&impl Serialize` | `()` | Stores a JSON record. | | `set_record_raw_with_options(key, bytes, content_type, options)` / `set_record_json_with_options(key, value, options)` | as above plus `SetRecordOptions { content_encoding, do_not_retry_timeouts }` | `()` | Like the methods above; `do_not_retry_timeouts` gives up on the first transport timeout instead of retrying it. `content_encoding` is for raw bytes only. | | `delete_record(key)` | `&str` | `()` | Deletes a record. | diff --git a/src/client.rs b/src/client.rs index 0f2ce85..d6f0a58 100644 --- a/src/client.rs +++ b/src/client.rs @@ -171,11 +171,10 @@ impl ApifyClientBuilder { /// Selects the built-in algorithm used to compress large request bodies (default /// [`RequestCompression::Brotli`]), at its default quality. /// - /// The client compresses request bodies of at least 1024 bytes; this chooses whether that - /// uses brotli (`Content-Encoding: br`) or gzip (`Content-Encoding: gzip`). Brotli is - /// preferred (matching the reference client); select [`RequestCompression::Gzip`] for - /// environments or intermediaries that do not handle brotli. To tune the quality or use - /// another algorithm, pass an [`HttpCompressor`] to [`http_compressor`](Self::http_compressor). + /// The client compresses request bodies of at least 1024 bytes with brotli (`br`), gzip + /// (`gzip`) or zstd (`zstd`). Select [`RequestCompression::Gzip`] for environments or + /// intermediaries that do not handle brotli. To tune the quality or use another algorithm, + /// pass an [`HttpCompressor`] to [`http_compressor`](Self::http_compressor). pub fn request_compression(mut self, compression: RequestCompression) -> Self { self.compressor = compression.into_compressor(); self diff --git a/src/clients/actor_server.rs b/src/clients/actor_server.rs index 761a77c..7dc7db5 100644 --- a/src/clients/actor_server.rs +++ b/src/clients/actor_server.rs @@ -28,6 +28,9 @@ use crate::timeouts::Timeout; /// returns one itself gets the request again. const SERVER_RETRY_STATUS_CODES: &[u16] = &[429, 503]; +/// Header that carries the ID of a request the platform forwards to a Server Actor. +const ACTOR_REQUEST_ID_HEADER: &str = "X-Actor-Request-ID"; + /// The tag of the build that serves the server interface when the Actor's `actorStandby.build` /// names none. const DEFAULT_SERVER_BUILD: &str = "latest"; @@ -72,6 +75,10 @@ pub struct ActorServerRequestOptions { pub headers: HashMap, /// The request body. pub body: Option, + /// The `X-Actor-Request-ID` header of the request your Server Actor is handling. It is sent on + /// as the same header, replacing one in `headers`, so the called Actor charges at that + /// request's pricing tier. + pub parent_request_id: Option, /// Compression of the request body. The body is sent uncompressed when it's `None`, since the /// Actor's server may not decode a compressed body. Set it only if the Actor's server decodes /// the resulting `Content-Encoding`. The body is compressed under the same conditions as for @@ -311,6 +318,10 @@ impl ActorServerClient { )); let mut headers = options.headers; + if let Some(parent_request_id) = options.parent_request_id { + headers.retain(|name, _| !name.eq_ignore_ascii_case(ACTOR_REQUEST_ID_HEADER)); + headers.insert(ACTOR_REQUEST_ID_HEADER.to_string(), parent_request_id); + } let body = match options.body { None => None, Some(ActorServerBody::Json(value)) => { diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 5db3bfa..1bf2979 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -59,8 +59,8 @@ pub struct GetRecordOptions { /// [`KeyValueStoreClient::set_record_json_with_options`]. #[derive(Debug, Default, Clone)] pub struct SetRecordOptions { - /// `Content-Encoding` of a value that is already compressed (`gzip`, `br`, `deflate` or - /// `identity`). The client then sends the bytes as they are, instead of compressing them + /// `Content-Encoding` of a value that is already compressed (`gzip`, `br`, `zstd`, `deflate` + /// or `identity`). The client then sends the bytes as they are, instead of compressing them /// again. pub content_encoding: Option, /// Give up on the first transport timeout instead of retrying it. A retry of an upload that @@ -234,7 +234,7 @@ impl KeyValueStoreClient { /// `Content-Encoding` header so it is uploaded (and later served) exactly as given, without /// the client compressing it again. /// - /// The API accepts `gzip`, `br`, `deflate`, and `identity`. Unlike the reference client's + /// The API accepts `gzip`, `br`, `zstd`, `deflate`, and `identity`. Unlike the reference client's /// `setRecord({ contentEncoding })`, there is no separate runtime check rejecting a /// non-identity encoding on a string or JSON value: this method, like /// [`set_record_raw`](Self::set_record_raw), only ever accepts bytes that are already the diff --git a/src/http_client.rs b/src/http_client.rs index f4067f0..c504c75 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -594,8 +594,7 @@ impl HttpClient { self.stats.add_call(); self.apply_headers(&mut request, options.authorization_header); - // Compress the request body once (not per attempt) when it is large enough, mirroring the - // reference client. The API accepts both brotli- and gzip-encoded request bodies. + // Compress the request body once, not per attempt, when it is large enough. self.compress_body(&mut request, &options.compression)?; let method_str = request.method.as_str().to_string(); diff --git a/src/http_compressors.rs b/src/http_compressors.rs index d669d12..14a32ea 100644 --- a/src/http_compressors.rs +++ b/src/http_compressors.rs @@ -2,8 +2,8 @@ //! //! The client compresses request bodies of at least 1024 bytes whose content type //! is not already compressed and that carry no `Content-Encoding` of their own. The algorithm is an -//! [`HttpCompressor`]: the built-in [`BrotliHttpCompressor`] (the default) and -//! [`GzipHttpCompressor`], or a custom implementation passed to +//! [`HttpCompressor`]: the built-in [`BrotliHttpCompressor`] (the default), +//! [`GzipHttpCompressor`] and [`ZstdHttpCompressor`], or a custom implementation passed to //! [`ApifyClientBuilder::http_compressor`](crate::ApifyClientBuilder::http_compressor). use std::collections::HashMap; @@ -31,6 +31,8 @@ const BROTLI_DEFAULT_QUALITY: u32 = 6; const BROTLI_WINDOW_SIZE: u32 = 22; /// Internal buffer size for the brotli encoder. const BROTLI_BUFFER_SIZE: usize = 4096; +/// `Content-Encoding` value used for zstd-compressed request bodies. +const CONTENT_ENCODING_ZSTD: &str = "zstd"; /// Lowest gzip quality, the fastest with the least compression. const GZIP_MIN_QUALITY: u32 = 1; /// Highest gzip quality, the slowest with the best compression. @@ -39,6 +41,15 @@ const GZIP_MAX_QUALITY: u32 = 9; /// client uses too. The levels above it roughly double the CPU cost for about a percent fewer /// bytes. const GZIP_DEFAULT_QUALITY: u32 = 6; +/// Lowest zstd level, the fastest with the least compression. +const ZSTD_MIN_QUALITY: u32 = 1; +/// Highest zstd level the compressor accepts. zstd itself goes up to 22, but levels 20 to 22 use a +/// window above the 8 MB that RFC 9659 requires HTTP decoders to support, so a body compressed at +/// those levels could fail to decode. +const ZSTD_MAX_QUALITY: u32 = 19; +/// Zstd level used by [`ZstdHttpCompressor::default`]: zstd's own default, which the reference +/// client uses too. +const ZSTD_DEFAULT_QUALITY: u32 = 3; /// Media-type prefixes whose payloads already carry their own compression, so running them /// through brotli/gzip burns CPU and memory for a result that is usually no smaller — and the @@ -201,9 +212,48 @@ impl HttpCompressor for GzipHttpCompressor { } } +/// Zstd compressor (`Content-Encoding: zstd`). Fast at its default level, with a ratio close to +/// brotli at the higher ones. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ZstdHttpCompressor { + quality: u32, +} + +impl ZstdHttpCompressor { + /// Creates a zstd compressor with the given level, from 1 (fastest, least compression) to 19 + /// (slowest, best compression). Returns [`ApifyClientError::InvalidArgument`] when `quality` + /// is out of range. + pub fn new(quality: u32) -> ApifyClientResult { + check_quality("zstd", quality, ZSTD_MIN_QUALITY, ZSTD_MAX_QUALITY)?; + Ok(Self { quality }) + } +} + +impl Default for ZstdHttpCompressor { + /// Zstd at level 3. + fn default() -> Self { + Self { + quality: ZSTD_DEFAULT_QUALITY, + } + } +} + +impl HttpCompressor for ZstdHttpCompressor { + fn content_encoding(&self) -> &str { + CONTENT_ENCODING_ZSTD + } + + fn compress(&self, data: &[u8]) -> ApifyClientResult> { + // The level is validated in `new` and fits `i32`. + zstd::bulk::compress(data, self.quality as i32) + .map_err(|error| ApifyClientError::Http(format!("zstd compression failed: {error}"))) + } +} + /// Built-in compression algorithm selected by name, at its default quality. A shorthand for -/// [`BrotliHttpCompressor::default`] and [`GzipHttpCompressor::default`]; use the compressor -/// structs directly to tune the quality, or implement [`HttpCompressor`] for another algorithm. +/// [`BrotliHttpCompressor::default`], [`GzipHttpCompressor::default`] and +/// [`ZstdHttpCompressor::default`]; use the compressor structs directly to tune the quality, or +/// implement [`HttpCompressor`] for another algorithm. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[non_exhaustive] pub enum RequestCompression { @@ -214,6 +264,8 @@ pub enum RequestCompression { /// Gzip (`Content-Encoding: gzip`). Choose this for environments or intermediaries that do not /// handle brotli. Gzip, + /// Zstd (`Content-Encoding: zstd`). Fast at a good ratio. + Zstd, } impl RequestCompression { @@ -222,6 +274,7 @@ impl RequestCompression { match self { RequestCompression::Brotli => std::sync::Arc::new(BrotliHttpCompressor::default()), RequestCompression::Gzip => std::sync::Arc::new(GzipHttpCompressor::default()), + RequestCompression::Zstd => std::sync::Arc::new(ZstdHttpCompressor::default()), } } } @@ -332,6 +385,10 @@ mod tests { assert!(GzipHttpCompressor::new(1).is_ok()); assert!(GzipHttpCompressor::new(0).is_err()); assert!(GzipHttpCompressor::new(10).is_err()); + assert!(ZstdHttpCompressor::new(1).is_ok()); + assert!(ZstdHttpCompressor::new(19).is_ok()); + assert!(ZstdHttpCompressor::new(0).is_err()); + assert!(ZstdHttpCompressor::new(20).is_err()); } #[test] @@ -361,6 +418,22 @@ mod tests { fn built_in_compressors_report_their_content_encoding() { assert_eq!(BrotliHttpCompressor::default().content_encoding(), "br"); assert_eq!(GzipHttpCompressor::default().content_encoding(), "gzip"); + assert_eq!(ZstdHttpCompressor::default().content_encoding(), "zstd"); + } + + #[test] + fn zstd_output_decodes_to_the_input() { + let data: Vec = (0..4096u32).map(|i| (i % 251) as u8).collect(); + for compressor in [ + ZstdHttpCompressor::default(), + ZstdHttpCompressor::new(19).unwrap(), + ] { + let compressed = compressor.compress(&data).unwrap(); + assert_eq!( + zstd::bulk::decompress(&compressed, data.len()).unwrap(), + data + ); + } } #[test] diff --git a/src/lib.rs b/src/lib.rs index 38683ce..72fe82f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -76,6 +76,7 @@ pub use common::{ListOptions, PaginationList, StorageListOptions}; pub use error::{ApiError, ApifyClientError, ApifyClientResult}; pub use http_compressors::{ BrotliHttpCompressor, GzipHttpCompressor, HttpCompressor, RequestCompression, + ZstdHttpCompressor, }; pub use statistics::Statistics; pub use timeouts::Timeout; diff --git a/src/version.rs b/src/version.rs index 3c83068..e91ab73 100644 --- a/src/version.rs +++ b/src/version.rs @@ -10,4 +10,4 @@ pub const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// and verified against. /// /// This corresponds to the `info.version` field of the Apify OpenAPI document. -pub const API_SPEC_VERSION: &str = "v2-2026-11-04T151257Z"; +pub const API_SPEC_VERSION: &str = "v2-2026-11-09T143926Z"; diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 6cc36fd..08e754e 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -348,6 +348,49 @@ async fn set_record_with_content_encoding_round_trips_without_corruption() { ); } +/// A large record uploaded by a client that compresses with zstd reads back unchanged. +#[tokio::test(flavor = "multi_thread")] +async fn set_record_compressed_with_zstd() { + use apify_client::RequestCompression; + + let client = common::make_client_builder().map(|builder| { + builder + .request_compression(RequestCompression::Zstd) + .build() + }); + let Some(client) = client else { + eprintln!("Skipping: APIFY_TOKEN is not set"); + return; + }; + let name = common::unique_name("kvs-zstd"); + let store = client + .key_value_stores() + .get_or_create(Some(&name)) + .await + .expect("create store"); + + let cleanup_client = client.clone(); + let id = store.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.key_value_store(&id).delete().await; + }); + + let store_client = client.key_value_store(&store.id); + let payload = json!({ "data": "x".repeat(4096) }); + store_client + .set_record_json("zstd-record", &payload) + .await + .expect("set zstd-compressed record"); + + let record = store_client + .get_record("zstd-record") + .await + .expect("get record") + .expect("record should exist"); + let stored: serde_json::Value = serde_json::from_slice(&record.value).expect("JSON record"); + assert_eq!(stored, payload); +} + /// Builds a record public URL and confirms it is well-formed and fetchable without auth. /// /// The URL points at the public records endpoint; when the store exposes a URL-signing diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 5f18e88..f7d0e83 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -569,6 +569,32 @@ async fn large_request_body_is_gzip_compressed_when_selected() { assert_eq!(decompressed, original, "gzip round-trip must be lossless"); } +/// With the zstd encoding selected, a large request body is sent with `Content-Encoding: zstd` and +/// decodes back to the original bytes. +#[tokio::test] +async fn large_request_body_is_zstd_compressed_when_selected() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"".to_vec())]); + let client = client_with_compression(backend.clone(), RequestCompression::Zstd); + + let original = vec![b'a'; 4096]; + client + .key_value_store("me~store") + .set_record_raw("key", original.clone(), "text/plain") + .await + .expect("ok"); + + assert_eq!( + backend.last_header("Content-Encoding").as_deref(), + Some("zstd") + ); + let sent = backend.last_body().expect("a body was sent"); + assert!(sent.len() < original.len()); + assert_eq!( + zstd::bulk::decompress(&sent, original.len()).expect("decode"), + original + ); +} + /// When `request_compression()` is never called on the builder, large bodies are brotli-compressed: /// the builder's default encoding is brotli, so the backend sees `Content-Encoding: br`. This /// exercises the builder default via the public builder path (not the `Default` derive), guarding diff --git a/tests/unit_server.rs b/tests/unit_server.rs index a7ce74c..a31f74d 100644 --- a/tests/unit_server.rs +++ b/tests/unit_server.rs @@ -155,6 +155,31 @@ async fn the_token_goes_in_the_x_apify_authorization_header_and_own_headers_reac assert_eq!(backend.follow_redirects(), vec![true, false]); } +#[tokio::test] +async fn parent_request_id_is_sent_as_the_actor_request_id_header_over_one_in_headers() { + let backend = MockBackend::new(vec![actor(json!({})), server_response(200, "ok")]); + let client = client_with(backend.clone(), 0); + + let mut options = ActorServerRequestOptions { + parent_request_id: Some("parent-id".to_string()), + ..Default::default() + }; + options + .headers + .insert("x-actor-request-id".to_string(), "stale".to_string()); + client + .actor("a") + .server() + .request("/", options) + .await + .unwrap(); + + assert_eq!( + backend.last_header("X-Actor-Request-ID").as_deref(), + Some("parent-id") + ); +} + #[tokio::test] async fn the_token_does_not_travel_in_the_authorization_header_by_default() { let backend = MockBackend::new(vec![actor(json!({})), server_response(200, "ok")]); diff --git a/tests/unit_timeouts.rs b/tests/unit_timeouts.rs index 2afe3bf..b89311c 100644 --- a/tests/unit_timeouts.rs +++ b/tests/unit_timeouts.rs @@ -14,7 +14,7 @@ use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, ReqwestBac use apify_client::timeouts::NO_TIMEOUT; use apify_client::{ ActorStartOptions, ApifyClient, ApifyClientError, BrotliHttpCompressor, GzipHttpCompressor, - HttpCompressor, Timeout, + HttpCompressor, Timeout, ZstdHttpCompressor, }; use common::mock_backend::{client_with, MockBackend, MockOutcome}; use serde_json::{json, Value}; @@ -415,6 +415,21 @@ async fn gzip_quality_is_configurable() { assert_eq!(decompressed, vec![b'a'; COMPRESSIBLE_BODY_BYTES]); } +#[tokio::test] +async fn zstd_quality_is_configurable() { + let compressor = ZstdHttpCompressor::new(19).expect("19 is a valid zstd level"); + let backend = upload_with(Arc::new(compressor)).await; + assert_eq!( + backend.last_header("Content-Encoding").as_deref(), + Some("zstd") + ); + let sent = backend.last_body().unwrap(); + assert_eq!( + zstd::bulk::decompress(&sent, COMPRESSIBLE_BODY_BYTES).expect("decode"), + vec![b'a'; COMPRESSIBLE_BODY_BYTES] + ); +} + #[test] fn a_quality_outside_the_range_of_the_algorithm_is_rejected() { assert!(matches!( @@ -425,6 +440,10 @@ fn a_quality_outside_the_range_of_the_algorithm_is_rejected() { GzipHttpCompressor::new(0), Err(ApifyClientError::InvalidArgument(_)) )); + assert!(matches!( + ZstdHttpCompressor::new(20), + Err(ApifyClientError::InvalidArgument(_)) + )); } #[tokio::test] From 9284efa3cea7fe081703e663eab6feb53776e717 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:17:53 +0000 Subject: [PATCH 2/6] docs: remove provenance and redundant comments and shorten documentation Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- CHANGELOG.md | 2 +- README.md | 52 +++----- docs/README.md | 10 +- docs/actors.md | 4 +- docs/storages.md | 14 +- examples/log_redirection.rs | 6 +- src/client.rs | 50 ++------ src/clients/actor.rs | 53 ++++---- src/clients/actor_env_var_collection.rs | 4 +- src/clients/actor_server.rs | 66 +++++----- src/clients/actor_version_collection.rs | 4 +- src/clients/base.rs | 113 +++++++---------- src/clients/dataset.rs | 51 +++----- src/clients/key_value_store.rs | 79 ++++-------- src/clients/log.rs | 16 +-- src/clients/pagination.rs | 74 ++++------- src/clients/request_queue.rs | 106 ++++++---------- src/clients/run.rs | 51 ++++---- src/clients/run_collection.rs | 11 +- src/clients/streamed_log.rs | 5 +- src/clients/task.rs | 8 +- src/common.rs | 162 +++++++++--------------- src/error.rs | 16 +-- src/http_client.rs | 93 +++++--------- src/http_compressors.rs | 40 ++---- src/lib.rs | 24 ++-- src/models.rs | 10 +- tests/common/mod.rs | 50 +++----- tests/examples.rs | 7 +- tests/key_value_store.rs | 31 ++--- tests/task.rs | 27 ++-- tests/unit_http.rs | 48 +++---- tests/user.rs | 19 +-- 33 files changed, 478 insertions(+), 828 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 744bceb..e395282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ to [Semantic Versioning](https://semver.org/). among the `Content-Encoding` values and describes `actorPath` on Actor versions. - **Breaking:** `ActorServerRequestOptions` has the new field `parent_request_id`, so a struct literal without `..Default::default()` no longer compiles. -- Removed comments and documentation that repeated the code or described how a change was made. +- Shortened code comments and documentation, and removed the references to other clients. ## [0.16.0] - 2026-11-06 diff --git a/README.md b/README.md index 0d419d0..839c4e9 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,8 @@ > relying on it in production and report issues on the repository. An idiomatic Rust client for the [Apify API](https://docs.apify.com/api/v2). -It provides a resource-oriented, async interface that mirrors the official -[JavaScript](https://github.com/apify/apify-client-js) and -[Python](https://github.com/apify/apify-client-python) clients. +It provides a resource-oriented, async interface that follows the +[JavaScript client](https://github.com/apify/apify-client-js). - Async (Tokio-friendly), built on `reqwest`. - Transparent authentication, retries with exponential backoff, and timeouts. @@ -194,10 +193,9 @@ it: `calls()` (each call counts once), `requests()` (every attempt counts) and Every request carries a `User-Agent` of the form `ApifyClient/{client_version} ({os}; Rust/{rust_version}); isAtHome/{true|false}`, plus any -`user_agent_suffix` you set. The `{os}` token matches the reference clients' `os.platform()` value -(the client maps Rust's `macos`/`windows`/`solaris`/`illumos` to `darwin`/`win32`/`sunos`; -`linux`, `android`, and the rest pass through), so it is identical across all Apify clients. -`isAtHome` reflects the `APIFY_IS_AT_HOME` environment variable. +`user_agent_suffix` you set. The `{os}` token is the Node.js `os.platform()` value (Rust's +`macos`, `windows`, `solaris` and `illumos` become `darwin`, `win32`, `sunos` and `sunos`; the +rest pass through). `isAtHome` reflects the `APIFY_IS_AT_HOME` environment variable. ## Resource clients @@ -318,23 +316,18 @@ See [`docs/`](docs) for the full API documentation. ## Scope -This client deliberately mirrors the public surface of the official JavaScript reference -client. A few documented API endpoints are intentionally **not** exposed because the -reference client does not expose them either, keeping the clients consistent: +The client has the same public surface as the JavaScript client. These documented endpoints are +not exposed, because the JavaScript client does not expose them either: - Synchronous run endpoints (`run-sync`, `run-sync-get-dataset-items`). - Cryptographic tools (`/tools/encode-and-sign`, `/tools/decode-and-verify`). - `/browser-info`. - The `POST` create-with-explicit-key variants of Actor versions (`POST /v2/actors/{actorId}/versions/{versionNumber}`) and version env-vars - (`POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}`). The reference - client creates versions/env-vars via `POST` to the collection and upserts via `PUT` on the - keyed path (both supported here as `versions().create(...)` / `version(n).update(...)` and - the env-var equivalents), so the redundant keyed-`POST` create is intentionally omitted. + (`POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}`). Create them with + `versions().create(...)` and the env-var equivalent, and update them with `version(n).update(...)`. - The `POST` store-record variant (`POST /v2/key-value-stores/{storeId}/records/{recordKey}`), - which the spec defines as behaving identically to the covered `PUT` variant. Records are - stored via `set_record_raw(...)` / `set_record_json(...)` (`PUT`), matching the reference - client; the redundant keyed-`POST` alias is intentionally omitted. + which behaves like the `PUT` variant. Use `set_record_raw(...)` or `set_record_json(...)`. If you need these, call them directly through a custom `HttpBackend` or open an issue. @@ -369,7 +362,7 @@ match client.actor("nonexistent~actor").get().await { ``` `get`/`delete` operations resolve a missing resource (any `404`) to `Ok(None)` / a successful -no-op, matching the reference clients, but only on a client that names its resource by id. A +no-op, but only on a client that names its resource by id. A client chained from a parent without an id of its own, such as `run.dataset()` or `run.log()`, returns the `404` as an error: the response cannot tell a missing run from a missing dataset. The same goes for methods that address a fixed sub-path of a resource (`get_statistics`, `get_input`, @@ -377,26 +370,21 @@ same goes for methods that address a fixed sub-path of a resource (`get_statisti `ApiError` also exposes the `is_invalid_request()` (400), `is_unauthorized()` (401), `is_forbidden()` (403), `is_not_found()` (404), `is_conflict()` (409), `is_rate_limited()` (429) and -`is_server_error()` (5xx) predicates classifying it by HTTP status — the idiomatic equivalent of the reference -clients' `NotFoundError`/`RateLimitError`/... error subclasses, since Rust has no exception -hierarchy to mirror them with. +`is_server_error()` (5xx) predicates that classify it by HTTP status. ### Cancellation -The reference JavaScript client accepts an `AbortSignal` option on most methods. Rust futures do -nothing until polled, so dropping one — e.g. via `tokio::select!`, or letting a -`tokio::time::timeout` elapse — stops the underlying request with no separate API needed; that is -the idiomatic equivalent here. +Methods take no cancellation parameter. Drop the future a method returns, for example through +`tokio::select!` or `tokio::time::timeout`, to stop the request. ## Custom HTTP transport -The transport is a replaceable component: any type implementing `http_client::HttpBackend` can be -injected with `ApifyClientBuilder::http_backend`. The most common customization is to reuse the -built-in `ReqwestBackend` with a `reqwest::Client` you pre-configure (custom proxy, TLS, connection -pool, …), as shown below; to route requests through an entirely different HTTP stack, implement -`HttpBackend` on your own type instead — just `async fn send`; `send_streaming` (used by -`log().stream()` and `download_items_stream()`) has a default implementation that buffers the -whole response via `send`, so it is optional unless you want true incremental streaming. A request +Any type that implements `http_client::HttpBackend` can be injected with +`ApifyClientBuilder::http_backend`. To customize the proxy, TLS or connection pool, wrap a +`reqwest::Client` you configure in the built-in `ReqwestBackend`, as shown below. To use another +HTTP stack, implement `HttpBackend` yourself: `send` is required, and `send_streaming` (used by +`log().stream()` and `download_items_stream()`) buffers the whole response through `send` unless +you override it for incremental streaming. A request whose `timeout` is `timeouts::NO_TIMEOUT` (see [Timeouts](#timeouts)) may run for as long as it takes, so a backend must check for it before it applies a timeout of its own. A request whose `follow_redirects` is `false`, such as one to an Actor's server, carries a token that must not diff --git a/docs/README.md b/docs/README.md index 2970db2..870b1a5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -166,12 +166,10 @@ let client = ApifyClient::builder() The `User-Agent` header has the form `ApifyClient/{client_version} ({os}; Rust/{rust_version}); isAtHome/{true|false}` where the `{os}` -token matches the reference client's `os.platform()` value (e.g. `darwin`, `win32`, `linux`) so it -is identical across all Apify clients, and `isAtHome` reflects the `APIFY_IS_AT_HOME` environment -variable (the platform variable the reference clients read; rendered lowercase to match them). To -reach those values the client maps Rust's native `std::env::consts::OS` spellings to Node's -(`macos` → `darwin`, `windows` → `win32`, `solaris`/`illumos` → `sunos`); all other tokens -(`linux`, `android`, `freebsd`, …) are already identical and pass through unchanged. +token is the Node.js `os.platform()` value (e.g. `darwin`, `win32`, `linux`), and `isAtHome` is +`true` or `false` according to the `APIFY_IS_AT_HOME` environment variable. The client maps Rust's +`macos`, `windows`, `solaris` and `illumos` to `darwin`, `win32`, `sunos` and `sunos`; other tokens +pass through unchanged. ### Timeouts diff --git a/docs/actors.md b/docs/actors.md index f9c3566..fb2a413 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -54,7 +54,7 @@ documents neither. | `content_type` | `Option` | Content type of the input body. Defaults to `application/json` for `start`/`call`, or `application/octet-stream` for `start_raw`/`call_raw`, when unset. | | `restart_on_error` | `Option` | Whether to restart the run if it fails. | | `force_permission_level` | `Option` | Override the Actor's permission level for this run. | -| `webhooks` | `Option>` | Ad-hoc webhooks to attach to this run. Encoded as base64 JSON in the `webhooks` query parameter, matching the reference clients. | +| `webhooks` | `Option>` | Ad-hoc webhooks to attach to this run. Encoded as base64 JSON in the `webhooks` query parameter. | | `wait_for_resources` | `Option` | Retry the start while the account lacks the memory or a concurrent-run slot for it (`ApiError::error_type` of `actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`). See below. | ### `ActorCallOptions` @@ -224,7 +224,7 @@ if let Some(actor) = client.actor("apify~hello-world").get().await? { | `image_digest` | `Option` | Digest of the built Docker image manifest (without the `sha256:` prefix); compare two builds' digests to see if their image contents differ. `None` if not yet available. | | `extra` | `Extra` | Any other fields returned by the API. | -`Build::is_terminal()` reports whether `status` is a terminal value, mirroring `ActorRun`. +`Build::is_terminal()` reports whether `status` is a terminal value. ```rust,no_run # use apify_client::ApifyClient; diff --git a/docs/storages.md b/docs/storages.md index 3bfba85..af7f036 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -233,11 +233,10 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: | `delete_request_lock(id, forefront)` | `&str`, `bool` | `()` | Release a lock. | | `unlock_requests()` | — | `UnlockRequestsResult` | Release all this client's locks. | -`paginate_requests(page_limit)` returns a `RequestQueueRequestsIterator` — a lazy, page-fetching -iterator (parity with the Store iterator in [Store, users and logs](misc.md#apify-store--clientstore)). -It is named `paginate_requests` (rather than an `iterate_*` verb like the dataset/key-value-store -iterators) to mirror the reference JavaScript client's `paginateRequests` method, keeping the -public interface consistent across the two clients. +`paginate_requests(page_limit)` returns a `RequestQueueRequestsIterator`, a lazy iterator that +fetches pages on demand, like the Store iterator in +[Store, users and logs](misc.md#apify-store--clientstore). It is named after the `paginateRequests` +method of the JavaScript client. Its `next()` is `async` and fallible, returning `ApifyClientResult>`, fetching the next page on demand and yielding `Ok(None)` once the queue is exhausted. `page_limit` bounds the requests fetched per page (`None` @@ -282,8 +281,7 @@ them at the **back**. ### `batch_add_requests` `batch_add_requests(requests, options)` is the efficient way to add many requests at once — -significantly cheaper than calling `add_request` in a loop. It mirrors the reference client's -`batchAddRequests`: +significantly cheaper than calling `add_request` in a loop: - The input is automatically split into chunks that respect both the API's per-call request-count limit (25) and its request-body byte-size limit (~9 MiB), so there is no need to chunk manually. @@ -326,7 +324,7 @@ println!( # } ``` -`batch_delete_requests` does **not** auto-chunk (matching the reference client): it accepts at +`batch_delete_requests` does **not** auto-chunk: it accepts at most 25 requests per call, identified by `id` and/or `unique_key` (not the full `RequestQueueRequest` shape), and returns `ApifyClientError::InvalidArgument` for a larger input. diff --git a/examples/log_redirection.rs b/examples/log_redirection.rs index 3a02e09..5f13a56 100644 --- a/examples/log_redirection.rs +++ b/examples/log_redirection.rs @@ -1,9 +1,7 @@ //! Run another Actor with log redirection turned on. //! -//! "Log redirection" is the convenience of taking a *separate* Actor run's log and piping it, -//! live, into the calling program's own output as it is produced. This mirrors the JS reference -//! client's `StreamedLog`, which prefixes every message with `"{Actor name} runId:{run id} -> "` -//! so the redirected output is attributable to its source Actor. +//! Log redirection pipes the live log of a separate Actor run into your own output. Every message +//! is prefixed with `"{Actor name} runId:{run id} -> "`, so you can tell which Actor it came from. //! //! The first part lets `ActorClient::call` do it, which redirects the log to the standard output //! by default. The second part starts the run itself and redirects its log to a custom sink. diff --git a/src/client.rs b/src/client.rs index d6f0a58..723d852 100644 --- a/src/client.rs +++ b/src/client.rs @@ -36,7 +36,7 @@ const DEFAULT_BASE_URL: &str = "https://api.apify.com"; /// Apify's API version path segment, appended to a configured base URL unless it is already /// present there (see [`to_api_base_url`]). const API_VERSION_PATH: &str = "/v2"; -/// Default maximum number of retries, matching the reference clients. +/// Default maximum number of retries. const DEFAULT_MAX_RETRIES: u32 = 8; /// Default minimum delay between retries. const DEFAULT_MIN_DELAY_BETWEEN_RETRIES: Duration = Duration::from_millis(500); @@ -91,10 +91,9 @@ impl ApifyClientBuilder { self } - /// Overrides the base URL of the API, with or without the `/v2` version path — it is - /// appended automatically when not already the URL's final path segment, so passing a URL - /// that already ends in `/v2` (e.g. one read back from [`ApifyClient::api_base_url`]) does - /// not double it up. + /// Overrides the base URL of the API, with or without the `/v2` version path. The path is + /// appended unless the URL already ends in it, so a URL read back from + /// [`ApifyClient::api_base_url`] can be passed as it is. /// /// Defaults to `https://api.apify.com`. pub fn base_url(mut self, base_url: impl Into) -> Self { @@ -102,9 +101,9 @@ impl ApifyClientBuilder { self } - /// Overrides the base URL used when building public, shareable resource URLs (e.g. a - /// signed dataset-items URL). Defaults to the API base URL. Like [`base_url`](Self::base_url), - /// the `/v2` version path is appended automatically only when not already present. + /// Overrides the base URL used to build public, shareable resource URLs, such as a signed + /// dataset-items URL. Defaults to the API base URL. The `/v2` path is handled like in + /// [`base_url`](Self::base_url). pub fn public_base_url(mut self, public_base_url: impl Into) -> Self { self.public_base_url = Some(public_base_url.into()); self @@ -199,8 +198,6 @@ impl ApifyClientBuilder { } /// Replaces the default [`reqwest`]-based HTTP backend with a custom implementation. - /// - /// This is the seam that makes the transport a replaceable component. pub fn http_backend(mut self, backend: Arc) -> Self { self.http_backend = Some(backend); self @@ -304,8 +301,6 @@ impl ApifyClient { &self.base_url } - // ----- Actor accessors ------------------------------------------------- - /// Returns a client for the Actor collection (list & create Actors). pub fn actors(&self) -> ActorCollectionClient { ActorCollectionClient::new(self.http(), &self.base_url) @@ -316,8 +311,6 @@ impl ApifyClient { ActorClient::new(self.clone(), self.http(), &self.base_url, &id.into()) } - // ----- Build accessors ------------------------------------------------- - /// Returns a client for the Actor build collection (list builds). pub fn builds(&self) -> BuildCollectionClient { BuildCollectionClient::new(self.http(), &self.base_url) @@ -328,8 +321,6 @@ impl ApifyClient { BuildClient::new(self.http(), &self.base_url, &id.into()) } - // ----- Run accessors --------------------------------------------------- - /// Returns a client for the Actor run collection (list runs). pub fn runs(&self) -> RunCollectionClient { RunCollectionClient::new(self.http(), &self.base_url, "actor-runs") @@ -346,8 +337,6 @@ impl ApifyClient { ) } - // ----- Dataset accessors ---------------------------------------------- - /// Returns a client for the dataset collection (list & get-or-create datasets). pub fn datasets(&self) -> DatasetCollectionClient { DatasetCollectionClient::new(self.http(), &self.base_url) @@ -359,8 +348,6 @@ impl ApifyClient { .with_public_base(&self.public_base_url) } - // ----- Key-value store accessors -------------------------------------- - /// Returns a client for the key-value store collection. pub fn key_value_stores(&self) -> KeyValueStoreCollectionClient { KeyValueStoreCollectionClient::new(self.http(), &self.base_url) @@ -372,8 +359,6 @@ impl ApifyClient { .with_public_base(&self.public_base_url) } - // ----- Request queue accessors ---------------------------------------- - /// Returns a client for the request queue collection. pub fn request_queues(&self) -> RequestQueueCollectionClient { RequestQueueCollectionClient::new(self.http(), &self.base_url) @@ -384,8 +369,6 @@ impl ApifyClient { RequestQueueClient::new(self.http(), &self.base_url, "request-queues", &id.into()) } - // ----- Task accessors -------------------------------------------------- - /// Returns a client for the Actor task collection (list & create tasks). pub fn tasks(&self) -> TaskCollectionClient { TaskCollectionClient::new(self.http(), &self.base_url) @@ -396,8 +379,6 @@ impl ApifyClient { TaskClient::new(self.clone(), self.http(), &self.base_url, &id.into()) } - // ----- Schedule accessors --------------------------------------------- - /// Returns a client for the schedule collection (list & create schedules). pub fn schedules(&self) -> ScheduleCollectionClient { ScheduleCollectionClient::new(self.http(), &self.base_url) @@ -408,8 +389,6 @@ impl ApifyClient { ScheduleClient::new(self.http(), &self.base_url, &id.into()) } - // ----- Webhook accessors ---------------------------------------------- - /// Returns a client for the webhook collection (list & create webhooks). pub fn webhooks(&self) -> WebhookCollectionClient { WebhookCollectionClient::new(self.http(), &self.base_url) @@ -430,8 +409,6 @@ impl ApifyClient { WebhookDispatchClient::new(self.http(), &self.base_url, &id.into()) } - // ----- Misc accessors -------------------------------------------------- - /// Returns a client for browsing the Apify Store. pub fn store(&self) -> StoreCollectionClient { StoreCollectionClient::new(self.http(), &self.base_url) @@ -478,13 +455,10 @@ impl ApifyClient { } } -/// Appends [`API_VERSION_PATH`] to `url` unless it is already the final path segment, mirroring -/// the reference client's `toApiBaseUrl`. +/// Appends [`API_VERSION_PATH`] to `url` unless it is already the final path segment. /// -/// A trailing slash (or several) is trimmed first, so `https://host/v2/` and `https://host/v2//` -/// both normalize to `https://host/v2` rather than growing a double slash. The check looks only -/// at the URL's *path*, not at the string as a whole: `https://v2` has no path (`v2` is the -/// host), so it still becomes `https://v2/v2` even though the raw string ends in `v2`. +/// Trailing slashes are trimmed first. Only the path is checked: `https://v2` has no path (`v2` is +/// the host), so it becomes `https://v2/v2`. fn to_api_base_url(url: &str) -> String { let (authority, path) = split_authority(url); let path = path.trim_end_matches('/'); @@ -515,10 +489,8 @@ fn split_authority(url: &str) -> (&str, &str) { mod tests { use super::to_api_base_url; - /// Mirrors the reference client's `toApiBaseUrl` test matrix exactly, so the two clients - /// agree on every case, including the `https://v2` host-vs-path edge case. #[test] - fn to_api_base_url_matches_reference_matrix() { + fn to_api_base_url_appends_the_version_path_once() { let cases = [ ("https://example.com", "https://example.com/v2"), ("https://example.com/", "https://example.com/v2"), diff --git a/src/clients/actor.rs b/src/clients/actor.rs index ce804d0..b1369e9 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -61,7 +61,7 @@ pub struct ActorStartOptions { /// Override the Actor's permission level for this run. pub force_permission_level: Option, /// Ad-hoc webhooks to attach to this run. Serialized to base64-encoded JSON as the - /// `webhooks` query parameter, matching the reference clients. + /// `webhooks` query parameter. pub webhooks: Option>, /// Retry the start while the account lacks the resources for the run, i.e. while the API /// rejects it with an [`ApiError`](crate::ApiError) whose `error_type` is @@ -79,8 +79,8 @@ pub struct ActorStartOptions { } impl ActorStartOptions { - /// Serializes these options into run-start query parameters. Shared by the Actor and - /// task start methods (DRY). + /// Serializes these options into run-start query parameters, for the Actor and task start + /// methods. pub(crate) fn apply(&self, params: &mut QueryParams) { params .add_str("build", self.build.clone()) @@ -247,11 +247,9 @@ impl ActorClient { /// Starts the Actor with a raw request body instead of a JSON-serializable value. /// - /// Use this for a non-JSON input, e.g. a ZIP archive paired with - /// `options.content_type = Some("application/zip".into())`; the bytes are sent exactly as - /// given, with no JSON serialization. For an object or array input, use - /// [`start`](Self::start) instead, which handles the serialization. Mirrors the reference - /// client, whose `ActorInput` accepts a plain object, an array of them, or raw bytes. + /// Use this for a non-JSON input, e.g. a ZIP archive with + /// `options.content_type = Some("application/zip".into())`. The bytes are sent as given. For an + /// object or array input, use [`start`](Self::start). pub async fn start_raw( &self, input: &[u8], @@ -266,10 +264,9 @@ impl ActorClient { .await } - /// Shared implementation of [`start`](Self::start) and [`start_raw`](Self::start_raw): - /// applies the run-start options as query parameters and posts the given body, falling back - /// to `default_content_type` only when the caller did not set `options.content_type`. `tier` - /// is the timeout of the request unless the client was given one with `with_timeout`. + /// Implements [`start`](Self::start) and [`start_raw`](Self::start_raw). `default_content_type` + /// applies unless `options.content_type` is set, and `tier` is the timeout of the request unless + /// the client has one from `with_timeout`. async fn start_with_body( &self, body: Option>, @@ -308,10 +305,9 @@ impl ActorClient { /// to the log sink instead (without the `"{actor name} runId:{run id} -> "` prefix of the /// default redirection). One that cannot be set up is reported on the standard error. /// - /// Like the reference client, the start request has no client-side timeout, because an - /// aborted start would be retried and could create a second run; set a timeout with the - /// client's `with_timeout` to change that, and it applies to the polls as well. Without one, - /// the polls are timed as [`RunClient::wait_for_finish`] describes. + /// The start request has no client-side timeout, since an aborted start would be retried and + /// could create a second run. A timeout set with `with_timeout` changes that and applies to the + /// polls as well. Without one, the polls are timed as [`RunClient::wait_for_finish`] describes. pub async fn call( &self, input: Option<&T>, @@ -349,7 +345,6 @@ impl ActorClient { let run = self .start_with_body(body, default_content_type, start, Timeout::NoTimeout) .await?; - // Use the root client's run client so polling targets the canonical run route. let mut run_client = self.root.run(run.id); if let Some(timeout) = self.ctx.timeout { run_client = run_client.with_timeout(timeout); @@ -407,7 +402,7 @@ impl ActorClient { /// Resolves the Actor's default build and returns a client for it. /// /// `wait_for_finish` optionally bounds how long (in seconds) the API waits for the build - /// to finish before responding, matching the reference client's `defaultBuild(options)`. + /// to finish before responding. pub async fn default_build( &self, wait_for_finish: Option, @@ -440,10 +435,8 @@ impl ActorClient { /// Validates the given input against the input schema of a specific Actor build. /// - /// `build` is the optional tag or number of the Actor build whose input schema is used for - /// validation (e.g. `"latest"` or `"1.2.34"`); passing `None` uses the default build, which - /// is equivalent to [`ActorClient::validate_input`]. This maps to the spec's optional `build` - /// query parameter on `POST /v2/actors/{actorId}/validate-input`. + /// `build` is the tag or number of the build whose input schema is used (e.g. `"latest"` or + /// `"1.2.34"`). `None` uses the default build, like [`ActorClient::validate_input`]. pub async fn validate_input_for_build( &self, input: &T, @@ -452,8 +445,7 @@ impl ActorClient { let body = serde_json::to_vec(input)?; let mut params = QueryParams::new(); params.add_str("build", build); - // `validate-input` returns a bare `{ "valid": ... }` object, *not* the usual - // `{ "data": ... }` envelope, so it must skip `parse_data_envelope`. + // `validate-input` returns a bare `{ "valid": ... }` object, without the `data` envelope. crate::clients::base::post_action_raw( &self.ctx, Some("validate-input"), @@ -468,9 +460,9 @@ impl ActorClient { /// Returns a client for the last run of this Actor, optionally filtered by run status. /// /// `status` filters by run status (e.g. `"SUCCEEDED"`, `"FAILED"`, `"RUNNING"`); pass `None` - /// to leave it unfiltered. This maps to the `status` query parameter on - /// `GET /v2/actors/{actorId}/runs/last` and mirrors the reference client's `lastRun({ status })`. - /// To also filter by `origin`, use [`ActorClient::last_run_with_options`]. + /// to leave it unfiltered. It is the `status` query parameter of + /// `GET /v2/actors/{actorId}/runs/last`. To also filter by `origin`, use + /// [`ActorClient::last_run_with_options`]. pub fn last_run(&self, status: Option<&str>) -> RunClient { self.last_run_with_options(LastRunOptions { status: status.map(str::to_owned), @@ -482,10 +474,9 @@ impl ActorClient { /// (e.g. [`LastRunOptions::status`] and/or [`LastRunOptions::origin`]). /// /// `status` filters by run status (e.g. `"SUCCEEDED"`, `"FAILED"`, `"RUNNING"`); `origin` filters - /// by how the run was started, with accepted values being the platform's run origins (e.g. - /// `"DEVELOPMENT"`, `"WEB"`, `"API"`, `"SCHEDULER"`). Both are documented optional query - /// parameters on `GET /v2/actors/{actorId}/runs/last` and match the reference client's - /// `lastRun({ status, origin })`; leave a field as `None` to omit it. + /// by how the run was started (e.g. `"DEVELOPMENT"`, `"WEB"`, `"API"`, `"SCHEDULER"`). Both + /// are query parameters of `GET /v2/actors/{actorId}/runs/last`; leave a field as `None` to omit + /// it. pub fn last_run_with_options(&self, options: LastRunOptions) -> RunClient { let mut client = RunClient::new( self.root.clone(), diff --git a/src/clients/actor_env_var_collection.rs b/src/clients/actor_env_var_collection.rs index 5dc2042..13fff4b 100644 --- a/src/clients/actor_env_var_collection.rs +++ b/src/clients/actor_env_var_collection.rs @@ -31,8 +31,8 @@ impl ActorEnvVarCollectionClient { /// Lazily iterates over the Actor version's environment variables. /// /// The listing is not offset-paginated (the API returns every variable in a single page), so this - /// yields all of them from that one page and then completes. It exists for interface parity - /// with the other collection clients and the reference client. + /// yields all of them from that one page. It exists for parity with the other collection + /// clients. pub fn iterate(&self) -> ListIterator { single_page_iterator!(self) } diff --git a/src/clients/actor_server.rs b/src/clients/actor_server.rs index 7dc7db5..ac1572c 100644 --- a/src/clients/actor_server.rs +++ b/src/clients/actor_server.rs @@ -154,11 +154,10 @@ impl ActorServerClient { /// Returns the URL of the Actor's server. /// - /// The URL is the Actor's `standbyUrl`, fetched from the API by the first call of this method - /// or [`get_open_api_spec`](Self::get_open_api_spec) and reused by later calls on the clients - /// of the same [`ActorClient`](crate::clients::actor::ActorClient). Use it to call the server - /// with another HTTP client, for example one generated from the server's OpenAPI - /// specification. + /// The URL is the Actor's `standbyUrl`. The first call of this method or of + /// [`get_open_api_spec`](Self::get_open_api_spec) fetches it, and the clients of the same + /// [`ActorClient`](crate::clients::actor::ActorClient) reuse it. Use it to call the server with + /// another HTTP client, for example one generated from the server's OpenAPI specification. /// /// Fails with [`ApifyClientError::InvalidArgument`] when the Actor is not a Server Actor, and /// with a `404` [`ApiError`](crate::ApiError) when the Actor does not exist. @@ -172,18 +171,16 @@ impl ActorServerClient { /// Returns the OpenAPI specification of the Actor's server, or `None` if the Actor's /// definition doesn't include one. /// - /// The specification is the web server schema from the Actor's definition, taken from the - /// build that serves the server interface: the build set in `actorStandby.build`, or `latest` - /// if none is set. Use it to generate an HTTP client for the server, and point that client at - /// the URL returned by [`get_url`](Self::get_url). + /// It is the web server schema in the definition of the build that serves the server + /// interface: the one set in `actorStandby.build`, or `latest`. Use it to generate an HTTP + /// client for the server, and point that client at [`get_url`](Self::get_url). /// /// Fails with [`ApifyClientError::InvalidArgument`] when the Actor is not a Server Actor, or /// the build that serves its server interface doesn't exist, and with a `404` /// [`ApiError`](crate::ApiError) when the Actor does not exist. pub async fn get_open_api_spec(&self) -> ApifyClientResult> { let (actor, server_url) = self.fetch_server_actor().await?; - // Lets a later `get_url` and `request` skip fetching the Actor again. The URL is already - // set when an earlier call got there first, which is fine. + // Saves a later `get_url` and `request` from fetching the Actor again. let _ = self.server_url.set(server_url); let build_ref = actor .actor_standby @@ -227,29 +224,27 @@ impl ActorServerClient { /// Sends an HTTP request to the Actor's server and buffers the response. /// - /// The request goes to the URL returned by [`get_url`](Self::get_url) and carries the client's - /// token in the `X-Apify-Authorization` header, which the platform consumes, so it never - /// reaches the Actor's code. Headers in the options, `Authorization` included, reach the - /// Actor's server as they are. The first request to an idle Actor waits for the platform to - /// start a server run, which is why the timeout defaults to the `Long` tier. + /// The request goes to the URL of [`get_url`](Self::get_url). The client's token is sent in the + /// `X-Apify-Authorization` header, which the platform consumes, so it never reaches the Actor's + /// code. Headers in the options, `Authorization` included, reach the server as they are. The + /// first request to an idle Actor waits for a server run to start, so the timeout defaults to + /// the `Long` tier. /// - /// `path` is the path on the Actor's server, for example `/search`. It is always resolved - /// against the server URL, so the request, and the token with it, can't be sent to another - /// host. Pass the query in [`ActorServerRequestOptions::params`]. A fragment (`#...`) is - /// dropped, since it's never sent to the server. + /// `path` is the path on the server, for example `/search`. It is resolved against the server + /// URL, so the request and the token can't go to another host. Pass the query in + /// [`ActorServerRequestOptions::params`]. A fragment (`#...`) is dropped. /// /// The method returns any response of the server, error statuses included, so check - /// [`HttpResponse::status`] before you use the body. A rate-limit (`429`) or Service - /// Unavailable (`503`) response is retried with the client's retry settings, and the last one - /// is returned once the retries run out. The Actor's code can return these statuses too, and - /// then it gets the request again. Network errors and timeouts are not retried, since the - /// Actor may have already handled the request. A redirect is returned as it is, since - /// following it would send the token to a host the Actor's code picked. + /// [`HttpResponse::status`] before you use the body. `429` and `503` responses are retried with + /// the client's retry settings, and the last one is returned when the retries run out. The + /// Actor's code can return these statuses too, and then it gets the request again. Network + /// errors and timeouts are not retried, since the Actor may have handled the request already. A + /// redirect is returned as it is, since following it would send the token to a host the + /// Actor's code picked. /// - /// It fails only when it gets no response from the server: with - /// [`ApifyClientError::InvalidArgument`] when the Actor is not a Server Actor, with a `404` - /// [`ApiError`](crate::ApiError) when the Actor does not exist, and with the transport error - /// when the request fails or times out. + /// It fails only when it gets no response: with [`ApifyClientError::InvalidArgument`] when the + /// Actor is not a Server Actor, with a `404` [`ApiError`](crate::ApiError) when the Actor does + /// not exist, and with the transport error when the request fails or times out. pub async fn request( &self, path: &str, @@ -271,12 +266,11 @@ impl ActorServerClient { /// Like [`request`](Self::request), but returns the response body as a stream instead of /// buffering it. /// - /// The response is returned with whatever status the server answers, and is retried like - /// [`request`](Self::request) until it starts. Once the body is being read, it is not, since a - /// stream that is already open can't be replayed. The stream holds its connection open until - /// you read it to the end or drop it. The timeout of the request bounds only the wait for the - /// server to start responding, so a server that keeps the connection open and stays silent - /// between two chunks, such as one that sends events, is not cut off. + /// The response has whatever status the server answers, and is retried like + /// [`request`](Self::request) until it starts, but not once the body is being read. The stream + /// holds its connection until you read it to the end or drop it. The timeout bounds only the + /// wait for the response to start, so a server that stays silent between chunks, such as one + /// that sends events, is not cut off. pub async fn request_stream( &self, path: &str, diff --git a/src/clients/actor_version_collection.rs b/src/clients/actor_version_collection.rs index 7326154..42efa26 100644 --- a/src/clients/actor_version_collection.rs +++ b/src/clients/actor_version_collection.rs @@ -34,8 +34,8 @@ impl ActorVersionCollectionClient { /// Lazily iterates over the Actor's versions. /// /// The listing is not offset-paginated (the API returns every version in a single page), so this - /// yields all of them from that one page and then completes. It exists for interface parity - /// with the other collection clients and the reference client. + /// yields all of them from that one page. It exists for parity with the other collection + /// clients. pub fn iterate(&self) -> ListIterator { single_page_iterator!(self) } diff --git a/src/clients/base.rs b/src/clients/base.rs index 29605b2..7054466 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -1,9 +1,7 @@ //! Shared internal machinery for resource clients. //! -//! [`ResourceContext`] holds the resolved URL and the [`HttpClient`] for a single -//! resource (or sub-resource). The free functions in this module implement the CRUD -//! and wait-for-finish primitives once, so that every resource client stays small and -//! consistent (the DRY principle). +//! [`ResourceContext`] holds the URL and the [`HttpClient`] of a resource or sub-resource. The +//! free functions implement the CRUD and wait-for-finish primitives that the resource clients share. use std::collections::HashMap; use std::time::Duration; @@ -22,18 +20,16 @@ use crate::timeouts::Timeout; /// How long to wait between polls while waiting for a run/build to finish. const WAIT_FOR_FINISH_POLL_INTERVAL: Duration = Duration::from_millis(250); -/// The `waitForFinish` value (in seconds) sent on each poll; the API caps server-side -/// waiting at 60 seconds, so we poll in chunks of this size. +/// The `waitForFinish` value (in seconds) sent on each poll. The API holds a response for at most +/// 60 seconds. const WAIT_FOR_FINISH_REQUEST_SECS: i64 = 60; -/// How long a run/build may keep answering `404` before [`wait_for_finish`] gives up and returns -/// a not-found error, instead of polling forever. A brand-new run/build can briefly 404 while the -/// write that created it has not yet reached the replica a poll happens to land on; a success -/// within this window resets it. Matches the reference clients' `DEFAULT_WAIT_WHEN_JOB_NOT_EXIST`. -/// Without this bound, a run/build ID that was simply never valid (e.g. a typo, or the resource -/// was deleted) would poll indefinitely when `wait_secs` is `None`. +/// How long a run or build may keep answering `404` before [`wait_for_finish`] gives up with a +/// not-found error. A new run or build can briefly 404 until its write reaches the replica a poll +/// lands on, and a success resets the window. Without the bound, an ID that never existed would be +/// polled forever when `wait_secs` is `None`. const NOT_FOUND_GRACE_PERIOD: Duration = Duration::from_secs(3); -/// Path segment standing in for an ID that failed validation. The URL is never requested: -/// the [`HttpClient`] of such a context is poisoned and fails every call first. +/// Path segment standing in for an ID that failed validation. It is never requested: the +/// [`HttpClient`] of such a context fails every call first. const INVALID_ID_PLACEHOLDER: &str = "invalid-id"; /// Why a zero timeout is refused. @@ -44,7 +40,7 @@ const ZERO_TIMEOUT_MESSAGE: &str = "a timeout and a timeout cap must be longer t #[derive(Debug, Clone)] pub(crate) struct ResourceContext { pub(crate) http: HttpClient, - /// The fully-qualified base URL of the resource, e.g. `https://api.apify.com/v2/actors/ID`. + /// The base URL of the resource, e.g. `https://api.apify.com/v2/actors/ID`. pub(crate) url: String, /// Parameters inherited from a parent resource (e.g. `status` on `actor.last_run()`). pub(crate) base_params: QueryParams, @@ -90,8 +86,7 @@ impl ResourceContext { ) } - /// Internal constructor that derives the API origin from `base_url`. The public origin - /// defaults to the API origin and can be overridden with [`with_public_origin`]. + /// Derives the API origin from `base_url`. The public origin defaults to it. fn new(http: HttpClient, url: String, base_url: &str, has_id: bool) -> Self { let api_origin = origin_of(base_url); Self { @@ -115,10 +110,9 @@ impl ResourceContext { /// Fails with [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument) /// when a request cannot be sent: the client was addressed with an invalid id (which follows - /// every client chained from it), or was given a zero timeout or cap. A zero timeout makes - /// every attempt fail at once and can never grow by doubling, and `Timeout::NoTimeout` is the - /// way to run without one. The timeout and cap of this client alone are checked, so a zero one - /// is neither inherited by chained clients nor sticky once `with_timeout` replaces it. + /// every client chained from it), or was given a zero timeout or cap. A zero timeout fails + /// every attempt at once, and `Timeout::NoTimeout` is the way to run without one. Only the + /// timeout and cap of this client are checked, so chained clients don't inherit a zero one. fn check_request_allowed(&self) -> ApifyClientResult<()> { self.http.fail_if_poisoned()?; let zero_timeout = self.timeout == Some(Timeout::Custom(Duration::ZERO)) @@ -307,8 +301,7 @@ impl ResourceContext { } } -/// Generates the public `with_timeout` method of a resource client holding a [`ResourceContext`] -/// in its `ctx` field. +/// Generates `with_timeout` for a resource client with a [`ResourceContext`] in its `ctx` field. macro_rules! impl_with_timeout { ($client:ident) => { impl $client { @@ -350,15 +343,14 @@ impl QueryParams { } } -/// Header map for a request with a JSON body. +/// Headers of a request with a JSON body. fn json_headers() -> HashMap { HashMap::from([("Content-Type".to_string(), "application/json".to_string())]) } -/// Swallows a `404` and returns `Ok(None)` only when the client names its resource by ID. A -/// chained client without one, such as `run.dataset()`, requests a path where a `404` can mean -/// either the parent or the default sub-resource is missing, and the response cannot tell the two -/// apart, so the error propagates. +/// Turns a `404` into `Ok(None)` only when the client names its resource by ID. For a chained +/// client without one, such as `run.dataset()`, a `404` can mean that the parent or the +/// sub-resource is missing, so the error propagates. pub(crate) fn catch_not_found_for( ctx: &ResourceContext, result: ApifyClientResult, @@ -522,10 +514,8 @@ pub(crate) async fn post_creating( parse_data_envelope(&response.body) } -/// Sends a `POST` and returns the raw response body, deserialized directly **without** -/// unwrapping a `data` envelope. A few endpoints (e.g. `validate-input`) return a bare JSON -/// object rather than the usual `{ "data": ... }` envelope, so they must not go through -/// [`parse_data_envelope`]. +/// Sends a `POST` and deserializes the body without unwrapping a `data` envelope, for the few +/// endpoints, such as `validate-input`, that return a bare JSON object. pub(crate) async fn post_action_raw( ctx: &ResourceContext, sub_path: Option<&str>, @@ -547,8 +537,7 @@ pub(crate) async fn post_action_raw( Ok(serde_json::from_slice(&response.body)?) } -/// Shared `POST` sender used by [`post_action`] and [`post_action_raw`]. Builds the URL with -/// merged query params, sets the optional `Content-Type`, and returns the raw response. +/// The `POST` sender of [`post_action`] and [`post_action_raw`]. async fn post_send( ctx: &ResourceContext, sub_path: Option<&str>, @@ -613,12 +602,10 @@ pub(crate) async fn head_exists( Ok(catch_not_found(result)?.is_some()) } -/// A `PUT` with raw bytes and a content type (used for KVS record uploads). +/// A `PUT` with raw bytes and a content type, used for key-value store records. /// -/// `content_encoding`, when set, is sent as the `Content-Encoding` header for an -/// already-compressed value; it makes [`HttpClient::call_with_options`]'s automatic request compression skip -/// the body (it already refuses to double-encode a request that carries a `Content-Encoding`), so -/// the bytes reach the API exactly as given. +/// `content_encoding` is sent as the `Content-Encoding` header for an already-compressed value. +/// The automatic compression skips such a body, so the bytes reach the API as given. pub(crate) async fn put_raw( ctx: &ResourceContext, sub_path: Option<&str>, @@ -652,13 +639,9 @@ pub(crate) struct RawUpload<'a> { pub(crate) content_encoding: Option<&'a str>, } -/// Opens a streaming `GET` connection to `url` (already fully built, including query string), -/// for a caller that wants to process a large body (a log or a dataset export) incrementally -/// instead of waiting for [`HttpClient::call_with_options`] to buffer the whole response. Delegates to -/// [`HttpClient::call_streaming`], which — unlike a one-off raw `reqwest` call — goes through -/// the configured [`HttpBackend`](crate::http_client::HttpBackend), so a custom backend (or a -/// `ReqwestBackend` wrapping a caller-provided, proxy/TLS-configured `reqwest::Client`) is -/// honoured for streaming requests exactly as it is for buffered ones. +/// Opens a streaming `GET` connection to the complete `url`, for a large body such as a log or a +/// dataset export. It goes through [`HttpClient::call_streaming`], so a custom +/// [`HttpBackend`](crate::http_client::HttpBackend) serves it like any other request. pub(crate) async fn get_byte_stream( ctx: &ResourceContext, url: String, @@ -677,8 +660,8 @@ pub(crate) async fn get_byte_stream( .await } -/// A `POST` with a raw body that returns an enveloped resource. Used by `actor.start` -/// and `run.metamorph` where the input is arbitrary user JSON. +/// A `POST` with a raw body that returns an enveloped resource, for the arbitrary JSON input of +/// `actor.start` and `run.metamorph`. pub(crate) async fn post_with_body( ctx: &ResourceContext, sub_path: Option<&str>, @@ -711,16 +694,13 @@ pub(crate) async fn delete_with_body( parse_data_envelope(&response.body) } -/// Polls a `GET` endpoint with `waitForFinish` until the resource reaches a terminal -/// state or `wait_secs` elapses. Used by `RunClient` and `BuildClient`. +/// Polls a `GET` endpoint with `waitForFinish` until the resource is terminal, as `is_terminal` +/// decides, or `wait_secs` elapses. Returns the latest resource. /// -/// `is_terminal` extracts whether the fetched resource is finished. Returns the latest -/// resource (finished, or still running if the wait budget was exhausted). -/// -/// With `wait_secs: None`, a poll has no client-side timeout unless the client was given one with -/// `with_timeout`, like the reference client. With `Some`, a poll is allowed the `Medium` tier plus -/// the hold it asks the API for, so a stalled connection cannot outlast the wait budget by more -/// than that. A timeout set with `with_timeout` always wins. +/// With `wait_secs: None`, a poll has no client-side timeout unless the client has one from +/// `with_timeout`. With `Some`, a poll gets the `Medium` tier plus the hold it asks the API for, so +/// a stalled connection can't outlast the wait by more than that. A timeout from `with_timeout` +/// always wins. pub(crate) async fn wait_for_finish( ctx: &ResourceContext, wait_secs: Option, @@ -732,8 +712,7 @@ where { let start = tokio::time::Instant::now(); let budget = wait_secs.map(|s| Duration::from_secs(s.max(0) as u64)); - // When the fetch below starts hitting `404`, the instant the *first* one was seen; reset on - // any successful fetch. `None` means the resource has not 404'd (yet). + // When the first of the current run of `404` responses was seen, if any. let mut not_found_since: Option = None; loop { @@ -768,9 +747,7 @@ where if !is_not_found { return Err(err); } - // A run/build that keeps 404ing past the grace period was never going to appear - // (bad ID, or it was deleted); without this, `wait_secs: None` would poll it - // forever instead of surfacing the not-found error. + // Past the grace period the resource is not going to appear. let now = tokio::time::Instant::now(); let first_seen = *not_found_since.get_or_insert(now); if now.duration_since(first_seen) >= NOT_FOUND_GRACE_PERIOD { @@ -779,7 +756,7 @@ where } } - // Brief pause to let replicas catch up, then poll again. + // Pause before polling again. crate::http_client::sleep_public(WAIT_FOR_FINISH_POLL_INTERVAL).await; } } @@ -801,8 +778,7 @@ async fn poll( parse_data_envelope(&response.body) } -/// Cooldown between retries of a run start the API rejected for lack of resources, matching the -/// reference clients. +/// Cooldown between retries of a run start the API rejected for lack of resources. const WAIT_FOR_RESOURCES_COOLDOWN: Duration = Duration::from_secs(10); /// `ApiError::error_type` values the API rejects a run start with while the account has no free /// memory or concurrent-run slot for it. Both clear as other runs/builds finish. @@ -813,10 +789,9 @@ const RESOURCE_LIMIT_ERROR_TYPES: [&str; 2] = [ /// Retries `attempt` while it fails with a resource-limit [`ApiError`](crate::ApiError), per /// `wait_for_resources` (see [`WaitForResources`](crate::clients::actor::WaitForResources)). -/// `None` makes exactly one attempt. Any other error is returned immediately, unretried. Shared -/// by [`ActorClient::start`](crate::clients::actor::ActorClient::start) and -/// [`TaskClient::start`](crate::clients::task::TaskClient::start), whose run-start options and -/// retry behavior are otherwise identical (the don't-repeat-yourself goal of this module). +/// `None` makes exactly one attempt. Any other error is returned at once. Shared by +/// [`ActorClient::start`](crate::clients::actor::ActorClient::start) and +/// [`TaskClient::start`](crate::clients::task::TaskClient::start). pub(crate) async fn retry_while_resource_limited( wait_for_resources: Option, mut attempt: F, diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index 2e12efd..3065031 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -325,24 +325,17 @@ impl DatasetClient { /// Lazily iterates over all items in the dataset, fetching pages on demand. /// - /// The idiomatic-Rust counterpart of the reference client's async-iterable - /// `listItems`/`iterateItems`: yields one deserialized item of type `T` at a time, - /// transparently paging. The caller's `options.limit` caps the total number of items yielded - /// (unset = all); use [`ListIterator::with_chunk_size`] to control the per-page fetch size. Dataset - /// items have no default page limit, so without a page size the whole dataset, or all `limit` - /// items, is loaded in one response: set one to iterate a large dataset. + /// Yields one deserialized item of type `T` at a time and fetches pages on demand. + /// `options.limit` caps the total number of items yielded (unset means all). Use + /// [`ListIterator::with_chunk_size`] to set the page size: without one, the whole dataset, or + /// all `limit` items, is loaded in one response. /// - /// Server-side filters (`skip_empty`/`clean`/`skip_hidden`) and `unwind` are forwarded on - /// every page request. Paging advances the offset by the number of underlying rows each page - /// *scanned* (the `x-apify-pagination-count` response header), not by `items.len()`, matching - /// the reference JavaScript client: a filter can scan more rows than it returns, and `unwind` - /// can return more items than rows scanned, so advancing by the returned count alone would - /// re-scan or skip rows. The listing's reported `total` is never consulted, so a dataset a - /// still-running Actor keeps pushing to is read to the end: iteration keeps requesting pages - /// until one scans no rows (one extra, empty request past the last page with data), rather - /// than stopping at the size the dataset had when the loop started. - /// - /// If `x-apify-pagination-count` is ever absent, the scanned count falls back to `items.len()`. + /// The filters (`skip_empty`, `clean`, `skip_hidden`) and `unwind` apply to every page. The + /// offset advances by the number of rows a page scanned (the `x-apify-pagination-count` + /// header, or `items.len()` when it is absent), since a filter can scan more rows than it + /// returns and `unwind` can return more items than rows. The reported `total` is not used, so a + /// dataset that an Actor is still filling is read to the end: iteration stops at the first page + /// that scans no rows. pub fn iterate_items( &self, options: DatasetListItemsOptions, @@ -388,17 +381,13 @@ impl DatasetClient { /// Like [`download_items`](Self::download_items), but streams the export as it arrives /// instead of buffering the whole response in memory. /// - /// Returns a `Stream` of byte chunks; drive it with `futures_util::StreamExt::next()` (or - /// pipe it through an `AsyncRead` adapter) to process a large export in constant memory. A - /// gap between chunks longer than the [`Timeout::Long`] tier (6 minutes by default) ends the - /// stream with [`ApifyClientError::Timeout`] — an - /// inactivity timeout, not a cap on the export's total duration. Use + /// Returns a `Stream` of byte chunks, which processes a large export in constant memory. A gap + /// between chunks longer than the [`Timeout::Long`] tier (6 minutes by default) ends the + /// stream with [`ApifyClientError::Timeout`]; the total duration is not capped. Use /// [`with_timeout`](Self::with_timeout) to change it. /// - /// Goes through the configured `HttpBackend`'s streaming path instead of the buffered one, - /// to keep the connection open — like [`crate::clients::log::LogClient::stream`], a failed - /// request (e.g. a `404`) still surfaces the same structured [`crate::error::ApiError`] as - /// the buffered endpoints, from reading the error body before the stream is handed back. + /// A failed request, such as a `404`, returns the same structured [`crate::error::ApiError`] + /// as the buffered endpoints, before the stream is returned. pub async fn download_items_stream( &self, format: DownloadItemsFormat, @@ -447,11 +436,10 @@ impl DatasetClient { /// Builds a public URL for downloading this dataset's items. /// - /// Mirrors the reference client's `createItemsPublicUrl`: it fetches the dataset, and if - /// the dataset exposes a URL-signing secret key (i.e. it is private), appends an - /// HMAC-SHA256 `signature` so the URL grants access without an API token. `expires_in_secs` - /// optionally bounds the validity of a signed URL. The URL is built from the configured - /// public base URL. The served items are JSON; to request another export format, use + /// It fetches the dataset and, for a private dataset with a URL-signing secret key, appends an + /// HMAC-SHA256 `signature` so the URL works without a token. `expires_in_secs` bounds the + /// validity of a signed URL. The URL is built from the configured public base URL and serves + /// JSON. For another format, use /// [`create_items_public_url_with_format`](Self::create_items_public_url_with_format). /// /// `expires_in_secs` must be positive and at most @@ -485,7 +473,6 @@ impl DatasetClient { options.apply(&mut params); params.add_str("format", format.map(|f| f.as_str().to_string())); - // Like the reference client, fetches the dataset with the `Long` tier of this method. let dataset: Option = get_resource(&self.ctx, None, &QueryParams::new(), Timeout::Long).await?; if let Some(dataset) = dataset { diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index 1bf2979..1d34d91 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -49,7 +49,7 @@ pub struct ListKeysOptions { #[derive(Debug, Default, Clone)] pub struct GetRecordOptions { /// Request the record with a `Content-Disposition: attachment` response header. When unset, - /// the client sends `attachment=true`, matching the reference client's unconditional default. + /// the client sends `attachment=true`. pub attachment: Option, /// URL-signing signature granting access to a record in a private store. pub signature: Option, @@ -135,22 +135,14 @@ impl KeyValueStoreClient { /// Lazily iterates over every key in the store, fetching pages on demand. /// - /// Returns a [`KeyValueStoreKeysIterator`]; call its `next()` to get one key at a time, - /// transparently fetching the following page once the local buffer drains, until the store - /// is exhausted. This is the auto-paginating counterpart to the single-page - /// [`list_keys`](Self::list_keys), matching the reference client's `listKeys()` - /// `AsyncIterable`. + /// Returns a [`KeyValueStoreKeysIterator`]; call its `next()` to get one key at a time. It + /// fetches the next page when the buffer is empty, until the store is exhausted. It is the + /// auto-paginating counterpart of [`list_keys`](Self::list_keys). /// - /// Key-value stores use cursor-based (not offset) pagination: each page is anchored by the - /// previous page's `nextExclusiveStartKey`, so the iterator threads that cursor through - /// automatically. The `prefix`, `collection` and `signature` filters from `options` are - /// carried into every page. - /// - /// `options.limit` caps the *total* number of keys yielded across all pages; leaving it unset - /// (or `0`) iterates the entire store, matching the reference client. It is honoured across as - /// many pages as needed — each individual request is bounded to the endpoint's maximum page - /// size ([`KEY_LIST_MAX_LIMIT`]), so a cap larger than one page still works. - /// `options.exclusive_start_key`, when set, resumes iteration after that key. + /// The `prefix`, `collection` and `signature` filters of `options` apply to every page. + /// `options.limit` caps the total number of keys yielded across all pages, and leaving it unset + /// (or `0`) iterates the whole store. Each request is limited to the endpoint's maximum page + /// size ([`KEY_LIST_MAX_LIMIT`]). `options.exclusive_start_key` resumes after that key. pub fn iterate_keys(&self, options: ListKeysOptions) -> KeyValueStoreKeysIterator { let remaining = options.limit.filter(|&l| l > 0); KeyValueStoreKeysIterator { @@ -177,13 +169,11 @@ impl KeyValueStoreClient { /// Gets a record's raw value (and content type), or `None` if it does not exist. /// - /// The value is returned as the bytes the API sent: unlike the reference client's - /// `getRecord`, it is never parsed according to its content type, so a malformed JSON body - /// is not an error here. Use [`KeyValueStoreRecord::as_text`] or `serde_json` to decode it. + /// The value is returned as the bytes the API sent, never parsed according to its content + /// type. Use [`KeyValueStoreRecord::as_text`] or `serde_json` to decode it. /// - /// Like the reference client's `getRecord`, this sends `attachment=true`. Use - /// [`get_record_with_options`](Self::get_record_with_options) to override the attachment - /// behaviour or to pass a URL-signing `signature` for a private store. + /// This sends `attachment=true`. Use [`get_record_with_options`](Self::get_record_with_options) + /// to override it or to pass a URL-signing `signature` for a private store. pub async fn get_record(&self, key: &str) -> ApifyClientResult> { self.get_record_with_options(key, GetRecordOptions::default()) .await @@ -192,7 +182,7 @@ impl KeyValueStoreClient { /// Gets a record with explicit options. /// /// The `attachment` option controls the `Content-Disposition: attachment` response header; - /// when unset it defaults to `true`, matching the reference client's unconditional behaviour. + /// when unset it defaults to `true`. /// `signature` supplies a URL-signing signature for accessing a record in a private store. pub async fn get_record_with_options( &self, @@ -200,7 +190,6 @@ impl KeyValueStoreClient { options: GetRecordOptions, ) -> ApifyClientResult> { let mut params = QueryParams::new(); - // Default to `attachment=true` (reference parity); honour an explicit override. params .add_bool("attachment", Some(options.attachment.unwrap_or(true))) .add_str("signature", options.signature); @@ -230,17 +219,12 @@ impl KeyValueStoreClient { .await } - /// Stores a record with raw, already-compressed bytes, sending `content_encoding` as the - /// `Content-Encoding` header so it is uploaded (and later served) exactly as given, without - /// the client compressing it again. + /// Stores a record with already-compressed bytes. `content_encoding` is sent as the + /// `Content-Encoding` header, and the record is stored and served as given, without the client + /// compressing it again. /// - /// The API accepts `gzip`, `br`, `zstd`, `deflate`, and `identity`. Unlike the reference client's - /// `setRecord({ contentEncoding })`, there is no separate runtime check rejecting a - /// non-identity encoding on a string or JSON value: this method, like - /// [`set_record_raw`](Self::set_record_raw), only ever accepts bytes that are already the - /// wire payload, so that case cannot arise here — pass pre-serialized/pre-compressed bytes, - /// not a value for the client to serialize. For plain (uncompressed) raw bytes, use - /// [`set_record_raw`](Self::set_record_raw); for a JSON value, use + /// The API accepts `gzip`, `br`, `zstd`, `deflate`, and `identity`. For uncompressed bytes use + /// [`set_record_raw`](Self::set_record_raw), and for a JSON value /// [`set_record_json`](Self::set_record_json). pub async fn set_record_raw_with_content_encoding( &self, @@ -328,9 +312,8 @@ impl KeyValueStoreClient { /// Builds a public URL for reading the record with the given key. /// - /// Mirrors the reference client's `getRecordPublicUrl`: it fetches the store, and if the - /// store exposes a URL-signing secret key (private store), appends an HMAC-SHA256 - /// `signature` over the record key so the URL works without an API token. The URL is + /// It fetches the store and, for a private store with a URL-signing secret key, appends an + /// HMAC-SHA256 `signature` over the record key so the URL works without a token. The URL is /// built from the configured public base URL. pub async fn get_record_public_url(&self, key: &str) -> ApifyClientResult { let path = record_path(key)?; @@ -400,16 +383,11 @@ pub const KEY_LIST_MAX_LIMIT: i64 = 1000; /// A lazy, page-fetching async iterator over the keys in a key-value store. /// /// Created by [`KeyValueStoreClient::iterate_keys`]. Each call to [`next`](Self::next) returns -/// the next key, fetching another page from the API when the local buffer is exhausted, until -/// every key has been yielded (or the caller's total-key cap is reached). +/// the next key and fetches another page when the buffer is empty. /// -/// Unlike the offset/limit-paginated [`ListIterator`](crate::ListIterator), key-value stores use -/// cursor-based pagination: each page is anchored by the previous page's -/// `nextExclusiveStartKey`. The walk stops once the page reports `isTruncated == false` (the -/// authoritative end-of-data signal), a page comes back empty, the API stops returning a next -/// cursor, or the caller's `limit` is exhausted. This yields the same result as the reference -/// client's `listKeys()` async-iterable (which loops on the cursor); leading with `isTruncated` -/// additionally avoids a wasted empty fetch when a final page still carries a cursor. +/// Key-value stores use cursor pagination: each page starts at the `nextExclusiveStartKey` of the +/// previous one. The walk stops when a page reports `isTruncated == false`, a page is empty, the +/// API returns no next cursor, or the caller's `limit` is reached. pub struct KeyValueStoreKeysIterator { client: KeyValueStoreClient, /// Base listing options. The `prefix`/`collection`/`signature` filters are carried into every @@ -471,12 +449,9 @@ impl KeyValueStoreKeysIterator { } self.next_exclusive_start_key = page.next_exclusive_start_key; - // Stop when the page is empty; the model reports no more keys (`is_truncated == false`, - // the authoritative "more data?" signal); the API returns no next cursor to advance on - // (a robustness fallback in case the flag and cursor disagree — e.g. `isTruncated:true` - // with a null cursor — which also prevents re-fetching the same first page); or the - // caller's cap is reached. Leading with `is_truncated` avoids one wasted empty fetch when - // a final page still carries a cursor, matching the model semantics. + // Stop on an empty page, on `is_truncated == false`, without a next cursor (the flag and + // the cursor can disagree, and without a cursor the first page would be fetched again), + // or at the caller's cap. if received == 0 || !is_truncated || self.next_exclusive_start_key.is_none() diff --git a/src/clients/log.rs b/src/clients/log.rs index ecc2552..2893175 100644 --- a/src/clients/log.rs +++ b/src/clients/log.rs @@ -19,7 +19,7 @@ use crate::timeouts::Timeout; /// /// Covers the spec's optional `raw` and `download` query parameters on the log endpoints /// (`GET /v2/logs/{buildOrRunId}`, `GET /v2/actor-runs/{runId}/log`, and the last-run log -/// variants), matching the reference client's `LogOptions`. +/// variants). #[derive(Debug, Default, Clone)] pub struct LogOptions { /// If `true`, return the raw log content without any server-side processing (e.g. without @@ -105,12 +105,11 @@ impl LogClient { /// Opens a streaming connection to the log, yielding chunks of bytes as they arrive, or /// `None` if the log does not exist. /// - /// This powers real-time log redirection: callers can forward each chunk to their own - /// logger/stdout while a run is still in progress. The stream completes when the log ends - /// (i.e. the run finishes) — as long as a new chunk arrives at least every 6 minutes (the - /// [`Timeout::Long`] tier, see [`with_timeout`](Self::with_timeout)); a run that legitimately - /// logs nothing for that long errors with - /// [`ApifyClientError::Timeout`](crate::error::ApifyClientError::Timeout) instead. + /// Use it to forward the log of a run that is still in progress. The stream ends when the log + /// ends, that is, when the run finishes, as long as a chunk arrives within the + /// [`Timeout::Long`] tier (6 minutes by default, see [`with_timeout`](Self::with_timeout)). A + /// run that logs nothing for longer fails the stream with + /// [`ApifyClientError::Timeout`](crate::error::ApifyClientError::Timeout). /// /// A log client chained from a run or build, such as `run.log()`, has no ID of its own, so it /// returns the `404` as an error: the response cannot tell a missing run from a missing log. @@ -127,8 +126,7 @@ impl LogClient { /// chunks of bytes as they arrive, or `None` if the log does not exist. /// /// Like [`LogClient::stream`], but lets the caller request the raw log via - /// [`LogOptions::raw`] (as the reference client's log redirection does, which streams - /// `{ raw: true }`). + /// [`LogOptions::raw`], as the log redirection does. pub async fn stream_with_options( &self, options: LogOptions, diff --git a/src/clients/pagination.rs b/src/clients/pagination.rs index ecc7f75..6a52bf5 100644 --- a/src/clients/pagination.rs +++ b/src/clients/pagination.rs @@ -1,11 +1,7 @@ //! Generic lazy pagination shared by every collection client. //! -//! The reference JavaScript client returns an `AsyncIterable` from every collection `list()` -//! (via its base `_listPaginatedFromCallback`), so callers can iterate across all pages without -//! manually tracking offsets. Rust cannot return a value that is simultaneously a `Future` and a -//! `Stream`, so the idiomatic equivalent here is a dedicated `iterate()` method on each collection -//! client that returns a [`ListIterator`]. The paging logic lives here once so every client stays -//! a thin wrapper over it (don't-repeat-yourself). +//! A collection client's `iterate()` method returns a [`ListIterator`], which walks all pages +//! without the caller tracking offsets. The paging logic lives here once for every client. use std::collections::VecDeque; use std::future::Future; @@ -73,8 +69,7 @@ macro_rules! single_page_iterator { pub(crate) use single_page_iterator; /// Returns the smaller of two optional positive limits, treating a non-positive value as "no -/// limit" (`None`). Mirrors the reference client's `minForLimitParam`, where the API treats `0` -/// as an absent limit. +/// limit" (`None`), since the API treats `0` as an absent limit. fn min_positive_limit(a: Option, b: Option) -> Option { let a = a.filter(|&x| x > 0); let b = b.filter(|&x| x > 0); @@ -88,30 +83,23 @@ fn min_positive_limit(a: Option, b: Option) -> Option { /// A lazy, page-fetching async iterator over an offset/limit-paginated list endpoint. /// /// Created by a collection client's `iterate()` method. Each call to [`next`](Self::next) -/// returns the next item, transparently fetching the following page from the API once the -/// local buffer drains, until the listing is exhausted (or the caller's total-item cap is hit). +/// returns the next item and fetches the following page when the buffer is empty, until the +/// listing is exhausted or the caller's cap is reached. /// -/// # `limit` vs. page size +/// # `limit` and page size /// -/// The caller's `limit` (from the list options passed to `iterate()`) is a **cap on the total -/// number of items the iterator yields**, matching the reference JavaScript client's -/// `_listPaginatedFromCallback`, where `options.limit` bounds the whole async-iterable and a -/// separate `chunkSize` controls page size. Leaving `limit` unset (or `0`) iterates the entire -/// listing. The page size is a distinct concern: set it with [`with_chunk_size`](Self::with_chunk_size); -/// when unset, the API's default page size is used. So `iterate(opts{ limit: 10 })` yields at most -/// 10 items, and `iterate(opts).with_chunk_size(50)` fetches 50 per request while yielding -/// everything. Without a page size, each request gets the API's default page size, capped by -/// the remaining `limit`. Dataset items have no default limit, so iterating -/// [`DatasetClient::iterate_items`](crate::clients::dataset::DatasetClient::iterate_items) -/// without a page size loads the whole dataset, or all `limit` items, in one response: set one -/// to iterate a large dataset. +/// The `limit` of the list options is a cap on the total number of items the iterator yields. +/// Leaving it unset (or `0`) iterates the whole listing. The page size is separate: set it with +/// [`with_chunk_size`](Self::with_chunk_size), or each request gets the API's default page size, +/// capped by the remaining `limit`. So `iterate(opts{ limit: 10 })` yields at most 10 items, and +/// `iterate(opts).with_chunk_size(50)` fetches 50 per request and yields everything. Dataset items +/// have no default limit, so +/// [`DatasetClient::iterate_items`](crate::clients::dataset::DatasetClient::iterate_items) without +/// a page size loads the whole dataset, or all `limit` items, in one response. /// -/// **Large caps and the first page.** When a total cap is set but no page size is, the first page -/// requests `limit == cap` (the reference client does the same, via -/// `minForLimitParam(options.limit, options.chunkSize)`). If you set a very large cap — larger -/// than the endpoint's maximum `limit` — also call [`with_chunk_size`](Self::with_chunk_size) with -/// a value at or below that maximum, so the first request stays within the endpoint's accepted -/// range rather than asking for the whole cap up front. +/// **Large caps.** With a cap but no page size, the first page requests `limit == cap`. If the cap +/// is larger than the endpoint's maximum `limit`, also call +/// [`with_chunk_size`](Self::with_chunk_size) with a value at or below that maximum. /// /// # Example /// ```no_run @@ -134,8 +122,7 @@ pub struct ListIterator { /// Number of items still allowed under the caller's total-item cap; `None` = uncapped. /// Decremented by each page's item count as pages are fetched. remaining: Option, - /// Page size to request per fetch (the reference client's `chunkSize`); `None` = let the API - /// choose its default page size. + /// Page size to request per fetch; `None` = let the API choose its default page size. chunk_size: Option, /// When `true`, the underlying endpoint is not offset-paginated: the first fetch returns the /// whole result set, so the iterator stops after it rather than requesting a second page. @@ -171,10 +158,9 @@ impl ListIterator { } } - /// Sets the page size (items requested per API call) for this iteration — the reference - /// client's `chunkSize`. This controls only how many items each page fetch requests, never how - /// many the iterator yields in total (that is the caller's `limit`; see the type docs). A - /// non-positive value lets the API choose its default page size. + /// Sets the page size, the number of items requested per API call. It never changes how many + /// items the iterator yields in total, which is the caller's `limit`. A non-positive value lets + /// the API choose its default page size. pub fn with_chunk_size(mut self, chunk_size: i64) -> Self { self.chunk_size = (chunk_size > 0).then_some(chunk_size); self @@ -197,10 +183,8 @@ impl ListIterator { /// Fetches the page at the current offset into the buffer and advances the offset. async fn fetch_next_page(&mut self) -> ApifyClientResult<()> { - // Request the smaller of the items still allowed under the caller's cap (`remaining`) and - // the configured page size (`chunk_size`); `None` lets the API pick its default page size. - // On the first page `remaining` is the full cap, matching the reference client's initial - // `minForLimitParam(options.limit, options.chunkSize)`. + // Request the smaller of the items still allowed (`remaining`) and the page size + // (`chunk_size`); `None` lets the API pick its default page size. let page_limit = min_positive_limit(self.remaining, self.chunk_size); let page = (self.fetch)(self.next_offset, page_limit).await?; let received = page.items.len() as i64; @@ -233,14 +217,10 @@ impl ListIterator { *rem -= scanned; } - // Decide whether more pages remain. `total` is deliberately never consulted: it changes - // as the listed resource grows (e.g. a dataset a still-running Actor keeps pushing to), - // so stopping at the `total` a page happened to report can end the walk before rows added - // later are ever read. A short page is not treated as terminal either (server-side filters - // can legitimately make a full, non-final window come back short) — only a page that - // scanned nothing, or the caller's own cap, ends the walk. This matches the reference - // client, which pages until a request scans zero rows and so always sends one extra - // request at the end to find that empty page. + // `total` is not used: it grows with the resource, so stopping at it could miss rows added + // later. A short page is not terminal either, since a filter can shorten a non-final page. + // Only a page that scanned nothing, or the caller's cap, ends the walk, which costs one + // extra request at the end. if self.single_page { // Non-paginated endpoint: everything came back in one response. self.exhausted = true; diff --git a/src/clients/request_queue.rs b/src/clients/request_queue.rs index 36645eb..be9f85e 100644 --- a/src/clients/request_queue.rs +++ b/src/clients/request_queue.rs @@ -21,29 +21,24 @@ use crate::models::{ use crate::timeouts::Timeout; /// Maximum number of requests the API accepts in a single `requests/batch` call. Larger -/// `batch_add_requests` inputs are split into chunks of at most this size (matching the -/// reference client's `REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION`); `batch_delete_requests` -/// does not auto-chunk (matching the reference client) and instead rejects larger inputs. +/// `batch_add_requests` inputs are split into chunks of this size, while `batch_delete_requests` +/// rejects them. const MAX_REQUESTS_PER_BATCH_OPERATION: usize = 25; -/// Maximum accepted size (bytes) of a request body, mirroring the platform-wide -/// `MAX_PAYLOAD_SIZE_BYTES` (9 MiB) that the reference client chunks `batch_add_requests` calls -/// against, on top of the per-call request-count limit. +/// Maximum size in bytes (9 MiB) of a request body. `batch_add_requests` chunks by it as well as by +/// the request count. const MAX_PAYLOAD_SIZE_BYTES: usize = 9_437_184; /// Bytes of the `[` and `]` around a JSON array. const ARRAY_BRACKETS_BYTES: usize = 2; -/// Fraction of [`MAX_PAYLOAD_SIZE_BYTES`] held back as a safety margin (0.01%), matching the -/// reference client's `SAFETY_BUFFER_PERCENT`. +/// Fraction of [`MAX_PAYLOAD_SIZE_BYTES`] held back as a safety margin (0.01%). const SAFETY_BUFFER_PERCENT: f64 = 0.0001; /// Default number of batch-add API calls [`RequestQueueClient::batch_add_requests`] keeps in -/// flight at once, matching the reference client's `DEFAULT_PARALLEL_BATCH_ADD_REQUESTS`. +/// flight at once. const DEFAULT_MAX_PARALLEL_BATCH_ADD_REQUESTS: usize = 5; /// Default number of retry attempts for requests a batch-add call reports as `unprocessed` -/// (typically rate-limited), matching the reference client's -/// `DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS`. +/// (typically rate-limited). const DEFAULT_MAX_UNPROCESSED_REQUESTS_RETRIES: u32 = 3; -/// Default minimum delay before the first unprocessed-request retry; doubles (with jitter) on -/// each subsequent retry, matching the reference client's -/// `DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS`. +/// Default minimum delay before the first unprocessed-request retry. It doubles, with jitter, on +/// each later retry. const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES: Duration = Duration::from_millis(500); /// The path of the request with the given ID, relative to the queue. An ID that is empty or a dot @@ -55,11 +50,10 @@ fn request_path(id: &str) -> ApifyClientResult { /// Options for [`RequestQueueClient::batch_add_requests`]. /// -/// Mirrors the reference client's retrying, chunked, parallel `batchAddRequests`: large inputs -/// are split by count (max `MAX_REQUESTS_PER_BATCH_OPERATION`) and by JSON byte size (max -/// `MAX_PAYLOAD_SIZE_BYTES`, minus a safety margin), chunks are sent with up to `max_parallel` -/// requests in flight at once, and any request an API call reports as `unprocessed` (typically -/// due to rate limiting) is retried with exponential backoff. +/// Large inputs are split by count (`MAX_REQUESTS_PER_BATCH_OPERATION`) and by JSON size +/// (`MAX_PAYLOAD_SIZE_BYTES` minus a safety margin), up to `max_parallel` chunks are sent at once, +/// and requests reported as `unprocessed`, typically because of rate limiting, are retried with +/// exponential backoff. #[derive(Debug, Default, Clone)] pub struct BatchAddRequestsOptions { /// If `true`, adds all requests to the front of the queue. @@ -80,9 +74,8 @@ pub struct BatchAddRequestsOptions { /// request against the payload limit and is joined into the batch body that is sent, so a request /// is never serialized twice, whichever chunk or retry it ends up in. struct SerializedRequest { - /// Key correlating the request across the retry loop: its explicit `unique_key` if set, - /// otherwise its `url` — matching the API's own fallback (a request added without a - /// `unique_key` is deduplicated by its raw `url`). + /// Key that matches the request across retries: its `unique_key`, or its `url` if it has none, + /// like the API's own deduplication. dedup_key: String, url: String, method: Option, @@ -125,13 +118,10 @@ fn batch_body(requests: &[SerializedRequest]) -> Vec { /// is only used to name the offending item in the error message, so it should be the slice's /// absolute position within the caller's full input. /// -/// The first item is always included regardless of size (once its own size has been checked -/// against `max_bytes`), guaranteeing a non-empty result for a non-empty input — unlike the -/// reference implementation, which can return an empty slice (and loop forever) when a single -/// item's size leaves no room under `max_bytes` for even itself plus the array wrapper. +/// The first item is always included once its own size has been checked against `max_bytes`, so a +/// non-empty input never yields an empty chunk. /// -/// Returns [`ApifyClientError::InvalidArgument`] if a single request's JSON exceeds `max_bytes` -/// on its own (mirroring the reference client's thrown error). +/// Returns [`ApifyClientError::InvalidArgument`] if a single request's JSON exceeds `max_bytes`. fn count_requests_fitting_byte_length( requests: &[SerializedRequest], max_bytes: usize, @@ -166,7 +156,7 @@ fn count_requests_fitting_byte_length( /// Splits `requests` into chunks that each satisfy both the per-call count limit /// ([`MAX_REQUESTS_PER_BATCH_OPERATION`]) and the payload byte-size limit -/// (`max_bytes`), mirroring the reference client's chunking loop in `batchAddRequests`. +/// (`max_bytes`). fn chunk_requests_for_batch_add( requests: Vec, max_bytes: usize, @@ -202,8 +192,7 @@ pub struct ListRequestsOptions { /// Opaque pagination cursor returned by a previous call. pub cursor: Option, /// Restrict the returned requests to the given states. The spec defines this as an array of - /// the enum values `"locked"` and `"pending"`; multiple values are sent comma-joined (matching - /// the JS reference, which serializes `filter: Array<'locked' | 'pending'>` via `join(',')`). + /// the enum values `"locked"` and `"pending"`; multiple values are sent comma-joined. pub filter: Option>, } @@ -383,24 +372,18 @@ impl RequestQueueClient { /// Adds multiple requests to the queue in a single logical operation. /// /// This is significantly more efficient than calling [`add_request`](Self::add_request) - /// once per request, especially for large batches: the input is automatically split into - /// chunks that respect both the API's per-call request-count limit - /// (`MAX_REQUESTS_PER_BATCH_OPERATION`) and its request-body byte-size limit - /// (`MAX_PAYLOAD_SIZE_BYTES`), chunks are sent with up to `options.max_parallel` API calls - /// in flight at once, and any request an API call reports as `unprocessed` (typically due to - /// rate limiting) is retried with exponential backoff — matching the reference client's - /// `batchAddRequests`. Every request must be identifiable by [`RequestQueueRequest::unique_key`] - /// (or, if left unset, by `url`, the API's own fallback) so a retried request can be matched - /// back to the original input. + /// once per request, especially for large batches: the input is split into chunks that respect + /// the API's limits on the request count (`MAX_REQUESTS_PER_BATCH_OPERATION`) and the body size + /// (`MAX_PAYLOAD_SIZE_BYTES`), up to `options.max_parallel` API calls run at once, and requests + /// an API call reports as `unprocessed`, typically because of rate limiting, are retried with + /// exponential backoff. Every request must be identifiable by + /// [`RequestQueueRequest::unique_key`], or by `url` if it has none, so a retried request can be + /// matched to the input. /// - /// Unlike most methods here, this does not propagate per-chunk API errors: a chunk that fails - /// even after retries has its requests reported in the result's `unprocessed_requests` - /// instead, so a batch add of many requests never fails outright over one bad chunk (matching - /// the reference client). It still returns [`ApifyClientError::InvalidArgument`] up front, - /// before any request is sent, for an empty `requests` (matching - /// [`batch_delete_requests`](Self::batch_delete_requests) and the reference client, which - /// validates both as non-empty) or if a single request's JSON is too large to ever fit in a - /// chunk. + /// A chunk that fails even after retries does not fail the call: its requests are reported in + /// the result's `unprocessed_requests`. Before any request is sent, it fails with + /// [`ApifyClientError::InvalidArgument`] for an empty `requests`, or for a single request whose + /// JSON is too large to fit in a chunk. pub async fn batch_add_requests( &self, requests: &[RequestQueueRequest], @@ -450,8 +433,8 @@ impl RequestQueueClient { Ok(merged) } - /// Sends one already-byte/count-limited chunk, retrying requests reported as `unprocessed` - /// with exponential backoff (matching the reference client's `_batchAddRequestsWithRetries`). + /// Sends one chunk that is within the count and size limits, retrying requests reported as + /// `unprocessed` with exponential backoff. /// /// Never returns an `Err`: a transport/API failure that survives `HttpClient`'s own retries /// marks every request still outstanding in this chunk as unprocessed instead of propagating, @@ -498,9 +481,8 @@ impl RequestQueueClient { } } Err(_) => { - // A hard failure (already retried by `HttpClient` for transient errors): - // treat every request still outstanding in this chunk as unprocessed rather - // than propagating, matching the reference client's "never throws" contract. + // A failure that `HttpClient` already retried: report every outstanding + // request as unprocessed instead of failing the whole call. return BatchRequestsOperationResult { processed_requests: processed, unprocessed_requests: remaining @@ -510,9 +492,7 @@ impl RequestQueueClient { }; } } - // Exponential backoff with jitter before the next retry, matching the reference - // client's `(1 + random()) * 2^attempt * minDelay` (see `randomized_delay`, which - // returns a value in `[base, 2*base)`, i.e. `(1 + random()) * base`). + // `randomized_delay` adds jitter to the exponential backoff. let backoff = min_delay.saturating_mul(2u32.saturating_pow(attempt)); sleep_public(crate::http_client::randomized_delay(backoff)).await; } @@ -552,9 +532,8 @@ impl RequestQueueClient { /// Deletes multiple requests in a single batch operation. /// /// Unlike [`batch_add_requests`](Self::batch_add_requests), this does not auto-chunk: the API - /// accepts at most `MAX_REQUESTS_PER_BATCH_OPERATION` requests per call (matching the - /// reference client, which validates rather than chunks), so a larger `requests` returns - /// [`ApifyClientError::InvalidArgument`] before any request is sent. + /// accepts at most `MAX_REQUESTS_PER_BATCH_OPERATION` requests per call, so a larger + /// `requests` fails with [`ApifyClientError::InvalidArgument`] before any request is sent. pub async fn batch_delete_requests( &self, requests: &[T], @@ -636,8 +615,7 @@ impl RequestQueueClient { /// Lazily paginates over all requests in the queue, fetching pages on demand. /// /// Returns a [`RequestQueueRequestsIterator`]; call its `next()` to get one request at a - /// time. Pagination uses the API's opaque `nextCursor` token: every page after the first is - /// fetched with `cursor` (matching the JS reference). + /// time. Every page after the first is fetched with the `nextCursor` token of the previous one. pub fn paginate_requests(&self, page_limit: Option) -> RequestQueueRequestsIterator { RequestQueueRequestsIterator { client: self.clone(), @@ -686,8 +664,7 @@ impl RequestQueueRequestsIterator { return Ok(None); } - // Every page after the first is fetched with the opaque `cursor` token, matching the JS - // reference. Here we only ever paginate from the queue head, so the first page has none. + // The first page has no cursor: pagination starts at the queue head. let page = self .client .list_requests(ListRequestsOptions { @@ -739,8 +716,7 @@ mod batch_add_tests { .collect() } - /// A request without an explicit `unique_key` is correlated by `url`, matching the API's own - /// deduplication fallback. + /// A request without a `unique_key` is matched by `url`, like the API's own deduplication. #[test] fn dedup_key_falls_back_to_url() { let with_key = SerializedRequest::new(&request("https://example.com", Some("k1"))).unwrap(); diff --git a/src/clients/run.rs b/src/clients/run.rs index 4479b57..6689af9 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -21,7 +21,7 @@ use crate::models::ActorRun; use crate::timeouts::Timeout; use std::sync::Arc; -/// Header the API uses to deduplicate charge requests (matching the reference client). +/// Header the API uses to deduplicate charge requests. const CHARGE_IDEMPOTENCY_HEADER: &str = "idempotency-key"; /// Nanoseconds in a millisecond: the sub-millisecond part of a timestamp is below it. const NANOS_PER_MILLI: u32 = 1_000_000; @@ -32,8 +32,7 @@ const NANOS_PER_MILLI: u32 = 1_000_000; /// /// Both are sent as query parameters to the last-run endpoints /// (`GET /v2/actors/{actorId}/runs/last`, `GET /v2/actor-tasks/{actorTaskId}/runs/last`). -/// `status` and `origin` are both documented optional filters on those endpoints and match the -/// reference client's `lastRun({ status, origin })`. +/// `status` and `origin` are optional filters of those endpoints. #[derive(Debug, Default, Clone)] pub struct LastRunOptions { /// Filter by run status (e.g. `"SUCCEEDED"`, `"FAILED"`, `"RUNNING"`). `None` leaves it @@ -81,8 +80,8 @@ pub struct RunChargeOptions { /// Idempotency key deduplicating the charge across retries. The API requires this header /// on every charge request and forgets each key 3 minutes after the charge, so a retry sent /// past that window with the same key creates a new charge rather than being deduplicated. - /// If `None`, one is auto-generated as `{runId}-{eventName}-{timestampMillis}-{random}`, - /// matching the reference client, so a transport-retried charge is applied at most once. + /// If `None`, one is generated as `{runId}-{eventName}-{timestampMillis}-{random}`, so a + /// charge that the transport retries is applied at most once. pub idempotency_key: Option, /// ID of the Server Actor request being charged, taken from the `X-Actor-Request-ID` header /// of the incoming request. Required for multi-tenant Server Actor runs, where it maps the @@ -159,10 +158,9 @@ impl RunClient { delete_resource(&self.ctx, None, Timeout::Short).await } - /// Aborts the run. `gracefully` is optional, matching the reference client's optional - /// `gracefully` option and the Go sibling's `Option`: `Some(true)` lets the run - /// perform cleanup first, `Some(false)` aborts immediately, and `None` omits the parameter - /// entirely so the server applies its default (immediate abort). + /// Aborts the run. `Some(true)` for `gracefully` lets the run clean up first, `Some(false)` + /// aborts immediately, and `None` omits the parameter, so the API applies its default + /// (immediate abort). pub async fn abort(&self, gracefully: Option) -> ApifyClientResult { let mut params = QueryParams::new(); params.add_bool("gracefully", gracefully); @@ -179,9 +177,9 @@ impl RunClient { /// Transforms the run into a run of another Actor (metamorph). /// - /// `options.content_type` sets the content type of the input body (defaulting to - /// `application/json`), matching the reference client's `metamorph(..., { contentType })`. - /// To send a non-JSON input as raw bytes instead, use [`metamorph_raw`](Self::metamorph_raw). + /// `options.content_type` sets the content type of the input body (default + /// `application/json`). To send a non-JSON input as raw bytes, use + /// [`metamorph_raw`](Self::metamorph_raw). pub async fn metamorph( &self, target_actor_id: &str, @@ -288,12 +286,10 @@ impl RunClient { /// Charges the run for a pay-per-event run, recording occurrences of a named event. /// - /// An idempotency key is always sent (auto-generated when `options.idempotency_key` is - /// `None`), so a charge that is retried by the transport is applied at most once — matching - /// the reference client and preventing double-charging. + /// An idempotency key is always sent (generated when `options.idempotency_key` is `None`), so + /// a charge that the transport retries is applied at most once. /// - /// The charge endpoint returns an empty body on success, so this issues the request - /// directly and treats any 2xx response as success (errors still surface normally). + /// The charge endpoint returns an empty body on success, so any 2xx response is a success. pub async fn charge(&self, options: RunChargeOptions) -> ApifyClientResult<()> { let count = options.count.unwrap_or(1); let idempotency_key = options @@ -325,9 +321,8 @@ impl RunClient { } /// Builds a per-charge idempotency key of the form - /// `{runId}-{eventName}-{timestampMillis}-{random}`, matching the reference client. The - /// suffix only needs to be unique enough to avoid collisions within the same millisecond; - /// it is derived from the sub-millisecond part of the current time (no crypto needed). + /// `{runId}-{eventName}-{timestampMillis}-{random}`. The suffix only has to differ within the + /// same millisecond, so it comes from the sub-millisecond part of the current time. fn generate_idempotency_key(&self, event_name: &str) -> String { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -397,16 +392,14 @@ impl RunClient { /// /// Returns a [`RunDatasetItemsIterator`]; call its `next()` to get one item at a time. Unlike /// [`DatasetClient::iterate_items`], which stops at the end of the dataset as it is, this keeps - /// polling until the run finishes, so a consumer can process the items of a long-running Actor - /// as they appear. See [`RunDatasetItemsIterator`] for how the rows are read, and its - /// `with_chunk_size` and `with_poll_interval` for the page size and the polling interval. + /// polling until the run finishes, so you can process the items of a long-running Actor as they + /// appear. See [`RunDatasetItemsIterator`] for how rows are read, and its `with_chunk_size` and + /// `with_poll_interval` for the page size and the polling interval. /// - /// `options` filters and projects the items exactly like - /// [`DatasetClient::list_items`]: `offset` is the number of dataset rows to skip and `limit` - /// the maximum number of rows to scan. `desc: Some(true)` is rejected with + /// `options` works like in [`DatasetClient::list_items`]: `offset` is the number of rows to skip + /// and `limit` the maximum number of rows to scan. `desc: Some(true)` fails with /// [`ApifyClientError::InvalidArgument`](crate::error::ApifyClientError::InvalidArgument), since - /// the end of a dataset that is still growing is not known: the rows are always read oldest - /// first. + /// the end of a growing dataset is not known: rows are read oldest first. /// /// # Example /// ```no_run @@ -434,7 +427,7 @@ impl RunClient { /// [`LogRedirect::Disabled`] or, for [`LogRedirect::Default`], the run does not exist. With a /// [`LogRedirect::Sink`] nothing is requested, so a missing run shows up as the error of /// [`StreamedLog::stop`]. Call [`StreamedLog::start`] to begin redirecting and - /// [`StreamedLog::stop`] to end it. Mirrors the reference client's `getStreamedLog`. + /// [`StreamedLog::stop`] to end it. /// /// With [`LogRedirect::Default`], the messages go to the standard output prefixed with /// `"{actor name} runId:{run id} -> "`, which costs a request for the run and one for its diff --git a/src/clients/run_collection.rs b/src/clients/run_collection.rs index a7129ee..16a327f 100644 --- a/src/clients/run_collection.rs +++ b/src/clients/run_collection.rs @@ -11,15 +11,13 @@ use crate::timeouts::Timeout; /// Filtering options for [`RunCollectionClient::list`], covering the spec query parameters of /// `GET /v2/actor-runs` and `GET /v2/actors/{actorId}/runs`. /// -/// As in the reference client, the same options type is reused for the task-scoped run -/// collection (`GET /v2/actor-tasks/{actorTaskId}/runs`), whose spec does not define -/// `startedAfter`/`startedBefore`; those params are simply ignored server-side when set on a -/// task-scoped list. This deliberate reference-parity reuse keeps the API surface uniform. +/// The task-scoped run collection (`GET /v2/actor-tasks/{actorTaskId}/runs`) reuses this type. Its +/// spec has no `startedAfter` and `startedBefore`, so the API ignores them there. #[derive(Debug, Default, Clone)] pub struct RunListOptions { /// Only return runs with one of these statuses (e.g. `SUCCEEDED`, `RUNNING`). The API /// accepts multiple statuses; they are sent as a comma-separated list. Empty means no - /// status filter. Matches the reference client, whose `status` accepts a string or array. + /// status filter. pub status: Vec, /// Only return runs started at or after this ISO 8601 timestamp. (Actor-scoped lists only.) pub started_after: Option, @@ -91,8 +89,7 @@ impl RunCollectionClient { /// Lazily iterates over all runs matching `options`/`filter`, fetching pages on demand. /// - /// The idiomatic-Rust counterpart of the reference client's async-iterable run listing; - /// yields one [`ActorRun`] at a time across all pages. + /// Yields one [`ActorRun`] at a time across all pages. /// /// `options.limit` caps the *total* number of runs yielded across all pages, unlike /// [`list`](Self::list) where `limit` is a single page's size. Set the per-page fetch size diff --git a/src/clients/streamed_log.rs b/src/clients/streamed_log.rs index d787ea4..f3f25d2 100644 --- a/src/clients/streamed_log.rs +++ b/src/clients/streamed_log.rs @@ -1,6 +1,6 @@ //! Log redirection: forwarding the live log of a run to a [`LogSink`]. //! -//! [`StreamedLog`] is the equivalent of the reference client's `StreamedLog`. It follows the log +//! [`StreamedLog`] follows the log //! of a run in a background task, splits it into messages at the timestamps the Actor's log lines //! start with, and hands every message to a [`LogSink`]. [`ActorClient::call`] uses it to redirect //! the log of the run it waits for. @@ -147,8 +147,7 @@ struct Running { /// [`start`](Self::start) spawns a task that follows the log, and [`stop`](Self::stop) ends it. /// Dropping a started `StreamedLog` cancels the task without reading the rest of the log. /// -/// Compared with the reference client, a line that continues a message in a later chunk is -/// emitted as a message of its own, instead of being dropped. +/// A line that continues a message in a later chunk is emitted as a message of its own. pub struct StreamedLog { log: LogClient, sink: Arc, diff --git a/src/clients/task.rs b/src/clients/task.rs index c0df615..4744ef7 100644 --- a/src/clients/task.rs +++ b/src/clients/task.rs @@ -184,8 +184,8 @@ impl TaskClient { /// /// `status` filters by run status (e.g. `"SUCCEEDED"`, `"FAILED"`, `"RUNNING"`); pass `None` /// to leave it unfiltered. This maps to the `status` query parameter on - /// `GET /v2/actor-tasks/{actorTaskId}/runs/last` and mirrors the reference client's - /// `lastRun({ status })`. To also filter by `origin`, use [`TaskClient::last_run_with_options`]. + /// `GET /v2/actor-tasks/{actorTaskId}/runs/last`. To also filter by `origin`, use + /// [`TaskClient::last_run_with_options`]. pub fn last_run(&self, status: Option<&str>) -> RunClient { self.last_run_with_options(LastRunOptions { status: status.map(str::to_owned), @@ -199,8 +199,8 @@ impl TaskClient { /// `status` filters by run status (e.g. `"SUCCEEDED"`, `"FAILED"`, `"RUNNING"`); `origin` filters /// by how the run was started, with accepted values being the platform's run origins (e.g. /// `"DEVELOPMENT"`, `"WEB"`, `"API"`, `"SCHEDULER"`). Both are documented optional query - /// parameters on `GET /v2/actor-tasks/{actorTaskId}/runs/last` and match the reference client's - /// `lastRun({ status, origin })`; leave a field as `None` to omit it. + /// parameters on `GET /v2/actor-tasks/{actorTaskId}/runs/last`; leave a field as `None` to + /// omit it. pub fn last_run_with_options(&self, options: LastRunOptions) -> RunClient { let mut client = RunClient::new( self.root.clone(), diff --git a/src/common.rs b/src/common.rs index 65669b3..fadf27b 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,5 +1,5 @@ -//! Shared building blocks used across resource clients: the response envelope, -//! query-parameter helpers, pagination types and the `User-Agent` builder. +//! Building blocks shared by the resource clients: the response envelope, query parameters, +//! pagination types and the `User-Agent` builder. use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -7,8 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::error::{ApifyClientError, ApifyClientResult, NOT_FOUND_STATUS_CODE}; use crate::version::CLIENT_VERSION; -/// Most Apify endpoints wrap their payload in a top-level `data` property. -/// This envelope unwraps `{ "data": ... }` into the inner type. +/// The `{ "data": ... }` envelope around the payload of most endpoints. #[derive(Debug, Deserialize)] pub(crate) struct DataEnvelope { pub data: T, @@ -20,11 +19,8 @@ pub(crate) fn parse_data_envelope(body: &[u8]) -> ApifyClie Ok(envelope.data) } -/// Translates a "not found" API error into `Ok(None)`, re-raising any other error. -/// -/// This mirrors `catchNotFoundOrThrow` in the reference clients: a `get`/`delete` on a -/// missing resource resolves to `None` rather than raising, whatever the error `type` of the -/// `404` response. +/// Translates a "not found" API error into `Ok(None)` and returns any other error. A `get` or +/// `delete` of a missing resource thus resolves to `None`, whatever the error `type` of the `404`. pub(crate) fn catch_not_found(result: ApifyClientResult) -> ApifyClientResult> { match result { Ok(value) => Ok(Some(value)), @@ -33,8 +29,7 @@ pub(crate) fn catch_not_found(result: ApifyClientResult) -> ApifyClientRes } } -/// A mutable collection of query parameters that serializes booleans as `0`/`1` and -/// omits `None` values, matching the Apify API conventions. +/// A collection of query parameters that serializes booleans as `0`/`1` and omits `None` values. #[derive(Debug, Default, Clone)] pub(crate) struct QueryParams { pairs: Vec<(String, String)>, @@ -89,12 +84,12 @@ impl QueryParams { self } - /// Internal: read access to the raw pairs (used when merging parent params). + /// The raw key-value pairs. pub(crate) fn pairs_ref(&self) -> &[(String, String)] { &self.pairs } - /// Internal: push an already-stringified key/value pair. + /// Adds an already-stringified pair. pub(crate) fn push_raw(&mut self, key: String, value: String) { self.pairs.push((key, value)); } @@ -128,30 +123,21 @@ fn url_encode(input: &str) -> String { out } -/// Percent-encodes a single URL *path segment*, so that values interpolated into the path -/// (such as key-value-store record keys or request IDs) cannot break out of the segment or -/// inject a query string. -/// -/// Encodes everything except the RFC 3986 "unreserved" characters. Notably `/`, `?`, `#`, -/// space and any non-ASCII bytes are escaped — unlike a raw `format!(".../{key}")`, which -/// would leave them intact and produce a malformed or wrong-endpoint URL. +/// Percent-encodes a single URL path segment, such as a record key or a request ID, so it can't +/// leave the segment or add a query string. Everything except the RFC 3986 unreserved characters +/// is escaped. /// -/// This does not reject a segment that is empty or a dot segment (`.` or `..`), which a URL -/// parser resolves to another endpoint even when percent-encoded; the client methods that take -/// such a value validate it with `to_path_segment` instead. +/// It does not reject an empty segment or a dot segment (`.` or `..`), which a URL parser resolves +/// to another endpoint even when percent-encoded. Methods that take such a value use +/// `to_path_segment` instead. pub(crate) fn encode_path_segment(input: &str) -> String { - // Path segments and query components share the same unreserved set here, so reuse the - // query encoder. (`url_encode` already escapes `/`, `?`, `#`, space and non-ASCII.) + // Path segments and query components share the unreserved set. url_encode(input) } -/// Validates and percent-encodes a caller-supplied URL path segment so it cannot restructure -/// the request path. -/// -/// An empty value is rejected because it would address the collection instead of a resource, and -/// so is `.` or `..`: URL parsers resolve dot segments after percent-decoding, so -/// `records/%2E%2E` collapses to the parent endpoint just like `records/..`. The error is -/// [`ApifyClientError::InvalidArgument`]. +/// Validates and percent-encodes a caller-supplied path segment. An empty segment, which would +/// address the collection, and `.` or `..`, which URL parsers resolve even when percent-encoded, +/// fail with [`ApifyClientError::InvalidArgument`]. pub(crate) fn to_path_segment(input: &str) -> ApifyClientResult { if input.is_empty() || input == "." || input == ".." { return Err(ApifyClientError::InvalidArgument(format!( @@ -161,7 +147,7 @@ pub(crate) fn to_path_segment(input: &str) -> ApifyClientResult { Ok(encode_path_segment(input)) } -/// Standard offset/limit pagination options shared by most list endpoints. +/// Offset/limit pagination options of most list endpoints. #[derive(Debug, Default, Clone)] pub struct ListOptions { /// Number of items to skip from the beginning of the list. @@ -172,9 +158,8 @@ pub struct ListOptions { pub desc: Option, } -/// Options shared by the storage collection list endpoints (`GET /v2/datasets`, -/// `/v2/key-value-stores`, `/v2/request-queues`), which add `unnamed` and `ownership` -/// filters on top of the standard offset/limit pagination. +/// Options of the storage collection list endpoints (datasets, key-value stores and request +/// queues): the offset/limit pagination plus `unnamed` and `ownership` filters. #[derive(Debug, Default, Clone)] pub struct StorageListOptions { /// Number of items to skip from the beginning of the list. @@ -237,23 +222,17 @@ fn env_var_set(name: &str) -> bool { /// Builds the `User-Agent` header value mandated by the client requirements: /// `ApifyClient/{version} ({os}; {language version}); isAtHome/{isAtHome}`. pub(crate) fn build_user_agent(suffix: Option<&str>) -> String { - // The OS token must be byte-for-byte identical across all Apify clients, which means it must - // equal the reference JS client's `os.platform()` value. `std::env::consts::OS` uses Rust's - // own spellings (`macos`, `windows`, `solaris`, `illumos`), so map those to the Node - // `os.platform()` tokens (`darwin`, `win32`, `sunos`); the rest already agree. + // The OS token is the `os.platform()` value of Node.js, as in every Apify client. + // `std::env::consts::OS` spells a few differently. let os = map_os_to_node_platform(std::env::consts::OS); - // The `isAtHome` flag signals whether the client runs on the Apify platform. Per the - // requirements it is `true`/`false` based solely on the `APIFY_IS_AT_HOME` environment - // variable (`false` when the variable is missing), matching the JS reference, which reads - // only `APIFY_IS_AT_HOME` and renders a lowercase boolean. + // `isAtHome` depends only on `APIFY_IS_AT_HOME` and is `false` when it is missing. let is_at_home = if env_var_set("APIFY_IS_AT_HOME") { "true" } else { "false" }; - // Rust has no stable runtime-version API, so report the compiler version captured at build - // time by `build.rs` (the closest analogue to a "language/runtime version"). Falls back to - // "unknown" only if the build script could not invoke rustc. + // Rust has no runtime-version API, so this is the compiler version `build.rs` captured, or + // "unknown" if it could not run rustc. let rust_version = option_env!("BUILD_RUSTC_VERSION").unwrap_or("unknown"); let mut ua = format!("ApifyClient/{CLIENT_VERSION} ({os}; Rust/{rust_version}); isAtHome/{is_at_home}"); @@ -268,12 +247,9 @@ pub(crate) fn build_user_agent(suffix: Option<&str>) -> String { /// Maps a Rust `std::env::consts::OS` value to the corresponding Node.js `os.platform()` token. /// -/// The User-Agent OS field must be identical across every Apify client, and the reference client -/// derives it from Node's `os.platform()`. Rust and Node agree on most tokens (`linux`, `android`, -/// `freebsd`, `openbsd`, `netbsd`, `aix`, ...), which pass through unchanged; the four Rust -/// spellings that differ (`macos`, `windows`, `solaris`, `illumos`) are translated to Node's -/// spelling. Note both `solaris` and `illumos` map to Node's single `sunos` token. Any unknown -/// value passes through as-is (best effort). +/// Rust and Node.js agree on most tokens, which pass through unchanged. The four that differ +/// (`macos`, `windows`, `solaris`, `illumos`) are translated to `darwin`, `win32`, `sunos` and +/// `sunos`. An unknown value passes through as it is. fn map_os_to_node_platform(os: &str) -> &str { match os { "macos" => "darwin", @@ -290,21 +266,17 @@ pub(crate) fn to_safe_id(id: &str) -> String { id.replace('/', "~") } -/// Version tag embedded in storage-content signatures. Matches the upstream library default -/// (`0`), which the reference clients rely on by not passing an explicit version. +/// Version tag embedded in storage-content signatures: the default of `@apify/utilities`. const STORAGE_CONTENT_SIGNATURE_VERSION: &str = "0"; /// Number of leading hex characters of the HMAC digest used by `create_hmac_signature`. const HMAC_SIGNATURE_HEX_LEN: usize = 30; -/// base62 alphabet (`0-9a-zA-Z`, lowercase first) used to encode the truncated HMAC. This -/// ordering matches upstream `@apify/utilities`. +/// base62 alphabet (`0-9a-zA-Z`, lowercase first) used to encode the truncated HMAC, in the order +/// `@apify/utilities` uses. const BASE62_ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; -/// Computes an Apify URL-signing signature, byte-for-byte compatible with the platform's -/// `@apify/utilities` `createHmacSignature`. -/// -/// The algorithm is: `HMAC-SHA256(secret_key, message)` as lowercase hex, take the first 30 -/// hex characters, interpret them as a big integer, then base62-encode (alphabet `0-9a-zA-Z`). -/// Used to sign key-value-store record keys for public access. +/// Computes an Apify URL-signing signature, compatible with `createHmacSignature` of +/// `@apify/utilities`: the first 30 hex characters of `HMAC-SHA256(secret_key, message)`, +/// base62-encoded. It signs the record keys of public key-value-store URLs. pub(crate) fn create_hmac_signature(secret_key: &str, message: &str) -> String { use hmac::{Hmac, Mac}; use sha2::Sha256; @@ -314,7 +286,7 @@ pub(crate) fn create_hmac_signature(secret_key: &str, message: &str) -> String { mac.update(message.as_bytes()); let digest = mac.finalize().into_bytes(); - // Hex-encode and keep the leading 30 hex chars (120 bits → fits in u128). + // 30 hex characters are 120 bits, which fit in a `u128`. let mut hex = String::with_capacity(digest.len() * 2); for byte in digest.iter() { hex.push_str(&format!("{byte:02x}")); @@ -343,16 +315,14 @@ fn to_base62(mut value: u128) -> String { /// Milliseconds in a second. const MILLIS_PER_SECOND: i64 = 1000; -/// The longest validity of a signed public URL, in seconds: `4611686018427387`, about 146 million -/// years. It is half of what fits in milliseconds in an `i64`, which keeps the current time plus -/// the validity from overflowing. The `expires_in_secs` of `create_items_public_url`, -/// `create_items_public_url_with_format` and `create_keys_public_url` can be at most this. +/// The longest validity of a signed public URL, in seconds (about 146 million years): half of what +/// fits in milliseconds in an `i64`, so the current time plus the validity can't overflow. It is +/// the upper bound of the `expires_in_secs` of the public URL methods. pub const MAX_EXPIRES_IN_SECS: i64 = i64::MAX / MILLIS_PER_SECOND / 2; -/// Checks the validity of a signed public URL before any request is sent. `None` is a URL that -/// does not expire. A validity has to be positive, since `0` and negative values would sign a URL -/// that has already expired, and at most [`MAX_EXPIRES_IN_SECS`], so that its expiry can't -/// overflow. Fails with [`ApifyClientError::InvalidArgument`] otherwise. +/// Checks the validity of a signed public URL before any request is sent. `None` never expires. +/// A validity must be positive, since `0` or less signs a URL that has already expired, and at +/// most [`MAX_EXPIRES_IN_SECS`]. Fails with [`ApifyClientError::InvalidArgument`] otherwise. pub(crate) fn validate_expires_in_secs(expires_in_secs: Option) -> ApifyClientResult<()> { match expires_in_secs { Some(secs) if secs <= 0 => Err(ApifyClientError::InvalidArgument(format!( @@ -365,10 +335,10 @@ pub(crate) fn validate_expires_in_secs(expires_in_secs: Option) -> ApifyCli } } -/// The expiry that a signed public URL gets, as the absolute time in milliseconds: the current -/// time plus the validity, or `0` for a URL that does not expire. Call it when the URL is signed, -/// so that the time the request before it took doesn't come off the validity, with a validity that -/// passed [`validate_expires_in_secs`]. +/// The expiry of a signed public URL as an absolute time in milliseconds: now plus the validity, +/// or `0` for a URL that does not expire. Call it when the URL is signed, so the time of earlier +/// requests doesn't come off the validity. The validity must have passed +/// [`validate_expires_in_secs`]. pub(crate) fn public_url_expiry_millis(expires_in_secs: Option) -> i64 { match expires_in_secs { Some(secs) => chrono::Utc::now().timestamp_millis() + secs * MILLIS_PER_SECOND, @@ -376,13 +346,12 @@ pub(crate) fn public_url_expiry_millis(expires_in_secs: Option) -> i64 { } } -/// Builds a storage-content signature for a resource's public URL, byte-for-byte compatible -/// with the platform's `@apify/utilities` `createStorageContentSignature`. +/// Builds a storage-content signature for the public URL of dataset items or a key list, +/// compatible with `createStorageContentSignature` of `@apify/utilities`. /// -/// It signs the message `"{version}.{expiresAtMillis}.{resourceId}"` (where `expiresAtMillis` -/// is the absolute expiry in milliseconds, as [`public_url_expiry_millis`] returns it) with -/// [`create_hmac_signature`], then returns the base64url (no padding) encoding of -/// `"{version}.{expiresAtMillis}.{hmac}"`. Used for dataset-items and key-list public URLs. +/// It signs `"{version}.{expiresAtMillis}.{resourceId}"` with [`create_hmac_signature`] and returns +/// the unpadded base64url encoding of `"{version}.{expiresAtMillis}.{hmac}"`. `expiresAtMillis` is +/// what [`public_url_expiry_millis`] returns. pub(crate) fn sign_storage_content( secret_key: &str, resource_id: &str, @@ -401,10 +370,8 @@ pub(crate) fn sign_storage_content( mod user_agent_tests { use super::{build_user_agent, map_os_to_node_platform}; - // `build_user_agent` keys the flag solely on the `APIFY_IS_AT_HOME` env var (per the - // requirements and the JS reference). A bare `isAtHome` env var must NOT affect it. The value - // is the lowercase `true`/`false` the JS reference emits. Env vars are process-global, so this - // test owns both names for its duration and restores them afterwards. + // Only `APIFY_IS_AT_HOME` sets the flag, not a bare `isAtHome` variable. Environment variables + // are process-global, so this test owns both names and restores them afterwards. #[test] fn is_at_home_reads_only_apify_is_at_home() { let prev_apify = std::env::var("APIFY_IS_AT_HOME").ok(); @@ -444,11 +411,8 @@ mod user_agent_tests { } } - // The OS token in the emitted User-Agent must match the reference JS client's `os.platform()` - // value (so every Apify client reports the same token). This test checks the token that - // `build_user_agent` actually produces against an independent oracle — the set of documented - // Node `os.platform()` values — and asserts it is lowercase and never a Rust-native spelling. - // The exact per-platform translation is covered by `os_token_maps_rust_spellings_to_node_platform`. + // The OS token must be a documented Node.js `os.platform()` value. The per-platform mapping is + // covered by `os_token_maps_rust_spellings_to_node_platform`. #[test] fn user_agent_os_token_matches_node_platform() { // The values Node's `os.platform()` can return (independent of our mapping code). @@ -490,9 +454,7 @@ mod user_agent_tests { } } - // The Rust->Node platform mapping must translate the four tokens that differ (both `solaris` - // and `illumos` collapse to Node's `sunos`) and leave every shared token untouched, so the - // emitted value is byte-for-byte identical to the reference. + // The four differing tokens are translated and the shared ones are left untouched. #[test] fn os_token_maps_rust_spellings_to_node_platform() { assert_eq!(map_os_to_node_platform("macos"), "darwin"); @@ -505,9 +467,7 @@ mod user_agent_tests { } } - // The `{language version}` segment must carry a real compiler version captured by build.rs, - // not the literal "unknown" the MSRV-based source used to produce. We assert the `Rust/` - // token is followed by a digit (e.g. `Rust/1.94.1`). + // The `Rust/` token must be followed by a compiler version such as `1.94.1`, not "unknown". #[test] fn user_agent_reports_real_rust_version() { let ua = build_user_agent(None); @@ -527,12 +487,10 @@ mod user_agent_tests { mod signature_tests { use super::{create_hmac_signature, sign_storage_content, to_base62}; - // Known-answer test pinning the `createHmacSignature` algorithm against upstream - // `@apify/utilities`: HMAC-SHA256 → hex → first 30 hex chars → big integer → base62 with - // the lowercase-first alphabet `0-9a-zA-Z`. For key="secret", msg="message" the upstream - // output is `11GYWmGxviysIBMtnQHBk`. + // Known answer of `createHmacSignature` in `@apify/utilities` for key "secret" and message + // "message". #[test] - fn hmac_signature_matches_upstream_scheme() { + fn hmac_signature_matches_the_known_answer() { let sig = create_hmac_signature("secret", "message"); assert_eq!(sig, "11GYWmGxviysIBMtnQHBk"); // base62 output uses only `0-9a-zA-Z`. diff --git a/src/error.rs b/src/error.rs index e958697..60f12b7 100644 --- a/src/error.rs +++ b/src/error.rs @@ -38,10 +38,9 @@ pub(crate) struct ApiErrorDetail { /// An error response returned by the Apify API. /// -/// This is raised for HTTP requests that reach the API but return a non-success -/// status code. It mirrors the `ApifyApiError` of the reference clients and exposes -/// the parsed error `type`, the human-readable `message`, the HTTP `status_code`, -/// the number of the final `attempt`, and the request `http_method`/`path`. +/// Returned when a request reaches the API and gets a non-success status. It holds the error +/// `type` and `message`, the HTTP `status_code`, the number of the final `attempt`, and the request +/// `http_method` and `path`. #[derive(Debug, Clone)] pub struct ApiError { /// HTTP status code of the error response. @@ -76,10 +75,6 @@ impl std::error::Error for ApiError {} impl ApiError { /// `true` for HTTP 400 Bad Request, typically because the request failed validation. - /// - /// The reference clients throw a distinct `InvalidRequestError` subclass for this status; - /// since Rust has no exception hierarchy to mirror, these `is_*` predicates are the - /// idiomatic equivalent classification on the one [`ApiError`] type. pub fn is_invalid_request(&self) -> bool { self.status_code == BAD_REQUEST_STATUS_CODE } @@ -124,13 +119,12 @@ impl ApiError { } } -/// The top-level error type for all client operations: every method returns -/// `Result`, so one `match`/`if let` already catches every kind of failure. +/// The error type of every client method. #[derive(Debug, thiserror::Error)] pub enum ApifyClientError { /// The API returned a non-success status code with a structured error body. /// - /// Boxed to keep the overall `Result` size small (the error path is rare). + /// Boxed to keep `Result` small. #[error(transparent)] Api(Box), diff --git a/src/http_client.rs b/src/http_client.rs index c504c75..9ecff37 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -1,15 +1,12 @@ //! The HTTP layer of the client. //! -//! The [`HttpBackend`] trait defines the minimal contract for sending a single HTTP -//! request and receiving a response. It is the *replaceable component* of the client: -//! the default implementation [`ReqwestBackend`] uses [`reqwest`], but a custom -//! backend (e.g. for testing or for a different runtime) can be plugged in via +//! The [`HttpBackend`] trait sends a single HTTP request and returns its response. The default +//! implementation, [`ReqwestBackend`], uses [`reqwest`]. Plug in another one with //! [`ApifyClientBuilder::http_backend`](crate::ApifyClientBuilder::http_backend). //! -//! The client wraps a backend in an internal `HttpClient` that adds the cross-cutting concerns shared by every -//! endpoint: authentication, the `User-Agent` header, query-parameter serialization, -//! timeouts and retries with exponential backoff (mirroring the JavaScript and Python -//! reference clients). +//! The client wraps a backend in an internal `HttpClient` that adds what every endpoint shares: +//! authentication, the `User-Agent` header, query parameters, timeouts, and retries with +//! exponential backoff. use std::collections::HashMap; use std::sync::Arc; @@ -32,8 +29,7 @@ const AUTHORIZATION_HEADER: &str = "Authorization"; /// The header that carries the API token to an Actor's server. The platform removes it before the /// request reaches the Actor, which keeps `Authorization` free for the Actor's own authentication. const X_APIFY_AUTHORIZATION_HEADER: &str = "X-Apify-Authorization"; -/// Multiplier applied to the inter-retry delay after each attempt (exponential backoff). -/// Matches the reference client's `async-retry` default factor of 2. +/// Multiplier applied to the delay between retries after each attempt. const BACKOFF_FACTOR: u32 = 2; /// HTTP method of a request. @@ -73,10 +69,8 @@ impl HttpMethod { } } -/// A fully-resolved HTTP request, ready to be sent by an [`HttpBackend`]. -/// -/// All cross-cutting concerns (auth header, user agent, retry policy) are applied by -/// the client before the request reaches the backend. +/// An HTTP request ready to be sent by an [`HttpBackend`]. The client has already applied the auth +/// header, the user agent and the compression. #[derive(Debug, Clone)] pub struct HttpRequest { /// The HTTP method. @@ -146,11 +140,8 @@ pub struct StreamingHttpResponse { std::pin::Pin>> + Send>>, } -/// The replaceable transport contract. -/// -/// Implementors are responsible only for sending a single request and returning the -/// raw response. Retries, authentication and serialization are handled by -/// the client, so a backend only needs to perform one network round-trip. +/// The transport that sends requests. An implementation performs one network round-trip per +/// request: the client handles retries, authentication and serialization. #[async_trait] pub trait HttpBackend: Send + Sync + std::fmt::Debug { /// Sends a single HTTP request and returns the response. @@ -163,11 +154,8 @@ pub trait HttpBackend: Send + Sync + std::fmt::Debug { /// Like [`send`](Self::send), but for a request whose body the caller wants to process /// incrementally (a live log or a large export) instead of fully buffered. /// - /// The default implementation just buffers the whole response via [`send`](Self::send) and - /// yields it as a single chunk: correct for a custom backend that does not override this - /// (streaming still works end-to-end, just without a memory-usage benefit), while the - /// default [`ReqwestBackend`] overrides it for real constant-memory streaming through - /// `reqwest`'s own chunked body reader. + /// The default implementation buffers the whole response with [`send`](Self::send) and yields + /// it as one chunk. [`ReqwestBackend`] overrides it to stream the body in constant memory. async fn send_streaming( &self, request: HttpRequest, @@ -301,11 +289,9 @@ impl HttpBackend for ReqwestBackend { ) -> ApifyClientResult { let method = to_reqwest_method(request.method); - // Deliberately no `.timeout(request.timeout)` here: reqwest's per-request timeout bounds - // the *entire* request including reading the whole body (it wraps the body stream in a - // `TotalTimeoutBody`), which would cut off a long-but-active stream (e.g. live log - // redirection). `HttpClient` applies its own response-start bound and, unless the call - // opts out, a per-chunk-inactivity bound instead, uniformly across backends. + // No `.timeout(request.timeout)`: reqwest's timeout covers reading the whole body, which + // would cut off a long but active stream such as a live log. `HttpClient` bounds the wait + // for the response and, unless the call opts out, the silence between chunks instead. let mut builder = self.client_for(&request).request(method, &request.url); for (key, value) in &request.headers { builder = builder.header(key, value); @@ -546,10 +532,9 @@ impl HttpClient { /// Opens a streaming connection through the configured [`HttpBackend`]'s /// [`HttpBackend::send_streaming`], with authentication, the user-agent header and the retry /// policy applied like [`call_with_options`](Self::call_with_options), but with no request - /// compression. Like the reference clients, it retries until the response starts (an error - /// status or a transport error before it opens), and never once the body is being read: an - /// open stream, such as a log redirection or a large export already in flight, can't be - /// safely replayed from the start. + /// compression. It retries until the response starts (an error status or a transport error + /// before it opens), and never once the body is being read, since an open stream can't be + /// replayed from the start. /// /// On a non-success status, reads the body (bounded by the same inactivity timeout as a /// successful body below) and returns the same structured [`ApiError`] [`call_with_options`](Self::call_with_options) @@ -654,11 +639,8 @@ impl HttpClient { return Err(error); } - // Sleep with randomized exponential backoff before the next attempt. The backoff - // doubles each retry (matching the reference client, which uses `async-retry` with - // a factor of 2) and is capped at `timeout_max`. + // The backoff doubles each retry and is capped at `timeout_max`. sleep(randomized_delay(delay)).await; - // `saturating_mul` mirrors the saturating arithmetic in `attempt_timeout`. delay = delay.saturating_mul(BACKOFF_FACTOR).min(self.tiers.max); attempt += 1; } @@ -668,8 +650,7 @@ impl HttpClient { /// /// The first attempt uses the `base` timeout of the request; each retry doubles it so a /// slow-but-progressing connection gets more time, while never exceeding the client's - /// `timeout_max`. A request without a timeout ([`NO_TIMEOUT`]) stays without one. Mirrors the - /// reference clients. + /// `timeout_max`. A request without a timeout ([`NO_TIMEOUT`]) stays without one. fn attempt_timeout(&self, base: Duration, attempt: u32) -> Duration { if base == NO_TIMEOUT { return NO_TIMEOUT; @@ -848,9 +829,8 @@ fn extract_path(url: &str) -> Option { .map(|idx| after_scheme[idx..].to_string()) } -/// We retry `429` (rate limit) and `5xx` (internal server errors), matching the -/// reference client policy, unless the request lists its own `retry_status_codes`. Other `4xx` -/// statuses are caller errors and are not retried. +/// Retries `429` (rate limit) and `5xx`, unless the request lists its own `retry_status_codes`. +/// Other `4xx` statuses are caller errors and are not retried. fn is_status_retryable(status: u16, retry_status_codes: Option<&[u16]>) -> bool { match retry_status_codes { Some(codes) => codes.contains(&status), @@ -858,9 +838,8 @@ fn is_status_retryable(status: u16, retry_status_codes: Option<&[u16]>) -> bool } } -/// Only transport-level failures are retryable. Programming errors (serde, invalid -/// argument) and already-classified API errors are handled elsewhere, so they are not -/// retried here — matching the reference clients, which retry only network/timeout errors. +/// Only transport failures (network errors and timeouts) are retried. Serialization errors, +/// invalid arguments and API errors are not. fn is_error_retryable(err: &ApifyClientError) -> bool { matches!(err, ApifyClientError::Http(_) | ApifyClientError::Timeout) } @@ -903,11 +882,8 @@ fn build_api_error( } } -/// Returns a delay chosen randomly from the interval `[delay, 2*delay)`, matching the -/// exponential-backoff-with-jitter algorithm described in the API docs. -/// -/// `pub(crate)` so other retrying call sites (e.g. `RequestQueueClient::batch_add_requests`'s -/// unprocessed-request retries) can reuse the same jitter source instead of duplicating it. +/// Returns a delay chosen randomly from the interval `[delay, 2*delay)`, the jitter of the API's +/// recommended exponential backoff. Shared with the retries of `batch_add_requests`. pub(crate) fn randomized_delay(delay: Duration) -> Duration { let base = delay.as_millis() as u64; if base == 0 { @@ -917,20 +893,15 @@ pub(crate) fn randomized_delay(delay: Duration) -> Duration { Duration::from_millis(base + extra) } -/// A process-wide pseudo-random source for backoff jitter. -/// -/// Backoff jitter does not need cryptographic quality, but it must be well-distributed and -/// uncorrelated across concurrent retries (otherwise many clients retry in lockstep). A -/// shared atomically-advanced SplitMix64 generator, seeded once from the clock, gives each -/// caller a distinct value without pulling in a heavyweight RNG dependency. +/// A process-wide SplitMix64 source for backoff jitter. It is not cryptographic, but it gives +/// concurrent retries distinct values so they don't run in lockstep, without an RNG dependency. fn next_jitter() -> u64 { use std::sync::atomic::{AtomicU64, Ordering}; static STATE: AtomicU64 = AtomicU64::new(0); const GOLDEN_GAMMA: u64 = 0x9E3779B97F4A7C15; - // Lazily seed from the clock on first use. A racing double-seed is harmless: both - // candidate seeds are valid SplitMix64 stream starting points. + // Seed from the clock on first use. Two racing seeds are both valid starting points. if STATE.load(Ordering::Relaxed) == 0 { let seed = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -940,9 +911,7 @@ fn next_jitter() -> u64 { let _ = STATE.compare_exchange(0, seed, Ordering::Relaxed, Ordering::Relaxed); } - // SplitMix64: advance the shared state by the golden-ratio increment in a single atomic - // read-modify-write (`fetch_add`) so concurrent callers each observe a distinct value — - // a plain load-then-store could hand two racing retries the same number. Then scramble. + // A single atomic `fetch_add` gives each concurrent caller a distinct value. let mut z = STATE .fetch_add(GOLDEN_GAMMA, Ordering::Relaxed) .wrapping_add(GOLDEN_GAMMA); @@ -951,7 +920,7 @@ fn next_jitter() -> u64 { z ^ (z >> 31) } -/// Sleeps for the given duration (public crate-internal helper for poll loops). +/// Sleeps for the given duration. pub(crate) async fn sleep_public(duration: Duration) { sleep(duration).await; } diff --git a/src/http_compressors.rs b/src/http_compressors.rs index 14a32ea..8c7aeb6 100644 --- a/src/http_compressors.rs +++ b/src/http_compressors.rs @@ -12,9 +12,8 @@ use std::io::Write; use crate::error::{ApifyClientError, ApifyClientResult}; -/// Request bodies at least this many bytes are compressed before sending. Smaller bodies are -/// left uncompressed because the CPU cost outweighs the transfer savings. Matches the reference -/// client's `MIN_COMPRESS_BYTES` threshold. +/// Request bodies of at least this many bytes are compressed. For smaller ones the CPU cost +/// outweighs the transfer savings. pub(crate) const MIN_COMPRESS_BYTES: usize = 1024; /// `Content-Encoding` value used for brotli-compressed request bodies. const CONTENT_ENCODING_BROTLI: &str = "br"; @@ -24,8 +23,7 @@ const CONTENT_ENCODING_GZIP: &str = "gzip"; const BROTLI_MIN_QUALITY: u32 = 0; /// Highest brotli quality, the slowest with the best compression. const BROTLI_MAX_QUALITY: u32 = 11; -/// Brotli quality used by [`BrotliHttpCompressor::default`]. Quality 6 balances the compression -/// ratio against CPU cost, matching the reference client. +/// Brotli quality used by [`BrotliHttpCompressor::default`], balancing ratio against CPU cost. const BROTLI_DEFAULT_QUALITY: u32 = 6; /// Brotli sliding-window size (log2), 22 is the library default (a 4 MiB window). const BROTLI_WINDOW_SIZE: u32 = 22; @@ -37,9 +35,8 @@ const CONTENT_ENCODING_ZSTD: &str = "zstd"; const GZIP_MIN_QUALITY: u32 = 1; /// Highest gzip quality, the slowest with the best compression. const GZIP_MAX_QUALITY: u32 = 9; -/// Gzip quality used by [`GzipHttpCompressor::default`]: zlib's own default, which the reference -/// client uses too. The levels above it roughly double the CPU cost for about a percent fewer -/// bytes. +/// Gzip quality used by [`GzipHttpCompressor::default`]: zlib's own default. The levels above it +/// roughly double the CPU cost for about a percent fewer bytes. const GZIP_DEFAULT_QUALITY: u32 = 6; /// Lowest zstd level, the fastest with the least compression. const ZSTD_MIN_QUALITY: u32 = 1; @@ -47,19 +44,15 @@ const ZSTD_MIN_QUALITY: u32 = 1; /// window above the 8 MB that RFC 9659 requires HTTP decoders to support, so a body compressed at /// those levels could fail to decode. const ZSTD_MAX_QUALITY: u32 = 19; -/// Zstd level used by [`ZstdHttpCompressor::default`]: zstd's own default, which the reference -/// client uses too. +/// Zstd level used by [`ZstdHttpCompressor::default`]: zstd's own default. const ZSTD_DEFAULT_QUALITY: u32 = 3; -/// Media-type prefixes whose payloads already carry their own compression, so running them -/// through brotli/gzip burns CPU and memory for a result that is usually no smaller — and the -/// request also keeps the intended `Content-Type`, rather than becoming an unreadable -/// `Content-Encoding: br` blob the destination may not expect for these formats. -/// Matches the reference client's `ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES`. +/// Media-type prefixes whose payloads already carry their own compression, so compressing them +/// again costs CPU and memory for a result that is usually no smaller. const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES: [&str; 3] = ["audio/", "image/", "video/"]; /// Exact media types (outside the prefixes above) whose payloads already carry their own -/// compression. Matches the reference client's `ALREADY_COMPRESSED_MEDIA_TYPES`. +/// compression. const ALREADY_COMPRESSED_MEDIA_TYPES: [&str; 19] = [ "application/epub+zip", "application/gzip", @@ -83,7 +76,7 @@ const ALREADY_COMPRESSED_MEDIA_TYPES: [&str; 19] = [ ]; /// Uncompressed media types that sit under an already-compressed prefix above, so compressing -/// them still pays off. Matches the reference client's `COMPRESSIBLE_MEDIA_TYPES`. +/// them still pays off. const COMPRESSIBLE_MEDIA_TYPES: [&str; 16] = [ "audio/aiff", "audio/basic", @@ -104,8 +97,7 @@ const COMPRESSIBLE_MEDIA_TYPES: [&str; 16] = [ ]; /// Structured-syntax suffixes that mark a media type as text even under an already-compressed -/// prefix (e.g. `image/svg+xml`). Matches the reference client's -/// `COMPRESSIBLE_MEDIA_TYPE_SUFFIXES`. +/// prefix (e.g. `image/svg+xml`). const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES: [&str; 2] = ["+json", "+xml"]; /// Strategy for compressing request bodies. @@ -257,8 +249,7 @@ impl HttpCompressor for ZstdHttpCompressor { #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[non_exhaustive] pub enum RequestCompression { - /// Brotli (`Content-Encoding: br`). The default: best compression ratio, and the encoding the - /// reference client prefers. + /// Brotli (`Content-Encoding: br`). The default, with the best compression ratio. #[default] Brotli, /// Gzip (`Content-Encoding: gzip`). Choose this for environments or intermediaries that do not @@ -292,9 +283,6 @@ fn check_quality(algorithm: &str, quality: u32, min: u32, max: u32) -> ApifyClie /// Compresses `body` with `compressor` and adds the matching `Content-Encoding` header, when the /// body is large enough, no `Content-Encoding` is already set, and its content type is not /// already compressed. Returns the (possibly unchanged) body. -/// -/// The size threshold and the "compress once, before retries" behaviour mirror the reference -/// client. pub(crate) fn maybe_compress( compressor: &dyn HttpCompressor, headers: &mut HashMap, @@ -337,7 +325,7 @@ pub(crate) fn maybe_compress( /// brotli or gzip burns CPU, holds a second full copy of the body in memory, and usually produces /// output no smaller than the input (sometimes larger). Formats that are raw despite such a media /// type, e.g. `image/bmp` or `audio/wav`, are still compressed. A body with no content type is -/// assumed to be compressible. Matches the reference client's `isCompressibleContentType`. +/// assumed to be compressible. fn is_compressible_content_type(content_type: Option<&str>) -> bool { let Some(content_type) = content_type else { return true; @@ -392,7 +380,7 @@ mod tests { } #[test] - fn default_qualities_are_the_zlib_and_brotli_defaults_of_the_reference_client() { + fn default_qualities_are_the_documented_defaults() { let data: Vec = (0..4096u32).map(|i| (i % 251) as u8).collect(); let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(6)); diff --git a/src/lib.rs b/src/lib.rs index 72fe82f..4010cba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,10 +2,9 @@ //! //! An idiomatic Rust client for the [Apify API](https://docs.apify.com/api/v2). //! -//! It provides a resource-oriented interface that mirrors the official -//! [JavaScript](https://github.com/apify/apify-client-js) and Python clients: start from -//! an [`ApifyClient`], then drill down into resources (Actors, runs, datasets, key-value -//! stores, request queues, tasks, schedules, webhooks, the store, users and logs). +//! It follows the interface of the [JavaScript client](https://github.com/apify/apify-client-js): +//! start from an [`ApifyClient`], then drill down into resources (Actors, runs, datasets, +//! key-value stores, request queues, tasks, schedules, webhooks, the store, users and logs). //! //! ## Quick start //! @@ -37,23 +36,17 @@ //! //! - **Public interface**: [`ApifyClient`] and the resource clients in [`clients`]. //! - **Replaceable transport**: the [`http_client::HttpBackend`] trait, with a default -//! [`http_client::ReqwestBackend`]. Swap it via -//! [`ApifyClientBuilder::http_backend`]. -//! - **Cross-cutting behaviour** (auth, `User-Agent`, retries with exponential backoff, -//! timeouts, request compression) lives in the client's internal HTTP layer and is applied to every -//! request. +//! [`http_client::ReqwestBackend`]. Swap it via [`ApifyClientBuilder::http_backend`]. +//! - **Shared behaviour** (auth, `User-Agent`, retries with exponential backoff, timeouts, request +//! compression) is applied to every request by the client's internal HTTP layer. //! - **Timeouts**: every method uses a [`Timeout`] tier (`Short`, `Medium` or `Long`) whose //! duration is configured on the [`ApifyClientBuilder`]. A resource client's `with_timeout` //! overrides the tier of all its requests. //! //! ## Cancellation //! -//! The reference JavaScript client accepts an `AbortSignal` option on most methods so a caller -//! can cancel an in-flight request. Rust futures are cancel-safe by construction: they do -//! nothing until polled, so dropping one (e.g. via `tokio::select!`, or letting a -//! `tokio::time::timeout` elapse) stops the underlying request without any API for it. This -//! client therefore exposes no separate cancellation parameter — dropping the future returned -//! by any method is the idiomatic equivalent: the request in flight ends and no retry or further +//! Methods take no cancellation parameter. Drop the future a method returns, for example through +//! `tokio::select!` or `tokio::time::timeout`, to end the request in flight: no retry or further //! poll follows. The built-in waits, such as [`RunClient::wait_for_finish`], are futures too. //! //! [`RunClient::wait_for_finish`]: crate::clients::run::RunClient::wait_for_finish @@ -82,7 +75,6 @@ pub use statistics::Statistics; pub use timeouts::Timeout; pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; -// Re-export the most commonly used option/parameter types for ergonomic access. pub use clients::actor::{ ActorBuildOptions, ActorCallOptions, ActorStartOptions, TaskStartOptions, WaitForResources, }; diff --git a/src/models.rs b/src/models.rs index 61d1b24..1950504 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,10 +1,8 @@ //! Data models for Apify API resources. //! -//! Each resource is modelled with the fields most commonly used by clients, mirroring -//! the reference JavaScript client. To remain forward-compatible with additive changes to -//! the API, none of the models set `deny_unknown_fields`, so unknown API fields are ignored -//! rather than breaking deserialization. Most resource models additionally capture any unknown -//! fields in an `extra` map via `#[serde(flatten)]` so they remain accessible to callers. +//! Each resource is modelled with its most commonly used fields. No model sets +//! `deny_unknown_fields`, so a new API field never breaks deserialization. Most resource models +//! also keep unknown fields in an `extra` map. use std::collections::HashMap; @@ -642,7 +640,7 @@ pub struct ProcessedRequest { /// The request's unique key. #[serde(default, skip_serializing_if = "Option::is_none")] pub unique_key: Option, - /// The request's ID, as returned by a batch **add** (mirrors the API's `requestId`). + /// The request's ID, as returned by a batch **add**. #[serde(default, skip_serializing_if = "Option::is_none")] pub request_id: Option, /// Whether the request was already present in the queue (batch add only). diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f4bc744..dcf2467 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,12 +1,10 @@ //! Shared helpers for the integration test suite. //! -//! All integration tests require a valid `APIFY_TOKEN` for the test account. The API -//! base URL is taken from `APIFY_API_URL` (which includes the `/v2` suffix) and falls -//! back to `https://api.apify.com/v2`. +//! The integration tests need an `APIFY_TOKEN`. The API URL is `APIFY_API_URL`, including the `/v2` +//! suffix, and defaults to `https://api.apify.com/v2`. //! -//! Tests are designed to run concurrently — including against the same test account from -//! several language clients at once — so every test creates uniquely-named resources and -//! cleans them up afterwards. +//! The tests can run concurrently with each other and with other clients' tests on the same +//! account, so each one creates uniquely named resources and cleans them up. #![allow(dead_code)] @@ -67,12 +65,8 @@ macro_rules! require_client { }}; } -/// A panic-safe cleanup guard. -/// -/// Holds a deferred cleanup action that runs when the guard is dropped — including when a -/// test panics partway through (a failed `assert!`/`expect`), which would otherwise leak the -/// created resource on the shared test account. The action is run to completion on the test's -/// own (multi-thread) Tokio runtime, so it works inside `#[tokio::test]` bodies. +/// Runs a cleanup action when dropped, also when the test panics, so a failed test doesn't leak +/// its resources. It runs to completion on the test's multi-thread Tokio runtime. /// /// Usage: /// ```ignore @@ -154,22 +148,13 @@ pub const ITER_RETRY_ATTEMPTS: usize = 32; /// Delay between the attempts made by [`iter_contains_eventually`]. pub const ITER_RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_secs(3); -/// Like [`iter_contains`], but tolerant of replica lag in collection LIST endpoints. -/// -/// The list endpoints read from database secondaries that may trail the primary by up to 90 -/// seconds, so a resource created through a write endpoint is not always immediately reflected in -/// its collection's LIST response. A create-then-iterate test that scans the collection exactly -/// once therefore races that lag and flakes when the just-created entity has not yet propagated. -/// -/// This helper rebuilds a fresh iterator via `make_iter` and re-scans it with [`iter_contains`] up -/// to [`ITER_RETRY_ATTEMPTS`] times, sleeping [`ITER_RETRY_BACKOFF`] between attempts, returning -/// `true` as soon as `pred` matches. When the entity is already visible it matches on the first -/// attempt and returns immediately with no sleeping — so it is a no-op in the common -/// already-consistent case and only pays the backoff on the rare lagging run. +/// Like [`iter_contains`], but tolerates replica lag. The list endpoints read from secondaries that +/// can trail the primary by up to 90 seconds, so a just-created resource may be missing from the +/// first scan. /// -/// Budget: `(ITER_RETRY_ATTEMPTS - 1) * ITER_RETRY_BACKOFF` = 93s of retrying before giving up, -/// which covers the 90-second staleness window. The cost lands only on lagging or -/// genuinely-failing runs; a consistent account never sleeps. +/// It scans a fresh iterator from `make_iter` up to [`ITER_RETRY_ATTEMPTS`] times, sleeping +/// [`ITER_RETRY_BACKOFF`] between attempts, and returns `true` as soon as `pred` matches. The +/// retries add up to 93 seconds, which covers the staleness window. pub async fn iter_contains_eventually(mut make_iter: Mk, mut pred: F) -> bool where Mk: FnMut() -> apify_client::ListIterator, @@ -205,14 +190,9 @@ const ACTOR_NAME_HEAD_CAP: usize = 8; /// Generates a unique name that satisfies Apify's (stricter) Actor naming rules: starts with a /// letter and contains no hyphens, unlike the names [`unique_name`] produces for other resources. /// -/// Built directly from `prefix` (filtered to alphanumerics) plus a full -/// [`ACTOR_NAME_SUFFIX_LEN`]-hex-char random suffix, rather than truncating an already-formatted -/// `unique_name(prefix)` string: that would fix the random suffix's *position* but not its -/// *presence* - a short length cap could still truncate into the fixed `rust-test-` boilerplate -/// `unique_name` prepends, silently keeping the same literal head (and thus the same account-wide -/// collision domain, dropping the caller's `prefix` entirely) for every call regardless of -/// `prefix`. Building the name from `prefix` directly keeps collision resistance and -/// traceability-to-caller both guaranteed by construction. +/// It is the alphanumeric part of `prefix`, capped at [`ACTOR_NAME_HEAD_CAP`] characters, plus a +/// random suffix of [`ACTOR_NAME_SUFFIX_LEN`] hex characters. Truncating a `unique_name` instead +/// could cut into its fixed `rust-test-` head and drop the `prefix`. pub fn unique_actor_name(prefix: &str) -> String { let sanitized: String = prefix.chars().filter(char::is_ascii_alphanumeric).collect(); let head = &sanitized[..sanitized.len().min(ACTOR_NAME_HEAD_CAP)]; diff --git a/tests/examples.rs b/tests/examples.rs index 7994c48..caa68f9 100644 --- a/tests/examples.rs +++ b/tests/examples.rs @@ -1,10 +1,7 @@ //! Smoke tests that run the documentation examples end-to-end. //! -//! These ensure every example in `examples/` actually compiles and runs successfully -//! against the live API, satisfying the requirement that each documentation example has a -//! CI test. Each example is executed as a subprocess via `cargo run --example`. -//! -//! The longer Actor-build example is included but given a generous time budget. +//! Each example in `examples/` runs as a subprocess with `cargo run --example` against the live +//! API. The Actor-build example gets a longer time budget. mod common; diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 08e754e..28b4623 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -80,13 +80,9 @@ async fn iterate_key_value_stores() { /// Iteration: `iterate_keys` yields every key in the store. /// -/// Creates a store, writes several records, then drives the `KeyValueStoreKeysIterator` to -/// completion and asserts every written key is yielded exactly once. With only a handful of keys -/// this fits in the endpoint's single default page, so it validates the helper end to end but -/// does not cross a page boundary — `iterate_keys` has no per-page-size knob (its `limit` is a -/// total cap, matching the JS reference), so forcing >1 page would need >1000 keys. The -/// multi-page cursor-threading path (`exclusiveStartKey`/`nextExclusiveStartKey`) is covered -/// hermetically by `iterate_keys_walks_cursor_pages` in `tests/unit_http.rs`. +/// Writes several records and asserts that the iterator yields every key exactly once. The keys fit +/// in one page, since `iterate_keys` has no page size option, so the paging across several pages +/// is covered by `iterate_keys_walks_cursor_pages` in `tests/unit_http.rs`. #[tokio::test(flavor = "multi_thread")] async fn iterate_keys_yields_all_keys() { let client = require_client!(); @@ -142,9 +138,8 @@ async fn iterate_keys_yields_all_keys() { /// reserved in a URL path must round-trip correctly, proving the path segment is /// percent-encoded rather than interpolated raw. /// -/// The Apify API restricts record keys to `a-zA-Z0-9!-_.'()`; several of those (`!`, `'`, -/// `(`, `)`) are not RFC 3986 unreserved characters, so they must be percent-encoded in the -/// URL path. A raw `format!(".../{key}")` would send them unescaped. +/// The API allows `!`, `'`, `(` and `)` in record keys. They are not RFC 3986 unreserved +/// characters, so the client must percent-encode them. #[tokio::test(flavor = "multi_thread")] async fn record_key_with_special_chars_round_trips() { let client = require_client!(); @@ -278,18 +273,10 @@ async fn key_value_store_crud_flow() { .is_none()); } -/// `set_record_raw_with_content_encoding` uploads an already-compressed value without the -/// client compressing it a second time, and the value round-trips without corruption. -/// -/// The payload is well above the client's own auto-compression size threshold, so this also -/// exercises that the `Content-Encoding` header (not just the small-body skip) is what the -/// client relies on to avoid double-compression; `set_record_with_content_encoding_is_not_compressed_again` -/// in `tests/unit_http.rs` pins that exact wire behaviour hermetically. What the API does with -/// the encoded upload server-side (store the compressed bytes as-is per -/// `KeyValueStoreClient::set_record_raw_with_content_encoding`'s docs, or transparently decode -/// it before storing) is outside the client's control, so this only asserts the value it reads -/// back decodes to the original payload either way, never that the bytes are corrupted or -/// double-compressed. +/// A value uploaded with `set_record_raw_with_content_encoding` reads back as the original payload. +/// The payload is above the compression threshold. The API may store the bytes as they are or +/// decode them, so the test accepts either. The wire behavior is covered by +/// `set_record_with_content_encoding_is_not_compressed_again` in `tests/unit_http.rs`. #[tokio::test(flavor = "multi_thread")] async fn set_record_with_content_encoding_round_trips_without_corruption() { use flate2::{write::GzEncoder, Compression}; diff --git a/tests/task.rs b/tests/task.rs index d02f826..f6bc83d 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -87,13 +87,10 @@ async fn iterate_tasks() { ); } -/// `publish`/`unpublish` are thin wrappers around `update` that flip `isPublic`. Unpublishing an -/// already-unpublished task is a documented no-op, so it round-trips cleanly; publishing a task -/// (and, per the spec, editing `publicConfig` at all) requires write permission over its Actor. -/// Here that's the shared, Apify-owned `apify/hello-world` (per [`task_definition`]), so both are -/// expected to be rejected with `insufficient-permissions` - this exercises the same request path -/// without needing a private Actor the test account can actually publish. See -/// [`task_public_config_update`] for the `publicConfig`-editing path against an owned Actor. +/// `publish` and `unpublish` flip `isPublic` through `update`. Unpublishing an unpublished task is +/// a no-op. Publishing needs write permission to the task's Actor, which `apify/hello-world` (see +/// [`task_definition`]) does not give, so the API is expected to reject it with +/// `insufficient-permissions`. [`task_public_config_update`] covers an Actor the account owns. #[tokio::test(flavor = "multi_thread")] async fn task_publish_unpublish() { let client = require_client!(); @@ -157,17 +154,13 @@ fn owned_actor_definition(name: &str) -> serde_json::Value { }) } -/// Exercises `Task::public_config` (`TaskPublicConfig`): editing it via `update` requires write -/// permission to the task's Actor, so this uses a private Actor owned by the test account. Sets -/// the `seoTitle`/`seoDescription` metadata fields and reads them back typed - `inputSchemaFields`/ -/// `datasetView` are omitted since the API validates them against the Actor's input/dataset -/// schema (which this minimal Actor doesn't declare) even on a plain `publicConfig` edit, not -/// only on publish. Also checks that `unpublish` (a no-op here, since the task was never -/// published) leaves `publicConfig` untouched. +/// Edits `Task::public_config` of a task of a private Actor the account owns, which `update` +/// requires. It sets `seoTitle` and `seoDescription` and reads them back. `inputSchemaFields` and +/// `datasetView` are left out, since the API validates them against schemas this Actor lacks. +/// `unpublish` of the unpublished task must leave `publicConfig` untouched. /// -/// Task creation resolves the Actor's default (`latest`) build tag up front, so - unlike the -/// other Actor-owning tests in this suite, which only need the Actor to exist - the Actor must -/// actually be built first (mirrors `tests/build.rs::build_actor_flow`'s build-and-wait). +/// Creating a task resolves the `latest` build, so the Actor is built first, as in +/// `tests/build.rs::build_actor_flow`. #[tokio::test(flavor = "multi_thread")] async fn task_public_config_update() { let client = require_client!(); diff --git a/tests/unit_http.rs b/tests/unit_http.rs index f7d0e83..4f7eeff 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -227,9 +227,7 @@ async fn unparsable_error_body_is_retried_by_status_code() { ); } -/// `ApiError`'s `is_*` status-classification predicates (the idiomatic equivalent of the -/// reference client's `NotFoundError`/`RateLimitError`/... subclasses) match the response status, -/// and only that status's predicate is `true`. +/// The `is_*` predicates of `ApiError` match the response status, and only that one is `true`. #[tokio::test] async fn api_error_status_predicates_classify_by_status_code() { async fn error_for(status: u16) -> apify_client::ApiError { @@ -340,11 +338,8 @@ async fn zero_retries_single_attempt() { assert_eq!(backend.call_count(), 1); } -/// Regression guard for the validate-input envelope-skip fix: the `validate-input` endpoint -/// returns a **bare** `{"valid":...}` object with no `{"data":...}` envelope, so it must NOT be -/// routed through `parse_data_envelope` (which would fail with `missing field 'data'`). This is -/// token-free, so a future refactor that re-introduces envelope unwrapping is caught even in a -/// run without `APIFY_TOKEN`. +/// The `validate-input` endpoint returns a bare `{"valid":...}` object without the `data` +/// envelope, which must not fail to parse. #[tokio::test] async fn validate_input_does_not_unwrap_data_envelope() { let backend = MockBackend::new(vec![MockOutcome::Status( @@ -432,10 +427,9 @@ async fn log_get_sends_raw_query_param() { ); } -/// `log().stream()` goes through the configured `HttpBackend` — including a fully custom one, -/// not just `ReqwestBackend` — and applies the same auth/user-agent headers as a buffered call. -/// `MockBackend` does not override `send_streaming`, so this also exercises -/// `HttpBackend::send_streaming`'s default (buffer-via-`send`-and-yield-one-chunk) fallback. +/// `log().stream()` goes through a custom `HttpBackend` and sends the same auth and user-agent +/// headers as a buffered call. `MockBackend` does not override `send_streaming`, so this also +/// covers its default implementation. #[tokio::test] async fn log_stream_goes_through_the_configured_backend() { let backend = MockBackend::new(vec![MockOutcome::Status(200, b"some log output".to_vec())]); @@ -468,9 +462,8 @@ async fn log_stream_goes_through_the_configured_backend() { ); } -/// A failed `log().stream()` through a custom `HttpBackend` (not just `ReqwestBackend`) still -/// surfaces the same structured `ApiError` a buffered call would, built from the mock's 404 -/// response — proving the error path also goes through the configured backend. +/// A failed `log().stream()` through a custom `HttpBackend` returns the same structured +/// `ApiError` as a buffered call. #[tokio::test] async fn log_stream_through_mock_backend_surfaces_structured_error() { let backend = MockBackend::new(vec![MockOutcome::Status( @@ -595,10 +588,7 @@ async fn large_request_body_is_zstd_compressed_when_selected() { ); } -/// When `request_compression()` is never called on the builder, large bodies are brotli-compressed: -/// the builder's default encoding is brotli, so the backend sees `Content-Encoding: br`. This -/// exercises the builder default via the public builder path (not the `Default` derive), guarding -/// against the default silently changing. +/// Without `request_compression()`, large bodies are brotli-compressed. #[tokio::test] async fn builder_default_request_compression_is_brotli() { let backend = MockBackend::new(vec![MockOutcome::Status(200, b"".to_vec())]); @@ -643,10 +633,8 @@ async fn small_request_body_is_not_compressed() { ); } -/// A large body whose content type already carries its own compression (e.g. a ZIP archive) is -/// sent uncompressed, with no `Content-Encoding`: compressing it again would burn CPU for little -/// to no size reduction. Exercises the raw-bytes path via `ActorClient::start_raw`, which is the -/// main way a caller sets a non-JSON content type on a large body. +/// A large body whose content type is already compressed, such as a ZIP archive, is sent as it is, +/// through `ActorClient::start_raw`. #[tokio::test] async fn already_compressed_content_type_is_not_compressed() { let backend = MockBackend::new(vec![MockOutcome::Status( @@ -679,9 +667,8 @@ async fn already_compressed_content_type_is_not_compressed() { ); } -/// `set_record_raw_with_content_encoding` sends the caller's `Content-Encoding` verbatim and -/// does not compress the body a second time, even above the auto-compression size threshold: -/// `maybe_compress_request` skips any body that already carries a `Content-Encoding` header. +/// `set_record_raw_with_content_encoding` sends the caller's `Content-Encoding` as it is and does +/// not compress the body again, even above the compression threshold. #[tokio::test] async fn set_record_with_content_encoding_is_not_compressed_again() { let backend = MockBackend::new(vec![MockOutcome::Status(201, b"{}".to_vec())]); @@ -847,13 +834,8 @@ async fn last_run_sends_status_and_origin_query_params() { ); } -/// `iterate_items` must keep paging even when the dataset-items response omits the -/// `X-Apify-Pagination-Total` header. With the header absent, `list_items` reports `total = 0` -/// ("unknown"), so the iterator falls through to the short-page/empty-page backstop and walks -/// every page. The `MockBackend` returns no headers, reproducing the missing-total case. Before -/// the fix, `list_items` fell back to `total = count`, which the iterator read as a completed -/// total after the first full page (`next_offset == total`) and silently dropped every later -/// item — this test scripts three data pages then an empty one and asserts all items are yielded. +/// `iterate_items` keeps paging when the response has no `X-Apify-Pagination-Total` header, which +/// `MockBackend` never sends. The test scripts three data pages and an empty one. #[tokio::test] async fn iterate_items_without_total_header_walks_all_pages() { let backend = MockBackend::new(vec![ diff --git a/tests/user.rs b/tests/user.rs index b7d2dfb..31693bd 100644 --- a/tests/user.rs +++ b/tests/user.rs @@ -30,12 +30,9 @@ async fn get_monthly_usage() { /// Simple GET: the current account's monthly usage for a specific month (the spec's optional /// `date` query parameter on `GET /v2/users/me/usage/monthly`). /// -/// Asserts the returned billing cycle actually contains the requested date, which proves the -/// `date` query parameter was honored (a dropped param would return the *current* cycle -/// instead, which would not contain a fixed past date). The client unwraps the `{ data: ... }` -/// envelope, so the response is `{ usageCycle: { startAt, endAt }, ... }`. Billing cycles are -/// not calendar-month aligned, so we compare on day (`YYYY-MM-DD`) granularity rather than a -/// month prefix or a full timestamp. +/// Asserts that the returned billing cycle contains the requested date. A dropped `date` parameter +/// would return the current cycle, which does not contain a fixed past date. Billing cycles are +/// not aligned to calendar months, so the comparison is by day. #[tokio::test(flavor = "multi_thread")] async fn get_monthly_usage_for_date() { let client = require_client!(); @@ -60,13 +57,9 @@ async fn get_monthly_usage_for_date() { .and_then(|v| v.as_str()) .expect("usageCycle.endAt should be a string"); - // Compare on calendar-day granularity: the requested day must fall within the cycle's day - // span [startAt_day, endAt_day]. RFC3339 dates are zero-padded `YYYY-MM-DD`, so the first 10 - // chars sort lexicographically as dates. Using day granularity (not the full timestamp) - // avoids false failures when a cycle boundary lands on the requested calendar day but at a - // non-midnight instant (billing cycles are not calendar-aligned and start at the signup - // time of day). If the `date` param were dropped, the API would return the *current* cycle, - // which cannot span a fixed 2026-03 day. + // The requested day must fall within [startAt_day, endAt_day]. The first 10 characters of an + // RFC 3339 timestamp, `YYYY-MM-DD`, sort as dates. A cycle can start or end during the + // requested day, so comparing full timestamps could fail. let start_day = start .get(..10) .expect("usageCycle.startAt should be an RFC3339 date at least 10 chars long"); From a66b8a672798cfe77698462f8de8bb455d99a66a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:28:57 +0000 Subject: [PATCH 3/6] feat: make zstd an optional feature and address review feedback Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- .github/workflows/rust-integration-tests.yml | 4 ++ CHANGELOG.md | 5 ++- Cargo.toml | 6 ++- README.md | 9 ++-- docs/README.md | 4 +- docs/actors.md | 35 ++++++++++++++++ src/client.rs | 2 +- src/clients/base.rs | 2 +- src/common.rs | 2 +- src/http_client.rs | 4 +- src/http_compressors.rs | 44 ++++++++++---------- src/lib.rs | 3 +- tests/dataset.rs | 5 +-- tests/key_value_store.rs | 1 + tests/request_queue.rs | 4 +- tests/unit_http.rs | 9 ++-- tests/unit_timeouts.rs | 11 +++-- 17 files changed, 98 insertions(+), 52 deletions(-) diff --git a/.github/workflows/rust-integration-tests.yml b/.github/workflows/rust-integration-tests.yml index 6cae9c8..0c573e0 100644 --- a/.github/workflows/rust-integration-tests.yml +++ b/.github/workflows/rust-integration-tests.yml @@ -61,6 +61,10 @@ jobs: - name: Clippy with rustls run: cargo clippy --no-default-features --features rustls -- -D warnings + # The `zstd` feature is on by default, so the build without it needs its own check. + - name: Clippy without zstd + run: cargo clippy --no-default-features --features native-tls --all-targets -- -D warnings + # Fail fast if the integration-test secret is missing or empty. Without this guard the # integration tests silently "pass" (they early-return when APIFY_TOKEN is unset), so a # green run would not prove the API logic actually executed. diff --git a/CHANGELOG.md b/CHANGELOG.md index e395282..b742e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,9 @@ to [Semantic Versioning](https://semver.org/). ## [0.17.0] - 2026-11-10 ### Added -- `ZstdHttpCompressor` (level 1 to 19, default 3) and `RequestCompression::Zstd`, which send request - bodies with `Content-Encoding: zstd`. +- `ZstdHttpCompressor` (quality 1 to 19, default 3) and `RequestCompression::Zstd`, which send + request bodies with `Content-Encoding: zstd`. They need the new `zstd` cargo feature, which is + on by default. - `ActorServerRequestOptions::parent_request_id`, sent as the `X-Actor-Request-ID` header so a Server Actor that calls another one passes on the pricing tier of the request it handles. diff --git a/Cargo.toml b/Cargo.toml index 0aef697..98b643f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ hmac = "0.12" sha2 = "0.10" brotli = "7" flate2 = "1" -zstd = "0.13" +zstd = { version = "0.13", optional = true } # Minimum-version floors for the TLS trust stack behind the `rustls` feature. The crates are not # used directly: `reqwest` pulls them in, and the floor here makes Cargo resolve at least the # releases that no longer trust the compromised intermediate CA certificates. @@ -39,6 +39,8 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread", "time", "test- uuid = { version = "1", features = ["v4"] } [features] -default = ["native-tls"] +default = ["native-tls", "zstd"] native-tls = ["reqwest/native-tls"] rustls = ["reqwest/rustls-tls", "dep:rustls-webpki", "dep:webpki-roots"] +# Zstd request compression. `zstd` compiles a C library, so turn it off if you do not need it. +zstd = ["dep:zstd"] diff --git a/README.md b/README.md index 839c4e9..35fa162 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ let client = ApifyClient::builder() .timeout_max(Duration::from_secs(360)) // default: 360s, cap on a single attempt .user_agent_suffix("my-app/1.0") .default_header("X-My-Header", "value") // sent with every request - .request_compression(RequestCompression::Brotli) // default: Brotli; or Gzip, Zstd + .request_compression(RequestCompression::Brotli) // default: Brotli; or Gzip or Zstd .build(); ``` @@ -163,8 +163,9 @@ or `RequestCompression::Zstd` (`zstd`) for fast compression at a good ratio. (`use apify_client::RequestCompression;`). To tune the quality or use another algorithm, pass an `HttpCompressor` to `ApifyClientBuilder::http_compressor`: the built-in `BrotliHttpCompressor` (quality 0 to 11, default 6), `GzipHttpCompressor` (quality 1 to 9, -default 6) and `ZstdHttpCompressor` (level 1 to 19, default 3), or your own implementation of the -trait. +default 6), and `ZstdHttpCompressor` (quality 1 to 19, default 3), or your own implementation of +the trait. Zstd needs the `zstd` cargo feature, which is on by default and compiles a C library, so +turn it off with `default-features = false` if you don't need it. ```rust,no_run use std::sync::Arc; @@ -380,7 +381,7 @@ Methods take no cancellation parameter. Drop the future a method returns, for ex ## Custom HTTP transport Any type that implements `http_client::HttpBackend` can be injected with -`ApifyClientBuilder::http_backend`. To customize the proxy, TLS or connection pool, wrap a +`ApifyClientBuilder::http_backend`. To customize the proxy, TLS, or connection pool, wrap a `reqwest::Client` you configure in the built-in `ReqwestBackend`, as shown below. To use another HTTP stack, implement `HttpBackend` yourself: `send` is required, and `send_streaming` (used by `log().stream()` and `download_items_stream()`) buffers the whole response through `send` unless diff --git a/docs/README.md b/docs/README.md index 870b1a5..41cc5de 100644 --- a/docs/README.md +++ b/docs/README.md @@ -147,8 +147,8 @@ Builder options: | `timeout_max(d)` | `360s` | Cap on the timeout of a single attempt. `timeout(d)` is the same option under its original name. | | `user_agent_suffix(s)` | none | Extra text appended to the `User-Agent` header. | | `default_header(name, value)` | none | Header sent with every request. It wins over the built-in `User-Agent` and `Authorization` headers; a header a request sets itself wins over it. Names are compared case-insensitively. | -| `request_compression(c)` | `Brotli` | Encoding for large request bodies: `RequestCompression::Brotli` (`br`), `Gzip` (`gzip`) or `Zstd` (`zstd`). | -| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11, default 6), `GzipHttpCompressor::new(quality)` (1 to 9, default 6), `ZstdHttpCompressor::new(level)` (1 to 19, default 3), or your own implementation. | +| `request_compression(c)` | `Brotli` | Encoding for large request bodies: `RequestCompression::Brotli` (`br`), `Gzip` (`gzip`), or `Zstd` (`zstd`; needs the `zstd` feature, on by default). | +| `http_compressor(c)` | Brotli, quality 6 | A custom `Arc`: `BrotliHttpCompressor::new(quality)` (0 to 11, default 6), `GzipHttpCompressor::new(quality)` (1 to 9, default 6), `ZstdHttpCompressor::new(quality)` (1 to 19, default 3), or your own implementation. | | `http_backend(b)` | reqwest | Replaceable HTTP transport (`apify_client::http_client::{HttpBackend, ReqwestBackend}`). A request whose `timeout` is `timeouts::NO_TIMEOUT` may run for as long as it takes, so a backend must check for it before it applies a timeout of its own. A request whose `follow_redirects` is `false`, such as one to an Actor's server, carries a token that must not follow a redirect to another host, so a backend must return a redirect response as it is. | Request bodies of at least 1024 bytes are compressed once (before retries) with the selected diff --git a/docs/actors.md b/docs/actors.md index fb2a413..4b8fd5c 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -328,6 +328,41 @@ How the request is sent: timeouts are not retried, since the Actor may have already handled the request, and they fail the call. +#### Calling a Server Actor from a Server Actor + +A multi-tenant Server Actor shares the developer's runs among its users, so the platform tells +requests apart by an ID it sends in the `X-Actor-Request-ID` header of each request it forwards to +the Actor. The Actor uses that ID to charge the caller of the request, and to pass the caller's +pricing tier on to the Actors it calls. + +When your multi-tenant Server Actor calls another one while it handles a request, pass the incoming +`X-Actor-Request-ID` header in `parent_request_id`. The client sends it as the same header, so the +called Actor charges at the pricing tier of your incoming request. Use a client created with the +run's `APIFY_TOKEN`. For details, see +[Compose multi-tenant Server Actors](https://docs.apify.com/platform/actors/development/programming-interface/standby#compose-server-actors). + +```rust,no_run +use apify_client::{ActorServerRequestOptions, ApifyClient}; + +# async fn run(incoming_request_id: String) -> Result<(), Box> { +let client = ApifyClient::new("my-api-token"); +let response = client + .actor("username/other-actor") + .server() + .request( + "/search", + ActorServerRequestOptions { + params: vec![("q".to_string(), "apify".to_string())], + parent_request_id: Some(incoming_request_id), + ..Default::default() + }, + ) + .await?; +println!("{}", response.text()); +# Ok(()) +# } +``` + `get_open_api_spec` returns the web server schema from the build that serves the server interface: the build in the Actor's `actor_standby.build` field (a tag or a build number), or `latest`. It fails with `InvalidArgument` if that build doesn't exist. diff --git a/src/client.rs b/src/client.rs index 723d852..04fac11 100644 --- a/src/client.rs +++ b/src/client.rs @@ -171,7 +171,7 @@ impl ApifyClientBuilder { /// [`RequestCompression::Brotli`]), at its default quality. /// /// The client compresses request bodies of at least 1024 bytes with brotli (`br`), gzip - /// (`gzip`) or zstd (`zstd`). Select [`RequestCompression::Gzip`] for environments or + /// (`gzip`), or zstd (`zstd`). Select [`RequestCompression::Gzip`] for environments or /// intermediaries that do not handle brotli. To tune the quality or use another algorithm, /// pass an [`HttpCompressor`] to [`http_compressor`](Self::http_compressor). pub fn request_compression(mut self, compression: RequestCompression) -> Self { diff --git a/src/clients/base.rs b/src/clients/base.rs index 7054466..973eb13 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -756,7 +756,7 @@ where } } - // Pause before polling again. + // Let the replicas catch up before polling again. crate::http_client::sleep_public(WAIT_FOR_FINISH_POLL_INTERVAL).await; } } diff --git a/src/common.rs b/src/common.rs index fadf27b..08d0e23 100644 --- a/src/common.rs +++ b/src/common.rs @@ -158,7 +158,7 @@ pub struct ListOptions { pub desc: Option, } -/// Options of the storage collection list endpoints (datasets, key-value stores and request +/// Options of the storage collection list endpoints (datasets, key-value stores, and request /// queues): the offset/limit pagination plus `unnamed` and `ownership` filters. #[derive(Debug, Default, Clone)] pub struct StorageListOptions { diff --git a/src/http_client.rs b/src/http_client.rs index 9ecff37..f531625 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -70,7 +70,7 @@ impl HttpMethod { } /// An HTTP request ready to be sent by an [`HttpBackend`]. The client has already applied the auth -/// header, the user agent and the compression. +/// header, the user agent, and the compression. #[derive(Debug, Clone)] pub struct HttpRequest { /// The HTTP method. @@ -141,7 +141,7 @@ pub struct StreamingHttpResponse { } /// The transport that sends requests. An implementation performs one network round-trip per -/// request: the client handles retries, authentication and serialization. +/// request: the client handles retries, authentication, and serialization. #[async_trait] pub trait HttpBackend: Send + Sync + std::fmt::Debug { /// Sends a single HTTP request and returns the response. diff --git a/src/http_compressors.rs b/src/http_compressors.rs index 8c7aeb6..9c7dbcd 100644 --- a/src/http_compressors.rs +++ b/src/http_compressors.rs @@ -3,7 +3,7 @@ //! The client compresses request bodies of at least 1024 bytes whose content type //! is not already compressed and that carry no `Content-Encoding` of their own. The algorithm is an //! [`HttpCompressor`]: the built-in [`BrotliHttpCompressor`] (the default), -//! [`GzipHttpCompressor`] and [`ZstdHttpCompressor`], or a custom implementation passed to +//! [`GzipHttpCompressor`], and `ZstdHttpCompressor`, or a custom implementation passed to //! [`ApifyClientBuilder::http_compressor`](crate::ApifyClientBuilder::http_compressor). use std::collections::HashMap; @@ -30,6 +30,7 @@ const BROTLI_WINDOW_SIZE: u32 = 22; /// Internal buffer size for the brotli encoder. const BROTLI_BUFFER_SIZE: usize = 4096; /// `Content-Encoding` value used for zstd-compressed request bodies. +#[cfg(feature = "zstd")] const CONTENT_ENCODING_ZSTD: &str = "zstd"; /// Lowest gzip quality, the fastest with the least compression. const GZIP_MIN_QUALITY: u32 = 1; @@ -39,12 +40,15 @@ const GZIP_MAX_QUALITY: u32 = 9; /// roughly double the CPU cost for about a percent fewer bytes. const GZIP_DEFAULT_QUALITY: u32 = 6; /// Lowest zstd level, the fastest with the least compression. +#[cfg(feature = "zstd")] const ZSTD_MIN_QUALITY: u32 = 1; /// Highest zstd level the compressor accepts. zstd itself goes up to 22, but levels 20 to 22 use a /// window above the 8 MB that RFC 9659 requires HTTP decoders to support, so a body compressed at /// those levels could fail to decode. +#[cfg(feature = "zstd")] const ZSTD_MAX_QUALITY: u32 = 19; -/// Zstd level used by [`ZstdHttpCompressor::default`]: zstd's own default. +/// Zstd level used by `ZstdHttpCompressor::default`: zstd's own default. +#[cfg(feature = "zstd")] const ZSTD_DEFAULT_QUALITY: u32 = 3; /// Media-type prefixes whose payloads already carry their own compression, so compressing them @@ -205,12 +209,14 @@ impl HttpCompressor for GzipHttpCompressor { } /// Zstd compressor (`Content-Encoding: zstd`). Fast at its default level, with a ratio close to -/// brotli at the higher ones. +/// brotli at the higher ones. Available with the `zstd` feature, which is on by default. +#[cfg(feature = "zstd")] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ZstdHttpCompressor { quality: u32, } +#[cfg(feature = "zstd")] impl ZstdHttpCompressor { /// Creates a zstd compressor with the given level, from 1 (fastest, least compression) to 19 /// (slowest, best compression). Returns [`ApifyClientError::InvalidArgument`] when `quality` @@ -221,6 +227,7 @@ impl ZstdHttpCompressor { } } +#[cfg(feature = "zstd")] impl Default for ZstdHttpCompressor { /// Zstd at level 3. fn default() -> Self { @@ -230,6 +237,7 @@ impl Default for ZstdHttpCompressor { } } +#[cfg(feature = "zstd")] impl HttpCompressor for ZstdHttpCompressor { fn content_encoding(&self) -> &str { CONTENT_ENCODING_ZSTD @@ -243,8 +251,8 @@ impl HttpCompressor for ZstdHttpCompressor { } /// Built-in compression algorithm selected by name, at its default quality. A shorthand for -/// [`BrotliHttpCompressor::default`], [`GzipHttpCompressor::default`] and -/// [`ZstdHttpCompressor::default`]; use the compressor structs directly to tune the quality, or +/// [`BrotliHttpCompressor::default`], [`GzipHttpCompressor::default`], and +/// `ZstdHttpCompressor::default`; use the compressor structs directly to tune the quality, or /// implement [`HttpCompressor`] for another algorithm. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[non_exhaustive] @@ -255,7 +263,9 @@ pub enum RequestCompression { /// Gzip (`Content-Encoding: gzip`). Choose this for environments or intermediaries that do not /// handle brotli. Gzip, - /// Zstd (`Content-Encoding: zstd`). Fast at a good ratio. + /// Zstd (`Content-Encoding: zstd`). Fast at a good ratio. Available with the `zstd` feature, + /// which is on by default. + #[cfg(feature = "zstd")] Zstd, } @@ -265,6 +275,7 @@ impl RequestCompression { match self { RequestCompression::Brotli => std::sync::Arc::new(BrotliHttpCompressor::default()), RequestCompression::Gzip => std::sync::Arc::new(GzipHttpCompressor::default()), + #[cfg(feature = "zstd")] RequestCompression::Zstd => std::sync::Arc::new(ZstdHttpCompressor::default()), } } @@ -373,6 +384,11 @@ mod tests { assert!(GzipHttpCompressor::new(1).is_ok()); assert!(GzipHttpCompressor::new(0).is_err()); assert!(GzipHttpCompressor::new(10).is_err()); + } + + #[cfg(feature = "zstd")] + #[test] + fn zstd_level_outside_the_range_is_rejected() { assert!(ZstdHttpCompressor::new(1).is_ok()); assert!(ZstdHttpCompressor::new(19).is_ok()); assert!(ZstdHttpCompressor::new(0).is_err()); @@ -406,24 +422,10 @@ mod tests { fn built_in_compressors_report_their_content_encoding() { assert_eq!(BrotliHttpCompressor::default().content_encoding(), "br"); assert_eq!(GzipHttpCompressor::default().content_encoding(), "gzip"); + #[cfg(feature = "zstd")] assert_eq!(ZstdHttpCompressor::default().content_encoding(), "zstd"); } - #[test] - fn zstd_output_decodes_to_the_input() { - let data: Vec = (0..4096u32).map(|i| (i % 251) as u8).collect(); - for compressor in [ - ZstdHttpCompressor::default(), - ZstdHttpCompressor::new(19).unwrap(), - ] { - let compressed = compressor.compress(&data).unwrap(); - assert_eq!( - zstd::bulk::decompress(&compressed, data.len()).unwrap(), - data - ); - } - } - #[test] fn small_bodies_are_left_uncompressed() { let mut headers = headers(&[("Content-Type", "application/json")]); diff --git a/src/lib.rs b/src/lib.rs index 4010cba..c341b28 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,9 +67,10 @@ mod version; pub use client::{ApifyClient, ApifyClientBuilder}; pub use common::{ListOptions, PaginationList, StorageListOptions}; pub use error::{ApiError, ApifyClientError, ApifyClientResult}; +#[cfg(feature = "zstd")] +pub use http_compressors::ZstdHttpCompressor; pub use http_compressors::{ BrotliHttpCompressor, GzipHttpCompressor, HttpCompressor, RequestCompression, - ZstdHttpCompressor, }; pub use statistics::Statistics; pub use timeouts::Timeout; diff --git a/tests/dataset.rs b/tests/dataset.rs index e834c4f..917e737 100644 --- a/tests/dataset.rs +++ b/tests/dataset.rs @@ -227,9 +227,8 @@ async fn dataset_crud_flow() { .expect("list items"); assert_eq!(items.items.len(), 3, "should read back the 3 pushed items"); - // The `signature` query param must be settable on list_items (spec + JS reference both - // expose it for fetching items from a private dataset with a pre-shared signature). On this - // owned dataset the signature is unnecessary, but the request must still succeed. + // `list_items` accepts a `signature` for fetching items of a private dataset. It is not needed + // for this owned dataset, but the request must still succeed. let signed = dataset_client .list_items::(apify_client::DatasetListItemsOptions { signature: Some("test-signature".to_string()), diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index 28b4623..ba5ed95 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -336,6 +336,7 @@ async fn set_record_with_content_encoding_round_trips_without_corruption() { } /// A large record uploaded by a client that compresses with zstd reads back unchanged. +#[cfg(feature = "zstd")] #[tokio::test(flavor = "multi_thread")] async fn set_record_compressed_with_zstd() { use apify_client::RequestCompression; diff --git a/tests/request_queue.rs b/tests/request_queue.rs index ce02db8..50854d4 100644 --- a/tests/request_queue.rs +++ b/tests/request_queue.rs @@ -285,8 +285,8 @@ async fn request_queue_batch_add_and_delete() { .expect("batch delete requests"); assert_eq!(delete_result.processed_requests.len(), TOTAL); - // `batch_delete_requests` rejects more than 25 requests per call client-side, matching the - // reference client's validation (it does not auto-chunk deletes). + // `batch_delete_requests` rejects more than 25 requests per call client-side. It does not + // auto-chunk. let too_many: Vec = (0..26) .map(|i| json!({ "uniqueKey": format!("toomany-{i}") })) .collect(); diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 4f7eeff..694cd03 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -564,6 +564,7 @@ async fn large_request_body_is_gzip_compressed_when_selected() { /// With the zstd encoding selected, a large request body is sent with `Content-Encoding: zstd` and /// decodes back to the original bytes. +#[cfg(feature = "zstd")] #[tokio::test] async fn large_request_body_is_zstd_compressed_when_selected() { let backend = MockBackend::new(vec![MockOutcome::Status(200, b"".to_vec())]); @@ -755,8 +756,8 @@ async fn start_raw_defaults_content_type_and_sends_bytes_verbatim() { } /// The status-only `last_run(status)` and the options-based `last_run_with_options` thread the -/// optional `status` and `origin` filters into the last-run request as query parameters, matching -/// the JS reference's `lastRun({ status, origin })`. Leaving a filter as `None` omits it. +/// optional `status` and `origin` filters into the last-run request as query parameters. Leaving a +/// filter as `None` omits it. #[tokio::test] async fn last_run_sends_status_and_origin_query_params() { let backend = MockBackend::new(vec![MockOutcome::Status( @@ -1059,8 +1060,8 @@ async fn iterate_keys_stops_on_is_truncated_even_with_trailing_cursor() { ); } -/// The `iterate_keys` `limit` is a total cap that also bounds the first page's request size -/// (reference parity): with `limit=2` the first request asks for `limit=2` and, once two keys +/// The `iterate_keys` `limit` is a total cap that also bounds the first page's request size: +/// with `limit=2` the first request asks for `limit=2` and, once two keys /// are consumed, the walk stops without a second fetch even though the API advertised more keys. #[tokio::test] async fn iterate_keys_limit_caps_the_walk() { diff --git a/tests/unit_timeouts.rs b/tests/unit_timeouts.rs index b89311c..d1bed5d 100644 --- a/tests/unit_timeouts.rs +++ b/tests/unit_timeouts.rs @@ -12,9 +12,11 @@ use std::time::Duration; use apify_client::clients::key_value_store::SetRecordOptions; use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, ReqwestBackend}; use apify_client::timeouts::NO_TIMEOUT; +#[cfg(feature = "zstd")] +use apify_client::ZstdHttpCompressor; use apify_client::{ ActorStartOptions, ApifyClient, ApifyClientError, BrotliHttpCompressor, GzipHttpCompressor, - HttpCompressor, Timeout, ZstdHttpCompressor, + HttpCompressor, Timeout, }; use common::mock_backend::{client_with, MockBackend, MockOutcome}; use serde_json::{json, Value}; @@ -415,6 +417,7 @@ async fn gzip_quality_is_configurable() { assert_eq!(decompressed, vec![b'a'; COMPRESSIBLE_BODY_BYTES]); } +#[cfg(feature = "zstd")] #[tokio::test] async fn zstd_quality_is_configurable() { let compressor = ZstdHttpCompressor::new(19).expect("19 is a valid zstd level"); @@ -440,10 +443,6 @@ fn a_quality_outside_the_range_of_the_algorithm_is_rejected() { GzipHttpCompressor::new(0), Err(ApifyClientError::InvalidArgument(_)) )); - assert!(matches!( - ZstdHttpCompressor::new(20), - Err(ApifyClientError::InvalidArgument(_)) - )); } #[tokio::test] @@ -629,7 +628,7 @@ async fn statistics_count_calls_requests_and_rate_limit_errors_by_attempt() { } /// Methods that read or write whole stores and datasets, or build a public URL from one, use the -/// `Long` tier, like the reference client. +/// `Long` tier. #[tokio::test] async fn key_value_store_writes_and_public_urls_use_the_long_tier() { let backend = MockBackend::new(vec![MockOutcome::Status( From 902e2944150672ae672ac8165cc6b829994db159 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:40:19 +0000 Subject: [PATCH 4/6] docs: fix install snippets, feature instructions and polling comment Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- README.md | 12 +++++++----- docs/README.md | 2 +- docs/actors.md | 2 +- src/clients/base.rs | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 35fa162..13e101e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ It provides a resource-oriented, async interface that follows the ```toml [dependencies] -apify-client = "0.15" +apify-client = "0.17" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } serde_json = "1" # for the `serde_json::Value` responses used in the Quick start ``` @@ -44,10 +44,11 @@ project needs `serde_json`. Two more dependencies are needed only for specific f snippet needs it as a direct dependency, and its version must match this crate's `reqwest` version (`0.12`) so the `reqwest::Client` types are compatible. -By default the client uses the system TLS (`native-tls`). To use rustls instead: +By default the client uses the system TLS (`native-tls`) and has the `zstd` feature. To use rustls +instead: ```toml -apify-client = { version = "0.15", default-features = false, features = ["rustls"] } +apify-client = { version = "0.17", default-features = false, features = ["rustls", "zstd"] } ``` The `rustls` feature requires `rustls-webpki` 0.103.16 or newer and `webpki-roots` 1.0.10 or @@ -164,8 +165,9 @@ or `RequestCompression::Zstd` (`zstd`) for fast compression at a good ratio. `HttpCompressor` to `ApifyClientBuilder::http_compressor`: the built-in `BrotliHttpCompressor` (quality 0 to 11, default 6), `GzipHttpCompressor` (quality 1 to 9, default 6), and `ZstdHttpCompressor` (quality 1 to 19, default 3), or your own implementation of -the trait. Zstd needs the `zstd` cargo feature, which is on by default and compiles a C library, so -turn it off with `default-features = false` if you don't need it. +the trait. Zstd needs the `zstd` cargo feature, which is on by default and compiles a C library. +To build without it, disable the default features and name the TLS backend you use, for example +`default-features = false, features = ["native-tls"]`. ```rust,no_run use std::sync::Arc; diff --git a/docs/README.md b/docs/README.md index 41cc5de..28e42e0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,7 +31,7 @@ Add the crate and an async runtime: ```toml [dependencies] -apify-client = "0.15" +apify-client = "0.17" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } ``` diff --git a/docs/actors.md b/docs/actors.md index 4b8fd5c..d098b55 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -345,7 +345,7 @@ run's `APIFY_TOKEN`. For details, see use apify_client::{ActorServerRequestOptions, ApifyClient}; # async fn run(incoming_request_id: String) -> Result<(), Box> { -let client = ApifyClient::new("my-api-token"); +let client = ApifyClient::new(std::env::var("APIFY_TOKEN")?); let response = client .actor("username/other-actor") .server() diff --git a/src/clients/base.rs b/src/clients/base.rs index 973eb13..48caaa5 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -756,7 +756,7 @@ where } } - // Let the replicas catch up before polling again. + // The pause also gives a replica that answered `404` time to catch up. crate::http_client::sleep_public(WAIT_FOR_FINISH_POLL_INTERVAL).await; } } From c5642b1f750934beac6ee670839f02a568f5dbf0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:43:46 +0000 Subject: [PATCH 5/6] docs: say that the zstd entry of the rustls snippet is optional Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- Cargo.toml | 3 ++- README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 98b643f..d2be488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,5 +42,6 @@ uuid = { version = "1", features = ["v4"] } default = ["native-tls", "zstd"] native-tls = ["reqwest/native-tls"] rustls = ["reqwest/rustls-tls", "dep:rustls-webpki", "dep:webpki-roots"] -# Zstd request compression. `zstd` compiles a C library, so turn it off if you do not need it. +# Zstd request compression. It compiles a C library. To build without it, disable the default +# features and name a TLS backend (`native-tls` or `rustls`). zstd = ["dep:zstd"] diff --git a/README.md b/README.md index 13e101e..9dc79c2 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ project needs `serde_json`. Two more dependencies are needed only for specific f version (`0.12`) so the `reqwest::Client` types are compatible. By default the client uses the system TLS (`native-tls`) and has the `zstd` feature. To use rustls -instead: +instead, disable the default features and name `rustls`. The `"zstd"` entry keeps zstd compression, +so drop it if you don't need it: ```toml apify-client = { version = "0.17", default-features = false, features = ["rustls", "zstd"] } From aa5b6ea607a874d5ed1f2beca3ccd2b7dd74179b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:45:52 +0000 Subject: [PATCH 6/6] docs: spell out a contraction in the README Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAJkLeAK9WauXA9zCTU --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9dc79c2..1be575d 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ project needs `serde_json`. Two more dependencies are needed only for specific f By default the client uses the system TLS (`native-tls`) and has the `zstd` feature. To use rustls instead, disable the default features and name `rustls`. The `"zstd"` entry keeps zstd compression, -so drop it if you don't need it: +so drop it if you do not need it: ```toml apify-client = { version = "0.17", default-features = false, features = ["rustls", "zstd"] }