diff --git a/README.md b/README.md index cce5cd8..939647f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Available as both a **Rust library** and a **CLI tool**. ## Features - **Swagger 2.0 support** – Swagger 2.0 sources are converted to OpenAPI 3.0.3 automatically; anything that had to be approximated is reported as a warning +- **External `$ref`s** – specs split across files or URLs are bundled per source before merging (CLI sandboxed to the config directory) - **Multiple source types** – local YAML files, local JSON files, and HTTP endpoints (with custom headers) - **Config-file driven** – define sources and merge options in a single YAML config - **Conflict resolution** – choose between `error`, `overwrite`, or `rename` strategies for duplicate paths and component names @@ -128,6 +129,17 @@ A referenced variable that is not set is an error. Each source can define `additional_blocks` as any YAML/JSON object. It is deep-merged into that source document before merge, so you can inject vendor extensions (for example API gateway related `x-...` blocks) or other custom OpenAPI fragments without adding provider-specific fields. +### External `$ref`s + +A source may point at other files or URLs (`$ref: ./schemas/pet.yaml#/Pet`, `$ref: https://example.com/defs.yaml#/Error`). Before merging, each source is **bundled**: every external target is copied into that source's own `components` and the `$ref` is rewritten to `#/components/...`. Circular references work, and everything downstream (conflict strategies, renaming) sees a normal self-contained spec. Specs without external refs are unchanged. + +- **Where targets land:** a pointer like `#/components/schemas/Pet` keeps its type and name; `#/definitions/Pet` and `#/$defs/Pet` become schemas; otherwise the type comes from where the `$ref` sits (a `parameters` entry becomes a parameter, a response becomes a response, ...) and the name is the last pointer segment (or the file name for a whole-file ref). If the name is taken, the file name is added (`pet_Pet`) and a warning says so. A path item that is a `$ref` is inlined. +- **Relative refs** resolve against the referencing file or URL. +- **Sandbox (CLI and library):** external file refs may only read inside the config directory and the directory of each file source. Anything else, including absolute paths and symlinks pointing elsewhere, is an error. Pass `--allow-outside-config-dir` (or `AggregateOptions { allow_outside_config_dir: true, .. }` with `aggregate_with_options`) to lift it. +- **URLs:** a URL source's `headers` are only sent to that URL's origin; refs to other hosts are fetched without them. A document fetched over HTTP can never cause a local file read, and `file:` URLs are not supported. +- **Limit:** at most 100 external documents per source. +- **Playground:** referenced files are looked up in the workspace by name, so add each one under exactly the name the `$ref` resolves to (the error message tells you which). Remote refs follow the same "Allow fetching" rule as remote sources. + ## Library Usage ```rust diff --git a/crates/openapi-aggregator-core/src/bundle.rs b/crates/openapi-aggregator-core/src/bundle.rs new file mode 100644 index 0000000..4594f4c --- /dev/null +++ b/crates/openapi-aggregator-core/src/bundle.rs @@ -0,0 +1,953 @@ +//! Bundle external `$ref`s into a source's own `components`. +//! +//! Pure and I/O-free: front ends discover the documents a spec needs with +//! [`missing_documents`], load them, and call [`bundle`]. + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +use crate::error::Error; + +/// Whether `location` is an absolute `http(s)` URL. +pub fn is_url(location: &str) -> bool { + location.starts_with("http://") || location.starts_with("https://") +} + +/// Resolve `reference` (document part only, no fragment) against the location of the +/// referencing document `base`. +pub fn resolve_location(base: &str, reference: &str) -> String { + if reference.is_empty() { + return base.to_string(); + } + if is_url(reference) || reference.starts_with("file:") { + return reference.to_string(); + } + if is_url(base) { + let (origin, path) = split_origin(base); + let path = path.split(['?', '#']).next().unwrap_or(""); + let joined = if reference.starts_with('/') { + reference.to_string() + } else { + let dir = &path[..path.rfind('/').map_or(0, |i| i + 1)]; + format!("{dir}{reference}") + }; + return format!("{origin}{}", normalize(&joined, true)); + } + if reference.starts_with('/') { + return reference.to_string(); + } + let dir = &base[..base.rfind('/').map_or(0, |i| i + 1)]; + normalize(&format!("{dir}{reference}"), base.starts_with('/')) +} + +/// `("https://host:80", "/path")` for an absolute URL. +fn split_origin(url: &str) -> (&str, &str) { + let after_scheme = url.find("://").map_or(0, |i| i + 3); + match url[after_scheme..].find('/') { + Some(i) => url.split_at(after_scheme + i), + None => (url, "/"), + } +} + +/// Collapse `.` and `..` segments. Relative paths keep leading `..`; URL paths +/// (`rooted`) cannot go above the root. +fn normalize(path: &str, rooted: bool) -> String { + let mut out: Vec<&str> = Vec::new(); + for segment in path.split('/') { + match segment { + "" | "." => {} + ".." => { + if matches!(out.last(), Some(last) if *last != "..") { + out.pop(); + } else if !rooted { + out.push(".."); + } + } + other => out.push(other), + } + } + let joined = out.join("/"); + if rooted { + format!("/{joined}") + } else { + joined + } +} + +/// Split `"a.yaml#/x"` into `("a.yaml", "/x")`. +pub(crate) fn split_ref(reference: &str) -> (&str, &str) { + match reference.find('#') { + Some(i) => (&reference[..i], &reference[i + 1..]), + None => (reference, ""), + } +} + +/// Decode a JSON pointer (as found after `#`) into segments. +pub(crate) fn decode_pointer(pointer: &str) -> Vec { + let Some(rest) = pointer.strip_prefix('/') else { + return Vec::new(); + }; + rest.split('/') + .map(|segment| { + percent_decode(segment) + .replace("~1", "/") + .replace("~0", "~") + }) + .collect() +} + +fn percent_decode(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + let hex = std::str::from_utf8(&bytes[i + 1..i + 3]).ok(); + if let Some(byte) = hex.and_then(|h| u8::from_str_radix(h, 16).ok()) { + out.push(byte); + i += 3; + continue; + } + } + out.push(bytes[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// A document that a spec (transitively) references but that is not loaded yet. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MissingDocument { + pub location: String, + pub referenced_from: String, +} + +/// Keys whose children are user-chosen names, not OpenAPI keywords. +const NAME_MAPS: [&str; 17] = [ + "properties", + "patternProperties", + "definitions", + "$defs", + "schemas", + "parameters", + "responses", + "headers", + "examples", + "requestBodies", + "links", + "callbacks", + "securitySchemes", + "paths", + "content", + "encoding", + "mapping", +]; + +/// Keys that hold example data, not spec structure. +const DATA_KEYS: [&str; 4] = ["example", "default", "enum", "value"]; + +/// Visit every `$ref` string, skipping example/default/enum data. +pub(crate) fn walk_refs(value: &Value, f: &mut dyn FnMut(&str)) { + walk(value, false, f); +} + +fn walk(value: &Value, names: bool, f: &mut dyn FnMut(&str)) { + match value { + Value::Object(map) => { + for (key, child) in map { + if !names && DATA_KEYS.contains(&key.as_str()) { + continue; + } + match child { + Value::String(target) if !names && key == "$ref" => f(target), + _ => walk(child, !names && NAME_MAPS.contains(&key.as_str()), f), + } + } + } + Value::Array(items) => items.iter().for_each(|item| walk(item, false, f)), + _ => {} + } +} + +/// The documents `spec` (located at `base`) needs, following references through the +/// documents already in `docs`, that are not in `docs` yet. Deduplicated, in discovery order. +pub fn missing_documents( + spec: &Value, + base: &str, + docs: &HashMap, +) -> Vec { + let mut missing = Vec::new(); + let mut seen_missing = HashSet::new(); + let mut visited = HashSet::new(); + visited.insert(base.to_string()); + let mut queue: Vec<(String, &Value)> = vec![(base.to_string(), spec)]; + + while let Some((location, document)) = queue.pop() { + let mut found = Vec::new(); + walk_refs(document, &mut |reference| { + let (doc_part, _) = split_ref(reference); + if !doc_part.is_empty() { + found.push(resolve_location(&location, doc_part)); + } + }); + for target in found { + if let Some(loaded) = docs.get(&target) { + if visited.insert(target.clone()) { + queue.push((target, loaded)); + } + } else if target != base && seen_missing.insert(target.clone()) { + missing.push(MissingDocument { + location: target, + referenced_from: location.clone(), + }); + } + } + } + missing +} + +/// Maximum number of external documents a single source may pull in. +pub const MAX_DOCUMENTS: usize = 100; + +/// Fail when a source has pulled in more than [`MAX_DOCUMENTS`] external documents. +pub fn check_document_cap(count: usize, source_name: &str) -> Result<(), Error> { + if count > MAX_DOCUMENTS { + return Err(Error::InvalidSpec { + name: source_name.to_string(), + reason: format!("too many external documents (limit {MAX_DOCUMENTS})"), + }); + } + Ok(()) +} + +/// Bundle every external `$ref` in `spec` (located at `base`) into its `components`. +/// Returns the bundled spec and notes about names that had to be changed. +pub fn bundle( + mut spec: Value, + base: &str, + docs: &HashMap, +) -> Result<(Value, Vec), Error> { + let mut needs_work = false; + walk_refs(&spec, &mut |reference| { + let (doc_part, _) = split_ref(reference); + needs_work |= !doc_part.is_empty(); + }); + if !needs_work { + return Ok((spec, Vec::new())); + } + + let mut bundler = Bundler::new(base, docs, &spec); + bundler.inline_path_items(&mut spec)?; + bundler.process(&mut spec, base, &mut Vec::new(), false)?; + let notes = bundler.finish(&mut spec); + Ok((spec, notes)) +} + +/// The component type a ref's position implies, from the keys leading to it +/// (`"[]"` marks an array element). +fn infer_type(stack: &[String]) -> &'static str { + let by_container = |key: &str| match key { + "parameters" => "parameters", + "responses" => "responses", + "headers" => "headers", + "examples" => "examples", + "links" => "links", + "callbacks" => "callbacks", + "requestBodies" => "requestBodies", + "securitySchemes" => "securitySchemes", + _ => "schemas", + }; + let keys: Vec<&str> = stack.iter().map(String::as_str).collect(); + match keys.as_slice() { + [.., parent, "[]"] => by_container(parent), + [.., "requestBody"] => "requestBodies", + [.., "schema" | "items" | "additionalProperties" | "not"] => "schemas", + [.., parent, _] => by_container(parent), + _ => "schemas", + } +} + +const COMPONENT_TYPES: [&str; 9] = [ + "schemas", + "parameters", + "responses", + "requestBodies", + "headers", + "examples", + "links", + "callbacks", + "securitySchemes", +]; + +/// File name without directories, query or extension: `a/b/pet.yaml?x` -> `pet`. +fn file_stem(location: &str) -> String { + let name = location.rsplit('/').next().unwrap_or(location); + let name = name.split(['?', '#']).next().unwrap_or(name); + match name.rfind('.') { + Some(i) if i > 0 => name[..i].to_string(), + _ => name.to_string(), + } +} + +fn sanitize_name(name: &str) -> String { + let cleaned: String = name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { + c + } else { + '_' + } + }) + .collect(); + if cleaned.is_empty() { + "component".to_string() + } else { + cleaned + } +} + +fn lookup<'v>(document: &'v Value, segments: &[String]) -> Option<&'v Value> { + let mut current = document; + for segment in segments { + current = match current { + Value::Object(map) => map.get(segment)?, + Value::Array(items) => items.get(segment.parse::().ok()?)?, + _ => return None, + }; + } + Some(current) +} + +struct Bundler<'a> { + base: String, + docs: &'a HashMap, + /// (document, pointer) -> (type, name) for everything already hoisted. + memo: HashMap<(String, String), (&'static str, String)>, + taken: HashMap<&'static str, HashSet>, + hoisted: Vec<(&'static str, String, Value)>, + notes: Vec, + inlining: Vec<(String, String)>, +} + +impl<'a> Bundler<'a> { + fn new(base: &str, docs: &'a HashMap, spec: &Value) -> Self { + let mut taken: HashMap<&'static str, HashSet> = HashMap::new(); + for ty in COMPONENT_TYPES { + if let Some(Value::Object(existing)) = spec.pointer(&format!("/components/{ty}")) { + taken + .entry(ty) + .or_default() + .extend(existing.keys().cloned()); + } + } + Self { + base: base.to_string(), + docs, + memo: HashMap::new(), + taken, + hoisted: Vec::new(), + notes: Vec::new(), + inlining: Vec::new(), + } + } + + fn unresolved(reference: &str, reason: &str) -> Error { + Error::UnresolvedRef { + reference: reference.to_string(), + reason: reason.to_string(), + } + } + + fn target_of(&self, location: &str, reference: &str) -> (String, String) { + let (doc_part, pointer) = split_ref(reference); + let target = if doc_part.is_empty() { + location.to_string() + } else { + resolve_location(location, doc_part) + }; + (target, pointer.to_string()) + } + + fn fetch(&self, target: &str, pointer: &str, reference: &str) -> Result { + let document = self.docs.get(target).ok_or_else(|| { + Self::unresolved(reference, "points to a document that was not loaded") + })?; + lookup(document, &decode_pointer(pointer)) + .cloned() + .ok_or_else(|| Self::unresolved(&format!("{target}#{pointer}"), "not found")) + } + + /// Replace `paths` entries that are external path-item refs by the referenced item. + fn inline_path_items(&mut self, spec: &mut Value) -> Result<(), Error> { + let base = self.base.clone(); + let Some(Value::Object(paths)) = spec.get_mut("paths") else { + return Ok(()); + }; + for (path, item) in paths.iter_mut() { + let Some(reference) = item.get("$ref").and_then(Value::as_str).map(String::from) else { + continue; + }; + let (doc_part, _) = split_ref(&reference); + if doc_part.is_empty() { + continue; + } + let (target, pointer) = self.target_of(&base, &reference); + if target == base { + continue; + } + let key = (target.clone(), pointer.clone()); + if self.inlining.contains(&key) { + return Err(Self::unresolved( + &reference, + "is a circular path item reference", + )); + } + let mut inlined = self.fetch(&target, &pointer, &reference)?; + self.inlining.push(key); + let mut stack = vec!["paths".to_string(), path.clone()]; + let result = self.process(&mut inlined, &target, &mut stack, false); + self.inlining.pop(); + result?; + if let (Value::Object(merged), Value::Object(own)) = (&mut inlined, &*item) { + for (k, v) in own { + if k != "$ref" { + merged.insert(k.clone(), v.clone()); + } + } + } + *item = inlined; + } + Ok(()) + } + + fn process( + &mut self, + value: &mut Value, + location: &str, + stack: &mut Vec, + names: bool, + ) -> Result<(), Error> { + match value { + Value::Object(map) => { + for (key, child) in map.iter_mut() { + if !names && DATA_KEYS.contains(&key.as_str()) { + continue; + } + match child { + Value::String(reference) if !names && key == "$ref" => { + if let Some(new) = self.rewrite_ref(reference, location, stack)? { + *reference = new; + } + } + _ => { + stack.push(key.clone()); + let child_names = !names && NAME_MAPS.contains(&key.as_str()); + let result = self.process(child, location, stack, child_names); + stack.pop(); + result?; + } + } + } + } + Value::Array(items) => { + for item in items { + stack.push("[]".to_string()); + let result = self.process(item, location, stack, false); + stack.pop(); + result?; + } + } + _ => {} + } + Ok(()) + } + + /// The replacement for `reference`, or `None` to leave it alone. + fn rewrite_ref( + &mut self, + reference: &str, + location: &str, + stack: &[String], + ) -> Result, Error> { + let (doc_part, _) = split_ref(reference); + let (target, pointer) = self.target_of(location, reference); + if target == self.base { + if location == self.base && doc_part.is_empty() { + return Ok(None); + } + if pointer.is_empty() { + return Err(Self::unresolved( + reference, + "refers to the root document itself", + )); + } + return Ok(Some(format!("#{pointer}"))); + } + + let key = (target.clone(), pointer.clone()); + if let Some((ty, name)) = self.memo.get(&key) { + return Ok(Some(format!("#/components/{ty}/{name}"))); + } + let mut value = self.fetch(&target, &pointer, reference)?; + + let segments = decode_pointer(&pointer); + let (ty, wanted) = match segments.as_slice() { + [c, ty, name] if c == "components" && COMPONENT_TYPES.contains(&ty.as_str()) => ( + *COMPONENT_TYPES.iter().find(|t| **t == ty).unwrap(), + name.clone(), + ), + [d, name] if d == "definitions" || d == "$defs" => ("schemas", name.clone()), + [.., last] => (infer_type(stack), last.clone()), + [] => (infer_type(stack), file_stem(&target)), + }; + let wanted = sanitize_name(&wanted); + let name = self.unique_name(ty, &wanted, &target, &pointer); + self.memo.insert(key, (ty, name.clone())); + + let mut nested = vec![ty.to_string(), name.clone()]; + self.process(&mut value, &target, &mut nested, false)?; + self.hoisted.push((ty, name.clone(), value)); + Ok(Some(format!("#/components/{ty}/{name}"))) + } + + fn unique_name( + &mut self, + ty: &'static str, + wanted: &str, + target: &str, + pointer: &str, + ) -> String { + let taken = self.taken.entry(ty).or_default(); + if taken.insert(wanted.to_string()) { + return wanted.to_string(); + } + let stem = sanitize_name(&file_stem(target)); + let mut candidate = format!("{stem}_{wanted}"); + let mut n = 2; + while !taken.insert(candidate.clone()) { + candidate = format!("{stem}_{wanted}_{n}"); + n += 1; + } + self.notes.push(format!( + "hoisted '{target}#{pointer}' as components.{ty}.{candidate} ('{wanted}' was taken)" + )); + candidate + } + + fn finish(self, spec: &mut Value) -> Vec { + if self.hoisted.is_empty() { + return self.notes; + } + if !spec.is_object() { + return self.notes; + } + let root = spec.as_object_mut().expect("checked object"); + let components = root + .entry("components") + .or_insert_with(|| Value::Object(Default::default())); + if let Value::Object(components) = components { + for (ty, name, value) in self.hoisted { + let section = components + .entry(ty) + .or_insert_with(|| Value::Object(Default::default())); + if let Value::Object(section) = section { + section.insert(name, value); + } + } + } + self.notes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolves_relative_file_locations() { + assert_eq!( + resolve_location("specs/pets.yaml", "./schemas/pet.yaml"), + "specs/schemas/pet.yaml" + ); + assert_eq!( + resolve_location("specs/pets.yaml", "../common/e.yaml"), + "common/e.yaml" + ); + assert_eq!(resolve_location("pets.yaml", "../x.yaml"), "../x.yaml"); + assert_eq!( + resolve_location("specs/pets.yaml", "/etc/passwd"), + "/etc/passwd" + ); + assert_eq!( + resolve_location("specs/pets.yaml", "https://b.test/x.yaml"), + "https://b.test/x.yaml" + ); + assert_eq!(resolve_location("specs/pets.yaml", ""), "specs/pets.yaml"); + } + + #[test] + fn absolute_file_bases_stay_absolute() { + assert_eq!( + resolve_location("/tmp/x/openapi.yaml", "./schemas/pet.yaml"), + "/tmp/x/schemas/pet.yaml" + ); + assert_eq!( + resolve_location("/tmp/x/openapi.yaml", "../y/d.yaml"), + "/tmp/y/d.yaml" + ); + assert_eq!( + resolve_location("/tmp/x/openapi.yaml", "../../../../d.yaml"), + "/d.yaml" + ); + } + + #[test] + fn resolves_relative_url_locations() { + let base = "https://a.test/api/v1/spec.json"; + assert_eq!( + resolve_location(base, "defs.yaml"), + "https://a.test/api/v1/defs.yaml" + ); + assert_eq!( + resolve_location(base, "../common.yaml"), + "https://a.test/api/common.yaml" + ); + assert_eq!( + resolve_location(base, "/root.yaml"), + "https://a.test/root.yaml" + ); + assert_eq!( + resolve_location(base, "https://b.test/x.yaml"), + "https://b.test/x.yaml" + ); + assert_eq!( + resolve_location(base, "file:///etc/passwd"), + "file:///etc/passwd" + ); + assert_eq!( + resolve_location("https://a.test:8080/s.json?x=1", "d.yaml"), + "https://a.test:8080/d.yaml" + ); + assert_eq!( + resolve_location("https://a.test/s.json", "../../d.yaml"), + "https://a.test/d.yaml" + ); + } + + #[test] + fn splits_refs() { + assert_eq!( + split_ref("a.yaml#/components/schemas/Pet"), + ("a.yaml", "/components/schemas/Pet") + ); + assert_eq!(split_ref("#/x"), ("", "/x")); + assert_eq!(split_ref("a.yaml"), ("a.yaml", "")); + } + + #[test] + fn decodes_pointers() { + assert_eq!(decode_pointer("/a~1b/c%20d/~0"), vec!["a/b", "c d", "~"]); + assert!(decode_pointer("").is_empty()); + } + + #[test] + fn is_url_detects_http_only() { + assert!(is_url("https://a.test/x") && is_url("http://a.test/x")); + assert!(!is_url("specs/a.yaml") && !is_url("file:///x")); + } + + use serde_json::json; + + #[test] + fn finds_missing_documents_transitively() { + let spec = json!({"a": {"$ref": "./schemas/pet.yaml#/Pet"}, "b": {"$ref": "#/components/schemas/Local"}}); + let mut docs = HashMap::new(); + assert_eq!( + missing_documents(&spec, "specs/pets.yaml", &docs), + vec![MissingDocument { + location: "specs/schemas/pet.yaml".into(), + referenced_from: "specs/pets.yaml".into() + }] + ); + + docs.insert("specs/schemas/pet.yaml".to_string(), json!({"Pet": {"$ref": "../common/err.yaml#/E"}, "Other": {"$ref": "../common/err.yaml#/F"}})); + assert_eq!( + missing_documents(&spec, "specs/pets.yaml", &docs), + vec![MissingDocument { + location: "specs/common/err.yaml".into(), + referenced_from: "specs/schemas/pet.yaml".into() + }] + ); + } + + #[test] + fn ignores_own_location_and_example_data() { + let spec = json!({ + "a": {"$ref": "pets.yaml#/x"}, + "b": {"example": {"$ref": "other.yaml"}}, + "c": {"properties": {"default": {"$ref": "real.yaml#/R"}, "example": {"$ref": "real2.yaml#/R"}}} + }); + let missing: Vec = missing_documents(&spec, "pets.yaml", &HashMap::new()) + .into_iter() + .map(|m| m.location) + .collect(); + assert_eq!(missing, vec!["real.yaml", "real2.yaml"]); + } + + #[test] + fn circular_documents_terminate() { + let mut docs = HashMap::new(); + docs.insert("a.yaml".to_string(), json!({"A": {"$ref": "b.yaml#/B"}})); + docs.insert("b.yaml".to_string(), json!({"B": {"$ref": "a.yaml#/A"}})); + let spec = json!({"$ref": "a.yaml#/A"}); + assert!(missing_documents(&spec, "root.yaml", &docs).is_empty()); + } + + fn run(spec: Value, base: &str, docs: Vec<(&str, Value)>) -> (Value, Vec) { + let docs: HashMap = + docs.into_iter().map(|(k, v)| (k.to_string(), v)).collect(); + bundle(spec, base, &docs).unwrap() + } + + fn root(schema_ref: &str) -> Value { + json!({"openapi":"3.0.3","info":{"title":"T","version":"1"},"paths":{"/p":{"get":{"responses":{"200":{ + "description":"ok","content":{"application/json":{"schema":{"$ref": schema_ref}}}}}}}}}) + } + + fn schema_ref(spec: &Value) -> &str { + spec["paths"]["/p"]["get"]["responses"]["200"]["content"]["application/json"]["schema"] + ["$ref"] + .as_str() + .unwrap() + } + + #[test] + fn hoists_schema_by_component_pointer() { + let target = json!({"type":"object","properties":{"m":{"type":"string"}}}); + let (out, notes) = run( + root("./e.yaml#/components/schemas/Error"), + "root.yaml", + vec![( + "e.yaml", + json!({"components":{"schemas":{"Error": target.clone()}}}), + )], + ); + assert_eq!(schema_ref(&out), "#/components/schemas/Error"); + assert_eq!(out["components"]["schemas"]["Error"], target); + assert!(notes.is_empty()); + } + + #[test] + fn hoists_definitions_and_defs_as_schemas() { + for pointer in ["/definitions/Pet", "/$defs/Pet"] { + let doc = if pointer.contains("defs/") && pointer.starts_with("/$") { + json!({"$defs":{"Pet":{"type":"object"}}}) + } else { + json!({"definitions":{"Pet":{"type":"object"}}}) + }; + let (out, _) = run( + root(&format!("d.yaml#{pointer}")), + "root.yaml", + vec![("d.yaml", doc)], + ); + assert_eq!(schema_ref(&out), "#/components/schemas/Pet"); + assert_eq!(out["components"]["schemas"]["Pet"]["type"], "object"); + } + } + + #[test] + fn infers_type_from_position() { + let spec = json!({"openapi":"3.0.3","paths":{"/p":{"post":{ + "parameters":[{"$ref":"d.yaml#/Limit"}], + "requestBody":{"$ref":"d.yaml#/Body"}, + "responses":{"404":{"$ref":"d.yaml#/NotFound"}}}}}}); + let d = json!({"Limit":{"name":"limit","in":"query"},"Body":{"content":{}},"NotFound":{"description":"nf"}}); + let (out, _) = run(spec, "root.yaml", vec![("d.yaml", d)]); + let op = &out["paths"]["/p"]["post"]; + assert_eq!(op["parameters"][0]["$ref"], "#/components/parameters/Limit"); + assert_eq!(op["requestBody"]["$ref"], "#/components/requestBodies/Body"); + assert_eq!( + op["responses"]["404"]["$ref"], + "#/components/responses/NotFound" + ); + assert!(out["components"]["parameters"]["Limit"].is_object()); + assert!(out["components"]["requestBodies"]["Body"].is_object()); + assert!(out["components"]["responses"]["NotFound"].is_object()); + } + + #[test] + fn whole_document_ref_is_named_after_the_file_stem() { + let (out, _) = run( + root("./models/pet.yaml"), + "root.yaml", + vec![("models/pet.yaml", json!({"type":"object"}))], + ); + assert_eq!(schema_ref(&out), "#/components/schemas/pet"); + } + + #[test] + fn internal_and_relative_refs_in_external_docs_are_bundled() { + let (out, _) = run( + root("./models/pet.yaml#/Pet"), + "root.yaml", + vec![ + ( + "models/pet.yaml", + json!({"Pet":{"type":"object","properties":{ + "owner":{"$ref":"#/Owner"},"tag":{"$ref":"./tag.yaml#/Tag"}}},"Owner":{"type":"string"}}), + ), + ("models/tag.yaml", json!({"Tag":{"type":"integer"}})), + ], + ); + let s = &out["components"]["schemas"]; + assert_eq!( + s["Pet"]["properties"]["owner"]["$ref"], + "#/components/schemas/Owner" + ); + assert_eq!( + s["Pet"]["properties"]["tag"]["$ref"], + "#/components/schemas/Tag" + ); + assert_eq!(s["Owner"]["type"], "string"); + assert_eq!(s["Tag"]["type"], "integer"); + } + + #[test] + fn circular_refs_terminate_and_hoist_once() { + let (out, _) = run( + root("a.yaml#/A"), + "root.yaml", + vec![ + ( + "a.yaml", + json!({"A":{"properties":{"b":{"$ref":"b.yaml#/B"}}}}), + ), + ( + "b.yaml", + json!({"B":{"properties":{"a":{"$ref":"a.yaml#/A"}}}}), + ), + ], + ); + let s = out["components"]["schemas"].as_object().unwrap(); + assert_eq!(s.len(), 2); + assert_eq!(s["B"]["properties"]["a"]["$ref"], "#/components/schemas/A"); + } + + #[test] + fn same_target_is_hoisted_once() { + let spec = + json!({"a":{"schema":{"$ref":"d.yaml#/X"}},"b":{"schema":{"$ref":"./d.yaml#/X"}}}); + let (out, _) = run( + spec, + "root.yaml", + vec![("d.yaml", json!({"X":{"type":"string"}}))], + ); + assert_eq!(out["a"]["schema"]["$ref"], out["b"]["schema"]["$ref"]); + assert_eq!(out["components"]["schemas"].as_object().unwrap().len(), 1); + } + + #[test] + fn name_clash_adds_file_stem_and_note() { + let mut spec = root("models/pet.yaml#/Pet"); + spec["components"] = json!({"schemas":{"Pet":{"type":"string"}}}); + spec["x"] = json!({"schema":{"$ref":"models/other/pet.yaml#/Pet"}}); + let (out, notes) = run( + spec, + "root.yaml", + vec![ + ("models/pet.yaml", json!({"Pet":{"type":"object"}})), + ("models/other/pet.yaml", json!({"Pet":{"type":"integer"}})), + ], + ); + assert_eq!(schema_ref(&out), "#/components/schemas/pet_Pet"); + assert_eq!(out["x"]["schema"]["$ref"], "#/components/schemas/pet_Pet_2"); + assert_eq!(out["components"]["schemas"]["Pet"]["type"], "string"); + assert!( + notes[0].contains( + "hoisted 'models/pet.yaml#/Pet' as components.schemas.pet_Pet ('Pet' was taken)" + ), + "{notes:?}" + ); + assert_eq!(notes.len(), 2); + } + + #[test] + fn root_internal_refs_are_left_alone_and_refs_back_into_root_become_internal() { + let mut spec = root("d.yaml#/D"); + spec["components"] = json!({"schemas":{"X":{"type":"string"}}}); + let (out, _) = run( + spec, + "root.yaml", + vec![( + "d.yaml", + json!({"D":{"properties":{"x":{"$ref":"root.yaml#/components/schemas/X"}}}}), + )], + ); + assert_eq!( + out["components"]["schemas"]["D"]["properties"]["x"]["$ref"], + "#/components/schemas/X" + ); + assert_eq!(out["components"]["schemas"].as_object().unwrap().len(), 2); + + let internal = root("#/components/schemas/X"); + let (same, notes) = run(internal.clone(), "root.yaml", vec![]); + assert_eq!(same, internal); + assert!(notes.is_empty()); + } + + #[test] + fn path_item_ref_is_inlined() { + let spec = + json!({"openapi":"3.0.3","paths":{"/x":{"$ref":"p.yaml#/paths/~1x","summary":"own"}}}); + let p = json!({"paths":{"/x":{"summary":"theirs","get":{"responses":{"200":{"description":"ok","content":{ + "application/json":{"schema":{"$ref":"#/definitions/T"}}}}}}}},"definitions":{"T":{"type":"string"}}}); + let (out, _) = run(spec, "root.yaml", vec![("p.yaml", p)]); + let item = &out["paths"]["/x"]; + assert!(item.get("$ref").is_none()); + assert_eq!(item["summary"], "own"); + assert_eq!( + item["get"]["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/T" + ); + assert_eq!(out["components"]["schemas"]["T"]["type"], "string"); + } + + #[test] + fn example_data_is_not_treated_as_a_ref() { + let spec = + json!({"a":{"example":{"$ref":"nope.yaml#/x"}},"b":{"schema":{"$ref":"d.yaml#/X"}}}); + let (out, _) = run(spec, "root.yaml", vec![("d.yaml", json!({"X":{}}))]); + assert_eq!(out["a"]["example"]["$ref"], "nope.yaml#/x"); + } + + #[test] + fn invalid_component_name_characters_are_replaced() { + let (out, _) = run( + root("d.yaml#/My%20Pet"), + "root.yaml", + vec![("d.yaml", json!({"My Pet":{"type":"object"}}))], + ); + assert_eq!(schema_ref(&out), "#/components/schemas/My_Pet"); + } + + #[test] + fn errors_for_missing_pointer_and_missing_document() { + let docs: HashMap = [("d.yaml".to_string(), json!({"X":{}}))].into(); + let err = bundle(root("d.yaml#/Nope"), "root.yaml", &docs) + .unwrap_err() + .to_string(); + assert!( + err.contains("d.yaml#/Nope") && err.contains("not found"), + "{err}" + ); + let err = bundle(root("missing.yaml#/X"), "root.yaml", &docs) + .unwrap_err() + .to_string(); + assert!( + err.contains("missing.yaml") && err.contains("not loaded"), + "{err}" + ); + } +} diff --git a/crates/openapi-aggregator-core/src/error.rs b/crates/openapi-aggregator-core/src/error.rs index a0ebf97..ff3252a 100644 --- a/crates/openapi-aggregator-core/src/error.rs +++ b/crates/openapi-aggregator-core/src/error.rs @@ -14,6 +14,9 @@ pub enum Error { #[error("configuration error: {0}")] Config(String), + #[error("$ref '{reference}' {reason}")] + UnresolvedRef { reference: String, reason: String }, + #[error("no sources provided")] NoSources, } diff --git a/crates/openapi-aggregator-core/src/lib.rs b/crates/openapi-aggregator-core/src/lib.rs index 3dcb72e..7380a8e 100644 --- a/crates/openapi-aggregator-core/src/lib.rs +++ b/crates/openapi-aggregator-core/src/lib.rs @@ -35,16 +35,23 @@ //! assert!(merged["paths"]["/orders"].is_object()); //! ``` +pub mod bundle; pub mod config; pub mod error; pub mod merge; pub mod prepare; pub mod swagger2; +pub use bundle::{ + bundle, check_document_cap, is_url, missing_documents, resolve_location, MissingDocument, + MAX_DOCUMENTS, +}; pub use config::{ Config, ConflictStrategy, InfoOverride, MergeConfig, OutputConfig, OutputFormat, ServerEntry, Source, TagEntry, TagPrefixStrategy, }; pub use error::Error; pub use merge::{merge_specs, merge_specs_with_report, MergeReport}; -pub use prepare::{parse_content, prepare_source, prepare_source_with_notes, PreparedSource}; +pub use prepare::{ + bundle_source, parse_content, prepare_source, prepare_source_with_notes, PreparedSource, +}; diff --git a/crates/openapi-aggregator-core/src/prepare.rs b/crates/openapi-aggregator-core/src/prepare.rs index dd49cbc..cbf7a3c 100644 --- a/crates/openapi-aggregator-core/src/prepare.rs +++ b/crates/openapi-aggregator-core/src/prepare.rs @@ -1,5 +1,8 @@ +use std::collections::HashMap; + use serde_json::Value; +use crate::bundle::bundle; use crate::config::Source; use crate::error::Error; use crate::merge::deep_merge; @@ -33,6 +36,26 @@ pub fn prepare_source_with_notes(content: &str, source: &Source) -> Result': `. +pub fn bundle_source( + mut prepared: PreparedSource, + base: &str, + docs: &HashMap, +) -> Result { + let name = prepared.name.clone(); + let (spec, notes) = bundle(prepared.spec, base, docs).map_err(|e| Error::InvalidSpec { + name: name.clone(), + reason: e.to_string(), + })?; + prepared.spec = spec; + prepared + .notes + .extend(notes.into_iter().map(|n| format!("source '{name}': {n}"))); + Ok(prepared) +} + fn parse_and_prepare(content: &str, source: &Source) -> Result<(Value, Vec), Error> { let mut value = parse_content(content)?; let mut notes = Vec::new(); @@ -265,4 +288,82 @@ mod tests { let err = parse_and_prepare(base, &source).unwrap_err().to_string(); assert!(err.contains("additional_blocks")); } + + fn docs_of(entries: Vec<(&str, Value)>) -> std::collections::HashMap { + entries + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect() + } + + #[test] + fn bundle_source_bundles_and_prefixes_notes() { + let source = file_source("pets", None); + let prepared = prepare_source_with_notes( + r#"{"openapi":"3.0.3","info":{"title":"T","version":"1"},"paths":{}, + "components":{"schemas":{"Pet":{"type":"string"}}}, + "x":{"schema":{"$ref":"m/pet.yaml#/Pet"}}}"#, + &source, + ) + .unwrap(); + let docs = docs_of(vec![("m/pet.yaml", json!({"Pet": {"type": "object"}}))]); + let bundled = bundle_source(prepared, "pets.yaml", &docs).unwrap(); + assert_eq!( + bundled.spec["x"]["schema"]["$ref"], + "#/components/schemas/pet_Pet" + ); + assert!(bundled + .notes + .iter() + .all(|n| n.starts_with("source 'pets': "))); + assert!(bundled.notes[0].contains("hoisted 'm/pet.yaml#/Pet'")); + } + + #[test] + fn bundle_source_errors_name_the_source() { + let source = file_source("pets", None); + let prepared = prepare_source_with_notes( + r#"{"openapi":"3.0.3","info":{"title":"T","version":"1"},"paths":{},"x":{"$ref":"d.yaml#/Nope"}}"#, + &source, + ) + .unwrap(); + let docs = docs_of(vec![("d.yaml", json!({}))]); + let err = bundle_source(prepared, "pets.yaml", &docs) + .unwrap_err() + .to_string(); + assert!(err.contains("pets") && err.contains("not found"), "{err}"); + } + + #[test] + fn swagger2_source_converts_then_bundles_external_definitions() { + let source = file_source("legacy", None); + let prepared = prepare_source_with_notes( + "swagger: '2.0'\ninfo: {title: T, version: '1'}\npaths:\n /p:\n get:\n responses:\n '200':\n description: ok\n schema:\n $ref: defs.yaml#/definitions/Pet\n", + &source, + ) + .unwrap(); + let docs = docs_of(vec![( + "defs.yaml", + json!({"definitions": {"Pet": {"type": "object"}}}), + )]); + let bundled = bundle_source(prepared, "legacy.yaml", &docs).unwrap(); + assert_eq!( + bundled.spec["paths"]["/p"]["get"]["responses"]["200"]["content"]["application/json"] + ["schema"]["$ref"], + "#/components/schemas/Pet" + ); + assert_eq!( + bundled.spec["components"]["schemas"]["Pet"]["type"], + "object" + ); + } + + #[test] + fn document_cap_allows_100_and_rejects_101() { + assert!(crate::bundle::check_document_cap(100, "x").is_ok()); + let err = crate::bundle::check_document_cap(101, "x") + .unwrap_err() + .to_string(); + assert!(err.contains("limit 100") && err.contains("x"), "{err}"); + } } diff --git a/crates/openapi-aggregator-wasm/src/lib.rs b/crates/openapi-aggregator-wasm/src/lib.rs index 4df4276..681a8ef 100644 --- a/crates/openapi-aggregator-wasm/src/lib.rs +++ b/crates/openapi-aggregator-wasm/src/lib.rs @@ -1,10 +1,11 @@ //! Browser bindings for openapi-aggregator. Fetching happens in JS; //! this crate only parses the config and merges already-loaded spec text. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use openapi_aggregator_core::{ - merge_specs_with_report, prepare_source_with_notes, Config, Error, OutputFormat, Source, + bundle_source, check_document_cap, merge_specs_with_report, missing_documents, parse_content, + prepare_source_with_notes, Config, Error, OutputFormat, PreparedSource, Source, }; use serde::Serialize; use wasm_bindgen::prelude::*; @@ -68,8 +69,48 @@ pub fn summarize_config(yaml: &str) -> Result { }) } -pub fn aggregate_contents(yaml: &str, contents: Vec) -> Result { - let config = Config::from_yaml(yaml).map_err(|e| e.to_string())?; +/// A document a source references that has not been supplied yet. +#[derive(Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct MissingDocumentOut { + pub location: String, + pub referenced_from: String, +} + +/// For each source, the external documents it still needs. +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct PendingRefs { + pub sources: Vec>, +} + +/// Strip leading `./` so a config `path:` matches a workspace file name. +fn workspace_key(path: &str) -> String { + let mut key = path; + while let Some(rest) = key.strip_prefix("./") { + key = rest; + } + key.to_string() +} + +/// The location a source's relative `$ref`s are resolved against. +fn base_of(source: &Source) -> String { + match source { + Source::File { path, .. } => workspace_key(&path.to_string_lossy()), + Source::Http { url, .. } => url.clone(), + } +} + +fn parse_docs(docs: HashMap) -> Result, String> { + docs.into_iter() + .map(|(location, text)| { + parse_content(&text) + .map(|value| (location.clone(), value)) + .map_err(|e| format!("document '{location}': {e}")) + }) + .collect() +} + +fn prepare_all(config: &Config, contents: &[String]) -> Result, String> { if config.sources.is_empty() { return Err(Error::NoSources.to_string()); } @@ -80,14 +121,60 @@ pub fn aggregate_contents(yaml: &str, contents: Vec) -> Result format!("source '{}': {e}", source.display_name()), + other => other.to_string(), + }) + }) + .collect() +} + +/// For each source, the external documents (by location) still missing from `docs`. +pub fn pending_refs( + yaml: &str, + contents: Vec, + docs: HashMap, +) -> Result { + let config = Config::from_yaml(yaml).map_err(|e| e.to_string())?; + let prepared = prepare_all(&config, &contents)?; + let docs = parse_docs(docs)?; + let sources = config + .sources + .iter() + .zip(&prepared) + .map(|(source, p)| { + missing_documents(&p.spec, &base_of(source), &docs) + .into_iter() + .map(|m| MissingDocumentOut { + location: m.location, + referenced_from: m.referenced_from, + }) + .collect() + }) + .collect(); + Ok(PendingRefs { sources }) +} + +pub fn aggregate_contents( + yaml: &str, + contents: Vec, + docs: HashMap, +) -> Result { + let config = Config::from_yaml(yaml).map_err(|e| e.to_string())?; + let prepared = prepare_all(&config, &contents)?; + let docs = parse_docs(docs)?; let mut notes = Vec::new(); let mut specs = Vec::new(); - for (source, content) in config.sources.iter().zip(&contents) { - let prepared = prepare_source_with_notes(content, source).map_err(|e| match e { - Error::Parse(_) => format!("source '{}': {e}", source.display_name()), - other => other.to_string(), - })?; + for (source, prepared) in config.sources.iter().zip(prepared) { + check_document_cap(docs.len(), &prepared.name).map_err(|e| e.to_string())?; + let prepared = + bundle_source(prepared, &base_of(source), &docs).map_err(|e| e.to_string())?; notes.extend(prepared.notes); specs.push((prepared.name, source.tag_prefix(), prepared.spec)); } @@ -119,11 +206,30 @@ pub fn parse_config(yaml: &str) -> Result { to_js(&summarize_config(yaml).map_err(|e| JsError::new(&e))?) } -/// Merge already-loaded spec texts (`contents[i]` belongs to `sources[i]`); -/// returns an `AggregateOutput` or throws. +fn docs_from_js(docs: JsValue) -> Result, JsError> { + if docs.is_undefined() || docs.is_null() { + return Ok(HashMap::new()); + } + serde_wasm_bindgen::from_value(docs) + .map_err(|e| JsError::new(&format!("invalid external documents: {e}"))) +} + +/// Which external documents each source still needs (`docs` maps location -> text); +/// returns `PendingRefs` or throws. +#[wasm_bindgen(js_name = pendingRefs)] +pub fn pending_refs_js( + yaml: &str, + contents: Vec, + docs: JsValue, +) -> Result { + to_js(&pending_refs(yaml, contents, docs_from_js(docs)?).map_err(|e| JsError::new(&e))?) +} + +/// Merge already-loaded spec texts (`contents[i]` belongs to `sources[i]`), bundling +/// external refs from `docs` (location -> text); returns an `AggregateOutput` or throws. #[wasm_bindgen] -pub fn aggregate(yaml: &str, contents: Vec) -> Result { - to_js(&aggregate_contents(yaml, contents).map_err(|e| JsError::new(&e))?) +pub fn aggregate(yaml: &str, contents: Vec, docs: JsValue) -> Result { + to_js(&aggregate_contents(yaml, contents, docs_from_js(docs)?).map_err(|e| JsError::new(&e))?) } #[cfg(test)] @@ -169,6 +275,7 @@ mod tests { let out = aggregate_contents( "sources:\n - path: pets.yaml\n - path: users.yaml\n", vec![PETS.into(), USERS.into()], + Default::default(), ) .unwrap(); assert!(out.text.starts_with("openapi: 3.0.3"), "{}", out.text); @@ -181,6 +288,7 @@ mod tests { let out = aggregate_contents( "sources:\n - path: pets.yaml\noutput: {format: json}\n", vec![PETS.into()], + Default::default(), ) .unwrap(); let parsed: serde_json::Value = serde_json::from_str(&out.text).unwrap(); @@ -189,14 +297,23 @@ mod tests { #[test] fn unparseable_content_names_the_source() { - let err = aggregate_contents("sources:\n - path: pets.yaml\n", vec!["{: nope".into()]) - .unwrap_err(); + let err = aggregate_contents( + "sources:\n - path: pets.yaml\n", + vec!["{: nope".into()], + Default::default(), + ) + .unwrap_err(); assert!(err.contains("source 'pets'"), "{err}"); } #[test] fn content_count_must_match_sources() { - let err = aggregate_contents("sources:\n - path: pets.yaml\n", vec![]).unwrap_err(); + let err = aggregate_contents( + "sources:\n - path: pets.yaml\n", + vec![], + Default::default(), + ) + .unwrap_err(); assert!(err.contains("expected 1"), "{err}"); } @@ -206,6 +323,7 @@ mod tests { let out = aggregate_contents( "sources:\n - name: legacy\n path: legacy.yaml\n - path: pets.yaml\n", vec![legacy.into(), PETS.into()], + Default::default(), ) .unwrap(); assert!(out.text.contains("/legacy:") && out.text.contains("/pets:")); @@ -217,4 +335,90 @@ mod tests { out.warnings ); } + + const ROOT_WITH_REF: &str = "openapi: 3.0.3\ninfo: {title: Root, version: '1'}\npaths:\n /p:\n get:\n responses:\n '200':\n description: ok\n content:\n application/json:\n schema:\n $ref: './a.yaml#/Pet'\n"; + + fn docs(entries: &[(&str, &str)]) -> HashMap { + entries + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn pending_refs_lists_missing_documents_per_source() { + let yaml = "sources:\n - path: ./root.yaml\n - path: pets.yaml\n"; + let pending = pending_refs( + yaml, + vec![ROOT_WITH_REF.into(), PETS.into()], + HashMap::new(), + ) + .unwrap(); + assert_eq!( + pending.sources, + vec![ + vec![MissingDocumentOut { + location: "a.yaml".into(), + referenced_from: "root.yaml".into() + }], + vec![] + ] + ); + + let a = "Pet:\n properties:\n t:\n $ref: 'b.yaml#/T'\n"; + let pending = pending_refs( + yaml, + vec![ROOT_WITH_REF.into(), PETS.into()], + docs(&[("a.yaml", a)]), + ) + .unwrap(); + assert_eq!( + pending.sources[0], + vec![MissingDocumentOut { + location: "b.yaml".into(), + referenced_from: "a.yaml".into() + }] + ); + + let json = serde_json::to_value(&pending).unwrap(); + assert_eq!(json["sources"][0][0]["referencedFrom"], "a.yaml"); + } + + #[test] + fn pending_refs_names_unparseable_documents() { + let err = pending_refs( + "sources:\n - path: root.yaml\n", + vec![ROOT_WITH_REF.into()], + docs(&[("a.yaml", "{: nope")]), + ) + .unwrap_err(); + assert!(err.contains("a.yaml"), "{err}"); + } + + #[test] + fn aggregate_bundles_external_documents() { + let out = aggregate_contents( + "sources:\n - path: root.yaml\n", + vec![ROOT_WITH_REF.into()], + docs(&[("a.yaml", "Pet:\n type: object\n")]), + ) + .unwrap(); + assert!( + out.text.contains("components:") && out.text.contains("Pet:"), + "{}", + out.text + ); + assert!(!out.text.contains("a.yaml#"), "{}", out.text); + } + + #[test] + fn aggregate_without_the_document_names_source_and_location() { + let err = aggregate_contents( + "sources:\n - path: root.yaml\n", + vec![ROOT_WITH_REF.into()], + HashMap::new(), + ) + .unwrap_err(); + assert!(err.contains("root") && err.contains("a.yaml"), "{err}"); + } } diff --git a/crates/openapi-aggregator/src/lib.rs b/crates/openapi-aggregator/src/lib.rs index ee677a3..86aba22 100644 --- a/crates/openapi-aggregator/src/lib.rs +++ b/crates/openapi-aggregator/src/lib.rs @@ -7,22 +7,34 @@ //! The crate also ships the `openapi-aggregator` command-line tool. pub mod error; +mod refs; pub mod source; pub use error::Error; pub use openapi_aggregator_core::{ - config, merge, merge_specs, merge_specs_with_report, parse_content, prepare_source, - prepare_source_with_notes, Config, ConflictStrategy, InfoOverride, MergeConfig, MergeReport, + bundle_source, check_document_cap, config, is_url, merge, merge_specs, merge_specs_with_report, + missing_documents, parse_content, prepare_source, prepare_source_with_notes, resolve_location, + Config, ConflictStrategy, InfoOverride, MergeConfig, MergeReport, MissingDocument, OutputConfig, OutputFormat, PreparedSource, ServerEntry, Source, TagEntry, TagPrefixStrategy, }; pub use source::load_source; -use std::path::Path; +use std::path::{Path, PathBuf}; use futures_util::future::try_join_all; use openapi_aggregator_core::Error as CoreError; use source::{http_client, load_source_with_client}; +/// Options for [`aggregate_with_options`]. +#[derive(Debug, Clone, Default)] +pub struct AggregateOptions { + /// Allow external `$ref`s to read files outside the config directory (and the + /// directories of the file sources). Off by default. + pub allow_outside_config_dir: bool, + /// The directory the config was loaded from; defaults to the current directory. + pub config_dir: Option, +} + /// Load all sources defined in `config` and merge them into a single OpenAPI spec. pub async fn aggregate(config: &Config) -> Result { aggregate_with_report(config) @@ -33,15 +45,29 @@ pub async fn aggregate(config: &Config) -> Result { /// Like [`aggregate`], but also returns non-fatal merge warnings. /// /// Sources are loaded concurrently; the merge order still follows `config.sources`. +/// External `$ref`s are resolved with the default (sandboxed) [`AggregateOptions`]. pub async fn aggregate_with_report(config: &Config) -> Result { + aggregate_with_options(config, &AggregateOptions::default()).await +} + +/// Like [`aggregate_with_report`], with explicit [`AggregateOptions`]. +pub async fn aggregate_with_options( + config: &Config, + options: &AggregateOptions, +) -> Result { if config.sources.is_empty() { return Err(CoreError::NoSources.into()); } let client = http_client()?; + let ctx = refs::RefContext { + client: &client, + options, + allowed_roots: refs::allowed_roots(&config.sources, options), + }; let prepared = try_join_all(config.sources.iter().map(|src| async { let prepared = load_source_with_client(src, &client).await?; - Ok::<_, Error>(prepared) + refs::resolve_external_refs(prepared, src, &ctx).await })) .await?; @@ -84,5 +110,11 @@ pub fn load_config(path: &Path) -> Result { /// Read a config file, resolve relative source paths against its directory, /// then aggregate. pub async fn aggregate_from_file(path: &Path) -> Result { - aggregate(&load_config(path)?).await + let options = AggregateOptions { + config_dir: path.parent().map(Path::to_path_buf), + ..Default::default() + }; + aggregate_with_options(&load_config(path)?, &options) + .await + .map(|report| report.spec) } diff --git a/crates/openapi-aggregator/src/main.rs b/crates/openapi-aggregator/src/main.rs index 10313c6..249f447 100644 --- a/crates/openapi-aggregator/src/main.rs +++ b/crates/openapi-aggregator/src/main.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use clap::Parser; -use openapi_aggregator::{aggregate_with_report, load_config, OutputFormat}; +use openapi_aggregator::{aggregate_with_options, load_config, AggregateOptions, OutputFormat}; #[derive(Parser)] #[command(name = "openapi-aggregator")] @@ -19,6 +19,10 @@ struct Cli { /// Output format (overrides config) #[arg(short, long, value_parser = parse_format)] format: Option, + + /// Let external `$ref`s read files outside the config directory + #[arg(long)] + allow_outside_config_dir: bool, } fn parse_format(s: &str) -> Result { @@ -43,7 +47,11 @@ async fn main() { async fn run(cli: Cli) -> Result<(), Box> { let config = load_config(&cli.config)?; - let report = aggregate_with_report(&config).await?; + let options = AggregateOptions { + allow_outside_config_dir: cli.allow_outside_config_dir, + config_dir: cli.config.parent().map(PathBuf::from), + }; + let report = aggregate_with_options(&config, &options).await?; for warning in &report.warnings { eprintln!("warning: {warning}"); } @@ -64,3 +72,17 @@ async fn run(cli: Cli) -> Result<(), Box> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_allow_outside_config_dir_flag() { + let cli = + Cli::try_parse_from(["openapi-aggregator", "--allow-outside-config-dir"]).unwrap(); + assert!(cli.allow_outside_config_dir); + let cli = Cli::try_parse_from(["openapi-aggregator"]).unwrap(); + assert!(!cli.allow_outside_config_dir); + } +} diff --git a/crates/openapi-aggregator/src/refs.rs b/crates/openapi-aggregator/src/refs.rs new file mode 100644 index 0000000..60fe53f --- /dev/null +++ b/crates/openapi-aggregator/src/refs.rs @@ -0,0 +1,174 @@ +//! Loading the external documents a source's `$ref`s point at, then bundling them. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use openapi_aggregator_core::{ + bundle_source, check_document_cap, is_url, missing_documents, parse_content, + Error as CoreError, MissingDocument, PreparedSource, Source, +}; +use serde_json::Value; + +use crate::error::Error; +use crate::source::expand_env; +use crate::AggregateOptions; + +pub(crate) struct RefContext<'a> { + pub client: &'a reqwest::Client, + pub options: &'a AggregateOptions, + /// Canonicalised directories external file refs may read from. + pub allowed_roots: Vec, +} + +/// The directories external file refs may point into: the config directory and the +/// directory of every file source. +pub(crate) fn allowed_roots(sources: &[Source], options: &AggregateOptions) -> Vec { + let config_dir = options + .config_dir + .clone() + .filter(|d| !d.as_os_str().is_empty()) + .unwrap_or_else(|| PathBuf::from(".")); + let mut roots = vec![config_dir]; + for source in sources { + if let Source::File { path, .. } = source { + let parent = path.parent().filter(|p| !p.as_os_str().is_empty()); + roots.push(parent.map_or_else(|| PathBuf::from("."), Path::to_path_buf)); + } + } + roots + .into_iter() + .filter_map(|r| std::fs::canonicalize(r).ok()) + .collect() +} + +fn invalid(name: &str, reason: String) -> Error { + CoreError::InvalidSpec { + name: name.to_string(), + reason, + } + .into() +} + +/// Load every document `prepared` references (transitively) and bundle them in. +pub(crate) async fn resolve_external_refs( + prepared: PreparedSource, + source: &Source, + ctx: &RefContext<'_>, +) -> Result { + let base = match source { + Source::File { path, .. } => path.to_string_lossy().into_owned(), + Source::Http { url, .. } => url.clone(), + }; + let mut docs: HashMap = HashMap::new(); + loop { + let missing = missing_documents(&prepared.spec, &base, &docs); + if missing.is_empty() { + break; + } + for document in missing { + check_document_cap(docs.len() + 1, &prepared.name)?; + let text = load_document(&document, &prepared.name, source, ctx).await?; + let value = parse_content(&text).map_err(|e| { + invalid( + &prepared.name, + format!( + "$ref document '{}' could not be parsed: {e}", + document.location + ), + ) + })?; + docs.insert(document.location, value); + } + } + Ok(bundle_source(prepared, &base, &docs)?) +} + +async fn load_document( + document: &MissingDocument, + name: &str, + source: &Source, + ctx: &RefContext<'_>, +) -> Result { + let location = &document.location; + let from = &document.referenced_from; + if location.starts_with("file:") { + return Err(invalid( + name, + format!("$ref '{location}' (from '{from}'): file: URLs are not supported"), + )); + } + if is_url(location) { + return load_url(location, source, ctx).await; + } + if is_url(from) { + return Err(invalid( + name, + format!( + "$ref '{location}' (from '{from}'): a remote document cannot reference local files" + ), + )); + } + load_file(location, from, name, ctx).await +} + +async fn load_file( + location: &str, + from: &str, + name: &str, + ctx: &RefContext<'_>, +) -> Result { + let unreadable = |e: std::io::Error| { + invalid( + name, + format!("$ref '{location}' (from '{from}') could not be read: {e}"), + ) + }; + let real = tokio::fs::canonicalize(location) + .await + .map_err(unreadable)?; + if !ctx.options.allow_outside_config_dir + && !ctx.allowed_roots.iter().any(|root| real.starts_with(root)) + { + return Err(invalid( + name, + format!( + "$ref '{location}' (from '{from}') is outside the config directory; \ + pass --allow-outside-config-dir to permit it" + ), + )); + } + tokio::fs::read_to_string(&real).await.map_err(unreadable) +} + +async fn load_url(location: &str, source: &Source, ctx: &RefContext<'_>) -> Result { + // Errors carry the unexpanded URL so `${SECRET}` values never reach logs. + let http_err = |e: reqwest::Error| Error::HttpRequest { + url: location.to_string(), + source: e.without_url(), + }; + let target = expand_env(location)?; + let mut request = ctx.client.get(&target); + if let Source::Http { url, headers, .. } = source { + if same_origin(&expand_env(url)?, &target) { + for (key, value) in headers { + request = request.header(key, expand_env(value)?); + } + } + } + request + .send() + .await + .map_err(http_err)? + .error_for_status() + .map_err(http_err)? + .text() + .await + .map_err(http_err) +} + +fn same_origin(a: &str, b: &str) -> bool { + match (reqwest::Url::parse(a), reqwest::Url::parse(b)) { + (Ok(a), Ok(b)) => a.origin() == b.origin(), + _ => false, + } +} diff --git a/crates/openapi-aggregator/src/source.rs b/crates/openapi-aggregator/src/source.rs index c42501a..0d06cfe 100644 --- a/crates/openapi-aggregator/src/source.rs +++ b/crates/openapi-aggregator/src/source.rs @@ -7,6 +7,8 @@ use openapi_aggregator_core::{ use crate::error::Error; +/// Load and prepare one source. External `$ref`s are left as they are; they are bundled by +/// [`aggregate_with_options`](crate::aggregate_with_options). pub async fn load_source(source: &Source) -> Result<(String, Value), Error> { let prepared = load_source_with_client(source, &http_client()?).await?; Ok((prepared.name, prepared.spec)) @@ -62,7 +64,7 @@ pub(crate) async fn load_source_with_client( } /// Replace `${NAME}` placeholders with environment variables. -fn expand_env(text: &str) -> Result { +pub(crate) fn expand_env(text: &str) -> Result { expand_env_with(text, |name| std::env::var(name).ok()) } diff --git a/crates/openapi-aggregator/tests/integration_test.rs b/crates/openapi-aggregator/tests/integration_test.rs index 176ac92..1884ed3 100644 --- a/crates/openapi-aggregator/tests/integration_test.rs +++ b/crates/openapi-aggregator/tests/integration_test.rs @@ -1,8 +1,8 @@ use std::path::Path; use openapi_aggregator::{ - aggregate, aggregate_from_file, aggregate_with_report, load_source, merge_specs, Config, - ConflictStrategy, MergeConfig, Source, + aggregate, aggregate_from_file, aggregate_with_options, aggregate_with_report, load_config, + load_source, merge_specs, AggregateOptions, Config, ConflictStrategy, MergeConfig, Source, }; fn fixtures() -> &'static Path { @@ -496,3 +496,313 @@ async fn swagger2_source_merges_with_openapi3_and_refs_resolve() { assert!(report.spec.pointer(pointer).is_some(), "dangling $ref {r}"); } } + +// --------------------------------------------------------------------------- +// external $refs +// --------------------------------------------------------------------------- + +const ROOT_SPEC: &str = "openapi: 3.0.3\ninfo: {title: Root, version: '1'}\npaths:\n /pets:\n get:\n responses:\n '200':\n description: ok\n content:\n application/json:\n schema:\n $ref: REF\n"; + +fn root_spec(reference: &str) -> String { + ROOT_SPEC.replace("REF", &format!("'{reference}'")) +} + +fn write(path: &std::path::Path, content: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, content).unwrap(); +} + +/// Aggregate the config at `dir/cfg.yaml` with default options (sandboxed) or with the escape hatch. +async fn run_config( + dir: &std::path::Path, + allow_outside: bool, +) -> Result { + let config = load_config(&dir.join("cfg.yaml")).unwrap(); + let options = AggregateOptions { + allow_outside_config_dir: allow_outside, + config_dir: Some(dir.to_path_buf()), + }; + aggregate_with_options(&config, &options).await +} + +fn assert_refs_resolve(spec: &serde_json::Value) { + let mut refs = Vec::new(); + collect_refs(spec, &mut refs); + assert!(!refs.is_empty()); + for r in refs { + assert!( + spec.pointer(r.trim_start_matches('#')).is_some(), + "dangling $ref {r}" + ); + } +} + +#[tokio::test] +async fn multi_file_spec_merges_and_refs_resolve() { + let dir = tempfile::tempdir().unwrap(); + write( + &dir.path().join("cfg.yaml"), + "sources:\n - name: pets\n path: openapi.yaml\n", + ); + write( + &dir.path().join("openapi.yaml"), + &root_spec("./schemas/pet.yaml#/Pet"), + ); + write( + &dir.path().join("schemas/pet.yaml"), + "Pet:\n type: object\n properties:\n owner:\n $ref: './owner.yaml#/Owner'\n", + ); + write( + &dir.path().join("schemas/owner.yaml"), + "Owner:\n type: string\n", + ); + + let report = run_config(dir.path(), false).await.unwrap(); + assert_refs_resolve(&report.spec); + assert_eq!( + report.spec["components"]["schemas"]["Owner"]["type"], + "string" + ); +} + +#[tokio::test] +async fn ref_outside_config_dir_is_rejected_unless_allowed() { + let dir = tempfile::tempdir().unwrap(); + let cfg = dir.path().join("cfg"); + write( + &cfg.join("cfg.yaml"), + "sources:\n - name: pets\n path: openapi.yaml\n", + ); + write(&cfg.join("openapi.yaml"), &root_spec("../outside.yaml#/X")); + write(&dir.path().join("outside.yaml"), "X:\n type: string\n"); + + let err = run_config(&cfg, false).await.unwrap_err().to_string(); + assert!(err.contains("is outside the config directory"), "{err}"); + assert!(err.contains("--allow-outside-config-dir"), "{err}"); + + let report = run_config(&cfg, true).await.unwrap(); + assert_eq!(report.spec["components"]["schemas"]["X"]["type"], "string"); +} + +#[cfg(unix)] +#[tokio::test] +async fn symlink_escape_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let cfg = dir.path().join("cfg"); + write( + &cfg.join("cfg.yaml"), + "sources:\n - name: pets\n path: openapi.yaml\n", + ); + write(&cfg.join("openapi.yaml"), &root_spec("./link.yaml#/X")); + write(&dir.path().join("outside.yaml"), "X:\n type: string\n"); + std::os::unix::fs::symlink(dir.path().join("outside.yaml"), cfg.join("link.yaml")).unwrap(); + + let err = run_config(&cfg, false).await.unwrap_err().to_string(); + assert!(err.contains("is outside the config directory"), "{err}"); +} + +#[tokio::test] +async fn absolute_path_ref_outside_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let cfg = dir.path().join("cfg"); + let outside = dir.path().join("outside.yaml"); + write(&outside, "X:\n type: string\n"); + write( + &cfg.join("cfg.yaml"), + "sources:\n - name: pets\n path: openapi.yaml\n", + ); + write( + &cfg.join("openapi.yaml"), + &root_spec(&format!("{}#/X", outside.display())), + ); + + let err = run_config(&cfg, false).await.unwrap_err().to_string(); + assert!(err.contains("is outside the config directory"), "{err}"); +} + +#[tokio::test] +async fn source_outside_config_dir_may_reference_its_siblings() { + let dir = tempfile::tempdir().unwrap(); + let cfg = dir.path().join("cfg"); + write( + &cfg.join("cfg.yaml"), + "sources:\n - name: shared\n path: ../shared/spec.yaml\n", + ); + write( + &dir.path().join("shared/spec.yaml"), + &root_spec("./defs.yaml#/Def"), + ); + write( + &dir.path().join("shared/defs.yaml"), + "Def:\n type: string\n", + ); + + let report = run_config(&cfg, false).await.unwrap(); + assert_eq!( + report.spec["components"]["schemas"]["Def"]["type"], + "string" + ); +} + +#[tokio::test] +async fn missing_ref_file_names_source_and_referrer() { + let dir = tempfile::tempdir().unwrap(); + write( + &dir.path().join("cfg.yaml"), + "sources:\n - name: pets\n path: openapi.yaml\n", + ); + write( + &dir.path().join("openapi.yaml"), + &root_spec("./nope.yaml#/X"), + ); + + let err = run_config(dir.path(), false).await.unwrap_err().to_string(); + assert!( + err.contains("pets") && err.contains("nope.yaml") && err.contains("openapi.yaml"), + "{err}" + ); +} + +#[tokio::test] +async fn too_many_external_documents_is_an_error() { + let dir = tempfile::tempdir().unwrap(); + write( + &dir.path().join("cfg.yaml"), + "sources:\n - name: pets\n path: openapi.yaml\n", + ); + write(&dir.path().join("openapi.yaml"), &root_spec("./d1.yaml#/D")); + for i in 1..=101 { + write( + &dir.path().join(format!("d{i}.yaml")), + &format!( + "D:\n properties:\n n:\n $ref: './d{}.yaml#/D'\n", + i + 1 + ), + ); + } + let err = run_config(dir.path(), false).await.unwrap_err().to_string(); + assert!(err.contains("limit 100"), "{err}"); +} + +fn spec_json(reference: &str) -> String { + serde_json::json!({ + "openapi": "3.0.3", + "info": {"title": "Remote", "version": "1"}, + "paths": {"/r": {"get": {"responses": {"200": {"description": "ok", + "content": {"application/json": {"schema": {"$ref": reference}}}}}}}} + }) + .to_string() +} + +fn http_config(url: String, headers: &[(&str, &str)]) -> Config { + Config { + sources: vec![Source::Http { + name: Some("remote".into()), + url, + headers: headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + tag_prefix: None, + additional_blocks: None, + }], + output: Default::default(), + merge: MergeConfig::default(), + } +} + +#[tokio::test] +async fn url_source_resolves_relative_refs_over_http() { + let mut server = mockito::Server::new_async().await; + let _spec = server + .mock("GET", "/api/spec.json") + .with_body(spec_json("defs.yaml#/Def")) + .create_async() + .await; + let _defs = server + .mock("GET", "/api/defs.yaml") + .with_body("Def:\n type: string\n") + .create_async() + .await; + + let config = http_config(format!("{}/api/spec.json", server.url()), &[]); + let report = aggregate_with_options(&config, &AggregateOptions::default()) + .await + .unwrap(); + assert_eq!( + report.spec["components"]["schemas"]["Def"]["type"], + "string" + ); +} + +#[tokio::test] +async fn headers_are_only_sent_to_the_same_origin() { + let mut a = mockito::Server::new_async().await; + let mut b = mockito::Server::new_async().await; + let reference = format!("{}/x.yaml#/Foreign", b.url()); + let _spec = a + .mock("GET", "/spec.json") + .match_header("Authorization", "Bearer t") + .with_body(spec_json("./local.yaml#/Local")) + .create_async() + .await; + let _spec2 = a + .mock("GET", "/spec2.json") + .match_header("Authorization", "Bearer t") + .with_body(spec_json(&reference)) + .create_async() + .await; + let _local = a + .mock("GET", "/local.yaml") + .match_header("Authorization", "Bearer t") + .with_body("Local:\n type: string\n") + .create_async() + .await; + let foreign = b + .mock("GET", "/x.yaml") + .match_header("Authorization", mockito::Matcher::Missing) + .with_body("Foreign:\n type: integer\n") + .create_async() + .await; + + let same = http_config( + format!("{}/spec.json", a.url()), + &[("Authorization", "Bearer t")], + ); + let report = aggregate_with_options(&same, &AggregateOptions::default()) + .await + .unwrap(); + assert_eq!( + report.spec["components"]["schemas"]["Local"]["type"], + "string" + ); + + let cross = http_config( + format!("{}/spec2.json", a.url()), + &[("Authorization", "Bearer t")], + ); + let report = aggregate_with_options(&cross, &AggregateOptions::default()) + .await + .unwrap(); + assert_eq!( + report.spec["components"]["schemas"]["Foreign"]["type"], + "integer" + ); + foreign.assert_async().await; +} + +#[tokio::test] +async fn url_source_cannot_read_local_files() { + let mut server = mockito::Server::new_async().await; + let _spec = server + .mock("GET", "/spec.json") + .with_body(spec_json("file:///etc/hostname#/x")) + .create_async() + .await; + let config = http_config(format!("{}/spec.json", server.url()), &[]); + let err = aggregate_with_options(&config, &AggregateOptions::default()) + .await + .unwrap_err() + .to_string(); + assert!(err.contains("file:"), "{err}"); +} diff --git a/web/src/app.ts b/web/src/app.ts index a2e7fdb..b2966a7 100644 --- a/web/src/app.ts +++ b/web/src/app.ts @@ -1,4 +1,4 @@ -import { aggregate, parseConfig } from './engine'; +import { aggregate, parseConfig, pendingRefs } from './engine'; import { DEFAULT_EXAMPLE_ID, EXAMPLES, findExample, type Example } from './examples'; import { latestOnly, runPipeline, type PipelineResult } from './pipeline'; import { errorMessage, fetchText, guardLoader, memoizeLoader } from './resolve'; @@ -77,7 +77,7 @@ export async function startApp(): Promise { () => remoteAllowed, ); let load = newLoader(); - const pipeline = latestOnly(() => runPipeline(state.config, state.files, { parseConfig, aggregate, load })); + const pipeline = latestOnly(() => runPipeline(state.config, state.files, { parseConfig, aggregate, pendingRefs, load })); const configEditor = createYamlEditor($('#config-editor'), '', (text) => { state.config = text; diff --git a/web/src/engine.ts b/web/src/engine.ts index 6f5cb6f..543e71b 100644 --- a/web/src/engine.ts +++ b/web/src/engine.ts @@ -2,8 +2,9 @@ import init, { aggregate as wasmAggregate, initSync, parseConfig as wasmParseConfig, + pendingRefs as wasmPendingRefs, } from './wasm/openapi_aggregator_wasm.js'; -import type { AggregateOutput, ConfigSummary } from './types'; +import type { AggregateOutput, ConfigSummary, PendingRefs } from './types'; export async function initEngine(): Promise { await init(); @@ -18,7 +19,18 @@ export function parseConfig(yaml: string): ConfigSummary { return wasmParseConfig(yaml) as ConfigSummary; } -/** Merge spec texts; `contents[i]` belongs to `parseConfig(yaml).sources[i]`. Throws on failure. */ -export function aggregate(yaml: string, contents: string[]): AggregateOutput { - return wasmAggregate(yaml, contents) as AggregateOutput; +/** + * Which external documents each source still needs. `docs` maps a location (as returned in a + * previous result) to its text. Throws on config or parse errors. + */ +export function pendingRefs(yaml: string, contents: string[], docs: Record): PendingRefs { + return wasmPendingRefs(yaml, contents, docs) as PendingRefs; +} + +/** + * Merge spec texts; `contents[i]` belongs to `parseConfig(yaml).sources[i]`. `docs` holds the + * external documents the sources reference (location -> text). Throws on failure. + */ +export function aggregate(yaml: string, contents: string[], docs: Record = {}): AggregateOutput { + return wasmAggregate(yaml, contents, docs) as AggregateOutput; } diff --git a/web/src/examples.ts b/web/src/examples.ts index 1f6e708..aa8cdd9 100644 --- a/web/src/examples.ts +++ b/web/src/examples.ts @@ -180,6 +180,39 @@ definitions: name: { type: string } `; +const MULTI_ROOT = `openapi: 3.0.3 +info: + title: Library API + version: 1.0.0 +paths: + /books: + get: + summary: List books + responses: + '200': + description: Books + content: + application/json: + schema: + type: array + items: + $ref: './schemas/book.yaml#/Book' +`; + +const MULTI_BOOK = `Book: + type: object + properties: + title: { type: string } + author: + $ref: './author.yaml#/Author' +`; + +const MULTI_AUTHOR = `Author: + type: object + properties: + name: { type: string } +`; + export const EXAMPLES: Example[] = [ { id: 'rename', @@ -293,6 +326,22 @@ sources: { name: 'specs/users.yaml', content: USERS }, ], }, + { + id: 'multi-file', + title: 'Multi-file spec ($ref to files)', + config: `# The root spec points at other files with $ref. Each external target is +# hoisted into components, and the refs are rewritten to match. +# Add the referenced files to the workspace under exactly the names the refs use. +sources: + - name: library + path: ./specs/library.yaml +`, + files: [ + { name: 'specs/library.yaml', content: MULTI_ROOT }, + { name: 'specs/schemas/book.yaml', content: MULTI_BOOK }, + { name: 'specs/schemas/author.yaml', content: MULTI_AUTHOR }, + ], + }, { id: 'remote', title: 'Remote URL + local file', diff --git a/web/src/pipeline.ts b/web/src/pipeline.ts index ec23d0d..98a81d8 100644 --- a/web/src/pipeline.ts +++ b/web/src/pipeline.ts @@ -1,5 +1,8 @@ -import { errorMessage, resolveSources, type Loader } from './resolve'; -import type { AggregateOutput, ConfigSummary, Problem, WorkspaceFile } from './types'; +import { errorMessage, resolveDocuments, resolveSources, type Loader, type WantedDocument } from './resolve'; +import type { AggregateOutput, ConfigSummary, PendingRefs, Problem, WorkspaceFile } from './types'; + +/** Mirrors the engine's limit on external documents per config. */ +const MAX_DOCUMENTS = 100; export interface PipelineResult { text: string | null; @@ -9,7 +12,8 @@ export interface PipelineResult { export interface PipelineDeps { parseConfig(yaml: string): ConfigSummary; - aggregate(yaml: string, contents: string[]): AggregateOutput; + pendingRefs(yaml: string, contents: string[], docs: Record): PendingRefs; + aggregate(yaml: string, contents: string[], docs: Record): AggregateOutput; load: Loader; } @@ -31,8 +35,24 @@ export async function runPipeline(config: string, files: WorkspaceFile[], deps: const { contents, problems } = await resolveSources(summary.sources, files, deps.load); if (!contents) return { text: null, format: summary.format, problems }; + const docs: Record = {}; try { - const out = deps.aggregate(config, contents); + for (;;) { + const wanted = collectWanted(deps.pendingRefs(config, contents, docs), summary); + if (!wanted.length) break; + if (Object.keys(docs).length + wanted.length > MAX_DOCUMENTS) { + throw new Error(`too many external documents (limit ${MAX_DOCUMENTS})`); + } + const loaded = await resolveDocuments(wanted, files, deps.load); + if (loaded.problems.length) return { text: null, format: summary.format, problems: loaded.problems }; + Object.assign(docs, loaded.docs); + } + } catch (e) { + return { text: null, format: summary.format, problems: [{ level: 'error', kind: 'merge', message: errorMessage(e) }] }; + } + + try { + const out = deps.aggregate(config, contents, docs); const warnings: Problem[] = out.warnings.map((message) => ({ level: 'warning', kind: 'merge', message })); return { text: out.text, format: summary.format, problems: [...problems, ...warnings] }; } catch (e) { @@ -44,6 +64,20 @@ export async function runPipeline(config: string, files: WorkspaceFile[], deps: } } +/** Flatten per-source pending documents, keeping the first source that needs each location. */ +function collectWanted(pending: PendingRefs, summary: ConfigSummary): WantedDocument[] { + const seen = new Set(); + const wanted: WantedDocument[] = []; + pending.sources.forEach((missing, index) => { + for (const doc of missing) { + if (seen.has(doc.location)) continue; + seen.add(doc.location); + wanted.push({ ...doc, source: summary.sources[index] }); + } + }); + return wanted; +} + /** Wrap an async function so only the most recent call's result is delivered. */ export function latestOnly(fn: (...args: A) => Promise): (...args: A) => Promise { let latest = 0; diff --git a/web/src/resolve.ts b/web/src/resolve.ts index 2de1ad1..eec3fce 100644 --- a/web/src/resolve.ts +++ b/web/src/resolve.ts @@ -1,5 +1,5 @@ import { workspaceKey } from './workspace'; -import type { Problem, ProblemKind, Settings, SourceSummary, WorkspaceFile } from './types'; +import type { MissingDocument, Problem, ProblemKind, Settings, SourceSummary, WorkspaceFile } from './types'; export type Loader = (url: string, headers: Record) => Promise; export type FetchFn = (input: string, init?: RequestInit) => Promise; @@ -111,3 +111,56 @@ export async function resolveSources( if (problems.length) return { contents: null, problems }; return { contents: results.map((r) => (r as { content: string }).content), problems }; } + +export interface WantedDocument extends MissingDocument { + /** The source whose spec (transitively) needs this document. */ + source: SourceSummary; +} + +function originOf(url: string): string | null { + const match = /^(https?:\/\/[^/?#]+)/i.exec(url); + return match ? match[1].toLowerCase() : null; +} + +/** + * Load external documents: workspace files by key, `http(s)` URLs through `load`. A source's + * headers are only sent to its own origin. Every failure becomes a problem (none are thrown). + */ +export async function resolveDocuments( + wanted: WantedDocument[], + files: WorkspaceFile[], + load: Loader, +): Promise<{ docs: Record; problems: Problem[] }> { + const results = await Promise.all( + wanted.map(async ({ location, referencedFrom, source }): Promise<{ location: string; text: string } | { problem: Problem }> => { + try { + if (/^https?:\/\//i.test(location)) { + const sameOrigin = source.kind === 'url' && originOf(source.url ?? '') === originOf(location); + return { location, text: await load(location, sameOrigin ? source.headers : {}) }; + } + if (/^file:/i.test(location)) { + throw new SourceError('missing-file', `'${location}' (referenced from '${referencedFrom}'): file: references are not supported.`); + } + const key = workspaceKey(location); + const file = files.find((f) => workspaceKey(f.name) === key); + if (!file) { + throw new SourceError( + 'missing-file', + `No workspace file named '${key}' (referenced from '${referencedFrom}'). Add it to the workspace with exactly that name.`, + ); + } + return { location, text: file.content }; + } catch (e) { + const kind = e instanceof SourceError ? e.kind : 'http'; + return { problem: { level: 'error', kind, source: source.name, message: errorMessage(e) } }; + } + }), + ); + const docs: Record = {}; + const problems: Problem[] = []; + for (const r of results) { + if ('problem' in r) problems.push(r.problem); + else docs[r.location] = r.text; + } + return { docs, problems }; +} diff --git a/web/src/types.ts b/web/src/types.ts index 46c28bd..a7c2391 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -21,6 +21,16 @@ export interface ConfigSummary { format: 'yaml' | 'json'; } +export interface MissingDocument { + location: string; + referencedFrom: string; +} + +/** For each source (same order as the config), the external documents it still needs. */ +export interface PendingRefs { + sources: MissingDocument[][]; +} + export interface AggregateOutput { text: string; warnings: string[]; diff --git a/web/tests/engine.test.ts b/web/tests/engine.test.ts index bf13a02..93e5995 100644 --- a/web/tests/engine.test.ts +++ b/web/tests/engine.test.ts @@ -1,5 +1,5 @@ import { beforeAll, describe, expect, it } from 'vitest'; -import { aggregate, parseConfig } from '../src/engine'; +import { aggregate, parseConfig, pendingRefs } from '../src/engine'; import { loadEngineForTests } from './helpers'; const PETS = "openapi: 3.0.3\ninfo: {title: Pets, version: '1'}\npaths:\n /pets:\n get: {summary: list}\n"; @@ -24,4 +24,12 @@ describe('engine', () => { expect(out.text).toContain('/pets:'); expect(out.warnings).toEqual([]); }); + + it('lists the external documents a source still needs, then bundles them', () => { + const root = "openapi: 3.0.3\ninfo: {title: R, version: '1'}\npaths: {}\ncomponents:\n schemas:\n A: {$ref: './a.yaml#/A'}\n"; + const config = 'sources:\n - path: ./root.yaml\n'; + expect(pendingRefs(config, [root], {})).toEqual({ sources: [[{ location: 'a.yaml', referencedFrom: 'root.yaml' }]] }); + expect(pendingRefs(config, [root], { 'a.yaml': 'A:\n type: string\n' })).toEqual({ sources: [[]] }); + expect(aggregate(config, [root], { 'a.yaml': 'A:\n type: string\n' }).text).toContain('type: string'); + }); }); diff --git a/web/tests/examples.test.ts b/web/tests/examples.test.ts index d22dff4..f25011e 100644 --- a/web/tests/examples.test.ts +++ b/web/tests/examples.test.ts @@ -1,5 +1,6 @@ import { beforeAll, describe, expect, it } from 'vitest'; -import { aggregate, parseConfig } from '../src/engine'; +import { aggregate, parseConfig, pendingRefs } from '../src/engine'; +import { runPipeline } from '../src/pipeline'; import { DEFAULT_EXAMPLE_ID, EXAMPLES, findExample } from '../src/examples'; import { resolveSources } from '../src/resolve'; import { loadEngineForTests } from './helpers'; @@ -27,12 +28,9 @@ describe('examples', () => { const usesNetwork = parseConfigSafe(example.config).some((s) => s.kind === 'url'); const expectsWarnings = example.id === 'swagger2'; it.skipIf(usesNetwork || expectsWarnings)(`${example.id}: merges without errors or warnings`, async () => { - const summary = parseConfig(example.config); - const { contents, problems } = await resolveSources(summary.sources, example.files, noNetwork); - expect(problems).toEqual([]); - const out = aggregate(example.config, contents!); - expect(out.warnings).toEqual([]); - expect(out.text.length).toBeGreaterThan(0); + const result = await runPipeline(example.config, example.files, { parseConfig, aggregate, pendingRefs, load: noNetwork }); + expect(result.problems).toEqual([]); + expect(result.text!.length).toBeGreaterThan(0); }); } @@ -56,6 +54,16 @@ describe('examples', () => { expect(out.warnings.some((w) => w.includes('converted from Swagger 2.0'))).toBe(true); }); + it('multi-file example bundles refs from workspace files', async () => { + const ex = findExample('multi-file'); + expect(ex.id).toBe('multi-file'); + const result = await runPipeline(ex.config, ex.files, { parseConfig, aggregate, pendingRefs, load: noNetwork }); + expect(result.problems).toEqual([]); + expect(result.text).toContain('components:'); + expect(result.text).toMatch(/\$ref: '#\/components\/schemas\/\w+'/); + expect(result.text).not.toMatch(/\.yaml#/); + }); + it('per-operation example combines GET and POST on one path', async () => { const ex = findExample('per-operation'); const { contents } = await resolveSources(parseConfig(ex.config).sources, ex.files, noNetwork); diff --git a/web/tests/pipeline.test.ts b/web/tests/pipeline.test.ts index a973af2..b12bf0a 100644 --- a/web/tests/pipeline.test.ts +++ b/web/tests/pipeline.test.ts @@ -1,11 +1,12 @@ import { beforeAll, describe, expect, it } from 'vitest'; -import { aggregate, parseConfig } from '../src/engine'; +import { aggregate, parseConfig, pendingRefs } from '../src/engine'; +import { SourceError } from '../src/resolve'; import { latestOnly, lineOf, runPipeline, type PipelineDeps } from '../src/pipeline'; import { loadEngineForTests } from './helpers'; beforeAll(loadEngineForTests); -const deps = (load: PipelineDeps['load'] = async () => 'unused'): PipelineDeps => ({ parseConfig, aggregate, load }); +const deps = (load: PipelineDeps['load'] = async () => 'unused'): PipelineDeps => ({ parseConfig, aggregate, pendingRefs, load }); const spec = (title: string, paths = ' /a:\n get: {summary: a}\n') => `openapi: 3.0.3\ninfo: {title: ${title}, version: '1'}\npaths:\n${paths}`; @@ -91,3 +92,69 @@ describe('latestOnly', () => { expect(await first).toBeUndefined(); }); }); + +describe('external refs', () => { + const root = (ref: string) => + `openapi: 3.0.3\ninfo: {title: Root, version: '1'}\npaths:\n /p:\n get:\n responses:\n '200':\n description: ok\n content:\n application/json:\n schema:\n $ref: '${ref}'\n`; + const config = 'sources:\n - name: root\n path: specs/root.yaml\n'; + + it('bundles refs resolved from workspace files, over several rounds', async () => { + const result = await runPipeline( + config, + [ + { name: 'specs/root.yaml', content: root('./schemas/pet.yaml#/Pet') }, + { name: 'specs/schemas/pet.yaml', content: "Pet:\n type: object\n properties:\n owner:\n $ref: './owner.yaml#/Owner'\n" }, + { name: 'specs/schemas/owner.yaml', content: 'Owner:\n type: string\n' }, + ], + deps(), + ); + expect(result.problems).toEqual([]); + expect(result.text).toContain('Owner:'); + expect(result.text).toContain("'#/components/schemas/Owner'"); + }); + + it('names the missing workspace file and who referenced it', async () => { + const result = await runPipeline(config, [{ name: 'specs/root.yaml', content: root('./schemas/pet.yaml#/Pet') }], deps()); + expect(result.text).toBeNull(); + expect(result.problems).toEqual([ + expect.objectContaining({ + level: 'error', + kind: 'missing-file', + source: 'root', + message: + "No workspace file named 'specs/schemas/pet.yaml' (referenced from 'specs/root.yaml'). Add it to the workspace with exactly that name.", + }), + ]); + }); + + it('reports a blocked remote ref instead of fetching', async () => { + const result = await runPipeline( + config, + [{ name: 'specs/root.yaml', content: root('https://schemas.test/pet.yaml#/Pet') }], + deps(async () => { + throw new SourceError('blocked', 'Not fetched: https://schemas.test/pet.yaml.'); + }), + ); + expect(result.text).toBeNull(); + expect(result.problems[0]).toMatchObject({ level: 'error', kind: 'blocked' }); + }); + + it('sends source headers only to the same origin', async () => { + const urlRoot = + "openapi: 3.0.3\ninfo: {title: R, version: '1'}\npaths: {}\ncomponents:\n schemas:\n A: {$ref: './local.yaml#/Local'}\n B: {$ref: 'https://b.test/x.yaml#/Foreign'}\n"; + const calls: Array<[string, Record]> = []; + const load = async (url: string, headers: Record) => { + calls.push([url, headers]); + if (url.endsWith('/spec.yaml')) return urlRoot; + return url.includes('local') ? 'Local:\n type: string\n' : 'Foreign:\n type: integer\n'; + }; + const result = await runPipeline( + 'sources:\n - name: r\n url: https://a.test/api/spec.yaml\n headers: {Authorization: Bearer t}\n', + [], + deps(load), + ); + expect(result.problems).toEqual([]); + expect(calls.find(([u]) => u === 'https://a.test/api/local.yaml')?.[1]).toEqual({ Authorization: 'Bearer t' }); + expect(calls.find(([u]) => u === 'https://b.test/x.yaml')?.[1]).toEqual({}); + }); +}); diff --git a/web/tests/resolve.test.ts b/web/tests/resolve.test.ts index 7a66c28..3ff69c9 100644 --- a/web/tests/resolve.test.ts +++ b/web/tests/resolve.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { fetchText, guardLoader, memoizeLoader, resolveSources, substituteVars } from '../src/resolve'; +import { fetchText, guardLoader, memoizeLoader, resolveDocuments, resolveSources, substituteVars } from '../src/resolve'; import type { Settings, SourceSummary } from '../src/types'; const settings = (over: Partial = {}): Settings => ({ proxy: '', variables: {}, ...over }); @@ -119,3 +119,54 @@ describe('guardLoader', () => { expect(await load('https://x.test/a', {})).toBe('spec'); }); }); + +describe('resolveDocuments', () => { + const urlRoot: SourceSummary = { name: 'r', kind: 'url', path: null, url: 'https://a.test/spec.yaml', headers: { Authorization: 'Bearer t' } }; + const file = (name: string, content = 'X: {}') => ({ name, content }); + + it('reads workspace files by normalised key', async () => { + const { docs, problems } = await resolveDocuments( + [{ location: 'specs/a.yaml', referencedFrom: 'root.yaml', source: fileSource('root.yaml') }], + [file('./specs/a.yaml', 'A: {}')], + async () => 'unused', + ); + expect(problems).toEqual([]); + expect(docs).toEqual({ 'specs/a.yaml': 'A: {}' }); + }); + + it('reports every missing file, not just the first', async () => { + const { problems } = await resolveDocuments( + [ + { location: 'a.yaml', referencedFrom: 'root.yaml', source: fileSource('root.yaml') }, + { location: 'b.yaml', referencedFrom: 'a.yaml', source: fileSource('root.yaml') }, + ], + [], + async () => 'unused', + ); + expect(problems.map((p) => p.kind)).toEqual(['missing-file', 'missing-file']); + expect(problems[1].message).toContain("'b.yaml' (referenced from 'a.yaml')"); + }); + + it('sends headers only to the source origin and rejects file: refs', async () => { + const seen: Array<[string, Record]> = []; + const load = async (url: string, headers: Record) => { + seen.push([url, headers]); + return 'X: {}'; + }; + const { problems } = await resolveDocuments( + [ + { location: 'https://a.test/d.yaml', referencedFrom: 'https://a.test/spec.yaml', source: urlRoot }, + { location: 'https://b.test/d.yaml', referencedFrom: 'https://a.test/spec.yaml', source: urlRoot }, + { location: 'file:///etc/passwd', referencedFrom: 'https://a.test/spec.yaml', source: urlRoot }, + ], + [], + load, + ); + expect(seen).toEqual([ + ['https://a.test/d.yaml', { Authorization: 'Bearer t' }], + ['https://b.test/d.yaml', {}], + ]); + expect(problems).toHaveLength(1); + expect(problems[0].message).toContain('file:'); + }); +});