diff --git a/Cargo.lock b/Cargo.lock index 2d526e4719..e703ad6948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,6 +187,12 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.8" @@ -397,6 +403,19 @@ dependencies = [ "walkdir", ] +[[package]] +name = "by_resource" +version = "0.0.0" +dependencies = [ + "ruff_python_stdlib", + "rustc-hash", + "saphyr", + "saphyr-parser", + "serde", + "serde_json", + "toml_edit", +] + [[package]] name = "by_rt" version = "0.0.0" @@ -423,6 +442,7 @@ dependencies = [ name = "by_transforms" version = "0.0.0" dependencies = [ + "by_resource", "indexmap", "indoc", "ruff_db", @@ -683,9 +703,9 @@ checksum = "d669bb552908e336ad5681789752033b45566b7e591aeaac7a614e58e5d6d8f2" dependencies = [ "nix", "terminfo", - "thiserror 2.0.19", + "thiserror 2.0.20", "which", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -804,7 +824,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -1100,7 +1120,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1154,7 +1174,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -1221,6 +1241,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1234,7 +1263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -2060,7 +2089,7 @@ dependencies = [ "paste", "peg", "regex", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2231,7 +2260,7 @@ dependencies = [ "serde", "similar 3.1.2", "smallvec", - "thiserror 2.0.19", + "thiserror 2.0.20", "toml 1.1.4+spec-1.1.0", "toml_parser", "tracing", @@ -2443,6 +2472,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0" +dependencies = [ + "num-traits", +] + [[package]] name = "ordermap" version = "1.2.0" @@ -2614,7 +2652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21e0a3a33733faeaf8651dfee72dd0f388f0c8e5ad496a3478fa5a922f49cfa8" dependencies = [ "memchr", - "thiserror 2.0.19", + "thiserror 2.0.20", "ucd-trie", ] @@ -2881,7 +2919,7 @@ dependencies = [ "pep440_rs", "pep508_rs", "serde", - "thiserror 2.0.19", + "thiserror 2.0.20", "toml 0.9.12+spec-1.1.0", ] @@ -2896,7 +2934,7 @@ dependencies = [ "newtype-uuid", "quick-xml", "strip-ansi-escapes", - "thiserror 2.0.19", + "thiserror 2.0.20", "uuid", ] @@ -3071,7 +3109,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -3238,7 +3276,7 @@ dependencies = [ "strum", "tempfile", "test-case", - "thiserror 2.0.19", + "thiserror 2.0.20", "tikv-jemallocator", "toml 1.1.4+spec-1.1.0", "tracing", @@ -3304,6 +3342,7 @@ version = "0.0.8" dependencies = [ "anstyle", "arc-swap", + "by_resource", "camino", "compact_str", "dashmap", @@ -3336,7 +3375,7 @@ dependencies = [ "similar 3.1.2", "supports-hyperlinks", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "tracing", "tracing-subscriber", "ty_static", @@ -3500,7 +3539,7 @@ dependencies = [ "strum_macros", "tempfile", "test-case", - "thiserror 2.0.19", + "thiserror 2.0.20", "toml 1.1.4+spec-1.1.0", "typed-arena", "unicode-normalization", @@ -3572,7 +3611,7 @@ dependencies = [ "serde", "serde_json", "test-case", - "thiserror 2.0.19", + "thiserror 2.0.20", "uuid", ] @@ -3606,7 +3645,7 @@ dependencies = [ "serde", "serde_json", "thin-vec", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -3661,7 +3700,7 @@ dependencies = [ "similar 3.1.2", "smallvec", "static_assertions", - "thiserror 2.0.19", + "thiserror 2.0.20", "tracing", ] @@ -3828,7 +3867,7 @@ dependencies = [ "shellexpand", "smallvec", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "toml 1.1.4+spec-1.1.0", "tracing", "tracing-log", @@ -3958,7 +3997,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -4026,6 +4065,29 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saphyr" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372edd44fb1d21f97d7160ca1fb3ed522b7f7315b2369cd10e5127379db5af33" +dependencies = [ + "encoding_rs", + "hashlink", + "ordered-float", + "saphyr-parser", + "thiserror 2.0.20", +] + +[[package]] +name = "saphyr-parser" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b1b76852c5643f558e7347e4de549474c3ecb7154295a189f4668428c1b453" +dependencies = [ + "arraydeque", + "thiserror 2.0.20", +] + [[package]] name = "schemars" version = "1.2.2" @@ -4368,7 +4430,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -4378,7 +4440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -4452,11 +4514,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -4472,9 +4534,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -4867,7 +4929,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "toml 1.1.4+spec-1.1.0", "tracing", "ty_vendored", @@ -4911,7 +4973,7 @@ dependencies = [ "shellexpand", "strum", "strum_macros", - "thiserror 2.0.19", + "thiserror 2.0.20", "toml 1.1.4+spec-1.1.0", "tracing", "ty_combine", @@ -4962,6 +5024,7 @@ version = "0.0.8" dependencies = [ "anyhow", "bitflags 2.13.1", + "by_resource", "camino", "char_str", "compact_str", @@ -5000,7 +5063,7 @@ dependencies = [ "strum", "strum_macros", "test-case", - "thiserror 2.0.19", + "thiserror 2.0.20", "tracing", "ty_combine", "ty_module_resolver", @@ -5045,7 +5108,7 @@ dependencies = [ "smallvec", "strum", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "tracing", "tracing-subscriber", "ty_combine", @@ -5577,7 +5640,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a86e0670eb..6a37f50608 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ by_codegen_c = { path = "crates/by_codegen_c" } by_ir = { path = "crates/by_ir" } by_irbuild = { path = "crates/by_irbuild" } by_opt = { path = "crates/by_opt" } +by_resource = { path = "crates/by_resource" } by_rt = { path = "crates/by_rt" } by_stage = { path = "crates/by_stage" } by_transforms = { path = "crates/by_transforms" } @@ -179,6 +180,8 @@ salsa = { version = "0.28.2", default-features = false, features = [ ] } same-file = { version = "1.0.6" } schemars = { version = "1.0.4" } +saphyr = { version = "0.0.12" } +saphyr-parser = { version = "0.0.12" } seahash = { version = "4.1.0" } serde = { version = "1.0.197", features = ["derive"] } serde-wasm-bindgen = { version = "0.6.4" } diff --git a/crates/by_resource/Cargo.toml b/crates/by_resource/Cargo.toml new file mode 100644 index 0000000000..6a15c663df --- /dev/null +++ b/crates/by_resource/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "by_resource" +version = "0.0.0" +publish = false +authors = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +license = "MIT" + +[dependencies] +ruff_python_stdlib = { workspace = true } + +rustc-hash = { workspace = true } +saphyr = { workspace = true } +saphyr-parser = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +toml_edit = { workspace = true, features = ["parse"] } + +[lints] +workspace = true diff --git a/crates/by_resource/src/lib.rs b/crates/by_resource/src/lib.rs new file mode 100644 index 0000000000..08a5cdc201 --- /dev/null +++ b/crates/by_resource/src/lib.rs @@ -0,0 +1,26 @@ +//! importing a json, toml or yaml file as a value with typed dot access. +//! +//! `import "data/config.yaml" as config` says the file is a *static* resource: +//! part of the program, fixed at build time, read through named attributes +//! rather than opened at runtime. this crate is what that claim rests on. it +//! reads the document ([`parse`]) and renders it as python ([`render`]) — a +//! mapping as a class, a sequence as a tuple, a scalar as a `Final` literal. +//! +//! both halves of the language read a resource through this one rendering: the +//! type checker infers the rendered module to answer what `config.a.b[1]` is, +//! and the transpiler writes the same rendering into the python it emits. there +//! is no second description of what a document means, so the type and the object +//! cannot disagree. + +mod parse; +mod render; +mod value; + +pub use parse::{Format, ParseError, parse}; +pub use render::{REQUIRED_IMPORT, Rendered, binding_name, render}; +pub use value::Value; + +/// read `text` as `format` and render it as python bound to `root`. +pub fn transpile(format: Format, text: &str, root: &str) -> Result { + Ok(render(&parse(format, text)?, root)) +} diff --git a/crates/by_resource/src/parse.rs b/crates/by_resource/src/parse.rs new file mode 100644 index 0000000000..7f80d8c9a5 --- /dev/null +++ b/crates/by_resource/src/parse.rs @@ -0,0 +1,676 @@ +//! reading json, toml and yaml into [`Value`]. + +use std::fmt; + +use rustc_hash::FxHashMap; + +use crate::value::Value; + +/// a data format a static resource can be written in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Format { + Json, + Toml, + Yaml, +} + +impl Format { + /// the format a file extension names, or `None` when the extension is not + /// one a static resource can be written in. + pub fn from_extension(extension: &str) -> Option { + match extension { + "json" => Some(Format::Json), + "toml" => Some(Format::Toml), + "yaml" | "yml" => Some(Format::Yaml), + _ => None, + } + } + + /// every extension a static resource can have, for listing in a diagnostic. + pub const EXTENSIONS: &'static [&'static str] = &["json", "toml", "yaml", "yml"]; + + /// what to call the format in a message. + pub fn name(self) -> &'static str { + match self { + Format::Json => "json", + Format::Toml => "toml", + Format::Yaml => "yaml", + } + } +} + +/// why a document could not be read as a static resource. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseError { + message: String, +} + +impl ParseError { + fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +/// the largest document that will be turned into python. +/// +/// the generated source is parsed and type checked like any other module, so a +/// resource is bounded the way a hand-written module is. the limit is far above +/// any configuration file and far below anything that would make the type +/// checker crawl. +const MAX_NODES: usize = 100_000; + +/// how deeply a document may nest. +/// +/// each level of mapping becomes a level of nested `class`, and python's parser +/// gives up long before this. no configuration file comes close. +const MAX_DEPTH: usize = 32; + +/// what is left of a document's budget. +/// +/// yaml is read against this as the tree is built rather than after, for two +/// reasons that both end the process rather than the parse. the walk is +/// recursive, so a document nested a hundred thousand deep overflows the stack +/// before anything can measure it. and an anchor is expanded wherever it is +/// used, so a two-hundred byte file can name more values than there is memory +/// to hold — the size has to be counted while the copies are being made. +/// +/// json and toml need no such thing here: their own parsers refuse to recurse +/// past a limit of their own (128 levels and a comparable one), which bounds +/// the tree they hand over, and neither format can name a value twice. +struct Budget { + nodes: usize, + depth: usize, +} + +impl Budget { + fn new() -> Self { + Self { + nodes: MAX_NODES, + depth: MAX_DEPTH, + } + } + + /// charge one value, and enter it. + fn enter(&mut self) -> Result<(), ParseError> { + self.nodes = self + .nodes + .checked_sub(1) + .ok_or_else(|| ParseError::new(too_large()))?; + self.depth = self + .depth + .checked_sub(1) + .ok_or_else(|| ParseError::new(too_deep()))?; + Ok(()) + } + + fn leave(&mut self) { + self.depth += 1; + } + + /// charge `nodes` values without entering them. + fn charge(&mut self, nodes: usize) -> Result<(), ParseError> { + self.nodes = self + .nodes + .checked_sub(nodes) + .ok_or_else(|| ParseError::new(too_large()))?; + Ok(()) + } +} + +fn too_large() -> String { + format!( + "the document holds more than {MAX_NODES} values, which is too large to import as a static resource" + ) +} + +fn too_deep() -> String { + format!( + "the document nests more than {MAX_DEPTH} levels deep, which is too deep to import as a static resource" + ) +} + +/// read `text` as `format`. +pub fn parse(format: Format, text: &str) -> Result { + let value = match format { + Format::Json => json(text), + Format::Toml => toml(text), + Format::Yaml => yaml(text), + }?; + + if value.size() > MAX_NODES { + return Err(ParseError::new(too_large())); + } + if value.depth() > MAX_DEPTH { + return Err(ParseError::new(too_deep())); + } + + Ok(value) +} + +/// a mapping being read, in document order and without duplicates. +/// +/// json and yaml both allow a document to name a key twice and both read the +/// last one, so the last one is what the reader of the generated python should +/// find too — under one name, because two `Final` declarations of the same +/// attribute is not something python would accept. +/// +/// the earlier entry is emptied rather than removed, and the empties are dropped +/// at the end. scanning the entries for each key instead would be quadratic, and +/// a document is allowed a hundred thousand of them. +#[derive(Default)] +struct Entries { + entries: Vec>, + positions: FxHashMap, +} + +impl Entries { + fn with_capacity(capacity: usize) -> Self { + Self { + entries: Vec::with_capacity(capacity), + positions: FxHashMap::default(), + } + } + + fn insert(&mut self, key: String, value: Value) { + if let Some(previous) = self.positions.insert(key.clone(), self.entries.len()) { + self.entries[previous] = None; + } + self.entries.push(Some((key, value))); + } + + fn into_value(self) -> Value { + Value::Map(self.entries.into_iter().flatten().collect()) + } +} + +/// json, read through serde rather than through `serde_json::Value`. +/// +/// `serde_json::Value` holds its object in a `BTreeMap`, which would hand the +/// keys over sorted; a visitor sees them in the order the document lists them. +fn json(text: &str) -> Result { + serde_json::from_str::(text) + .map(|document| document.0) + .map_err(|error| ParseError::new(error.to_string())) +} + +/// a document being read by serde. +struct Document(Value); + +impl<'de> serde::Deserialize<'de> for Document { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(DocumentVisitor).map(Document) + } +} + +struct DocumentVisitor; + +impl<'de> serde::de::Visitor<'de> for DocumentVisitor { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a json value") + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Value::Int(value)) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(i64::try_from(value).map_or_else( + // python's integers have no upper bound, so the digits go through + // as they were written rather than through an `f64` + |_| Value::BigInt(value.to_string()), + Value::Int, + )) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Value::Float(value)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Value::Str(value.to_string())) + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut items = Vec::new(); + while let Some(Document(item)) = access.next_element()? { + items.push(item); + } + Ok(Value::Seq(items)) + } + + fn visit_map>(self, mut access: A) -> Result { + let mut entries = Entries::default(); + while let Some((key, Document(value))) = access.next_entry::()? { + entries.insert(key, value); + } + Ok(entries.into_value()) + } +} + +/// toml, read through `toml_edit` rather than through `toml`. +/// +/// `toml::Value` holds its tables sorted, and `toml_edit` keeps them in document +/// order — the same reason json is read through a visitor. +fn toml(text: &str) -> Result { + let document = + toml_edit::Document::parse(text).map_err(|error| ParseError::new(error.to_string()))?; + Ok(from_toml_table(document.as_table())) +} + +fn from_toml_table(table: &toml_edit::Table) -> Value { + let mut entries = Entries::with_capacity(table.len()); + for (key, item) in table { + entries.insert(key.to_string(), from_toml_item(item)); + } + entries.into_value() +} + +fn from_toml_item(item: &toml_edit::Item) -> Value { + match item { + toml_edit::Item::Value(value) => from_toml_value(value), + toml_edit::Item::Table(table) => from_toml_table(table), + toml_edit::Item::ArrayOfTables(tables) => { + Value::Seq(tables.iter().map(from_toml_table).collect()) + } + // a key that was written without a value; toml's parser rejects that, so + // nothing reaching here has one + toml_edit::Item::None => Value::Null, + } +} + +fn from_toml_value(value: &toml_edit::Value) -> Value { + match value { + toml_edit::Value::String(value) => Value::Str(value.value().clone()), + toml_edit::Value::Integer(value) => Value::Int(*value.value()), + toml_edit::Value::Float(value) => Value::Float(*value.value()), + toml_edit::Value::Boolean(value) => Value::Bool(*value.value()), + // toml is the only one of the three formats with a date type, and python + // has no literal for one. the text the document holds is what survives, + // which is at least exactly what was written + toml_edit::Value::Datetime(value) => Value::Str(value.value().to_string()), + toml_edit::Value::Array(items) => Value::Seq(items.iter().map(from_toml_value).collect()), + toml_edit::Value::InlineTable(table) => { + let mut entries = Entries::with_capacity(table.len()); + for (key, value) in table { + entries.insert(key.to_string(), from_toml_value(value)); + } + entries.into_value() + } + } +} + +/// yaml, read from the parser's event stream rather than from a loaded tree. +/// +/// the tree a loader hands back is built by recursing once per level, and a +/// document can nest deeper than there is stack to do that in — the load +/// overflows before anything is in a position to measure it. reading the events +/// with a stack of our own has no such limit, and it is also where an anchor is +/// expanded, which is the only way to charge the copies against the budget +/// before they are made. +fn yaml(text: &str) -> Result { + let mut reader = YamlReader { + budget: Budget::new(), + frames: Vec::new(), + anchors: FxHashMap::default(), + document: None, + documents: 0, + }; + + for event in saphyr_parser::Parser::new_from_str(text) { + let (event, _span) = event.map_err(|error| ParseError::new(error.to_string()))?; + reader.read(event)?; + } + + // an empty file is a document holding nothing + Ok(reader.document.unwrap_or(Value::Null)) +} + +/// a collection the reader is in the middle of. +enum Frame { + Sequence { + anchor: usize, + items: Vec, + }, + Mapping { + anchor: usize, + entries: Entries, + /// the key this mapping is waiting for a value for + key: Option, + }, +} + +struct YamlReader { + budget: Budget, + frames: Vec, + /// what each anchor in the document was defined as, to copy at an alias + anchors: FxHashMap, + document: Option, + documents: usize, +} + +impl YamlReader { + fn read(&mut self, event: saphyr_parser::Event<'_>) -> Result<(), ParseError> { + use saphyr_parser::Event; + + match event { + Event::DocumentStart(_) => { + self.documents += 1; + if self.documents > 1 { + return Err(ParseError::new( + "the file holds more than one yaml document, and a static resource is a single value", + )); + } + } + Event::Scalar(text, style, anchor, tag) => { + self.budget.enter()?; + self.budget.leave(); + let scalar = saphyr::Scalar::parse_from_cow_and_metadata(text, style, tag.as_ref()) + .ok_or_else(|| { + ParseError::new("the document holds a value yaml could not read") + })?; + self.place(scalar_value(&scalar), anchor)?; + } + Event::SequenceStart(anchor, _) => { + self.budget.enter()?; + self.frames.push(Frame::Sequence { + anchor, + items: Vec::new(), + }); + } + Event::MappingStart(anchor, _) => { + self.budget.enter()?; + self.frames.push(Frame::Mapping { + anchor, + entries: Entries::default(), + key: None, + }); + } + Event::SequenceEnd | Event::MappingEnd => { + self.budget.leave(); + let (value, anchor) = match self.frames.pop() { + Some(Frame::Sequence { anchor, items }) => (Value::Seq(items), anchor), + Some(Frame::Mapping { + anchor, entries, .. + }) => (entries.into_value(), anchor), + // the parser pairs every end with a start + None => return Ok(()), + }; + self.place(value, anchor)?; + } + Event::Alias(anchor) => { + let Some(value) = self.anchors.get(&anchor) else { + return Err(ParseError::new( + "the document uses a yaml alias for an anchor it never defined", + )); + }; + // an alias is a copy, so the copy is what the budget is charged + // for — this is the only thing standing between a handful of + // anchors and a document that names more values than there is + // memory to hold + self.budget.charge(value.size())?; + let value = value.clone(); + self.place(value, 0)?; + } + _ => {} + } + + Ok(()) + } + + /// record `value` under `anchor` if it has one, and hand it to whatever is + /// waiting for it. + fn place(&mut self, value: Value, anchor: usize) -> Result<(), ParseError> { + // anchor 0 is the parser's way of saying there is no anchor + if anchor != 0 { + self.anchors.insert(anchor, value.clone()); + } + + match self.frames.last_mut() { + Some(Frame::Sequence { items, .. }) => items.push(value), + Some(Frame::Mapping { entries, key, .. }) => match key.take() { + Some(name) => entries.insert(name, value), + None => match value { + Value::Str(name) => *key = Some(name), + // a mapping key that is not a string cannot name an + // attribute, and pretending otherwise would silently drop it + _ => { + return Err(ParseError::new( + "a mapping key is not a string, and a static resource is read through its keys", + )); + } + }, + }, + None => self.document = Some(value), + } + + Ok(()) + } +} + +fn scalar_value(scalar: &saphyr::Scalar<'_>) -> Value { + use saphyr::Scalar; + + match scalar { + Scalar::Null => Value::Null, + Scalar::Boolean(value) => Value::Bool(*value), + Scalar::Integer(value) => Value::Int(*value), + Scalar::FloatingPoint(value) => Value::Float(value.into_inner()), + Scalar::String(value) => Value::Str(value.to_string()), + } +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use super::*; + + fn map(entries: [(&str, Value); 1]) -> Value { + Value::Map( + entries + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(), + ) + } + + #[test] + fn json_document() { + let value = parse(Format::Json, r#"{"a": {"b": [1, 2]}}"#).unwrap(); + assert_eq!( + value, + map([( + "a", + map([("b", Value::Seq(vec![Value::Int(1), Value::Int(2)]))]) + )]) + ); + } + + #[test] + fn toml_document() { + let value = parse(Format::Toml, "[a]\nb = [1, 2]\n").unwrap(); + assert_eq!( + value, + map([( + "a", + map([("b", Value::Seq(vec![Value::Int(1), Value::Int(2)]))]) + )]) + ); + } + + #[test] + fn yaml_document() { + let value = parse(Format::Yaml, "a:\n b:\n - 1\n - 2\n").unwrap(); + assert_eq!( + value, + map([( + "a", + map([("b", Value::Seq(vec![Value::Int(1), Value::Int(2)]))]) + )]) + ); + } + + #[test] + fn yaml_scalars() { + let value = parse(Format::Yaml, "a: ~\nb: true\nc: 1.5\nd: text\n").unwrap(); + assert_eq!( + value, + Value::Map(vec![ + ("a".to_string(), Value::Null), + ("b".to_string(), Value::Bool(true)), + ("c".to_string(), Value::Float(1.5)), + ("d".to_string(), Value::Str("text".to_string())), + ]) + ); + } + + #[test] + fn empty_yaml_is_null() { + assert_eq!(parse(Format::Yaml, "").unwrap(), Value::Null); + } + + #[test] + fn a_second_yaml_document_is_rejected() { + let error = parse(Format::Yaml, "a: 1\n---\nb: 2\n").unwrap_err(); + assert!(error.to_string().contains("more than one yaml document")); + } + + #[test] + fn a_yaml_anchor_is_expanded() { + let value = parse(Format::Yaml, "a: &anchor 1\nb: *anchor\n").unwrap(); + assert_eq!( + value, + Value::Map(vec![ + ("a".to_string(), Value::Int(1)), + ("b".to_string(), Value::Int(1)), + ]) + ); + } + + #[test] + fn the_last_of_two_keys_with_one_name_wins() { + let value = parse(Format::Json, r#"{"a": 1, "b": 2, "a": 3}"#).unwrap(); + assert_eq!( + value, + Value::Map(vec![ + ("b".to_string(), Value::Int(2)), + ("a".to_string(), Value::Int(3)), + ]) + ); + } + + #[test] + fn keys_keep_the_order_the_document_lists_them_in() { + let value = parse(Format::Json, r#"{"b": 1, "a": 2}"#).unwrap(); + assert_eq!( + value, + Value::Map(vec![ + ("b".to_string(), Value::Int(1)), + ("a".to_string(), Value::Int(2)), + ]) + ); + + let value = parse(Format::Toml, "b = 1\na = 2\n").unwrap(); + assert_eq!( + value, + Value::Map(vec![ + ("b".to_string(), Value::Int(1)), + ("a".to_string(), Value::Int(2)), + ]) + ); + } + + #[test] + fn a_toml_array_of_tables() { + let value = parse(Format::Toml, "[[a]]\nb = 1\n\n[[a]]\nb = 2\n").unwrap(); + assert_eq!( + value, + map([( + "a", + Value::Seq(vec![ + Value::Map(vec![("b".to_string(), Value::Int(1))]), + Value::Map(vec![("b".to_string(), Value::Int(2))]), + ]) + )]) + ); + } + + #[test] + fn a_non_string_yaml_key_is_rejected() { + let error = parse(Format::Yaml, "1: one\n").unwrap_err(); + assert!(error.to_string().contains("not a string")); + } + + #[test] + fn json_integers_larger_than_i64() { + let value = parse(Format::Json, r#"{"a": 18446744073709551615}"#).unwrap(); + assert_eq!( + value, + map([("a", Value::BigInt("18446744073709551615".to_string()))]) + ); + } + + #[test] + fn a_toml_datetime_keeps_its_text() { + let value = parse(Format::Toml, "a = 1979-05-27\n").unwrap(); + assert_eq!(value, map([("a", Value::Str("1979-05-27".to_string()))])); + } + + #[test] + fn a_document_that_nests_too_deeply_is_rejected() { + let text = "[".repeat(MAX_DEPTH + 1) + &"]".repeat(MAX_DEPTH + 1); + let error = parse(Format::Json, &text).unwrap_err(); + assert!(error.to_string().contains("too deep")); + } + + /// the depth is counted as the tree is built, because the walk that would + /// measure it afterwards is the one that overflows the stack + #[test] + fn a_yaml_document_nested_past_any_stack_is_rejected() { + let text = "- ".repeat(100_000) + "1\n"; + let error = parse(Format::Yaml, &text).unwrap_err(); + assert!(error.to_string().contains("too deep"), "{error}"); + } + + /// an anchor is expanded wherever it is used, so the size has to be counted + /// while the copies are being made — a few hundred bytes name more values + /// than there is memory to hold + #[test] + fn a_yaml_document_whose_anchors_expand_without_end_is_rejected() { + let mut text = String::from("a0: &a0 [1, 1, 1, 1, 1, 1, 1, 1, 1]\n"); + for level in 1..12 { + let refs = (0..9) + .map(|_| format!("*a{previous}", previous = level - 1)) + .collect::>() + .join(", "); + let _ = writeln!(text, "a{level}: &a{level} [{refs}]"); + } + let error = parse(Format::Yaml, &text).unwrap_err(); + assert!(error.to_string().contains("too large"), "{error}"); + } + + #[test] + fn extensions() { + assert_eq!(Format::from_extension("json"), Some(Format::Json)); + assert_eq!(Format::from_extension("yml"), Some(Format::Yaml)); + assert_eq!(Format::from_extension("txt"), None); + } +} diff --git a/crates/by_resource/src/render.rs b/crates/by_resource/src/render.rs new file mode 100644 index 0000000000..8c4d16a897 --- /dev/null +++ b/crates/by_resource/src/render.rs @@ -0,0 +1,473 @@ +//! turning a [`Value`] into the python that stands for it. +//! +//! a mapping becomes a class, because a class is the only python object whose +//! attributes a type checker knows one by one without anybody writing them +//! down twice. a sequence becomes a tuple, so an index reaches an element the +//! checker can name rather than the union of everything in the collection. a +//! scalar becomes a literal, annotated `Final` so it keeps the value it was +//! written with instead of widening to its class. +//! +//! this is the only place that decides what a resource means. the type checker +//! reads a resource by inferring the module rendered here, and the transpiler +//! writes the very same rendering into the python it emits, so a value's type +//! and the object the program actually gets cannot drift apart. + +use std::fmt::Write as _; + +use ruff_python_stdlib::identifiers::is_identifier; + +use crate::value::Value; + +/// the import the rendered statements need. +pub const REQUIRED_IMPORT: &str = "from typing import Final"; + +const INDENT: &str = " "; + +/// python standing for one document. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Rendered { + /// module-level statements, in the order they must run + pub source: String, + /// the name the statements bind the document to + pub root: String, + /// paths of the keys that could not be given a name, in document order + pub unusable_keys: Vec, +} + +impl Rendered { + /// the statements as a module of their own, import included. + pub fn module_source(&self) -> String { + format!("{REQUIRED_IMPORT}\n\n{}", self.source) + } +} + +/// render `value` as python that binds it to `root`. +/// +/// `root` must be a valid identifier; [`binding_name`] makes one out of a file +/// name. +pub fn render(value: &Value, root: &str) -> Rendered { + let mut renderer = Renderer { + prefix: format!("{HELPER_PREFIX}{root}_"), + helpers: Vec::new(), + unusable_keys: Vec::new(), + next_helper: 0, + }; + + // a mapping at the top is the document's own namespace, so it becomes the + // class the name is bound to rather than a class the name points at + let binding = match value { + Value::Map(entries) => renderer.class(root, entries, "", ""), + _ => { + let expression = renderer.expression(value); + format!("{root}: Final = {expression}\n") + } + }; + + let mut source = String::new(); + for helper in &renderer.helpers { + source.push_str(helper); + source.push('\n'); + } + source.push_str(&binding); + + Rendered { + source, + root: root.to_string(), + unusable_keys: renderer.unusable_keys, + } +} + +/// the name to bind a resource file's document to, made from its file stem. +/// +/// the stem is used as written when it can be, so the class the checker reports +/// is the one the reader would guess from the file name. +pub fn binding_name(stem: &str) -> String { + if is_identifier(stem) { + return stem.to_string(); + } + + let mut name = String::with_capacity(stem.len() + 1); + for character in stem.chars() { + if character.is_alphanumeric() || character == '_' { + name.push(character); + } else { + name.push('_'); + } + } + if !is_identifier(&name) { + name.insert_str(0, "resource_"); + } + name +} + +/// whether a mapping key can be reached through an attribute of that name. +/// +/// a key that python cannot spell as an attribute is left out entirely rather +/// than renamed: a reader comparing the document against the code has to be +/// able to trust that a name in one is the same name in the other. +/// +/// a name with two leading underscores is left out as well. python mangles +/// `__x` inside a class body, so the attribute the reader would write is not +/// the one that would exist, and `__x__` would collide with what a class object +/// carries of its own. +/// +/// two names the rendering itself needs are left out for a third reason. an +/// attribute is what a class body resolves a bare name to, ahead of the module +/// around it, so a key called `Final` would be the `Final` every sibling after +/// it is annotated with, and a key called `_by_…` would be the helper class a +/// sibling names — the value would then be the number in the document rather +/// than the class the checker described. +fn is_usable_key(key: &str) -> bool { + is_identifier(key) + && !key.starts_with("__") + && !key.starts_with(HELPER_PREFIX) + && key != "Final" +} + +/// what the classes this rendering needs for itself are named after. +const HELPER_PREFIX: &str = "_by_"; + +struct Renderer { + /// what the helper classes this document needs are named after + prefix: String, + /// helper class definitions, in the order they must run + helpers: Vec, + unusable_keys: Vec, + next_helper: usize, +} + +impl Renderer { + /// a `class` statement binding `name` to `entries`, indented by `indent`. + /// + /// `path` is what the keys of `entries` are reached through, so an + /// unusable one can be reported where the document holds it rather than by + /// a bare name that may appear at several depths. + fn class( + &mut self, + name: &str, + entries: &[(String, Value)], + indent: &str, + path: &str, + ) -> String { + let body_indent = format!("{indent}{INDENT}"); + let mut body = String::new(); + + for (key, value) in entries { + if !is_usable_key(key) { + self.unusable_keys.push(format!("{path}{key}")); + continue; + } + match value { + // a mapping under a name of its own nests, so the class the + // checker reports for `config.a` is called `a` + Value::Map(entries) => { + body.push_str(&self.class( + key, + entries, + &body_indent, + &format!("{path}{key}."), + )); + } + _ => { + let expression = self.expression(value); + let _ = writeln!(body, "{body_indent}{key}: Final = {expression}"); + } + } + } + + if body.is_empty() { + body = format!("{body_indent}pass\n"); + } + + format!("{indent}class {name}:\n{body}") + } + + /// an expression for `value`. + /// + /// a mapping has no expression form, so one is defined as a class of its own + /// first and named here. + fn expression(&mut self, value: &Value) -> String { + match value { + Value::Null => "None".to_string(), + Value::Bool(true) => "True".to_string(), + Value::Bool(false) => "False".to_string(), + Value::Int(value) => value.to_string(), + Value::BigInt(digits) => digits.clone(), + Value::Float(value) => float_literal(*value), + Value::Str(value) => string_literal(value), + Value::Seq(items) => { + let parts: Vec<_> = items.iter().map(|item| self.expression(item)).collect(); + match parts.len() { + 0 => "()".to_string(), + // a one-element tuple needs its comma + 1 => format!("({},)", parts[0]), + _ => format!("({})", parts.join(", ")), + } + } + Value::Map(entries) => { + let name = format!("{}{}", self.prefix, self.next_helper); + self.next_helper += 1; + // a class body runs when the class is defined, so a helper this + // one names has to exist by then. rendering first and pushing + // afterwards puts the helpers it needed ahead of it + let definition = self.class(&name, entries, "", ""); + self.helpers.push(definition); + name + } + } + } +} + +/// a python float literal for `value`. +fn float_literal(value: f64) -> String { + if value.is_nan() { + return "float(\"nan\")".to_string(); + } + if value.is_infinite() { + return if value.is_sign_negative() { + "float(\"-inf\")".to_string() + } else { + "float(\"inf\")".to_string() + }; + } + // `{:?}` is the shortest text that reads back as the same `f64`, and it + // always writes a `.` or an `e`, so the result is a float literal rather + // than an integer one + format!("{value:?}") +} + +/// a python string literal for `value`. +fn string_literal(value: &str) -> String { + let mut literal = String::with_capacity(value.len() + 2); + literal.push('"'); + for character in value.chars() { + match character { + '\\' => literal.push_str("\\\\"), + '"' => literal.push_str("\\\""), + '\n' => literal.push_str("\\n"), + '\r' => literal.push_str("\\r"), + '\t' => literal.push_str("\\t"), + character if (character as u32) < 0x20 || character as u32 == 0x7f => { + let _ = write!(literal, "\\x{:02x}", character as u32); + } + character => literal.push(character), + } + } + literal.push('"'); + literal +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parse::{Format, parse}; + + fn rendered(format: Format, text: &str) -> Rendered { + render(&parse(format, text).unwrap(), "config") + } + + fn source(format: Format, text: &str) -> String { + rendered(format, text).source + } + + #[test] + fn a_nested_mapping_nests() { + assert_eq!( + source(Format::Yaml, "a:\n b:\n - 1\n - 2\n"), + "\ +class config: + class a: + b: Final = (1, 2) +" + ); + } + + #[test] + fn scalars() { + assert_eq!( + source( + Format::Yaml, + "a: 1\nb: text\nc: true\nd: ~\ne: 1.5\nf: -2\n" + ), + "\ +class config: + a: Final = 1 + b: Final = \"text\" + c: Final = True + d: Final = None + e: Final = 1.5 + f: Final = -2 +" + ); + } + + #[test] + fn a_mapping_inside_a_sequence_becomes_a_helper_class() { + assert_eq!( + source( + Format::Json, + r#"{"servers": [{"host": "a"}, {"host": "b"}]}"# + ), + "\ +class _by_config_0: + host: Final = \"a\" + +class _by_config_1: + host: Final = \"b\" + +class config: + servers: Final = (_by_config_0, _by_config_1) +" + ); + } + + #[test] + fn a_helper_a_helper_needs_is_defined_first() { + assert_eq!( + source(Format::Json, r#"{"a": [{"b": [{"c": 1}]}]}"#), + "\ +class _by_config_1: + c: Final = 1 + +class _by_config_0: + b: Final = (_by_config_1,) + +class config: + a: Final = (_by_config_0,) +" + ); + } + + #[test] + fn a_document_that_is_not_a_mapping_binds_the_value() { + assert_eq!(source(Format::Json, "[1, 2]"), "config: Final = (1, 2)\n"); + assert_eq!(source(Format::Json, "5"), "config: Final = 5\n"); + assert_eq!(source(Format::Yaml, ""), "config: Final = None\n"); + } + + #[test] + fn a_mapping_at_the_top_of_a_sequence_still_gets_a_helper() { + assert_eq!( + source(Format::Json, r#"[{"a": 1}]"#), + "\ +class _by_config_0: + a: Final = 1 + +config: Final = (_by_config_0,) +" + ); + } + + #[test] + fn an_empty_mapping_has_an_empty_body() { + assert_eq!( + source(Format::Json, r#"{"a": {}}"#), + "\ +class config: + class a: + pass +" + ); + assert_eq!(source(Format::Json, "{}"), "class config:\n pass\n"); + } + + #[test] + fn an_empty_sequence_is_an_empty_tuple() { + assert_eq!( + source(Format::Json, r#"{"a": []}"#), + "class config:\n a: Final = ()\n" + ); + } + + #[test] + fn a_key_python_cannot_spell_is_left_out() { + let rendered = rendered( + Format::Json, + r#"{"a-b": 1, "class": 2, "__x": 3, "1": 4, "ok": 5}"#, + ); + assert_eq!(rendered.source, "class config:\n ok: Final = 5\n"); + assert_eq!(rendered.unusable_keys, ["a-b", "class", "__x", "1"]); + } + + /// the same name can be a key at several depths, so a report that named one + /// of them alone would not say which + #[test] + fn an_unusable_key_is_reported_where_the_document_holds_it() { + let rendered = rendered(Format::Json, r#"{"a": {"b": {"c-d": 1}}, "c-d": 2}"#); + assert_eq!(rendered.unusable_keys, ["a.b.c-d", "c-d"]); + } + + #[test] + fn a_key_the_rendering_needs_the_name_of_is_left_out() { + let rendered = rendered(Format::Json, r#"{"Final": 1, "a": 2}"#); + assert_eq!(rendered.source, "class config:\n a: Final = 2\n"); + assert_eq!(rendered.unusable_keys, ["Final"]); + } + + /// a class body resolves a bare name to its own attributes first, so a key + /// named after a helper class would be what the sibling that names that + /// helper actually gets — the number in the document, not the class + #[test] + fn a_key_that_would_shadow_a_helper_class_is_left_out() { + let rendered = rendered(Format::Json, r#"{"_by_config_0": 1, "x": [{"y": 2}]}"#); + assert_eq!( + rendered.source, + "\ +class _by_config_0: + y: Final = 2 + +class config: + x: Final = (_by_config_0,) +" + ); + assert_eq!(rendered.unusable_keys, ["_by_config_0"]); + } + + #[test] + fn a_dunder_key_is_left_out() { + let rendered = rendered(Format::Json, r#"{"__x__": 1}"#); + assert_eq!(rendered.unusable_keys, ["__x__"]); + } + + #[test] + fn strings_are_escaped() { + assert_eq!( + source(Format::Json, r#"{"a": "he said \"hi\"\n\tc:\\d\u0001"}"#), + "class config:\n a: Final = \"he said \\\"hi\\\"\\n\\tc:\\\\d\\x01\"\n" + ); + } + + #[test] + fn non_ascii_survives() { + assert_eq!( + source(Format::Json, r#"{"a": "héllo — ok"}"#), + "class config:\n a: Final = \"héllo — ok\"\n" + ); + } + + #[test] + fn floats_read_back_as_floats() { + assert_eq!(float_literal(1.0), "1.0"); + assert_eq!(float_literal(-0.5), "-0.5"); + assert_eq!(float_literal(f64::INFINITY), "float(\"inf\")"); + assert_eq!(float_literal(f64::NEG_INFINITY), "float(\"-inf\")"); + assert_eq!(float_literal(f64::NAN), "float(\"nan\")"); + } + + #[test] + fn a_module_carries_its_import() { + assert_eq!( + rendered(Format::Json, r#"{"a": 1}"#).module_source(), + "from typing import Final\n\nclass config:\n a: Final = 1\n" + ); + } + + #[test] + fn binding_names() { + assert_eq!(binding_name("config"), "config"); + assert_eq!(binding_name("my-config"), "my_config"); + assert_eq!(binding_name("2020"), "resource_2020"); + assert_eq!(binding_name("class"), "resource_class"); + assert_eq!(binding_name("_hidden"), "_hidden"); + } +} diff --git a/crates/by_resource/src/value.rs b/crates/by_resource/src/value.rs new file mode 100644 index 0000000000..f3f8011a4a --- /dev/null +++ b/crates/by_resource/src/value.rs @@ -0,0 +1,44 @@ +//! the one shape every supported format is read into. + +/// a value read out of a static resource. +/// +/// json, toml and yaml disagree about a great many things, but they agree about +/// this much: a document is a tree of scalars, sequences and string-keyed +/// mappings. everything a format offers beyond that (a toml datetime, a yaml +/// anchor) is resolved or rejected while parsing, so the renderer never has to +/// know which format it came from. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Int(i64), + /// an integer too large for [`Value::Int`], kept as the digits it was + /// written with so nothing is lost to a round trip through `f64` + BigInt(String), + Float(f64), + Str(String), + Seq(Vec), + /// insertion-ordered, because a reader of the generated python should find + /// the keys in the order the document lists them + Map(Vec<(String, Value)>), +} + +impl Value { + /// how many nodes the tree holds, counting itself. + pub(crate) fn size(&self) -> usize { + match self { + Value::Seq(items) => 1 + items.iter().map(Value::size).sum::(), + Value::Map(entries) => 1 + entries.iter().map(|(_, v)| v.size()).sum::(), + _ => 1, + } + } + + /// how deeply the tree nests. + pub(crate) fn depth(&self) -> usize { + match self { + Value::Seq(items) => 1 + items.iter().map(Value::depth).max().unwrap_or(0), + Value::Map(entries) => 1 + entries.iter().map(|(_, v)| v.depth()).max().unwrap_or(0), + _ => 1, + } + } +} diff --git a/crates/by_stage/src/record.rs b/crates/by_stage/src/record.rs index a36d757301..dfe43967ee 100644 --- a/crates/by_stage/src/record.rs +++ b/crates/by_stage/src/record.rs @@ -19,9 +19,11 @@ //! lowering flags were folded in, and a re-stage reads it back rather than //! deriving anything. +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use by_transforms::SoundnessPositions; +use by_transforms::config::FloatLiteralLowering; use by_transforms::config::{Config, PythonVersion}; use crate::staging::Staging; @@ -55,6 +57,15 @@ pub struct ConfigRecord { pub soundness: String, pub runtime_raises_checks: bool, pub unique_loop_bindings: bool, + /// how a float or complex literal type was spelled, as the option takes it + pub float_literals: String, + /// the `build:` stamps the build settled, so a re-stage emits the same ones + /// + /// without this a re-staged file would fall back to the block's defaults and + /// claim a different commit than the tree around it. defaulted, because a + /// tree written before stamps existed genuinely had none + #[serde(default)] + pub stamps: BTreeMap, } /// What a build tree records about itself. @@ -107,6 +118,8 @@ impl BuildRecord { soundness: spell_soundness(config.soundness), runtime_raises_checks: config.runtime_raises_checks, unique_loop_bindings: config.unique_loop_bindings, + float_literals: spell_float_literals(config.float_literals), + stamps: config.stamps.clone(), }, } } @@ -153,6 +166,8 @@ impl BuildRecord { soundness: parse_soundness(&self.config.soundness)?, runtime_raises_checks: self.config.runtime_raises_checks, unique_loop_bindings: self.config.unique_loop_bindings, + float_literals: parse_float_literals(&self.config.float_literals)?, + stamps: self.config.stamps.clone(), ..Config::default() }) } @@ -210,6 +225,27 @@ pub fn parse_soundness(spec: &str) -> anyhow::Result { /// list, which round-trips exactly — and it has to round-trip exactly rather than /// approximately, since these positions decide which runtime checks the emitted /// python carries. +/// The spelling a [`FloatLiteralLowering`] is written down as, matching the +/// `lowering.float-literals` option a project sets it with. +pub fn spell_float_literals(lowering: FloatLiteralLowering) -> String { + match lowering { + FloatLiteralLowering::Nominal => "nominal".to_owned(), + FloatLiteralLowering::Literal => "literal".to_owned(), + } +} + +/// Read back what [`spell_float_literals`] wrote. +pub fn parse_float_literals(spelled: &str) -> anyhow::Result { + match spelled { + "nominal" => Ok(FloatLiteralLowering::Nominal), + "literal" => Ok(FloatLiteralLowering::Literal), + other => anyhow::bail!( + "{BY_BUILD_FILENAME} spells float literals {other:?}, \ + which is not `nominal` or `literal`" + ), + } +} + pub fn spell_soundness(positions: SoundnessPositions) -> String { if positions == SoundnessPositions::defaults() { return "default".to_owned(); @@ -237,9 +273,11 @@ pub fn spell_soundness(positions: SoundnessPositions) -> String { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::{BuildRecord, ConfigRecord, parse_soundness, spell_soundness}; use by_transforms::SoundnessPositions; - use by_transforms::config::Config; + use by_transforms::config::{Config, FloatLiteralLowering}; use std::path::{Path, PathBuf}; /// the wire form, exactly as the plugin and `bpd` read it. nothing else here @@ -316,6 +354,7 @@ mod tests { soundness, runtime_raises_checks: true, unique_loop_bindings: false, + float_literals: FloatLiteralLowering::Literal, ..Config::default() }; @@ -326,6 +365,7 @@ mod tests { assert_eq!(recovered.soundness, soundness); assert!(recovered.runtime_raises_checks); assert!(!recovered.unique_loop_bindings); + assert_eq!(recovered.float_literals, FloatLiteralLowering::Literal); assert_eq!(recovered.min_version, config.min_version); Ok(()) } @@ -371,6 +411,8 @@ mod tests { soundness: "none".to_owned(), runtime_raises_checks: false, unique_loop_bindings: true, + float_literals: "nominal".to_owned(), + stamps: BTreeMap::new(), }, }; assert!(record.config().is_err()); diff --git a/crates/by_transforms/Cargo.toml b/crates/by_transforms/Cargo.toml index 1a6432d768..5286f3478b 100644 --- a/crates/by_transforms/Cargo.toml +++ b/crates/by_transforms/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } license = "MIT OR Apache-2.0" [dependencies] +by_resource = { workspace = true } indexmap = { workspace = true } ruff_db = { workspace = true } ruff_diagnostics = { workspace = true } diff --git a/crates/by_transforms/src/config.rs b/crates/by_transforms/src/config.rs index 40f53d840c..94deced5c3 100644 --- a/crates/by_transforms/src/config.rs +++ b/crates/by_transforms/src/config.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + pub use ruff_python_ast::PythonVersion; #[derive(Debug, Clone)] @@ -46,12 +48,52 @@ pub struct Config { /// saw. a clause with no faithful runtime test — `raises ...`, or a set with /// no runtime spelling such as a negation — is never guarded pub runtime_raises_checks: bool, + /// the values a `build:` block's stamps take, supplied by whatever drove + /// this transpile — `by build` and `by run` settle them once, ahead of the + /// pipeline. + /// + /// they arrive as config rather than being read here because transpile + /// output has to stay a function of the source plus this config. a pipeline + /// that asked git for the commit itself would give a different answer to a + /// re-stage than it gave the build that re-stage is patching, and the two + /// trees would disagree about what they are + /// + /// a stamp with no entry here falls back to the default written in the + /// block; one with neither is a transpile error, which is the whole point of + /// declaring it + pub stamps: BTreeMap, /// when true (the default), a closure created inside a loop captures the /// loop's bindings *by value*, so each iteration's closure sees that /// iteration's element instead of every closure sharing the one cell /// python leaves behind. Tests that compare exact transpile output should /// set this to `false` unless they exercise the capture itself pub unique_loop_bindings: bool, + /// how a float or complex literal written in a type position is spelled in + /// the emitted python. see [`FloatLiteralLowering`] + pub float_literals: FloatLiteralLowering, +} + +/// What a float or complex literal type becomes in the emitted python. +/// +/// basedpython reads `a: 1.5` as a literal type, and python has no spelling for +/// one: PEP 586 admits only `None`, `int`, `bool`, `str`, `bytes` and enum +/// members into `Literal[...]`, so `Literal[1.5]` — while perfectly happy at +/// runtime, since `typing` does not check what it is handed — is rejected by +/// every type checker that reads the output, ty included. +/// +/// Leaving the literal bare is not an option: `a: int | 3.5` would be a +/// `TypeError` the moment the annotation is evaluated. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum FloatLiteralLowering { + /// the nominal type the literal is one of — `a: 1.5` becomes `a: float`, + /// `a: 2j` becomes `a: complex`. precision is lost, and the output is + /// something every checker accepts + #[default] + Nominal, + /// the literal kept inside `Literal[...]` — `a: 1.5` becomes + /// `a: Literal[1.5]`. the precision survives and the output still runs, but + /// a checker reading it reports the argument as invalid + Literal, } /// Per-position toggles for the runtime type-soundness checks. Each field @@ -149,6 +191,8 @@ impl Default for Config { soundness: SoundnessPositions::defaults(), runtime_raises_checks: false, unique_loop_bindings: true, + float_literals: FloatLiteralLowering::Nominal, + stamps: BTreeMap::new(), } } } diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index fc52c48555..3029d6b991 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -4,7 +4,7 @@ pub mod source_map; mod transforms; pub(crate) mod type_info; -pub use config::{Config, PythonVersion, SoundnessPositions}; +pub use config::{Config, FloatLiteralLowering, PythonVersion, SoundnessPositions}; use std::collections::{BTreeSet, HashSet}; diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index ed2637a832..60526ffa6a 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -3,6 +3,7 @@ use ruff_python_ast::{Expr, PythonVersion, Stmt}; use ruff_text_size::Ranged; use crate::Config; +use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::transforms::{literal_types, optional_type}; @@ -40,6 +41,7 @@ pub(crate) struct TupleLiteralType<'src> { source: &'src str, types: &'src dyn TypeInfo, min_version: PythonVersion, + float_literals: FloatLiteralLowering, /// set when a lowering spelled an `Unpack`, so the pass can ask for the import pub(crate) needs_unpack_import: std::cell::Cell, pub(crate) edits: Vec, @@ -50,11 +52,13 @@ impl<'src> TupleLiteralType<'src> { source: &'src str, types: &'src dyn TypeInfo, min_version: PythonVersion, + float_literals: FloatLiteralLowering, ) -> Self { Self { source, types, min_version, + float_literals, needs_unpack_import: std::cell::Cell::new(false), edits: Vec::new(), } @@ -79,7 +83,7 @@ impl<'src> TupleLiteralType<'src> { /// Source text for `expr`, with literal-type rewrites applied if needed. fn fallback_src(&self, expr: &Expr) -> String { - literal_types::rewrite_type_expr(self.source, self.types, expr) + literal_types::rewrite_type_expr(self.source, self.types, expr, self.float_literals) .unwrap_or_else(|| self.src(expr.range()).to_owned()) } @@ -342,7 +346,12 @@ impl<'src> TupleLiteralTypePass<'src> { impl TypeAwarePass for TupleLiteralTypePass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = TupleLiteralType::new(self.source, types, self.config.min_version); + let mut inner = TupleLiteralType::new( + self.source, + types, + self.config.min_version, + self.config.float_literals, + ); walk_type_positions(stmts, Some(types), &mut inner); let mut wraps_literal = false; for fix in inner.edits { diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index ba41ba7d71..e909acc23b 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -168,8 +168,8 @@ impl<'src> AnonNamedTuple<'src> { Self { source, types, + callable: CallableSyntax::new(source, config.float_literals).with_types(types), config, - callable: CallableSyntax::new(source).with_types(types), edits: Vec::new(), shapes: indexmap::IndexMap::new(), range_to_class: Vec::new(), diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 9b0c47f724..924e130807 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -36,19 +36,20 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::source_util::preamble_offset; use super::{ - annotation, anon_named_tuple, auto_quote, callable, character_type, checked_cast, + annotation, anon_named_tuple, auto_quote, build_stamps, callable, character_type, checked_cast, class_pattern_star, coalesce, coalesce_chain, compat, conformance, context_params, conversion, decl_site_variance, decorated_binding, decorator_keyword, dedent_string, destructure, django_lookup, dynamic_keyword, empty_declarations, export_import, extension, flexible_keyword, float_const, force_unwrap, frameworks, generic_call, generics, grapheme_string, identity_swap, if_let, implicit_receiver, implicit_typing, inferred_annotation, init_method, just_float, kw_subscript, literal_string, literal_types, local_once, main_function, match_type, modifiers, - mutable_defaults, none_chain, optional_type, overload, parametric_is, postfix_await, - private_method, propagate, properties, protocol_type, raises_clause, reified_class, - reified_generic, repeated_underscore, return_value_use, runtime_union, sentinel, some_ctor, - soundness, statement_expression, string_tag, super_keyword, symbolic_type_op, template_type, - top_star, trailing_lambda, tuple_index, type_fn, type_is, type_reification, typed_dict_literal, - typed_lambda, typeof_keyword, unique_loop_bindings, unpack, use_site_variance, + module_api, mutable_defaults, none_chain, optional_type, overload, parametric_is, + postfix_await, private_method, propagate, properties, protocol_type, raises_clause, + reified_class, reified_generic, repeated_underscore, return_value_use, runtime_union, sentinel, + some_ctor, soundness, statement_expression, static_resource, string_tag, super_keyword, + symbolic_type_op, template_type, top_star, trailing_lambda, tuple_index, type_fn, type_is, + type_reification, typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, + unpack, use_site_variance, }; use crate::Config; use crate::type_info::TypeInfo; @@ -461,6 +462,14 @@ pub(crate) fn run_against_source<'a>( // blanked copy the passes walk no longer has a `literal` keyword in it let literal_string_rewrites = literal_string::collect(parsed_handle.suite(), &semantic_model); + // a static resource import has no python spelling at all, so the document it + // names is read now — from the db's own parse, which is the one the checker + // answered about — and written into the module in the import's place + let static_resource_pass = static_resource::StaticResource::new( + static_resource::collect(parsed_handle.suite(), &semantic_model), + source_ref, + ); + let symbolic_folds = symbolic_type_op::collect_symbolic_folds(parsed_handle.suite(), &semantic_model); let symbolic_needs_literal_import = symbolic_folds.needs_literal_import; @@ -537,7 +546,7 @@ pub(crate) fn run_against_source<'a>( config.min_version, config.inject_future_annotations, ); - let init_method_pass = init_method::InitMethod::new(source_ref); + let init_method_pass = init_method::InitMethod::new(source_ref, config.float_literals); let properties_pass = properties::PropertiesPass::new(source_ref, accessor_value_ranges); let local_once_pass = local_once::LocalOncePass::new(source_ref); let raises_strip_pass = raises_clause::RaisesStripPass::new(source_ref); @@ -551,6 +560,7 @@ pub(crate) fn run_against_source<'a>( let match_type_pass = match_type::MatchTypePass::new(source_ref); let modifiers_pass = modifiers::ModifiersPass::new(source_ref); let main_function_pass = main_function::MainFunction::new(source_ref, config.is_stub); + let build_stamps_pass = build_stamps::BuildStampsPass::new(source_ref, config.stamps.clone()); let empty_declarations_pass = empty_declarations::EmptyDeclarations::new(); let overload_pass = overload::Overload::new(source_ref, config.is_stub); let decorator_keyword_pass = decorator_keyword::DecoratorKeyword::new(source_ref); @@ -572,8 +582,8 @@ pub(crate) fn run_against_source<'a>( let inferred_annotation_pass = inferred_annotation::InferredAnnotationPass::new(); let template_type_pass = template_type::TemplateTypePass; let tuple_types_pass = annotation::TupleLiteralTypePass::new(source_ref, config.clone()); - let literal_types_pass = literal_types::LiteralTypePass::new(source_ref); - let callable_pass = callable::CallableSyntaxPass::new(source_ref); + let literal_types_pass = literal_types::LiteralTypePass::new(source_ref, config.float_literals); + let callable_pass = callable::CallableSyntaxPass::new(source_ref, config.float_literals); let protocol_type_pass = protocol_type::ProtocolTypePass::new(source_ref, config.clone()); let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref); let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); @@ -586,6 +596,7 @@ pub(crate) fn run_against_source<'a>( let generics_pass = generics::GenericPolyfillPass::new(source_ref, config.clone()); let soundness_pass = soundness::SoundnessPass::new(source_ref, config); let checked_cast_pass = checked_cast::CheckedCastPass; + let module_api_pass = module_api::ModuleApiPass::new(source_ref); let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); let if_let_pass = if_let::IfLetPass::new(source_ref); let class_pattern_star_pass = class_pattern_star::ClassPatternStarPass::new(source_ref); @@ -650,6 +661,9 @@ pub(crate) fn run_against_source<'a>( // erase `type def` declarations; their applications were already folded to // the resolved type by the symbolic pass above &type_fn_pass, + // erase `implements` declarations — a whole-line source deletion, so it + // has to read ranges before any AST-mutation pass zeroes them + &module_api_pass, // replace a match type's `case` blocks with a runtime value, and strip // `TypeVarTuple` bounds wherever they appear &match_type_pass, @@ -664,6 +678,10 @@ pub(crate) fn run_against_source<'a>( // emits, and before the AST-mutation passes so `main`'s decorator // ranges are still valid for the `private` check &main_function_pass, + // replaces a whole `build:` block with the class it lowers to, reading + // the block's own source for each default — so it has to run while those + // ranges still mean something, ahead of every mutation pass + &build_stamps_pass, &empty_declarations_pass, &overload_pass, &decorator_keyword_pass, @@ -680,6 +698,14 @@ pub(crate) fn run_against_source<'a>( &sentinel_pass, &repeated_underscore_pass, &typed_lambda_pass, + // a static resource import is replaced whole by the document it names, + // in the source *and* in the AST. last, because the statements it + // splices in are parsed from the rendering and carry that text's + // ranges, which name nothing in this file — no pass after it may read + // them. it declares no change of its own, so the source edit is what + // normally lands and the AST rewrite only matters when another pass + // has already forced the statement to be re-rendered + &static_resource_pass, ]; for pass in passes { pass.run(&mut module, &mut ctx); diff --git a/crates/by_transforms/src/transforms/build_stamps.rs b/crates/by_transforms/src/transforms/build_stamps.rs new file mode 100644 index 0000000000..101f448a38 --- /dev/null +++ b/crates/by_transforms/src/transforms/build_stamps.rs @@ -0,0 +1,369 @@ +//! AST pass: a `build:` block is the program's build stamps — values settled +//! when the artifact was produced rather than read at startup. +//! +//! the block parses to a `ClassDef` carrying a synthetic `build_def` marker, +//! whose members are annotation-only. this pass fills each one in from +//! [`Config::stamps`] and emits an ordinary class, so `build.GIT_SHA` needs no +//! runtime support at all: +//! +//! ```text +//! build: class build: +//! GIT_SHA: str -> GIT_SHA: str = "e6f9ac1d" +//! PORT: int = 8000 PORT: int = 8000 +//! ``` +//! +//! the values come in through the config because the pipeline must never go +//! looking for them itself. asking git here would make the emitted python a +//! function of the working tree as well as the source, and a re-stage — which +//! re-transpiles one file into a tree an earlier build wrote — would quietly +//! disagree with the rest of that tree about what commit it is +//! +//! a stamp with no supplied value falls back to the default written in the +//! block. one with neither is a hard error: declaring a stamp without a default +//! is precisely the claim that the build has to supply it, and that claim is the +//! whole reason to write it down +//! +//! there is deliberately no reverse transform. what this emits is an ordinary +//! class, indistinguishable from one somebody wrote by hand, and turning a class +//! named `build` back into a block would throw away the values it holds — which +//! for the one class where the values *are* the point is the worst direction to +//! be lossy in +//! +//! [`Config::stamps`]: crate::Config + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::{Expr, ModModule, Stmt, StmtClassDef}; +use ruff_text_size::{Ranged, TextRange}; + +use super::ast_driver::{AstPass, PassContext}; +use super::source_util::python_string_literal; + +/// the annotations a stamp may be declared with. +/// +/// a stamp arrives from the build as text — a commit hash, the output of `git +/// status`, a number a CI job counted — so the set is the types that text has +/// one obvious reading as. anything else would need a convention about how the +/// string becomes the value, and inventing one silently is worse than saying the +/// annotation is not supported +#[derive(Clone, Copy)] +enum StampType { + Str, + Int, + Bool, +} + +impl StampType { + fn from_annotation(annotation: &Expr) -> Option { + match annotation.as_name_expr()?.id.as_str() { + "str" => Some(Self::Str), + "int" => Some(Self::Int), + "bool" => Some(Self::Bool), + _ => None, + } + } + + fn spelled(self) -> &'static str { + match self { + Self::Str => "str", + Self::Int => "int", + Self::Bool => "bool", + } + } + + /// the python literal `value` stands for, or `None` when the text the build + /// supplied is not one of this type at all + fn literal(self, value: &str) -> Option { + match self { + Self::Str => Some(python_string_literal(value)), + // rendered back from the parsed number rather than passed through: + // `007` and `+3` are values a shell hands over quite naturally, and + // neither is a python integer literal + Self::Int => value.parse::().ok().map(|parsed| parsed.to_string()), + Self::Bool => match value.trim().to_ascii_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Some("True".to_owned()), + "false" | "0" | "no" | "off" | "" => Some("False".to_owned()), + _ => None, + }, + } + } +} + +pub(crate) struct BuildStampsPass<'a> { + source: &'a str, + stamps: BTreeMap, +} + +impl<'a> BuildStampsPass<'a> { + pub(crate) fn new(source: &'a str, stamps: BTreeMap) -> Self { + Self { source, stamps } + } + + /// the lowered class, or the reasons it could not be lowered + fn lower(&self, class: &StmtClassDef) -> Result> { + let mut rendered = String::from("class build:"); + let mut errors = Vec::new(); + + for statement in &class.body { + match statement { + Stmt::AnnAssign(declaration) => { + let Some(target) = declaration.target.as_name_expr() else { + errors + .push("a `build` stamp is a plain name with an annotation".to_owned()); + continue; + }; + let name = target.id.as_str(); + + let Some(stamp_type) = StampType::from_annotation(&declaration.annotation) + else { + errors.push(format!( + "the stamp `{name}` is annotated `{}`, which the build has no way to \ + supply — a stamp reaches the program as text, so it must be \ + annotated `str`, `int` or `bool`", + self.text(declaration.annotation.range()) + )); + continue; + }; + + let value = match self.stamps.get(name) { + Some(supplied) => match stamp_type.literal(supplied) { + Some(literal) => literal, + None => { + errors.push(format!( + "the stamp `{name}` is declared `{}`, but the build supplied \ + `{supplied}`, which is not one", + stamp_type.spelled() + )); + continue; + } + }, + // no value: the default stands in, and its source is + // carried across as written + None => match &declaration.value { + Some(default) => self.text(default.range()).to_owned(), + None => { + errors.push(format!( + "the build supplied no value for the stamp `{name}`, and it \ + has no default" + )); + continue; + } + }, + }; + + let _ = write!(rendered, "\n {name}: {} = {value}", stamp_type.spelled()); + } + // a docstring describes the block and belongs to the class it + // becomes + Stmt::Expr(expression) if expression.value.is_string_literal_expr() => { + let _ = write!(rendered, "\n {}", self.text(expression.range())); + } + other => errors.push(format!( + "a `build` block holds stamp declarations and nothing else, but this is \ + `{}`", + self.text(other.range()).lines().next().unwrap_or_default() + )), + } + } + + if errors.is_empty() { + Ok(rendered) + } else { + Err(errors) + } + } + + fn text(&self, range: TextRange) -> &str { + self.source + .get(usize::from(range.start())..usize::from(range.end())) + .unwrap_or_default() + } +} + +impl AstPass for BuildStampsPass<'_> { + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { + let mut seen = false; + for statement in &module.body { + if let Stmt::ClassDef(class) = statement + && class.is_build_stamps() + { + // both lower to `class build`, so the second would shadow the + // first and every stamp the first declared would quietly stop + // being there + if seen { + ctx.errors.push( + "a module declares its build stamps once, and this is a second `build` \ + block — the stamps of both belong in one" + .to_owned(), + ); + continue; + } + seen = true; + match self.lower(class) { + Ok(rendered) => ctx.text_edits.push((class.range(), rendered)), + Err(errors) => ctx.errors.extend(errors), + } + } + } + + // a block anywhere but the module body describes the same one program's + // build, from a place its readers cannot see. rather than lower it into + // a class nobody can reach, say so + let mut nested = Nested { depth: 0, found: 0 }; + for statement in &module.body { + nested.visit_stmt(statement); + } + for _ in 0..nested.found { + ctx.errors.push( + "a `build` block declares the whole program's stamps, so it belongs at the top \ + level of a module" + .to_owned(), + ); + } + } +} + +/// finds `build:` blocks below the module body +struct Nested { + depth: usize, + found: usize, +} + +impl<'ast> Visitor<'ast> for Nested { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if self.depth > 0 + && let Stmt::ClassDef(class) = stmt + && class.is_build_stamps() + { + self.found += 1; + } + self.depth += 1; + walk_stmt(self, stmt); + self.depth -= 1; + } +} + +#[cfg(test)] +mod tests { + use crate::{Config, transpile}; + + fn with_stamps(source: &str, stamps: &[(&str, &str)]) -> Result { + let mut config = Config::test_default(); + config.stamps = stamps + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + transpile(source, &config) + } + + #[test] + fn a_supplied_stamp_becomes_a_literal() { + let out = with_stamps("build:\n GIT_SHA: str\n", &[("GIT_SHA", "e6f9ac1")]).unwrap(); + assert!( + out.contains("class build:") && out.contains("GIT_SHA: str = \"e6f9ac1\""), + "{out}" + ); + } + + #[test] + fn a_supplied_value_beats_the_default() { + let out = with_stamps( + "build:\n VERSION: str = \"0.0.0+dev\"\n", + &[("VERSION", "1.4.0")], + ) + .unwrap(); + assert!(out.contains("VERSION: str = \"1.4.0\""), "{out}"); + } + + #[test] + fn an_unsupplied_stamp_falls_back_to_its_default() { + let out = with_stamps("build:\n VERSION: str = \"0.0.0+dev\"\n", &[]).unwrap(); + assert!(out.contains("VERSION: str = \"0.0.0+dev\""), "{out}"); + } + + #[test] + fn an_unsupplied_stamp_with_no_default_is_an_error() { + let error = with_stamps("build:\n GIT_SHA: str\n", &[]).unwrap_err(); + assert!( + error.contains("supplied no value for the stamp `GIT_SHA`"), + "{error}" + ); + } + + #[test] + fn a_bool_stamp_reads_the_spellings_a_shell_produces() { + for (supplied, expected) in [ + ("true", "True"), + ("1", "True"), + ("false", "False"), + ("0", "False"), + ("", "False"), + ] { + let out = with_stamps("build:\n DIRTY: bool\n", &[("DIRTY", supplied)]).unwrap(); + assert!( + out.contains(&format!("DIRTY: bool = {expected}")), + "{supplied:?} -> {out}" + ); + } + } + + #[test] + fn an_int_stamp_is_rendered_from_its_value() { + // `007` is a perfectly ordinary thing for a build system to hand over, + // and a python literal it is not + let out = with_stamps("build:\n RUN: int\n", &[("RUN", "007")]).unwrap(); + assert!(out.contains("RUN: int = 7"), "{out}"); + } + + #[test] + fn a_value_that_is_not_the_declared_type_is_an_error() { + let error = with_stamps("build:\n RUN: int\n", &[("RUN", "later")]).unwrap_err(); + assert!( + error.contains("`RUN` is declared `int`, but the build supplied `later`"), + "{error}" + ); + } + + #[test] + fn an_unsupported_annotation_is_an_error() { + let error = with_stamps("build:\n WHEN: float\n", &[("WHEN", "1.0")]).unwrap_err(); + assert!(error.contains("annotated `float`"), "{error}"); + } + + #[test] + fn a_statement_that_is_not_a_stamp_is_an_error() { + let error = with_stamps("build:\n def f(self): ...\n", &[]).unwrap_err(); + assert!( + error.contains("holds stamp declarations and nothing else"), + "{error}" + ); + } + + #[test] + fn a_docstring_is_carried_across() { + let out = with_stamps( + "build:\n \"what this build was\"\n V: str = \"x\"\n", + &[], + ) + .unwrap(); + assert!(out.contains("\"what this build was\""), "{out}"); + } + + #[test] + fn a_second_block_is_an_error() { + let error = with_stamps( + "build:\n A: str = \"a\"\n\nbuild:\n B: str = \"b\"\n", + &[], + ) + .unwrap_err(); + assert!(error.contains("second `build` block"), "{error}"); + } + + #[test] + fn a_nested_block_is_an_error() { + let error = with_stamps("def f():\n build:\n V: str = \"x\"\n", &[]).unwrap_err(); + assert!(error.contains("belongs at the top level"), "{error}"); + } +} diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index 8c693905ce..ce3b9f7aa5 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -31,6 +31,7 @@ use super::ast_driver::{PassContext, TypeAwarePass}; use super::intersection::{collect_intersect, collect_union, is_intersection_node}; use super::just_float::rewrite_type_expr_with_imports; use super::wrapped_runtime::OPTIONAL_RUNTIME; +use crate::config::FloatLiteralLowering; use crate::type_info::{TypeInfo, UnpackedKwargsLowering}; #[expect( @@ -45,6 +46,7 @@ pub(crate) struct CallableSyntax<'src> { /// it would re-render its `typeof`/operator surface and the wider edit /// would clobber the fold, so `rewrite` leaves it for the fold's own edit claimed_ranges: &'src [TextRange], + float_literals: FloatLiteralLowering, pub(crate) edits: Vec, pub(crate) needs_import: bool, pub(crate) needs_concatenate_import: bool, @@ -81,11 +83,12 @@ struct ProtocolShape { } impl<'src> CallableSyntax<'src> { - pub(crate) fn new(source: &'src str) -> Self { + pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { Self { source, types: None, claimed_ranges: &[], + float_literals, edits: Vec::new(), needs_import: false, needs_concatenate_import: false, @@ -202,7 +205,8 @@ impl<'src> CallableSyntax<'src> { return self.sweep_substitutions(expr.range()); } if let Some(types) = self.types - && let Some((text, imports)) = rewrite_type_expr_with_imports(self.source, types, expr) + && let Some((text, imports)) = + rewrite_type_expr_with_imports(self.source, types, expr, self.float_literals) { self.extra_imports.extend(imports); return text; @@ -817,8 +821,9 @@ pub(crate) fn lower_type_expr_full( types: &dyn TypeInfo, expr: &Expr, substitutions: &[(TextRange, String)], + float_literals: FloatLiteralLowering, ) -> Option { - let mut inner = CallableSyntax::new(source).with_types(types); + let mut inner = CallableSyntax::new(source, float_literals).with_types(types); for (range, name) in substitutions { inner.add_substitution(*range, name.clone()); } @@ -835,7 +840,7 @@ pub(crate) fn lower_type_expr_full( } // no structural type-form — fall back to the per-leaf composer // (`float` → `JustFloat`, a literal → `Literal[…]`, `dynamic` → `Any`) - rewrite_type_expr_with_imports(source, types, expr).map(|(text, _)| text) + rewrite_type_expr_with_imports(source, types, expr, float_literals).map(|(text, _)| text) } /// if `expr` is `Subscript(Name("__let__"|"__classvar__"|"__final__"), slice)`, @@ -904,11 +909,15 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> pub(crate) struct CallableSyntaxPass<'src> { source: &'src str, + float_literals: FloatLiteralLowering, } impl<'src> CallableSyntaxPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + Self { + source, + float_literals, + } } } @@ -947,7 +956,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { // owned copy so `inner`'s borrow doesn't pin `ctx` against the mutable // `required_imports` / `edits` uses below let claimed = ctx.claimed_type_op_ranges.clone(); - let mut inner = CallableSyntax::new(self.source) + let mut inner = CallableSyntax::new(self.source, self.float_literals) .with_types(types) .with_claimed_ranges(&claimed); crate::transforms::type_expr_walker::walk_type_positions_skipping( diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 7193a5ce7b..a4ab2e5f45 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -449,6 +449,7 @@ impl<'src> GenericPolyfill<'src> { self.types, bound, &self.subsume_within(bound.range()), + self.config.float_literals, ) .unwrap_or_else(|| self.src(bound.range()).to_owned()) }; @@ -462,6 +463,7 @@ impl<'src> GenericPolyfill<'src> { self.types, default, &self.subsume_within(default.range()), + self.config.float_literals, ) .unwrap_or_else(|| self.src(default.range()).to_owned()); if self.config.min_version < PythonVersion::PY313 { @@ -894,8 +896,14 @@ impl<'src> GenericPolyfill<'src> { let substitutions: Vec<(TextRange, String)> = folded.iter().cloned().chain(renames).collect(); - let value_src = lower_type_expr_full(self.source, self.types, &alias.value, &substitutions) - .unwrap_or(raw_value_src); + let value_src = lower_type_expr_full( + self.source, + self.types, + &alias.value, + &substitutions, + self.config.float_literals, + ) + .unwrap_or(raw_value_src); self.needed_imports.typealias_type = true; diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 6555320956..ebf343691a 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -30,15 +30,20 @@ use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use super::callable::lower_type_expr_full; use super::mutable_defaults::parameter_guards; use super::source_util::{PrologueStatement, first_body_statement}; +use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; pub(crate) struct InitMethod<'src> { source: &'src str, + float_literals: FloatLiteralLowering, } impl<'src> InitMethod<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + Self { + source, + float_literals, + } } } @@ -48,6 +53,7 @@ impl TypeAwarePass for InitMethod<'_> { source: self.source, types, symbolic_substitutions: ctx.symbolic_substitutions.clone(), + float_literals: self.float_literals, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), errors: RefCell::new(Vec::new()), @@ -92,6 +98,7 @@ struct State<'src> { /// line is fresh output, so a fold inside a parameter annotation is dropped /// unless it is spliced in here symbolic_substitutions: Vec<(TextRange, String)>, + float_literals: FloatLiteralLowering, edits: RefCell>, templates: RefCell)>>, errors: RefCell>, @@ -148,7 +155,14 @@ impl State<'_> { .filter(|(range, _)| ann.range().contains_range(*range)) .cloned() .collect(); - lower_type_expr_full(self.source, self.types, ann, &substitutions).unwrap_or_else(|| { + lower_type_expr_full( + self.source, + self.types, + ann, + &substitutions, + self.float_literals, + ) + .unwrap_or_else(|| { self.source[usize::from(ann.range().start())..usize::from(ann.range().end())].to_owned() }) } diff --git a/crates/by_transforms/src/transforms/just_float.rs b/crates/by_transforms/src/transforms/just_float.rs index 3dc9b7f84f..3324605ef8 100644 --- a/crates/by_transforms/src/transforms/just_float.rs +++ b/crates/by_transforms/src/transforms/just_float.rs @@ -22,6 +22,7 @@ use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::{Expr, ExprName, Stmt}; use ruff_text_size::Ranged; +use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::literal_types::LiteralType; use crate::transforms::type_expr_walker::{ @@ -138,11 +139,12 @@ pub(crate) fn rewrite_type_expr_with_imports( source: &str, types: &dyn TypeInfo, expr: &Expr, + float_literals: FloatLiteralLowering, ) -> Option<(String, Vec)> { let mut all_edits: Vec = Vec::new(); let mut imports: Vec = Vec::new(); - let mut lt = LiteralType::new(source, types); + let mut lt = LiteralType::new(source, types, float_literals); lt.emit_type_edits(expr, true); if lt.needs_literal_import { imports.push("from typing import Literal".to_owned()); diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index 160bbe4074..f740305f32 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -5,6 +5,11 @@ //! `a: 5` → `a: Literal[5]` //! `X[1 | 2]` where X is a type → `X[Literal[1, 2]]` //! +//! A float or complex literal has no `Literal[...]` spelling python's own rules +//! admit, so what it becomes is [`FloatLiteralLowering`]'s to say: the nominal +//! `float` / `complex` by default, or `Literal[1.5]` where the project would +//! rather keep the precision than keep the output checkable. +//! //! Type-expression context is determined structurally (annotations, function //! return types) or via `TypeInfo` lookup (subscript slices where the value //! resolves to a class, type alias, or imported/unknown name). @@ -13,6 +18,7 @@ use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::{Expr, ExprSubscript, Operator, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use crate::config::FloatLiteralLowering; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping, @@ -22,15 +28,21 @@ use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct LiteralType<'src> { source: &'src str, types: &'src dyn TypeInfo, + float_literals: FloatLiteralLowering, pub(crate) edits: Vec, pub(crate) needs_literal_import: bool, } impl<'src> LiteralType<'src> { - pub(crate) fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { + pub(crate) fn new( + source: &'src str, + types: &'src dyn TypeInfo, + float_literals: FloatLiteralLowering, + ) -> Self { Self { source, types, + float_literals, edits: Vec::new(), needs_literal_import: false, } @@ -73,11 +85,15 @@ impl<'src> LiteralType<'src> { return None; } - if is_literal_expr(expr) { + if is_literal_expr(expr, self.float_literals) { self.needs_literal_import = true; return Some(format!("Literal[{}]", self.src(expr.range()))); } + if let Some(nominal) = nominal_float_type(expr, self.float_literals) { + return Some(nominal.to_owned()); + } + if let Expr::BinOp(b) = expr { if matches!(b.op, Operator::BitOr) { return self.transform_union(expr); @@ -113,7 +129,7 @@ impl<'src> LiteralType<'src> { } else { pending_none = true; } - } else if is_literal_expr(p) { + } else if is_literal_expr(p, self.float_literals) { let s = self.src(p.range()).to_owned(); if pending_none { pending_none = false; @@ -247,7 +263,7 @@ impl<'src> LiteralType<'src> { if matches!(expr, Expr::NoneLiteral(_)) { return; } - if is_literal_expr(expr) { + if is_literal_expr(expr, self.float_literals) { self.needs_literal_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( format!("Literal[{}]", self.src(expr.range())), @@ -255,6 +271,13 @@ impl<'src> LiteralType<'src> { ))); return; } + if let Some(nominal) = nominal_float_type(expr, self.float_literals) { + self.edits.push(Fix::safe_edit(Edit::range_replacement( + nominal.to_owned(), + expr.range(), + ))); + return; + } if let Expr::BinOp(b) = expr { if matches!(b.op, Operator::BitOr) { self.emit_union_group_edits(expr); @@ -313,8 +336,11 @@ impl<'src> LiteralType<'src> { /// Each edit covers only `first_literal.start..last_literal.end`, so /// non-literal name nodes between groups are left at their original ranges. fn emit_union_group_edits(&mut self, union_expr: &Expr) { + let float_literals = self.float_literals; let parts = flatten_union(union_expr); - if !parts.iter().any(|p| is_literal_expr(p)) { + if !parts.iter().any(|p| { + is_literal_expr(p, float_literals) || nominal_float_type(p, float_literals).is_some() + }) { return; } @@ -345,7 +371,7 @@ impl<'src> LiteralType<'src> { } else { pending_none_start = Some(p.range().start()); } - } else if is_literal_expr(p) { + } else if is_literal_expr(p, float_literals) { if group_start.is_none() { if let Some(pn) = pending_none_start.take() { group_start = Some(pn); @@ -371,17 +397,21 @@ impl<'src> LiteralType<'src> { pub(crate) struct LiteralTypePass<'src> { source: &'src str, + float_literals: FloatLiteralLowering, } impl<'src> LiteralTypePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, float_literals: FloatLiteralLowering) -> Self { + Self { + source, + float_literals, + } } } impl TypeAwarePass for LiteralTypePass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = LiteralType::new(self.source, types); + let mut inner = LiteralType::new(self.source, types, self.float_literals); walk_type_positions_skipping(stmts, Some(types), &ctx.claimed_type_op_ranges, &mut inner); if inner.needs_literal_import && !literal_already_imported(types) { ctx.required_imports @@ -409,15 +439,15 @@ impl TypeExprVisitor for LiteralType<'_> { } } -fn is_literal_expr(expr: &Expr) -> bool { +fn is_literal_expr(expr: &Expr, float_literals: FloatLiteralLowering) -> bool { + // a float or complex literal is a basedpython-only literal type, and PEP 586 + // does not admit one into `Literal[...]`. wrapping it anyway is what + // `FloatLiteralLowering::Literal` asks for; otherwise it is not a literal as + // far as this pass is concerned and `nominal_float_type` names it instead + let literal_floats = float_literals == FloatLiteralLowering::Literal; match expr { - // float/complex literals are basedpython-only literal types — Python's - // `Literal[...]` rejects them per PEP 586. leave bare so the output is - // valid Python (`a: 1.5` becomes `a: 1.5` would also be invalid in - // python; we don't currently rewrite to `float`, but skipping the - // `Literal[]` wrap is at least no worse than wrapping) Expr::NumberLiteral(n) => { - matches!(n.value, ruff_python_ast::Number::Int(_)) + literal_floats || matches!(n.value, ruff_python_ast::Number::Int(_)) } Expr::StringLiteral(_) | Expr::BooleanLiteral(_) @@ -427,13 +457,40 @@ fn is_literal_expr(expr: &Expr) -> bool { matches!(u.op, UnaryOp::USub | UnaryOp::UAdd) && matches!( u.operand.as_ref(), - Expr::NumberLiteral(n) if matches!(n.value, ruff_python_ast::Number::Int(_)) + Expr::NumberLiteral(n) + if literal_floats || matches!(n.value, ruff_python_ast::Number::Int(_)) ) } _ => false, } } +/// the builtin a float or complex literal type is one of, when the project +/// spells such a literal with its nominal type rather than with `Literal[...]`. +/// +/// a signed literal is the same type as the literal it negates, so `-1.5` is a +/// `float` just as `1.5` is +fn nominal_float_type(expr: &Expr, float_literals: FloatLiteralLowering) -> Option<&'static str> { + if float_literals != FloatLiteralLowering::Nominal { + return None; + } + let number = match expr { + Expr::NumberLiteral(n) => &n.value, + Expr::UnaryOp(u) if matches!(u.op, UnaryOp::USub | UnaryOp::UAdd) => { + match u.operand.as_ref() { + Expr::NumberLiteral(n) => &n.value, + _ => return None, + } + } + _ => return None, + }; + match number { + ruff_python_ast::Number::Float(_) => Some("float"), + ruff_python_ast::Number::Complex { .. } => Some("complex"), + ruff_python_ast::Number::Int(_) => None, + } +} + fn flatten_union(expr: &Expr) -> Vec<&Expr> { let mut parts = Vec::new(); flatten_into(expr, &mut parts); @@ -466,13 +523,19 @@ pub(crate) fn literal_already_imported(types: &dyn TypeInfo) -> bool { /// uses the original source text for sub-expressions, so callers must not /// then apply incompatible edits (like name renames) on top of overlapping /// ranges. -pub(crate) fn rewrite_type_expr(source: &str, types: &dyn TypeInfo, expr: &Expr) -> Option { - let mut t = LiteralType::new(source, types); +pub(crate) fn rewrite_type_expr( + source: &str, + types: &dyn TypeInfo, + expr: &Expr, + float_literals: FloatLiteralLowering, +) -> Option { + let mut t = LiteralType::new(source, types, float_literals); t.transform_type_expr(expr, true) } #[cfg(test)] mod tests { + use crate::config::FloatLiteralLowering; use crate::python_passthrough::unchanged; use crate::{Config, transpile}; use indoc::indoc; @@ -484,6 +547,77 @@ mod tests { ); } + /// `check`, for a project that keeps a float literal inside `Literal[...]` + /// rather than widening it to the type it is one of + fn check_literal_floats(input: &str, expected: &str) { + let config = Config { + float_literals: FloatLiteralLowering::Literal, + ..Config::test_default() + }; + assert_eq!( + transpile(input, &config).unwrap(), + crate::python_passthrough::lazify_expected(expected) + ); + } + + // ------------------------------------------------------------------------- + // Float and complex literal types, which PEP 586 has no `Literal[...]` for. + // ------------------------------------------------------------------------- + + #[test] + fn a_float_literal_is_the_type_it_is_one_of() { + check("a: 3.5\n", "a: float\n"); + check("a: 2j\n", "a: complex\n"); + check("a: -1.5\n", "a: float\n"); + } + + /// left bare, `int | 3.5` is a `TypeError` the moment the annotation is + /// evaluated — `type.__or__` has nothing to do with a float + #[test] + fn a_float_arm_of_a_union_is_lowered_too() { + check("a: int | 3.5\n", "a: int | float\n"); + check("a: int | 2j\n", "a: int | complex\n"); + check("a: list[3.5]\n", "a: list[float]\n"); + } + + #[test] + fn a_project_may_keep_the_float_literal_instead() { + check_literal_floats( + "a: 3.5\n", + indoc! {" + from typing import Literal + a: Literal[3.5] + "}, + ); + check_literal_floats( + "a: int | 3.5\n", + indoc! {" + from typing import Literal + a: int | Literal[3.5] + "}, + ); + } + + /// an int literal has a `Literal[...]` spelling of its own, so neither + /// setting moves it + #[test] + fn an_int_literal_is_unaffected_by_the_setting() { + check( + "a: int | 5\n", + indoc! {" + from typing import Literal + a: int | Literal[5] + "}, + ); + check_literal_floats( + "a: int | 5\n", + indoc! {" + from typing import Literal + a: int | Literal[5] + "}, + ); + } + // ------------------------------------------------------------------------- // Basic literal unions — the core feature. // ------------------------------------------------------------------------- diff --git a/crates/by_transforms/src/transforms/main_function.rs b/crates/by_transforms/src/transforms/main_function.rs index e4e6d0ccd8..7a0b83ec08 100644 --- a/crates/by_transforms/src/transforms/main_function.rs +++ b/crates/by_transforms/src/transforms/main_function.rs @@ -15,12 +15,10 @@ //! existing `__main__` guard or a bare top-level `main()` call — so the entry //! point never runs twice -use std::fmt::Write as _; - use ruff_python_ast::{self as ast, CmpOp, Expr, ModModule, Parameters, Stmt, StmtFunctionDef}; use super::ast_driver::{AstPass, PassContext}; -use super::source_util::is_synthetic_decorator; +use super::source_util::{is_synthetic_decorator, python_string_literal}; /// Parses `sys.argv` into `main`'s parameters. Driven by a spec of /// `(name, converter, kind, required)` tuples emitted from the signature, so @@ -368,32 +366,6 @@ fn trailing_name(expr: &Expr) -> Option<&str> { } } -/// `value` as a python string literal. -/// -/// Rust's own debug spelling is not python: it escapes a control character as -/// `\u{7f}`, which python does not read. Everything printable is written as -/// itself, non-ascii included — the emitted file is the utf-8 python reads by -/// default. -fn python_string_literal(value: &str) -> String { - let mut out = String::with_capacity(value.len() + 2); - out.push('"'); - for character in value.chars() { - match character { - '"' => out.push_str("\\\""), - '\\' => out.push_str("\\\\"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - control if control.is_control() => { - let _ = write!(out, "\\x{:02x}", control as u32); - } - other => out.push(other), - } - } - out.push('"'); - out -} - /// the operands of a `|` union, flattened — `a | b | c` nests to the left fn union_operands(annotation: &Expr) -> Vec<&Expr> { fn walk<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index cd1af5416f..0f9bbba945 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod annotation; pub(crate) mod anon_named_tuple; pub(crate) mod ast_driver; pub(crate) mod auto_quote; +pub(crate) mod build_stamps; pub(crate) mod callable; pub(crate) mod character_type; pub(crate) mod checked_cast; @@ -50,6 +51,7 @@ pub(crate) mod main_function; pub(crate) mod match_polyfill; pub(crate) mod match_type; pub(crate) mod modifiers; +pub(crate) mod module_api; pub(crate) mod mutable_defaults; pub(crate) mod none_chain; pub(crate) mod not_type; @@ -72,6 +74,7 @@ pub(crate) mod some_ctor; pub(crate) mod soundness; pub(crate) mod source_util; pub(crate) mod statement_expression; +pub(crate) mod static_resource; pub(crate) mod string_tag; pub(crate) mod super_keyword; pub(crate) mod symbolic_type_op; diff --git a/crates/by_transforms/src/transforms/module_api.rs b/crates/by_transforms/src/transforms/module_api.rs new file mode 100644 index 0000000000..6979188448 --- /dev/null +++ b/crates/by_transforms/src/transforms/module_api.rs @@ -0,0 +1,222 @@ +//! AST pass that erases `implements` declarations. +//! +//! ```by +//! implements Backend +//! +//! def connect(url: str) -> str: ... +//! ``` +//! +//! lowers to +//! +//! ```python +//! def connect(url: str) -> str: ... +//! ``` +//! +//! the declaration is checked entirely by ty and has no runtime meaning, so +//! nothing is left behind. The import that named the interface stays: it may be +//! needed by annotations, and removing it would change the emitted module's own +//! surface. +//! +//! A declaration that is not at module level has nothing to attach to, and ty +//! reports it as an error of its own. It is erased all the same: leaving it in +//! would emit a call to a name that does not exist at runtime, which is a worse +//! failure than the one the author already has to fix. + +use ruff_python_ast::helpers::implements_declaration; +use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt}; +use ruff_python_ast::{ModModule, Stmt}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use super::ast_driver::{AstPass, PassContext}; + +pub(crate) struct ModuleApiPass<'a> { + source: &'a str, +} + +impl<'a> ModuleApiPass<'a> { + pub(crate) fn new(source: &'a str) -> Self { + Self { source } + } +} + +impl AstPass for ModuleApiPass<'_> { + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { + collect(&module.body, self.source, ctx); + } +} + +/// Collects the erasures for one body, and for every body nested inside it. +fn collect(body: &[Stmt], source: &str, ctx: &mut PassContext) { + // erasing every statement of a body would leave a block with nothing under + // it, which does not parse — the first erasure keeps the block alive by + // leaving a `pass` behind instead + let mut keep_block_alive = !body.is_empty() + && body + .iter() + .all(|stmt| implements_declaration(stmt).is_some()); + + for stmt in body { + if implements_declaration(stmt).is_some() { + let range = erased_range(stmt, source); + let replacement = if keep_block_alive { + keep_block_alive = false; + let indent = usize::from(stmt.range().start() - range.start()); + format!("{}pass\n", " ".repeat(indent)) + } else { + String::new() + }; + ctx.text_edits.push((range, replacement)); + } else { + let mut nested = Nested { source, ctx }; + walk_stmt(&mut nested, stmt); + } + } +} + +/// Walks into the bodies a statement carries, so a declaration written inside one +/// is erased too. +struct Nested<'a, 'ctx> { + source: &'a str, + ctx: &'ctx mut PassContext, +} + +impl<'a> StatementVisitor<'a> for Nested<'_, '_> { + fn visit_body(&mut self, body: &'a [Stmt]) { + collect(body, self.source, self.ctx); + for stmt in body { + if implements_declaration(stmt).is_none() { + walk_stmt(self, stmt); + } + } + } +} + +/// The range to delete: the declaration's whole line, terminator included, and +/// the blank lines that followed it. +/// +/// From the line start rather than the statement start, because the declaration's +/// own indentation is not part of its node range and leaving it behind would +/// prepend it to whatever line follows. Through the blank lines after it because a +/// declaration is written with a gap under it, and leaving the gap behind opens +/// the emitted module with an empty line. +fn erased_range(stmt: &Stmt, source: &str) -> TextRange { + let start = usize::from(stmt.range().start()); + let line_start = source[..start].rfind('\n').map_or(0, |newline| newline + 1); + let end = usize::from(stmt.range().end()); + let mut end = match source[end..].find('\n') { + Some(newline) => end + newline + 1, + None => source.len(), + }; + // a declaration is only ever at module level, so what follows is at column + // zero too and there is no block for a swallowed blank line to close + while let Some(newline) = source[end..].find('\n') { + if !source[end..end + newline].trim().is_empty() { + break; + } + end += newline + 1; + } + TextRange::new( + TextSize::try_from(line_start).unwrap_or_default(), + TextSize::try_from(end).unwrap_or_default(), + ) +} + +#[cfg(test)] +mod tests { + use crate::{Config, transpile}; + use indoc::indoc; + + fn check(input: &str, expected: &str) { + assert_eq!( + transpile(input, &Config::test_default()).unwrap(), + crate::python_passthrough::lazify_expected(expected) + ); + } + + #[test] + fn a_declaration_is_erased() { + check( + indoc! {" + implements Backend + + def connect(url: str) -> str: + return url + "}, + indoc! {" + def connect(url: str) -> str: + return url + "}, + ); + } + + #[test] + fn a_rule_is_erased() { + check( + indoc! {" + implements Backend, Migratable for \".*\", \"!.base\" + + x = 1 + "}, + indoc! {" + x = 1 + "}, + ); + } + + #[test] + fn a_declaration_inside_a_body_is_erased_too() { + // ty reports it; emitting a call to a name that does not exist at runtime + // would be a second, worse problem + check( + indoc! {" + def f(): + implements Backend + "}, + indoc! {" + def f(): + pass + "}, + ); + } + + #[test] + fn a_trailing_comment_goes_with_the_declaration() { + check( + indoc! {" + implements Backend # the plugin interface + + x = 1 + "}, + indoc! {" + x = 1 + "}, + ); + } + + #[test] + fn consecutive_declarations_are_erased() { + check( + indoc! {" + implements Backend + implements Migratable + x = 1 + "}, + indoc! {" + x = 1 + "}, + ); + } + + #[test] + fn a_declaration_at_the_end_of_a_file_is_erased() { + check( + indoc! {" + x = 1 + implements Backend + "}, + indoc! {" + x = 1 + "}, + ); + } +} diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index e4e5c5786b..a9f11b5784 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -39,6 +39,7 @@ use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::{Expr, ExprProtocolType, Stmt}; use ruff_text_size::{Ranged, TextRange}; +use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; use super::ast_driver::{PassContext, TypeAwarePass}; @@ -107,10 +108,15 @@ struct ProtocolTypeLowering<'src> { } impl<'src> ProtocolTypeLowering<'src> { - fn new(source: &'src str, types: &'src dyn TypeInfo, claimed: &'src [TextRange]) -> Self { + fn new( + source: &'src str, + types: &'src dyn TypeInfo, + claimed: &'src [TextRange], + float_literals: FloatLiteralLowering, + ) -> Self { Self { source, - callable: CallableSyntax::new(source) + callable: CallableSyntax::new(source, float_literals) .with_types(types) .with_claimed_ranges(claimed), edits: Vec::new(), @@ -377,7 +383,7 @@ fn lower<'src>( stmts: &[Stmt], config: &crate::Config, ) -> ProtocolTypeLowering<'src> { - let mut inner = ProtocolTypeLowering::new(source, types, claimed); + let mut inner = ProtocolTypeLowering::new(source, types, claimed, config.float_literals); { let mut walker = TypevarScopeWalker { config: config.clone(), diff --git a/crates/by_transforms/src/transforms/source_util.rs b/crates/by_transforms/src/transforms/source_util.rs index cd06f47b27..0c933c8774 100644 --- a/crates/by_transforms/src/transforms/source_util.rs +++ b/crates/by_transforms/src/transforms/source_util.rs @@ -1,4 +1,5 @@ use std::fmt::Display; +use std::fmt::Write as _; use super::ast_driver::Fragment; @@ -301,6 +302,32 @@ pub(crate) fn binding_start( .map_or(last.range().end(), |token| token.range.start()) } +/// `value` as a python string literal. +/// +/// Rust's own debug spelling is not python: it escapes a control character as +/// `\u{7f}`, which python does not read. Everything printable is written as +/// itself, non-ascii included — the emitted file is the utf-8 python reads by +/// default. +pub(crate) fn python_string_literal(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for character in value.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + control if control.is_control() => { + let _ = write!(out, "\\x{:02x}", control as u32); + } + other => out.push(other), + } + } + out.push('"'); + out +} + /// Invoke `on_ann` on every annotation expression reachable from `stmt`. /// Covers `AnnAssign` targets, `TypeAlias` RHS, function parameter /// annotations (regular, vararg, kwarg), return annotations, and recurses diff --git a/crates/by_transforms/src/transforms/static_resource.rs b/crates/by_transforms/src/transforms/static_resource.rs new file mode 100644 index 0000000000..2f555f7cbf --- /dev/null +++ b/crates/by_transforms/src/transforms/static_resource.rs @@ -0,0 +1,414 @@ +//! Lowers a static resource import into the python its document stands for. +//! +//! `import "data/config.yaml" as config` has no python spelling: python imports +//! modules, and the file is not one. What python does have is the document +//! itself, written out — a mapping as a class, a sequence as a tuple, a scalar +//! as a `Final` literal — and that is what the import becomes: +//! +//! ```python +//! class config: +//! class a: +//! b: Final = (1, 2) +//! ``` +//! +//! The rendering is the one `by_resource` produces, which is also the one the +//! type checker infers, so what the program gets is what the checker described. +//! +//! # Why the document is written into the importing module +//! +//! Rather than into a module of its own that importers share. A resource is +//! read at build time and has no runtime existence, so a file that imports one +//! is complete on its own: `by transpile` on a single file emits python that +//! runs, with no second file to place beside it and no import edge that the +//! source did not have. +//! +//! Two modules importing one resource therefore get two objects rather than +//! one. Nothing about a value read through attributes notices, except identity: +//! `a.config.x is b.config.x` is false where a shared module would make it true. +//! +//! The class this emits is named after the binding rather than after the file, +//! so the reader of the output sees the name the import gave it. The checker +//! names the same class after the file — a resource file is rendered once, and +//! a name in it cannot depend on who imported it — which is visible only in +//! `__name__`. +//! +//! # Why there is no reverse transform +//! +//! Every other lowering here has one, so that python read back as basedpython +//! comes back as the idiom it started from. This one cannot: the python it +//! emits is a tree of classes holding constants, which is also what a +//! hand-written tree of classes holding constants looks like. Recognising one +//! as a document would mean claiming a file the source never mentions, and +//! guessing which of json, toml and yaml it had been. A resource import is +//! written by hand, and read back as the classes it became. + +use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::{Alias, ModModule, Stmt, StmtImport}; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use thin_vec::ThinVec; + +use super::ast_driver::{AstPass, PassContext}; +use crate::type_info::TypeInfo; + +/// What each static resource import in a file lowers to. +#[derive(Default)] +pub(crate) struct ResourceLowerings { + /// `(the import statement's range, the python that replaces it)`. + /// + /// The python is written at module level; the statement it replaces may be + /// indented, and [`indented`] puts it where the statement was. + replacements: Vec<(TextRange, String)>, + /// documents that could not be read; the checker reports these too, and the + /// transpile has nothing to emit for them + errors: Vec, +} + +/// Lower every static resource import in `stmts`. +/// +/// `stmts` must come from the same parse `types` answers for: the lowering is +/// keyed by source range, and a range from another parse names other text. +pub(crate) fn collect(stmts: &[Stmt], types: &dyn TypeInfo) -> ResourceLowerings { + let mut collector = Collector { + types, + lowerings: ResourceLowerings::default(), + }; + for stmt in stmts { + collector.visit_stmt(stmt); + } + collector.lowerings +} + +struct Collector<'a> { + types: &'a dyn TypeInfo, + lowerings: ResourceLowerings, +} + +impl Visitor<'_> for Collector<'_> { + fn visit_stmt(&mut self, stmt: &Stmt) { + if let Stmt::Import(import) = stmt + && import.names.iter().any(|alias| alias.is_resource) + { + self.lower(import); + return; + } + walk_stmt(self, stmt); + } +} + +impl Collector<'_> { + fn lower(&mut self, import: &StmtImport) { + // the parser rejects a statement that mixes a resource with a module, so + // every alias here names a resource + let mut lowered = String::new(); + + for alias in &import.names { + let Some(binding) = binding_of(alias) else { + // no `as` clause: the parser has already said so, and there is + // no name to bind the document to + return; + }; + match self.types.static_resource(&alias.name.id, binding) { + Ok(rendered) => lowered.push_str(&rendered), + Err(message) => { + self.lowerings.errors.push(format!( + "cannot read static resource `{path}`: {message}", + path = alias.name.id + )); + return; + } + } + } + + self.lowerings.replacements.push((import.range(), lowered)); + } +} + +/// The name an alias binds, which for a resource is always its `as` clause. +fn binding_of(alias: &Alias) -> Option<&str> { + Some(alias.asname.as_ref()?.id.as_str()) +} + +/// The whitespace at the start of the line `offset` is on. +fn indentation_of(source: &str, offset: TextSize) -> String { + let line_start = source[..usize::from(offset)] + .rfind('\n') + .map_or(0, |index| index + 1); + source[line_start..usize::from(offset)] + .chars() + .take_while(|character| character.is_whitespace()) + .collect() +} + +/// `source` with every non-empty line indented by `indent`. +/// +/// The first line is left alone: it takes the place of the statement, which is +/// already at that indentation. +fn indented(source: &str, indent: &str) -> String { + if indent.is_empty() { + return source.to_string(); + } + let mut indented = String::with_capacity(source.len()); + for (index, line) in source.lines().enumerate() { + if index > 0 { + indented.push_str(indent); + } + indented.push_str(line); + indented.push('\n'); + } + indented +} + +/// Replaces each static resource import with the document it names. +/// +/// The import goes from the source text *and* from the AST, because either one +/// alone leaves the other standing. A statement no pass mutated keeps its source +/// bytes, and ignores the AST; a statement some pass did mutate is re-rendered +/// from the AST, and the text edit inside it is dropped. An `import` statement +/// that survived to the output is not python at all — a path is not a module +/// name — so phase 3 rejects the whole file. +/// +/// The AST rewrite deliberately does not declare the statement changed. Saying +/// so would re-render every enclosing `def` through the code generator, losing +/// the comments and the formatting of everything else in it; leaving it undeclared +/// means the rewrite matters only when something else already forced a re-render. +pub(crate) struct StaticResource { + lowerings: ResourceLowerings, + source: String, +} + +impl StaticResource { + pub(crate) fn new(lowerings: ResourceLowerings, source: &str) -> Self { + Self { + lowerings, + source: source.to_string(), + } + } + + fn rendering_for(&self, range: TextRange) -> Option<&str> { + self.lowerings + .replacements + .iter() + .find_map(|(replaced, rendering)| (*replaced == range).then_some(rendering.as_str())) + } + + /// Replace every lowered import in `body` with the statements it renders to. + fn rewrite_body(&self, body: &mut ThinVec) { + let mut index = 0; + while index < body.len() { + let rendered = match &body[index] { + Stmt::Import(import) if import.names.iter().any(|alias| alias.is_resource) => { + self.rendering_for(import.range()).and_then(|rendering| { + let parsed = ruff_python_parser::parse_module(rendering).ok()?; + Some(parsed.into_syntax().body) + }) + } + _ => None, + }; + + if let Some(rendered) = rendered { + let count = rendered.len(); + body.splice(index..=index, rendered); + index += count; + continue; + } + + self.rewrite_within(&mut body[index]); + index += 1; + } + } + + /// Rewrite the bodies `stmt` holds, if it holds any. + fn rewrite_within(&self, stmt: &mut Stmt) { + match stmt { + Stmt::FunctionDef(node) => self.rewrite_body(&mut node.body), + Stmt::ClassDef(node) => self.rewrite_body(&mut node.body), + Stmt::For(node) => { + self.rewrite_body(&mut node.body); + self.rewrite_body(&mut node.orelse); + } + Stmt::While(node) => { + self.rewrite_body(&mut node.body); + self.rewrite_body(&mut node.orelse); + } + Stmt::If(node) => { + self.rewrite_body(&mut node.body); + for clause in &mut node.elif_else_clauses { + self.rewrite_body(&mut clause.body); + } + } + Stmt::With(node) => self.rewrite_body(&mut node.body), + Stmt::Match(node) => { + for case in &mut node.cases { + self.rewrite_body(&mut case.body); + } + } + Stmt::Try(node) => { + self.rewrite_body(&mut node.body); + for handler in &mut node.handlers { + let ruff_python_ast::ExceptHandler::ExceptHandler(handler) = handler; + self.rewrite_body(&mut handler.body); + } + self.rewrite_body(&mut node.orelse); + self.rewrite_body(&mut node.finalbody); + } + // every other statement holds expressions, and an import is not one + _ => {} + } + } +} + +impl AstPass for StaticResource { + fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { + for error in &self.lowerings.errors { + ctx.errors.push(error.clone()); + } + if self.lowerings.replacements.is_empty() { + return; + } + + ctx.required_imports + .push(by_resource::REQUIRED_IMPORT.to_string()); + for (range, rendering) in &self.lowerings.replacements { + let indent = indentation_of(&self.source, range.start()); + // the rendering ends in a newline, and it replaces a statement the + // splice will follow with one of its own + let replacement = indented(rendering, &indent) + .trim_end_matches('\n') + .to_string(); + ctx.text_edits.push((*range, replacement)); + } + + self.rewrite_body(&mut module.body); + } +} + +#[cfg(test)] +mod tests { + use ruff_db::files::system_path_to_file; + use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; + use ty_project::{ProjectMetadata, TestDb}; + + use super::*; + use crate::{Config, transpile_typed}; + + /// transpile `/proj/main.by`, with `files` written around it. + fn transpile(files: &[(&str, &str)]) -> Result { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/proj"), + )); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + db.init_program().expect("program init failed"); + let file = system_path_to_file(&db, "/proj/main.by").expect("file not in db"); + transpile_typed(&db, file, &Config::test_default(), None).map_err(|error| error.to_string()) + } + + #[test] + fn a_document_is_written_into_the_module_that_imports_it() { + let output = transpile(&[ + ("/proj/data/config.yaml", "a:\n b:\n - 1\n - 2\n"), + ( + "/proj/main.by", + "import \"data/config.yaml\" as config\n\nprint(config.a.b[1])\n", + ), + ]) + .expect("transpile should succeed"); + + assert!(output.contains("from typing import Final"), "{output}"); + assert!( + output.contains("class config:\n class a:\n b: Final = (1, 2)"), + "{output}" + ); + assert!(output.contains("print(config.a.b[1])"), "{output}"); + } + + #[test] + fn an_import_inside_a_function_is_indented_to_match() { + let output = transpile(&[ + ("/proj/data/config.json", r#"{"a": 1}"#), + ( + "/proj/main.by", + "def f():\n import \"data/config.json\" as config\n return config.a\n", + ), + ]) + .expect("transpile should succeed"); + + assert!( + output.contains(" class config:\n a: Final = 1"), + "{output}" + ); + } + + /// an ast-mutating pass re-renders the whole top-level statement it touched, + /// through the code generator rather than from source. a resource import + /// inside one is re-spelled by the generator, and a path written as a name + /// is not python + #[test] + fn an_import_inside_a_statement_another_pass_rewrites_survives() { + let output = transpile(&[ + ("/proj/data/config.json", r#"{"a": 1}"#), + ( + "/proj/main.by", + "def f(_, _):\n import \"data/config.json\" as config\n return config.a\n", + ), + ]) + .expect("transpile should succeed"); + + assert!(output.contains("def f(_, _2):"), "{output}"); + assert!(output.contains("class config:"), "{output}"); + assert!(!output.contains("import data/config.json"), "{output}"); + } + + /// the statements spliced into the AST are parsed from the rendering and + /// carry its ranges, which name nothing in this file. a pass reading one as + /// a range in the source slices out of bounds + #[test] + fn a_rewritten_statement_does_not_poison_the_passes_around_it() { + let output = transpile(&[ + ("/proj/data/config.json", r#"{"a": 1}"#), + ( + "/proj/main.by", + "let value = 1\n\n\ndef f(_, _):\n import \"data/config.json\" as config\n return config.a\n", + ), + ]) + .expect("transpile should succeed"); + + assert!(output.contains("value: Final = 1"), "{output}"); + assert!(output.contains("class config:"), "{output}"); + } + + #[test] + fn a_document_that_cannot_be_read_fails_the_transpile() { + let error = transpile(&[ + ("/proj/data/broken.json", "{ \"a\": }"), + ("/proj/main.by", "import \"data/broken.json\" as config\n"), + ]) + .expect_err("a document that is not json has nothing to emit"); + + assert!(error.contains("data/broken.json"), "{error}"); + } + + #[test] + fn a_path_that_names_nothing_fails_the_transpile() { + let error = transpile(&[("/proj/main.by", "import \"data/gone.json\" as config\n")]) + .expect_err("a path that names nothing has nothing to emit"); + + assert!(error.contains("data/gone.json"), "{error}"); + } + + #[test] + fn indentation() { + assert_eq!(indentation_of("import x", 0.into()), ""); + assert_eq!(indentation_of("def f():\n import x", 13.into()), " "); + } + + #[test] + fn indenting_leaves_the_first_line_alone() { + assert_eq!( + indented("class a:\n b: Final = 1\n", " "), + "class a:\n b: Final = 1\n" + ); + } +} diff --git a/crates/by_transforms/src/transforms/template_type.rs b/crates/by_transforms/src/transforms/template_type.rs index 5e5a7fce68..38fb28267b 100644 --- a/crates/by_transforms/src/transforms/template_type.rs +++ b/crates/by_transforms/src/transforms/template_type.rs @@ -180,4 +180,31 @@ mod tests { "}, ); } + + #[test] + fn a_type_alias_hole_folds() { + check( + "type Name = \"foo\" | \"bar\"\ntype Title = f\"the {Name}\"\n", + indoc! {" + from typing import Literal + from typing_extensions import TypeAliasType + Name = TypeAliasType(\"Name\", Literal[\"foo\", \"bar\"]) + Title = TypeAliasType(\"Title\", Literal[\"the foo\", \"the bar\"]) + "}, + ); + } + + #[test] + fn an_alias_of_an_alias_hole_folds() { + check( + "type Inner = \"foo\" | \"bar\"\ntype Outer = Inner | \"baz\"\ntype Title = f\"the {Outer}\"\n", + indoc! {" + from typing import Literal + from typing_extensions import TypeAliasType + Inner = TypeAliasType(\"Inner\", Literal[\"foo\", \"bar\"]) + Outer = TypeAliasType(\"Outer\", Inner | Literal[\"baz\"]) + Title = TypeAliasType(\"Title\", Literal[\"the foo\", \"the bar\", \"the baz\"]) + "}, + ); + } } diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 3774cc04b1..cf9df35f79 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -562,6 +562,14 @@ pub(crate) trait TypeInfo { /// soundness check against the field annotation would always fail at /// runtime — the transpiler must not emit one fn is_field_specifier(&self, expr: &Expr) -> bool; + + /// basedpython: the python a static resource import lowers to, bound to + /// `binding` — the document at `path` written out as classes, tuples and + /// `Final` literals — or why the document cannot be read. + /// + /// the rendering is the one the checker infers the import's type from, so + /// asking here is asking the same question the type answered + fn static_resource(&self, path: &str, binding: &str) -> Result; } /// re-export of the ty-side check plan so transforms name a single type @@ -1268,6 +1276,14 @@ impl TypeInfo for SemanticModel<'_> { }) } + fn static_resource(&self, path: &str, binding: &str) -> Result { + let file = ty_python_semantic::resolve_static_resource(self.db(), self.file(), path) + .map_err(|error| error.to_string())?; + ty_python_semantic::render_as(self.db(), file, binding) + .map(|rendered| rendered.source) + .map_err(|error| error.to_string()) + } + fn is_field_specifier(&self, expr: &Expr) -> bool { matches!( expr.inferred_type(self), diff --git a/crates/by_transforms/tests/build_stamps_runtime.rs b/crates/by_transforms/tests/build_stamps_runtime.rs new file mode 100644 index 0000000000..8d53bc483c --- /dev/null +++ b/crates/by_transforms/tests/build_stamps_runtime.rs @@ -0,0 +1,121 @@ +//! Runtime test for `build:` stamps. +//! +//! The block lowers to a class whose members hold literals the build supplied, +//! so what matters is that the emitted python *is* python: a stamp value is +//! text a build system handed over, and text that reached the output unescaped +//! is a `SyntaxError` at import rather than anything `by check` or an mdtest +//! would notice. The awkward values are the point of this test. + +mod common; + +use std::collections::BTreeMap; +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +fn config(stamps: &[(&str, &str)]) -> Config { + Config { + min_version: PythonVersion::PY313, + stamps: stamps + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect::>(), + ..Config::default() + } +} + +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn run(program: &str, stamps: &[(&str, &str)]) { + let Some(python) = common::python() else { + eprintln!("skipping build stamp runtime test: no interpreter found"); + return; + }; + + let transpiled = transpile(program, &config(stamps)).expect("transpile should succeed"); + let output = Command::new(&python) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "stamped program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +fn a_stamp_reaches_the_program_as_its_declared_type() { + run( + r#" +build: + GIT_SHA: str + GIT_DIRTY: bool + BUILD_NUMBER: int + +assert build.GIT_SHA == "e6f9ac1d", build.GIT_SHA +assert build.GIT_DIRTY is True, build.GIT_DIRTY +assert build.BUILD_NUMBER == 417, build.BUILD_NUMBER +print("ok") +"#, + &[ + ("GIT_SHA", "e6f9ac1d"), + ("GIT_DIRTY", "true"), + ("BUILD_NUMBER", "417"), + ], + ); +} + +#[test] +fn a_value_holding_quotes_and_newlines_survives() { + // `git log -1 --format=%s` hands over whatever the commit message says, and + // a subject line with a quote in it is not unusual + run( + r#" +build: + SUBJECT: str + +assert build.SUBJECT == "fix the \"one\" case\nand a second line", repr(build.SUBJECT) +print("ok") +"#, + &[("SUBJECT", "fix the \"one\" case\nand a second line")], + ); +} + +#[test] +fn a_value_that_looks_like_code_is_not_code() { + // whatever a build hands over is data. this one would be an import and a + // call if it were ever spliced in unescaped + run( + r#" +build: + DESCRIBE: str + +assert build.DESCRIBE == "\" + __import__('os').getcwd() + \"", repr(build.DESCRIBE) +print("ok") +"#, + &[("DESCRIBE", "\" + __import__('os').getcwd() + \"")], + ); +} + +#[test] +fn an_unsupplied_stamp_runs_on_its_default() { + run( + r#" +build: + GIT_SHA: str = "unreleased" + DIRTY: bool = False + +assert build.GIT_SHA == "unreleased", build.GIT_SHA +assert build.DIRTY is False, build.DIRTY +print("ok") +"#, + &[], + ); +} diff --git a/crates/by_transforms/tests/static_resource_runtime.rs b/crates/by_transforms/tests/static_resource_runtime.rs new file mode 100644 index 0000000000..fb9c67754c --- /dev/null +++ b/crates/by_transforms/tests/static_resource_runtime.rs @@ -0,0 +1,220 @@ +//! Runtime tests for static resource imports. +//! +//! The point of the feature is that `config.a.b[1]` has a type *and* a value, +//! and that they are the same thing. The mdtests hold up the type half; only a +//! real interpreter holds up the other. Every assertion below is written so +//! that a rendering which type checked but did not run — a tuple missing its +//! trailing comma, a helper class named before it was defined, a string whose +//! escaping did not survive — fails here. + +#![expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use by_transforms::{Config, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; + +mod common; + +/// transpile `/proj/main.by` with `files` written around it, into a fresh +/// directory under the cargo temp dir. +fn build_case(case: &str, files: &[(&str, &str)]) -> PathBuf { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/proj"), + )); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + db.init_program().expect("program init failed"); + + let file = system_path_to_file(&db, "/proj/main.by").expect("file not in db"); + let transpiled = + transpile_typed(&db, file, &Config::default(), None).expect("transpile should succeed"); + + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); + // a stale directory from an earlier run would mask a transpile failure + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create case dir"); + fs::write(dir.join("main.py"), transpiled).expect("write module"); + dir +} + +/// run `main.py` in `dir`, asserting it exits cleanly and prints `ok` +fn run_main(python: &str, dir: &Path) { + let output = Command::new(python) + .arg("main.py") + .current_dir(dir) + .output() + .expect("failed to spawn python"); + + assert!( + output.status.success(), + "transpiled program failed:\n--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +const YAML: &str = "\ +name: ty +port: 8080 +ratio: 0.5 +debug: true +missing: ~ +quote: 'he said \"hi\"' +nested: + deep: + - 1 + - 2 +one: + - 9 +empty_list: [] +empty_map: {} +"; + +const YAML_MAIN: &str = r#" +import "data/config.yaml" as config + +assert config.name == "ty", "a string" +assert config.port == 8080, "an integer" +assert config.ratio == 0.5, "a float" +assert config.debug is True, "a boolean" +assert config.missing is None, "a null" +assert config.quote == 'he said "hi"', "a string holding a quote" +assert config.nested.deep == (1, 2), "a nested sequence" +assert config.nested.deep[1] == 2, "and an index into it" +assert config.one == (9,), "a one-element sequence is still a sequence" +assert config.empty_list == (), "an empty sequence" +assert [n for n in vars(config.empty_map) if not n.startswith("__")] == [], "an empty mapping holds nothing" +print("ok") +"#; + +/// a mapping inside a sequence inside a mapping inside a sequence: every one of +/// those becomes a class defined beside the value that names it, and a class +/// body runs when it is defined, so an ordering mistake is a `NameError` here +const JSON: &str = r#"{ + "servers": [ + { "host": "a", "tags": [{ "name": "x" }] }, + { "host": "b", "tags": [] } + ], + "build-backend": "left out", + "root": "." +}"#; + +const JSON_MAIN: &str = r#" +import "data/config.json" as config + +assert config.servers[0].host == "a", "a mapping in a sequence" +assert config.servers[1].host == "b", "and the one after it" +assert config.servers[0].tags[0].name == "x", "a mapping in a sequence in a mapping in a sequence" +assert config.servers[1].tags == (), "an empty sequence beside a full one" +assert config.root == ".", "a key beside one python cannot name" +assert not hasattr(config, "build-backend"), "a key python cannot name is left out" +print("ok") +"#; + +const TOML: &str = "\ +[server] +host = \"localhost\" +ports = [80, 443] + +[[server.routes]] +path = \"/a\" + +[[server.routes]] +path = \"/b\" +"; + +const TOML_MAIN: &str = r#" +import "data/config.toml" as config + +assert config.server.host == "localhost", "a table" +assert config.server.ports == (80, 443), "an array" +assert config.server.routes[1].path == "/b", "an array of tables" +print("ok") +"#; + +/// the document is read where it is imported, so an import inside a function is +/// a class defined inside that function +const SCOPED_MAIN: &str = r#" +def load() -> str: + import "data/config.json" as config + return config.servers[0].tags[0].name + + +assert load() == "x", "a resource imported inside a function" +print("ok") +"#; + +#[test] +fn a_yaml_document_reads_back_as_it_was_written() { + let Some(python) = common::python() else { + eprintln!("skipping static resource runtime test: no python interpreter found"); + return; + }; + let dir = build_case( + "resource_yaml", + &[ + ("/proj/data/config.yaml", YAML), + ("/proj/main.by", YAML_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_mapping_nested_in_a_sequence_is_defined_before_it_is_named() { + let Some(python) = common::python() else { + eprintln!("skipping static resource runtime test: no python interpreter found"); + return; + }; + let dir = build_case( + "resource_json", + &[ + ("/proj/data/config.json", JSON), + ("/proj/main.by", JSON_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_toml_document_reads_back_as_it_was_written() { + let Some(python) = common::python() else { + eprintln!("skipping static resource runtime test: no python interpreter found"); + return; + }; + let dir = build_case( + "resource_toml", + &[ + ("/proj/data/config.toml", TOML), + ("/proj/main.by", TOML_MAIN), + ], + ); + run_main(&python, &dir); +} + +#[test] +fn a_resource_imported_inside_a_function_runs_there() { + let Some(python) = common::python() else { + eprintln!("skipping static resource runtime test: no python interpreter found"); + return; + }; + let dir = build_case( + "resource_scoped", + &[ + ("/proj/data/config.json", JSON), + ("/proj/main.by", SCOPED_MAIN), + ], + ); + run_main(&python, &dir); +} diff --git a/crates/ruff_db/Cargo.toml b/crates/ruff_db/Cargo.toml index 52d668e9af..ae5037ff34 100644 --- a/crates/ruff_db/Cargo.toml +++ b/crates/ruff_db/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } license = { workspace = true } [dependencies] +by_resource = { workspace = true } ruff_annotate_snippets = { workspace = true } ruff_cache = { workspace = true, optional = true } ruff_diagnostics = { workspace = true } diff --git a/crates/ruff_db/src/lib.rs b/crates/ruff_db/src/lib.rs index 0f3eb8d109..de3b624947 100644 --- a/crates/ruff_db/src/lib.rs +++ b/crates/ruff_db/src/lib.rs @@ -19,6 +19,7 @@ pub mod file_revision; pub mod files; pub mod panic; pub mod parsed; +pub mod resource; pub mod source; pub mod system; #[cfg(feature = "testing")] diff --git a/crates/ruff_db/src/parsed.rs b/crates/ruff_db/src/parsed.rs index 55b7fed03a..c22b9308b5 100644 --- a/crates/ruff_db/src/parsed.rs +++ b/crates/ruff_db/src/parsed.rs @@ -47,7 +47,6 @@ pub(super) fn disable_lru(db: &mut dyn Db) { } fn parsed_module_impl(db: &dyn Db, file: File, target_version: PythonVersion) -> Parsed { - let source = source_text(db, file); let ty = file.source_type(db); let is_basedpython = matches!(file.path(db).extension(), Some("by" | "byi")); @@ -55,6 +54,8 @@ fn parsed_module_impl(db: &dyn Db, file: File, target_version: PythonVersion) -> .with_target_version(target_version) .with_basedpython(is_basedpython); + let source = source_text(db, file); + // Notebooks parse each cell as an independent module so a syntax error confined to one cell is // surfaced instead of being masked by a later cell's content. Regular files take the existing // single-parse path. diff --git a/crates/ruff_db/src/resource.rs b/crates/ruff_db/src/resource.rs new file mode 100644 index 0000000000..d32c8b89a9 --- /dev/null +++ b/crates/ruff_db/src/resource.rs @@ -0,0 +1,131 @@ +//! basedpython static resources: a data file read as the python it stands for. +//! +//! `import "data/config.yaml" as config` imports a json, toml or yaml file as a +//! value. everything that answers a question about that value — what type +//! `config.a.b[1]` has, what the transpiled program binds `config` to — reads it +//! through the python [`by_resource`] renders the document into, so the file is +//! parsed and inferred exactly like a hand-written module. +//! +//! # why the python gets a file of its own +//! +//! The rendering could have been served as the document's own contents, and for +//! a while it was. But a position in the rendering is not a position in the +//! document: the python is longer, and its lines are somewhere else entirely. +//! Everything that takes a range and a file and expects the range to name text +//! in that file — a diagnostic, a hover, a definition an editor is asked to jump +//! to — was then holding two halves of different things, and a range past the +//! end of the document is not a wrong answer but a panic. +//! +//! So the rendering is [a file of its own](resource_module), at a path no +//! document has, whose contents are the python and whose ranges therefore mean +//! what they say. [`source_text`] keeps returning what each file actually holds, +//! which matters because plenty of the toml and json in a project is not a +//! resource at all — `pyproject.toml` is read through it. +//! +//! [`source_text`]: crate::source::source_text + +use by_resource::{Format, Rendered}; + +use crate::Db; +use crate::files::File; +use crate::system::{SystemVirtualPath, SystemVirtualPathBuf}; + +/// the scheme the rendering of a document is filed under. +/// +/// a virtual path, because there is no file on disk holding this python and +/// nothing should go looking for one. the document's own path follows, so the +/// two are one-to-one and either can be found from the other. +const SCHEME: &str = "by-resource:"; + +/// what a file offers when it is read as a static resource. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Resource { + /// the file's extension is not one a static resource is written in, so the + /// file is whatever it already was + NotAResource, + /// the document, as python + Rendered(Rendered), + /// the file is written in a resource format but could not be read as one + Unreadable(String), +} + +/// the format an extension names, if it names one. +pub fn resource_format(extension: Option<&str>) -> Option { + Format::from_extension(extension?) +} + +/// the file holding `document`'s rendering. +/// +/// the same file every time it is asked for, so the semantic index built over +/// it is reused rather than rebuilt. +pub fn resource_module(db: &dyn Db, document: File) -> File { + let path = SystemVirtualPathBuf::from(format!("{SCHEME}{}", document.path(db))); + if let Some(existing) = db.files().try_virtual_file(&path) { + return existing.file(); + } + db.files().virtual_file(db, &path).file() +} + +/// the document a rendering was made from, if `file` is a rendering. +pub fn resource_document(db: &dyn Db, file: File) -> Option { + let crate::files::FilePath::SystemVirtual(path) = file.path(db) else { + return None; + }; + let document = path.as_str().strip_prefix(SCHEME)?; + crate::files::system_path_to_file(db, document).ok() +} + +/// whether `path` names a rendering rather than a file anyone wrote. +pub fn is_resource_module(path: &SystemVirtualPath) -> bool { + path.as_str().starts_with(SCHEME) +} + +/// read `file` as a static resource. +/// +/// the result is the same one the type checker infers and the transpiler emits, +/// so there is no way for the two to be looking at different renderings of the +/// same document. +#[salsa::tracked(returns(ref), heap_size=ruff_memory_usage::heap_size)] +pub fn resource(db: &dyn Db, file: File) -> Resource { + let path = file.path(db); + let Some(format) = resource_format(path.extension()) else { + return Resource::NotAResource; + }; + + let text = crate::source::source_text(db, file); + if let Some(error) = text.read_error() { + return Resource::Unreadable(error.to_string()); + } + + // the name the document is bound to comes from the file name, so the class + // the checker names in a message is the one a reader would guess + let stem = path + .as_str() + .rsplit(['/', '\\']) + .next() + .and_then(|name| name.rsplit_once('.')) + .map_or("resource", |(stem, _)| stem); + + match by_resource::transpile(format, text.as_str(), &by_resource::binding_name(stem)) { + Ok(rendered) => Resource::Rendered(rendered), + Err(error) => Resource::Unreadable(error.to_string()), + } +} + +impl get_size2::GetSize for Resource { + fn get_heap_size(&self) -> usize { + match self { + Resource::NotAResource => 0, + Resource::Rendered(rendered) => { + rendered.source.len() + + rendered.root.len() + + rendered + .unusable_keys + .iter() + .map(|key| key.len() + size_of::()) + .sum::() + } + Resource::Unreadable(message) => message.len(), + } + } +} diff --git a/crates/ruff_db/src/source.rs b/crates/ruff_db/src/source.rs index 511a7e7fa6..d4eb1372c9 100644 --- a/crates/ruff_db/src/source.rs +++ b/crates/ruff_db/src/source.rs @@ -22,6 +22,15 @@ pub fn source_text(db: &dyn Db, file: File) -> SourceText { return source.clone(); } + // basedpython: a static resource's rendering is a file of its own, holding + // the python its document stands for. every other file — the document + // included — reads as what it actually holds. see `crate::resource` + if let FilePath::SystemVirtual(virtual_path) = path + && crate::resource::is_resource_module(virtual_path) + { + return rendered_resource(db, file); + } + let kind = if is_notebook(db.system(), path) { file.read_to_notebook(db) .unwrap_or_else(|error| { @@ -47,6 +56,29 @@ pub fn source_text(db: &dyn Db, file: File) -> SourceText { } } +/// The python a static resource's rendering holds. +/// +/// Empty when the document cannot be read: the import is where that is +/// reported, and an empty module leaves nothing else to say about it. +fn rendered_resource(db: &dyn Db, file: File) -> SourceText { + let source = crate::resource::resource_document(db, file) + .map(|document| crate::resource::resource(db, document)) + .and_then(|resource| match resource { + crate::resource::Resource::Rendered(rendered) => Some(rendered.module_source()), + crate::resource::Resource::Unreadable(_) | crate::resource::Resource::NotAResource => { + None + } + }) + .unwrap_or_default(); + + SourceText { + inner: Arc::new(SourceTextInner { + kind: source.into(), + read_error: None, + }), + } +} + fn is_notebook(system: &dyn System, path: &FilePath) -> bool { let source_type = match path { FilePath::System(path) => system.source_type(path), diff --git a/crates/ruff_linter/resources/test/fixtures/pep8_naming/N806_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pep8_naming/N806_basedpython.by new file mode 100644 index 0000000000..b764689718 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pep8_naming/N806_basedpython.by @@ -0,0 +1,23 @@ +# a bare `case` name that is an `enum class` variant is spelled the way its +# declaration spells it — it names a member, not a variable, so the naming rule +# has nothing to say about it +enum class Colour: + case Red + case Green + + +def enum_case_names(c: Colour) -> int: + match c: + case Red: + return 1 + case Green: + return 2 + + +# a capture that matches no variant is an ordinary variable and still fires +def ordinary_capture(value: object) -> int: + match value: + case Unmatched: # N806 + return 1 + case _: + return 0 diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by index b47938af5c..57e96f2e30 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by @@ -20,3 +20,28 @@ def mixed() -> str: context typed: str = "hello" plain = "dead" # F841 return f() + + +# a bare `case` name that is an `enum class` variant is a reference to that +# member, not the capture python spells it as — so there is nothing for it to +# leave unused +enum class Colour: + case Red + case Green + + +def enum_case_names(c: Colour) -> int: + match c: + case Red: + return 1 + case Green: + return 2 + + +# a name matching no variant is an ordinary capture and still fires +def ordinary_capture(value: object) -> int: + match value: + case Unmatched: # F841 + return 1 + case _: + return 0 diff --git a/crates/ruff_linter/src/checkers/ast/analyze/deferred_scopes.rs b/crates/ruff_linter/src/checkers/ast/analyze/deferred_scopes.rs index b53d8c8022..42a633a917 100644 --- a/crates/ruff_linter/src/checkers/ast/analyze/deferred_scopes.rs +++ b/crates/ruff_linter/src/checkers/ast/analyze/deferred_scopes.rs @@ -164,6 +164,10 @@ pub(crate) fn deferred_scopes(checker: &Checker) { && !binding.is_nonlocal() && !binding.is_global() && !is_context_declaration(binding, checker.semantic()) + // basedpython: a bare `case Red:` that names an + // `enum class` variant binds nothing, so there is + // nothing for it to leave unused + && !binding.is_based_enum_case_name() && !checker.settings().ignores_unused_binding(name) && !matches!( name, diff --git a/crates/ruff_linter/src/checkers/ast/analyze/pattern.rs b/crates/ruff_linter/src/checkers/ast/analyze/pattern.rs index f0d25f9b6d..0e00eb0f9d 100644 --- a/crates/ruff_linter/src/checkers/ast/analyze/pattern.rs +++ b/crates/ruff_linter/src/checkers/ast/analyze/pattern.rs @@ -25,6 +25,13 @@ pub(crate) fn pattern(pattern: &Pattern, checker: &Checker) { } fn check_pattern_name(checker: &Checker, name: &Identifier) { + // basedpython: a bare `case Red:` naming an `enum class` variant is a + // reference to that member, not a variable — so the naming rules, which are + // about what a *variable* should be called, have nothing to say about it. + // `Red` is spelled the way its declaration spells it, which is the point + if checker.semantic().is_based_enum_case_name(name.as_str()) { + return; + } if checker.is_rule_enabled(Rule::NonLowercaseVariableInFunction) && checker.semantic().current_scope().kind.is_function() { diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index b199c7db2d..ebcaf04679 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -2521,12 +2521,19 @@ impl<'a> Visitor<'a> for Checker<'a> { rest: Some(name), .. }) = pattern { - self.add_binding( - name, - name.range(), - BindingKind::Assignment, - BindingFlags::empty(), - ); + // basedpython: a bare `case Red:` whose name is an `enum class` + // variant is a *reference* to that member, not the capture python + // spells it as. Whether this subject's type really admits it takes the + // type, which the linter does not have — so the same split the rest of + // `is_basedpython_transpile_resolved_name` makes applies: a name that + // matches no variant at all stays an ordinary capture, and ty reports + // one whose subject does not accept it + let flags = if self.semantic.is_based_enum_case_name(name.as_str()) { + BindingFlags::BASED_ENUM_CASE_NAME + } else { + BindingFlags::empty() + }; + self.add_binding(name, name.range(), BindingKind::Assignment, flags); } // Step 2: Traversal diff --git a/crates/ruff_linter/src/rules/isort/block.rs b/crates/ruff_linter/src/rules/isort/block.rs index 1dfe96dc51..c984c7d598 100644 --- a/crates/ruff_linter/src/rules/isort/block.rs +++ b/crates/ruff_linter/src/rules/isort/block.rs @@ -168,7 +168,14 @@ impl<'a> StatementVisitor<'a> for BlockBuilder<'a> { } // Track imports. + // + // basedpython: a static resource import (`import "data/config.yaml" as + // config`) is left out. isort rewrites a block it sorts by re-spelling + // each name, and a path is not a name — re-spelling one would emit + // `import data/config.yaml as config`. it ends the block instead, the + // way any other non-import statement does if matches!(stmt, Stmt::Import(_) | Stmt::ImportFrom(_)) + && !matches!(stmt, Stmt::Import(import) if import.names.iter().any(|alias| alias.is_resource)) && !self .exclusions .iter() diff --git a/crates/ruff_linter/src/rules/pep8_naming/mod.rs b/crates/ruff_linter/src/rules/pep8_naming/mod.rs index a80d382e72..e2cf544221 100644 --- a/crates/ruff_linter/src/rules/pep8_naming/mod.rs +++ b/crates/ruff_linter/src/rules/pep8_naming/mod.rs @@ -25,6 +25,7 @@ mod tests { #[test_case(Rule::InvalidFirstArgumentNameForClassMethod, Path::new("N804.py"))] #[test_case(Rule::InvalidFirstArgumentNameForMethod, Path::new("N805.py"))] #[test_case(Rule::NonLowercaseVariableInFunction, Path::new("N806.py"))] + #[test_case(Rule::NonLowercaseVariableInFunction, Path::new("N806_basedpython.by"))] #[test_case(Rule::DunderFunctionName, Path::new("N807.py"))] #[test_case(Rule::DunderFunctionName, Path::new("N807_basedpython.by"))] #[test_case(Rule::ConstantImportedAsNonConstant, Path::new("N811.py"))] diff --git a/crates/ruff_linter/src/rules/pep8_naming/snapshots/ruff_linter__rules__pep8_naming__tests__N806_N806_basedpython.by.snap b/crates/ruff_linter/src/rules/pep8_naming/snapshots/ruff_linter__rules__pep8_naming__tests__N806_N806_basedpython.by.snap new file mode 100644 index 0000000000..9dad5159e3 --- /dev/null +++ b/crates/ruff_linter/src/rules/pep8_naming/snapshots/ruff_linter__rules__pep8_naming__tests__N806_N806_basedpython.by.snap @@ -0,0 +1,13 @@ +--- +source: crates/ruff_linter/src/rules/pep8_naming/mod.rs +--- +N806 Variable `Unmatched` in function should be lowercase + --> N806_basedpython.by:20:14 + | +18 | def ordinary_capture(value: object) -> int: +19 | match value: +20 | case Unmatched: # N806 + | ^^^^^^^^^ +21 | return 1 +22 | case _: + | diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/multiple_imports_on_one_line.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/multiple_imports_on_one_line.rs index 2cc09d521e..838170a71f 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/multiple_imports_on_one_line.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/multiple_imports_on_one_line.rs @@ -61,6 +61,15 @@ pub(crate) fn multiple_imports_on_one_line(checker: &Checker, stmt: &Stmt, names } } +/// How an imported name is written in source. +fn spelling(name: &ruff_python_ast::Identifier, is_resource: bool) -> String { + if is_resource { + format!("\"{name}\"") + } else { + name.to_string() + } +} + /// Generate a [`Fix`] to split the imports across multiple statements. fn split_imports( stmt: &Stmt, @@ -79,8 +88,12 @@ fn split_imports( node_index: _, name, asname, + is_resource, } = alias; + // basedpython: a static resource is imported by a quoted path, + // and the quotes are not part of the name it is held under + let name = spelling(name, *is_resource); if let Some(asname) = asname { format!("import {name} as {asname}") } else { @@ -104,8 +117,10 @@ fn split_imports( node_index: _, name, asname, + is_resource, } = alias; + let name = spelling(name, *is_resource); if let Some(asname) = asname { format!("{indentation}import {name} as {asname}") } else { diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F841_F841_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F841_F841_basedpython.by.snap index 3761ad930d..da1004479d 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F841_F841_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F841_F841_basedpython.by.snap @@ -17,3 +17,15 @@ help: Remove assignment to unused variable `plain` 21 | return f() | note: This is an unsafe fix and may change runtime behavior + +F841 Local variable `Unmatched` is assigned to but never used + --> F841_basedpython.by:44:14 + | +42 | def ordinary_capture(value: object) -> int: +43 | match value: +44 | case Unmatched: # F841 + | ^^^^^^^^^ +45 | return 1 +46 | case _: + | +help: Remove assignment to unused variable `Unmatched` diff --git a/crates/ruff_linter/src/rules/pylint/rules/manual_import_from.rs b/crates/ruff_linter/src/rules/pylint/rules/manual_import_from.rs index 5a706a556c..bdcf1b7791 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/manual_import_from.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/manual_import_from.rs @@ -56,6 +56,13 @@ impl Violation for ManualFromImport { /// PLR0402 pub(crate) fn manual_from_import(checker: &Checker, stmt: &Stmt, alias: &Alias, names: &[Alias]) { + // basedpython: a static resource is imported by a path, and `from` has no + // spelling for one — `"data/settings.toml" as toml` reads as a dotted name + // whose last part is the binding, and rewriting it would name a module that + // does not exist + if alias.is_resource { + return; + } let Some(asname) = &alias.asname else { return; }; @@ -91,6 +98,7 @@ pub(crate) fn manual_from_import(checker: &Checker, stmt: &Stmt, alias: &Alias, names: vec![Alias { name: asname.clone(), asname: None, + is_resource: false, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }], diff --git a/crates/ruff_python_ast/src/helpers.rs b/crates/ruff_python_ast/src/helpers.rs index 0839e823ee..a2b0c91e65 100644 --- a/crates/ruff_python_ast/src/helpers.rs +++ b/crates/ruff_python_ast/src/helpers.rs @@ -1393,6 +1393,58 @@ pub fn declaration_annotation_type(annotation: &Expr) -> Option<&Expr> { .then(|| subscript.slice.as_ref()) } +/// basedpython: an `implements A, B` declaration, and the `for` clause that says +/// which modules it obliges. +/// +/// The parser encodes the statement as a call to the synthetic `__implements__` +/// marker, so the interfaces stay ordinary load expressions — an import that +/// exists only for the declaration still counts as used, and an interface that +/// does not exist is still an undefined name. A `for` clause's module patterns +/// follow the interfaces in the same argument list, and are told apart from them +/// by being string literals, which is why an interface may only be written as a +/// name or a dotted name. +pub struct ImplementsDeclaration<'a> { + /// the range of the `implements` keyword itself, which is where a diagnostic + /// about the declaration belongs + pub keyword_range: TextRange, + /// the interfaces the declaration names + pub interfaces: &'a [Expr], + /// the module patterns of a `for` clause, empty when there is none + pub patterns: &'a [Expr], +} + +/// The [`ImplementsDeclaration`] `stmt` is, if it is one. +pub fn implements_declaration(stmt: &Stmt) -> Option> { + let Stmt::Expr(statement) = stmt else { + return None; + }; + implements_declaration_expression(&statement.value) +} + +/// The [`ImplementsDeclaration`] an expression carries, if it is the marker call +/// a declaration parses to. +pub fn implements_declaration_expression(expression: &Expr) -> Option> { + let Expr::Call(call) = expression else { + return None; + }; + let Expr::Name(marker) = call.func.as_ref() else { + return None; + }; + if marker.id.as_str() != "__implements__" || !matches!(marker.ctx, ExprContext::Invalid) { + return None; + } + let arguments = &call.arguments.args; + let first_pattern = arguments + .iter() + .position(Expr::is_string_literal_expr) + .unwrap_or(arguments.len()); + Some(ImplementsDeclaration { + keyword_range: marker.range, + interfaces: &arguments[..first_pattern], + patterns: &arguments[first_pattern..], + }) +} + /// basedpython: the type a declaration writes, wherever it writes it. /// /// A plain `x: int` writes its annotation directly; a keyword declaration writes it diff --git a/crates/ruff_python_ast/src/node.rs b/crates/ruff_python_ast/src/node.rs index 67ed0ef5b2..929e51c464 100644 --- a/crates/ruff_python_ast/src/node.rs +++ b/crates/ruff_python_ast/src/node.rs @@ -442,6 +442,7 @@ impl Alias { node_index: _, name, asname, + is_resource: _, } = self; visitor.visit_identifier(name); diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs index d17c108f12..7ab7016862 100644 --- a/crates/ruff_python_ast/src/nodes.rs +++ b/crates/ruff_python_ast/src/nodes.rs @@ -99,9 +99,21 @@ impl StmtClassDef { /// `Invalid` ctx distinguishes it from a real decorator of the same /// spelling. pub fn has_synthetic_marker(&self, marker: &str) -> bool { - self.decorator_list.iter().any(|dec| { - matches!(&dec.expression, Expr::Name(name) if name.is_invalid() && name.id == marker) - }) + self.synthetic_marker_range(marker).is_some() + } + + /// The range of the keyword a synthetic basedpython marker was written as — + /// the `build` of `build:`, the `enum class` of `enum class Color:`. The + /// marker carries the keyword's own range, which is the only span in the + /// source that names the construct: the class name the parser synthesizes for + /// it spans nothing. + pub(crate) fn synthetic_marker_range(&self, marker: &str) -> Option { + self.decorator_list + .iter() + .find_map(|dec| match &dec.expression { + Expr::Name(name) if name.is_invalid() && name.id == marker => Some(name.range), + _ => None, + }) } /// True for a based-enum declaration (`enum class Shape:`). @@ -131,6 +143,21 @@ impl StmtClassDef { self.has_synthetic_marker("extension_def") } + /// True for a build-stamp declaration (`build:`) — the values the build + /// settles when the artifact is produced. Parses to a class so that + /// `build.GIT_SHA` is ordinary attribute access; the members are + /// annotation-only until lowering fills each one in. + pub fn is_build_stamps(&self) -> bool { + self.has_synthetic_marker("build_def") + } + + /// The range of the `build` keyword this block was written as, for a + /// diagnostic that has to point at the construct rather than at the class it + /// desugars to. + pub fn build_stamps_range(&self) -> Option { + self.synthetic_marker_range("build_def") + } + /// True for a based-enum variant (`Circle(radius: float)`, `Empty`) — the /// nested class defs inside a based enum. pub fn is_enum_variant(&self) -> bool { @@ -3294,6 +3321,11 @@ pub struct Alias { pub node_index: AtomicNodeIndex, pub name: Identifier, pub asname: Option, + /// basedpython: the alias names a static resource by path, + /// `import "data/config.yaml" as config`. `name` then holds the path as it + /// was written rather than a module name, and its range covers the string + /// literal it was written as + pub is_resource: bool, } /// See also [withitem](https://docs.python.org/3/library/ast.html#ast.withitem) diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index 78b6e76fbe..f56a6f497e 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -2145,7 +2145,14 @@ impl<'a> Generator<'a> { } fn unparse_alias(&mut self, alias: &Alias) { - self.p_id(&alias.name); + if alias.is_resource { + // basedpython: `import "data/config.yaml" as config`. the path is + // held under `name` without the quotes it was written with, and + // writing it as a name would emit `import data/config.yaml` + self.p_str_repr(alias.name.as_str(), StringLiteralFlags::empty()); + } else { + self.p_id(&alias.name); + } if let Some(asname) = &alias.asname { self.p(" as "); self.p_id(asname); diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/build_stamps.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/build_stamps.by new file mode 100644 index 0000000000..6d83c4047d --- /dev/null +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/build_stamps.by @@ -0,0 +1,13 @@ +build: + """what this artifact was built from""" + + GIT_SHA: str + GIT_SHA_SHORT: str + GIT_DIRTY: bool + BUILT_AT: str + VERSION: str = "0.0.0+dev" + BUILD_NUMBER: int = 0 + + +def describe() -> str: + return f"{build.GIT_SHA_SHORT}{'-dirty' if build.GIT_DIRTY else ''}" diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/module_api.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/module_api.by new file mode 100644 index 0000000000..5c32a90857 --- /dev/null +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/module_api.by @@ -0,0 +1,15 @@ +implements Backend + +implements Backend, Migratable + +implements api.Backend + +implements Backend for ".*" + +implements Backend for ".*", "!.base" + +implements Backend, api.Migratable for ".**", "._named" + + +class A: + implements Backend diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource.by new file mode 100644 index 0000000000..c4c2e8c501 --- /dev/null +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource.by @@ -0,0 +1,10 @@ +import "data/config.yaml" as config +import "data/ports.json" as ports +import 'data/settings.toml' as settings +import "a.json" as a, "b.json" as b + + +def f(): + import "data/config.yaml" as inner + return inner + diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource_escapes.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource_escapes.by new file mode 100644 index 0000000000..eb6640093e --- /dev/null +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource_escapes.by @@ -0,0 +1,4 @@ +import "a\nb.json" as newline +import "c\td.json" as tab +import "eéf.json" as unicode_escape +import 'plain.json' as plain diff --git a/crates/ruff_python_formatter/src/other/alias.rs b/crates/ruff_python_formatter/src/other/alias.rs index 9b997565fa..34e1c36577 100644 --- a/crates/ruff_python_formatter/src/other/alias.rs +++ b/crates/ruff_python_formatter/src/other/alias.rs @@ -1,5 +1,6 @@ use ruff_formatter::write; use ruff_python_ast::Alias; +use ruff_text_size::Ranged; use crate::comments::trailing_comments; use crate::other::identifier::DotDelimitedIdentifier; @@ -15,8 +16,25 @@ impl FormatNodeRule for FormatAlias { node_index: _, name, asname, + is_resource, } = item; - write!(f, [DotDelimitedIdentifier::new(name)])?; + + if *is_resource { + // basedpython: `import "data/config.yaml" as config`. the path lives + // in `name` without the quotes it was written with, so it is quoted + // again here — with the double quotes every other string is + // normalized to. a path holding anything a plain double-quoted + // literal cannot spell keeps the source's own spelling, which is the + // one thing certain to still mean the same path + if name.id.chars().any(needs_escaping) { + write!(f, [source_text_slice(name.range())])?; + } else { + let quoted = std::format!("\"{}\"", name.id); + write!(f, [text("ed)])?; + } + } else { + write!(f, [DotDelimitedIdentifier::new(name)])?; + } let comments = f.context().comments().clone(); @@ -102,3 +120,8 @@ impl FormatNodeRule for FormatAlias { Ok(()) } } + +/// Whether a plain double-quoted string literal cannot hold `character` as it is. +fn needs_escaping(character: char) -> bool { + matches!(character, '"' | '\\') || character.is_control() +} diff --git a/crates/ruff_python_formatter/src/statement/stmt_class_def.rs b/crates/ruff_python_formatter/src/statement/stmt_class_def.rs index 96d9c373d7..72dfa4f49b 100644 --- a/crates/ruff_python_formatter/src/statement/stmt_class_def.rs +++ b/crates/ruff_python_formatter/src/statement/stmt_class_def.rs @@ -84,25 +84,35 @@ impl FormatNodeRule for FormatStmtClassDef { // are more than two, then `leading_comments` will preserve the correct number of newlines. empty_lines_after_leading_comments(comments.leading(item)).fmt(f)?; - // basedpython: `protocol Foo:` and `extension Foo:` introduce a class - // without the `class` keyword. the parser still synthesizes a `ClassDef` - // but tags it with a synthetic `protocol_class` / `extension_def` - // decorator (rendered verbatim as its keyword text). suppress the `class` - // keyword so the round-trip emits `protocol Foo:` / `extension Foo:` - // rather than `protocol class Foo:`. the body is ordinary class-body - // statements, so it formats normally (unlike `enum`, whose `case` - // variants have no printer and force verbatim) + // basedpython: `protocol Foo:`, `extension Foo:` and `build:` introduce a + // class without the `class` keyword. the parser still synthesizes a + // `ClassDef` but tags it with a synthetic `protocol_class` / + // `extension_def` / `build_def` decorator (rendered verbatim as its + // keyword text). suppress the `class` keyword so the round-trip emits + // `protocol Foo:` / `extension Foo:` / `build:` rather than + // `protocol class Foo:`. the body is ordinary class-body statements, so + // it formats normally (unlike `enum`, whose `case` variants have no + // printer and force verbatim) let suppress_class_keyword = f.options().is_basedpython() && decorator_list.iter().any(|d| { matches!(&d.expression, Expr::Name(n) - if matches!(n.id.as_str(), "protocol_class" | "extension_def")) + if matches!(n.id.as_str(), "protocol_class" | "extension_def" | "build_def")) }); + // `build` has no name after its keyword — the marker *is* the whole + // header. the synthesized `build` identifier spans nothing, so printing + // it would emit the name a second time right after the keyword + let suppress_name = f.options().is_basedpython() + && decorator_list + .iter() + .any(|d| matches!(&d.expression, Expr::Name(n) if n.id.as_str() == "build_def")); let format_header = format_with(|f| { if !suppress_class_keyword { write!(f, [token("class"), space()])?; } - write!(f, [name.format()])?; + if !suppress_name { + write!(f, [name.format()])?; + } if let Some(type_params) = type_params.as_deref() { write!(f, [type_params.format()])?; diff --git a/crates/ruff_python_formatter/src/statement/stmt_expr.rs b/crates/ruff_python_formatter/src/statement/stmt_expr.rs index e5073a7b35..0c66597368 100644 --- a/crates/ruff_python_formatter/src/statement/stmt_expr.rs +++ b/crates/ruff_python_formatter/src/statement/stmt_expr.rs @@ -1,4 +1,6 @@ +use ruff_formatter::write; use ruff_python_ast as ast; +use ruff_python_ast::helpers::implements_declaration_expression; use ruff_python_ast::{Expr, Operator, StmtExpr}; use crate::expression::maybe_parenthesize_expression; @@ -13,6 +15,32 @@ impl FormatNodeRule for FormatStmtExpr { fn fmt_fields(&self, item: &StmtExpr, f: &mut PyFormatter) -> FormatResult<()> { let StmtExpr { value, .. } = item; + // basedpython: `implements A, B for ".*"` parses as a call to the + // synthetic `__implements__` marker, whose func has no source text of its + // own, so the declaration has to be written back out rather than printed + // as the call it is stored as + if f.options().is_basedpython() + && let Some(declaration) = implements_declaration_expression(value) + { + write!(f, [token("implements"), space()])?; + for (position, interface) in declaration.interfaces.iter().enumerate() { + if position > 0 { + write!(f, [token(","), space()])?; + } + interface.format().fmt(f)?; + } + if !declaration.patterns.is_empty() { + write!(f, [space(), token("for"), space()])?; + for (position, pattern) in declaration.patterns.iter().enumerate() { + if position > 0 { + write!(f, [token(","), space()])?; + } + pattern.format().fmt(f)?; + } + } + return Ok(()); + } + if is_arithmetic_like(value) { maybe_parenthesize_expression(value, item, Parenthesize::Optional).fmt(f)?; } else { diff --git a/crates/ruff_python_formatter/tests/snapshots/format@build_stamps.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@build_stamps.by.snap new file mode 100644 index 0000000000..822844bb6e --- /dev/null +++ b/crates/ruff_python_formatter/tests/snapshots/format@build_stamps.by.snap @@ -0,0 +1,37 @@ +--- +source: crates/ruff_python_formatter/tests/fixtures.rs +input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/build_stamps.by +--- +## Input +```bython +build: + """what this artifact was built from""" + + GIT_SHA: str + GIT_SHA_SHORT: str + GIT_DIRTY: bool + BUILT_AT: str + VERSION: str = "0.0.0+dev" + BUILD_NUMBER: int = 0 + + +def describe() -> str: + return f"{build.GIT_SHA_SHORT}{'-dirty' if build.GIT_DIRTY else ''}" +``` + +## Output +```bython +build: + """what this artifact was built from""" + + GIT_SHA: str + GIT_SHA_SHORT: str + GIT_DIRTY: bool + BUILT_AT: str + VERSION: str = "0.0.0+dev" + BUILD_NUMBER: int = 0 + + +def describe() -> str: + return f"{build.GIT_SHA_SHORT}{'-dirty' if build.GIT_DIRTY else ''}" +``` diff --git a/crates/ruff_python_formatter/tests/snapshots/format@module_api.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@module_api.by.snap new file mode 100644 index 0000000000..ccf3a6222b --- /dev/null +++ b/crates/ruff_python_formatter/tests/snapshots/format@module_api.by.snap @@ -0,0 +1,41 @@ +--- +source: crates/ruff_python_formatter/tests/fixtures.rs +input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/module_api.by +--- +## Input +```bython +implements Backend + +implements Backend, Migratable + +implements api.Backend + +implements Backend for ".*" + +implements Backend for ".*", "!.base" + +implements Backend, api.Migratable for ".**", "._named" + + +class A: + implements Backend +``` + +## Output +```bython +implements Backend + +implements Backend, Migratable + +implements api.Backend + +implements Backend for ".*" + +implements Backend for ".*", "!.base" + +implements Backend, api.Migratable for ".**", "._named" + + +class A: + implements Backend +``` diff --git a/crates/ruff_python_formatter/tests/snapshots/format@static_resource.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@static_resource.by.snap new file mode 100644 index 0000000000..37b9f24c77 --- /dev/null +++ b/crates/ruff_python_formatter/tests/snapshots/format@static_resource.by.snap @@ -0,0 +1,31 @@ +--- +source: crates/ruff_python_formatter/tests/fixtures.rs +input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource.by +--- +## Input +```bython +import "data/config.yaml" as config +import "data/ports.json" as ports +import 'data/settings.toml' as settings +import "a.json" as a, "b.json" as b + + +def f(): + import "data/config.yaml" as inner + return inner + +``` + +## Output +```bython +import "data/config.yaml" as config +import "data/ports.json" as ports +import "data/settings.toml" as settings +import "a.json" as a, "b.json" as b + + +def f(): + import "data/config.yaml" as inner + + return inner +``` diff --git a/crates/ruff_python_formatter/tests/snapshots/format@static_resource_escapes.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@static_resource_escapes.by.snap new file mode 100644 index 0000000000..09b3d97ff1 --- /dev/null +++ b/crates/ruff_python_formatter/tests/snapshots/format@static_resource_escapes.by.snap @@ -0,0 +1,19 @@ +--- +source: crates/ruff_python_formatter/tests/fixtures.rs +input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/static_resource_escapes.by +--- +## Input +```bython +import "a\nb.json" as newline +import "c\td.json" as tab +import "eéf.json" as unicode_escape +import 'plain.json' as plain +``` + +## Output +```bython +import "a\nb.json" as newline +import "c\td.json" as tab +import "eéf.json" as unicode_escape +import "plain.json" as plain +``` diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index a33e5e7f64..6a9afb4d3c 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -1553,7 +1553,11 @@ impl RecoveryContextKind { RecoveryContextKind::Except => p.at(TokenKind::Except), RecoveryContextKind::AssignmentTargets => p.at(TokenKind::Equal), RecoveryContextKind::TypeParams => p.at_type_param(), - RecoveryContextKind::ImportNames => p.at_name_or_soft_keyword(), + // basedpython: a static resource is imported by its path, so a + // string starts an import name as well as a name does + RecoveryContextKind::ImportNames => { + p.at_name_or_soft_keyword() || (p.options.is_basedpython && p.at(TokenKind::String)) + } RecoveryContextKind::ImportFromAsNames(_) => { p.at(TokenKind::Star) || p.at_name_or_soft_keyword() } diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 496f647eb1..ce52181025 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -1,5 +1,7 @@ use std::fmt::{Display, Write}; +use thin_vec::ThinVec; + use ruff_python_ast::helpers::{is_compound_statement, written_annotation_type}; use ruff_python_ast::name::Name; use ruff_python_ast::token::TokenKind; @@ -754,6 +756,25 @@ impl<'src> Parser<'src> { ); return Some(self.parse_extension_def(start)); } + // `build:` — the values the build stamps into the program. the colon has + // to be followed by a newline: `build: int` is an annotated assignment of + // an ordinary name, and only a bare `build:` opening a block is unclaimed + // syntax today + if kw == "build" + && self.peek() == TokenKind::Colon + && self.peek_nth(1).0 == TokenKind::Newline + { + self.error_if_not_basedpython( + "`build` declarations are not valid in .py files".to_string(), + ); + return Some(self.parse_build_def(start)); + } + if kw == "implements" && self.peek() == TokenKind::Name { + self.error_if_not_basedpython( + "`implements` declarations are not valid in .py files".to_string(), + ); + return Some(self.parse_implements_decl(start)); + } // `enum class E:` / `enum class E[T]:` — a "based enum" (an algebraic // sum type when its body has payload variants, an idiomatic `Enum` when // its variants are all unit). the `class` keyword is part of the @@ -1902,6 +1923,210 @@ impl<'src> Parser<'src> { }) } + /// Parses a `build:` declaration — the values the build stamps into the + /// program, settled when the artifact was produced rather than read at + /// startup. + /// + /// Produces a [`ClassDef`] named `build` carrying a synthetic `build_def` + /// marker decorator, so a use site is ordinary class-attribute access + /// (`build.GIT_SHA`) and needs no resolution rule of its own. + /// + /// The name is given an empty range at the end of the keyword. There is no + /// identifier in the source to point at, and a name sharing the keyword's + /// span would highlight twice and offer a rename of something nobody wrote; + /// an empty range is skipped by everything that reports on a span. + /// + /// [`ClassDef`]: ast::StmtClassDef + fn parse_build_def(&mut self, start: TextSize) -> Stmt { + let keyword_start = self.current_token_range().start(); + self.bump(TokenKind::Name); // consume "build" + let keyword_range = TextRange::new(keyword_start, self.current_token_range().start()); + + let mut decorators = DecoratorList::new(); + decorators.push(ast::Decorator { + expression: Expr::Name(ast::ExprName { + id: Name::new_static("build_def"), + ctx: ExprContext::Invalid, + range: keyword_range, + node_index: AtomicNodeIndex::NONE, + }), + range: keyword_range, + node_index: AtomicNodeIndex::NONE, + }); + + let name = ast::Identifier { + id: Name::new_static("build"), + range: TextRange::empty(keyword_range.end()), + node_index: AtomicNodeIndex::NONE, + }; + + // the dispatch that got here required the colon, so there is no missing + // one to recover from + self.bump(TokenKind::Colon); + let body = self.parse_body(Clause::Class); + + Stmt::ClassDef(ast::StmtClassDef { + range: self.node_range(start), + decorator_list: decorators, + name, + type_params: None, + arguments: None, + body, + node_index: AtomicNodeIndex::NONE, + }) + } + + /// Parses an `implements A, B` declaration, and its optional `for` clause + /// naming the modules the obligation is imposed on + /// (`implements Backend for ".*", "!.base"`). + /// + /// Produces an expression statement calling the synthetic `__implements__` + /// marker. The interfaces are ordinary loads, so an import that exists only + /// for the declaration still counts as used, and a `for` clause's patterns + /// follow them in the same argument list as string literals. The two are told + /// apart by kind, which is why an interface here may only be a name or a + /// dotted name + fn parse_implements_decl(&mut self, start: TextSize) -> Stmt { + let keyword_range = self.current_token_range(); + self.bump(TokenKind::Name); // consume "implements" + + let mut args: Vec = Vec::new(); + loop { + if !self.at(TokenKind::Name) { + self.add_error( + ParseErrorType::OtherError("`implements` takes an interface name".to_string()), + self.current_token_range(), + ); + break; + } + args.push(self.parse_interface_reference()); + if !self.eat(TokenKind::Comma) { + break; + } + // a comma introduces another interface. `implements A, for "…"` is a + // list with a hole in it, not a shorter list + if self.at(TokenKind::For) { + self.add_error( + ParseErrorType::OtherError( + "`implements` takes an interface name after `,`".to_string(), + ), + self.current_token_range(), + ); + break; + } + } + + if self.eat(TokenKind::For) { + loop { + if !self.at(TokenKind::String) { + self.add_error( + ParseErrorType::OtherError( + "a `for` clause takes module patterns, written as strings".to_string(), + ), + self.current_token_range(), + ); + break; + } + let pattern_range = self.current_token_range(); + let pattern = self.parse_strings(); + if pattern.is_string_literal_expr() { + args.push(pattern); + } else { + self.add_error( + ParseErrorType::OtherError( + "a module pattern is a plain string".to_string(), + ), + pattern_range, + ); + } + if !self.eat(TokenKind::Comma) { + break; + } + } + } + + // nothing else belongs on the line. a subscripted interface + // (`implements Backend[int]`) is the shape this catches: leaving the + // bracket to be re-parsed would silently drop the specialization and turn + // the obligation into one the author did not write + if !self.at_declaration_end() { + self.add_error( + ParseErrorType::OtherError( + "an `implements` declaration takes interface names, and patterns after `for`" + .to_string(), + ), + self.current_token_range(), + ); + // consume what is left, so it does not re-parse as a statement of its + // own and report a second, more confusing error + let mut progress = ParserProgress::default(); + while !self.at_declaration_end() { + progress.assert_progressing(self); + self.bump_any(); + } + } + + let range = self.node_range(start); + self.eat_declaration_terminator(); + + let marker = Expr::Name(ast::ExprName { + id: Name::new_static("__implements__"), + ctx: ExprContext::Invalid, + range: keyword_range, + node_index: AtomicNodeIndex::NONE, + }); + Stmt::Expr(ast::StmtExpr { + value: Box::new(Expr::Call(ast::ExprCall { + func: Box::new(marker), + arguments: ast::Arguments { + args: args.into_boxed_slice(), + keywords: ThinVec::new(), + range: TextRange::new(keyword_range.end(), range.end()), + node_index: AtomicNodeIndex::NONE, + }, + range_start: start, + cast_kind: None, + is_string_tag: false, + node_index: AtomicNodeIndex::NONE, + })), + range, + node_index: AtomicNodeIndex::NONE, + }) + } + + /// Is the parser at something that ends a simple statement? + fn at_declaration_end(&self) -> bool { + matches!( + self.current_token_kind(), + TokenKind::Newline | TokenKind::Semi | TokenKind::EndOfFile | TokenKind::Dedent + ) + } + + /// Parses the `A` or `pkg.A` naming an interface in an `implements` + /// declaration, as an ordinary load expression + fn parse_interface_reference(&mut self) -> Expr { + let start = self.node_start(); + let name = self.parse_identifier(); + let mut expr = Expr::Name(ast::ExprName { + id: name.id, + ctx: ExprContext::Load, + range: name.range, + node_index: AtomicNodeIndex::NONE, + }); + while self.eat(TokenKind::Dot) { + let attr = self.parse_identifier(); + expr = Expr::Attribute(ast::ExprAttribute { + value: Box::new(expr), + attr, + ctx: ExprContext::Load, + optional: false, + range: self.node_range(start), + node_index: AtomicNodeIndex::NONE, + }); + } + expr + } + /// Parses a `type def Name[X]:` declaration — a user-defined type function /// whose body is executed to produce a type at each application. /// @@ -2932,6 +3157,29 @@ impl<'src> Parser<'src> { self.add_error(ParseErrorType::EmptyImportNames, self.current_token_range()); } + // basedpython: a static resource is a document written into the module + // that imports it, not a module the runtime goes and finds, so it shares + // a statement neither with a module import nor with `lazy` + if let Some(resource) = names.iter().find(|alias| alias.is_resource) { + if let Some(module) = names.iter().find(|alias| !alias.is_resource) { + self.add_error( + ParseErrorType::OtherError( + "A static resource is imported by a statement of its own".to_string(), + ), + module.range(), + ); + } + if is_lazy { + self.add_error( + ParseErrorType::OtherError( + "A static resource is read while the program is built, so it cannot be lazy" + .to_string(), + ), + resource.range(), + ); + } + } + ast::StmtImport { names, is_lazy, @@ -3142,11 +3390,19 @@ impl<'src> Parser<'src> { node_index: AtomicNodeIndex::NONE, }, asname: None, + is_resource: false, range, node_index: AtomicNodeIndex::NONE, }; } + if self.options.is_basedpython + && matches!(style, ImportStyle::Import) + && self.at(TokenKind::String) + { + return self.parse_resource_alias(start); + } + let name = match style { ImportStyle::Import => self.parse_dotted_name(), ImportStyle::ImportFrom => self.parse_identifier(), @@ -3177,6 +3433,63 @@ impl<'src> Parser<'src> { range: self.node_range(start), name, asname, + is_resource: false, + node_index: AtomicNodeIndex::NONE, + } + } + + /// Parses the `"data/config.yaml" as config` of a basedpython static + /// resource import. + /// + /// The path is held in the alias's `name`, which is why the name it binds + /// has to be written: a path is not a name, so there is nothing to fall back + /// on the way `import foo` falls back on `foo`. + fn parse_resource_alias(&mut self, start: TextSize) -> ast::Alias { + let path_range = self.current_token_range(); + let path = match self.parse_strings() { + Expr::StringLiteral(string) => Name::new(string.value.to_str()), + path => { + self.add_error( + ParseErrorType::OtherError( + "A static resource is imported by a plain string path".to_string(), + ), + path.range(), + ); + Name::empty() + } + }; + + let name = ast::Identifier { + id: path, + range: self.node_range(path_range.start()), + node_index: AtomicNodeIndex::NONE, + }; + + let asname = if self.eat(TokenKind::As) { + if self.at_name_or_soft_keyword() { + Some(self.parse_identifier()) + } else { + self.add_error( + ParseErrorType::OtherError("Expected symbol after `as`".to_string()), + self.current_token_range(), + ); + None + } + } else { + self.add_error( + ParseErrorType::OtherError( + "Expected `as` and a name to bind the static resource to".to_string(), + ), + self.current_token_range(), + ); + None + }; + + ast::Alias { + range: self.node_range(start), + name, + asname, + is_resource: true, node_index: AtomicNodeIndex::NONE, } } diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 2845fc6e37..9174ade0a9 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -702,6 +702,51 @@ fn basedpython_lone_starred_group_without_arrow_still_errors() { ); } +#[test] +fn basedpython_build_parses_to_marked_class() { + let parsed = parse_basedpython_module("build:\n GIT_SHA: str\n"); + let [Stmt::ClassDef(class)] = parsed.syntax().body.as_slice() else { + panic!("expected a single ClassDef"); + }; + assert!(class.is_build_stamps()); + assert_eq!(class.name.as_str(), "build"); + // the name stands for no source identifier, so it spans nothing + assert!(class.name.range().is_empty()); + let [marker] = class.decorator_list.as_slice() else { + panic!("expected the one synthetic marker"); + }; + // the marker covers the keyword itself, which is what highlights it + assert_eq!( + &"build:\n GIT_SHA: str\n" + [usize::from(marker.range().start())..usize::from(marker.range().end())], + "build" + ); + assert!(matches!( + class.body.as_slice(), + [Stmt::AnnAssign(a)] if a.target.as_name_expr().is_some_and(|n| n.id == "GIT_SHA") + )); +} + +#[test] +fn basedpython_build_annotation_is_still_an_assignment() { + // `build` is an ordinary name everywhere but in front of a block-opening + // colon, so an annotated assignment that happens to use it still parses as one + let parsed = parse_basedpython_module("build: int = 3\n"); + assert!(matches!( + parsed.syntax().body.as_slice(), + [Stmt::AnnAssign(_)] + )); +} + +#[test] +fn build_block_is_rejected_in_python() { + let parsed = parse( + "build:\n GIT_SHA: str\n", + ParseOptions::from(Mode::Module), + ); + assert!(parsed.is_err(), "a `build` block is basedpython-only"); +} + #[test] fn basedpython_extension_parses_to_marked_class() { let parsed = parse_basedpython_module( diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@debug_shadow_import.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@debug_shadow_import.py.snap index ca510e6d23..419bf5463a 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@debug_shadow_import.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@debug_shadow_import.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -49,6 +50,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -75,6 +77,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -109,6 +112,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@dotted_name_multiple_dots.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@dotted_name_multiple_dots.py.snap index ebad60ec0a..8a935953b8 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@dotted_name_multiple_dots.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@dotted_name_multiple_dots.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -43,6 +44,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_dotted_names.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_dotted_names.py.snap index 6ef0484859..d7cc3717c1 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_dotted_names.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_dotted_names.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -59,6 +60,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 33..34, @@ -69,6 +71,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -97,6 +100,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 52..53, @@ -107,6 +111,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 54..55, @@ -117,6 +122,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 57..58, @@ -127,6 +133,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 60..61, @@ -137,6 +144,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 62..63, @@ -147,6 +155,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 65..66, @@ -157,6 +166,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_module.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_module.py.snap index e5b8da2347..cc87be3e63 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_module.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_module.py.snap @@ -36,6 +36,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap index 91931f5735..05acb31726 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 18..19, @@ -41,6 +42,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -100,6 +102,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 44..45, @@ -110,6 +113,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_parenthesized_star.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_parenthesized_star.py.snap index 30839311f4..e753b94f4e 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_parenthesized_star.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_parenthesized_star.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_star_with_other_names.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_star_with_other_names.py.snap index 12e70a6758..1c071a2e20 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_star_with_other_names.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_star_with_other_names.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 17..18, @@ -41,6 +42,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -69,6 +71,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 36..37, @@ -79,6 +82,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 39..40, @@ -89,6 +93,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -117,6 +122,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 58..64, @@ -133,6 +139,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], level: 0, @@ -161,6 +168,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 82..83, @@ -171,6 +179,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 85..86, @@ -181,6 +190,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_unparenthesized_trailing_comma.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_unparenthesized_trailing_comma.py.snap index e3142b3641..0980435ded 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_unparenthesized_trailing_comma.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_unparenthesized_trailing_comma.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -65,6 +66,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], level: 0, @@ -93,6 +95,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 56..57, @@ -103,6 +106,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_alias_missing_asname.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_alias_missing_asname.py.snap index 47f3fd9311..f42d750936 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_alias_missing_asname.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_alias_missing_asname.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_from_star.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_from_star.py.snap index b3e52b7258..80349447ba 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_from_star.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_from_star.py.snap @@ -55,6 +55,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -101,6 +102,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -156,6 +158,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 2, @@ -213,6 +216,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 142..143, @@ -223,6 +227,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_star_import.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_star_import.py.snap index 89a4ac84b0..359b843c1f 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_star_import.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_star_import.py.snap @@ -53,6 +53,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_trailing_comma.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_trailing_comma.py.snap index 37a9842027..bc24b75fb9 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_trailing_comma.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@import_stmt_trailing_comma.py.snap @@ -32,6 +32,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 19..20, @@ -42,6 +43,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@invalid_future_feature.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@invalid_future_feature.py.snap index 64f6b1d6fa..cf0dec27ec 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@invalid_future_feature.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@invalid_future_feature.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -59,6 +60,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 75..90, @@ -69,6 +71,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -97,6 +100,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 133..150, @@ -107,6 +111,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_context_py315.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_context_py315.py.snap index c014b6a345..10559513e6 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_context_py315.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_context_py315.py.snap @@ -29,6 +29,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -109,6 +110,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -161,6 +163,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -216,6 +219,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -255,6 +259,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -314,6 +319,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_from_py315.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_from_py315.py.snap index 92145e8fa9..f900b4211d 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_from_py315.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_invalid_from_py315.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -59,6 +60,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -111,6 +113,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_stmt_py314.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_stmt_py314.py.snap index 1e8c6b1536..8c58639ff5 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_stmt_py314.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@lazy_import_stmt_py314.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -50,6 +51,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@debug_rename_import.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@debug_rename_import.py.snap index 68c231d480..6b6fc9a3a5 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@debug_rename_import.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@debug_rename_import.py.snap @@ -30,6 +30,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -56,6 +57,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -90,6 +92,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@dotted_name_normalized_spaces.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@dotted_name_normalized_spaces.py.snap index 6bb8c42dcb..c96ea4e575 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@dotted_name_normalized_spaces.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@dotted_name_normalized_spaces.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -43,6 +44,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_no_space.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_no_space.py.snap index 4df87cf39b..d9e77dcb5b 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_no_space.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_no_space.py.snap @@ -25,6 +25,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 1, @@ -47,6 +48,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 3, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_soft_keyword_module_name.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_soft_keyword_module_name.py.snap index af4c7fba1e..d6a72ce711 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_soft_keyword_module_name.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_soft_keyword_module_name.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -59,6 +60,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -87,6 +89,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -115,6 +118,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -143,6 +147,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_stmt_terminator.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_stmt_terminator.py.snap index cdb2bf926a..5f5cbb2d89 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_stmt_terminator.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@from_import_stmt_terminator.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 18..19, @@ -41,6 +42,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -69,6 +71,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 39..40, @@ -79,6 +82,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -143,6 +147,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 65..66, @@ -153,6 +158,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -217,6 +223,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 90..91, @@ -227,6 +234,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_as_name_soft_keyword.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_as_name_soft_keyword.py.snap index 0b9b20b8ce..e60a5b7f16 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_as_name_soft_keyword.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_as_name_soft_keyword.py.snap @@ -30,6 +30,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -55,6 +56,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -80,6 +82,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -105,6 +108,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_from_star.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_from_star.py.snap index 279c5674fd..7a37ce2c3f 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_from_star.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_from_star.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_stmt_terminator.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_stmt_terminator.py.snap index 6a694c9e14..037bec1418 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_stmt_terminator.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@import_stmt_terminator.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 10..11, @@ -34,6 +35,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -53,6 +55,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 23..24, @@ -63,6 +66,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -82,6 +86,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 35..36, @@ -92,6 +97,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_relative_py315.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_relative_py315.py.snap index ab4319e684..701965f87b 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_relative_py315.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_relative_py315.py.snap @@ -25,6 +25,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 1, @@ -53,6 +54,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 98..99, @@ -63,6 +65,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 1, @@ -85,6 +88,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 122..123, @@ -95,6 +99,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 1, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_semantic_ok_py315.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_semantic_ok_py315.py.snap index 146065838c..6a5acfe757 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_semantic_ok_py315.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_semantic_ok_py315.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -92,6 +93,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -170,6 +172,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_soft_keyword_split_py315.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_soft_keyword_split_py315.py.snap index fee62ba5af..3d17b7da99 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_soft_keyword_split_py315.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_soft_keyword_split_py315.py.snap @@ -38,6 +38,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -78,6 +79,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_stmt_py315.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_stmt_py315.py.snap index a4b5218929..f3cc7a6ce9 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_stmt_py315.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@lazy_import_stmt_py315.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: true, @@ -49,6 +50,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: true, @@ -75,6 +77,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -109,6 +112,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], level: 0, @@ -162,6 +166,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -188,6 +193,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@param_with_star_annotation_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@param_with_star_annotation_py310.py.snap index a45357da2c..5402d5f630 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@param_with_star_annotation_py310.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@param_with_star_annotation_py310.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 199..206, @@ -41,6 +42,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@simple_stmts_with_semicolons.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@simple_stmts_with_semicolons.py.snap index a074b65673..2634e160a4 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@simple_stmts_with_semicolons.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@simple_stmts_with_semicolons.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -57,6 +58,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__from_import.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__from_import.py.snap index 34787d882b..505460c077 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__from_import.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__from_import.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -53,6 +54,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 1, @@ -87,6 +89,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, Alias { range: 73..85, @@ -103,6 +106,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], level: 0, @@ -131,6 +135,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 1, @@ -153,6 +158,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 3, @@ -175,6 +181,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 26, @@ -203,6 +210,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 26, @@ -231,6 +239,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 231..237, @@ -247,6 +256,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, Alias { range: 239..240, @@ -257,6 +267,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, @@ -285,6 +296,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__import.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__import.py.snap index 6a04816af6..db338b54fb 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__import.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__import.py.snap @@ -24,6 +24,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -43,6 +44,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -68,6 +70,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, @@ -87,6 +90,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 50..51, @@ -97,6 +101,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, Alias { range: 53..54, @@ -107,6 +112,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], is_lazy: false, @@ -132,6 +138,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, Alias { range: 76..91, @@ -148,6 +155,7 @@ Module( node_index: NodeIndex(None), }, ), + is_resource: false, }, ], is_lazy: false, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@valid_future_feature.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@valid_future_feature.py.snap index c4b1d564a7..49dde7d3ff 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@valid_future_feature.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@valid_future_feature.py.snap @@ -31,6 +31,7 @@ Module( node_index: NodeIndex(None), }, asname: None, + is_resource: false, }, ], level: 0, diff --git a/crates/ruff_python_semantic/src/binding.rs b/crates/ruff_python_semantic/src/binding.rs index 83d42a20f2..4f3200d34f 100644 --- a/crates/ruff_python_semantic/src/binding.rs +++ b/crates/ruff_python_semantic/src/binding.rs @@ -108,6 +108,13 @@ impl<'a> Binding<'a> { self.flags.intersects(BindingFlags::UNPACKED_ASSIGNMENT) } + /// basedpython: return `true` if this [`Binding`] is a bare `case` name that + /// names an `enum class` variant, and so binds nothing — see + /// [`BindingFlags::BASED_ENUM_CASE_NAME`]. + pub const fn is_based_enum_case_name(&self) -> bool { + self.flags.intersects(BindingFlags::BASED_ENUM_CASE_NAME) + } + /// Return `true` if this [`Binding`] represents an unbound variable /// (e.g., `x` in `x = 1; del x`). pub const fn is_unbound(&self) -> bool { @@ -431,6 +438,28 @@ bitflags! { /// ``` const IN_ASSERT_STATEMENT = 1 << 13; + /// basedpython: the binding is a bare `case` name that names an + /// [`enum class`] variant, so it binds nothing at runtime. + /// + /// ```by + /// enum class Color: + /// case Red + /// + /// def f(c: Color) -> int: + /// match c: + /// case Red: # `Color.Red`, not a capture + /// return 1 + /// ``` + /// + /// Python spells this as a capture and the parser has to parse it as one, + /// so the binding exists — but nothing is assigned and nothing goes + /// unused, and the name is a member's name rather than a variable's. Rules + /// that read a capture as a variable have to skip it or they report + /// correct code. + /// + /// [`enum class`]: https://docs.basedpython.org/features/enums + const BASED_ENUM_CASE_NAME = 1 << 14; + /// The binding represents any type alias. const TYPE_ALIAS = Self::ANNOTATED_TYPE_ALIAS.bits() | Self::DEFERRED_TYPE_ALIAS.bits(); } diff --git a/crates/ruff_python_semantic/src/model.rs b/crates/ruff_python_semantic/src/model.rs index e19fb013f0..d53e7703a5 100644 --- a/crates/ruff_python_semantic/src/model.rs +++ b/crates/ruff_python_semantic/src/model.rs @@ -849,6 +849,17 @@ impl<'a> SemanticModel<'a> { /// linter does not have — so it defers, exactly as it does for a receiver /// block's `self`. A name matching no variant at all is still reported, and /// ty reports one whose expected type does not accept it. + /// basedpython: whether a bare `case :` names an `enum class` variant + /// in this file rather than binding a capture. + /// + /// The public form of `is_based_enum_variant`, for the one caller + /// outside name resolution: a pattern name has to be *bound* either way, + /// since python's grammar spells a member match as a capture, so the rules + /// that read a capture as a variable need to be told which it is + pub fn is_based_enum_case_name(&self, name: &str) -> bool { + self.in_basedpython_file() && self.is_based_enum_variant(name) + } + fn is_based_enum_variant(&self, name: &str) -> bool { self.global_scope().binding_ids().any(|binding_id| { let BindingKind::ClassDefinition(scope_id) = self.bindings[binding_id].kind else { diff --git a/crates/ruff_python_stdlib/src/identifiers.rs b/crates/ruff_python_stdlib/src/identifiers.rs index 950b128c98..fc0ac0e715 100644 --- a/crates/ruff_python_stdlib/src/identifiers.rs +++ b/crates/ruff_python_stdlib/src/identifiers.rs @@ -24,15 +24,15 @@ pub fn is_identifier(name: &str) -> bool { true } -// Checks if the character c is a valid starting character as described -// in https://docs.python.org/3/reference/lexical_analysis.html#identifiers -fn is_identifier_start(c: char) -> bool { +/// Checks if the character c is a valid starting character as described +/// in +pub fn is_identifier_start(c: char) -> bool { matches!(c, 'a'..='z' | 'A'..='Z' | '_') || is_xid_start(c) } -// Checks if the character c is a valid continuation character as described -// in https://docs.python.org/3/reference/lexical_analysis.html#identifiers -fn is_identifier_continuation(c: char) -> bool { +/// Checks if the character c is a valid continuation character as described +/// in +pub fn is_identifier_continuation(c: char) -> bool { // Arrange things such that ASCII codepoints never // result in the slower `is_xid_continue` getting called. if c.is_ascii() { diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 1e37fdc501..b01c3d6d6e 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -259,7 +259,8 @@ by run [OPTIONS] [MODULE] [ARGS]...

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

-

[default: default]

+

[default: default]

--stamp name=value

supply a build: stamp, as NAME=VALUE. repeatable. overrides whatever this build could have worked out for itself, which is what a CI job that knows its own commit wants

+
## by init @@ -309,7 +310,8 @@ by build [OPTIONS]

input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

-

[default: default]

--wheels

Build one publishable wheel per python version, and a source distribution, into dist/.

+

[default: default]

--stamp name=value

supply a build: stamp, as NAME=VALUE. repeatable. overrides whatever this build could have worked out for itself, which is what a CI job that knows its own commit wants

+
--wheels

Build one publishable wheel per python version, and a source distribution, into dist/.

Each wheel is lowered to the version it is tagged for, so an installer hands every interpreter the best wheel it can use rather than one lowered to the oldest python the project supports. Needs uv, which does the packaging.

@@ -370,7 +372,8 @@ by compile [OPTIONS] [FILE]...

Stricter than --no-any, and a different question: --no-any asks whether the module is fully typed, this asks whether it compiles entirely.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

-

[default: default]

--verbose

Report every function that was not lowered natively, with the reason

+

[default: default]

--stamp name=value

supply a build: stamp, as NAME=VALUE. repeatable. overrides whatever this build could have worked out for itself, which is what a CI job that knows its own commit wants

+
--verbose

Report every function that was not lowered natively, with the reason

## by generate-api-file @@ -429,7 +432,8 @@ by transpile [OPTIONS] [FILE]
--reverse

convert Python source into basedpython idioms (instead of the default by → py direction)

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

-

[default: default]

+

[default: default]

--stamp name=value

supply a build: stamp, as NAME=VALUE. repeatable. overrides whatever this build could have worked out for itself, which is what a CI job that knows its own commit wants

+
## by help diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index 9d6a267cff..f4fdacf6c0 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -1382,6 +1382,123 @@ bundled as a zip file in the binary --- +## `experimental` + +Features that are still being designed, and are off unless the project asks +for them by name. + +An experimental feature may change or be withdrawn without the deprecation +period a stable one gets. Opting in says you would rather have it than that +guarantee. + +### `build-stamps` + +Whether a `build:` block declares build stamps. + +`build:` declares the values a build settles when it produces the artifact +— the commit it was built from, the time it was built at — and each is read +as `build.NAME` at the type it declares. With this off the block still +parses and still lowers, so a program that reads a stamp keeps working, but +writing one is reported: nothing settles a stamp the project has not asked +for, so it would silently stand for its default, or for nothing. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.experimental] + # let the program read the commit it was built from + build-stamps = true + ``` + +=== "ty.toml" + + ```toml + [experimental] + # let the program read the commit it was built from + build-stamps = true + ``` + +--- + +### `module-api` + +Whether an `implements` declaration is enforced. + +`implements Backend` obliges the module that writes it to answer the +protocol, and a `for` clause in a package's `__init__` imposes the same +obligation on the modules its patterns name. With this off the declaration +still parses and still lowers, but nothing is checked against it — and a +declaration written anyway is reported, rather than quietly doing nothing. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.experimental] + # hold every module in `backends` to the `Backend` protocol + module-api = true + ``` + +=== "ty.toml" + + ```toml + [experimental] + # hold every module in `backends` to the `Backend` protocol + module-api = true + ``` + +--- + +## `lowering` + +### `float-literals` + +How a float or complex literal type is spelled in the transpiled python. + +basedpython reads `a: 1.5` as a literal type, and python has no spelling for one: +PEP 586 admits only `None`, `int`, `bool`, `str`, `bytes` and enum members into +`Literal[...]`. + +* `nominal` (the default) writes the type the literal is one of — `a: 1.5` becomes + `a: float`, `a: 2j` becomes `a: complex`. The precision is lost, and every checker + that reads the output accepts it. +* `literal` keeps the literal, writing `a: Literal[1.5]`. The precision survives and + the output still runs, because `typing` does not check what it is handed — but a + checker reading it reports the argument as invalid. + +**Default value**: `"nominal"` + +**Type**: `"nominal" | "literal"` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.lowering] + float-literals = "literal" + ``` + +=== "ty.toml" + + ```toml + [lowering] + float-literals = "literal" + ``` + +--- + ## `overrides` Configuration override that applies to specific files based on glob patterns. diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 4e63b13cc3..700a118e2f 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -8,7 +8,7 @@ Default level: error · Added in 0.0.64 · Related issues · -View source +View source @@ -44,7 +44,7 @@ class Base(ABC): Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -90,7 +90,7 @@ class Derived(Base): # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -123,7 +123,7 @@ f(1, b=s1) # ok — explicit Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -159,7 +159,7 @@ report(Celsius()) # error: two conversions apply Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -194,7 +194,7 @@ extension list: Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -258,7 +258,7 @@ class SubProto(BaseProto, Protocol): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -341,7 +341,7 @@ value = unknown # ty: ignore[unresolved-reference] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -384,7 +384,7 @@ a4 = True + 1 # ok — a boolean used as a boolean Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -439,7 +439,7 @@ Foo.method() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -467,7 +467,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.7 · Related issues · -View source +View source @@ -502,7 +502,7 @@ def f(x: object): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -536,7 +536,7 @@ a = 1 # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -571,7 +571,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -607,7 +607,7 @@ class B(A): ... # error Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -643,7 +643,7 @@ type B = A # error Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -680,7 +680,7 @@ class Example: Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -719,7 +719,7 @@ old_func() # error: [deprecated] Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -752,7 +752,7 @@ false positives it can produce. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -783,7 +783,7 @@ class B(A, A): ... # error Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -826,7 +826,7 @@ class A: # error Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -876,7 +876,7 @@ def bar() -> str: # error: [empty-body] Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -937,7 +937,7 @@ def h(x: object): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -1039,7 +1039,7 @@ def foo() -> "intt\b": ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1078,7 +1078,7 @@ def f(local fn: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1119,7 +1119,7 @@ for x in [1, 2, 3]: Default level: warn · Added in 0.0.50 · Related issues · -View source +View source @@ -1159,7 +1159,7 @@ def g(value: ~A) -> None: ... # error: [experimental-syntax] Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -1194,7 +1194,7 @@ def my_function() -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.40 · Related issues · -View source +View source @@ -1229,7 +1229,7 @@ let a = 1 Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -1345,7 +1345,7 @@ def test() -> "Literal[5]": Default level: ignore · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -1400,7 +1400,7 @@ an item of an unpacking — is not a declaration, and is never reported. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -1475,7 +1475,7 @@ print(Labelled) # warning: prints `` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1511,7 +1511,7 @@ class C(A, B): ... # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1541,7 +1541,7 @@ t[3] # error Default level: warn · Added in 0.0.1-alpha.33 · Related issues · -View source +View source @@ -1578,7 +1578,7 @@ class MyClass: ... Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -1679,7 +1679,7 @@ an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1711,7 +1711,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1742,7 +1742,7 @@ a: int = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1800,7 +1800,7 @@ C.instance_only_var = 56 # error Default level: error · Added in 0.0.33 · Related issues · -View source +View source @@ -1846,7 +1846,7 @@ class Sub(Base): Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -1888,7 +1888,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1909,13 +1909,51 @@ Class definitions with bases like this will lead to `TypeError` being raised at class A(42): ... # error: [invalid-base] ``` +## `invalid-build-stamps` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.79 · +Related issues · +View source + + + +**What it does** + +Checks that a `build:` block is one the project has asked for. Build +stamps are experimental, so a project opts in by name: + +```toml +# basedpython.toml +[experimental] +build-stamps = true +``` + +**Why is this bad?** + +The block parses and lowers whether or not the project opted in, because a +program that reads `build.GIT_SHA` has to keep working when the feature is +turned off — so nothing at the point of use says the value was never +settled. A stamp declared without a default fails the transpile, and one +with a default quietly stands for that default in an artifact that claims +to know what commit it came from. + +**Example** + + +```by +build: # error: `build` is an experimental feature, and is off + GIT_SHA: str +``` + ## `invalid-conformance` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.5 · Related issues · -View source +View source @@ -1952,7 +1990,7 @@ extension str(A): # error: `str` does not answer every member of `A` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1982,7 +2020,7 @@ with 1: # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -2015,7 +2053,7 @@ class Fahrenheit: Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2068,7 +2106,7 @@ See: Default level: error · Added in 0.0.13 · Related issues · -View source +View source @@ -2104,7 +2142,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2136,7 +2174,7 @@ a: str # error Default level: warn · Added in 0.0.20 · Related issues · -View source +View source @@ -2193,7 +2231,7 @@ class Pet(Enum): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2257,7 +2295,7 @@ This rule corresponds to Ruff's [`except-with-non-exception-classes` (`B030`)](h Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -2310,7 +2348,7 @@ class D(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -2343,7 +2381,7 @@ extension list[T: int]: # error: `list` declares no type parameter `T` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -2372,7 +2410,7 @@ Author.objects.filter(name__startswith=1) # error: lookup wants `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -2408,7 +2446,7 @@ def test_user(user: int) -> None: # error: fixture provides `str` Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.68 · Related issues · -View source +View source @@ -2456,7 +2494,7 @@ f"{'name':>10}" # ok Default level: error · Added in 0.0.1-alpha.35 · Related issues · -View source +View source @@ -2507,7 +2545,7 @@ class NonFrozenChild(FrozenBase): # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2562,7 +2600,7 @@ class E(Generic[V]): # error Default level: error · Added in 0.0.12 · Related issues · -View source +View source @@ -2658,7 +2696,7 @@ a = 20 / 0 # type: ignore Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -2706,7 +2744,7 @@ carol = Person(name="Carol", aeg=25) # typo! Default level: warn · Added in 0.0.15 · Related issues · -View source +View source @@ -2768,7 +2806,7 @@ def f(x, y, /): # Python 3.8+ syntax Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2808,7 +2846,7 @@ def f(t: TypeVar("U")): ... # ty: ignore[invalid-type-form] Default level: error · Added in 0.0.18 · Related issues · -View source +View source @@ -2860,7 +2898,7 @@ match object(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2895,7 +2933,7 @@ class B(metaclass=42): ... # error Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -3007,13 +3045,56 @@ Correct use of `@override` is enforced by ty's [`invalid-explicit-override`](#in [liskov-substitution-principle]: https://en.wikipedia.org/wiki/Liskov_substitution_principle [override]: https://docs.python.org/3/library/typing.html#typing.override +## `invalid-module-api` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.72 · +Related issues · +View source + + + +**What it does** + +Checks that an `implements` declaration is one a module can be held to: +that it names a protocol, that a `for` clause is written in a package's +`__init__` with patterns relative to that package and reaching something, +and that the module it obliges can actually be checked. + +**Why is this bad?** + +An obligation nothing can check is worse than no obligation: it reads, in +review and in the editor, as a promise that is being enforced. A rule +whose patterns match no module, or one written where no module will look +for it, enforces nothing at all. + +This also covers a declaration written while the feature is off. `implements` +is experimental, so a project opts in by name: + +```toml +# basedpython.toml +[experimental] +module-api = true +``` + +**Example** + + +```by +class Backend: + def connect(self) -> None: ... + +implements Backend # error: `Backend` is not a protocol +``` + ## `invalid-named-tuple` Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -3080,7 +3161,7 @@ TypeError: typing.ClassVar[int] is not valid as type argument Default level: warn · Added in 0.0.31 · Related issues · -View source +View source @@ -3128,7 +3209,7 @@ admin[0] # "Alice" Default level: error · Added in 0.0.1-alpha.27 · Related issues · -View source +View source @@ -3166,7 +3247,7 @@ Baz = NewType("Baz", int | str) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3223,7 +3304,7 @@ def foo(x: int) -> int: ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3252,7 +3333,7 @@ def f(a: int = ""): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3285,7 +3366,7 @@ def test_add(a: int, b: int) -> None: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3321,7 +3402,7 @@ P2 = ParamSpec() # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3357,7 +3438,7 @@ TypeError: Protocols can only inherit from other protocols, got Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3428,7 +3509,7 @@ def g(): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -3456,7 +3537,7 @@ def f() raises int: # error: `int` is not an exception Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -3489,7 +3570,7 @@ if m := re.match("(a)(b)", "ab"): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -3521,7 +3602,7 @@ type Alias[reified T] = list[T] # error: an alias's parameters are erased Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3661,13 +3742,52 @@ def detail(request, pk: str): ... # warning: the converter gives an `int` def detail(request, pk: int): ... # ok ``` +## `invalid-static-resource` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.79 · +Related issues · +View source + + + +**What it does** + + +Checks for basedpython static resource imports that cannot be read. + +**Why is this bad?** + + +`import "data/config.yaml" as config` says the file is part of the program. A +path that names nothing, a path that names a place on one machine, a file in a +format that is not `.json`, `.toml`, `.yaml` or `.yml`, and a document the +format's own parser rejects all leave the import with no value to bind. + +**Examples** + + +`main.by`: + +```by +# error: [invalid-static-resource] +import "data/config.txt" as config + +# error: [invalid-static-resource] +import "/etc/hosts.json" as hosts + +# error: [invalid-static-resource] +import "data/missing.json" as missing +``` + ## `invalid-super-argument` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3778,7 +3898,7 @@ class C: ... Default level: error · Added in 0.0.10 · Related issues · -View source +View source @@ -3829,7 +3949,7 @@ class MyClass: Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -3875,7 +3995,7 @@ InvalidAlias = TypeAliasType("InvalidAlias", list[T], type_params=(list[T],)) # Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -3942,7 +4062,7 @@ Bar[int] # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -3975,7 +4095,7 @@ TYPE_CHECKING = "" # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4011,7 +4131,7 @@ b: Annotated[int] # error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -4068,7 +4188,7 @@ class C: Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -4112,7 +4232,7 @@ def g[U, T: U](): ... # error: [invalid-type-variable-bound] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4169,7 +4289,7 @@ V = TypeVar("V", list[int], int) # valid constrained Type Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -4211,7 +4331,7 @@ U = TypeVar("U", int, str, default=bytes) # error: [invalid-type-variable-defau Default level: error · Added in 0.0.28 · Related issues · -View source +View source @@ -4247,7 +4367,7 @@ class Child(Base): Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -4290,7 +4410,7 @@ def f(options: dict[str, object]): Default level: error · Added in 0.0.9 · Related issues · -View source +View source @@ -4325,7 +4445,7 @@ class Foo(TypedDict): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -4366,7 +4486,7 @@ type Alias[out T] = list[T] # error: `list` is invariant Default level: error · Added in 0.0.25 · Related issues · -View source +View source @@ -4401,7 +4521,7 @@ def gen() -> Iterator[int]: Default level: error · Added in 0.0.14 · Related issues · -View source +View source @@ -4468,7 +4588,7 @@ def h(arg2: type): Default level: error · Added in 0.0.15 · Related issues · -View source +View source @@ -4518,7 +4638,7 @@ def g(arg: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -4549,7 +4669,7 @@ def f(s: str): Default level: warn · Added in 0.0.30 · Related issues · -View source +View source @@ -4592,7 +4712,7 @@ Movie = TypedDict("Film", {"title": str}) # error: [mismatched-type-name] Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -4660,7 +4780,7 @@ and nothing is reported. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4691,7 +4811,7 @@ func() # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -4722,7 +4842,7 @@ f(1) # ok — `s` is passed implicitly Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -4750,7 +4870,7 @@ from django.db import models # warning: install `django-stubs` for precise type Default level: error · Level under ty-compatible: ignore · Added in 0.0.41 · Related issues · -View source +View source @@ -4809,7 +4929,7 @@ class ExplicitChild(Parent): Default level: error · Level under ty-compatible: ignore · Added in 0.0.45 · Related issues · -View source +View source @@ -4848,7 +4968,7 @@ def handle(m: re.Match[str]) -> str: Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -4887,7 +5007,7 @@ alice["age"] # KeyError Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4921,7 +5041,7 @@ def f(a: int | None): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -4959,7 +5079,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.30 · Related issues · -View source +View source @@ -4997,7 +5117,7 @@ class Sub(Super): ... # error: [non-callable-init-subclass] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -5028,7 +5148,7 @@ def f(x: int | str) -> int: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5057,7 +5177,7 @@ def f(a: object): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5101,7 +5221,7 @@ def g(o: object, shape: Shape): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5130,7 +5250,7 @@ for i in 34: # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5158,7 +5278,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5187,7 +5307,7 @@ def f(once done: () -> None): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5215,7 +5335,7 @@ def f(once done: () -> None): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.61 · Related issues · -View source +View source @@ -5253,7 +5373,7 @@ def f(x: int?): Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5308,7 +5428,7 @@ def g(name: str | None): Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -5345,7 +5465,7 @@ class B(A): Default level: error · Added in 0.0.16 · Related issues · -View source +View source @@ -5382,7 +5502,7 @@ class B(A): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.38 · Related issues · -View source +View source @@ -5425,7 +5545,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5456,7 +5576,7 @@ f(1, x=2) # error Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -5487,7 +5607,7 @@ f(x=1) # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -5526,7 +5646,7 @@ A.c # error Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -5565,7 +5685,7 @@ A()[0] # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -5611,7 +5731,7 @@ from module import a # error Default level: warn · Added in 0.0.23 · Related issues · -View source +View source @@ -5643,7 +5763,7 @@ html.parser # error Default level: error · Level under ty-compatible: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5680,7 +5800,7 @@ print(x) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5712,7 +5832,7 @@ from helpers import Key # error: `Key` is private to `helpers` Default level: warn · Added in 0.0.60 · Related issues · -View source +View source @@ -5787,7 +5907,7 @@ def test() -> "int": Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5829,7 +5949,7 @@ def g(a: bool | None): Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -5864,7 +5984,7 @@ cast(int, f()) # error Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -5918,7 +6038,7 @@ if sys.version_info >= (3, 12): # ok — artificially constant Default level: warn · Added in 0.0.18 · Related issues · -View source +View source @@ -5956,7 +6076,7 @@ class C: Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6011,7 +6131,7 @@ class Sub(Base): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.62 · Related issues · -View source +View source @@ -6061,7 +6181,7 @@ def f(value: int | str) -> int: Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -6126,7 +6246,7 @@ def g(values: tuple[int, ...]) -> None: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -6159,7 +6279,7 @@ class C: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.72 · Related issues · -View source +View source @@ -6195,7 +6315,7 @@ class C[T]: Default level: warn · Added in 0.0.71 · Related issues · -View source +View source @@ -6238,7 +6358,7 @@ def build(t: Tag) -> None: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -6282,7 +6402,7 @@ class Outer[T]: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6317,7 +6437,7 @@ static_assert(int(2.0 * 3.0) == 6) # error Default level: warn · Added in 0.0.39 · Related issues · -View source +View source @@ -6368,7 +6488,7 @@ Consider using [`functools.total_ordering`][total_ordering] instead, which does Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6402,7 +6522,7 @@ class B(A): ... # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6434,7 +6554,7 @@ class Circle(Shape): ... # error: `Shape` is sealed in another workspace Default level: error · Added in 0.0.1-alpha.30 · Related issues · -View source +View source @@ -6543,7 +6663,7 @@ class Book: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6573,7 +6693,7 @@ f("foo") # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6612,7 +6732,7 @@ def find(items: list[int]) -> int: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6659,7 +6779,7 @@ g: Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6694,7 +6814,7 @@ f: # error: the block returns `None`, not `str` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6733,7 +6853,7 @@ def _(x: int): Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.4 · Related issues · -View source +View source @@ -6764,7 +6884,7 @@ class User(BaseModel): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -6822,7 +6942,7 @@ class A: Default level: error · Added in 0.0.20 · Related issues · -View source +View source @@ -6895,7 +7015,7 @@ the project registers with `@register.simple_block_tag`. Default level: warn · Added in 0.0.1-alpha.39 · Related issues · -View source +View source @@ -6961,7 +7081,7 @@ what the projects depending on it read. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -6990,7 +7110,7 @@ def f() raises TypeError: Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7019,7 +7139,7 @@ reveal_type(1) # revealed: Literal[1] Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.37 · Related issues · -View source +View source @@ -7049,7 +7169,7 @@ def main(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7080,7 +7200,7 @@ f(x=1, y=2) # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.36 · Related issues · -View source +View source @@ -7285,13 +7405,47 @@ page does not render at all. {% endfor %} ``` +## `unmet-module-api` + + +Default level: error · basedpython only, so absent under ty-compatible · +Added in 0.0.72 · +Related issues · +View source + + + +**What it does** + +Checks a module against the interfaces it is obliged to answer, whether it +declared one itself with `implements` or a package it lives in imposed one +with `implements ... for`. + +**Why is this bad?** + +A module that is meant to be used through an interface — a plugin, a +backend, a settings module — is otherwise only checked where something +assigns it to an interface-typed place. A plugin loaded by name is never +checked at all, and an error that does surface lands on the consumer +rather than on the module that broke. + +**Example** + + +```by +protocol Backend: + static def connect(url: str) -> None + +implements Backend # error: this module does not answer `Backend` +``` + ## `unresolved-attribute` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7324,7 +7478,7 @@ A().foo # error Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -7399,7 +7553,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7428,7 +7582,7 @@ import foo # error Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7458,7 +7612,7 @@ def check(value: int | None) -> asserts values: # error: `values` is nothing Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -7576,7 +7730,7 @@ is one whose template set cannot be established. Default level: error · Added in 0.0.71 · Related issues · -View source +View source @@ -7617,7 +7771,7 @@ def f(a: object, b: int, c: Any): Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -7760,7 +7914,7 @@ Python code. Default level: ignore · Added in 0.0.70 · Related issues · -View source +View source @@ -7903,7 +8057,7 @@ generator boundaries. Default level: error · basedpython only, so absent under ty-compatible · Added in 0.0.1-alpha.3 · Related issues · -View source +View source @@ -7953,7 +8107,7 @@ A() # error: nothing says which specialization this is Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -8000,7 +8154,7 @@ class D(C): ... # error: [unsupported-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8049,7 +8203,7 @@ b1 < b2 < b1 # error Default level: warn · Level under ty-compatible: ignore · Added in 0.0.12 · Related issues · -View source +View source @@ -8096,7 +8250,7 @@ def factory(base: type[Base]) -> type: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -8123,13 +8277,58 @@ class A: ... A() + A() # error ``` +## `unusable-resource-key` + + +Default level: warn · basedpython only, so absent under ty-compatible · +Added in 0.0.79 · +Related issues · +View source + + + +**What it does** + + +Checks for keys in an imported static resource that python cannot name. + +**Why is this bad?** + + +A static resource is read through attributes, so a key that is not a valid +python identifier — `build-backend`, `class`, `2` — has no attribute to be read +through, and is left out of the value the import binds. The document still holds +it; nothing in the program can reach it. + +Names with two leading underscores are left out for the same reason: python +mangles `__x` inside a class body, so the attribute the reader would write is +not the one that would exist. + +**Examples** + + +`data/project.json`: + +```json +{ "build-backend": "hatchling.build", "root": "." } +``` + +`main.by`: + +```by +# error: [unusable-resource-key] +import "data/project.json" as project + +reveal_type(project.root) # revealed: "." +``` + ## `unused-awaitable` Default level: warn · Added in 0.0.21 · Related issues · -View source +View source @@ -8208,7 +8407,7 @@ to `false` to prevent this rule from reporting unused `type: ignore` comments. Default level: warn · basedpython only, so absent under ty-compatible · Added in 0.0.71 · Related issues · -View source +View source @@ -8293,7 +8492,7 @@ to `false`. Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -8372,7 +8571,7 @@ def foo(x: int | str) -> int | str: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index da7f6a6b96..4521696b04 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -297,6 +297,11 @@ pub(crate) struct LoweringArgs { /// was made in #[arg(long)] pub(crate) no_unique_loop_bindings: bool, + /// supply a `build:` stamp, as `NAME=VALUE`. repeatable. overrides + /// whatever this build could have worked out for itself, which is what a + /// CI job that knows its own commit wants + #[arg(long = "stamp", value_name = "NAME=VALUE")] + pub(crate) stamps: Vec, } #[derive(Debug, Parser)] diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index aa4c7c9717..406065095a 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -21,6 +21,7 @@ use walkdir::WalkDir; use crate::ExitStatus; use crate::args::LoweringArgs; +use crate::by_lowering::SettledLowering; use by_stage::emit::{CheckGate, Transpiled, is_unusable_source, transpile_bug_diagnostic}; use by_stage::project::{ BY_SOURCES, COMPILABLE_SOURCES, Rebuilder, build_project_db, may_contain_sources, module_roots, @@ -61,12 +62,60 @@ pub(crate) fn parse_version(s: &str) -> anyhow::Result { } impl LoweringArgs { - /// Fold the lowering options into a config already carrying its target + /// Fold the lowering options this command was given into a config already + /// carrying its target version. + fn apply(&self, config: &mut Config, cwd: &Path) -> anyhow::Result<()> { + self.fold(&SettledLowering::from(self), config, cwd) + } + + /// [`apply`](Self::apply) for `by build`, which is the only command a + /// `--wheels` release runs inside itself. + /// + /// A release settles these once and hands them down, so a build that is one + /// of several inside one lowers the way the release asked rather than the + /// way its own command line reads — the backend passes no lowering options + /// on, so that command line is not the user's anyway. + /// + /// Every other command keeps [`apply`](Self::apply). These options change + /// what the emitted python does, and a variable left in the environment + /// must not silently re-lower a `by transpile` that was given options of its + /// own. + fn apply_for_build(&self, config: &mut Config, cwd: &Path) -> anyhow::Result<()> { + let settled = crate::by_lowering::settled_by_the_release() + .unwrap_or_else(|| SettledLowering::from(self)); + self.fold(&settled, config, cwd) + } + + /// Fold settled lowering options into a config already carrying its target /// version. - fn apply(&self, config: &mut Config) -> anyhow::Result<()> { - config.soundness = parse_soundness(&self.soundness)?; - config.runtime_raises_checks = self.runtime_raises_checks; - config.unique_loop_bindings = !self.no_unique_loop_bindings; + /// + /// `cwd` is where the project is looked for: the lowerings a project + /// configures belong to every command that emits python, so they are read + /// here rather than at each command's own config. Outside a project the + /// transpiler's own defaults stand. + fn fold( + &self, + lowering: &SettledLowering, + config: &mut Config, + cwd: &Path, + ) -> anyhow::Result<()> { + // destructured for the same reason the arguments are on the way in: an + // option carried across but never applied is dropped just as quietly as + // one never carried + let SettledLowering { + soundness, + runtime_raises_checks, + no_unique_loop_bindings, + } = lowering; + config.soundness = parse_soundness(soundness)?; + config.runtime_raises_checks = *runtime_raises_checks; + config.unique_loop_bindings = !*no_unique_loop_bindings; + if let Ok(project) = ResolvedProject::discover(cwd) { + config.float_literals = project.float_literals(); + } + // only what was asked for explicitly. what the build can work out for + // itself needs the project root, which is settled after this + config.stamps = crate::by_stamps::parse_explicit(&self.stamps)?; Ok(()) } } @@ -132,7 +181,9 @@ pub(crate) fn cmd_run( ); } } - lowering.apply(&mut config)?; + lowering.apply(&mut config, &cwd)?; + let target = config.min_version.to_string(); + crate::by_stamps::fill_discovered(&mut config.stamps, project.root(), Some(&target)); let tmp = tempfile::TempDir::new().context("failed to create temp directory")?; let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, None)?; @@ -626,6 +677,25 @@ impl ResolvedProject { &self.root } + /// How this project spells a float or complex literal type in the python it + /// emits. + fn float_literals(&self) -> by_transforms::FloatLiteralLowering { + match self + .metadata + .options() + .lowering + .as_ref() + .and_then(|lowering| lowering.float_literals.as_deref()) + { + Some(ty_project::metadata::options::FloatLiteralLowering::Literal) => { + by_transforms::FloatLiteralLowering::Literal + } + Some(ty_project::metadata::options::FloatLiteralLowering::Nominal) | None => { + by_transforms::FloatLiteralLowering::Nominal + } + } + } + /// The environment the project configures, as an absolute path. fn configured_environment(&self) -> Option { let sys_root = SystemPath::from_std_path(&self.root)?; @@ -772,7 +842,9 @@ pub(crate) fn cmd_build( ) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; let mut config = version_config(min_version, &cwd)?; - lowering.apply(&mut config)?; + lowering.apply_for_build(&mut config, &cwd)?; + let target = config.min_version.to_string(); + crate::by_stamps::fill_discovered(&mut config.stamps, &cwd, Some(&target)); // the output directory is settled before the project is read, because it is // the one directory the project must not be read *from*: it holds a copy of @@ -908,10 +980,16 @@ pub(crate) fn cmd_compile( // a declined function runs from the transpiled fallback, so it has to be // transpiled with the same options a `transpile` of this module would use — // otherwise the compiled half and the interpreted half check different things + let cwd = std::env::current_dir().context("failed to get current directory")?; let mut fallback = Config::default(); - lowering.apply(&mut fallback)?; + lowering.apply(&mut fallback, &cwd)?; + // no target python: `by_build` transpiles the interpreted twin for whatever + // interpreter the toolchain turns out to be, replacing this config's version + // as it goes — so naming one here would stamp a python the output was not + // lowered for. a program that wants `PYTHON_VERSION` under `by compile` has + // to say which, and gets told so rather than told the wrong one + crate::by_stamps::fill_discovered(&mut fallback.stamps, &cwd, None); options.fallback = Some(fallback); - let cwd = std::env::current_dir().context("failed to get current directory")?; let sources: Vec = if files.is_empty() { compilable_files(&cwd) } else { @@ -1113,7 +1191,9 @@ pub(crate) fn cmd_transpile( ) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; let mut config = version_config(min_version, &cwd)?; - lowering.apply(&mut config)?; + lowering.apply(&mut config, &cwd)?; + let target = config.min_version.to_string(); + crate::by_stamps::fill_discovered(&mut config.stamps, &cwd, Some(&target)); // a directory argument transpiles the whole tree in place: forward turns // every `.by` into a `.py` (type-aware, one shared project db); reverse diff --git a/crates/ty/src/by_lowering.rs b/crates/ty/src/by_lowering.rs new file mode 100644 index 0000000000..82df0fca5c --- /dev/null +++ b/crates/ty/src/by_lowering.rs @@ -0,0 +1,185 @@ +//! The lowering options a release settles for the builds inside it. +//! +//! `by build --wheels` is not one build. It calls the packaging frontend once +//! for the source distribution and once per python version, and each of those +//! reaches a fresh `by build` through the PEP 517 backend. Those inner builds +//! are where the transpiling actually happens, so a lowering option that stops +//! at the outer command changes nothing at all — `--soundness none` would be +//! accepted and then quietly produce wheels full of soundness checks. +//! +//! So the outer command settles the options once and hands them down, the same +//! way and for much the same reason as the stamps in [`crate::by_stamps`]: the +//! wheels of one release have to be one artifact set, and wheels lowered under +//! different options are not that. +//! +//! Only `by build` reads this, because `by build` is the only command the +//! backend runs. These options change what the emitted python *does*, so a +//! variable left in the environment must not quietly re-lower an unrelated +//! `by transpile` that was given options of its own. +//! +//! The two `by` executables involved are not necessarily the same build. A +//! project's `requires` names `basedpython` with a lower bound, so the frontend +//! resolves whatever is newest into the build environment, and the backend +//! prefers that one to whatever is on `PATH`. The outer `by` a user typed can +//! therefore be older than the inner one it hands these to — which is why every +//! field defaults rather than being required. A build that receives options +//! written by a `by` that had not heard of one of them takes the rest and +//! defaults that one, instead of failing to read the message and silently +//! lowering as though a release had settled nothing. +//! +//! The soundness spec crosses as the string the flag takes, so a *newer* outer +//! `by` can name a position an older inner one cannot parse. That fails the +//! build loudly, from inside the frontend, which is worse to read than it needs +//! to be but does not ship a wheel lowered other than as asked. + +use serde::{Deserialize, Serialize}; + +use crate::args::LoweringArgs; + +/// How one release hands its settled lowering options to the builds inside it. +pub(crate) const SETTLED_LOWERING: &str = "BY_BUILD_LOWERING"; + +/// The lowering options themselves, in the shape the command line spells them. +/// +/// Deliberately the same field names and senses as [`LoweringArgs`], including +/// the negated `no_unique_loop_bindings`: this is copied from one and applied as +/// the other, and a field that changed sense on the way through would invert a +/// lowering option with nothing to catch it. +/// +/// That is also why this is not [`by_stage::record::ConfigRecord`], which +/// carries the same three options for the build record. It spells the loop +/// option positively, renames its fields for its own file format, and refuses +/// unknown ones — all right for a record read back by the same `by` that wrote +/// it, and all wrong for a message crossing between two versions of `by`. +/// +/// [`by_stage::record::ConfigRecord`]: by_stage::record::ConfigRecord +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct SettledLowering { + /// Defaulted to the spelling the flag itself defaults to. An empty spec is + /// a valid one meaning *no checks at all*, so a missing field must not fall + /// back to `String::default`. + #[serde(default = "unset_soundness")] + pub(crate) soundness: String, + #[serde(default)] + pub(crate) runtime_raises_checks: bool, + #[serde(default)] + pub(crate) no_unique_loop_bindings: bool, +} + +/// What `--soundness` means when nothing asked for anything, matching the +/// flag's own `default_value`. +fn unset_soundness() -> String { + "default".to_owned() +} + +impl From<&LoweringArgs> for SettledLowering { + fn from(arguments: &LoweringArgs) -> Self { + // destructured rather than read field by field. a lowering option added + // to the arguments and forgotten here would be accepted by a `--wheels` + // release and then ignored by every build inside it — which is the bug + // this whole mechanism exists to close, so it should not be possible to + // reopen it quietly. written this way, it does not compile + let LoweringArgs { + soundness, + runtime_raises_checks, + no_unique_loop_bindings, + // not one of these: a stamp's value is discovered as well as given, + // so the release settles them itself rather than copying them across + stamps: _, + } = arguments; + Self { + soundness: soundness.clone(), + runtime_raises_checks: *runtime_raises_checks, + no_unique_loop_bindings: *no_unique_loop_bindings, + } + } +} + +/// The options an enclosing release settled, when this build is one of several +/// inside one. +pub(crate) fn settled_by_the_release() -> Option { + settled_from(std::env::var(SETTLED_LOWERING).ok().as_deref()) +} + +/// [`settled_by_the_release`] against a given value, so the reading of it can be +/// tested without writing to the process environment. +/// +/// Unreadable content is ignored rather than refused, for the same reason the +/// stamps are: the variable is only ever written by the outer command, so +/// anything else in it belongs to whatever else set it, and a build that +/// stopped over it would be refusing to run for a reason nobody could act on. +fn settled_from(raw: Option<&str>) -> Option { + serde_json::from_str(raw?).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// What one release writes, a build inside it has to be able to read. + #[test] + fn settled_options_round_trip_through_the_variable() { + let settled = SettledLowering { + soundness: "generic-calls,returns".to_owned(), + runtime_raises_checks: true, + no_unique_loop_bindings: true, + }; + let read = settled_from(Some(&serde_json::to_string(&settled).unwrap())) + .expect("a release's own message reads back"); + assert_eq!(read.soundness, "generic-calls,returns"); + assert!(read.runtime_raises_checks); + assert!(read.no_unique_loop_bindings); + } + + /// The two `by` executables in a release need not be the same build — the + /// outer one can be older than the inner one it hands these to. An option + /// the writer had never heard of must cost only itself, because the + /// alternative is reading nothing and lowering as though no release had + /// settled anything, which is the bug this mechanism exists to close. + #[test] + fn options_written_by_an_older_by_still_read() { + let read = settled_from(Some(r#"{"soundness":"none"}"#)).expect("the rest still reads"); + assert_eq!(read.soundness, "none"); + assert!(!read.runtime_raises_checks); + assert!(!read.no_unique_loop_bindings); + } + + /// An empty spec is a valid one meaning *no checks*, so the absent case has + /// to land on the flag's own default rather than on `String::default`. + #[test] + fn an_unstated_soundness_is_the_default_not_the_empty_spec() { + let read = settled_from(Some(r#"{"runtime_raises_checks":true}"#)).expect("reads"); + assert_eq!(read.soundness, "default"); + assert!(read.runtime_raises_checks); + } + + /// Whatever else set the variable, this is not a reason to refuse to build. + #[test] + fn content_that_is_not_settled_options_is_ignored() { + assert!(settled_from(None).is_none()); + assert!(settled_from(Some("not the outer command's")).is_none()); + assert!(settled_from(Some("[1, 2]")).is_none()); + } + + /// The arguments are carried across whole, and the negated one is the one + /// worth pinning: carried with its sense flipped it would turn the option + /// into its opposite in every wheel of the release. + #[test] + fn the_arguments_are_carried_across_unchanged() { + let arguments = LoweringArgs { + soundness: "none".to_owned(), + runtime_raises_checks: true, + no_unique_loop_bindings: true, + ..LoweringArgs::default() + }; + let settled = SettledLowering::from(&arguments); + assert_eq!(settled.soundness, "none"); + assert!(settled.runtime_raises_checks); + assert!(settled.no_unique_loop_bindings); + + let defaults = LoweringArgs::default(); + let settled = SettledLowering::from(&defaults); + assert!(!settled.runtime_raises_checks); + assert!(!settled.no_unique_loop_bindings); + } +} diff --git a/crates/ty/src/by_stamps.rs b/crates/ty/src/by_stamps.rs new file mode 100644 index 0000000000..6a29a1b024 --- /dev/null +++ b/crates/ty/src/by_stamps.rs @@ -0,0 +1,294 @@ +//! The values a `build:` block's stamps take. +//! +//! Discovery lives here, at the command layer, and never in the transpiler. The +//! emitted python has to be a function of the source and the transpile config; +//! a pipeline that asked git for the commit itself would answer differently on +//! every run, and a re-stage — which re-transpiles one file into a tree an +//! earlier build wrote — would put a module claiming one commit beside modules +//! claiming another. +//! +//! So a command settles the stamps once, hands them to the transpiler as +//! config, and the build record writes them down so a later re-stage reproduces +//! them exactly. +//! +//! Nothing here guesses. A project outside a git checkout, or on a machine with +//! no `git`, gets no git stamps at all — and a block that declared one with no +//! default then fails to transpile, which is the promise the declaration made. +//! Inventing `"unknown"` would let a build claim a commit it does not have. + +use std::collections::BTreeMap; +use std::path::Path; +use std::process::Command; + +use anyhow::Context; + +/// `SOURCE_DATE_EPOCH` — the ecosystem's agreement on what "now" is for a build +/// that has to come out the same twice. +/// +/// Honouring it is what lets a project with a `BUILT_AT` stamp still produce a +/// reproducible wheel; without it the artifact differs on every build and +/// defeats both caching and anyone trying to verify it was built from the +/// source it claims. +const SOURCE_DATE_EPOCH: &str = "SOURCE_DATE_EPOCH"; + +/// How one release hands its settled stamps to the builds inside it. +/// +/// `by build --wheels` is not one build: it calls the packaging frontend once +/// for the source distribution and once per python version, and each of those +/// reaches a fresh `by build` that would settle its own stamps. Two wheels of +/// one release stamped a second apart — or, if something landed mid-release, +/// two different commits — are not one artifact set, so the outer command +/// settles the stamps once and passes them down through here. +pub(crate) const SETTLED_STAMPS: &str = "BY_BUILD_STAMPS"; + +/// Parse the `--stamp NAME=VALUE` arguments into the map the transpiler takes. +/// +/// A value may contain `=`; only the first splits the pair, so +/// `--stamp DESCRIBE=v1.2-3-gabc=def` is one stamp. +pub(crate) fn parse_explicit(arguments: &[String]) -> anyhow::Result> { + let mut stamps = BTreeMap::new(); + for argument in arguments { + let (name, value) = argument.split_once('=').with_context(|| { + format!("`--stamp {argument}` is not a `NAME=VALUE` pair — a stamp needs both") + })?; + if name.is_empty() { + anyhow::bail!("`--stamp {argument}` names no stamp"); + } + if let Some(replaced) = stamps.insert(name.to_owned(), value.to_owned()) { + anyhow::bail!( + "`--stamp {name}` was given twice, as {replaced:?} and {value:?} — which one the \ + build meant is not something to guess at" + ); + } + } + Ok(stamps) +} + +/// Fill in the stamps this build can work out for itself, leaving anything +/// already in `stamps` alone. +/// +/// An explicit `--stamp` wins, because a CI job that knows the commit it was +/// dispatched for knows it better than a checkout that may be a detached head +/// or a shallow clone. +/// +/// `target_version` is the python the output is being lowered to, and `None` +/// says this command is not lowering for one — `by build --wheels` settles the +/// stamps its wheels share, and each wheel is lowered to a different python, so +/// `PYTHON_VERSION` is the one stamp the release must leave to them. +pub(crate) fn fill_discovered( + stamps: &mut BTreeMap, + project_root: &Path, + target_version: Option<&str>, +) { + let mut supply = |name: &str, value: Option| { + if let Some(value) = value + && !stamps.contains_key(name) + { + stamps.insert(name.to_owned(), value); + } + }; + + // a release that already settled its stamps says so, and this build is one + // of the several inside it rather than one of its own + if let Some(settled) = settled_by_the_release() { + for (name, value) in settled { + supply(&name, Some(value)); + } + } + + let head = git(project_root, &["rev-parse", "HEAD"]); + let head_found = head.is_some(); + supply("GIT_SHA_SHORT", head.as_ref().map(|sha| short(sha))); + supply("GIT_SHA", head); + supply( + "GIT_BRANCH", + git(project_root, &["rev-parse", "--abbrev-ref", "HEAD"]), + ); + // `--exact-match` so this is the tag *of this commit* and not the nearest + // one behind it: a stamp saying `v1.2.0` on three commits past the tag is a + // release claim nobody made. + // + // an untagged commit is the ordinary case rather than a failure to discover + // anything, so inside a repository the answer is the empty string. `describe` + // exits non-zero either way, which is why this asks whether there was a + // repository at all rather than reading its status + if head_found { + supply( + "GIT_TAG", + Some(git(project_root, &["describe", "--tags", "--exact-match"]).unwrap_or_default()), + ); + } + // asked separately from the sha, and reported rather than folded into it: a + // build from a tree with uncommitted changes is not the commit it names, and + // a program that cannot say so will eventually be asked to explain a stack + // trace that does not match its source + supply( + "GIT_DIRTY", + git(project_root, &["status", "--porcelain"]) + .map(|changes| if changes.is_empty() { "false" } else { "true" }.to_owned()), + ); + supply("BUILT_AT", Some(built_at())); + supply("PYTHON_VERSION", target_version.map(str::to_owned)); +} + +/// The stamps an enclosing release settled, when this build is one of several +/// inside one. +/// +/// Unreadable content is ignored rather than refused: the variable is only ever +/// written by the outer command, so anything else in it belongs to whatever else +/// set it, and a build that stopped over it would be refusing to run for a reason +/// nobody could act on. +fn settled_by_the_release() -> Option> { + serde_json::from_str(&std::env::var(SETTLED_STAMPS).ok()?).ok() +} + +/// The first twelve characters of a commit hash — long enough to be unambiguous +/// in any real repository, short enough to read in a `--version` line. +fn short(sha: &str) -> String { + sha.chars().take(12).collect() +} + +/// The moment the build happened, as an RFC 3339 timestamp in UTC. +/// +/// `SOURCE_DATE_EPOCH` replaces it when set, so a reproducible build stays +/// reproducible. +fn built_at() -> String { + built_at_from(std::env::var(SOURCE_DATE_EPOCH).ok().as_deref()) +} + +/// [`built_at`] against a given `SOURCE_DATE_EPOCH`, so the reading of it can be +/// tested without reaching into the process environment. +/// +/// A value that is not a number of seconds is ignored rather than refused: the +/// variable belongs to whatever invoked the build, and a build that stopped +/// because something upstream set it oddly would be harder to explain than one +/// that stamped the clock. +fn built_at_from(epoch: Option<&str>) -> String { + if let Some(epoch) = epoch + && let Ok(seconds) = epoch.trim().parse::() + && let Ok(timestamp) = jiff::Timestamp::from_second(seconds) + { + return timestamp.to_string(); + } + jiff::Timestamp::now().to_string() +} + +/// Run `git` in the project and return its trimmed output, or `None` for any +/// reason at all it did not answer: no git on the machine, no repository, a +/// repository with no commits, a `describe` that found no tag. +/// +/// Each of those is a real situation a build runs in — a source distribution +/// unpacked from a package index has no `.git` at all — and none of them is an error here. +/// The declaration is what decides whether a missing stamp is fatal. +fn git(project_root: &Path, arguments: &[&str]) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(project_root) + .args(arguments) + .output() + .ok()?; + if !output.status.success() { + return None; + } + Some(String::from_utf8(output.stdout).ok()?.trim().to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn settled_stamps_round_trip_through_the_variable() { + // what one release writes, a build inside it has to be able to read — + // values with `=`, quotes and newlines in them included + let mut settled = BTreeMap::new(); + settled.insert("GIT_SHA".to_owned(), "abc123".to_owned()); + settled.insert("SUBJECT".to_owned(), "fix \"one\"=two\nand more".to_owned()); + let rendered = serde_json::to_string(&settled).unwrap(); + let read: BTreeMap = serde_json::from_str(&rendered).unwrap(); + assert_eq!(read, settled); + } + + #[test] + fn a_pair_splits_on_its_first_equals() { + let stamps = parse_explicit(&["DESCRIBE=v1.2-3-gabc=def".to_owned()]).unwrap(); + assert_eq!(stamps["DESCRIBE"], "v1.2-3-gabc=def"); + } + + #[test] + fn a_stamp_without_a_value_is_refused() { + let error = parse_explicit(&["GIT_SHA".to_owned()]).unwrap_err(); + assert!(error.to_string().contains("is not a `NAME=VALUE` pair")); + } + + #[test] + fn the_same_stamp_twice_is_refused() { + let error = parse_explicit(&["A=1".to_owned(), "A=2".to_owned()]).unwrap_err(); + assert!(error.to_string().contains("was given twice")); + } + + #[test] + fn an_explicit_stamp_is_not_overwritten_by_discovery() { + let mut stamps = parse_explicit(&["GIT_SHA=deadbeef".to_owned()]).unwrap(); + fill_discovered(&mut stamps, Path::new("."), Some("3.13")); + assert_eq!(stamps["GIT_SHA"], "deadbeef"); + } + + #[test] + fn discovery_supplies_the_target_python() { + let mut stamps = BTreeMap::new(); + fill_discovered(&mut stamps, Path::new("."), Some("3.13")); + assert_eq!(stamps["PYTHON_VERSION"], "3.13"); + } + + /// A release settles what its wheels share. Each wheel is lowered to a + /// different python, so that one stamp is not the release's to settle. + #[test] + fn a_release_leaves_the_target_python_to_its_wheels() { + let mut stamps = BTreeMap::new(); + fill_discovered(&mut stamps, Path::new("."), None); + assert!(!stamps.contains_key("PYTHON_VERSION")); + assert!(stamps.contains_key("BUILT_AT")); + } + + #[test] + fn source_date_epoch_replaces_the_clock() { + // the epoch itself, and a moment well after it + assert_eq!(built_at_from(Some("0")), "1970-01-01T00:00:00Z"); + assert_eq!(built_at_from(Some(" 1750000000 ")), "2025-06-15T15:06:40Z"); + } + + #[test] + fn a_source_date_epoch_that_is_not_a_time_is_ignored() { + // the variable belongs to whatever ran the build; a build that stopped + // over it would be harder to explain than one that stamped the clock + for odd in ["", "yesterday", "1e9", "999999999999999999999"] { + assert_ne!(built_at_from(Some(odd)), "1970-01-01T00:00:00Z", "{odd:?}"); + } + } + + /// Ruff's own checkout is a repository with no tag on `HEAD` most of the + /// time, and an untagged commit is an ordinary state rather than a discovery + /// that failed — so the stamp is there and empty, not absent. + #[test] + fn an_untagged_commit_in_a_repository_still_has_a_tag_stamp() { + let mut stamps = BTreeMap::new(); + fill_discovered(&mut stamps, Path::new("."), Some("3.13")); + // only meaningful where there *is* a repository to be in + if stamps.contains_key("GIT_SHA") { + assert!(stamps.contains_key("GIT_TAG")); + } + } + + #[test] + fn a_directory_that_is_no_repository_gets_no_git_stamps() { + let outside = tempfile::TempDir::new().unwrap(); + let mut stamps = BTreeMap::new(); + fill_discovered(&mut stamps, outside.path(), Some("3.13")); + // `git` walks upward, so the only thing that reliably holds here is that + // nothing was invented: whatever it did or did not find, no stamp says + // "unknown" + assert!(stamps.values().all(|value| value != "unknown")); + assert!(stamps.contains_key("BUILT_AT")); + } +} diff --git a/crates/ty/src/by_wheels.rs b/crates/ty/src/by_wheels.rs index 7ace66e683..85795aa77b 100644 --- a/crates/ty/src/by_wheels.rs +++ b/crates/ty/src/by_wheels.rs @@ -27,6 +27,9 @@ use ty_project::{Db, ProjectDatabase, ProjectMetadata}; use ty_site_packages::PythonEnvironment; use crate::ExitStatus; +use crate::args::LoweringArgs; +use crate::by_lowering::SettledLowering; +use by_stage::record::parse_soundness; /// The tag every wheel carries when it was not lowered for one python. /// @@ -35,11 +38,50 @@ use crate::ExitStatus; /// generic wheel silently wins over the whole set. const UNTAGGED_WHEEL_MARKER: &str = "-py3-none-any.whl"; +/// What a release settles once and hands to every build inside it, each as the +/// json its environment variable carries. +/// +/// One value rather than two strings side by side. They are both json and both +/// bound for an environment variable, so passing them positionally would let a +/// swap compile — and a swap is silent at run time too, because each reader +/// ignores a value it cannot parse and falls back to its own default. +struct SettledRelease { + stamps: String, + lowering: String, +} + #[allow(clippy::print_stderr)] -pub(crate) fn cmd_build_wheels(out: Option<&Path>) -> anyhow::Result { +pub(crate) fn cmd_build_wheels( + out: Option<&Path>, + lowering: &LoweringArgs, +) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; let destination = cwd.join(out.unwrap_or(Path::new("dist"))); + // everything the command line asked for is read before any of it is acted + // on, so a spec it cannot parse is one error from the command the user ran + // — not the same error from each of the builds inside the release, and not + // one arriving after a `git` discovery has already run + let mut stamps = crate::by_stamps::parse_explicit(&lowering.stamps)?; + let lowering_options = SettledLowering::from(lowering); + let _ = parse_soundness(&lowering_options.soundness)?; + + // a release is one build, however many times the frontend is called inside + // it. what it settles is settled here and handed down, so every wheel and + // the source distribution report the same commit and the same moment rather + // than each reading the clock — and, if something lands mid-release, the + // same commit rather than two — and every one of them lowers the way the + // release was asked to rather than the way the backend happens to call it. + // `PYTHON_VERSION` is left out on purpose: each wheel is lowered to a + // different python and settles that one for itself + crate::by_stamps::fill_discovered(&mut stamps, &cwd, None); + let settled = SettledRelease { + stamps: serde_json::to_string(&stamps) + .context("could not describe the stamps this release settled")?, + lowering: serde_json::to_string(&lowering_options) + .context("could not describe the lowering options this release settled")?, + }; + let uv = find_uv(&cwd)?; let versions = wheel_versions(&cwd)?; if versions.is_empty() { @@ -64,7 +106,7 @@ pub(crate) fn cmd_build_wheels(out: Option<&Path>) -> anyhow::Result // older one and no one is told let staging = tempfile::TempDir::new().context("failed to create temp directory")?; - run_uv(&uv, &cwd, &["build", "--sdist"], staging.path())?; + run_uv(&uv, &cwd, &["build", "--sdist"], staging.path(), &settled)?; for version in &versions { run_uv( &uv, @@ -76,6 +118,7 @@ pub(crate) fn cmd_build_wheels(out: Option<&Path>) -> anyhow::Result &format!("python-version={version}"), ], staging.path(), + &settled, )?; } @@ -310,12 +353,22 @@ fn which_uv() -> Option { } #[allow(clippy::print_stderr)] -fn run_uv(uv: &Path, cwd: &Path, arguments: &[&str], out: &Path) -> anyhow::Result<()> { +fn run_uv( + uv: &Path, + cwd: &Path, + arguments: &[&str], + out: &Path, + settled: &SettledRelease, +) -> anyhow::Result<()> { let status = Command::new(uv) .args(arguments) .arg("--out-dir") .arg(out) .current_dir(cwd) + // both reach the `by build` the frontend eventually runs, through the + // backend, which passes its environment on + .env(crate::by_stamps::SETTLED_STAMPS, &settled.stamps) + .env(crate::by_lowering::SETTLED_LOWERING, &settled.lowering) .status() .with_context(|| format!("could not run `{}`", uv.display()))?; if !status.success() { diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index e963f2a2cf..88fe69ad31 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -1,7 +1,9 @@ mod args; mod by_commands; mod by_init; +mod by_lowering; mod by_source_encoding; +mod by_stamps; mod by_wheels; mod logging; mod printer; @@ -149,7 +151,7 @@ fn run_command(command: Command) -> anyhow::Result { lowering, } => { if wheels { - by_wheels::cmd_build_wheels(out.as_deref()) + by_wheels::cmd_build_wheels(out.as_deref(), &lowering) } else { by_commands::cmd_build( min_version.as_deref(), diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 33c103b661..6beedbb68e 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -777,6 +777,218 @@ fn run_main_with_args(source: &str, args: &[&str]) -> (String, String, i32) { ) } +/// Build stamps are experimental, so a project that writes a `build:` block has +/// to ask for them by name — these tests are projects like any other. +const OPT_IN_TO_STAMPS: &str = "[experimental]\nbuild-stamps = true\n"; + +/// `--stamp` comes before the module: everything after the module name is the +/// program's own `sys.argv` +fn run_main_stamped(source: &str, stamps: &[&str]) -> (String, String, i32) { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), source).unwrap(); + fs::write(dir.path().join("basedpython.toml"), OPT_IN_TO_STAMPS).unwrap(); + + let mut command = Command::new(env!("CARGO_BIN_EXE_by")); + command.arg("run"); + for stamp in stamps { + command.args(["--stamp", stamp]); + } + let output = command + .arg("main") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + ( + String::from_utf8_lossy(&output.stdout).trim().to_owned(), + String::from_utf8_lossy(&output.stderr).trim().to_owned(), + output.status.code().unwrap_or(-1), + ) +} + +#[test] +fn run_supplies_a_stamp_the_command_line_gave_it() { + let source = r#" +build: + GIT_SHA: str + BUILD_NUMBER: int + +def main(): + print(build.GIT_SHA, build.BUILD_NUMBER + 1) +"#; + + let (stdout, stderr, code) = + run_main_stamped(source, &["GIT_SHA=deadbeef", "BUILD_NUMBER=416"]); + assert_eq!( + (stdout.as_str(), code), + ("deadbeef 417", 0), + "stderr:\n{stderr}" + ); +} + +/// a stamp with no default is a claim the build has to satisfy, and nothing +/// supplies a name this one invented — so the transpile has to say so rather +/// than reach for a value +#[test] +fn run_refuses_a_required_stamp_nothing_supplied() { + let source = r#" +build: + RELEASE_CHANNEL: str + +def main(): + print(build.RELEASE_CHANNEL) +"#; + + let (_, stderr, code) = run_main_stamped(source, &[]); + assert_ne!(code, 0, "a stamp nothing supplied must not build"); + assert!( + stderr.contains("supplied no value for the stamp `RELEASE_CHANNEL`"), + "stderr:\n{stderr}" + ); +} + +#[test] +fn run_falls_back_to_a_stamp_default() { + let source = r#" +build: + RELEASE_CHANNEL: str = "dev" + +def main(): + print(build.RELEASE_CHANNEL) +"#; + + let (stdout, stderr, code) = run_main_stamped(source, &[]); + assert_eq!((stdout.as_str(), code), ("dev", 0), "stderr:\n{stderr}"); +} + +/// the python the output was lowered to is one of the values a build knows +/// without being told, and `by run` lowers for the interpreter it runs on +#[test] +fn run_discovers_the_python_it_lowered_for() { + let source = r#" +import sys + +build: + PYTHON_VERSION: str + +def main(): + print(build.PYTHON_VERSION == f"{sys.version_info[0]}.{sys.version_info[1]}") +"#; + + let (stdout, stderr, code) = run_main_stamped(source, &[]); + assert_eq!((stdout.as_str(), code), ("True", 0), "stderr:\n{stderr}"); +} + +/// An interpreter that can host a native build, or `None` to skip. +/// +/// `by compile` needs 3.11 or later, so on a host whose ambient python is older +/// the compiled leg cannot run at all — see `by_build::MINIMUM_PYTHON`. +fn native_interpreter() -> Option { + let candidates = std::env::var("PYTHON") + .map(|python| vec![python]) + .unwrap_or_else(|_| { + [ + "python3.14", + "python3.13", + "python3.12", + "python3.11", + "python3", + ] + .map(String::from) + .to_vec() + }); + candidates.into_iter().find(|python| { + Command::new(python) + .args([ + "-c", + "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)", + ]) + .status() + .is_ok_and(|status| status.success()) + }) +} + +/// A stamp has to survive native compilation with its value intact. +/// +/// The entry module always runs interpreted, so the block goes in an imported +/// module — the one that actually gets compiled. A stamp reaching the native +/// leg as an annotation with no value would leave `build.GIT_SHA` an +/// `AttributeError` that nothing else in this suite would catch. +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn a_stamp_survives_native_compilation() { + let Some(python) = native_interpreter() else { + eprintln!("skipping: no interpreter new enough for a native build"); + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("stamps.by"), + r#" +build: + GIT_SHA: str + GIT_DIRTY: bool + +def describe() -> str: + return build.GIT_SHA + ("-dirty" if build.GIT_DIRTY else "") +"#, + ) + .unwrap(); + fs::write( + dir.path().join("main.by"), + "from stamps import describe + +def main(): + print(describe()) +", + ) + .unwrap(); + fs::write(dir.path().join("basedpython.toml"), OPT_IN_TO_STAMPS).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "--compiled", "--python"]) + .arg(&python) + .args([ + "--stamp", + "GIT_SHA=abc123", + "--stamp", + "GIT_DIRTY=true", + "main", + ]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.contains("C toolchain") || stderr.contains("no working C compiler") { + eprintln!("skipping: no working C toolchain"); + return; + } + assert!( + output.status.success(), + "by run --compiled failed:\n{stderr}" + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "abc123-dirty", + "stderr:\n{stderr}" + ); +} + +#[test] +fn a_stamp_that_is_not_a_pair_is_refused() { + let (_, stderr, code) = run_main_stamped("def main(): ...\n", &["GIT_SHA"]); + assert_ne!(code, 0); + assert!( + stderr.contains("is not a `NAME=VALUE` pair"), + "stderr:\n{stderr}" + ); +} + #[test] fn run_fills_main_parameter_positionally_or_by_name() { let source = "def main(name: str):\n print(name)\n"; @@ -3838,6 +4050,237 @@ fn building_wheels_without_a_frontend_says_what_is_missing() { ); } +/// one source touched by all three lowering options, so a test can tell which +/// of them crossed the wire and which quietly did not +/// +/// each moves in a different direction, so a payload of non-defaults cannot +/// pass by accident: the soundness check disappears, the raises guard appears, +/// and the loop's per-iteration wrapper disappears +const LOWERED_THREE_WAYS: &str = "\ +items: list[int] = [] +fns: list[object] = [] + + +def t[T]() -> T: + raise NotImplementedError + + +def uses(): + a: str = t() + + +def f() raises ValueError: + raise ValueError + + +for i in items: + fns.append(lambda: i) +"; + +/// build `source` in a fresh project and return the emitted python +fn build_emitting(source: &str, settled: Option<&str>) -> String { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), source).unwrap(); + + let mut command = Command::new(env!("CARGO_BIN_EXE_by")); + command + .args(["build", "--out", "out"]) + .current_dir(dir.path()); + match settled { + Some(settled) => command.env("BY_BUILD_LOWERING", settled), + // whatever the developer's own environment holds must not decide what + // this observes + None => command.env_remove("BY_BUILD_LOWERING"), + }; + let output = command.output().expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + assert!(output.status.success(), "by build failed:\n{stderr}"); + fs::read_to_string(dir.path().join("out").join("main.py")).expect("emitted python") +} + +/// the builds inside a `--wheels` release are the ones that transpile, so every +/// lowering option has to reach them. one that stopped at the outer command +/// would be accepted and then change nothing about the wheels +#[test] +fn a_build_inside_a_release_lowers_the_way_the_release_settled() { + // on its own the build lowers with the defaults, which is what makes each + // difference below mean something + let alone = build_emitting(LOWERED_THREE_WAYS, None); + assert!( + alone.contains("_soundness_check(t(), str)"), + "checks are on by default:\n{alone}" + ); + assert!( + !alone.contains("_by_raises"), + "the raises guard is off by default:\n{alone}" + ); + assert!( + alone.contains("fns.append((lambda i: lambda: i)(i))"), + "a loop binds per iteration by default:\n{alone}" + ); + + let inside_a_release = build_emitting( + LOWERED_THREE_WAYS, + Some(r#"{"soundness":"none","runtime_raises_checks":true,"no_unique_loop_bindings":true}"#), + ); + assert!( + !inside_a_release.contains("_soundness_check"), + "the release settled `--soundness none`:\n{inside_a_release}" + ); + assert!( + inside_a_release.contains("_by_raises"), + "the release settled `--runtime-raises-checks`:\n{inside_a_release}" + ); + assert!( + inside_a_release.contains("fns.append(lambda: i)"), + "the release settled `--no-unique-loop-bindings`:\n{inside_a_release}" + ); +} + +/// the two `by` executables in a release need not be the same build, because a +/// project asks for `basedpython` by a lower bound and the backend prefers the +/// one the frontend resolved. so a build has to read a message written by a +/// `by` that had never heard of one of these options, and take the rest of it +#[test] +fn a_build_reads_settled_lowering_written_by_an_older_by() { + let emitted = build_emitting(LOWERED_THREE_WAYS, Some(r#"{"soundness":"none"}"#)); + assert!( + !emitted.contains("_soundness_check"), + "the option that was written still applies:\n{emitted}" + ); + assert!( + !emitted.contains("_by_raises"), + "and one that was not written takes its own default:\n{emitted}" + ); +} + +/// the variable is the outer command's, so anything else in it is not this +/// build's to refuse: it lowers the way it would have anyway rather than +/// stopping for a reason nobody could act on +#[test] +fn a_build_ignores_settled_lowering_it_cannot_read() { + let emitted = build_emitting(LOWERED_THREE_WAYS, Some("not the outer command's")); + assert!( + emitted.contains("_soundness_check(t(), str)"), + "an unreadable value leaves the lowering alone:\n{emitted}" + ); +} + +/// only `by build` is ever run inside a release, and these options change what +/// the emitted python does — so a variable left in the environment must not +/// silently re-lower a `by transpile` that was given options of its own +#[test] +fn only_a_build_takes_its_lowering_from_the_environment() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), LOWERED_THREE_WAYS).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["transpile", "main.by"]) + .current_dir(dir.path()) + .env( + "BY_BUILD_LOWERING", + r#"{"soundness":"none","runtime_raises_checks":false,"no_unique_loop_bindings":false}"#, + ) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by transpile failed:\n{stderr}"); + let emitted = String::from_utf8_lossy(&output.stdout); + assert!( + emitted.contains("_soundness_check(t(), str)"), + "a stray variable must not re-lower an unrelated transpile:\n{emitted}" + ); +} + +/// the release settles the options once, so a spec it cannot parse is one error +/// from the command the user ran — not the same error from each of the builds +/// inside it, and not a release that got as far as driving the frontend +#[test] +fn building_wheels_refuses_a_soundness_spec_it_cannot_parse() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--wheels", "--soundness", "nonsense"]) + .current_dir(dir.path()) + // refused before the frontend is even looked for, so this holds on a + // machine with no `uv` as well as on one with it + .env("PATH", "") + .env_remove("VIRTUAL_ENV") + .output() + .expect("failed to spawn by"); + + assert!(!output.status.success(), "expected a failure"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("unknown soundness position"), + "the message has to name what was wrong with the spec:\n{stderr}" + ); +} + +/// what the outer command actually writes, which nothing else here observes: +/// every test above sets the variables by hand, so a release that wrote them to +/// the wrong names — or swapped them — would pass all of them +/// +/// this drives the real `cmd_build_wheels` with a stub standing in for `uv`, so +/// it needs no frontend and no network. the release fails afterwards, because +/// the stub builds nothing, but by then it has recorded what it was handed +#[cfg(unix)] +#[test] +fn a_release_hands_each_build_the_stamps_and_the_lowering() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\nrequires-python = \">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + + let bin = dir.path().join("bin"); + fs::create_dir_all(&bin).unwrap(); + let report = dir.path().join("handed.txt"); + let uv = bin.join("uv"); + fs::write( + &uv, + "#!/bin/sh\n\ + printf 'STAMPS=%s\\nLOWERING=%s\\n' \"$BY_BUILD_STAMPS\" \"$BY_BUILD_LOWERING\" \ + >> \"$BY_TEST_REPORT\"\n", + ) + .unwrap(); + fs::set_permissions(&uv, fs::Permissions::from_mode(0o755)).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--wheels", "--soundness", "none"]) + .current_dir(dir.path()) + // the stub is the only `uv` reachable, so this cannot accidentally + // drive the developer's real one + .env("PATH", &bin) + .env_remove("VIRTUAL_ENV") + .env("BY_TEST_REPORT", &report) + .output() + .expect("failed to spawn by"); + + let handed = fs::read_to_string(&report).unwrap_or_else(|_| { + panic!( + "the release never ran the frontend:\n{}", + String::from_utf8_lossy(&output.stderr) + ) + }); + + assert!( + handed.contains(r#"LOWERING={"soundness":"none""#), + "the release has to hand down what it was asked to lower to:\n{handed}" + ); + // and the stamps still travel beside them, under their own name + assert!( + handed.contains(r#"STAMPS={"#) && handed.contains("BUILT_AT"), + "the stamps must not have been displaced:\n{handed}" + ); +} + /// `--wheels` produces a release, `--min-version` produces one tree lowered to /// one python. asking for both is asking for two different things at once #[test] diff --git a/crates/ty/tests/by_restage.rs b/crates/ty/tests/by_restage.rs index 716018fa78..061ff2769b 100644 --- a/crates/ty/tests/by_restage.rs +++ b/crates/ty/tests/by_restage.rs @@ -46,6 +46,56 @@ fn restage(dir: &Path, file: &str) -> (bool, serde_json::Value) { (out.status.success(), json) } +/// A re-stage is handed no stamps, so it can only reproduce the build's if the +/// build wrote them down. +/// +/// This is the reason `build:` stamps arrive as transpile config rather than +/// being discovered inside the pipeline. A re-stage that went looking for the +/// commit itself would answer with whatever `HEAD` is *now* — and drop a module +/// claiming one commit into a tree of modules claiming another, with nothing to +/// say they disagree. +#[test] +fn restaging_reproduces_the_stamps_the_build_settled() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("pyproject.toml"), + "[project] +name = \"p\" +version = \"0.1.0\" +requires-python = \">=3.13\" + +# build stamps are experimental, so a project that writes a `build:` block asks +# for them by name +[tool.ty.experimental] +build-stamps = true +", + ) + .unwrap(); + std::fs::write( + dir.path().join("main.by"), + "build:\n GIT_SHA: str\n\nprint(build.GIT_SHA)\n", + ) + .unwrap(); + + let status = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--out", "out", "--stamp", "GIT_SHA=abc123"]) + .current_dir(dir.path()) + .status() + .expect("`by build` should run"); + assert!(status.success(), "`by build` failed"); + + let on_disk = std::fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + assert!( + on_disk.contains(r#"GIT_SHA: str = "abc123""#), + "the build should have stamped the value:\n{on_disk}" + ); + + let (ok, answer) = restage(dir.path(), "main.by"); + assert!(ok, "an unedited file should re-stage: {answer}"); + assert_eq!(answer["content"].as_str().unwrap(), on_disk); + assert_eq!(answer["changed"].as_bool(), Some(false)); +} + /// **The property the whole design rests on.** /// /// A re-stage has to produce the bytes the build itself would have written, or the debugger is diff --git a/crates/ty/tests/cli/config_option.rs b/crates/ty/tests/cli/config_option.rs index 1bedb9c496..5bd32f5454 100644 --- a/crates/ty/tests/cli/config_option.rs +++ b/crates/ty/tests/cli/config_option.rs @@ -123,7 +123,7 @@ fn cli_config_args_invalid_option() -> anyhow::Result<()> { | 1 | bad-option=true | ^^^^^^^^^^ - unknown field `bad-option`, expected one of `type-checking-preset`, `environment`, `src`, `rules`, `terminal`, `analysis`, `run`, `build`, `editor`, `overrides` + unknown field `bad-option`, expected one of `type-checking-preset`, `environment`, `src`, `rules`, `terminal`, `analysis`, `experimental`, `run`, `build`, `lowering`, `editor`, `overrides` Usage: by diff --git a/crates/ty_ide/src/all_symbols.rs b/crates/ty_ide/src/all_symbols.rs index 5b9edffedb..7a9fbee0d3 100644 --- a/crates/ty_ide/src/all_symbols.rs +++ b/crates/ty_ide/src/all_symbols.rs @@ -35,34 +35,10 @@ pub fn all_symbols<'db>( let results = all_modules(db, resolver_environment) .into_par_iter() .map_with_db(db, |db, module| { - let name = module.name(db); - - // Note that this will always consider namespace - // packages to be "not firsty party." This isn't - // necessarily correct, and we can probably improve - // on this in response to user feedback. (At time - // of writing, 2026-02-13, we don't really handle - // namespace packages in auto-import anyway.) - let is_non_first_party = module.search_path(db).is_none_or(|sp| !sp.is_first_party()); - - // Filter out non-first-party test and private modules, while retaining private - // typeshed packages that are useful when writing type annotations. - if is_non_first_party - && (name.is_test_module() || name.is_private() && !module.is_type_check_only(db)) - { - return Vec::new(); - } - - // A distribution the project only has because something else needed - // it is not one it can import. Auto-import searches by symbol name, - // so there is no way for the user to ask for one on purpose here; - // offering them means offering imports that break on a fresh install. - if matches!( - dependencies::import_standing(db, importing_file, module), - ImportStanding::Undeclared { .. } - ) { + if !is_importable_module(db, importing_file, module) { return Vec::new(); } + let non_first_party = is_non_first_party(db, module); let Some(file) = module.file(db) else { return Vec::new(); @@ -86,7 +62,7 @@ pub fn all_symbols<'db>( for (_, symbol) in symbols_for_file_global_only(db, program_file).search(query) { // Test functions (starting with `test_`) in third-party // packages are almost never useful to import. - if is_non_first_party && symbol.name.starts_with("test_") { + if non_first_party && symbol.name.starts_with("test_") { continue; } if private.contains(symbol.name.as_ref()) { @@ -107,6 +83,64 @@ pub fn all_symbols<'db>( merge::merge(db, results) } +/// The modules an `import` could bring into scope under `name`. +/// +/// A module binds its own last component when it is imported from its package, so `pkg.sub` is +/// one of the answers for `sub`. The full dotted name is not the question here: `import pkg.sub` +/// binds `pkg`, so a file writing `sub.` is asking about the `from pkg import sub` instead. +pub(crate) fn modules_binding_name<'db>( + db: &'db dyn Db, + importing_from: ProgramFile<'db>, + name: &str, +) -> Vec> { + if name.is_empty() { + return Vec::new(); + } + let importing_file = importing_from.file(db); + let resolver_environment = importing_from.resolver_environment(db); + all_modules(db, resolver_environment) + .into_par_iter() + .map_with_db(db, |db, module| { + (module.name(db).last_component() == name + && module.file(db).is_some() + && is_importable_module(db, importing_file, module)) + .then_some(module) + }) + .flat_map_iter(|module| module) + .collect() +} + +/// Whether the project is allowed to write an import of `module`. +fn is_importable_module<'db>(db: &'db dyn Db, importing_file: File, module: Module<'db>) -> bool { + let name = module.name(db); + + // Filter out non-first-party test and private modules, while retaining private + // typeshed packages that are useful when writing type annotations. + if is_non_first_party(db, module) + && (name.is_test_module() || name.is_private() && !module.is_type_check_only(db)) + { + return false; + } + + // A distribution the project only has because something else needed + // it is not one it can import. Auto-import searches by symbol name, + // so there is no way for the user to ask for one on purpose here; + // offering them means offering imports that break on a fresh install. + !matches!( + dependencies::import_standing(db, importing_file, module), + ImportStanding::Undeclared { .. } + ) +} + +/// Whether `module` comes from somewhere other than the project's own code. +/// +/// Note that this will always consider namespace packages to be "not first party." This isn't +/// necessarily correct, and we can probably improve on this in response to user feedback. (At +/// time of writing, 2026-02-13, we don't really handle namespace packages in auto-import anyway.) +fn is_non_first_party<'db>(db: &'db dyn Db, module: Module<'db>) -> bool { + module.search_path(db).is_none_or(|sp| !sp.is_first_party()) +} + /// A symbol found in the workspace and dependencies, including the /// file it was found in. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 640574a25f..94ce59b147 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -17,6 +17,7 @@ use ruff_python_literal::escape::{Escape, UnicodeEscape}; use ruff_python_literal::format::FormatSpec; use ruff_python_literal::mini_language::FormatSpecComponent; use ruff_python_literal::strftime; +use ruff_python_stdlib::identifiers::{is_identifier_continuation, is_identifier_start}; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustc_hash::FxHashSet; use ty_module_resolver::{ @@ -35,6 +36,7 @@ use ty_python_semantic::{ types::{CycleDetector, KnownClass, Type}, }; +use crate::all_symbols::modules_binding_name; use crate::common_aliases; use crate::django_template::django_string_completions; use crate::docstring::Docstring; @@ -72,6 +74,14 @@ pub fn completion<'db>( } if !matches!(context.kind, ContextKind::Keywords(_)) && context.cursor.is_in_string() { + // a name written inside a plain string's braces, which the string only + // reads as a name once it is an f-string + if let Some(completions) = + string_field_completions(db, env, program_file, &model, &source, &context.cursor) + { + return completions.into_completions(); + } + let Some(string_expr) = context.cursor.enclosing_string_literal_expr() else { return vec![]; }; @@ -131,11 +141,25 @@ pub fn completion<'db>( &mut completions, ); if settings.auto_import { + let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()); + let importer = + Importer::new(db, &stylist, program_file, source.as_str(), &parsed); + let members = importer.members_in_scope_at(expr.into(), expr.start()); add_aliased_module_completions( db, program_file, - &parsed, &model, + &importer, + &members, + expr, + &mut completions, + ); + add_unimported_attribute_completions( + db, + program_file, + &model, + &importer, + &members, expr, &mut completions, ); @@ -183,7 +207,7 @@ pub fn completion<'db>( &in_scope, &mut completions, ); - add_compound_keyword_completions(&context.cursor, source_type, &mut completions); + add_statement_keyword_completions(&context.cursor, source_type, &mut completions); add_context_sensitive_completions( &context.cursor, &model, @@ -348,6 +372,11 @@ impl<'db> Completions<'db> { .collect() } + /// Whether nothing has been added to this collection. + fn is_empty(&self) -> bool { + self.items.is_empty() + } + /// Attempts to adds the given completion to this collection. /// /// When added, `true` is returned. @@ -503,6 +532,13 @@ pub struct Completion<'db> { /// An import statement to insert (or ensure is already /// present) when this completion is selected. pub import: Option, + /// a further edit the client should apply along with this completion, + /// without which the completion would not mean what it says + /// + /// a name completed inside the braces of a plain string is the one that + /// needs this: `"hello {na"` only reads `name` as a name once the string + /// carries the `f` that makes it an f-string, so that `f` rides along + pub additional_edit: Option, /// Whether this suggestion came from builtins or not. /// /// At time of writing (2025-06-26), this information @@ -564,6 +600,7 @@ struct CompletionBuilder<'db> { kind: Option, module_name: Option<&'db ModuleName>, import: Option, + additional_edit: Option, builtin: bool, is_context_specific: bool, is_type_check_only: bool, @@ -592,6 +629,7 @@ impl<'db> CompletionBuilder<'db> { kind: None, module_name: None, import: None, + additional_edit: None, builtin: false, is_context_specific: false, is_type_check_only: false, @@ -683,29 +721,41 @@ impl<'db> CompletionBuilder<'db> { .kind .or_else(|| self.ty.and_then(|ty| completion_kind_from_type(db, ty))); let relevance = Relevance::new(db, program_file, collection_context, query, &self); - let (label, insert, insert_text_format, command) = - if collection_context.should_complete_callable_parentheses(kind) { - let label = self.insert.unwrap_or_else(|| self.name.clone()); - if collection_context.capabilities.snippets { - let insert = compact_str::format_compact!("{label}($0)"); - ( - Some(label), - Some(insert), - CompletionInsertTextFormat::Snippet, - Some(CompletionCommand::TriggerSignatureHelp), - ) - } else { - let insert = compact_str::format_compact!("{label}()"); - ( - Some(label), - Some(insert), - CompletionInsertTextFormat::PlainText, - None, - ) - } + // a completion that writes its own text is only a candidate for + // parentheses when that text is a plain (possibly qualified) name. + // `typing.is_typeddict` wants them; a template that already spells out a + // whole construct does not — an `override def greet(self) -> str:` came + // back as `override def greet(self) -> str:()` before this said so, and + // so would every postfix rewrite + let insert_is_a_bare_name = self.insert.as_ref().is_none_or(|insert| { + insert + .chars() + .all(|c| c.is_alphanumeric() || c == '_' || c == '.') + }); + let (label, insert, insert_text_format, command) = if insert_is_a_bare_name + && collection_context.should_complete_callable_parentheses(kind) + { + let label = self.insert.unwrap_or_else(|| self.name.clone()); + if collection_context.capabilities.snippets { + let insert = compact_str::format_compact!("{label}($0)"); + ( + Some(label), + Some(insert), + CompletionInsertTextFormat::Snippet, + Some(CompletionCommand::TriggerSignatureHelp), + ) } else { - (self.label, self.insert, self.insert_text_format, None) - }; + let insert = compact_str::format_compact!("{label}()"); + ( + Some(label), + Some(insert), + CompletionInsertTextFormat::PlainText, + None, + ) + } + } else { + (self.label, self.insert, self.insert_text_format, None) + }; Completion { name: self.name, label, @@ -720,6 +770,7 @@ impl<'db> CompletionBuilder<'db> { module_name: self.module_name, module_dependency_kind: self.module_dependency_kind, import: self.import, + additional_edit: self.additional_edit, builtin: self.builtin, is_type_check_only: self.is_type_check_only, is_context_specific: self.is_context_specific, @@ -785,6 +836,11 @@ impl<'db> CompletionBuilder<'db> { self } + fn additional_edit(mut self, edit: Edit) -> CompletionBuilder<'db> { + self.additional_edit = Some(edit); + self + } + fn deprecated(mut self, deprecated: bool) -> CompletionBuilder<'db> { self.deprecated = deprecated; self @@ -916,7 +972,12 @@ impl Default for CompletionSettings { fn default() -> CompletionSettings { CompletionSettings { auto_import: true, - complete_function_parentheses: false, + // basedpython defaults this on where upstream ty leaves it off. A + // callable completion exists to be called, and typing the parentheses + // back is work the editor already knows how to do — the machinery + // below writes `name($0)` and asks for signature help, so accepting + // one leaves the caret where the first argument goes + complete_function_parentheses: true, } } } @@ -1230,6 +1291,23 @@ impl<'m> ContextCursor<'m> { } } + /// returns the string literal expression the cursor is in, when that + /// string is a plain one + /// + /// an f-string and a t-string are already read for the names they hold, + /// and a bytes literal can never hold one, so only a plain string has + /// anything to gain from becoming an f-string + fn enclosing_plain_string_literal_expr(&self) -> Option<&'m ast::ExprStringLiteral> { + if self + .tokens_before + .last() + .is_none_or(|token| token.kind() != TokenKind::String) + { + return None; + } + self.enclosing_string_literal_expr() + } + /// Returns the quote style of the string literal that the cursor is positioned within, if any. fn string_quote_style(&self) -> Option { self.tokens_before @@ -3018,11 +3096,13 @@ fn add_override_completions<'db>( } } -/// A statement opener spelled with more than one keyword. +/// A statement opener the completion list has to carry itself. /// -/// Each word of one is a keyword the completion list already carries on its -/// own, so the pair is only worth offering where the whole construct is valid. -struct CompoundKeyword { +/// Either a construct spelled with several keywords, each of which the list +/// already offers on its own, or a basedpython opener that python's keyword list +/// has never heard of. Both are only worth offering where the construct they +/// open is valid. +struct StatementKeyword { text: &'static str, /// Where the construct it opens may be written. position: KeywordPosition, @@ -3030,7 +3110,7 @@ struct CompoundKeyword { basedpython: bool, } -/// The positions a [`CompoundKeyword`] is offered in. +/// The positions a [`StatementKeyword`] is offered in. enum KeywordPosition { /// Wherever a statement may start. Anywhere, @@ -3040,136 +3120,166 @@ enum KeywordPosition { AsyncFunction, } -const COMPOUND_KEYWORDS: &[CompoundKeyword] = &[ - CompoundKeyword { +const STATEMENT_KEYWORDS: &[StatementKeyword] = &[ + StatementKeyword { text: "async def", position: KeywordPosition::Anywhere, basedpython: false, }, - CompoundKeyword { + StatementKeyword { text: "async for", position: KeywordPosition::AsyncFunction, basedpython: false, }, - CompoundKeyword { + StatementKeyword { text: "async with", position: KeywordPosition::AsyncFunction, basedpython: false, }, - CompoundKeyword { + StatementKeyword { text: "data class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "frozen data class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "enum class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "final class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "abstract class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "open class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "sealed class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "private class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "export class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "public class", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "private def", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "export def", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "public def", position: KeywordPosition::Anywhere, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "class def", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "static def", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "override def", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "abstract def", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "final def", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "static var", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "static let", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "late var", position: KeywordPosition::ClassBody, basedpython: true, }, - CompoundKeyword { + StatementKeyword { text: "private type", position: KeywordPosition::Anywhere, basedpython: true, }, + // the single-word openers. python's own keyword list covers `class`, `def` + // and the rest; these are basedpython's, and they are statements in their own + // right rather than the first word of something longer + StatementKeyword { + text: "extension", + position: KeywordPosition::Anywhere, + basedpython: true, + }, + StatementKeyword { + text: "let", + position: KeywordPosition::Anywhere, + basedpython: true, + }, + StatementKeyword { + text: "var", + position: KeywordPosition::Anywhere, + basedpython: true, + }, + StatementKeyword { + text: "init", + position: KeywordPosition::ClassBody, + basedpython: true, + }, ]; -/// Adds the statement openers spelled with more than one keyword. -fn add_compound_keyword_completions( +/// Adds the keywords a statement may open with, beyond the ones +/// [`add_keyword_completions`] offers unconditionally. +/// +/// Those are python's keyword list verbatim, so every basedpython statement +/// opener belongs here — the compound ones (`enum class`, `static let`) and the +/// single words alike. A word offered in only one of the two places is offered +/// only where it happens to head a compound form, which is how `extension` came +/// to be offered nowhere at all. +fn add_statement_keyword_completions( cursor: &ContextCursor<'_>, source_type: PySourceType, completions: &mut Completions<'_>, @@ -3177,7 +3287,7 @@ fn add_compound_keyword_completions( if !cursor.is_at_statement_start() { return; } - for keyword in COMPOUND_KEYWORDS { + for keyword in STATEMENT_KEYWORDS { if keyword.basedpython && !source_type.is_basedpython() { continue; } @@ -3618,6 +3728,226 @@ fn position_of(component: FormatSpecComponent) -> usize { .unwrap_or(0) } +/// completions for a name being typed inside a plain string's braces +/// +/// `"hello {na"` is only text to python, so the `na` in it names nothing and +/// nothing is offered for it today. but the braces say what the user is +/// reaching for, so the names in scope are offered, and taking one writes the +/// `f` that makes the string an f-string along with the brace that closes the +/// replacement field: +/// +/// ```python +/// name = "john" +/// f"hello {name}" +/// ``` +/// +/// `None` when the string says nothing about a field, and also when it says +/// so but nothing in scope matches what has been typed. the string is then +/// still an ordinary string to everything that reads one, so the completions +/// it would otherwise offer — a django name, an expected literal — are the +/// ones the user gets +fn string_field_completions<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + model: &SemanticModel<'db>, + source: &str, + cursor: &ContextCursor<'_>, +) -> Option> { + let string = cursor.enclosing_plain_string_literal_expr()?; + // the text is checked first because it rejects most cursors outright, and + // it costs nothing next to the semantic question below + let field = StringField::being_typed(source, cursor, string)?; + if !can_become_an_fstring(model, cursor, string) { + return None; + } + + let mut completions = Completions::new( + db, + file, + CollectionContext::none(), + UserQuery::fuzzy(Some(&source[field.typed])), + ); + // the brace the user has yet to write. one they already wrote is left + // alone, so completing into `{na}` doesn't leave `{name}}` + let closing = if field.closed { "" } else { "}" }; + for semantic in model.scoped_completions(string.into()) { + let name = semantic.name.clone(); + let module_dependency_kind = if semantic.builtin { + ModuleDependencyKind::Builtin + } else { + ModuleDependencyKind::Current + }; + completions.add( + CompletionBuilder::from_semantic_completion(db, env, semantic) + .label(name.clone()) + .insert(compact_str::format_compact!("{name}{closing}")) + .replace(field.replace) + .additional_edit(Edit::insertion("f".to_string(), field.prefix_at)) + .module_dependency_kind(module_dependency_kind), + ); + } + if completions.is_empty() { + return None; + } + Some(completions) +} + +/// a replacement field the user is part way through writing inside a plain +/// string +struct StringField { + /// where an `f` goes to make the string holding the field an f-string + /// + /// this is the start of the one part the cursor is in: only that part of + /// an implicitly concatenated string needs the prefix + prefix_at: TextSize, + /// the name written so far, between the brace and the cursor + typed: TextRange, + /// the name the completion stands in for, which runs past the cursor when + /// the user went back to correct one they had already written + replace: TextRange, + /// whether a brace already closes the field + closed: bool, +} + +impl StringField { + /// looks for the field the cursor sits in the middle of writing + /// + /// `None` when the cursor is somewhere in the string that says nothing + /// about a field: ordinary text, a brace pair that is already closed, or a + /// `{{` that spells one literal brace the moment the string becomes an + /// f-string + fn being_typed( + source: &str, + cursor: &ContextCursor<'_>, + string: &ast::ExprStringLiteral, + ) -> Option { + let part = string + .value + .iter() + .find(|part| part.content_range().contains_inclusive(cursor.offset))?; + // `u` and `f` cannot both prefix one string + if part.flags.prefix().is_unicode() { + return None; + } + let content = part.content_range(); + let before = source.get(content.start().to_usize()..cursor.offset.to_usize())?; + let open = before.rfind('{')?; + // a field the user has already closed + if before[open + 1..].contains('}') { + return None; + } + // braces pair up into escapes from the left, so the brace that opens a + // field is the last of a run of odd length. an even run is all escapes, + // and spells literal braces the user wants left as they are + let braces = before[..=open].bytes().rev().take_while(|b| *b == b'{'); + if braces.count() % 2 == 0 { + return None; + } + let typed = &before[open + 1..]; + if !is_name_prefix(typed) { + return None; + } + // the rest of a name the user went back into the middle of. it belongs + // to the field, so the completion stands in for it rather than pushing + // it along + let after = source.get(cursor.offset.to_usize()..content.end().to_usize())?; + let trailing = after + .find(|c: char| !is_identifier_continuation(c)) + .unwrap_or(after.len()); + Some(StringField { + prefix_at: part.start(), + typed: TextRange::new(cursor.offset - TextSize::of(typed), cursor.offset), + replace: TextRange::new( + cursor.offset - TextSize::of(typed), + cursor.offset + TextSize::try_from(trailing).ok()?, + ), + closed: after[trailing..].starts_with('}'), + }) + } +} + +/// whether the string the cursor sits in could be an f-string instead +/// +/// python reads a string in some positions for something other than the text +/// it holds, and an f-string is not read that way. a docstring stops being a +/// docstring, a `case` pattern and a type expression stop being valid, and a +/// `str.format` template already spells its fields for `format` to fill in +/// later +fn can_become_an_fstring( + model: &SemanticModel<'_>, + cursor: &ContextCursor<'_>, + string: &ast::ExprStringLiteral, +) -> bool { + !is_docstring(cursor, string) + && !is_match_pattern(cursor) + && !is_format_template(cursor) + && !cursor.is_in_type_expression(model) +} + +/// whether the string under the cursor is the docstring of what encloses it +/// +/// the string has to be the whole of the first statement, not merely somewhere +/// inside it: `print("hi")` opening a body is not a docstring +fn is_docstring(cursor: &ContextCursor<'_>, string: &ast::ExprStringLiteral) -> bool { + let mut ancestors = ancestors_above_string(cursor); + let Some(AnyNodeRef::StmtExpr(statement)) = ancestors.next() else { + return false; + }; + if statement.value.range() != string.range() { + return false; + } + let body = match ancestors.next() { + Some(AnyNodeRef::StmtFunctionDef(function)) => &function.body, + Some(AnyNodeRef::StmtClassDef(class)) => &class.body, + Some(AnyNodeRef::ModModule(module)) => &module.body, + _ => return false, + }; + body.first() + .is_some_and(|first| first.range() == statement.range()) +} + +/// whether the string under the cursor is part of a `case` pattern +/// +/// a pattern may only match a literal, so an f-string there is a syntax error +/// python raises and ty does not yet report +fn is_match_pattern(cursor: &ContextCursor<'_>) -> bool { + ancestors_above_string(cursor) + .take_while(|node| !node.is_statement()) + .any(AnyNodeRef::is_pattern) +} + +/// whether the string under the cursor is a template `str.format` fills in +fn is_format_template(cursor: &ContextCursor<'_>) -> bool { + matches!( + ancestors_above_string(cursor).next(), + Some(AnyNodeRef::ExprAttribute(attribute)) + if matches!(attribute.attr.as_str(), "format" | "format_map") + ) +} + +/// the nodes enclosing the string the cursor is in, nearest first +fn ancestors_above_string<'m>(cursor: &ContextCursor<'m>) -> impl Iterator> { + cursor + .covering_node + .ancestors() + .skip_while(|node| !matches!(node, AnyNodeRef::ExprStringLiteral(_))) + .skip(1) +} + +/// whether `text` could be the start of a name +/// +/// this is the text between the brace and the cursor, so it is a name only +/// part written: `na` on the way to `name`, and empty the moment the brace +/// itself is typed +fn is_name_prefix(text: &str) -> bool { + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return true; + }; + is_identifier_start(first) && chars.all(is_identifier_continuation) +} + /// Adds the django names the string literal under the cursor could be spelling. /// /// A template name and a url name are ordinary strings to python, so only the @@ -3841,8 +4171,9 @@ fn add_alias_completions<'db>( fn add_aliased_module_completions<'db>( db: &'db dyn Db, file: ProgramFile<'db>, - parsed: &ParsedModuleRef, model: &SemanticModel<'db>, + importer: &Importer<'_>, + members: &MembersInScope<'_>, expr: &ast::ExprAttribute, completions: &mut Completions<'db>, ) { @@ -3866,17 +4197,13 @@ fn add_aliased_module_completions<'db>( return; }; - let source = source_text(db, file.file(db)); - let stylist = Stylist::from_tokens(parsed.tokens(), source.as_str()); - let importer = Importer::new(db, &stylist, file, source.as_str(), parsed); - let members = importer.members_in_scope_at(expr.into(), expr.start()); if members.find_member(&name.id).is_some() { return; } let import_action = importer.import( ImportRequest::module_as(module_name.as_str(), &name.id), - &members, + members, ); let Some(import) = import_action.import() else { return; @@ -3893,6 +4220,185 @@ fn add_aliased_module_completions<'db>( } } +/// Adds the members of what an unimported dotted name would have named. +/// +/// `Asdf.n` in a file that never imported `Asdf` still names the class an import would bind, so +/// this offers that class's members, and carries the `from mod import Asdf` along with whatever +/// the user accepts. The same goes for a whole module: `mod.A` offers what `import mod` reaches. +fn add_unimported_attribute_completions<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + model: &SemanticModel<'db>, + importer: &Importer<'_>, + members: &MembersInScope<'_>, + expr: &ast::ExprAttribute, + completions: &mut Completions<'db>, +) { + // an import is only ever offered for something the user has begun to name, the same bargain + // auto-import strikes in a scope: a query that matches everything would answer a bare `mod.` + // with every member of every symbol spelled `mod` + if completions.query.will_match_everything() { + return; + } + let Some((root, attributes)) = attribute_chain(expr) else { + return; + }; + // a name that already means something is not waiting to be imported. an unbound one infers as + // `Unknown`, which is also what a binding of unknown type infers as, so the scope is asked + // about that case below + if root.inferred_type(model).is_some_and(|ty| !ty.is_unknown()) + || members.find_member(&root.id).is_some() + { + return; + } + + let source_file = file.file(db); + let env = model.program_environment(); + // several modules can export one name — `Mapping` is `typing`'s, and `collections.abc` and + // `typing_extensions` re-export it — and each would otherwise contribute a full copy of the + // same members, told apart only by an import edit the list does not show + let mut seen = FxHashSet::default(); + for candidate in root_candidates(db, file, &root.id) { + if candidate.file == source_file { + continue; + } + let (import_module, import_member) = candidate.import(db); + let request = match import_member { + Some(ref member) => ImportRequest::import_from(import_module.as_str(), member).force(), + None => ImportRequest::module(import_module.as_str()), + }; + let import_action = importer.import(request, members); + // the import has to bind the name the user already wrote: an importer that sidesteps a + // conflict by qualifying the symbol instead leaves the `Asdf.` in the file naming nothing + if import_action.symbol_text() != root.id.as_str() { + continue; + } + // the root is what the import writes, so the whole chain has to be walked from it: what + // the user is completing is the far end, and what they would be importing is the near one + let root_path: Vec<&str> = candidate.step.as_deref().into_iter().collect(); + let path: Vec<&str> = root_path + .iter() + .copied() + .chain(attributes.iter().copied()) + .collect(); + let (Some(root_ty), Some(ty)) = ( + model.path_type(candidate.module, &root_path), + model.path_type(candidate.module, &path), + ) else { + continue; + }; + if !seen.insert(ty) { + continue; + } + let module_name = candidate.module.name(db); + let module_dependency_kind = + ModuleDependencyKind::from_module(db, source_file, candidate.module); + // an import of a root that is only there while type checking is itself only there while + // type checking, however ordinary the member reached through it + let type_check_only = + is_type_check_only_module(db, file, candidate.module) || root_ty.is_type_check_only(db); + for semantic_completion in model.type_completions(ty) { + let type_check_only = type_check_only || semantic_completion.is_type_check_only; + completions.add( + CompletionBuilder::from_semantic_completion(db, &env, semantic_completion) + .type_check_only(type_check_only) + .deprecated(candidate.deprecated) + .module_name(module_name) + .import(import_action.import().cloned()) + .module_dependency_kind(module_dependency_kind), + ); + } + } +} + +/// Something an import could bind the root of a dotted name to. +struct RootCandidate<'db> { + /// The module whose members the root name reaches. + module: Module<'db>, + /// The step from that module to the root, absent when the root names the module itself. + step: Option, + /// The file the root is defined in. + file: File, + /// Whether what the import would bind is deprecated. + deprecated: bool, +} + +impl<'db> RootCandidate<'db> { + /// The module an import of this candidate names, and the member it takes from it. + /// + /// A module is imported from its package — `from pkg import sub` — because `import pkg.sub` + /// binds `pkg`, not the `sub` the file wrote. Only a module with no package left to name is + /// bound by an `import` of its own. + fn import(&self, db: &'db dyn Db) -> (ModuleName, Option) { + let name = self.module.name(db); + match self.step { + Some(ref step) => (name.clone(), Some(step.clone())), + None => match name.parent() { + Some(parent) => (parent, Some(Name::new(name.last_component()))), + None => (name.clone(), None), + }, + } + } +} + +/// Everything an import could bind `name` to. +fn root_candidates<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + name: &str, +) -> Vec> { + // `all_symbols` matches a module against the whole of its dotted name, which is the name a + // bare completion offers but not one an import can bind, so the modules come from the lookup + // that asks the binding question instead + let symbols = all_symbols(db, file, &QueryPattern::exactly(name)) + .into_iter() + // a builtin is in scope with no import at all, so it never reaches here as an unbound + // name — and offering one would be offering an import nothing needs + .filter(|symbol| !symbol.module().is_known(db, KnownModule::Builtins)) + .filter_map(|symbol| { + Some(RootCandidate { + module: symbol.module(), + step: Some(Name::new(symbol.name_in_file()?)), + file: symbol.file(), + deprecated: symbol.deprecated(), + }) + }); + let modules = modules_binding_name(db, file, name) + .into_iter() + .filter_map(|module| { + Some(RootCandidate { + module, + step: None, + file: module.file(db)?, + deprecated: false, + }) + }); + symbols.chain(modules).collect() +} + +/// The name a dotted expression starts from, and the attributes reached from it. +/// +/// The attribute under the cursor is not one of them, so `mod.Asdf.n` answers with `mod` and +/// `["Asdf"]`. A chain that starts from anything but a name — a call or a subscript, say — +/// has no name for an import to bind, and answers with nothing. +fn attribute_chain(expr: &ast::ExprAttribute) -> Option<(&ast::ExprName, Vec<&str>)> { + let mut path = vec![]; + let mut value = &*expr.value; + loop { + match value { + ast::Expr::Name(name) => { + path.reverse(); + return Some((name, path)); + } + ast::Expr::Attribute(attribute) => { + path.push(attribute.attr.as_str()); + value = &attribute.value; + } + _ => return None, + } + } +} + /// The kind of tokens identified under the cursor. #[derive(Debug)] enum CompletionTargetTokens<'t> { @@ -4900,7 +5406,7 @@ mod tests { use ruff_python_ast::helpers::is_dunder; use ruff_python_ast::token::{TokenKind, Tokens}; use ruff_python_parser::{Mode, ParseOptions}; - use ruff_text_size::{TextRange, TextSize}; + use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_module_resolver::ModuleName; use ty_project::metadata::options::{EditorOptions, Options}; @@ -5083,8 +5589,8 @@ type assert_snapshot!( test.type_signatures().skip_auto_import().build().snapshot(), @" - type :: - TypeError :: + type() :: + TypeError() :: ", ); } @@ -5379,7 +5885,7 @@ f assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), - @"foo", + @"foo()", ); } @@ -5409,7 +5915,7 @@ def foo(): ... ", ); - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo()"); } #[test] @@ -5425,7 +5931,7 @@ f assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), - @"foo", + @"foo()", ); } @@ -5440,7 +5946,7 @@ def foo(): ", ); - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo()"); } #[test] @@ -5456,8 +5962,8 @@ def foo(): assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo - foofoo + foo() + foofoo() "); } @@ -5489,7 +5995,7 @@ def foo(): // matches the current cursor's indentation. This seems fraught // however. It's not clear to me that we can always assume a // correspondence between scopes and indentation level. - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo()"); } #[test] @@ -5506,8 +6012,8 @@ def foo(): assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo - foofoo + foo() + foofoo() "); } @@ -5524,8 +6030,8 @@ def foo(): assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo - foofoo + foo() + foofoo() "); } @@ -5544,9 +6050,9 @@ def frob(): ... assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo - foofoo - frob + foo() + foofoo() + frob() "); } @@ -5565,8 +6071,8 @@ def frob(): ... assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo - frob + foo() + frob() "); } @@ -5585,10 +6091,10 @@ def frob(): ... assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo - foofoo - foofoofoo - frob + foo() + foofoo() + foofoofoo() + frob() "); } @@ -5612,7 +6118,7 @@ def foo(): // account for the indented whitespace, or some other technique // needs to be used to get the scope containing `foofoo` but not // `foofoofoo`. - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo()"); } #[test] @@ -5626,7 +6132,7 @@ def foo(): ); // FIXME: Should include `foofoo` (but not `foofoofoo`). - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"foo()"); } #[test] @@ -5643,8 +6149,8 @@ def frob(): ... // FIXME: Should include `foofoo` (but not `foofoofoo`). assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @" - foo - frob + foo() + frob() "); } @@ -5663,8 +6169,8 @@ def frob(): ... // FIXME: Should include `foofoo` (but not `foofoofoo`). assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @" - foo - frob + foo() + frob() "); } @@ -5684,8 +6190,8 @@ def frob(): ... // FIXME: Should include `foofoo` (but not `foofoofoo`). assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @" - foo - frob + foo() + frob() "); } @@ -5946,7 +6452,7 @@ class Foo: // // These don't work for similar reasons as other // tests above with the inside of whitespace. - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"Foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"Foo()"); } #[test] @@ -5963,7 +6469,7 @@ class Foo: // FIXME: Should include `bar`, `quux` and `frob`. // (Unclear if `Foo` should be included, but a false // positive isn't the end of the world.) - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"Foo"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"Foo()"); } #[test] @@ -6435,28 +6941,28 @@ quux. return quux :: postfix type(quux) :: postfix __annotations__ :: dict[str, Any] - __class__ :: type[Quux] - __delattr__ :: bound method Quux.__delattr__(name: str, /) + __class__() :: type[Quux] + __delattr__() :: bound method Quux.__delattr__(name: str, /) __dict__ :: dict[str, Any] - __dir__ :: bound method Quux.__dir__() -> Iterable[str] + __dir__() :: bound method Quux.__dir__() -> Iterable[str] __doc__ :: str | None - __eq__ :: bound method Quux.__eq__(value: object, /) -> bool - __format__ :: bound method Quux.__format__(format_spec: Literal[""], /) -> str - __getattribute__ :: bound method Quux.__getattribute__(name: str, /) -> Any - __getstate__ :: bound method Quux.__getstate__() -> object - __hash__ :: bound method Quux.__hash__() -> int - __init__ :: bound method Quux.__init__() - __init_subclass__ :: bound method type[Quux].__init_subclass__() + __eq__() :: bound method Quux.__eq__(value: object, /) -> bool + __format__() :: bound method Quux.__format__(format_spec: Literal[""], /) -> str + __getattribute__() :: bound method Quux.__getattribute__(name: str, /) -> Any + __getstate__() :: bound method Quux.__getstate__() -> object + __hash__() :: bound method Quux.__hash__() -> int + __init__() :: bound method Quux.__init__() + __init_subclass__() :: bound method type[Quux].__init_subclass__() __module__ :: str - __ne__ :: bound method Quux.__ne__(value: object, /) -> bool - __new__ :: def __new__[Self](cls) -> Self - __reduce__ :: bound method Quux.__reduce__() -> str | tuple[Any, ...] - __reduce_ex__ :: bound method Quux.__reduce_ex__(protocol: SupportsIndex, /) -> str | tuple[Any, ...] - __repr__ :: bound method Quux.__repr__() -> str - __setattr__ :: bound method Quux.__setattr__(name: str, value: Any, /) - __sizeof__ :: bound method Quux.__sizeof__() -> int - __str__ :: bound method Quux.__str__() -> str - __subclasshook__ :: bound method type[Quux].__subclasshook__(subclass: type, /) -> bool + __ne__() :: bound method Quux.__ne__(value: object, /) -> bool + __new__() :: def __new__[Self](cls) -> Self + __reduce__() :: bound method Quux.__reduce__() -> str | tuple[Any, ...] + __reduce_ex__() :: bound method Quux.__reduce_ex__(protocol: SupportsIndex, /) -> str | tuple[Any, ...] + __repr__() :: bound method Quux.__repr__() -> str + __setattr__() :: bound method Quux.__setattr__(name: str, value: Any, /) + __sizeof__() :: bound method Quux.__sizeof__() -> int + __str__() :: bound method Quux.__str__() -> str + __subclasshook__() :: bound method type[Quux].__subclasshook__(subclass: type, /) -> bool "#); } @@ -6479,9 +6985,9 @@ quux.b builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @" bar :: int baz :: int - __getattribute__ :: bound method Quux.__getattribute__(name: str, /) -> Any - __init_subclass__ :: bound method type[Quux].__init_subclass__() - __subclasshook__ :: bound method type[Quux].__subclasshook__(subclass: type, /) -> bool + __getattribute__() :: bound method Quux.__getattribute__(name: str, /) -> Any + __init_subclass__() :: bound method type[Quux].__init_subclass__() + __subclasshook__() :: bound method type[Quux].__subclasshook__(subclass: type, /) -> bool "); } @@ -6510,7 +7016,7 @@ C. match C: case :: postfix meta_attr :: int - mro :: bound method .mro() -> list[type] + mro() :: bound method .mro() -> list[type] not C :: postfix (C) :: postfix print(C) :: postfix @@ -6518,46 +7024,46 @@ C. repr(C) :: postfix return C :: postfix type(C) :: postfix - __annotate__ :: (() -> dict[str, AnnotationForm]) | None + __annotate__() :: (() -> dict[str, AnnotationForm]) | None __annotations__ :: dict[str, Any] __base__ :: type | None __bases__ :: tuple[type, ...] __basicsize__ :: int - __call__ :: bound method .__call__(...) -> Any - __class__ :: - __delattr__ :: def __delattr__(self, name: str, /) + __call__() :: bound method .__call__(...) -> Any + __class__() :: + __delattr__() :: def __delattr__(self, name: str, /) __dict__ :: dict[str, Any] __dictoffset__ :: int - __dir__ :: def __dir__(self) -> Iterable[str] + __dir__() :: def __dir__(self) -> Iterable[str] __doc__ :: str | None - __eq__ :: def __eq__(self, value: object, /) -> bool + __eq__() :: def __eq__(self, value: object, /) -> bool __flags__ :: int - __format__ :: def __format__(self, format_spec: Literal[""], /) -> str - __getattribute__ :: def __getattribute__(self, name: str, /) -> Any - __getstate__ :: def __getstate__(self) -> object - __hash__ :: def __hash__(self) -> int - __init__ :: def __init__(self) - __init_subclass__ :: bound method .__init_subclass__() - __instancecheck__ :: bound method .__instancecheck__(instance: Any, /) -> bool + __format__() :: def __format__(self, format_spec: Literal[""], /) -> str + __getattribute__() :: def __getattribute__(self, name: str, /) -> Any + __getstate__() :: def __getstate__(self) -> object + __hash__() :: def __hash__(self) -> int + __init__() :: def __init__(self) + __init_subclass__() :: bound method .__init_subclass__() + __instancecheck__() :: bound method .__instancecheck__(instance: Any, /) -> bool __itemsize__ :: int __module__ :: str __mro__ :: tuple[type, ...] __name__ :: str - __ne__ :: def __ne__(self, value: object, /) -> bool - __new__ :: def __new__[Self](cls) -> Self - __or__ :: bound method .__or__[Self](value: Any, /) -> UnionType | Self - __prepare__ :: bound method .__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object] + __ne__() :: def __ne__(self, value: object, /) -> bool + __new__() :: def __new__[Self](cls) -> Self + __or__() :: bound method .__or__[Self](value: Any, /) -> UnionType | Self + __prepare__() :: bound method .__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object] __qualname__ :: str - __reduce__ :: def __reduce__(self) -> str | tuple[Any, ...] - __reduce_ex__ :: def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...] - __repr__ :: def __repr__(self) -> str - __ror__ :: bound method .__ror__[Self](value: Any, /) -> UnionType | Self - __setattr__ :: def __setattr__(self, name: str, value: Any, /) - __sizeof__ :: def __sizeof__(self) -> int - __str__ :: def __str__(self) -> str - __subclasscheck__ :: bound method .__subclasscheck__(subclass: type, /) -> bool - __subclasses__ :: bound method .__subclasses__[Self]() -> list[Self] - __subclasshook__ :: bound method .__subclasshook__(subclass: type, /) -> bool + __reduce__() :: def __reduce__(self) -> str | tuple[Any, ...] + __reduce_ex__() :: def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...] + __repr__() :: def __repr__(self) -> str + __ror__() :: bound method .__ror__[Self](value: Any, /) -> UnionType | Self + __setattr__() :: def __setattr__(self, name: str, value: Any, /) + __sizeof__() :: def __sizeof__(self) -> int + __str__() :: def __str__(self) -> str + __subclasscheck__() :: bound method .__subclasscheck__(subclass: type, /) -> bool + __subclasses__() :: bound method .__subclasses__[Self]() -> list[Self] + __subclasshook__() :: bound method .__subclasshook__(subclass: type, /) -> bool __text_signature__ :: str | None __type_params__ :: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] __weakrefoffset__ :: int @@ -6596,7 +7102,7 @@ Meta. match Meta: case :: postfix meta_attr :: property - mro :: def mro(self) -> list[type] + mro() :: def mro(self) -> list[type] not Meta :: postfix (Meta) :: postfix print(Meta) :: postfix @@ -6607,40 +7113,40 @@ Meta. __base__ :: type | None __bases__ :: tuple[type, ...] __basicsize__ :: int - __call__ :: def __call__(self, *args: Any, **kwds: Any) -> Any - __class__ :: - __delattr__ :: def __delattr__(self, name: str, /) + __call__() :: def __call__(self, *args: Any, **kwds: Any) -> Any + __class__() :: + __delattr__() :: def __delattr__(self, name: str, /) __dict__ :: MappingProxyType[str, Any] __dictoffset__ :: int - __dir__ :: def __dir__(self) -> Iterable[str] + __dir__() :: def __dir__(self) -> Iterable[str] __doc__ :: str | None - __eq__ :: def __eq__(self, value: object, /) -> bool + __eq__() :: def __eq__(self, value: object, /) -> bool __flags__ :: int - __format__ :: def __format__(self, format_spec: Literal[""], /) -> str - __getattribute__ :: def __getattribute__(self, name: str, /) -> Any - __getstate__ :: def __getstate__(self) -> object - __hash__ :: def __hash__(self) -> int - __init__ :: Overload[(self, o: object, /) -> None, (self, name: str, bases: tuple[type, ...], dict: dict[str, Any], /, **kwds: Any) -> None] - __init_subclass__ :: bound method .__init_subclass__() - __instancecheck__ :: def __instancecheck__(self, instance: Any, /) -> bool + __format__() :: def __format__(self, format_spec: Literal[""], /) -> str + __getattribute__() :: def __getattribute__(self, name: str, /) -> Any + __getstate__() :: def __getstate__(self) -> object + __hash__() :: def __hash__(self) -> int + __init__() :: Overload[(self, o: object, /) -> None, (self, name: str, bases: tuple[type, ...], dict: dict[str, Any], /, **kwds: Any) -> None] + __init_subclass__() :: bound method .__init_subclass__() + __instancecheck__() :: def __instancecheck__(self, instance: Any, /) -> bool __itemsize__ :: int __module__ :: str __mro__ :: tuple[type, ...] __name__ :: str - __ne__ :: def __ne__(self, value: object, /) -> bool - __or__ :: def __or__[Self](self: Self, value: Any, /) -> UnionType | Self - __prepare__ :: bound method .__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object] + __ne__() :: def __ne__(self, value: object, /) -> bool + __or__() :: def __or__[Self](self: Self, value: Any, /) -> UnionType | Self + __prepare__() :: bound method .__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object] __qualname__ :: str - __reduce__ :: def __reduce__(self) -> str | tuple[Any, ...] - __reduce_ex__ :: def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...] - __repr__ :: def __repr__(self) -> str - __ror__ :: def __ror__[Self](self: Self, value: Any, /) -> UnionType | Self - __setattr__ :: def __setattr__(self, name: str, value: Any, /) - __sizeof__ :: def __sizeof__(self) -> int - __str__ :: def __str__(self) -> str - __subclasscheck__ :: def __subclasscheck__(self, subclass: type, /) -> bool - __subclasses__ :: def __subclasses__[Self](self: Self) -> list[Self] - __subclasshook__ :: bound method .__subclasshook__(subclass: type, /) -> bool + __reduce__() :: def __reduce__(self) -> str | tuple[Any, ...] + __reduce_ex__() :: def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...] + __repr__() :: def __repr__(self) -> str + __ror__() :: def __ror__[Self](self: Self, value: Any, /) -> UnionType | Self + __setattr__() :: def __setattr__(self, name: str, value: Any, /) + __sizeof__() :: def __sizeof__(self) -> int + __str__() :: def __str__(self) -> str + __subclasscheck__() :: def __subclasscheck__(self, subclass: type, /) -> bool + __subclasses__() :: def __subclasses__[Self](self: Self) -> list[Self] + __subclasshook__() :: bound method .__subclasshook__(subclass: type, /) -> bool __text_signature__ :: str | None __type_params__ :: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] __weakrefoffset__ :: int @@ -6681,28 +7187,28 @@ class Quux: return self type(self) __annotations__ - __class__ - __delattr__ + __class__() + __delattr__() __dict__ - __dir__ + __dir__() __doc__ - __eq__ - __format__ - __getattribute__ - __getstate__ - __hash__ - __init__ - __init_subclass__ + __eq__() + __format__() + __getattribute__() + __getstate__() + __hash__() + __init__() + __init_subclass__() __module__ - __ne__ - __new__ - __reduce__ - __reduce_ex__ - __repr__ - __setattr__ - __sizeof__ - __str__ - __subclasshook__ + __ne__() + __new__() + __reduce__() + __reduce_ex__() + __repr__() + __setattr__() + __sizeof__() + __str__() + __subclasshook__() "); } @@ -6746,7 +7252,7 @@ Quux. list(Quux) :: postfix match Quux: case :: postfix - mro :: bound method .mro() -> list[type] + mro() :: bound method .mro() -> list[type] not Quux :: postfix (Quux) :: postfix print(Quux) :: postfix @@ -6754,51 +7260,51 @@ Quux. repr(Quux) :: postfix return Quux :: postfix some_attribute :: int - some_class_method :: bound method .some_class_method() -> int - some_method :: def some_method(self) -> int + some_class_method() :: bound method .some_class_method() -> int + some_method() :: def some_method(self) -> int some_property :: property - some_static_method :: def some_static_method(self) -> int + some_static_method() :: def some_static_method(self) -> int type(Quux) :: postfix - __annotate__ :: (() -> dict[str, AnnotationForm]) | None + __annotate__() :: (() -> dict[str, AnnotationForm]) | None __annotations__ :: dict[str, Any] __base__ :: type | None __bases__ :: tuple[type, ...] __basicsize__ :: int - __call__ :: bound method .__call__(...) -> Any - __class__ :: - __delattr__ :: def __delattr__(self, name: str, /) + __call__() :: bound method .__call__(...) -> Any + __class__() :: + __delattr__() :: def __delattr__(self, name: str, /) __dict__ :: dict[str, Any] __dictoffset__ :: int - __dir__ :: def __dir__(self) -> Iterable[str] + __dir__() :: def __dir__(self) -> Iterable[str] __doc__ :: str | None - __eq__ :: def __eq__(self, value: object, /) -> bool + __eq__() :: def __eq__(self, value: object, /) -> bool __flags__ :: int - __format__ :: def __format__(self, format_spec: Literal[""], /) -> str - __getattribute__ :: def __getattribute__(self, name: str, /) -> Any - __getstate__ :: def __getstate__(self) -> object - __hash__ :: def __hash__(self) -> int - __init__ :: def __init__(self) - __init_subclass__ :: bound method .__init_subclass__() - __instancecheck__ :: bound method .__instancecheck__(instance: Any, /) -> bool + __format__() :: def __format__(self, format_spec: Literal[""], /) -> str + __getattribute__() :: def __getattribute__(self, name: str, /) -> Any + __getstate__() :: def __getstate__(self) -> object + __hash__() :: def __hash__(self) -> int + __init__() :: def __init__(self) + __init_subclass__() :: bound method .__init_subclass__() + __instancecheck__() :: bound method .__instancecheck__(instance: Any, /) -> bool __itemsize__ :: int __module__ :: str __mro__ :: tuple[type, ...] __name__ :: str - __ne__ :: def __ne__(self, value: object, /) -> bool - __new__ :: def __new__[Self](cls) -> Self - __or__ :: bound method .__or__[Self](value: Any, /) -> UnionType | Self - __prepare__ :: bound method .__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object] + __ne__() :: def __ne__(self, value: object, /) -> bool + __new__() :: def __new__[Self](cls) -> Self + __or__() :: bound method .__or__[Self](value: Any, /) -> UnionType | Self + __prepare__() :: bound method .__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object] __qualname__ :: str - __reduce__ :: def __reduce__(self) -> str | tuple[Any, ...] - __reduce_ex__ :: def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...] - __repr__ :: def __repr__(self) -> str - __ror__ :: bound method .__ror__[Self](value: Any, /) -> UnionType | Self - __setattr__ :: def __setattr__(self, name: str, value: Any, /) - __sizeof__ :: def __sizeof__(self) -> int - __str__ :: def __str__(self) -> str - __subclasscheck__ :: bound method .__subclasscheck__(subclass: type, /) -> bool - __subclasses__ :: bound method .__subclasses__[Self]() -> list[Self] - __subclasshook__ :: bound method .__subclasshook__(subclass: type, /) -> bool + __reduce__() :: def __reduce__(self) -> str | tuple[Any, ...] + __reduce_ex__() :: def __reduce_ex__(self, protocol: SupportsIndex, /) -> str | tuple[Any, ...] + __repr__() :: def __repr__(self) -> str + __ror__() :: bound method .__ror__[Self](value: Any, /) -> UnionType | Self + __setattr__() :: def __setattr__(self, name: str, value: Any, /) + __sizeof__() :: def __sizeof__(self) -> int + __str__() :: def __str__(self) -> str + __subclasscheck__() :: bound method .__subclasscheck__(subclass: type, /) -> bool + __subclasses__() :: bound method .__subclasses__[Self]() -> list[Self] + __subclasshook__() :: bound method .__subclasshook__(subclass: type, /) -> bool __text_signature__ :: str | None __type_params__ :: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] __weakrefoffset__ :: int @@ -6835,7 +7341,7 @@ Answer. list(Answer) :: postfix match Answer: case :: postfix - mro :: bound method .mro() -> list[type] + mro() :: bound method .mro() -> list[type] name :: enum.property not Answer :: postfix (Answer) :: postfix @@ -6849,59 +7355,59 @@ Answer. __base__ :: type | None __bases__ :: tuple[type, ...] __basicsize__ :: int - __bool__ :: bound method .__bool__() -> Literal[True] - __class__ :: - __contains__ :: bound method .__contains__(value: object) -> bool - __copy__ :: def __copy__[Self](self) -> Self - __deepcopy__ :: def __deepcopy__[Self](self, memo: Any) -> Self - __delattr__ :: def __delattr__(self, name: str, /) + __bool__() :: bound method .__bool__() -> Literal[True] + __class__() :: + __contains__() :: bound method .__contains__(value: object) -> bool + __copy__() :: def __copy__[Self](self) -> Self + __deepcopy__() :: def __deepcopy__[Self](self, memo: Any) -> Self + __delattr__() :: def __delattr__(self, name: str, /) __dict__ :: dict[str, Any] __dictoffset__ :: int - __dir__ :: def __dir__(self) -> list[str] + __dir__() :: def __dir__(self) -> list[str] __doc__ :: str | None - __eq__ :: def __eq__(self, value: object, /) -> bool + __eq__() :: def __eq__(self, value: object, /) -> bool __flags__ :: int - __format__ :: def __format__(self, format_spec: str) -> str - __getattribute__ :: def __getattribute__(self, name: str, /) -> Any - __getitem__ :: bound method .__getitem__[EnumMemberT](name: str) -> EnumMemberT - __getstate__ :: def __getstate__(self) -> object - __hash__ :: def __hash__(self) -> int - __init__ :: def __init__(self) - __init_subclass__ :: bound method .__init_subclass__() - __instancecheck__ :: bound method .__instancecheck__(instance: Any, /) -> bool + __format__() :: def __format__(self, format_spec: str) -> str + __getattribute__() :: def __getattribute__(self, name: str, /) -> Any + __getitem__() :: bound method .__getitem__[EnumMemberT](name: str) -> EnumMemberT + __getstate__() :: def __getstate__(self) -> object + __hash__() :: def __hash__(self) -> int + __init__() :: def __init__(self) + __init_subclass__() :: bound method .__init_subclass__() + __instancecheck__() :: bound method .__instancecheck__(instance: Any, /) -> bool __itemsize__ :: int - __iter__ :: bound method .__iter__[EnumMemberT]() -> Iterator[EnumMemberT] - __len__ :: bound method .__len__() -> int + __iter__() :: bound method .__iter__[EnumMemberT]() -> Iterator[EnumMemberT] + __len__() :: bound method .__len__() -> int __members__ :: MappingProxyType[str, Answer] __module__ :: str __mro__ :: tuple[type, ...] __name__ :: str - __ne__ :: def __ne__(self, value: object, /) -> bool - __new__ :: def __new__[Self](cls, value: object) -> Self - __or__ :: bound method .__or__[Self](value: Any, /) -> UnionType | Self + __ne__() :: def __ne__(self, value: object, /) -> bool + __new__() :: def __new__[Self](cls, value: object) -> Self + __or__() :: bound method .__or__[Self](value: Any, /) -> UnionType | Self __order__ :: str - __prepare__ :: bound method .__prepare__(cls: str, bases: tuple[type, ...], **kwds: Any) -> _EnumDict + __prepare__() :: bound method .__prepare__(cls: str, bases: tuple[type, ...], **kwds: Any) -> _EnumDict __qualname__ :: str - __reduce__ :: def __reduce__(self) -> str | tuple[Any, ...] - __repr__ :: def __repr__(self) -> str - __reversed__ :: bound method .__reversed__[EnumMemberT]() -> Iterator[EnumMemberT] - __ror__ :: bound method .__ror__[Self](value: Any, /) -> UnionType | Self - __setattr__ :: def __setattr__(self, name: str, value: Any, /) - __sizeof__ :: def __sizeof__(self) -> int - __str__ :: def __str__(self) -> str - __subclasscheck__ :: bound method .__subclasscheck__(subclass: type, /) -> bool - __subclasses__ :: bound method .__subclasses__[Self]() -> list[Self] - __subclasshook__ :: bound method .__subclasshook__(subclass: type, /) -> bool + __reduce__() :: def __reduce__(self) -> str | tuple[Any, ...] + __repr__() :: def __repr__(self) -> str + __reversed__() :: bound method .__reversed__[EnumMemberT]() -> Iterator[EnumMemberT] + __ror__() :: bound method .__ror__[Self](value: Any, /) -> UnionType | Self + __setattr__() :: def __setattr__(self, name: str, value: Any, /) + __sizeof__() :: def __sizeof__(self) -> int + __str__() :: def __str__(self) -> str + __subclasscheck__() :: bound method .__subclasscheck__(subclass: type, /) -> bool + __subclasses__() :: bound method .__subclasses__[Self]() -> list[Self] + __subclasshook__() :: bound method .__subclasshook__(subclass: type, /) -> bool __text_signature__ :: str | None __type_params__ :: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] __weakrefoffset__ :: int - _add_alias_ :: def _add_alias_(self, name: str) - _add_value_alias_ :: def _add_value_alias_(self, value: Any) - _generate_next_value_ :: def _generate_next_value_(name: str, start: int, count: int, last_values: list[Any]) -> Any + _add_alias_() :: def _add_alias_(self, name: str) + _add_value_alias_() :: def _add_value_alias_(self, value: Any) + _generate_next_value_() :: def _generate_next_value_(name: str, start: int, count: int, last_values: list[Any]) -> Any _ignore_ :: str | list[str] _member_map_ :: dict[str, Enum] _member_names_ :: list[str] - _missing_ :: bound method ._missing_(value: object) -> Any + _missing_() :: bound method ._missing_(value: object) -> Any _name_ :: str _order_ :: str _value2member_map_ :: dict[Any, Enum] @@ -6928,12 +7434,12 @@ quux. assert_snapshot!( builder.skip_keywords().skip_builtins().type_signatures().build().snapshot(), @r#" - count :: bound method Quux.count(value: Any, /) -> int + count() :: bound method Quux.count(value: Any, /) -> int for item in quux: :: postfix if quux: :: postfix - index :: bound method Quux.index(value: Any, start: SupportsIndex = 0, stop: SupportsIndex = ..., /) -> int + index() :: bound method Quux.index(value: Any, start: SupportsIndex = 0, stop: SupportsIndex = ..., /) -> int list(quux) :: postfix match quux: case :: postfix @@ -6946,50 +7452,50 @@ quux. type(quux) :: postfix x :: int y :: str - __add__ :: Overload[(value: tuple[int | str, ...], /) -> tuple[int | str, ...], [T](value: tuple[T, ...], /) -> tuple[int | str | T, ...]] + __add__() :: Overload[(value: tuple[int | str, ...], /) -> tuple[int | str, ...], [T](value: tuple[T, ...], /) -> tuple[int | str | T, ...]] __annotations__ :: dict[str, Any] - __class__ :: type[Quux] - __class_getitem__ :: bound method type[Quux].__class_getitem__(item: Any, /) -> GenericAlias - __contains__ :: bound method Quux.__contains__(key: Overlapping[int | str], /) -> bool - __delattr__ :: bound method Quux.__delattr__(name: str, /) + __class__() :: type[Quux] + __class_getitem__() :: bound method type[Quux].__class_getitem__(item: Any, /) -> GenericAlias + __contains__() :: bound method Quux.__contains__(key: Overlapping[int | str], /) -> bool + __delattr__() :: bound method Quux.__delattr__(name: str, /) __dict__ :: dict[str, Any] - __dir__ :: bound method Quux.__dir__() -> Iterable[str] + __dir__() :: bound method Quux.__dir__() -> Iterable[str] __doc__ :: str | None - __eq__ :: bound method Quux.__eq__(value: object, /) -> bool - __format__ :: bound method Quux.__format__(format_spec: Literal[""], /) -> str - __ge__ :: bound method Quux.__ge__(value: tuple[int | str, ...], /) -> bool - __getattribute__ :: bound method Quux.__getattribute__(name: str, /) -> Any - __getitem__ :: Overload[(index: Literal[-2, 0], /) -> int, (index: Literal[-1, 1], /) -> str, (index: SupportsIndex, /) -> int | str, (index: slice[SupportsIndex | None, SupportsIndex | None, SupportsIndex | None], /) -> tuple[int | str, ...]] - __getstate__ :: bound method Quux.__getstate__() -> object - __gt__ :: bound method Quux.__gt__(value: tuple[int | str, ...], /) -> bool - __hash__ :: bound method Quux.__hash__() -> int - __init__ :: bound method Quux.__init__() - __init_subclass__ :: bound method type[Quux].__init_subclass__() - __iter__ :: bound method Quux.__iter__() -> Iterator[int | str] - __le__ :: bound method Quux.__le__(value: tuple[int | str, ...], /) -> bool - __len__ :: () -> Literal[2] - __lt__ :: bound method Quux.__lt__(value: tuple[int | str, ...], /) -> bool + __eq__() :: bound method Quux.__eq__(value: object, /) -> bool + __format__() :: bound method Quux.__format__(format_spec: Literal[""], /) -> str + __ge__() :: bound method Quux.__ge__(value: tuple[int | str, ...], /) -> bool + __getattribute__() :: bound method Quux.__getattribute__(name: str, /) -> Any + __getitem__() :: Overload[(index: Literal[-2, 0], /) -> int, (index: Literal[-1, 1], /) -> str, (index: SupportsIndex, /) -> int | str, (index: slice[SupportsIndex | None, SupportsIndex | None, SupportsIndex | None], /) -> tuple[int | str, ...]] + __getstate__() :: bound method Quux.__getstate__() -> object + __gt__() :: bound method Quux.__gt__(value: tuple[int | str, ...], /) -> bool + __hash__() :: bound method Quux.__hash__() -> int + __init__() :: bound method Quux.__init__() + __init_subclass__() :: bound method type[Quux].__init_subclass__() + __iter__() :: bound method Quux.__iter__() -> Iterator[int | str] + __le__() :: bound method Quux.__le__(value: tuple[int | str, ...], /) -> bool + __len__() :: () -> Literal[2] + __lt__() :: bound method Quux.__lt__(value: tuple[int | str, ...], /) -> bool __match_args__ :: tuple[str, ...] __module__ :: str - __mul__ :: bound method Quux.__mul__(value: SupportsIndex, /) -> tuple[int | str, ...] - __ne__ :: bound method Quux.__ne__(value: object, /) -> bool - __new__ :: (x: int, y: str) -> Quux + __mul__() :: bound method Quux.__mul__(value: SupportsIndex, /) -> tuple[int | str, ...] + __ne__() :: bound method Quux.__ne__(value: object, /) -> bool + __new__() :: (x: int, y: str) -> Quux __orig_bases__ :: tuple[Any, ...] - __reduce__ :: bound method Quux.__reduce__() -> str | tuple[Any, ...] - __reduce_ex__ :: bound method Quux.__reduce_ex__(protocol: SupportsIndex, /) -> str | tuple[Any, ...] - __replace__ :: bound method NamedTupleFallback.__replace__(**kwargs: Any) -> NamedTupleFallback - __repr__ :: bound method Quux.__repr__() -> str - __reversed__ :: bound method Quux.__reversed__() -> Iterator[int | str] - __rmul__ :: bound method Quux.__rmul__(value: SupportsIndex, /) -> tuple[int | str, ...] - __setattr__ :: bound method Quux.__setattr__(name: str, value: Any, /) - __sizeof__ :: bound method Quux.__sizeof__() -> int - __str__ :: bound method Quux.__str__() -> str - __subclasshook__ :: bound method type[Quux].__subclasshook__(subclass: type, /) -> bool - _asdict :: bound method NamedTupleFallback._asdict() -> dict[str, Any] + __reduce__() :: bound method Quux.__reduce__() -> str | tuple[Any, ...] + __reduce_ex__() :: bound method Quux.__reduce_ex__(protocol: SupportsIndex, /) -> str | tuple[Any, ...] + __replace__() :: bound method NamedTupleFallback.__replace__(**kwargs: Any) -> NamedTupleFallback + __repr__() :: bound method Quux.__repr__() -> str + __reversed__() :: bound method Quux.__reversed__() -> Iterator[int | str] + __rmul__() :: bound method Quux.__rmul__(value: SupportsIndex, /) -> tuple[int | str, ...] + __setattr__() :: bound method Quux.__setattr__(name: str, value: Any, /) + __sizeof__() :: bound method Quux.__sizeof__() -> int + __str__() :: bound method Quux.__str__() -> str + __subclasshook__() :: bound method type[Quux].__subclasshook__(subclass: type, /) -> bool + _asdict() :: bound method NamedTupleFallback._asdict() -> dict[str, Any] _field_defaults :: dict[str, Any] _fields :: tuple[str, ...] - _make :: bound method type[NamedTupleFallback]._make(iterable: Iterable[Any]) -> NamedTupleFallback - _replace :: bound method NamedTupleFallback._replace(**kwargs: Any) -> NamedTupleFallback + _make() :: bound method type[NamedTupleFallback]._make(iterable: Iterable[Any]) -> NamedTupleFallback + _replace() :: bound method NamedTupleFallback._replace(**kwargs: Any) -> NamedTupleFallback "#); } @@ -7238,7 +7744,7 @@ bar( assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @" okay= - bar + bar() foo "); } @@ -7281,7 +7787,7 @@ class C: assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo + foo() self "); } @@ -7297,7 +7803,7 @@ class C: ", ); - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"C"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"C()"); } #[test] @@ -7317,7 +7823,7 @@ class C: assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), @" - foo + foo() self "); } @@ -7779,28 +8285,28 @@ q.foo.xyz return ... type(...) __annotations__ - __class__ - __delattr__ + __class__() + __delattr__() __dict__ - __dir__ + __dir__() __doc__ - __eq__ - __format__ - __getattribute__ - __getstate__ - __hash__ - __init__ - __init_subclass__ + __eq__() + __format__() + __getattribute__() + __getstate__() + __hash__() + __init__() + __init_subclass__() __module__ - __ne__ - __new__ - __reduce__ - __reduce_ex__ - __repr__ - __setattr__ - __sizeof__ - __str__ - __subclasshook__ + __ne__() + __new__() + __reduce__() + __reduce_ex__() + __repr__() + __setattr__() + __sizeof__() + __str__() + __subclasshook__() "); } @@ -7891,7 +8397,7 @@ def m(): pass ", ); - assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"m"); + assert_snapshot!(builder.skip_keywords().skip_builtins().build().snapshot(), @"m()"); } // Ref: https://github.com/astral-sh/ty/issues/572 @@ -7920,7 +8426,7 @@ Fo = float assert_snapshot!( builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), - @"Fo", + @"Fo()", ); } @@ -10128,23 +10634,378 @@ x: Literal["a"] = "a } #[test] - fn string_literal_completions_in_standalone_statement() { + fn string_field_writes_the_fstring_prefix_and_the_closing_brace() { let builder = completion_test_builder( r#" -from collections.abc import Callable -from typing import Literal +name = "john" +"hello {na" +"#, + ); -def func(callback: Callable[[], Literal["yes", "no"]]) -> None: ... + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.snapshot(), + @"name}", + ); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + f"hello {name}" + "#, + ); + } -x = y = func(lambda: "") + #[test] + fn string_field_leaves_a_brace_the_user_already_wrote() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {na}" "#, ); + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); assert_snapshot!( - builder.skip_keywords().skip_builtins().skip_auto_import().type_signatures().build().snapshot(), + test.apply("name"), @r#" - no :: Literal["no"] - yes :: Literal["yes"] + name = "john" + f"hello {name}" + "#, + ); + } + + #[test] + fn string_field_completes_a_local() { + let builder = completion_test_builder( + r#" +def greet(who: str) -> str: + return "hello {wh" +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("who"), + @r#" + def greet(who: str) -> str: + return f"hello {who}" + "#, + ); + } + + #[test] + fn string_field_prefixes_only_the_part_that_holds_it() { + let builder = completion_test_builder( + r#" +name = "john" +("hello " "there {na") +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + ("hello " f"there {name}") + "#, + ); + } + + #[test] + fn string_field_keeps_a_raw_prefix() { + let builder = completion_test_builder( + r#" +name = "john" +r"hello {na" +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + fr"hello {name}" + "#, + ); + } + + #[test] + fn string_field_replaces_the_rest_of_a_name_the_cursor_sits_inside() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {name}" +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + f"hello {name}" + "#, + ); + } + + #[test] + fn string_field_offers_a_second_field_beside_one_already_written() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {name} and {na" +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + f"hello {name} and {name}" + "#, + ); + } + + /// braces pair up into escapes from the left, so `{{{` opens a field after + /// one escaped brace + #[test] + fn string_field_reads_an_odd_run_of_braces_as_opening_one() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {{{na" +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + f"hello {{{name}" + "#, + ); + } + + #[test] + fn string_field_offers_nothing_without_a_brace() { + let builder = completion_test_builder( + r#" +name = "john" +"hello na" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_field_offers_nothing_for_a_closed_brace() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {name} na" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_field_offers_nothing_for_a_doubled_brace() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {{na" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_field_offers_nothing_for_something_that_is_not_a_name() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {name.up" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_field_offers_nothing_in_a_docstring() { + let builder = completion_test_builder( + r#" +name = "john" + +def greet() -> None: + """say hello to {na""" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + /// a string that merely opens a body is not a docstring, so a name in it + /// completes like any other + #[test] + fn string_field_completes_in_a_call_that_opens_a_body() { + let builder = completion_test_builder( + r#" +name = "john" + +def greet() -> None: + print("hi {na") +"#, + ); + + let builder = builder.skip_keywords().skip_builtins(); + let test = builder.build(); + assert_snapshot!( + test.apply("name"), + @r#" + name = "john" + + def greet() -> None: + print(f"hi {name}") + "#, + ); + } + + #[test] + fn string_field_offers_nothing_in_a_case_pattern() { + let builder = completion_test_builder( + r#" +name = "john" + +match name: + case "hello {na": + pass +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + /// the string is still an ordinary string, so a brace in it must not cost + /// the user the completions the string itself expects + #[test] + fn string_field_leaves_expected_literals_alone_when_nothing_matches() { + let builder = completion_test_builder( + r#" +from typing import Literal + +def render(template: Literal["{a}", "{b}"]) -> None: ... + +render("{zqzqzq") +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().skip_auto_import().build().snapshot(), + @r#" + {a} + {b} + "#, + ); + } + + #[test] + fn string_field_offers_nothing_in_a_format_template() { + let builder = completion_test_builder( + r#" +name = "john" +"hello {na".format(name=name) +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_field_offers_nothing_in_a_bytes_literal() { + let builder = completion_test_builder( + r#" +name = "john" +b"hello {na" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_field_offers_nothing_in_a_unicode_prefixed_string() { + let builder = completion_test_builder( + r#" +name = "john" +u"hello {na" +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().build().snapshot(), + @"", + ); + } + + #[test] + fn string_literal_completions_in_standalone_statement() { + let builder = completion_test_builder( + r#" +from collections.abc import Callable +from typing import Literal + +def func(callback: Callable[[], Literal["yes", "no"]]) -> None: ... + +x = y = func(lambda: "") +"#, + ); + + assert_snapshot!( + builder.skip_keywords().skip_builtins().skip_auto_import().type_signatures().build().snapshot(), + @r#" + no :: Literal["no"] + yes :: Literal["yes"] "#, ); } @@ -11604,7 +12465,7 @@ if foo: // long_nameb is currently imported and should be preferred. assert_snapshot!(snapshot, @" long_nameb :: Literal[1] :: - long_namea :: Unavailable :: foo + long_namea() :: Unavailable :: foo "); } @@ -11735,7 +12596,7 @@ if foo: .skip_auto_import() .build() .snapshot(); - assert_snapshot!(value_position, @"ignorable_return_value"); + assert_snapshot!(value_position, @"ignorable_return_value()"); let type_position = CursorTest::builder() .source("main.by", "a: ignorable_return_val") @@ -11797,14 +12658,17 @@ if foo: .not_contains("main"); } - /// Once the module has an entry point, `main` is a reference to it. + /// Once the module has an entry point, `main` is a reference to it — an + /// ordinary call, not the whole `def main(): …` the entry-point completion + /// writes when there is none. It carries the parentheses every callable + /// completion carries. #[test] fn basedpython_main_completion_skipped_when_defined() { let builder = CursorTest::builder() .source("main.by", "def main():\n pass\nmai") .completion_test_builder() .skip_auto_import(); - assert_eq!(inserted_text(&builder.build(), "main"), "main"); + assert_eq!(inserted_text(&builder.build(), "main"), "main()"); } /// Only a module-level `main` is the entry point. @@ -12001,7 +12865,42 @@ if foo: .completion_test_builder() .skip_auto_import() .filter(|c| c.detail.as_deref() == Some("extension")); - assert_snapshot!(builder.build().snapshot(), @"second"); + assert_snapshot!(builder.build().snapshot(), @"second()"); + } + + /// basedpython: an inline protocol's members. The type is structural, so it + /// has no class of its own to walk — without its interface being consulted + /// directly the list came back with nothing but postfix snippets. + #[test] + fn basedpython_inline_protocol_completions() { + let builder = CursorTest::builder() + .source( + "main.by", + "def f(x: protocol(a: int; def g(self) -> int)) -> int:\n return x.", + ) + .completion_test_builder() + .skip_auto_import() + .filter(|c| matches!(c.name.as_str(), "a" | "g")); + assert_snapshot!(builder.build().snapshot(), @" + a + g() + "); + } + + /// basedpython: `extension` is a statement opener like any other. The + /// single-word basedpython keywords were offered nowhere, because the only + /// basedpython-aware keyword table held the *compound* forms and the list + /// beside it is python's keywords verbatim. + #[test] + fn basedpython_single_word_statement_keywords() { + for keyword in ["extension", "let", "var"] { + CursorTest::builder() + .source("main.by", "") + .completion_test_builder() + .skip_auto_import() + .build() + .contains(keyword); + } } /// An extension in an imported module applies too. @@ -12749,7 +13648,7 @@ from .imp .iter() .all(|completion| completion.is_type_check_only) ); - assert_snapshot!(completions.snapshot(), @"static_assert :: ty_extensions"); + assert_snapshot!(completions.snapshot(), @"static_assert() :: ty_extensions"); } #[test] @@ -12766,7 +13665,7 @@ from .imp .iter() .all(|completion| completion.is_type_check_only) ); - assert_snapshot!(completions.snapshot(), @"TypedDictFallback :: _typeshed._type_checker_internals"); + assert_snapshot!(completions.snapshot(), @"TypedDictFallback() :: _typeshed._type_checker_internals"); } #[test] @@ -12798,8 +13697,8 @@ from .imp "runtime `typing_extensions` should not be downranked", ); assert_snapshot!(completions.snapshot(), @" - deprecated :: typing_extensions - deprecated :: warnings + deprecated() :: typing_extensions + deprecated() :: warnings "); } @@ -12831,8 +13730,8 @@ from .imp && completion.is_type_check_only })); assert_snapshot!(completions.snapshot(), @" - deprecated :: typing_extensions - deprecated :: warnings + deprecated() :: typing_extensions + deprecated() :: warnings "); } @@ -12907,7 +13806,7 @@ dependencies = ["requests"] .filter(|c| { c.name == "get" && c.module_name.map(ModuleName::as_str) == Some("requests") }); - assert_snapshot!(builder.build().snapshot(), @"get :: requests"); + assert_snapshot!(builder.build().snapshot(), @"get() :: requests"); } #[test] @@ -12972,8 +13871,8 @@ from collections import ChainMap as ChainMap ) }); assert_snapshot!(builder.build().snapshot(), @" - ChainMap :: collections - ChainMap :: thirdparty + ChainMap() :: collections + ChainMap() :: thirdparty "); } @@ -12999,8 +13898,8 @@ def no_type_check_decorator(): ) }); assert_snapshot!(builder.build().snapshot(), @" - no_type_check_decorator :: thirdparty - no_type_check_decorator :: typing + no_type_check_decorator() :: thirdparty + no_type_check_decorator() :: typing "); } @@ -13025,8 +13924,8 @@ from os import getpid as getpid ) }); assert_snapshot!(builder.build().snapshot(), @" - getpid :: os - getpid :: thirdparty + getpid() :: os + getpid() :: thirdparty "); } @@ -13069,8 +13968,8 @@ def prefix_name(): .filter(|c| matches!(c.name.as_str(), "a_prefix" | "prefix_name")); assert_snapshot!(builder.build().snapshot(), @" - prefix_name :: matching - a_prefix :: matching + prefix_name() :: matching + a_prefix() :: matching "); } @@ -13435,8 +14334,10 @@ value = Call ); } + /// basedpython turns callable parentheses on by default, and a class + /// constructor is a callable — see `CompletionSettings::default`. #[test] - fn complete_class_parentheses_disabled_by_default() { + fn complete_class_parentheses_enabled_by_default() { let builder = completion_test_builder( "\ class CallableType: ... @@ -13446,7 +14347,7 @@ value = Call ); assert_snapshot!( builder.skip_auto_import().skip_builtins().build().snapshot(), - @"CallableType", + @"CallableType()", ); } @@ -13749,10 +14650,10 @@ TypedDi builder.build().snapshot(), @" TypedDict :: , TypedDict - is_typeddict :: , is_typeddict + is_typeddict() :: , is_typeddict TypedDict :: from typing_extensions import TypedDict - is_typeddict :: from typing_extensions import is_typeddict + is_typeddict() :: from typing_extensions import is_typeddict ", ); } @@ -13768,7 +14669,7 @@ TypedDi .filter(|c| c.name == "Thing"); assert_snapshot!( builder.build().snapshot(), - @"Thing :: :: ", + @"Thing() :: :: ", ); } @@ -13783,7 +14684,7 @@ TypedDi .filter(|c| c.name == "Thing"); assert_snapshot!( builder.build().snapshot(), - @"Thing :: :: ", + @"Thing() :: :: ", ); } @@ -13805,7 +14706,7 @@ def f(): .filter(|c| c.name == "Thing"); assert_snapshot!( builder.build().snapshot(), - @"Thing :: :: ", + @"Thing() :: :: ", ); } @@ -13828,7 +14729,7 @@ def f(): .filter(|c| c.name == "Thing"); assert_snapshot!( builder.build().snapshot(), - @"Thing :: :: ", + @"Thing() :: :: ", ); } @@ -13845,7 +14746,7 @@ dt.timed .imports(); assert_snapshot!( builder.build().snapshot(), - @"timedelta :: datetime :: import datetime as dt", + @"timedelta() :: datetime :: import datetime as dt", ); } @@ -13914,7 +14815,7 @@ dt.timed .imports(); assert_snapshot!( builder.build().snapshot(), - @"timedelta :: :: ", + @"timedelta() :: :: ", ); } @@ -13943,7 +14844,7 @@ np.ara .imports(); assert_snapshot!( builder.build().snapshot(), - @"deposit :: bureau :: import bureau as bo", + @"deposit() :: bureau :: import bureau as bo", ); } @@ -13960,10 +14861,278 @@ np.ara .filter(|c| matches!(c.name.as_str(), "deposit" | "timedelta")); assert_snapshot!( builder.build().snapshot(), - @"deposit :: bureau :: import bureau as dt", + @"deposit() :: bureau :: import bureau as dt", ); } + /// A file that never imported `Asdf` still means the `Asdf` an import would bind, so the + /// class's members are offered, and accepting one writes that import. + #[test] + fn unimported_name_offers_its_members() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @"name :: mod :: from mod import Asdf"); + } + + /// The same goes for a module the file has not imported. + #[test] + fn unimported_module_offers_its_members() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "mod.As") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "Asdf"); + assert_snapshot!(builder.build().snapshot(), @"Asdf() :: mod :: import mod"); + } + + /// Every step of a longer chain is followed the same way. + #[test] + fn unimported_module_offers_the_members_of_its_members() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "mod.Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @"name :: mod :: import mod"); + } + + /// A name the file already binds means what the file says it means, whatever else could be + /// imported under that name. + #[test] + fn unimported_name_yields_to_a_binding() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "Asdf = 1\nAsdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @""); + } + + /// Once the file writes the import, the members come with no import attached. + #[test] + fn unimported_name_adds_no_import_once_it_is_written() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "from mod import Asdf\n\nAsdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @"name :: :: "); + } + + /// A chain that starts from anything but a name has no name for an import to bind. + #[test] + fn unimported_call_offers_nothing() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "Asdf().na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @""); + } + + /// A binding whose type is unknown infers exactly as an unbound name does, so it is the + /// scope, not the type, that has to say the name is already taken. + #[test] + fn unimported_name_yields_to_a_binding_of_unknown_type() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source( + "main.py", + "import broken\n\nAsdf = broken.thing\nAsdf.na", + ) + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @""); + } + + /// The class the same file writes in basedpython answers the same way, which is what makes + /// the two `private` cases below a test of `private` rather than of the language. + #[test] + fn unimported_basedpython_name_offers_its_members() { + let builder = CursorTest::builder() + .source("mod.by", "class Asdf:\n class name = \"Asdf\"\n") + .source("main.by", "Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @"name :: mod :: from mod import Asdf"); + } + + /// A `private` symbol is the module's implementation, and auto-import already refuses to + /// name one, so there is nothing here for a chain to start from either. + #[test] + fn unimported_private_name_offers_nothing() { + let builder = CursorTest::builder() + .source("mod.by", "private class Asdf:\n class name = \"Asdf\"\n") + .source("main.by", "Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @""); + } + + /// Reaching the same symbol as a step of a chain goes around auto-import's own filter, so the + /// walk has to refuse it again at every step. + #[test] + fn unimported_module_hides_its_private_members() { + let builder = CursorTest::builder() + .source("mod.by", "private class Asdf:\n class name = \"Asdf\"\n") + .source("main.by", "mod.Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @""); + } + + /// A module is imported from its package, because `import pkg.sub` binds `pkg` and the file + /// wrote `sub`. + #[test] + fn unimported_submodule_is_imported_from_its_package() { + let builder = CursorTest::builder() + .source("pkg/__init__.py", "") + .source("pkg/sub.py", "class Asdf: ...\n") + .source("main.py", "sub.As") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "Asdf"); + assert_snapshot!(builder.build().snapshot(), @"Asdf() :: pkg.sub :: from pkg import sub"); + } + + /// A submodule reached part-way along a chain is followed too, though nothing has imported it + /// and it is therefore not a member of its package yet. + #[test] + fn unimported_package_follows_a_submodule_step() { + let builder = CursorTest::builder() + .source("pkg/__init__.py", "") + .source("pkg/sub.py", "class Asdf: ...\n") + .source("main.py", "pkg.sub.As") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "Asdf"); + assert_snapshot!(builder.build().snapshot(), @"Asdf() :: pkg :: import pkg"); + } + + /// A file that already imports the module still gets the symbol offered. Reusing that import + /// would answer with `mod.Asdf`, which is not the name the file wrote, so the `from` import is + /// asked for outright. + #[test] + fn unimported_name_is_offered_beside_an_import_of_its_module() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = \"Asdf\"\n") + .source("main.py", "import mod\n\nAsdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @"name :: mod :: from mod import Asdf"); + } + + /// Two modules exporting a name of their own are two different offers, each carrying its own + /// import. + #[test] + fn unimported_name_is_offered_once_per_module_that_has_one() { + let builder = CursorTest::builder() + .source("a.py", "class Asdf:\n name = 1\n") + .source("b.py", "class Asdf:\n name = 2\n") + .source("main.py", "Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @" + name :: a :: from a import Asdf + + name :: b :: from b import Asdf + "); + } + + /// Re-exports are not. `typing`, `typing_extensions` and `collections.abc` all export the one + /// `Mapping`, and a copy of its members under each would differ only by an import edit the + /// list does not show. + #[test] + fn unimported_reexports_offer_one_copy_of_the_members() { + let builder = CursorTest::builder() + .source("main.py", "Mapping.ge") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "get"); + assert_snapshot!(builder.build().snapshot(), @"get() :: collections.abc :: from collections.abc import Mapping"); + } + + /// Nothing is offered for a bare `mod.`, on the same bargain auto-import strikes in a scope: a + /// query that matches everything would answer with every member of every symbol named `mod`. + #[test] + fn unimported_name_needs_a_query_to_match() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf: ...\n") + .source("main.py", "mod.") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.import.is_some()); + assert_snapshot!(builder.build().snapshot(), @""); + } + + /// A root that is only there while type checking makes the members reached through it the + /// same, however ordinary they are: the import is the type-check-only part. + #[test] + fn unimported_type_check_only_root_marks_its_members() { + let builder = CursorTest::builder() + .source( + "mod.pyi", + "from typing import type_check_only\n\n@type_check_only\nclass Asdf:\n name = 1\n", + ) + .source("main.py", "Asdf.na") + .completion_test_builder() + .module_names() + .imports() + .filter(|c| c.name == "name"); + let test = builder.build(); + assert!( + test.completions() + .iter() + .all(|completion| completion.is_type_check_only) + ); + assert_snapshot!(test.snapshot(), @"name :: mod :: from mod import Asdf"); + } + + /// Turning auto-import off turns these off, since an import is all they are. + #[test] + fn unimported_name_is_not_offered_without_auto_import() { + let builder = CursorTest::builder() + .source("mod.py", "class Asdf:\n name = 1\n") + .source("main.py", "Asdf.na") + .completion_test_builder() + .skip_auto_import() + .module_names() + .imports() + .filter(|c| c.name == "name"); + assert_snapshot!(builder.build().snapshot(), @""); + } + /// Options configuring just the common aliases. fn common_aliases(aliases: [(&str, &str); N]) -> Options { Options { @@ -14000,7 +15169,7 @@ Thi .filter(|c| c.name == "Thing"); assert_snapshot!( builder.build().snapshot(), - @"Thing :: :: ", + @"Thing() :: :: ", ); } @@ -14023,7 +15192,7 @@ Thi .filter(|c| c.name == "Thing"); assert_snapshot!( builder.build().snapshot(), - @"Thing :: :: ", + @"Thing() :: :: ", ); } @@ -14039,8 +15208,8 @@ Thi assert_snapshot!( builder.build().snapshot(), @" - Thing :: :: - Thing :: package :: from package import Thing + Thing() :: :: + Thing() :: package :: from package import Thing ", ); } @@ -14058,8 +15227,8 @@ Thi assert_snapshot!( builder.build().snapshot(), @" - Thing :: :: - package.Thing :: package :: import package + Thing() :: :: + package.Thing() :: package :: import package ", ); } @@ -14086,7 +15255,7 @@ def f(): builder.build().snapshot(), @" Thing :: :: - package.Thing :: package :: import package + package.Thing() :: package :: import package ", ); } @@ -14111,7 +15280,7 @@ class C: .filter(|c| c.name == "Thing" && c.import.is_some()); assert_snapshot!( builder.build().snapshot(), - @"Thing :: package :: from package import Thing", + @"Thing() :: package :: from package import Thing", ); } @@ -14128,8 +15297,8 @@ my_list[0].remove assert_snapshot!( builder.build().snapshot(), @" - removeprefix - removesuffix + removeprefix() + removesuffix() ", ); } @@ -14149,8 +15318,8 @@ def f(x: Any | str): assert_snapshot!( builder.build().snapshot(), @" - removeprefix - removesuffix + removeprefix() + removesuffix() ", ); } @@ -14171,8 +15340,8 @@ def f(x: Intersection[int, Any] | str): assert_snapshot!( builder.build().snapshot(), @" - removeprefix - removesuffix + removeprefix() + removesuffix() ", ); } @@ -14192,25 +15361,25 @@ def f(x: UnsafeUnion[int, str]): assert_snapshot!( builder.build().snapshot(), @" - is_integer - isalnum - isalpha - isascii - isdecimal - isdigit - isidentifier - islower - isnumeric - isprintable - isspace - istitle - isupper + is_integer() + isalnum() + isalpha() + isascii() + isdecimal() + isdigit() + isidentifier() + islower() + isnumeric() + isprintable() + isspace() + istitle() + isupper() list(x) raise x - splitlines + splitlines() __annotations__ - __contains__ - __init_subclass__ + __contains__() + __init_subclass__() ", ); } @@ -14414,6 +15583,7 @@ raise .collect(); CompletionTest { db: self.db(), + cursor: &self.cursor_test.cursor, original, filtered, type_signatures: self.type_signatures, @@ -14529,6 +15699,8 @@ raise struct CompletionTest<'db> { db: &'db ty_project::TestDb, + /// the file the completions were requested in, and where in it + cursor: &'db crate::tests::Cursor, /// The original completions returned before any additional /// test-specific filtering. We keep this around in order to /// slightly modify the test snapshot generated. This @@ -14600,6 +15772,47 @@ raise .join("\n") } + /// the document as it reads once the named completion is taken + /// + /// every edit the completion carries is applied, so this is what the + /// user would be left looking at + #[track_caller] + fn apply(&self, name: &str) -> String { + let completion = self + .filtered + .iter() + .find(|completion| completion.name == name) + .unwrap_or_else(|| panic!("Expected completions to include `{name}`")); + let replace = completion.replace.unwrap_or_else(|| { + panic!("`{name}` says nothing about what it replaces, so it cannot be applied") + }); + let inserted = completion.insert.as_deref().unwrap_or(completion.label()); + + let mut edits = vec![(replace, inserted.to_string())]; + for edit in completion + .import + .iter() + .chain(completion.additional_edit.iter()) + { + edits.push(( + edit.range(), + edit.content().map(ToString::to_string).unwrap_or_default(), + )); + } + // applying from the back keeps every range pointing at the text it + // was computed against + edits.sort_by_key(|(range, _)| std::cmp::Reverse(range.start())); + + let mut source = self.cursor.source.as_str().to_string(); + for (range, text) in edits { + source.replace_range( + range.start().to_usize()..range.end().to_usize(), + text.as_str(), + ); + } + source + } + #[track_caller] fn contains(&self, expected: &str) -> &CompletionTest<'db> { assert!( diff --git a/crates/ty_ide/src/document_symbols.rs b/crates/ty_ide/src/document_symbols.rs index 8d84d1212f..e9ad617a3f 100644 --- a/crates/ty_ide/src/document_symbols.rs +++ b/crates/ty_ide/src/document_symbols.rs @@ -25,7 +25,9 @@ mod tests { /// /// An enum's variants and an extension's methods are already carried by the /// ordinary class walk, since both are classes by the time the outline sees - /// them. + /// them. An extension is named the way it is written, because its header + /// names the class it extends rather than declaring one — listed as a bare + /// `list` it was a second symbol indistinguishable from the real class. #[test] fn basedpython_symbols() { let test = CursorTest::builder() @@ -61,7 +63,7 @@ class P: "Class Circle", "Field radius", "Class Square", - "Class list", + "Class extension list", "Method second", "Class P", "Property x", diff --git a/crates/ty_ide/src/goto.rs b/crates/ty_ide/src/goto.rs index d36f20226e..6aeddd4e90 100644 --- a/crates/ty_ide/src/goto.rs +++ b/crates/ty_ide/src/goto.rs @@ -399,16 +399,22 @@ impl<'db> Definitions<'db> { let focus_range = definition.focus_range(db, &module); let full_range = definition.full_range(db, &module); - NavigationTarget { - file: focus_range.file(), - focus_range: focus_range.range(), - full_range: full_range.range(), - } + NavigationTarget::create( + db, + focus_range.file(), + focus_range.range(), + full_range.range(), + ) } ResolvedDefinition::Module(file) => { NavigationTarget::new(file.file(db), TextRange::default()) } - ResolvedDefinition::FileWithRange(file_range) => NavigationTarget::from(file_range), + ResolvedDefinition::FileWithRange(file_range) => NavigationTarget::create( + db, + file_range.file(), + file_range.range(), + file_range.range(), + ), }) .collect() } @@ -637,9 +643,30 @@ impl GotoTarget<'_> { Some(defs) } - GotoTarget::ClassDef(class) => Some(vec![ResolvedDefinition::Definition( - class.definition(model), - )]), + GotoTarget::ClassDef(class) => { + // basedpython: an `extension Widget:` header names the class it + // extends rather than declaring one, so the name is a reference + // and goto belongs on `class Widget`. Resolving it as its own + // definition made the header self-referential, and left + // find-references on a class unable to list its extensions — + // which in a codebase that uses them is the thing worth asking + if class.is_extension() { + // the *statement* is what carries a scope the model can + // answer for; a class name's own identifier is not recorded + let extended = definitions_for_name( + model, + class.name.as_str(), + AnyNodeRef::StmtClassDef(class), + alias_resolution, + ); + if !extended.is_empty() { + return Some(Definitions::new(extended)); + } + } + Some(vec![ResolvedDefinition::Definition( + class.definition(model), + )]) + } GotoTarget::Parameter(parameter) => Some(vec![ResolvedDefinition::Definition( parameter.definition(model), @@ -718,6 +745,15 @@ impl GotoTarget<'_> { } GotoTarget::PatternMatchAsName(pattern_as) => pattern_as.name.as_ref().map(|name| { + // basedpython: a bare `case Red:` looks like a capture and is one + // only when the subject's type does not declare the name. Where it + // does, the pattern is a *reference* to that enum member, and + // answering with the pattern's own binding tells the reader the + // opposite of what the checker decided + let case = ty_python_semantic::definitions_for_case_name(model, name); + if !case.is_empty() { + return case; + } definitions_for_name( model, name.as_str(), @@ -1374,12 +1410,19 @@ fn definitions_for_expression<'db>( alias_resolution: ImportAliasResolution, ) -> Option>> { match expression { - ast::ExprRef::Name(name) => Some(definitions_for_name( - model, - name.id.as_str(), - expression.into(), - alias_resolution, - )), + ast::ExprRef::Name(name) => { + let definitions = + definitions_for_name(model, name.id.as_str(), expression.into(), alias_resolution); + if !definitions.is_empty() { + return Some(definitions); + } + // basedpython: nothing in scope binds a context-resolved enum member + // — `Red` in `c: Color = Red` is reached through the expected type — + // so the scope walk above has nothing to answer with + Some(ty_python_semantic::definitions_for_context_sensitive_name( + model, name, + )) + } ast::ExprRef::Attribute(attribute) => Some(ty_python_semantic::definitions_for_attribute( model, attribute, )), @@ -1466,15 +1509,105 @@ fn find_goto_target_impl<'a>( return None; } - let covering_node = covering_node(syntax, token.range()) - .find_first(|node| { - node.is_identifier() || node.is_expression() || node.is_stmt_import_from() - }) - .ok()?; + // basedpython: when nothing containing this offset is a name or an + // expression, a property accessor block's declaration line is what it is — + // see `property_declaration_name` + let Ok(covering_node) = covering_node(syntax, token.range()).find_first(|node| { + node.is_identifier() || node.is_expression() || node.is_stmt_import_from() + }) else { + return property_declaration_name(syntax, offset); + }; + + // basedpython: a synthesized marker's range deliberately spans the whole + // construct it stands for. A property accessor block becomes a `def` carrying + // a `__property__` decorator whose range covers everything from `var age: int` + // down to the last accessor, because the lowering replaces exactly that span. + // The decorator is visited before the body, so the covering-node search settles + // inside the marker for *every* position in the construct and then answers + // about the marker — which is why hovering `field` in a getter reported + // ``. `ExprContext::Invalid` is what makes a node synthetic; + // when the search lands on one, look again inside the body the marker swallowed + if is_synthetic_marker(covering_node.node()) + && let Some(target) = retry_inside_swallowed_body(model, &covering_node, offset, tokens) + { + return Some(target); + } GotoTarget::from_covering_node(model, &covering_node, offset, tokens) } +/// basedpython: the `def` an accessor-block property's declaration name belongs +/// to — the `age` of `var age: int = 0`. +/// +/// The construct lowers to a getter ranged over the accessor *below* the +/// declaration, carrying the name the author wrote on the declaration line. That +/// name is therefore outside its own function's range, and no node containing +/// that offset exists for the covering-node walk to descend into — so hovering a +/// property's own name answered nothing at all. +/// +/// The getter is a member of the enclosing class body, which is where this looks +/// for it. Answering with the `def` is what a python `@property` declaration +/// answers with, and reaches the same hover and the same definition. +fn property_declaration_name(syntax: AnyNodeRef<'_>, offset: TextSize) -> Option> { + let token_range = TextRange::new(offset, offset); + let class = covering_node(syntax, token_range) + .ancestors() + .find_map(ruff_python_ast::AnyNodeRef::stmt_class_def)?; + class.body.iter().find_map(|member| { + let function = member.as_function_def_stmt()?; + // only a *synthesized* property answers: an ordinary `def` keeps its name + // inside its own range, so the walk above would already have found it + let is_property = function.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, ast::Expr::Name(name) if is_synthetic_marker(name.into())) + }); + (is_property && function.name.range().contains_range(token_range)) + .then_some(GotoTarget::FunctionDef(function)) + }) +} + +/// Whether `node` is one of the marker expressions basedpython's parser +/// synthesizes to stand for surface syntax python has no node for. +/// +/// The ids differ per construct and are deliberately not matched on here — the +/// context is the marker, and a new marker gets this behaviour for free. +fn is_synthetic_marker(node: AnyNodeRef<'_>) -> bool { + matches!( + node, + AnyNodeRef::ExprName(ast::ExprName { + ctx: ruff_python_ast::ExprContext::Invalid, + .. + }) + ) +} + +/// Search again for a goto target, rooted at the body statements a synthetic +/// marker's construct-wide range hid from the first search. +fn retry_inside_swallowed_body<'a>( + model: &'a SemanticModel, + swallowed: &CoveringNode<'a>, + offset: TextSize, + tokens: &'a Tokens, +) -> Option> { + let function = swallowed + .ancestors() + .find_map(ruff_python_ast::AnyNodeRef::stmt_function_def)?; + let token_range = TextRange::new(offset, offset); + function + .body + .iter() + .filter(|stmt| stmt.range().contains_range(token_range)) + .find_map(|stmt| { + let inner = covering_node(AnyNodeRef::from(stmt), token_range) + .find_first(|node| { + node.is_identifier() || node.is_expression() || node.is_stmt_import_from() + }) + .ok()?; + (!is_synthetic_marker(inner.node())) + .then(|| GotoTarget::from_covering_node(model, &inner, offset, tokens)) + .flatten() + }) +} + /// Helper function to resolve a module name and create a navigation target. fn definitions_for_module<'db>( model: &SemanticModel<'db>, diff --git a/crates/ty_ide/src/goto_definition.rs b/crates/ty_ide/src/goto_definition.rs index bf8e326073..f9e13461e5 100644 --- a/crates/ty_ide/src/goto_definition.rs +++ b/crates/ty_ide/src/goto_definition.rs @@ -1700,6 +1700,442 @@ a = \"asdf\" "); } + /// basedpython: an `extension` member reached by a *bare* attribute access. + /// A call resolves through the call's own dispatch target and so found the + /// extension's function by accident; `xs.second` on its own has no call to + /// go through, and a property can never be a callee at all + #[test] + fn goto_definition_extension_member_without_a_call() { + let test = CursorTest::builder() + .source( + "main.by", + " +extension str: + def f(self) -> int: + return 1 + +\"asdf\".f +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @r#" + info[goto-definition]: Go to definition + --> main.by:6:8 + | + 6 | "asdf".f + | ^ Clicking here + info: Found 1 definition + --> main.by:3:9 + | + 3 | def f(self) -> int: + | - + "#); + } + + /// basedpython: an `extension` property. It is only ever read, never called, + /// so it reaches goto through the bare-access fallback and nothing else + #[test] + fn goto_definition_extension_property() { + let test = CursorTest::builder() + .source( + "main.by", + " +class Widget: ... + +extension Widget: + let size: int + get() = 1 + +w = Widget() +w.size +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:9:3 + | + 9 | w.size + | ^^^^ Clicking here + info: Found 1 definition + --> main.by:5:9 + | + 5 | let size: int + | ---- + "); + } + + /// basedpython: an implicit-receiver callable — `x.fn` where `fn` names a + /// receiver callable in scope rather than a member of `x`. The declaration + /// is an ordinary name in an enclosing scope, so the class-hierarchy walk + /// cannot reach it + #[test] + fn goto_definition_implicit_receiver_callable() { + let test = CursorTest::builder() + .source( + "main.by", + " +def apply(fn: int.() -> str) -> str: + receiver = 1 + return receiver.fn() +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:4:21 + | + 4 | return receiver.fn() + | ^^ Clicking here + info: Found 1 definition + --> main.by:2:11 + | + 2 | def apply(fn: int.() -> str) -> str: + | -- + "); + } + + /// basedpython: a name a `let` pattern binds. The pattern is the same node a + /// `match` case uses, so the binder is an ordinary capture + #[test] + fn goto_definition_let_destructuring_binder() { + let test = CursorTest::builder() + .source( + "main.by", + " +def f() -> tuple[int, str]: + return (1, \"a\") + +let (a, b) := f() +print(a) +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:6:7 + | + 6 | print(a) + | ^ Clicking here + info: Found 1 definition + --> main.by:5:6 + | + 5 | let (a, b) := f() + | - + "); + } + + /// basedpython: the name in an `extension Widget:` header. It denotes the + /// class the extension extends rather than declaring one, so goto belongs on + /// `class Widget` — answering with the header itself made it self-referential + /// and left find-references on a class unable to list its extensions + #[test] + fn goto_definition_extension_header_names_the_extended_class() { + let test = CursorTest::builder() + .source( + "main.by", + " +class Widget: ... + +extension Widget: + def go(self) -> int: + return 1 +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:4:11 + | + 4 | extension Widget: + | ^^^^^^ Clicking here + info: Found 1 definition + --> main.by:2:7 + | + 2 | class Widget: ... + | ------ + "); + } + + /// basedpython: a bare enum member reached through the expected type. + /// Nothing in scope binds `Red`, so the ordinary scope walk had nothing to + /// answer with even though hover already resolved it to `Color.Red` + #[test] + fn goto_definition_context_resolved_enum_member() { + let test = CursorTest::builder() + .source( + "main.by", + " +enum class Color: + case Red + case Green + +c: Color = Red +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:6:12 + | + 6 | c: Color = Red + | ^^^ Clicking here + info: Found 1 definition + --> main.by:3:10 + | + 3 | case Red + | --- + "); + } + + /// basedpython: a bare `case Red:` that names an enum member. It looks like + /// a capture and is one only where the subject's type does not declare the + /// name; answering with the pattern's own binding said the opposite of what + /// the checker decided + #[test] + fn goto_definition_enum_member_case_pattern() { + let test = CursorTest::builder() + .source( + "main.by", + " +enum class Color: + case Red + case Green + +def f(c: Color) -> int: + match c: + case Red: + return 1 + case Green: + return 2 +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:8:14 + | + 8 | case Red: + | ^^^ Clicking here + info: Found 1 definition + --> main.by:3:10 + | + 3 | case Red + | --- + "); + } + + /// basedpython: `field` inside a property getter. The getter carries the + /// `__property__` marker, whose range spans the whole construct so the + /// lowering knows what to replace — and the covering-node search settled + /// inside that marker for every position in the block, so a getter's `field` + /// answered about the property object while the setter's resolved correctly + #[test] + fn goto_definition_field_in_a_property_getter() { + let test = CursorTest::builder() + .source( + "main.by", + " +class Person: + var age: int = 0 + get() = field + set(value): + field = value +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:4:17 + | + 4 | get() = field + | ^^^^^ Clicking here + info: Found 1 definition + --> main.by:3:5 + | + 3 | var age: int = 0 + | - + "); + } + + /// basedpython: the *name* of an accessor-block property. The construct + /// lowers to a `def` ranged over the accessor *below* the declaration, which + /// carries the name the author wrote on the line above — so the name sits + /// outside its own function and the covering-node walk has nothing + /// containing that offset to descend into + #[test] + fn goto_definition_property_declaration_name() { + let test = CursorTest::builder() + .source( + "main.by", + " +class Person: + var age: int = 0 + get() = field + set(value): + field = value +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:3:9 + | + 3 | var age: int = 0 + | ^^^ Clicking here + info: Found 1 definition + --> main.by:3:9 + | + 3 | var age: int = 0 + | --- + "); + } + + /// The accessors may be written in either order, so neither the declaration + /// name nor a `field` may depend on the getter coming first + #[test] + fn goto_definition_property_with_the_setter_written_first() { + let test = CursorTest::builder() + .source( + "main.by", + " +class Person: + var age: int = 0 + set(value): + field = value + get() = field +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:5:13 + | + 5 | field = value + | ^^^^^ Clicking here + info: Found 1 definition + --> main.by:3:5 + | + 3 | var age: int = 0 + | - + "); + } + + /// A property is one declaration in the source and several `def`s in the + /// tree, each carrying the same name range, so goto offered the same place + /// twice. Python's own getter/setter pair sit at two different `def`s and + /// stay the two targets they are — see + /// `goto_definition_property_getter_and_setter_are_both_offered` + #[test] + fn goto_definition_property_use_is_not_offered_twice() { + let test = CursorTest::builder() + .source( + "main.by", + " +class Person: + var age: int = 0 + get() = field + set(value): + field = value + +p = Person() +print(p.age) +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:9:9 + | + 9 | print(p.age) + | ^^^ Clicking here + info: Found 1 definition + --> main.by:3:9 + | + 3 | var age: int = 0 + | --- + "); + } + + #[test] + fn goto_definition_property_getter_and_setter_are_both_offered() { + let test = CursorTest::builder() + .source( + "main.py", + " +class C: + @property + def p(self) -> int: + return self._p + @p.setter + def p(self, v: int) -> None: + self._p = v + +c = C() +print(c.p) +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.py:11:9 + | + 11 | print(c.p) + | ^ Clicking here + info: Found 2 definitions + --> main.py:4:9 + | + 4 | def p(self) -> int: + | - + 5 | return self._p + 6 | @p.setter + 7 | def p(self, v: int) -> None: + | - + "); + } + + /// basedpython: an inline protocol's member. The type is structural — two + /// written the same way anywhere are the same type — so it has no + /// declaration of its own, and the annotation the receiver was declared with + /// is the one place an editor can honestly point at + #[test] + fn goto_definition_inline_protocol_member() { + let test = CursorTest::builder() + .source( + "main.by", + " +def f(x: protocol(a: int; def g(self) -> int)) -> int: + return x.a +", + ) + .build(); + + assert_snapshot!(test.goto_definition(), @" + info[goto-definition]: Go to definition + --> main.by:3:14 + | + 3 | return x.a + | ^ Clicking here + info: Found 1 definition + --> main.by:2:19 + | + 2 | def f(x: protocol(a: int; def g(self) -> int)) -> int: + | - + "); + } + #[test] fn goto_definition_binary_operator_from_an_extension() { let test = CursorTest::builder() diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 4a170ccbfa..93fcfdcd5f 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -170,6 +170,30 @@ impl NavigationTarget { } } + /// Creates a `NavigationTarget`, sending one that lands in a basedpython + /// static resource to the head of the document instead. + /// + /// A resource's definitions live in the python its document stands for, + /// which is a file no editor can open, at a position no line of the document + /// is on. Until the rendering carries the document positions it came from, + /// the head of the document is both the most useful place to land and the + /// only one a reader can get to. + fn create( + db: &dyn ruff_db::Db, + file: File, + focus_range: TextRange, + full_range: TextRange, + ) -> Self { + if let Some(document) = ruff_db::resource::resource_document(db, file) { + return Self::new(document, TextRange::default()); + } + Self { + file, + focus_range, + full_range, + } + } + pub fn file(&self) -> File { self.file } @@ -363,11 +387,12 @@ impl HasNavigationTargets for TypeDefinition<'_> { return NavigationTargets::empty(); }; - NavigationTargets::single(NavigationTarget { - file: full_range.file(), - focus_range: self.focus_range(db).unwrap_or(full_range).range(), - full_range: full_range.range(), - }) + NavigationTargets::single(NavigationTarget::create( + db, + full_range.file(), + self.focus_range(db).unwrap_or(full_range).range(), + full_range.range(), + )) } } diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index b94daebcb5..a86ff6e4a5 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -184,6 +184,74 @@ mod tests { } } + /// basedpython: an `extension` method's call sites. `references` found them + /// all along; rename edited only the `def` and reported success, which left + /// every call broken. + #[test] + fn rename_extension_method_edits_its_call_sites() { + let test = CursorTest::builder() + .source( + "main.by", + "\ +extension list: + def second(self) -> int: + return 0 + +xs: list[int] = [1, 2, 3] +print(xs.second()) +print(xs.second()) +", + ) + .build(); + + assert_snapshot!(test.rename("second_item"), @" + info[rename]: Rename symbol (found 3 locations) + --> main.by:2:9 + | + 2 | def second(self) -> int: + | ^^^^^^ + | + ::: main.by:6:10 + | + 6 | print(xs.second()) + | ------ + 7 | print(xs.second()) + | ------ + "); + } + + /// basedpython: a name a `let` pattern binds. The binder had no scope + /// recorded for it, so rename edited the uses and left `let (a, b) :=` + /// naming something that no longer existed. + #[test] + fn rename_let_destructuring_binder_edits_the_binder() { + let test = CursorTest::builder() + .source( + "main.by", + "\ +def f() -> tuple[int, str]: + return (1, \"a\") + +let (a, b) := f() +print(a) +print(a) +", + ) + .build(); + + assert_snapshot!(test.rename("first"), @" + info[rename]: Rename symbol (found 3 locations) + --> main.by:4:6 + | + 4 | let (a, b) := f() + | ^ + 5 | print(a) + | - + 6 | print(a) + | - + "); + } + #[test] fn rename_does_not_mix_global_and_nonlocal_comprehension_walruses() { let test = cursor_test( diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index 98d129b1c0..61598941fe 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -547,6 +547,15 @@ impl<'db> SemanticTokenVisitor<'db> { TextRange::at(range.start(), text.trim_end().text_len()) } + /// The `for` of an `implements ... for ...` declaration, which sits between + /// the last interface and the first pattern and has no node of its own. + fn for_keyword_range(&self, after: TextSize, before: TextSize) -> Option { + let between = self.source.get(usize::from(after)..usize::from(before))?; + let offset = between.find("for")?; + let start = after + TextSize::try_from(offset).ok()?; + Some(TextRange::at(start, TextSize::from(3))) + } + /// True when `name` resolves to no binding, i.e. it is a bare keyword rather /// than a user-defined symbol shadowing one. fn is_unbound(&self, name: &ast::ExprName) -> bool { @@ -1715,6 +1724,27 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { } ast::Stmt::Import(import) => { for alias in &import.names { + if alias.is_resource { + // basedpython: `import "data/config.yaml" as config`. the + // path is a string and is highlighted as one — it names no + // module, and its range covers the quotes, which the + // dotted-name split below does not account for. what it + // binds is a value rather than a namespace + self.add_token( + alias.name.range(), + SemanticTokenType::String, + SemanticTokenModifier::empty(), + ); + if let Some(asname) = &alias.asname { + self.add_token( + asname.range(), + SemanticTokenType::Variable, + SemanticTokenModifier::empty(), + ); + } + continue; + } + // Create separate tokens for each part of a dotted module name self.add_dotted_name_tokens(&alias.name, SemanticTokenType::Namespace); @@ -1894,6 +1924,38 @@ impl SourceOrderVisitor<'_> for SemanticTokenVisitor<'_> { self.visit_body(&try_stmt.finalbody); } ast::Stmt::Expr(expr) => { + // basedpython: an `implements A for ".*"` declaration is stored as + // a call to a synthetic marker, so neither of its keywords has a + // node to highlight from, and the marker itself must not be + // highlighted as the name it is not + if self.is_basedpython + && let Some(declaration) = + ruff_python_ast::helpers::implements_declaration_expression(&expr.value) + { + self.add_token( + declaration.keyword_range, + SemanticTokenType::Keyword, + SemanticTokenModifier::empty(), + ); + for interface in declaration.interfaces { + self.visit_expr(interface); + } + if let (Some(last), Some(first)) = + (declaration.interfaces.last(), declaration.patterns.first()) + && let Some(range) = + self.for_keyword_range(last.range().end(), first.range().start()) + { + self.add_token( + range, + SemanticTokenType::Keyword, + SemanticTokenModifier::empty(), + ); + } + for pattern in declaration.patterns { + self.visit_expr(pattern); + } + return; + } if expecting_docstring && expr.value.is_string_literal_expr() { self.in_docstring = true; } @@ -6063,6 +6125,28 @@ from pathlib import Missing as Alias assert_snapshot!(test.to_snapshot(&tokens), @r#""pathlib" @ 6..13: Namespace"#); } + #[test] + fn semantic_tokens_static_resource_import() { + // the path is a string, not a dotted module name: highlighting it as a + // namespace would both miscolour it and land the tokens a byte early, + // since the name's range covers the quotes and its text does not + let test = SemanticTokenTest::new_by( + " +import \"data/config.yaml\" as config + +print(config) +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "\"data/config.yaml\"" @ 8..26: String + "config" @ 30..36: Variable + "print" @ 38..43: Function + "#); + } + #[test] fn semantic_tokens_trailing_lambda_callee() { // the block's callee rides on a synthetic decorator, but it is written @@ -6332,6 +6416,31 @@ enum class Color: "#); } + #[test] + fn semantic_tokens_build_stamps() { + // `build` highlights through its `build_def` marker like any other + // keyword the parser models as a synthetic decorator. the class name the + // parser synthesizes spans nothing, so it contributes no second token + // over the same text + let test = SemanticTokenTest::new_by( + " +build: + GIT_SHA: str + GIT_DIRTY: bool +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "build" @ 1..6: Keyword + "GIT_SHA" @ 12..19: Variable [definition, readonly] + "str" @ 21..24: Class + "GIT_DIRTY" @ 29..38: Variable [definition, readonly] + "bool" @ 40..44: Class + "#); + } + #[test] fn semantic_tokens_conformance_extension() { // the `extension` keyword highlights through its `extension_def` marker, @@ -6367,6 +6476,40 @@ extension str(Show): "#); } + #[test] + fn semantic_tokens_implements_declaration() { + // `implements` and the `for` of its clause are stored as a call to a + // synthetic marker, so neither has a node to highlight from — and the + // marker itself must not be highlighted as the name it is not + let test = SemanticTokenTest::new_by( + " +protocol Show: + def show(self) -> str + +implements Show + +implements Show for \".*\", \"!.base\" +", + ); + + let tokens = test.highlight_file(); + + assert_snapshot!(test.to_snapshot(&tokens), @r#" + "protocol" @ 1..9: Keyword + "Show" @ 10..14: Class [definition] + "show" @ 24..28: Method [definition] + "self" @ 29..33: SelfParameter [definition] + "str" @ 38..41: Class + "implements" @ 43..53: Keyword + "Show" @ 54..58: Class + "implements" @ 60..70: Keyword + "Show" @ 71..75: Class + "for" @ 76..79: Keyword + "\".*\"" @ 80..84: String + "\"!.base\"" @ 86..94: String + "#); + } + #[test] fn semantic_tokens_dynamic_keyword() { let test = SemanticTokenTest::new_by( diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index 31eafa148e..eb03666a39 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -1373,9 +1373,21 @@ impl<'db> SymbolVisitor<'db> { self.pop_symbol(); } ast::Stmt::ClassDef(class_def) => { + // basedpython: an `extension Widget:` block declares no class of + // its own — its name is a *reference* to the class it extends. Left + // as a bare `Widget` it becomes a second symbol indistinguishable + // from the real class, so an outline lists a class that does not + // exist and "go to symbol in workspace" returns each such class + // twice. Naming it the way it is written keeps its members reachable + // and still finds it when searching for the extended type + let name = if class_def.is_extension() { + format!("extension {}", class_def.name) + } else { + class_def.name.to_string() + }; let symbol = SymbolTree { parent: None, - name: class_def.name.to_string(), + name, kind: SymbolKind::Class, deprecated: Self::has_deprecated_decorator(&class_def.decorator_list), name_range: class_def.name.range(), diff --git a/crates/ty_project/src/db.rs b/crates/ty_project/src/db.rs index ec16d414b2..1c0786c4f0 100644 --- a/crates/ty_project/src/db.rs +++ b/crates/ty_project/src/db.rs @@ -22,7 +22,9 @@ use ty_python_core::ProgramFile; use ty_python_core::program::{FallibleStrategy, MisconfigurationStrategy, UseDefaultStrategy}; use ty_python_semantic::dependencies::DependencyManifest; use ty_python_semantic::lint::{LintRegistry, RuleSelection}; -use ty_python_semantic::{AnalysisSettings, Db as SemanticDb, PythonVersionWithSource}; +use ty_python_semantic::{ + AnalysisSettings, Db as SemanticDb, ExperimentalSettings, PythonVersionWithSource, +}; mod changes; @@ -585,6 +587,10 @@ impl SemanticDb for ProjectDatabase { settings.analysis(self) } + fn experimental_settings(&self) -> &ExperimentalSettings { + self.project().settings(self).experimental() + } + fn verbose(&self) -> bool { self.project().verbose(self) } @@ -704,7 +710,7 @@ pub(crate) mod testing { use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::dependencies::DependencyManifest; use ty_python_semantic::lint::{LintRegistry, RuleSelection}; - use ty_python_semantic::{AnalysisSettings, PythonVersionWithSource}; + use ty_python_semantic::{AnalysisSettings, ExperimentalSettings, PythonVersionWithSource}; use crate::db::Db; use crate::{Project, ProjectMetadata}; @@ -949,6 +955,10 @@ pub(crate) mod testing { self.project().settings(self).analysis() } + fn experimental_settings(&self) -> &ExperimentalSettings { + self.project().settings(self).experimental() + } + fn verbose(&self) -> bool { false } diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index a5e1d23751..bf19fe707d 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -42,9 +42,9 @@ use ty_python_core::platform::PythonPlatform; use ty_python_core::program::{MisconfigurationStrategy, ProgramSettings}; use ty_python_semantic::lint::{Level, LintSource, RuleSelection}; use ty_python_semantic::{ - AnalysisSettings, PythonEnvironment, PythonVersionFileSource, PythonVersionSource, - PythonVersionWithSource, SitePackagesPaths, SysPrefixPathOrigin, TypeCheckingPreset, - inferred_python_version_source_annotation, + AnalysisSettings, ExperimentalSettings, PythonEnvironment, PythonVersionFileSource, + PythonVersionSource, PythonVersionWithSource, SitePackagesPaths, SysPrefixPathOrigin, + TypeCheckingPreset, inferred_python_version_source_annotation, }; use ty_static::EnvVars; @@ -129,6 +129,10 @@ pub struct Options { #[option_group] pub analysis: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[option_group] + pub experimental: Option, + /// Configures how `by run` executes the project. #[serde(skip_serializing_if = "Option::is_none")] #[option_group] @@ -139,6 +143,11 @@ pub struct Options { #[option_group] pub build: Option, + /// Configures how basedpython spells constructs python has no spelling of its own for. + #[serde(skip_serializing_if = "Option::is_none")] + #[option_group] + pub lowering: Option, + /// Configures the parts of the editor experience that type checking does not decide. #[serde(skip_serializing_if = "Option::is_none")] #[option_group] @@ -537,6 +546,8 @@ impl Options { let analysis = strategy.fallback(analysis_result, |_| AnalysisSettings::from_preset(preset))?; + let experimental = self.experimental.or_default().to_settings(); + let overrides = self .to_overrides_settings(db, project_root, preset, &mut diagnostics) .map_err(|err| ToSettingsError { @@ -560,6 +571,7 @@ impl Options { src, build, analysis, + experimental, editor, overrides, }; @@ -1652,6 +1664,72 @@ pub struct RunOptions { pub main: Option>, } +#[derive( + Debug, + Default, + Clone, + Eq, + PartialEq, + Hash, + Combine, + Serialize, + Deserialize, + OptionsMetadata, + get_size2::GetSize, +)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct LoweringOptions { + /// How a float or complex literal type is spelled in the transpiled python. + /// + /// basedpython reads `a: 1.5` as a literal type, and python has no spelling for one: + /// PEP 586 admits only `None`, `int`, `bool`, `str`, `bytes` and enum members into + /// `Literal[...]`. + /// + /// * `nominal` (the default) writes the type the literal is one of — `a: 1.5` becomes + /// `a: float`, `a: 2j` becomes `a: complex`. The precision is lost, and every checker + /// that reads the output accepts it. + /// * `literal` keeps the literal, writing `a: Literal[1.5]`. The precision survives and + /// the output still runs, because `typing` does not check what it is handed — but a + /// checker reading it reports the argument as invalid. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#""nominal""#, + value_type = r#""nominal" | "literal""#, + example = r#" + float-literals = "literal" + "# + )] + pub float_literals: Option>, +} + +/// How a float or complex literal type reaches the transpiled python. +#[derive( + Copy, Clone, Debug, Default, Eq, PartialEq, Hash, Serialize, Deserialize, get_size2::GetSize, +)] +#[serde(rename_all = "kebab-case")] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum FloatLiteralLowering { + /// # Nominal + /// + /// The type the literal is one of: `1.5` becomes `float`, `2j` becomes `complex`. + #[default] + Nominal, + + /// # Literal + /// + /// The literal itself, inside `Literal[...]`. Runs, but no checker accepts it. + Literal, +} + +impl Combine for FloatLiteralLowering { + fn combine_with(&mut self, _other: Self) {} + + fn combine(self, _other: Self) -> Self { + self + } +} + #[derive( Debug, Default, @@ -1879,6 +1957,75 @@ impl FromIterator<(String, String)> for CommonAliases { } } +/// Features that are still being designed, and are off unless the project asks +/// for them by name. +/// +/// An experimental feature may change or be withdrawn without the deprecation +/// period a stable one gets. Opting in says you would rather have it than that +/// guarantee. +#[derive( + Debug, + Default, + Clone, + Eq, + PartialEq, + Hash, + Combine, + Serialize, + Deserialize, + OptionsMetadata, + get_size2::GetSize, +)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct ExperimentalOptions { + /// Whether an `implements` declaration is enforced. + /// + /// `implements Backend` obliges the module that writes it to answer the + /// protocol, and a `for` clause in a package's `__init__` imposes the same + /// obligation on the modules its patterns name. With this off the declaration + /// still parses and still lowers, but nothing is checked against it — and a + /// declaration written anyway is reported, rather than quietly doing nothing. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"false"#, + value_type = "bool", + example = r#" + # hold every module in `backends` to the `Backend` protocol + module-api = true + "# + )] + pub module_api: Option, + + /// Whether a `build:` block declares build stamps. + /// + /// `build:` declares the values a build settles when it produces the artifact + /// — the commit it was built from, the time it was built at — and each is read + /// as `build.NAME` at the type it declares. With this off the block still + /// parses and still lowers, so a program that reads a stamp keeps working, but + /// writing one is reported: nothing settles a stamp the project has not asked + /// for, so it would silently stand for its default, or for nothing. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"false"#, + value_type = "bool", + example = r#" + # let the program read the commit it was built from + build-stamps = true + "# + )] + pub build_stamps: Option, +} + +impl ExperimentalOptions { + pub(super) fn to_settings(&self) -> ExperimentalSettings { + ExperimentalSettings { + module_api: self.module_api.unwrap_or_default(), + build_stamps: self.build_stamps.unwrap_or_default(), + } + } +} + #[derive( Debug, Default, diff --git a/crates/ty_project/src/metadata/settings.rs b/crates/ty_project/src/metadata/settings.rs index dd4e2a6094..d5b06f64a5 100644 --- a/crates/ty_project/src/metadata/settings.rs +++ b/crates/ty_project/src/metadata/settings.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use ruff_db::files::File; use ty_combine::Combine; -use ty_python_semantic::AnalysisSettings; use ty_python_semantic::lint::RuleSelection; +use ty_python_semantic::{AnalysisSettings, ExperimentalSettings}; use crate::metadata::options::{FileOptions, InnerOverrideOptions, Options, OutputFormat}; use crate::metadata::script::script_metadata; @@ -32,6 +32,7 @@ pub struct Settings { pub(super) src: SrcSettings, pub(super) build: BuildSettings, pub(super) analysis: AnalysisSettings, + pub(super) experimental: ExperimentalSettings, pub(super) editor: EditorSettings, /// Settings for configuration overrides that apply to specific file patterns. @@ -71,6 +72,15 @@ impl Settings { &self.analysis } + /// The experimental features the project opted in to. + /// + /// Project-wide, and deliberately not part of [`OverrideSettings`]: an + /// experimental feature is a language feature, and a module's meaning cannot + /// depend on which file is asking about it. + pub fn experimental(&self) -> &ExperimentalSettings { + &self.experimental + } + pub fn editor(&self) -> &EditorSettings { &self.editor } diff --git a/crates/ty_python_core/src/builder.rs b/crates/ty_python_core/src/builder.rs index 7ed3bad8f3..1950fe7414 100644 --- a/crates/ty_python_core/src/builder.rs +++ b/crates/ty_python_core/src/builder.rs @@ -4380,7 +4380,14 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { for (alias_index, alias) in node.names.iter().enumerate() { // Mark the imported module, and all of its parents, as being imported in this // file. - if let Some(module_name) = ModuleName::new(&alias.name) { + // + // basedpython: a static resource names a file rather than a + // module, and a path can read as a module name even when it + // is not one — `"config.json"` has two valid identifiers in + // it and names no module at all + if !alias.is_resource + && let Some(module_name) = ModuleName::new(&alias.name) + { self.imported_modules.extend(module_name.ancestors()); } @@ -6463,6 +6470,19 @@ impl<'ast> Visitor<'ast> for SemanticIndexBuilder<'_, 'ast> { rest: Some(name), .. }) = pattern { + // A capture's own scope has to be recorded rather than left to the + // interval map to infer. That map merges consecutive same-scope + // entries into ranges, so a node with no entry of its own is only + // answered for when some *recorded* expression in the same scope sits + // on either side of it in node order. A `match` subject happens to sit + // before its patterns and so covered them by accident; basedpython's + // `let (a, b) := v` and `if let P := v` write the pattern *before* the + // value, and a `let (m, n): T` parameter binder has no neighbouring + // expression at all, so both fell outside every interval and every + // service that starts by asking which scope a name is in — goto, + // find-references, rename, highlight — answered nothing at the binder + self.scopes_by_expression + .record_expression(name, self.current_scope()); let symbol = self.add_symbol(name.id().clone()); let state = self.current_match_case.as_ref().unwrap(); let (pattern_node, predicate, case_name) = ( diff --git a/crates/ty_python_core/src/re_exports.rs b/crates/ty_python_core/src/re_exports.rs index c6743bd15c..580244c64b 100644 --- a/crates/ty_python_core/src/re_exports.rs +++ b/crates/ty_python_core/src/re_exports.rs @@ -104,6 +104,7 @@ impl<'db> Visitor<'db> for ExportFinder<'db> { asname, range: _, node_index: _, + is_resource: _, } = alias; let name = &name.id; diff --git a/crates/ty_python_semantic/Cargo.toml b/crates/ty_python_semantic/Cargo.toml index ace10a92d8..01986c5971 100644 --- a/crates/ty_python_semantic/Cargo.toml +++ b/crates/ty_python_semantic/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } license = { workspace = true } [dependencies] +by_resource = { workspace = true } ruff_db = { workspace = true } ruff_diagnostics = { workspace = true } ruff_index = { workspace = true, features = ["salsa"] } diff --git a/crates/ty_python_semantic/resources/lint_docs/invalid-static-resource.md b/crates/ty_python_semantic/resources/lint_docs/invalid-static-resource.md new file mode 100644 index 0000000000..557d11bba0 --- /dev/null +++ b/crates/ty_python_semantic/resources/lint_docs/invalid-static-resource.md @@ -0,0 +1,25 @@ +## What it does + +Checks for basedpython static resource imports that cannot be read. + +## Why is this bad? + +`import "data/config.yaml" as config` says the file is part of the program. A +path that names nothing, a path that names a place on one machine, a file in a +format that is not `.json`, `.toml`, `.yaml` or `.yml`, and a document the +format's own parser rejects all leave the import with no value to bind. + +## Examples + +`main.by`: + +```by +# error: [invalid-static-resource] +import "data/config.txt" as config + +# error: [invalid-static-resource] +import "/etc/hosts.json" as hosts + +# error: [invalid-static-resource] +import "data/missing.json" as missing +``` diff --git a/crates/ty_python_semantic/resources/lint_docs/unusable-resource-key.md b/crates/ty_python_semantic/resources/lint_docs/unusable-resource-key.md new file mode 100644 index 0000000000..a59fb53665 --- /dev/null +++ b/crates/ty_python_semantic/resources/lint_docs/unusable-resource-key.md @@ -0,0 +1,31 @@ +## What it does + +Checks for keys in an imported static resource that python cannot name. + +## Why is this bad? + +A static resource is read through attributes, so a key that is not a valid +python identifier — `build-backend`, `class`, `2` — has no attribute to be read +through, and is left out of the value the import binds. The document still holds +it; nothing in the program can reach it. + +Names with two leading underscores are left out for the same reason: python +mangles `__x` inside a class body, so the attribute the reader would write is +not the one that would exist. + +## Examples + +`data/project.json`: + +```json +{ "build-backend": "hatchling.build", "root": "." } +``` + +`main.by`: + +```by +# error: [unusable-resource-key] +import "data/project.json" as project + +reveal_type(project.root) # revealed: "." +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_build_stamps.md b/crates/ty_python_semantic/resources/mdtest/basedpython_build_stamps.md new file mode 100644 index 0000000000..73e9a7e3ed --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_build_stamps.md @@ -0,0 +1,119 @@ +# build stamps + +a `build:` block declares the values the build settles when it produces the artifact — the commit it +came from, whether that commit's tree was clean, when it happened. each one is read as an attribute +of `build`, at the type it was declared with + +(every stamp here carries a default, so that each example stands on its own without a build behind +it. whether a stamp has one changes what happens at build time, never its type.) + +the feature is experimental, so a project asks for it by name: + +```toml +[experimental] +build-stamps = true +``` + +## a stamp has the type it declares + +```by +build: + GIT_SHA: str = "unreleased" + GIT_DIRTY: bool = False + BUILD_NUMBER: int = 0 + +reveal_type(build.GIT_SHA) # revealed: str +reveal_type(build.GIT_DIRTY) # revealed: bool +reveal_type(build.BUILD_NUMBER) # revealed: int +``` + +## a default does not narrow the stamp + +the default stands in when the build supplies nothing, so it says what the value falls back to and +not what it is. a stamp annotated `str` is a `str` whichever of the two it ends up holding — it is +not the `Literal` the default would infer as on its own + +```by +build: + VERSION: str = "0.0.0+dev" + +reveal_type(build.VERSION) # revealed: str +``` + +## a stamp nothing declared is not there + +the block is the whole list of what the program stamps, so reaching for anything else is a mistake +the checker can catch — unlike an environment variable read through a mapping, which can only fail +at runtime + +```by +build: + GIT_SHA: str = "unreleased" + +build.GIT_BRANCH # error: [unresolved-attribute] +``` + +## stamps are ordinary values + +nothing about a stamp is special once it is read: it flows into anything its type fits, and is +rejected by anything it does not + +```by +build: + GIT_SHA: str = "unreleased" + GIT_DIRTY: bool = False + +def describe(sha: str, dirty: bool) -> str: + return f"{sha}{'-dirty' if dirty else ''}" + +describe(build.GIT_SHA, build.GIT_DIRTY) + +# error: [invalid-argument-type] "Argument to function `describe` is incorrect: Expected `str`, found `bool`" +describe(build.GIT_DIRTY, True) +``` + +## `build` is only a keyword in front of a block + +a name is not taken away by a declaration form that does not use it. `build` reads as an ordinary +identifier everywhere the block header is not + +```by +build = 3 +reveal_type(build) # revealed: 3 +``` + +## the feature is off unless the project asks for it + +a block written while the feature is off is reported rather than ignored. the block still lowers, so +a program that reads a stamp keeps working — which is exactly why nothing at the point of use would +say the value was never settled + +```toml +[experimental] +build-stamps = false +``` + +```by +# error: [invalid-build-stamps] "`build` is an experimental feature, and is off" +build: + GIT_SHA: str = "unreleased" + +reveal_type(build.GIT_SHA) # revealed: str +``` + +## a nested block is reported too + +the lowering fills a block in wherever it is written, so one nested inside a class is a stamp the +same way a module-level one is + +```toml +[experimental] +build-stamps = false +``` + +```by +class Program: + # error: [invalid-build-stamps] "`build` is an experimental feature, and is off" + build: + GIT_SHA: str = "unreleased" +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md b/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md index 3e972035d9..14245b340b 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_literal_annotations.md @@ -4,8 +4,9 @@ basedpython diverges from PEP 484 stringified-forward-reference and PEP 586 lite - a string in annotation/type position is `Literal[]`, not a forward reference. forward refs are unnecessary because basedpython annotations are always deferred -- float and complex literals are accepted in type position; they currently lower to the nominal - `float` / `complex` instance (no exact-literal narrowing yet) +- float and complex literals are accepted in type position. python's `Literal[...]` has no spelling + for one, so what they lower to is the `lowering.float-literals` option's to say: the nominal + `float` / `complex` instance by default - `A[T=int]` is a keyword type-arg binding, equivalent to `A[int]` for single-typevar generics ```toml @@ -29,6 +30,36 @@ x: Literal["a", "b"] = "a" reveal_type(x) # revealed: "a" ``` +## a union of string literals is a union of literal types + +`"foo" | "bar"` is two literal types joined, not a `str.__or__` that would fail at runtime: the +transpiler lowers the whole type expression to `Literal["foo", "bar"]` before python ever sees it. +that holds in an annotation and on the right-hand side of a type alias, whose value is the same +lowered expression. + +```by +type Name = "foo" | "bar" + +def f(a: Name, b: "spam" | "eggs") -> None: + reveal_type(a) # revealed: "foo" | "bar" + reveal_type(b) # revealed: "spam" | "eggs" + +# error: [invalid-assignment] +c: Name = "baz" +``` + +## a float literal in a union is a union arm like any other + +left bare, `int | 3.5` would be a `TypeError` the moment python evaluated the annotation. the +lowering is what keeps it running, and the checker reads the arm as the literal type it was written +as either way. + +```by +def f(a: int | 3.5, b: int | 2j) -> None: + reveal_type(a) # revealed: int | 3.5 + reveal_type(b) # revealed: int | 2j +``` + ## float literal in annotation is the literal type ```by diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_module_api.md b/crates/ty_python_semantic/resources/mdtest/basedpython_module_api.md new file mode 100644 index 0000000000..37947dc1b1 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_module_api.md @@ -0,0 +1,688 @@ +# basedpython: module api enforcement + +a module is a structural value, so it can answer a protocol through its public surface. an +`implements` declaration attaches that obligation to the module permanently, so a break is reported +in the module that broke rather than wherever something happens to assign it — or nowhere at all, +when nothing does. + +the feature is experimental, so a project asks for it by name: + +```toml +[experimental] +module-api = true +``` + +## a module answers the interface it declares + +the interface spells its members `static`, because a module's members are unbound. + +`api.by`: + +```by +protocol Backend: + name: str + static def connect(url: str) -> str +``` + +`postgres.by`: + +```by +from api import Backend + +implements Backend + +name: str = "postgres" + +def connect(url: str) -> str: + return url +``` + +## a class object stands in for a module + +a static-membered protocol is answered by a class object just as it is by a module, so a test can +substitute a fake without a module of its own. + +```by +protocol Backend: + name: str + static def connect(url: str) -> str + +class FakeBackend: + name: str = "fake" + static def connect(url: str) -> str: + return url + +def run(backend: Backend) -> None: ... + +run(FakeBackend) +``` + +## a member the module does not have + +the declaration is what the diagnostic points at, because that is what the module promised. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`postgres.by`: + +```by +from api import Backend + +# error: [unmet-module-api] "does not answer `Backend`" +implements Backend +``` + +## a member whose shape is wrong + + + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`postgres.by`: + +```by +from api import Backend + +# error: [unmet-module-api] "`postgres` does not answer `Backend`" +implements Backend + +def connect(url: int) -> str: + return str(url) +``` + +## a package imposes an interface on its submodules + +a `for` clause in a package's `__init__` obliges the modules its patterns name. the patterns are +relative to that package, spelled with a leading `.` as a relative import is. + + + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".*" +``` + +`backends/good.by`: + +```by +def connect(url: str) -> str: + return url +``` + +`backends/bad.by`: + +```by +x = 1 # error: [unmet-module-api] "`backends.bad` does not answer `Backend`" +``` + +## a package that imports its own submodules + +the shape a plugin package actually has: the `__init__` re-exports from the very modules its rule +obliges. + +`plug/api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`plug/__init__.by`: + +```by +from .api import Backend +from .good import connect + +implements Backend for ".*", "!.api" + +__all__ = ["connect"] +``` + +`plug/good.by`: + +```by +def connect(url: str) -> str: + return url +``` + +`plug/bad.by`: + +```by +x = 1 # error: [unmet-module-api] "`plug.bad` does not answer `Backend`" +``` + +## a rule does not oblige the package that wrote it + +patterns name what is *inside* the package, so `__init__` itself is never one of them. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".**" +``` + +`backends/good.by`: + +```by +def connect(url: str) -> str: + return url +``` + +## a pattern may carve a module back out + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".*", "!.registry" +``` + +`backends/good.by`: + +```by +def connect(url: str) -> str: + return url +``` + +`backends/registry.by`: + +```by +names: list[str] = [] +``` + +## a private submodule is reached only by name + +a leading underscore already means "not part of the surface", so `.*` passes a private helper by. +naming one exactly still reaches it. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".*", "._named" +``` + +`backends/_helper.by`: + +```by +def helper() -> int: + return 1 +``` + +`backends/_named.by`: + +```by +y = 2 # error: [unmet-module-api] "`backends._named` does not answer `Backend`" +``` + +## an interface has to be a protocol + +a module answers an interface through its public surface, which is what a protocol describes. a +concrete class promises state a module has no way to carry. + +```by +class Backend: + def connect(self) -> None: ... + +# error: [invalid-module-api] "`Backend` is not a protocol" +implements Backend +``` + +## a rule outside a package's `__init__` + +a module finds the rules imposed on it by walking the packages it is in, so a rule written anywhere +else would reach nothing. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`loose.by`: + +```by +from api import Backend + +# error: [invalid-module-api] "may only be written in a package's `__init__`" +implements Backend for ".*" +``` + +## a pattern that is not relative to its package + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +# error: [invalid-module-api] "is not a pattern relative to `backends`" +implements Backend for "backends.*" +``` + +## a rule that reaches nothing + +a pattern that matches no module enforces nothing, silently, which is worse than no rule at all. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +# error: [invalid-module-api] "reaches no module" +implements Backend for ".nothing_*" +``` + +`backends/postgres.by`: + +```by +def connect(url: str) -> str: + return url +``` + +## a declaration inside a body + +an obligation is about a module's surface, so there is nothing for one written inside a body to +attach to. + +```by +protocol Backend: + static def connect(url: str) -> str + +def f() -> None: + # error: [invalid-module-api] "belongs at module level" + implements Backend +``` + +## a module whose `__getattr__` answers everything + +every requirement would be met without anything being defined, so the obligation cannot be checked +at all. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`dynamic_module.by`: + +```by +from typing import Any + +from api import Backend + +# error: [invalid-module-api] "cannot be checked against `Backend`" +implements Backend + +def __getattr__(name: str) -> Any: ... +``` + +## a stub is the module's api + +when a module has a stub, the stub is what everything outside it reads, so the stub is what the +obligation is about. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`postgres.byi`: + +```byi +from api import Backend + +implements Backend + +def connect(url: str) -> str: ... +``` + +`postgres.by`: + +```by +def connect(url: str) -> str: + return url +``` + +## a declaration in a file its stub shadows + +nothing outside the module reads that file's surface, so a declaration there would be about a +surface nobody sees — and would leave the stub, which *is* the api, unchecked. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`postgres.byi`: + +```byi +def connect(url: str) -> str: ... +``` + +`postgres.by`: + +```by +from api import Backend + +# error: [invalid-module-api] "this module's api is its stub" +implements Backend + +def connect(url: str) -> str: + return url +``` + +## one interface, however many declarations ask for it + +a module inside a package that imposes an interface, which also declares it itself, has one thing +left to do about it — so it is told once. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".*" +implements Backend for ".**" +``` + +`backends/bad.by`: + +```by +from api import Backend + +# error: [unmet-module-api] "`backends.bad` does not answer `Backend`" +implements Backend +``` + +## a rule may name several interfaces + +each is checked on its own, so answering one says nothing about the other. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str + +protocol Named: + name: str +``` + +`backends/__init__.by`: + +```by +import api + +implements api.Backend, api.Named for ".*" +``` + +`backends/bad.by`: + +```by +name: str = "bad" # error: [unmet-module-api] "`backends.bad` does not answer `Backend`" +``` + +## a pattern that cannot be resolved does not disable the rest of the rule + +the malformed pattern is reported; the ones beside it go on meaning what they say. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +# error: [invalid-module-api] "is not a pattern relative to `backends`" +implements Backend for ".*", "backends.other" +``` + +`backends/bad.by`: + +```by +x = 1 # error: [unmet-module-api] "`backends.bad` does not answer `Backend`" +``` + +## a wildcard does not reach inside a private package + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".**" +``` + +`backends/good.by`: + +```by +def connect(url: str) -> str: + return url +``` + +`backends/_private/__init__.by`: + +```by +x = 1 +``` + +`backends/_private/helper.by`: + +```by +y = 2 +``` + +## a python submodule is obliged like any other + +a rule names modules, not files, so a `.py` module in the package answers for itself. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".*" +``` + +`backends/legacy.py`: + +```py +x = 1 # error: [unmet-module-api] "`backends.legacy` does not answer `Backend`" +``` + +## an interface and the module obliged to answer it may import each other + +resolving the interface a rule names infers the declaring module's code, which is how a package that +re-exports from the modules it obliges reaches back into them. + +`plug/api.by`: + +```by +from plug.good import connect as connect + +protocol Backend: + static def connect(url: str) -> str +``` + +`plug/__init__.by`: + +```by +from .api import Backend + +implements Backend for ".*", "!.api" +``` + +`plug/good.by`: + +```by +def connect(url: str) -> str: + return url +``` + +## a subscripted interface + +a specialization would be dropped silently, so it is a syntax error rather than an obligation the +author did not write. + +```by +protocol Backend[T]: + static def connect(url: T) -> str + +# error: [invalid-syntax] "an `implements` declaration takes interface names" +# error: [unmet-module-api] "does not answer `Backend[Unknown]`" +implements Backend[int] +``` + +## a comma with no interface after it + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +# error: [invalid-syntax] "takes an interface name after `,`" +implements Backend, for ".*" +``` + +## a namespace package between the rule and the module + +a directory with no `__init__` carries no rules of its own, and the walk continues past it to the +package that does. + +`api.by`: + +```by +protocol Backend: + static def connect(url: str) -> str +``` + +`backends/__init__.by`: + +```by +from api import Backend + +implements Backend for ".**" +``` + +`backends/space/deep.by`: + +```by +x = 1 # error: [unmet-module-api] "`backends.space.deep` does not answer `Backend`" +``` + +## the feature is off unless the project asks for it + +a declaration written while the feature is off is reported rather than ignored — an obligation +nothing checks is the failure this feature exists to remove. + +```toml +[experimental] +module-api = false +``` + +```by +protocol Backend: + static def connect(url: str) -> str + +# error: [invalid-module-api] "`implements` is an experimental feature, and is off" +implements Backend +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_static_resources.md b/crates/ty_python_semantic/resources/mdtest/basedpython_static_resources.md new file mode 100644 index 0000000000..f175eccb7c --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_static_resources.md @@ -0,0 +1,389 @@ +# Static resources + +A json, toml or yaml file can be imported by path, which binds its document to a name. + +## A mapping is read through its keys + +The document's mapping keys are the value's attributes, all the way down. A sequence is a tuple, so +an index reaches one element rather than the union of all of them. + +`data/config.yaml`: + +```yaml +a: + b: + - 1 + - 2 +``` + +`main.by`: + +```by +import "data/config.yaml" as config + +reveal_type(config.a.b[1]) # revealed: 2 +reveal_type(config.a.b) # revealed: (1, 2) +``` + +## Scalars keep the value they were written with + +`data/settings.yaml`: + +```yaml +name: ty +port: 8080 +ratio: 0.5 +debug: true +missing: ~ +``` + +`main.by`: + +```by +import "data/settings.yaml" as settings + +reveal_type(settings.name) # revealed: "ty" +reveal_type(settings.port) # revealed: 8080 +reveal_type(settings.ratio) # revealed: float +reveal_type(settings.debug) # revealed: True +reveal_type(settings.missing) # revealed: None +``` + +## An index past the end of a sequence is an error + +`data/ports.json`: + +```json +{ "ports": [80, 443] } +``` + +`main.by`: + +```by +import "data/ports.json" as config + +reveal_type(config.ports[1]) # revealed: 443 +# error: [index-out-of-bounds] +reveal_type(config.ports[2]) # revealed: Unknown +``` + +## A key the document does not have is an error + +`data/config.json`: + +```json +{ "a": 1 } +``` + +`main.by`: + +```by +import "data/config.json" as config + +# error: [unresolved-attribute] +reveal_type(config.b) # revealed: Unknown +``` + +## A value cannot be assigned to + +The document is fixed at build time, so its attributes are `Final`. + +`data/config.json`: + +```json +{ "a": 1 } +``` + +`main.by`: + +```by +import "data/config.json" as config + +config.a = 2 # snapshot +``` + +```snapshot +error[invalid-assignment]: Cannot assign to final attribute `a` on type `` + --> src/main.by:3:1 + | +3 | config.a = 2 # snapshot + | ^^^^^^^^ `Final` attributes can only be assigned in the class body or `__init__` +``` + +## toml + +`data/config.toml`: + +```toml +[server] +host = "localhost" +ports = [80, 443] +``` + +`main.by`: + +```by +import "data/config.toml" as config + +reveal_type(config.server.host) # revealed: "localhost" +reveal_type(config.server.ports[0]) # revealed: 80 +``` + +## A mapping inside a sequence + +`data/servers.json`: + +```json +{ "servers": [{ "host": "a" }, { "host": "b" }] } +``` + +`main.by`: + +```by +import "data/servers.json" as config + +reveal_type(config.servers[1].host) # revealed: "b" +``` + +## A document that is not a mapping + +The top of a document does not have to be a mapping; the name is then bound to whatever is there. + +`data/ports.json`: + +```json +[80, 443] +``` + +`main.by`: + +```by +import "data/ports.json" as ports + +reveal_type(ports[0]) # revealed: 80 +``` + +## The path is relative to the importing file + +`pkg/data/config.json`: + +```json +{ "a": 1 } +``` + +`pkg/inner/main.by`: + +```by +import "../data/config.json" as config + +reveal_type(config.a) # revealed: 1 +``` + +## A path that names nothing + +`main.by`: + +```by +# error: [invalid-static-resource] "Cannot read static resource `data/missing.json`" +import "data/missing.json" as config + +reveal_type(config) # revealed: Unknown +``` + +## A file that is not a resource format + +`data/notes.txt`: + +```text +hello +``` + +`main.by`: + +```by +# error: [invalid-static-resource] "Cannot read static resource `data/notes.txt`" +import "data/notes.txt" as notes +``` + +## An absolute path + +A resource is part of the program, so it is named the way the program is laid out rather than the +way one machine is. + +`main.by`: + +```by +# error: [invalid-static-resource] "Cannot read static resource `/etc/config.json`" +import "/etc/config.json" as config +``` + +## A document that cannot be read + +`data/broken.json`: + +```json +{ "a": } +``` + +`main.by`: + +```by +# error: [invalid-static-resource] "Cannot read static resource `data/broken.json`" +import "data/broken.json" as config +``` + +## A key python cannot name + +Such a key is left out of the value: there is no attribute it could be read through. + +`data/config.json`: + +```json +{ "build-backend": "hatchling.build", "root": "." } +``` + +`main.by`: + +```by +# error: [unusable-resource-key] "Key `build-backend` of `data/config.json` cannot be read as an attribute" +import "data/config.json" as project + +reveal_type(project.root) # revealed: "." +``` + +## A resource is imported by a statement of its own + +A module import is a name the runtime goes and resolves; a resource is a document written into the +program. The two do not share a statement. + +`data/config.json`: + +```json +{ "a": 1 } +``` + +`main.by`: + +```by +# error: [invalid-syntax] "A static resource is imported by a statement of its own" +import os, "data/config.json" as config +``` + +## A resource cannot be lazy + +`data/config.json`: + +```json +{ "a": 1 } +``` + +`main.by`: + +```by +# error: [invalid-syntax] "A static resource is read while the program is built, so it cannot be lazy" +lazy import "data/config.json" as config +``` + +## A resource has to say what it binds + +A path is not a name, so there is nothing for the import to fall back on. + +`data/config.json`: + +```json +{ "a": 1 } +``` + +`main.by`: + +```by +# error: [invalid-syntax] "Expected `as` and a name to bind the static resource to" +import "data/config.json" +``` + +## Names the rendering needs for itself are left out too + +`Final` is what the values are annotated with, and `_by_…` is what the classes a document needs for +itself are called. A key of either name would be what a sibling resolves to, so neither is exposed. +Nor is a name python would mangle inside a class body. + +`data/reserved.json`: + +```json +{ "Final": 1, "_by_reserved_0": 2, "__x": 3, "a": { "b-c": 4 }, "ok": 5 } +``` + +`main.by`: + +```by +# error: [unusable-resource-key] "4 keys of `data/reserved.json` cannot be read as attributes" +import "data/reserved.json" as reserved + +reveal_type(reserved.ok) # revealed: 5 +``` + +## A key written twice reads as its last value + +`data/twice.json`: + +```json +{ "a": 1, "b": 2, "a": 3 } +``` + +`main.by`: + +```by +import "data/twice.json" as twice + +reveal_type(twice.a) # revealed: 3 +reveal_type(twice.b) # revealed: 2 +``` + +## A string keeps what it holds + +`data/text.json`: + +```json +{ "quoted": "he said \"hi\"", "lines": "a\nb", "unicode": "héllo" } +``` + +`main.by`: + +```by +import "data/text.json" as text + +reveal_type(text.quoted) # revealed: "he said \"hi\"" +reveal_type(text.lines) # revealed: "a\nb" +reveal_type(text.unicode) # revealed: "héllo" +``` + +## A document too deep to read + +Counted as the document is read, so a file that nests past any stack is a diagnostic rather than a +crash. + +`data/deep.yaml`: + +```yaml +a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: + { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: { a: + { a: 1 } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } } +``` + +`main.by`: + +```by +# error: [invalid-static-resource] "Cannot read static resource `data/deep.yaml`" +import "data/deep.yaml" as deep +``` + +## Only basedpython has static resources + +`main.py`: + +```py +# error: [invalid-syntax] "Expected one or more symbol names after import" +# error: [invalid-syntax] "Expected a statement" +# error: [unresolved-reference] "Name `config` used when not defined" +import "data/config.json" as config +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_template_literal_types.md b/crates/ty_python_semantic/resources/mdtest/basedpython_template_literal_types.md index 78912288ef..abae1863fb 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_template_literal_types.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_template_literal_types.md @@ -59,6 +59,64 @@ def f(a: f"{str}", b: f"{Character}") -> None: reveal_type(b) # revealed: Character ``` +## a hole spelled as a type alias is the type it stands for + +a hole is read for what it means rather than for how it was written, so an alias distributes the +same way the type it names would. + +```by +type Name = "foo" | "bar" + +def f(a: f"the {Name}") -> None: + reveal_type(a) # revealed: "the foo" | "the bar" + +# error: [invalid-assignment] "Object of type `"the asdf"` is not assignable to `"the foo" | "the bar"`" +b: f"the {Name}" = "the asdf" +``` + +## an alias is followed as far as it goes + +a union written in a type expression keeps the names its arms were written with, so the type a hole +stands for can be an alias of an alias. every step is followed, and a pattern reached that way is +spliced in just as one written directly would be. + +```by +type Inner = "foo" | "bar" +type Outer = Inner | "baz" + +type Version = f"v{int}" +type Tagged = Version | "untagged" + +def f(a: f"the {Outer}", b: f"[{Tagged}]") -> None: + reveal_type(a) # revealed: "the foo" | "the bar" | "the baz" + reveal_type(b) # revealed: f"[v{int}]" | "[untagged]" +``` + +## an alias that names itself is followed only once + +```by +type Loop = Loop | "q" + +type Left = Right | int +type Right = Left | str + +def f(a: f"a{Loop}b", b: f"a{Right}b") -> None: + reveal_type(a) # revealed: "aqb" + reveal_type(b) # revealed: f"a{str}b" +``` + +## an alias of a type that is not a union is still that type + +resolving a hole is not a distribution — an alias naming one type leaves a pattern that is still a +pattern, spelled with the type rather than with the alias. + +```by +type Text = str + +def f(a: f"a{Text}b") -> None: + reveal_type(a) # revealed: f"a{str}b" +``` + ## a `None` hole renders as `str(None)` does ```by diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_module_a\342\200\246_-_basedpython___module_\342\200\246_-_a_member_whose_shape\342\200\246_(f51e9723fe7a9fe9).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_module_a\342\200\246_-_basedpython___module_\342\200\246_-_a_member_whose_shape\342\200\246_(f51e9723fe7a9fe9).snap" new file mode 100644 index 0000000000..06fb206fe9 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_module_a\342\200\246_-_basedpython___module_\342\200\246_-_a_member_whose_shape\342\200\246_(f51e9723fe7a9fe9).snap" @@ -0,0 +1,42 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_module_api.md - basedpython: module api enforcement - a member whose shape is wrong +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_module_api.md +--- + +# Source files + +## api.by + +``` +1 | protocol Backend: +2 | static def connect(url: str) -> str +``` + +## postgres.by + +``` +1 | from api import Backend +2 | +3 | # error: [unmet-module-api] "`postgres` does not answer `Backend`" +4 | implements Backend +5 | +6 | def connect(url: int) -> str: +7 | return str(url) +``` + +# Diagnostics + +``` +error[unmet-module-api]: `postgres` does not answer `Backend` + --> src/postgres.by:4:1 + | +4 | implements Backend + | ^^^^^^^^^^ +info: `connect` is `def connect(url: int) -> str`, but `Backend` declares it as `def connect(url: str) -> str` + +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_module_a\342\200\246_-_basedpython___module_\342\200\246_-_a_package_imposes_an\342\200\246_(79ae512d5793d1b1).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_module_a\342\200\246_-_basedpython___module_\342\200\246_-_a_package_imposes_an\342\200\246_(79ae512d5793d1b1).snap" new file mode 100644 index 0000000000..d9848329c9 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_module_a\342\200\246_-_basedpython___module_\342\200\246_-_a_package_imposes_an\342\200\246_(79ae512d5793d1b1).snap" @@ -0,0 +1,56 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_module_api.md - basedpython: module api enforcement - a package imposes an interface on its submodules +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_module_api.md +--- + +# Source files + +## api.by + +``` +1 | protocol Backend: +2 | static def connect(url: str) -> str +``` + +## backends/__init__.by + +``` +1 | from api import Backend +2 | +3 | implements Backend for ".*" +``` + +## backends/good.by + +``` +1 | def connect(url: str) -> str: +2 | return url +``` + +## backends/bad.by + +``` +1 | x = 1 # error: [unmet-module-api] "`backends.bad` does not answer `Backend`" +``` + +# Diagnostics + +``` +error[unmet-module-api]: `backends.bad` does not answer `Backend` + --> src/backends/bad.by:1:1 + | +1 | x = 1 # error: [unmet-module-api] "`backends.bad` does not answer `Backend`" + | ^ + | + ::: src/backends/__init__.by:3:1 + | +3 | implements Backend for ".*" + | ---------- required by this declaration +info: `connect` is missing + +``` diff --git a/crates/ty_python_semantic/src/db.rs b/crates/ty_python_semantic/src/db.rs index 27505f92c4..965347aa52 100644 --- a/crates/ty_python_semantic/src/db.rs +++ b/crates/ty_python_semantic/src/db.rs @@ -1,6 +1,6 @@ use crate::dependencies::DependencyManifest; use crate::lint::{LintRegistry, RuleSelection}; -use crate::{AnalysisSettings, PythonVersionWithSource}; +use crate::{AnalysisSettings, ExperimentalSettings, PythonVersionWithSource}; use ruff_db::diagnostic::Diagnostic; use ruff_db::files::File; use ty_python_core::{Db as PythonCoreDb, ProgramFile}; @@ -23,6 +23,12 @@ pub trait Db: PythonCoreDb { fn analysis_settings(&self, file: File) -> &AnalysisSettings; + /// The experimental features the project has opted in to. + /// + /// Project-wide rather than per-file: an experimental feature is a language + /// feature, and a module's meaning cannot depend on which file is asking. + fn experimental_settings(&self) -> &ExperimentalSettings; + /// Whether ty is running with logging verbosity INFO or higher (`-v` or more). fn verbose(&self) -> bool; @@ -119,6 +125,7 @@ pub(crate) mod tests { events: Events, rule_selection: Arc, analysis_settings: Arc, + experimental_settings: ExperimentalSettings, open_files: rustc_hash::FxHashSet, program_settings: ProgramSettings, } @@ -146,6 +153,12 @@ pub(crate) mod tests { TypeCheckingPreset::default(), )), analysis_settings: AnalysisSettings::default().into(), + // the in-crate test db is used for unit tests of the type system + // itself, where an experimental feature is what is under test + experimental_settings: ExperimentalSettings { + module_api: true, + build_stamps: true, + }, open_files: rustc_hash::FxHashSet::default(), program_settings, } @@ -269,6 +282,10 @@ pub(crate) mod tests { &self.analysis_settings } + fn experimental_settings(&self) -> &ExperimentalSettings { + &self.experimental_settings + } + fn verbose(&self) -> bool { false } diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index bcf1325b42..9b02a3e1a5 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -57,6 +57,7 @@ pub use types::extensions::{ExtensionAttributeInfo, ExtensionMemberKind}; pub use types::ide_support::{ ImplementationsFinder, ImportAliasResolution, OverridableMember, ResolvedDefinition, TypeHierarchyClass, contains_identifier, definitions_for_attribute, definitions_for_bin_op, + definitions_for_case_name, definitions_for_context_sensitive_name, definitions_for_django_lookup_root, definitions_for_imported_symbol, definitions_for_name, definitions_for_unary_op, map_stub_definition, type_hierarchy_prepare, type_hierarchy_subtypes, type_hierarchy_supertypes, @@ -65,6 +66,7 @@ pub use types::implicit_names::implicit_names; pub use types::reified_infer::{ ArgVariance, ErasedTargetReason, ErasedUnion, ParametricIsPlan, ProtocolMemberCheck, }; +pub use types::static_resource::{ResourceError, render_as, resolve_static_resource}; pub use types::template::finite_string_set; pub use types::visibility::private_symbols; pub use types::{DisplaySettings, ProgramEnvironment, TypeQualifiers}; @@ -120,6 +122,33 @@ fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&BLANKET_IGNORE_COMMENT); } +/// The experimental features a project has opted in to, from `[tool.ty.experimental]`. +/// +/// An experimental feature is off unless the project asks for it by name. It is +/// deliberately not part of [`AnalysisSettings`]: an analysis setting says how to +/// read code that is already understood, while this says whether a language +/// feature is on at all, and switching one on for some files and off for others +/// would make a module's meaning depend on which file asked. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, get_size2::GetSize)] +pub struct ExperimentalSettings { + /// Whether `implements` declarations are enforced. + /// + /// The declaration parses and lowers either way — it has to, or emitted python + /// would carry a name that does not exist at runtime — but nothing is checked + /// against it until the project opts in, and a declaration written with the + /// feature off says so rather than being quietly ignored. + pub module_api: bool, + + /// Whether a `build:` block declares build stamps. + /// + /// Like [`module_api`](Self::module_api), the block parses and lowers either + /// way — a program that reads `build.GIT_SHA` must keep working when the + /// project turns the feature off — but a block written while it is off is + /// reported, because a stamp nobody settles is a value that quietly stands + /// for nothing. + pub build_stamps: bool, +} + #[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] #[expect( clippy::struct_excessive_bools, diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 45283f2fb9..2c1100ff4e 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -1266,10 +1266,15 @@ impl<'db> SemanticModel<'db> { /// Returns completions for symbols available in a `object.` context. pub fn attribute_completions(&self, node: &ast::ExprAttribute) -> Vec> { - let db = self.db; let Some(ty) = node.value.inferred_type(self) else { return Vec::new(); }; + self.member_completions(ty) + } + + /// Returns completions for the members of `ty`. + fn member_completions(&self, ty: Type<'db>) -> Vec> { + let db = self.db; let private = self.foreign_private_symbols(ty); all_members(db, &self.program_environment(), ty) @@ -1284,6 +1289,66 @@ impl<'db> SemanticModel<'db> { .collect() } + /// Returns the type of what `path` reaches from `module`. + /// + /// This answers the question an unimported `Asdf.` asks: the file + /// never bound `Asdf`, so there is no expression here whose type could be + /// inferred, and the type has to come from the module an import would + /// reach instead. `path` names each step from that module, so `["Asdf"]` + /// is the `Asdf` a `from mod import Asdf` would bind, and an empty `path` + /// is the module itself. + pub fn path_type(&self, module: Module<'db>, path: &[&str]) -> Option> { + let env = self.program_environment(); + let mut ty = Type::module_literal(self.db, self.program_file(), module); + for name in path { + if self + .foreign_private_symbols(ty) + .is_some_and(|names| names.contains(*name)) + { + return None; + } + ty = ty + .member(self.db, &env, name) + .ignore_possibly_undefined() + .or_else(|| self.submodule_type(ty, name))?; + } + Some(ty) + } + + /// The type of `module.name`, when `module` is a module that has `name` as + /// a submodule. + /// + /// A submodule is not a member of the package until something imports it, + /// so this is the step `Type::member` cannot take on its own. Completing + /// `pkg.sub.` in a file that imported neither still has to reach `sub`. + fn submodule_type(&self, module: Type<'db>, name: &str) -> Option> { + let Type::ModuleLiteral(module) = module else { + return None; + }; + let mut submodule_name = module.module(self.db).name(self.db).clone(); + submodule_name.extend(&ModuleName::new(name)?); + let submodule = self.resolve_module_name(&submodule_name)?; + Some(Type::module_literal( + self.db, + self.program_file(), + submodule, + )) + } + + /// Returns completions for what can be written after a `ty.` in the source. + /// + /// A module's submodules are among them, which is the difference between + /// this and [`SemanticModel::attribute_completions`]: that one describes an + /// expression the file already wrote, and a file that reaches a module has + /// already imported whichever submodules it can see. + pub fn type_completions(&self, ty: Type<'db>) -> Vec> { + let mut completions = self.member_completions(ty); + if let Type::ModuleLiteral(module) = ty { + completions.extend(self.submodule_completions(&module.module(self.db))); + } + completions + } + /// Returns completions for symbols available in the scope containing the /// given expression. /// diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 2b28809522..8642c742c1 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -159,6 +159,7 @@ use ty_python_core::{ProgramFile, Truthiness, place_table, semantic_index, use_d mod attribute_write; mod bool; mod bound_super; +mod build_stamps; mod call; mod callable; pub mod character; @@ -198,6 +199,7 @@ mod match_pattern; pub(crate) mod match_type; mod member; mod method; +mod module_api; mod mro; pub(crate) mod name_fallback; pub(crate) mod narrow; @@ -217,6 +219,7 @@ mod set_theoretic; mod signatures; pub mod soundness; mod special_form; +pub mod static_resource; mod string_annotation; mod subclass_of; pub(crate) mod template; diff --git a/crates/ty_python_semantic/src/types/build_stamps.rs b/crates/ty_python_semantic/src/types/build_stamps.rs new file mode 100644 index 0000000000..ac5dc707db --- /dev/null +++ b/crates/ty_python_semantic/src/types/build_stamps.rs @@ -0,0 +1,77 @@ +//! basedpython: the experimental gate on a `build:` block. +//! +//! Everything a build stamp *does* happens in the transpiler, which fills each +//! declared stamp in from the values the build settled, so there is nothing here +//! for ty to check about a block the project has asked for. What is left is the +//! opposite case: a block written while the feature is off. +//! +//! It is reported rather than ignored, because ignoring it is the one outcome the +//! feature cannot afford. The block parses and lowers either way — a program that +//! reads `build.GIT_SHA` has to keep working when the project turns the feature +//! off — so nothing at the point of use says the value is unsettled. A stamp +//! declared without a default would then be a hard transpile error, and one with +//! a default would quietly stand for the default in an artifact that claims to +//! know what commit it came from. + +use ruff_python_ast as ast; +use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_text_size::TextRange; + +use crate::types::context::InferContext; +use crate::types::diagnostic::INVALID_BUILD_STAMPS; + +/// Reports every `build:` block in a project that has not opted in to the +/// feature. +pub(super) fn check_build_stamps<'ast>(context: &InferContext<'_, 'ast>, body: &'ast [ast::Stmt]) { + let db = context.db(); + if db.experimental_settings().build_stamps { + return; + } + // `build:` is basedpython-only syntax, so a python file cannot hold one + if !context + .program_file() + .file(db) + .source_type(db) + .is_basedpython() + { + return; + } + + let mut blocks = BuildBlocks::default(); + for statement in body { + blocks.visit_stmt(statement); + } + + for keyword in blocks.keywords { + let Some(builder) = context.report_lint(&INVALID_BUILD_STAMPS, keyword) else { + continue; + }; + let mut diagnostic = + builder.into_diagnostic("`build` is an experimental feature, and is off"); + diagnostic.info("nothing settles these stamps until the project opts in"); + diagnostic.help( + "Enable it with `build-stamps = true` under `[experimental]` in `basedpython.toml`", + ); + } +} + +/// The `build` keyword of every block in the file. +/// +/// A whole-tree walk rather than a scan of the module body, because the lowering +/// fills in a block wherever it is written — including one nested in a class or a +/// function, which is a stamp the same way a module-level one is. +#[derive(Default)] +struct BuildBlocks { + keywords: Vec, +} + +impl<'ast> Visitor<'ast> for BuildBlocks { + fn visit_stmt(&mut self, statement: &'ast ast::Stmt) { + if let ast::Stmt::ClassDef(class) = statement + && let Some(keyword) = class.build_stamps_range() + { + self.keywords.push(keyword); + } + walk_stmt(self, statement); + } +} diff --git a/crates/ty_python_semantic/src/types/context_sensitive.rs b/crates/ty_python_semantic/src/types/context_sensitive.rs index 91c02d8e7b..c462f7feda 100644 --- a/crates/ty_python_semantic/src/types/context_sensitive.rs +++ b/crates/ty_python_semantic/src/types/context_sensitive.rs @@ -59,6 +59,12 @@ impl<'db> ContextSensitiveMember<'db> { pub(crate) fn qualifier(self, db: &'db dyn Db) -> &'db Name { self.enum_class.name(db) } + + /// the enum the name was found on, for a caller that needs the declaration + /// rather than the spelling — goto-definition on a bare member + pub(crate) fn enum_class(self) -> ClassLiteral<'db> { + self.enum_class + } } /// what the expected type has to say about `name` before the resolution rules @@ -340,6 +346,22 @@ pub(crate) fn qualifier_for_unbound_name<'db>( name: &str, resolved_type: impl FnOnce() -> Option>, ) -> Option<&'db Name> { + enum_class_for_unbound_name(db, env, file, scope, name, resolved_type) + .map(|enum_class| enum_class.name(db)) +} + +/// [`qualifier_for_unbound_name`], as the enum itself rather than its spelling. +/// +/// Goto-definition wants the declaration the bare name resolves to, which means +/// walking the enum's body — the name alone cannot say where it was written +pub(crate) fn enum_class_for_unbound_name<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + scope: ScopeId<'db>, + name: &str, + resolved_type: impl FnOnce() -> Option>, +) -> Option> { // an ordinary reference to a name bound anywhere in the lexical chain, or to // a builtin, keeps its ordinary spelling. checked before the name's type is // asked for, so a file that uses no context-sensitive name is never inferred @@ -359,7 +381,7 @@ pub(crate) fn qualifier_for_unbound_name<'db>( if !declares_member(db, env, enum_class, name) { return None; } - is_nameable(db, file, scope, enum_class).then(|| enum_class.name(db)) + is_nameable(db, file, scope, enum_class).then_some(enum_class) } /// the enum an already-resolved member type belongs to diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index e5300c42a7..e6e147b391 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -116,6 +116,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&INVALID_NAMED_TUPLE); registry.register_lint(&INVALID_NAMED_TUPLE_OVERRIDE); registry.register_lint(&INVALID_RAISE); + registry.register_lint(&INVALID_STATIC_RESOURCE); registry.register_lint(&INVALID_SUPER_ARGUMENT); registry.register_lint(&INVALID_TYPE_ARGUMENTS); registry.register_lint(&INVALID_TYPE_CHECKING_CONSTANT); @@ -146,6 +147,9 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&INVALID_EXTENSION); registry.register_lint(&AMBIGUOUS_EXTENSION_MEMBER); registry.register_lint(&INVALID_CONFORMANCE); + registry.register_lint(&INVALID_MODULE_API); + registry.register_lint(&UNMET_MODULE_API); + registry.register_lint(&INVALID_BUILD_STAMPS); registry.register_lint(&INVALID_CONVERSION); registry.register_lint(&AMBIGUOUS_CONVERSION); registry.register_lint(&MISSING_FRAMEWORK_STUBS); @@ -209,6 +213,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&UNSUPPORTED_BASE); registry.register_lint(&UNSUPPORTED_DYNAMIC_BASE); registry.register_lint(&UNSUPPORTED_OPERATOR); + registry.register_lint(&UNUSABLE_RESOURCE_KEY); registry.register_lint(&UNUSED_AWAITABLE); registry.register_lint(&ZERO_STEPSIZE_IN_SLICE); registry.register_lint(&STATIC_ASSERT_ERROR); @@ -756,6 +761,26 @@ declare_lint! { } } +declare_lint! { + #[doc = include_str!("../../resources/lint_docs/invalid-static-resource.md")] + pub(crate) static INVALID_STATIC_RESOURCE = { + summary: "detects a static resource import that cannot be read", + status: LintStatus::stable("0.0.79"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + #[doc = include_str!("../../resources/lint_docs/unusable-resource-key.md")] + pub(crate) static UNUSABLE_RESOURCE_KEY = { + summary: "detects a key in a static resource that python cannot name", + status: LintStatus::stable("0.0.79"), + default_level: Level::Warn, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { #[doc = include_str!("../../resources/lint_docs/invalid-super-argument.md")] pub(crate) static INVALID_SUPER_ARGUMENT = { @@ -1192,6 +1217,106 @@ declare_lint! { } } +declare_lint! { + /// ## What it does + /// Checks that an `implements` declaration is one a module can be held to: + /// that it names a protocol, that a `for` clause is written in a package's + /// `__init__` with patterns relative to that package and reaching something, + /// and that the module it obliges can actually be checked. + /// + /// ## Why is this bad? + /// An obligation nothing can check is worse than no obligation: it reads, in + /// review and in the editor, as a promise that is being enforced. A rule + /// whose patterns match no module, or one written where no module will look + /// for it, enforces nothing at all. + /// + /// This also covers a declaration written while the feature is off. `implements` + /// is experimental, so a project opts in by name: + /// + /// ```toml + /// # basedpython.toml + /// [experimental] + /// module-api = true + /// ``` + /// + /// ## Example + /// + /// ```by + /// class Backend: + /// def connect(self) -> None: ... + /// + /// implements Backend # error: `Backend` is not a protocol + /// ``` + pub(crate) static INVALID_MODULE_API = { + summary: "detects invalid basedpython `implements` declarations", + status: LintStatus::stable("0.0.72"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + /// ## What it does + /// Checks a module against the interfaces it is obliged to answer, whether it + /// declared one itself with `implements` or a package it lives in imposed one + /// with `implements ... for`. + /// + /// ## Why is this bad? + /// A module that is meant to be used through an interface — a plugin, a + /// backend, a settings module — is otherwise only checked where something + /// assigns it to an interface-typed place. A plugin loaded by name is never + /// checked at all, and an error that does surface lands on the consumer + /// rather than on the module that broke. + /// + /// ## Example + /// + /// ```by + /// protocol Backend: + /// static def connect(url: str) -> None + /// + /// implements Backend # error: this module does not answer `Backend` + /// ``` + pub(crate) static UNMET_MODULE_API = { + summary: "detects a module that does not answer an interface it is obliged to", + status: LintStatus::stable("0.0.72"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + +declare_lint! { + /// ## What it does + /// Checks that a `build:` block is one the project has asked for. Build + /// stamps are experimental, so a project opts in by name: + /// + /// ```toml + /// # basedpython.toml + /// [experimental] + /// build-stamps = true + /// ``` + /// + /// ## Why is this bad? + /// The block parses and lowers whether or not the project opted in, because a + /// program that reads `build.GIT_SHA` has to keep working when the feature is + /// turned off — so nothing at the point of use says the value was never + /// settled. A stamp declared without a default fails the transpile, and one + /// with a default quietly stands for that default in an artifact that claims + /// to know what commit it came from. + /// + /// ## Example + /// + /// ```by + /// build: # error: `build` is an experimental feature, and is off + /// GIT_SHA: str + /// ``` + pub(crate) static INVALID_BUILD_STAMPS = { + summary: "detects a basedpython `build:` block the project has not opted in to", + status: LintStatus::stable("0.0.79"), + default_level: Level::Error, + ty_compat: TyCompat::BasedPython, + } +} + declare_lint! { /// ## What it does /// Checks that the basedpython conversion dunders have the shape their diff --git a/crates/ty_python_semantic/src/types/extensions.rs b/crates/ty_python_semantic/src/types/extensions.rs index e201299f24..f77ea02eb3 100644 --- a/crates/ty_python_semantic/src/types/extensions.rs +++ b/crates/ty_python_semantic/src/types/extensions.rs @@ -761,6 +761,31 @@ struct ApplicableMember<'db> { /// An extension of a *protocol* also applies to any class a visible conformance /// extension conforms to it — that is what makes a protocol extension's members /// reachable on a conforming type, exactly as they are on the protocol itself +/// can `extension` supply *any* member for a receiver of `receiver_class` — the +/// name-independent half of [`applicable_member`]. +/// +/// A completion list asks about every member name every extension in the file +/// declares, and `applicable_member` re-derives this for each one. Answering it +/// once per extension instead is what keeps a file's unrelated extensions from +/// costing anything: 300 `extension str` blocks contribute 300 names, and +/// without this each of those names walked all 300 again before concluding that +/// none of them extends an `int` +pub(crate) fn extension_can_apply<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + extension: StaticClassLiteral<'db>, + receiver_class: ClassType<'db>, +) -> bool { + if extension_applies(db, env, extension, receiver_class).is_some() { + return true; + } + let Some(target) = extended_class(db, extension) else { + return false; + }; + conformance::conformance_for(db, env, file, receiver_class, target).is_some() +} + fn applicable_member<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 4b29d05834..485523dc96 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -13,7 +13,9 @@ use crate::types::constraints::ConstraintSetBuilder; use crate::types::context_sensitive::for_each_candidate; use crate::types::dedicated::django; use crate::types::enums::enum_metadata; -use crate::types::extensions::{applicable_extensions, resolve_extension_member}; +use crate::types::extensions::{ + applicable_extensions, extension_can_apply, resolve_extension_member, resolve_extension_members, +}; use crate::types::function::FunctionDecorators; use crate::types::generics::GenericContext; use crate::types::implicit_names::{ImplicitNamePosition, implicit_name}; @@ -366,7 +368,94 @@ pub fn definitions_for_attribute<'db>( let Some(lhs_ty) = attribute.value.inferred_type(model) else { return Vec::new(); }; - definitions_for_member(model, lhs_ty, attribute.attr.as_str()) + let resolved = definitions_for_member(model, lhs_ty, attribute.attr.as_str()); + if !resolved.is_empty() { + return resolved; + } + definitions_for_fallback_attribute(model, attribute, lhs_ty) +} + +/// basedpython: the declarations of an attribute that no *declared* member +/// answers for — an `extension` member, or an implicit-receiver callable. +/// +/// Neither is a member of the receiver's class. An extension declares its +/// members in its own body, and a receiver callable is an ordinary name in an +/// enclosing scope, so the class-hierarchy walk `definitions_for_member` does +/// cannot reach either. Inference reaches them through two fallbacks that run +/// after member lookup comes up undefined, and goto has to take the same two +/// steps in the same order or the editor answers nothing about code the checker +/// resolved. +/// +/// Worth saying which access this is for, because the bug it fixes was subtle: +/// `xs.second()` already worked, since a *call* resolves through the call's own +/// dispatch target and lands on the extension's function that way. A bare +/// `xs.second` has no call to go through, and neither does a property — which +/// can never be a callee — so those answered nothing at all +/// +fn definitions_for_fallback_attribute<'db>( + model: &SemanticModel<'db>, + attribute: &ast::ExprAttribute, + lhs_ty: Type<'db>, +) -> Vec> { + let db = model.db(); + let env = model.program_environment(); + let name = attribute.attr.as_str(); + // an optional-chain link resolves against the chain's *present* type — the + // `None` it short-circuits with is not part of the receiver + let receiver_ty = if attribute.optional || receivers::spine_has_optional(&attribute.value) { + receivers::strip_none(db, &env, lhs_ty) + } else { + lhs_ty + }; + + let extensions = resolve_extension_members(db, &env, model.file(), receiver_ty, name); + if !extensions.is_empty() { + // every applicable extension is offered: when two supply the same name + // the checker reports the ambiguity at the access site, and an editor + // that lists both is telling the reader the same thing + return extensions + .iter() + .flat_map(|resolution| { + definitions_for_attribute_in_class_hierarchy( + &ClassLiteral::Static(resolution.extension), + model, + name, + ) + }) + .collect(); + } + + let protocol = definitions_for_inline_protocol_member(model, attribute); + if !protocol.is_empty() { + return protocol; + } + + let Some(scope) = model.scope(ast::AnyNodeRef::from(attribute)) else { + return Vec::new(); + }; + let scope = scope.to_scope_id(db, model.program_file()); + let Some((declaring_scope, _)) = receivers::resolve_receiver_attribute_in_scope( + db, + &env, + model.file(), + scope, + receiver_ty, + name, + ) else { + return Vec::new(); + }; + find_symbol_in_scope(db, declaring_scope, name) + .into_iter() + .flat_map(|definition| { + resolve_definition( + db, + &env, + definition, + Some(name), + ImportAliasResolution::ResolveAliases, + ) + }) + .collect() } /// Returns all resolved definitions for the member `name_str` of `lhs_ty`, the @@ -487,9 +576,170 @@ fn definitions_for_member<'db>( } } + // basedpython: a property accessor block is one declaration in the source + // and several `def`s in the tree, and each of them carries the *same* name + // range — the one the author wrote. Two entries pointing at one place are + // never a choice worth offering, so an editor should not open a + // disambiguation list to make it. Python's own getter/setter pair sit at + // two different `def`s and are left as the two targets they are + let mut seen = FxHashSet::default(); + resolved.retain(|definition| seen.insert(definition.focus_range(db))); resolved } +/// basedpython: the member declaration inside the [inline protocol] annotation +/// the receiver was declared with — the `a: int` of +/// `def f(x: protocol(a: int; def g(self) -> int))`, reached from `x.a`. +/// +/// An inline protocol is *structural*: two written the same way anywhere are the +/// same type, deliberately, so the type itself cannot say where any one of them +/// was written and there is no declaration for the ordinary member walk to find. +/// The annotation the receiver was declared with is the one place an editor can +/// honestly point at, so that is what this answers with. +/// +/// [inline protocol]: https://docs.basedpython.org/features/inline-protocol +fn definitions_for_inline_protocol_member<'db>( + model: &SemanticModel<'db>, + attribute: &ast::ExprAttribute, +) -> Vec> { + let db = model.db(); + let ast::Expr::Name(receiver) = attribute.value.as_ref() else { + return Vec::new(); + }; + let name = attribute.attr.as_str(); + let parsed = parsed_module(db, model.python_file()).load(db); + let mut found = Vec::new(); + for definition in definitions_for_name( + model, + receiver.id.as_str(), + ast::AnyNodeRef::from(receiver), + ImportAliasResolution::ResolveAliases, + ) { + let ResolvedDefinition::Definition(definition) = definition else { + continue; + }; + let Some(annotation) = declared_annotation(db, &parsed, definition) else { + continue; + }; + if let Some(range) = inline_protocol_member_range(annotation, name) { + found.push(ResolvedDefinition::FileWithRange(FileRange::new( + definition.file(db), + range, + ))); + } + } + found +} + +/// The annotation a definition was declared with, for the two forms that can +/// carry an inline protocol. +fn declared_annotation<'a>( + db: &dyn Db, + parsed: &'a ruff_db::parsed::ParsedModuleRef, + definition: Definition<'_>, +) -> Option<&'a ast::Expr> { + match definition.kind(db) { + DefinitionKind::Parameter(parameter) => match parameter { + ty_python_core::definition::ParameterDefinitionNodeKind::VariadicPositionalParameter( + parameter, + ) + | ty_python_core::definition::ParameterDefinitionNodeKind::VariadicKeywordParameter( + parameter, + ) => parameter.node(parsed).annotation.as_deref(), + ty_python_core::definition::ParameterDefinitionNodeKind::Parameter(parameter) => { + parameter.node(parsed).parameter.annotation.as_deref() + } + }, + DefinitionKind::AnnotatedAssignment(assignment) => Some(assignment.annotation(parsed)), + _ => None, + } +} + +/// The range of the member `name` declares itself at, anywhere inside an +/// annotation that writes an inline protocol. +/// +/// The search is over the whole annotation because a protocol can be written +/// nested — `list[protocol(a: int)]` — and the member is spelled the same way +/// wherever it sits. +fn inline_protocol_member_range(annotation: &ast::Expr, name: &str) -> Option { + match annotation { + ast::Expr::ProtocolType(protocol) => protocol.members.iter().find_map(|member| { + match member { + // `def g(self) -> int` + ast::Expr::ProtocolMethod(method) if method.name.as_str() == name => { + Some(method.name.range()) + } + // `a: int` + ast::Expr::Named(named) => named + .target + .as_name_expr() + .filter(|target| target.id.as_str() == name) + .map(ruff_text_size::Ranged::range), + _ => None, + } + }), + ast::Expr::Subscript(subscript) => inline_protocol_member_range(&subscript.slice, name), + ast::Expr::Tuple(tuple) => tuple + .elts + .iter() + .find_map(|element| inline_protocol_member_range(element, name)), + ast::Expr::BinOp(binop) => inline_protocol_member_range(&binop.left, name) + .or_else(|| inline_protocol_member_range(&binop.right, name)), + _ => None, + } +} + +/// basedpython: the declaration of the enum member a bare name resolves to +/// through context-sensitive resolution — the `Red` of `c: Color = Red`. +/// +/// Nothing in the enclosing scope binds that name; it is reached through the +/// expected type, so an ordinary scope walk finds nothing to point at. Empty for +/// every name that resolves the ordinary way. +pub fn definitions_for_context_sensitive_name<'db>( + model: &SemanticModel<'db>, + name: &ast::ExprName, +) -> Vec> { + let db = model.db(); + let env = model.program_environment(); + let Some(scope) = model.scope(ast::AnyNodeRef::from(name)) else { + return Vec::new(); + }; + let Some(enum_class) = crate::types::context_sensitive::enum_class_for_unbound_name( + db, + &env, + model.file(), + scope.to_scope_id(db, model.program_file()), + name.id.as_str(), + || name.inferred_type(model), + ) else { + return Vec::new(); + }; + definitions_for_attribute_in_class_hierarchy(&enum_class, model, name.id.as_str()) +} + +/// basedpython: the declaration of the enum member a bare `case Red:` names. +/// +/// The pattern looks like a capture and is one whenever the subject's type does +/// not declare the name, so the answer comes from the same resolution the +/// checker used rather than from the pattern's own binding +pub fn definitions_for_case_name<'db>( + model: &SemanticModel<'db>, + identifier: &ast::Identifier, +) -> Vec> { + let db = model.db(); + let env = model.program_environment(); + let index = ty_python_core::semantic_index(db, model.program_file()); + let Some(case_name) = index.case_name(ty_python_core::node_key::NodeKey::from_node(identifier)) + else { + return Vec::new(); + }; + let Some(member) = crate::types::context_sensitive::resolve_case_name(db, &env, case_name) + else { + return Vec::new(); + }; + definitions_for_attribute_in_class_hierarchy(&member.enum_class(), model, identifier.as_str()) +} + /// basedpython: the declarations of the model field the leading name of a django /// lookup expression names — `author` in `Book.objects.filter(author.name == "x")` /// @@ -3570,7 +3820,22 @@ pub fn extension_members<'db>( ) -> Vec<(Name, Type<'db>)> { let mut members = Vec::new(); let mut seen = FxHashSet::default(); + // whether an extension can supply anything for this receiver does not depend + // on the member name, so it is asked once per extension rather than once per + // name. Without it a file's extensions cost each other: every name any of + // them declares re-walked all of them, which is quadratic in the number of + // `extension` blocks in the file and paid even by blocks extending some + // unrelated type + let receiver_class = receiver + .erase_restriction(db) + .nominal_class(db, env) + .or_else(|| receiver.to_class_type(db)); for &extension in applicable_extensions(db, file) { + if let Some(receiver_class) = receiver_class + && !extension_can_apply(db, env, file, extension, receiver_class) + { + continue; + } for member in all_end_of_scope_members(db, extension.body_scope(db)) { let name = member.member.name; if !seen.insert(name.clone()) { diff --git a/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs b/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs index 3495215af5..d84cdcd6b3 100644 --- a/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs +++ b/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs @@ -3,11 +3,13 @@ use crate::reachability::is_reachable; use crate::types::function::FunctionDecorators; use crate::types::infer::function_known_decorator_flags; use get_size2::GetSize; -use ruff_db::parsed::parsed_module; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_python_ast::name::Name; use ruff_text_size::TextRange; use rustc_hash::FxHashSet; -use ty_python_core::definition::{DefinitionCategory, DefinitionKind, DefinitionState}; +use ty_python_core::definition::{ + DefinitionCategory, DefinitionKind, DefinitionState, ParameterDefinitionNodeKind, +}; use ty_python_core::place::ScopedPlaceId; use ty_python_core::scope::{FileScopeId, ScopeKind}; use ty_python_core::{ProgramFile, SemanticIndex, semantic_index}; @@ -78,6 +80,23 @@ fn comprehension_named_expression_is_local( }) } +/// basedpython: whether the parameter this definition binds was written +/// `context`. +fn parameter_is_context(kind: &DefinitionKind<'_>, parsed: &ParsedModuleRef) -> bool { + let DefinitionKind::Parameter(parameter) = kind else { + return false; + }; + match parameter { + ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter) + | ParameterDefinitionNodeKind::VariadicKeywordParameter(parameter) => { + parameter.node(parsed).is_context + } + ParameterDefinitionNodeKind::Parameter(parameter) => { + parameter.node(parsed).parameter.is_context + } + } +} + fn function_scope_is_overload_declaration( db: &dyn Db, index: &SemanticIndex<'_>, @@ -196,6 +215,19 @@ pub fn unused_bindings(db: &dyn Db, file: ProgramFile<'_>) -> Box<[UnusedBinding continue; } + // basedpython: a `context` parameter is not read the way an ordinary + // one is. It is filled implicitly at a call site that leaves it out, + // and forwarded implicitly to any nested call that wants one — so a + // function that "does not use" it is usually the case where it is + // being forwarded, and the forward happens during inference rather + // than as a name load the use-def map can see. Reporting it as unused + // is wrong there, and elsewhere it says nothing worth saying: the + // parameter is part of the contract its callers resolve against. The + // linter takes the same view of the declaration form, `context x: T` + if is_parameter && parameter_is_context(kind, &parsed) { + continue; + } + if name.starts_with('_') { continue; } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 8288ab4025..74c5d7020d 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -2152,6 +2152,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { crate::types::lifetimes::check_loop_variable_capture(&self.context, &module.body, |expr| { self.try_expression_type(expr) }); + + // basedpython: the interfaces this module is obliged to answer, whether it + // said so itself or a package it lives in said so for it + crate::types::module_api::check_module_api(&self.context, &module.body); + + // basedpython: a `build:` block only means something once the project has + // asked for build stamps + crate::types::build_stamps::check_build_stamps(&self.context, &module.body); } fn infer_type_alias_type_params(&mut self, type_alias: &ast::StmtTypeAlias) { diff --git a/crates/ty_python_semantic/src/types/infer/builder/class.rs b/crates/ty_python_semantic/src/types/infer/builder/class.rs index f4e155c7af..94cf157dde 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/class.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/class.rs @@ -204,6 +204,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { | "sealed" | "enum_def" | "extension_def" + | "build_def" | "variant_unit" | "variant_tuple" | "private" diff --git a/crates/ty_python_semantic/src/types/infer/builder/imports.rs b/crates/ty_python_semantic/src/types/infer/builder/imports.rs index 7aae03088c..4b39039f55 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/imports.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/imports.rs @@ -10,19 +10,20 @@ use crate::{ dependencies::{self, GroupName, ImportStanding}, place::{ DefinedPlace, Definedness, Place, PlaceAndQualifiers, TypeOrigin, - basedpython_typing_added_in, basedpython_warnings_added_in, typing_extensions_symbol, + basedpython_typing_added_in, basedpython_warnings_added_in, explicit_global_symbol, + typing_extensions_symbol, }, types::{ ModuleLiteralType, Type, TypeAndQualifiers, dedicated::EXTERNALLY_STUBBED_FRAMEWORKS, diagnostic::{ - MISPLACED_DEPENDENCY, MISSING_FRAMEWORK_STUBS, POSSIBLY_MISSING_IMPORT, PRIVATE_IMPORT, - UNDECLARED_DEPENDENCY, UNRESOLVED_IMPORT, - hint_if_stdlib_attribute_exists_on_other_versions, + INVALID_STATIC_RESOURCE, MISPLACED_DEPENDENCY, MISSING_FRAMEWORK_STUBS, + POSSIBLY_MISSING_IMPORT, PRIVATE_IMPORT, UNDECLARED_DEPENDENCY, UNRESOLVED_IMPORT, + UNUSABLE_RESOURCE_KEY, hint_if_stdlib_attribute_exists_on_other_versions, hint_if_stdlib_submodule_exists_on_other_versions, }, infer::{TypeInferenceBuilder, builder::DeclaredAndInferredType}, - infer_definition_types, + infer_definition_types, static_resource, visibility::private_symbols, }, }; @@ -179,8 +180,16 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { node_index: _, name, asname, + is_resource, } = alias; + // basedpython: `import "data/config.yaml" as config` names a file, not a + // module, so nothing about module resolution applies to it + if *is_resource { + self.infer_static_resource_definition(alias, definition); + return; + } + // The name of the module being imported let Some(full_module_name) = ModuleName::new(name) else { tracing::debug!("Failed to resolve import due to invalid syntax"); @@ -240,6 +249,106 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ); } + /// basedpython: `import "data/config.yaml" as config`. + /// + /// the document is rendered as python and that module is inferred, so the + /// type bound here is the type of a real declaration in a real file — which + /// is what makes `config.a.b[1]` an ordinary attribute access and an + /// ordinary index rather than a question about json. + fn infer_static_resource_definition( + &mut self, + alias: &ast::Alias, + definition: Definition<'db>, + ) { + let db = self.db(); + let path = alias.name.id.as_str(); + + let resolved = static_resource::resolve_static_resource(db, self.file(), path) + .and_then(|document| static_resource::rendered(db, document)); + + let (module, rendered) = match resolved { + Ok(resolved) => resolved, + Err(error) => { + if let Some(builder) = self + .context + .report_lint(&INVALID_STATIC_RESOURCE, alias.range()) + { + builder + .into_diagnostic(format_args!("Cannot read static resource `{path}`")) + .info(error.to_string()); + } + self.add_unknown_declaration_with_binding(alias.into(), definition); + return; + } + }; + + self.report_unusable_resource_keys(alias, &rendered.unusable_keys); + + let ty = explicit_global_symbol(db, db.program_file(module), &rendered.root) + .place + .ignore_possibly_undefined() + .unwrap_or_else(Type::unknown); + + self.add_declaration_with_binding( + alias.into(), + definition, + &DeclaredAndInferredType::are_the_same_type(ty), + ); + } + + /// how many unusable keys a diagnostic names before it starts counting. + const LISTED_UNUSABLE_KEYS: usize = 5; + + /// report the keys the document holds that the value cannot carry. + /// + /// they are not an error in the document — plenty of real configuration is + /// keyed by names python has no spelling for — but the program cannot reach + /// them, and finding that out by writing the attribute and being told it + /// does not exist is a worse way to learn it. + fn report_unusable_resource_keys(&mut self, alias: &ast::Alias, keys: &[String]) { + let Some((first, rest)) = keys.split_first() else { + return; + }; + let Some(builder) = self + .context + .report_lint(&UNUSABLE_RESOURCE_KEY, alias.range()) + else { + return; + }; + + let path = alias.name.id.as_str(); + // "cannot be read as an attribute" rather than "is not an identifier": + // a key is also left out when python would mangle it, and when it is a + // name the rendering needs for itself, and both of those are identifiers + let mut diagnostic = if rest.is_empty() { + builder.into_diagnostic(format_args!( + "Key `{first}` of `{path}` cannot be read as an attribute" + )) + } else { + builder.into_diagnostic(format_args!( + "{count} keys of `{path}` cannot be read as attributes", + count = keys.len() + )) + }; + + let listed = keys + .iter() + .take(Self::LISTED_UNUSABLE_KEYS) + .map(|key| format!("`{key}`")) + .collect::>() + .join(", "); + if keys.len() > Self::LISTED_UNUSABLE_KEYS { + diagnostic.info(format_args!( + "{listed} and {more} more are left out of the value", + more = keys.len() - Self::LISTED_UNUSABLE_KEYS + )); + } else if !rest.is_empty() { + diagnostic.info(format_args!("{listed} are left out of the value")); + } else { + diagnostic.info("It is left out of the value"); + } + } + pub(super) fn infer_import_from_statement(&mut self, import: &ast::StmtImportFrom) { let ast::StmtImportFrom { module: _, diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index b110934d6c..4b63f8529e 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -544,8 +544,14 @@ impl<'db> TypeInferenceBuilder<'db, '_> { let left_ty = self.infer_type_expression(&binary.left); let right_ty = self.infer_type_expression(&binary.right); - // Detect runtime errors from e.g. `int | "bytes"` on Python <3.14 without `__future__` annotations. - if !ignore_runtime_errors(self) { + // Detect runtime errors from e.g. `int | "bytes"` on Python <3.14 + // without `__future__` annotations — for a python file. A `.by` type + // expression is lowered before it is ever evaluated, and what the + // operands would mean *as values* says nothing about what actually + // runs: a bare `"foo"` is emitted as `Literal["foo"]`, a float literal + // as `float` or as `Literal[1.5]`, and a union that reaches an old + // target's runtime as `Union[...]`. every one of those supports `|`. + if !self.is_basedpython_file() && !ignore_runtime_errors(self) { let mut speculative_builder = self.speculate_without_diagnostics(); // If the left-hand side of the union is itself a PEP-604 union, // we'll already have checked whether it can be used with `|` in a previous inference step diff --git a/crates/ty_python_semantic/src/types/list_members.rs b/crates/ty_python_semantic/src/types/list_members.rs index 5016cc8bb3..6112ed4fb6 100644 --- a/crates/ty_python_semantic/src/types/list_members.rs +++ b/crates/ty_python_semantic/src/types/list_members.rs @@ -377,6 +377,30 @@ impl<'db> AllMembers<'db> { } } + // basedpython: an inline protocol — `protocol(a: int; def g(self) -> int)` + // — is structural, so it has no class of its own to walk. Its interface + // *is* the member list, and without this arm the fallthrough below finds + // a meta-type with no `class_origin` and offers nothing at all + Type::ProtocolInstance(protocol) if protocol.class_origin(db).is_none() => { + let names: Vec = protocol + .interface(db) + .members(db) + .map(|member| Name::new(member.name())) + .collect(); + for name in names { + if let Some(member_ty) = + ty.member(db, env, &name).place.ignore_possibly_undefined() + { + self.members.insert(Member { + name, + ty: member_ty, + is_type_check_only: false, + }); + } + } + self.extend_with_type(db, env, Type::object()); + } + Type::LiteralValue(_) | Type::PropertyInstance(_) | Type::FunctionLiteral(_) diff --git a/crates/ty_python_semantic/src/types/module_api.rs b/crates/ty_python_semantic/src/types/module_api.rs new file mode 100644 index 0000000000..b16776c270 --- /dev/null +++ b/crates/ty_python_semantic/src/types/module_api.rs @@ -0,0 +1,825 @@ +//! basedpython module api enforcement — the `implements` declaration +//! +//! A module is already a structural value: [`Type::ModuleLiteral`] answers a +//! protocol's members through the module's public surface, so +//! `backend: Backend = postgres` type-checks on its own. What this module adds is +//! a way to attach that obligation to a module *permanently*, so it is checked in +//! the file that carries it rather than wherever someone happens to assign it — +//! or nowhere at all, when nobody does, which is the case for a plugin that is +//! only ever loaded by name. +//! +//! An obligation is attached in one of two ways, and both arrive at +//! [`module_obligations`]: +//! +//! - the module says so itself, with a bare `implements Backend` +//! - a package it lives in says so, with `implements Backend for ".*"` written in +//! that package's `__init__` +//! +//! The second is what a plugin directory needs, and it is why obligations are +//! indexed by **containment**. A module has to be able to find the obligations +//! imposed on it, or the error would not appear in the file whose author can fix +//! it. Walking a module's ancestor packages costs a handful of lookups the module +//! resolver has already done. The two alternatives are worse: a rule anywhere in +//! the project would need a project-wide scan, making every file's check depend +//! on every file's declarations; and the import graph — the relation `extension` +//! visibility uses — is the wrong one here, because the whole point is imposing +//! on a module that does not import you. +//! +//! Containment also draws the ownership line in the right place: a package may +//! impose on modules inside itself, and nothing can reach into a package from +//! outside to add requirements to it. + +use ruff_db::diagnostic::{Annotation, Span}; +use ruff_db::files::FileRange; +use ruff_db::parsed::parsed_module; +use ruff_python_ast as ast; +use ruff_python_ast::helpers::{ImplementsDeclaration, implements_declaration}; +use ruff_text_size::{Ranged, TextRange}; +use ty_module_resolver::{ + ImportingFile, ModuleGlobSet, ModuleName, file_to_module, resolve_module, +}; + +use crate::Db; +use crate::place::global_symbol; +use crate::types::class::ClassType; +use crate::types::conformance::{interface_requirements, is_conformable}; +use crate::types::context::InferContext; +use crate::types::diagnostic::{INVALID_MODULE_API, UNMET_MODULE_API}; +use crate::types::{ProgramEnvironment, ProgramFile, Type}; + +/// An obligation on one module: an interface it has to answer, and every +/// declaration that asked for it. +/// +/// One interface, however many declarations name it — a module that declares +/// `implements Backend` inside a package whose rule says the same thing has one +/// obligation, not two, because it has one thing left to do about it. +#[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +struct Obligation<'db> { + /// the interface the module has to answer + interface: ClassType<'db>, + /// the module's own `implements`, when it declared this itself. The + /// diagnostic is anchored here, since it is the line the reader can act on + own: Option, + /// every declaration that asked for this interface, the module's own + /// included + #[get_size(ignore)] + sources: Box<[FileRange]>, +} + +impl Obligation<'_> { + /// Where a diagnostic about this obligation is anchored: the module's own + /// declaration, or the top of the file when a package imposed it and there is + /// nothing in the module to point at. + fn anchor(&self) -> TextRange { + self.own.unwrap_or_default() + } +} + +/// Collects obligations, merging the declarations that name the same interface. +#[derive(Default)] +struct Obligations<'db> { + collected: Vec>, +} + +impl<'db> Obligations<'db> { + fn add(&mut self, interface: ClassType<'db>, source: FileRange, own: Option) { + if let Some(existing) = self + .collected + .iter_mut() + .find(|obligation| obligation.interface == interface) + { + existing.own = existing.own.or(own); + let mut sources = existing.sources.to_vec(); + sources.push(source); + existing.sources = sources.into_boxed_slice(); + return; + } + self.collected.push(Obligation { + interface, + own, + sources: Box::from([source]), + }); + } +} + +/// A rule a package's `__init__` imposes on the modules in its subtree. +#[derive(Clone, Debug, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +struct PackageRule<'db> { + interface: ClassType<'db>, + /// the patterns exactly as written, still relative to the declaring package + patterns: Box<[Box]>, + /// the `implements` keyword the rule is written at + range: TextRange, +} + +/// The rules `file` imposes on its subtree, if it is a package `__init__` that +/// declares any. +/// +/// Cycles for the reason the conformance registry does: resolving the interface a +/// rule names infers the declaring module's code, and a package `__init__` +/// routinely imports the very submodules its rules govern. Recovery is the same — +/// no rules yet. +#[salsa::tracked( + returns(deref), + cycle_initial = |_, _, _| Box::default(), + heap_size = ruff_memory_usage::heap_size +)] +fn package_rules<'db>(db: &'db dyn Db, file: ProgramFile<'db>) -> Box<[PackageRule<'db>]> { + if !file.file(db).source_type(db).is_basedpython() { + return Box::default(); + } + let parsed = parsed_module(db, file.python_file(db)).load(db); + let mut rules = Vec::new(); + for statement in &parsed.syntax().body { + let Some(declaration) = implements_declaration(statement) else { + continue; + }; + if declaration.patterns.is_empty() { + continue; + } + let patterns: Box<[Box]> = declaration + .patterns + .iter() + .filter_map(|pattern| pattern.as_string_literal_expr()) + .map(|pattern| Box::from(pattern.value.to_str())) + .collect(); + for interface in declaration.interfaces { + let Some(interface) = resolve_interface(db, file, interface) else { + continue; + }; + if !is_conformable(db, interface) { + continue; + } + rules.push(PackageRule { + interface, + patterns: patterns.clone(), + range: declaration.keyword_range, + }); + } + } + rules.into_boxed_slice() +} + +/// The module `file` belongs to, as importers see it. +/// +/// Stub-preferred, so for a module with both a `.by` and a `.byi` this is the +/// stub — the file whose surface everything outside the module reads, and +/// therefore the only surface an obligation can sensibly be about. +fn api_module<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> Option> { + let module = file_to_module(db, file.resolver_file(db))?; + resolve_module( + db, + ImportingFile::File(file.file(db), file.resolver_environment(db)), + module.name(db), + ) +} + +/// Is `file` the file that *is* its module's api? +/// +/// False for an implementation file shadowed by a stub. Both files are checked by +/// the project, so without this the same obligation would be reported twice, once +/// against a surface nobody outside the module can see. +fn is_api_file<'db>(db: &'db dyn Db, file: ProgramFile<'db>) -> bool { + api_module(db, file) + .and_then(|module| module.file(db)) + .is_some_and(|api| api == file.file(db)) +} + +/// Every obligation that applies to `file`'s module: the ones it declares itself, +/// and the ones its ancestor packages impose on it. +#[salsa::tracked( + returns(deref), + cycle_initial = |_, _, _| Box::default(), + heap_size = ruff_memory_usage::heap_size +)] +fn module_obligations<'db>(db: &'db dyn Db, file: ProgramFile<'db>) -> Box<[Obligation<'db>]> { + let Some(module) = file_to_module(db, file.resolver_file(db)) else { + return Box::default(); + }; + // a rule cannot reach outside its own package, and nothing anyone can fix + // lives outside the project, so checking installed or vendored code would + // only produce noise + if module + .search_path(db) + .is_none_or(|path| !path.is_first_party()) + { + return Box::default(); + } + + // an implementation shadowed by a stub is not the module's api, and checking + // it too would report the same obligation twice, once against a surface + // nobody outside the module can see. `check_declarations` is what tells the + // author their declaration is in the wrong file + if !is_api_file(db, file) { + return Box::default(); + } + + let mut obligations = Obligations::default(); + + // the module's own declarations. a `for` clause on one of these is a rule, + // handled by `package_rules` for the modules it names, and says nothing about + // the file it is written in + if file.file(db).source_type(db).is_basedpython() { + let parsed = parsed_module(db, file.python_file(db)).load(db); + for statement in &parsed.syntax().body { + let Some(declaration) = implements_declaration(statement) else { + continue; + }; + if !declaration.patterns.is_empty() { + continue; + } + for interface in declaration.interfaces { + let Some(interface) = resolve_interface(db, file, interface) else { + continue; + }; + if !is_conformable(db, interface) { + continue; + } + obligations.add( + interface, + FileRange::new(file.file(db), declaration.keyword_range), + Some(declaration.keyword_range), + ); + } + } + } + + let name = module.name(db); + for ancestor in ancestor_packages(name) { + let Some(package) = resolve_module( + db, + ImportingFile::File(file.file(db), file.resolver_environment(db)), + &ancestor, + ) else { + continue; + }; + let Some(package_file) = package + .file(db) + .map(|package_file| ProgramFile::new(db, package_file, file.program(db))) + else { + continue; + }; + for rule in package_rules(db, package_file) { + if !Reach::new(&ancestor, &rule.patterns).includes(&ancestor, name) { + continue; + } + obligations.add( + rule.interface, + FileRange::new(package_file.file(db), rule.range), + None, + ); + } + } + + obligations.collected.into_boxed_slice() +} + +/// The strict ancestor packages of `name`, outermost first. +/// +/// Strict, so a package's own rules never oblige the package itself: patterns are +/// relative to the declaring package and name what is *inside* it. +fn ancestor_packages(name: &ModuleName) -> Vec { + let components: Vec<&str> = name.components().collect(); + (1..components.len()) + .filter_map(|end| ModuleName::new(&components[..end].join("."))) + .collect() +} + +/// The compiled reach of one rule: which modules its patterns include. +struct Reach { + /// every pattern the rule wrote + all: ModuleGlobSet, + /// only the include patterns whose last component is a literal name, which is + /// what decides whether a private module was reached deliberately + named: ModuleGlobSet, +} + +impl Reach { + /// A pattern that cannot be resolved against the package that wrote it is + /// skipped rather than poisoning the rule: [`check_patterns`] reports it, and + /// the patterns beside it go on meaning what they say. A rule whose patterns + /// are *all* unusable reaches nothing, which is what it says. + fn new(package: &ModuleName, patterns: &[Box]) -> Self { + let absolute: Vec = patterns + .iter() + .filter_map(|pattern| absolute_pattern(package, pattern)) + .collect(); + // a pattern reaches a private module only by naming it outright, so the + // ones that can are those with no wildcard anywhere + let named: Vec = patterns + .iter() + .filter(|pattern| !pattern.starts_with('!') && !pattern.contains('*')) + .filter_map(|pattern| absolute_pattern(package, pattern)) + .collect(); + Self { + all: Self::compile(&absolute), + named: Self::compile(&named), + } + } + + /// An invalid pattern is [`check_patterns`]' to report; a set that will not + /// compile simply matches nothing. + fn compile(patterns: &[String]) -> ModuleGlobSet { + ModuleGlobSet::from_patterns(patterns.iter().map(String::as_str)) + .unwrap_or_else(|_| ModuleGlobSet::empty()) + } + + fn includes(&self, package: &ModuleName, name: &ModuleName) -> bool { + if !self.all.matches(name).is_include() { + return false; + } + // a leading underscore already means "not part of the surface" everywhere + // else, and a private helper sitting among the plugins is the common case. + // every component below the declaring package counts, so a wildcard does + // not reach into a private *package* either + let private = name + .components() + .skip(package.components().count()) + .any(|component| component.starts_with('_')); + if !private { + return true; + } + self.named.matches(name).is_include() + } +} + +/// Whether a rule reaches something in its package — or whether that cannot be +/// answered, which counts the same. +/// +/// The whole subtree, not just the direct submodules: a pattern may name a +/// subpackage's contents (`".deep.*"`). +/// +/// A package with no submodules the resolver can enumerate is treated as +/// reaching something, because there is no evidence either way. That covers a +/// namespace portion inside the package, whose contents span directories and +/// search paths and are deliberately not enumerated +/// ([`ty_module_resolver::Module::all_submodules`] drops them). Accusing a rule +/// that does enforce is worse than missing a typo. +fn reaches_any<'db>( + db: &'db dyn Db, + package_module: ty_module_resolver::Module<'db>, + package: &ModuleName, + patterns: &[Box], +) -> bool { + reaches_subtree(db, package_module, package, &Reach::new(package, patterns)) +} + +fn reaches_subtree<'db>( + db: &'db dyn Db, + package_module: ty_module_resolver::Module<'db>, + package: &ModuleName, + reach: &Reach, +) -> bool { + let submodules = package_module.all_submodules(db); + if submodules.is_empty() { + return package_module.kind(db).is_package(); + } + submodules.iter().any(|&submodule| { + reach.includes(package, submodule.name(db)) + || reaches_subtree(db, submodule, package, reach) + }) +} + +/// How a pattern that is not relative should have been written. +/// +/// A pattern that spells the package out in full (`"pkg.*"` inside `pkg`) is the +/// common mistake, and its fix is not the same as an absolute pattern's. +fn relative_spelling(package: &ModuleName, written: &str) -> String { + let (negation, body) = match written.strip_prefix('!') { + Some(body) => ("!", body), + None => ("", written), + }; + let inside = body.strip_prefix(&format!("{package}.")).unwrap_or(body); + format!("{negation}.{}", inside.trim_start_matches('.')) +} + +/// A pattern as written, resolved against the package that declared it. +/// +/// `None` for a pattern that is not relative to its package — an absolute one, or +/// one that tries to climb — which [`check_module_api`] reports at the rule. +fn absolute_pattern(package: &ModuleName, pattern: &str) -> Option { + let (negation, body) = match pattern.strip_prefix('!') { + Some(body) => ("!", body), + None => ("", pattern), + }; + if !body.starts_with('.') || body.starts_with("..") { + return None; + } + Some(format!("{negation}{package}{body}")) +} + +/// The interface an `implements` declaration names, resolved in the file that +/// wrote it. +/// +/// By name rather than by inferring the expression, because a rule is read from a +/// *different* file than the one being checked, where there is no inference of +/// that file's expressions to read a type out of. +fn resolve_interface<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + expression: &ast::Expr, +) -> Option> { + resolve_reference(db, file, expression)?.to_class_type(db) +} + +/// The value a name or dotted name refers to, in the file that wrote it. +fn resolve_reference<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + expression: &ast::Expr, +) -> Option> { + match expression { + ast::Expr::Name(name) => global_symbol(db, file, name.id.as_str()) + .place + .ignore_possibly_undefined(), + ast::Expr::Attribute(attribute) => { + let value = resolve_reference(db, file, &attribute.value)?; + let env = ProgramEnvironment::from_file(file); + value + .member(db, &env, attribute.attr.id.as_str()) + .place + .ignore_possibly_undefined() + } + _ => None, + } +} + +/// Checks a module's `implements` declarations, and the obligations that apply to +/// it, at the end of inferring its module scope. +/// +/// Two directions, deliberately reported in different files. A declaration that +/// is malformed is the declaring file's problem, so it is reported there; an +/// obligation the module fails to answer is the *obliged* module's problem, even +/// when a package imposed it, so it is reported in the module — with a secondary +/// annotation on the rule, because an error saying a module must have `connect` +/// is useless without saying who says so. +pub(super) fn check_module_api<'ast>(context: &InferContext<'_, 'ast>, body: &'ast [ast::Stmt]) { + let db = context.db(); + let file = context.program_file(); + // an obligation cannot reach outside its own package, and a diagnostic about + // a file the project does not check is never shown — so doing any of this for + // installed or vendored code is pure waste + if file_to_module(db, file.resolver_file(db)) + .and_then(|module| module.search_path(db)) + .is_none_or(|path| !path.is_first_party()) + { + return; + } + // an experimental feature is off unless the project asked for it. a + // declaration written while it is off is *reported*, not ignored: an + // obligation nothing checks is the failure mode this whole feature exists to + // remove, and it would be a strange one to introduce here + if !db.experimental_settings().module_api { + if file.file(db).source_type(db).is_basedpython() { + report_disabled_declarations(context, body); + } + return; + } + + if file.file(db).source_type(db).is_basedpython() { + if !is_api_file(db, file) { + report_shadowed_declarations(context, body); + return; + } + check_declarations(context, body); + report_misplaced_declarations(context, body); + } + check_obligations(context); +} + +/// Reports every declaration in a project that has not opted in to the feature. +fn report_disabled_declarations<'ast>(context: &InferContext<'_, 'ast>, body: &'ast [ast::Stmt]) { + for statement in body { + let Some(declaration) = implements_declaration(statement) else { + continue; + }; + let Some(builder) = context.report_lint(&INVALID_MODULE_API, declaration.keyword_range) + else { + continue; + }; + let mut diagnostic = + builder.into_diagnostic("`implements` is an experimental feature, and is off"); + diagnostic.info("nothing is checked against this declaration until the project opts in"); + diagnostic.help( + "Enable it with `module-api = true` under `[experimental]` in `basedpython.toml`", + ); + } +} + +/// Reports a declaration written in an implementation file that a stub shadows. +/// +/// Nothing outside the module reads that file's surface, so an obligation +/// attached there would be about a surface that does not exist as far as anyone +/// else is concerned — and the stub, which *is* the module's api, would go +/// unchecked. Rather than check the wrong thing quietly, say where it belongs. +fn report_shadowed_declarations<'ast>(context: &InferContext<'_, 'ast>, body: &'ast [ast::Stmt]) { + let db = context.db(); + for statement in body { + let Some(declaration) = implements_declaration(statement) else { + continue; + }; + let Some(builder) = context.report_lint(&INVALID_MODULE_API, declaration.keyword_range) + else { + continue; + }; + let mut diagnostic = builder + .into_diagnostic("this module's api is its stub, so its declarations belong there"); + if let Some(stub) = + api_module(db, context.program_file()).and_then(|module| module.file(db)) + { + diagnostic.info(format_args!( + "everything outside this module reads `{}`", + stub.path(db) + )); + } + diagnostic.help("Move the declaration into the stub"); + } +} + +/// Reports what is wrong with the declarations `file` itself writes. +fn check_declarations<'ast>(context: &InferContext<'_, 'ast>, body: &'ast [ast::Stmt]) { + let db = context.db(); + let file = context.program_file(); + let module = file_to_module(db, file.resolver_file(db)); + let is_package = module.is_some_and(|module| module.kind(db).is_package()); + + for statement in body { + let Some(declaration) = implements_declaration(statement) else { + continue; + }; + + for interface in declaration.interfaces { + // an interface that does not resolve, or resolves to nothing usable — + // an undefined name, an unresolved import — has already been reported + // on the name itself, and saying it twice helps nobody + let Some(resolved) = resolve_reference(db, file, interface) else { + continue; + }; + if resolved.is_dynamic() { + continue; + } + let class = resolved.to_class_type(db); + if class.is_some_and(|class| is_conformable(db, class)) { + continue; + } + if let Some(builder) = context.report_lint(&INVALID_MODULE_API, interface) { + let mut diagnostic = builder.into_diagnostic(match class { + Some(class) => format!("`{}` is not a protocol", class.name(db)), + None => format!( + "`{}` is not a protocol", + resolved.display(db, context.program_environment()) + ), + }); + diagnostic.info( + "a module answers an interface through its public surface, which only a \ + protocol describes", + ); + } + } + + if declaration.patterns.is_empty() { + continue; + } + + // a rule is found by walking a module's ancestor packages, so a rule + // written anywhere else would govern nothing and silently enforce nothing + if !is_package + && let Some(builder) = + context.report_lint(&INVALID_MODULE_API, declaration.keyword_range) + { + let mut diagnostic = builder.into_diagnostic( + "a `for` clause may only be written in a package's `__init__`".to_string(), + ); + diagnostic.info( + "a module finds the rules imposed on it by walking the packages it is in, so a \ + rule anywhere else would reach nothing", + ); + diagnostic.help("Move the declaration into the package's `__init__`, or drop the `for` clause to oblige this module itself"); + } + + if let (true, Some(module)) = (is_package, module) { + check_patterns(context, &declaration, module.name(db)); + } + } +} + +/// Reports the patterns of one rule that cannot be resolved against the package +/// that wrote them, and the rule that reaches nothing. +fn check_patterns<'ast>( + context: &InferContext<'_, 'ast>, + declaration: &ImplementsDeclaration<'ast>, + package: &ModuleName, +) { + let db = context.db(); + let mut patterns = Vec::new(); + for pattern in declaration.patterns { + let Some(literal) = pattern.as_string_literal_expr() else { + continue; + }; + let written = literal.value.to_str(); + let Some(absolute) = absolute_pattern(package, written) else { + if let Some(builder) = context.report_lint(&INVALID_MODULE_API, pattern) { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{written}` is not a pattern relative to `{package}`" + )); + diagnostic.info( + "a rule reaches into the package it is written in, so its patterns start \ + with a `.`, as a relative import does", + ); + // a pattern with nothing left once the package prefix comes off + // has no spelling to suggest — `"."` is not one + let spelling = relative_spelling(package, written); + if spelling.trim_start_matches('!') != "." { + diagnostic.help(format_args!("Write `\"{spelling}\"`")); + } + } + continue; + }; + if let Err(error) = ModuleGlobSet::from_patterns([absolute.as_str()]) { + if let Some(builder) = context.report_lint(&INVALID_MODULE_API, pattern) { + builder.into_diagnostic(format_args!( + "`{written}` is not a valid module pattern: {error}" + )); + } + continue; + } + patterns.push(Box::from(written)); + } + + // a pattern that reaches nothing enforces nothing, silently, which is the + // worst thing a check can do. the package's own subtree answers this, so it + // costs a walk of the package rather than one of the project + if patterns.is_empty() { + return; + } + let Some(package_module) = resolve_module( + db, + ImportingFile::File( + context.file(), + context.program_file().resolver_environment(db), + ), + package, + ) else { + return; + }; + let reaches_something = reaches_any(db, package_module, package, &patterns); + if !reaches_something + && let Some(builder) = context.report_lint(&INVALID_MODULE_API, declaration.keyword_range) + { + let mut diagnostic = builder.into_diagnostic("this rule reaches no module".to_string()); + diagnostic.info(format_args!( + "nothing in `{package}` matches, so the declaration obliges nothing" + )); + } +} + +/// Reports an `implements` written somewhere other than module level. +/// +/// An obligation is about a module's surface, so a declaration inside a function +/// or a class body has nothing to attach to — and nothing looks for one there. +fn report_misplaced_declarations<'ast>(context: &InferContext<'_, 'ast>, body: &'ast [ast::Stmt]) { + struct Nested<'a, 'db, 'ast> { + context: &'a InferContext<'db, 'ast>, + } + + impl<'ast> ruff_python_ast::statement_visitor::StatementVisitor<'ast> for Nested<'_, '_, 'ast> { + fn visit_stmt(&mut self, stmt: &'ast ast::Stmt) { + if let Some(declaration) = implements_declaration(stmt) + && let Some(builder) = self + .context + .report_lint(&INVALID_MODULE_API, declaration.keyword_range) + { + let mut diagnostic = builder.into_diagnostic( + "an `implements` declaration belongs at module level".to_string(), + ); + diagnostic.info( + "an obligation is about a module's public surface, so there is nothing for \ + one written inside a body to attach to", + ); + } + ruff_python_ast::statement_visitor::walk_stmt(self, stmt); + } + } + + let mut visitor = Nested { context }; + for statement in body { + // the module's own statements are the well-placed ones; only what is + // nested inside them is misplaced + if implements_declaration(statement).is_none() { + ruff_python_ast::statement_visitor::walk_stmt(&mut visitor, statement); + } + } +} + +/// Checks the module against every obligation that applies to it. +fn check_obligations(context: &InferContext<'_, '_>) { + let db = context.db(); + let env = context.program_environment(); + let file = context.program_file(); + let obligations = module_obligations(db, file); + if obligations.is_empty() { + return; + } + + // the module as importers see it: with a stub, that is the stub's surface, + // which is the only one an obligation can be about + let Some(module) = api_module(db, file) else { + return; + }; + let module_ty = Type::module_literal(db, file, module); + + // a module-level `__getattr__` answers every name, so every requirement would + // be met vacuously. a silent pass is worse than no check at all + let has_getattr = global_symbol(db, file, "__getattr__") + .place + .ignore_possibly_undefined() + .is_some(); + + for obligation in obligations { + let interface = obligation.interface; + let interface_instance = Type::instance(db, env, interface); + + if has_getattr { + if let Some(builder) = context.report_lint(&INVALID_MODULE_API, obligation.anchor()) { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` cannot be checked against `{}`", + module.name(db), + interface_instance.display(db, env), + )); + diagnostic.info( + "this module's `__getattr__` answers every name, so every requirement would \ + be met without anything being defined", + ); + } + continue; + } + + if module_ty.is_assignable_to(db, env, interface_instance) { + continue; + } + + let mut missing = Vec::new(); + let mut mismatched = Vec::new(); + for requirement in interface_requirements(db, interface) { + let name = requirement.as_str(); + let Some(expected) = interface_instance + .member(db, env, name) + .place + .ignore_possibly_undefined() + else { + continue; + }; + match module_ty + .member(db, env, name) + .place + .ignore_possibly_undefined() + { + None => missing.push(requirement.clone()), + Some(actual) => { + if !actual.is_assignable_to(db, env, expected) { + mismatched.push((requirement.clone(), expected, actual)); + } + } + } + } + + let Some(builder) = context.report_lint(&UNMET_MODULE_API, obligation.anchor()) else { + continue; + }; + let mut diagnostic = builder.into_diagnostic(format_args!( + "`{}` does not answer `{}`", + module.name(db), + interface_instance.display(db, env), + )); + for name in &missing { + diagnostic.info(format_args!("`{name}` is missing")); + } + for (name, expected, actual) in &mismatched { + diagnostic.info(format_args!( + "`{name}` is `{}`, but `{}` declares it as `{}`", + actual.display(db, env), + interface_instance.display(db, env), + expected.display(db, env), + )); + } + if missing.is_empty() && mismatched.is_empty() { + diagnostic.info(format_args!( + "`{}` is not assignable to `{}`", + module_ty.display(db, env), + interface_instance.display(db, env), + )); + } + for source in &obligation.sources { + // the module's own declaration is already the primary annotation + if Some(source.range()) == obligation.own && source.file() == context.file() { + continue; + } + diagnostic.annotate( + Annotation::secondary(Span::from(*source)).message("required by this declaration"), + ); + } + } +} diff --git a/crates/ty_python_semantic/src/types/receivers.rs b/crates/ty_python_semantic/src/types/receivers.rs index a1dc0c4983..833197b563 100644 --- a/crates/ty_python_semantic/src/types/receivers.rs +++ b/crates/ty_python_semantic/src/types/receivers.rs @@ -105,6 +105,23 @@ pub(crate) fn resolve_receiver_attribute<'db>( receiver_ty: Type<'db>, name: &str, ) -> Option> { + Some(resolve_receiver_attribute_in_scope(db, env, file, scope, receiver_ty, name)?.1) +} + +/// [`resolve_receiver_attribute`], plus the scope whose declaration of `name` +/// answered. +/// +/// Goto-definition needs the scope: the callable's declaration is an ordinary +/// name in an enclosing scope rather than a member of anything, so the type +/// alone cannot say where it was written +pub(crate) fn resolve_receiver_attribute_in_scope<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + scope: ScopeId<'db>, + receiver_ty: Type<'db>, + name: &str, +) -> Option<(ScopeId<'db>, Type<'db>)> { let index = semantic_index(db, db.program_file(file)); for (ancestor_id, _) in index.visible_ancestor_scopes(scope.file_scope_id(db)) { let ancestor_scope = ancestor_id.to_scope_id(db, db.program_file(file)); @@ -128,7 +145,8 @@ pub(crate) fn resolve_receiver_attribute<'db>( ) .place .ignore_possibly_undefined()?; - return bind_receiver(db, env, declared, receiver_ty); + let bound = bind_receiver(db, env, declared, receiver_ty)?; + return Some((ancestor_scope, bound)); } None } diff --git a/crates/ty_python_semantic/src/types/static_resource.rs b/crates/ty_python_semantic/src/types/static_resource.rs new file mode 100644 index 0000000000..39f50685f8 --- /dev/null +++ b/crates/ty_python_semantic/src/types/static_resource.rs @@ -0,0 +1,140 @@ +//! basedpython: `import "data/config.yaml" as config`. +//! +//! a static resource is a json, toml or yaml file that is part of the program +//! rather than something the program opens. the import says so, and this module +//! is what turns the path it names into a value with a type. +//! +//! the document is not read into types directly. it is rendered as python — a +//! mapping as a class, a sequence as a tuple, a scalar as a `Final` literal — +//! and that python is inferred like any other module, so `config.a.b[1]` is +//! answered by the same machinery that answers it for a hand-written module. +//! the transpiler emits the very same rendering, which is why the type a value +//! has here and the object the program gets cannot disagree. + +use std::fmt; +use std::fmt::Write as _; + +use by_resource::Rendered; +use ruff_db::files::{File, FilePath, system_path_to_file}; +use ruff_db::resource::{Resource, resource, resource_format, resource_module}; +use ruff_db::source::source_text; +use ruff_db::system::{SystemPath, SystemPathBuf}; + +use crate::Db; + +/// why a static resource import could not be read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResourceError { + /// the path is absolute, so it names a place on one machine + NotRelative, + /// the importing file is not a file on the system — a stub out of the + /// vendored typeshed, say — so a path relative to it means nothing + Unanchored, + /// nothing is at the path + NotFound(SystemPathBuf), + /// something is at the path, but not in a format a resource is written in + UnsupportedFormat(String), + /// the document could not be read + Unreadable(String), +} + +impl fmt::Display for ResourceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ResourceError::NotRelative => { + f.write_str("a static resource is named by a path relative to the importing file") + } + ResourceError::Unanchored => { + f.write_str("this file is not on the file system, so it cannot name a path") + } + ResourceError::NotFound(path) => write!(f, "no file at `{path}`"), + ResourceError::UnsupportedFormat(path) => { + write!(f, "`{path}` is not {formats}", formats = list_of_formats()) + } + ResourceError::Unreadable(message) => f.write_str(message), + } + } +} + +/// the resource formats, as a phrase to put in a message. +fn list_of_formats() -> String { + let mut formats = String::from("a "); + for (index, extension) in by_resource::Format::EXTENSIONS.iter().enumerate() { + if index > 0 { + formats.push_str(if index + 1 == by_resource::Format::EXTENSIONS.len() { + " or " + } else { + ", " + }); + } + let _ = write!(formats, "`.{extension}`"); + } + formats.push_str(" file"); + formats +} + +/// the file a static resource import names, resolved against the file that +/// imports it. +/// +/// the path is relative to the importing file's own directory, the way an +/// import of a data file reads in every other language that has one. an +/// absolute path is refused because it names a place on the machine that +/// happened to build the program. +pub fn resolve_static_resource( + db: &dyn Db, + importing_file: File, + path: &str, +) -> Result { + if path.is_empty() || SystemPath::new(path).is_absolute() { + return Err(ResourceError::NotRelative); + } + + let FilePath::System(importing_path) = importing_file.path(db) else { + return Err(ResourceError::Unanchored); + }; + let Some(directory) = importing_path.parent() else { + return Err(ResourceError::Unanchored); + }; + + let resolved = SystemPath::absolute(path, directory); + if resource_format(resolved.extension()).is_none() { + return Err(ResourceError::UnsupportedFormat(resolved.to_string())); + } + + system_path_to_file(db, &resolved).map_err(|_| ResourceError::NotFound(resolved)) +} + +/// the python a resource file stands for, bound to `binding`. +/// +/// the file's own rendering — the one the type checker infers — binds the +/// document to a name taken from the file name. a caller that needs it under +/// another name, which is what an `as` clause asks for, gets it rendered again +/// rather than aliased, so the class a reader sees in the emitted python is +/// called what the import called it. +pub fn render_as(db: &dyn Db, file: File, binding: &str) -> Result { + let path = file.path(db); + let Some(format) = resource_format(path.extension()) else { + return Err(ResourceError::UnsupportedFormat(path.to_string())); + }; + + let text = source_text(db, file); + by_resource::transpile(format, text.as_str(), binding) + .map_err(|error| ResourceError::Unreadable(error.to_string())) +} + +/// the rendering the type checker infers for a resource file, and the file that +/// holds it. +/// +/// the rendering is a module in its own right, at a path of its own, so a range +/// in it names text that exists. reading a resource through the document's own +/// file would hand every consumer of a range — a diagnostic, a hover, a +/// definition to jump to — a position in text it is not looking at. +pub(crate) fn rendered(db: &dyn Db, document: File) -> Result<(File, &Rendered), ResourceError> { + match resource(db, document) { + Resource::Rendered(rendered) => Ok((resource_module(db, document), rendered)), + Resource::Unreadable(message) => Err(ResourceError::Unreadable(message.clone())), + Resource::NotAResource => Err(ResourceError::UnsupportedFormat( + document.path(db).to_string(), + )), + } +} diff --git a/crates/ty_python_semantic/src/types/template.rs b/crates/ty_python_semantic/src/types/template.rs index 9e2ff99627..9211971fc9 100644 --- a/crates/ty_python_semantic/src/types/template.rs +++ b/crates/ty_python_semantic/src/types/template.rs @@ -13,6 +13,7 @@ //! the constructor normalizes, so the variant only ever exists in a form that //! nothing else can already spell: //! +//! - a hole spelled as a type alias is the type that alias stands for //! - a hole whose type renders to one string is folded into the text, so a //! template with no holes left is a plain string literal //! - a union hole distributes, so `f"{Literal[1, 2]}"` is `Literal["1", "2"]` @@ -157,6 +158,17 @@ impl<'db> TemplateLiteralType<'db> { parts: Vec>, promotable: Promotable, ) -> Type<'db> { + // normalization reads a hole's own shape — whether it is a union to + // distribute, whether it renders to one string — and a type alias + // answers none of those questions for the type it stands for + let parts: Vec> = parts + .into_iter() + .map(|part| match part { + TemplatePart::Hole(hole) => TemplatePart::Hole(resolved_hole(db, env, hole)), + text @ TemplatePart::Text(_) => text, + }) + .collect(); + if parts .iter() .any(|part| matches!(part, TemplatePart::Hole(Type::Never))) @@ -164,20 +176,6 @@ impl<'db> TemplateLiteralType<'db> { return Type::Never; } - // a hole that is itself a pattern is that pattern spliced in: the string - // it stands for is built the same way, one level down - let parts: Vec> = parts - .into_iter() - .flat_map(|part| match part { - TemplatePart::Hole(Type::LiteralValue(literal)) - if let Some(nested) = literal.as_template() => - { - nested.parts(db).to_vec() - } - other => vec![other], - }) - .collect(); - let alternatives = distribute(db, env, &parts); let arms: Vec> = alternatives .into_iter() @@ -195,6 +193,22 @@ impl<'db> TemplateLiteralType<'db> { parts: Vec>, promotable: Promotable, ) -> Type<'db> { + // a hole that is itself a pattern is that pattern spliced in: the string + // it stands for is built the same way, one level down. this waits until + // after distribution, because a hole is only ever a single pattern once + // the union it may have been written as has been split into branches + let parts: Vec> = parts + .into_iter() + .flat_map(|part| match part { + TemplatePart::Hole(Type::LiteralValue(literal)) + if let Some(nested) = literal.as_template() => + { + nested.parts(db).to_vec() + } + other => vec![other], + }) + .collect(); + let mut folded: Vec> = Vec::with_capacity(parts.len()); for part in parts { let part = match part { @@ -340,6 +354,50 @@ impl<'db> TemplateLiteralType<'db> { } } +/// the type a hole really stands for. +/// +/// a hole may be written as a type alias, and a union written in a type +/// expression keeps its arms' aliases — [`UnionType::from_elements_leave_aliases`] +/// builds it that way so a union displays the names it was written with. so +/// reaching the union that [`distribute`] has to see, or the pattern that gets +/// spliced in, can take more than the one step [`Type::resolve_type_alias`] takes +fn resolved_hole<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + hole: Type<'db>, +) -> Type<'db> { + resolve_hole_arms(db, env, hole, &mut FxHashSet::default()) +} + +/// [`resolved_hole`], carrying the unions already being expanded further up. +/// +/// `type Cyc = Cyc | "q"` gives a union holding the very alias that produced it, +/// so following the arms of a union is not on its own a descent — without +/// remembering what is already open, that alias expands forever. the unions are +/// dropped again on the way out, so an arm that appears twice in different +/// branches is still expanded in both +fn resolve_hole_arms<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + hole: Type<'db>, + open: &mut FxHashSet>, +) -> Type<'db> { + let resolved = hole.resolve_type_alias(db); + let Type::Union(union) = resolved else { + return resolved; + }; + if !open.insert(resolved) { + return resolved; + } + let arms: Vec> = union + .elements(db) + .iter() + .map(|arm| resolve_hole_arms(db, env, *arm, open)) + .collect(); + open.remove(&resolved); + UnionType::from_elements(db, env, arms) +} + /// whether a hole type is itself a set of strings, so a pattern that is nothing /// but that hole is that type fn is_string_set<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, hole: Type<'db>) -> bool { diff --git a/crates/ty_python_semantic/tests/corpus.rs b/crates/ty_python_semantic/tests/corpus.rs index 771c5b31cf..de873f8b83 100644 --- a/crates/ty_python_semantic/tests/corpus.rs +++ b/crates/ty_python_semantic/tests/corpus.rs @@ -9,8 +9,8 @@ use ty_python_core::program::ProgramSettings; use ty_python_semantic::lint::{LintRegistry, RuleSelection}; use ty_python_semantic::pull_types::pull_types; use ty_python_semantic::{ - AnalysisSettings, Db as _, PythonVersionWithSource, TypeCheckingPreset, check_file_unwrap, - default_lint_registry, + AnalysisSettings, Db as _, ExperimentalSettings, PythonVersionWithSource, TypeCheckingPreset, + check_file_unwrap, default_lint_registry, }; use ruff_db::diagnostic::Diagnostic; @@ -147,6 +147,7 @@ pub struct CorpusDb { system: TestSystem, vendored: VendoredFileSystem, analysis_settings: Arc, + experimental_settings: ExperimentalSettings, program_settings: ProgramSettings, } @@ -165,6 +166,11 @@ impl CorpusDb { ), files: Files::default(), analysis_settings: Arc::new(AnalysisSettings::default()), + // the corpus is checked with every basedpython feature on + experimental_settings: ExperimentalSettings { + module_api: true, + build_stamps: true, + }, program_settings, } } @@ -250,6 +256,10 @@ impl ty_python_semantic::Db for CorpusDb { &self.analysis_settings } + fn experimental_settings(&self) -> &ExperimentalSettings { + &self.experimental_settings + } + fn dyn_clone(&self) -> Box { Box::new(self.clone()) } diff --git a/crates/ty_server/src/server/api/requests/completion.rs b/crates/ty_server/src/server/api/requests/completion.rs index d4c2a4d4db..72d15306dc 100644 --- a/crates/ty_server/src/server/api/requests/completion.rs +++ b/crates/ty_server/src/server/api/requests/completion.rs @@ -6,6 +6,7 @@ use lsp_types::{ CompletionParams, CompletionRequest, CompletionResponse, Documentation, InsertTextFormat, TextEdit, Uri, }; +use ruff_diagnostics::Edit; use ruff_source_file::OneIndexed; use ruff_text_size::Ranged; use ty_ide::{ @@ -105,7 +106,7 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { .ty .map(|ty| ty.display(db, &env).to_string()) .or_else(|| comp.detail.as_ref().map(ToString::to_string)); - let import_edit = comp.import.as_ref().and_then(|edit| { + let to_text_edit = |edit: &Edit| { let range = edit .range() .to_lsp_range(db, file, snapshot.encoding())? @@ -114,7 +115,16 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { range, new_text: edit.content().map(ToString::to_string).unwrap_or_default(), }) - }); + }; + let import_edit = comp.import.as_ref().and_then(to_text_edit); + // an import is not the only edit a completion can need: a + // name completed inside a plain string also brings the `f` + // that makes the string an f-string + let additional_text_edits: Vec = import_edit + .iter() + .cloned() + .chain(comp.additional_edit.as_ref().and_then(to_text_edit)) + .collect(); let label = comp.label().to_string(); let import_suffix = comp.module_name.and_then(|name| { @@ -179,7 +189,8 @@ impl BackgroundDocumentRequestHandler for CompletionRequestHandler { insert_text_format, filter_text: comp.filter.map(String::from), text_edit, - additional_text_edits: import_edit.map(|edit| vec![edit]), + additional_text_edits: (!additional_text_edits.is_empty()) + .then_some(additional_text_edits), documentation, command: comp .command diff --git a/crates/ty_server/src/session/options.rs b/crates/ty_server/src/session/options.rs index 563a93858f..d472c71796 100644 --- a/crates/ty_server/src/session/options.rs +++ b/crates/ty_server/src/session/options.rs @@ -427,7 +427,8 @@ impl CompletionOptions { fn into_settings(self) -> CompletionSettings { CompletionSettings { auto_import: self.auto_import.unwrap_or(true), - complete_function_parentheses: self.complete_function_parentheses.unwrap_or(false), + // basedpython defaults this on — see `CompletionSettings::default` + complete_function_parentheses: self.complete_function_parentheses.unwrap_or(true), } } } diff --git a/crates/ty_server/tests/e2e/completions.rs b/crates/ty_server/tests/e2e/completions.rs index 65f7462a1d..b0910dcdd4 100644 --- a/crates/ty_server/tests/e2e/completions.rs +++ b/crates/ty_server/tests/e2e/completions.rs @@ -31,7 +31,7 @@ walktr "label": "walktree (import inspect)", "kind": 3, "sortText": "0", - "insertText": "walktree", + "insertText": "walktree()", "additionalTextEdits": [ { "range": { @@ -79,8 +79,11 @@ walktr Ok(()) } +/// basedpython enables callable parentheses by default where upstream ty leaves +/// them off, so the no-options case is the *enabled* one — see +/// `CompletionSettings::default`. #[test] -fn complete_function_parentheses_disabled_by_default() -> Result<()> { +fn complete_function_parentheses_enabled_by_default() -> Result<()> { let workspace_root = SystemPath::new("src"); let foo = SystemPath::new("src/foo.py"); let foo_content = "\ @@ -106,7 +109,9 @@ complete_parenth "label": "complete_parentheses", "kind": 3, "detail": "def complete_parentheses()", - "sortText": "0" + "sortText": "0", + "insertText": "complete_parentheses($0)", + "insertTextFormat": 2 } ] "#); @@ -343,7 +348,7 @@ TypedDi "label": "typing.is_typeddict", "kind": 3, "sortText": "1", - "insertText": "typing.is_typeddict" + "insertText": "typing.is_typeddict()" }, { "label": "TypedDict (import typing_extensions)", @@ -370,7 +375,7 @@ TypedDi "label": "TypedDictFallback (import _typeshed._type_checker_internals)", "kind": 7, "sortText": "3", - "insertText": "TypedDictFallback", + "insertText": "TypedDictFallback()", "additionalTextEdits": [ { "range": { @@ -391,7 +396,7 @@ TypedDi "label": "is_typeddict (import typing_extensions)", "kind": 3, "sortText": "4", - "insertText": "is_typeddict", + "insertText": "is_typeddict()", "additionalTextEdits": [ { "range": { @@ -412,7 +417,7 @@ TypedDi "label": "_FilterConfigurationTypedDict (import logging.config)", "kind": 7, "sortText": "5", - "insertText": "_FilterConfigurationTypedDict", + "insertText": "_FilterConfigurationTypedDict()", "additionalTextEdits": [ { "range": { @@ -526,7 +531,7 @@ TypedDi "label": "is_typeddict (import typing)", "kind": 3, "sortText": "1", - "insertText": "is_typeddict", + "insertText": "is_typeddict()", "additionalTextEdits": [ { "range": { @@ -568,7 +573,7 @@ TypedDi "label": "TypedDictFallback (import _typeshed._type_checker_internals)", "kind": 7, "sortText": "3", - "insertText": "TypedDictFallback", + "insertText": "TypedDictFallback()", "additionalTextEdits": [ { "range": { @@ -589,7 +594,7 @@ TypedDi "label": "is_typeddict (import typing_extensions)", "kind": 3, "sortText": "4", - "insertText": "is_typeddict", + "insertText": "is_typeddict()", "additionalTextEdits": [ { "range": { @@ -610,7 +615,7 @@ TypedDi "label": "_FilterConfigurationTypedDict (import logging.config)", "kind": 7, "sortText": "5", - "insertText": "_FilterConfigurationTypedDict", + "insertText": "_FilterConfigurationTypedDict()", "additionalTextEdits": [ { "range": { @@ -714,7 +719,8 @@ re.match('', '', fla "kind": "plaintext", "value": "Floating-point operation failed.\n" }, - "sortText": "1" + "sortText": "1", + "insertText": "FloatingPointError()" }, { "label": "PythonFinalizationError", @@ -724,7 +730,8 @@ re.match('', '', fla "kind": "plaintext", "value": "Operation blocked during Python finalization.\n" }, - "sortText": "2" + "sortText": "2", + "insertText": "PythonFinalizationError()" }, { "label": "float", @@ -734,7 +741,8 @@ re.match('', '', fla "kind": "plaintext", "value": "Convert a string or number to a floating-point number, if possible.\n" }, - "sortText": "3" + "sortText": "3", + "insertText": "float()" } ] "#); @@ -782,6 +790,74 @@ x: Literal[\"apple\"] = \"app\" Ok(()) } +/// A name completed inside a plain string's braces reaches the client with the +/// `f` that makes the string an f-string as a further edit of its own. +#[test] +fn string_field_completion_carries_the_fstring_prefix() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +class Thing: + def __str__(self) -> str: ... + +zqzqzq = Thing() +\"hello {zqzq\" +"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options(ClientOptions::default().with_auto_import(false)) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, foo_content, 1); + + let completions = server.completion_request(&server.file_uri(foo), Position::new(4, 12)); + + insta::assert_json_snapshot!(completions, @r#" + [ + { + "label": "zqzqzq", + "kind": 22, + "detail": "Thing", + "sortText": "0", + "insertText": "zqzqzq}", + "textEdit": { + "range": { + "start": { + "line": 4, + "character": 8 + }, + "end": { + "line": 4, + "character": 12 + } + }, + "newText": "zqzqzq}" + }, + "additionalTextEdits": [ + { + "range": { + "start": { + "line": 4, + "character": 0 + }, + "end": { + "line": 4, + "character": 0 + } + }, + "newText": "f" + } + ] + } + ] + "#); + + Ok(()) +} + /// The type shown beside a suggestion is read as source, so in a basedpython /// file it is spelled the way it would be written there. #[test] diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index 9b87266767..08c3fdef4e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -40,7 +40,7 @@ Settings: WorkspaceSettings { }, completions: CompletionSettings { auto_import: true, - complete_function_parentheses: false, + complete_function_parentheses: true, }, configuration_file: None, override_options: None, @@ -254,6 +254,10 @@ Settings: Settings { "builtins.classmethod", ], }, + experimental: ExperimentalSettings { + module_api: false, + build_stamps: false, + }, editor: EditorSettings { common_aliases: [], }, diff --git a/crates/ty_test/src/config.rs b/crates/ty_test/src/config.rs index b863f8d051..3e30217156 100644 --- a/crates/ty_test/src/config.rs +++ b/crates/ty_test/src/config.rs @@ -40,6 +40,10 @@ pub(crate) struct MarkdownTestConfig { pub(crate) analysis: Option, + /// The experimental features the test opts in to, as `[tool.ty.experimental]` + /// does for a project. + pub(crate) experimental: Option, + /// The [`ruff_db::system::System`] to use for tests. /// /// Defaults to the case-sensitive [`ruff_db::system::InMemorySystem`]. @@ -216,6 +220,16 @@ pub(crate) struct Environment { python: Option, } +#[derive(Deserialize, Debug, Default, Clone)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) struct Experimental { + /// see [`ty_python_semantic::ExperimentalSettings::module_api`] + pub(crate) module_api: Option, + + /// see [`ty_python_semantic::ExperimentalSettings::build_stamps`] + pub(crate) build_stamps: Option, +} + #[derive(Deserialize, Default, Debug, Clone)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub(crate) struct Analysis { diff --git a/crates/ty_test/src/db.rs b/crates/ty_test/src/db.rs index 42b7f96612..71dd52db3f 100644 --- a/crates/ty_test/src/db.rs +++ b/crates/ty_test/src/db.rs @@ -1,4 +1,4 @@ -use crate::config::{Analysis, Rules, ScriptOptions}; +use crate::config::{Analysis, Experimental, Rules, ScriptOptions}; use camino::{Utf8Component, Utf8PathBuf}; use ruff_db::Db as SourceDb; use ruff_db::diagnostic::{Diagnostic, Severity}; @@ -20,8 +20,8 @@ use ty_python_core::{Db as _, ProgramFile, TestProgramDb}; use ty_python_semantic::dependencies::DependencyManifest; use ty_python_semantic::lint::{LintRegistry, RuleSelection}; use ty_python_semantic::{ - AnalysisSettings, Db as SemanticDb, PythonVersionWithSource, TypeCheckingPreset, - check_file_unwrap, default_lint_registry, django_settings, + AnalysisSettings, Db as SemanticDb, ExperimentalSettings, PythonVersionWithSource, + TypeCheckingPreset, check_file_unwrap, default_lint_registry, django_settings, }; #[salsa::db] @@ -96,6 +96,22 @@ impl Db { } } + pub(crate) fn update_experimental_options(&mut self, options: Option<&Experimental>) { + let experimental = ExperimentalSettings { + module_api: options + .and_then(|options| options.module_api) + .unwrap_or_default(), + build_stamps: options + .and_then(|options| options.build_stamps) + .unwrap_or_default(), + }; + + let settings = self.settings(); + if settings.experimental(self) != &experimental { + settings.set_experimental(self).to(experimental); + } + } + pub(crate) fn update_dependency_manifest(&mut self, manifest: Option) { let settings = self.settings(); if settings.dependency_manifest(self) != &manifest { @@ -211,6 +227,10 @@ impl SemanticDb for Db { file_settings(self, file).analysis(self) } + fn experimental_settings(&self) -> &ExperimentalSettings { + self.settings().experimental(self) + } + fn dependency_manifest(&self, _file: File) -> Option<&DependencyManifest> { self.settings().dependency_manifest(self).as_ref() } @@ -293,6 +313,9 @@ struct Settings { #[returns(ref)] analysis: AnalysisSettings, #[default] + #[returns(ref)] + experimental: ExperimentalSettings, + #[default] #[returns(deref)] rule_selection: MdtestRuleSelection, #[default] diff --git a/crates/ty_test/src/lib.rs b/crates/ty_test/src/lib.rs index e75dc4a2d1..ba9bcf3526 100644 --- a/crates/ty_test/src/lib.rs +++ b/crates/ty_test/src/lib.rs @@ -172,9 +172,9 @@ fn run_test( assert!( matches!( embedded.lang, - "py" | "pyi" | "python" | "ipynb" | "by" | "byi" | "bython" | "basedpython" | "text" | "cfg" | "pth" + "py" | "pyi" | "python" | "ipynb" | "by" | "byi" | "bython" | "basedpython" | "text" | "cfg" | "pth" | "json" | "toml" | "yaml" ), - "Supported file types are: py (or python), pyi, ipynb, by, bython, basedpython, byi, text, cfg, pth and ignore" + "Supported file types are: py (or python), pyi, ipynb, by, bython, basedpython, byi, text, cfg, pth, json, toml, yaml and ignore" ); let mut full_path = embedded.full_path(&project_root); @@ -310,6 +310,7 @@ fn run_test( db.update_program(settings); let preset = configuration.type_checking_preset(); db.update_analysis_options(preset, configuration.analysis.as_ref()); + db.update_experimental_options(configuration.experimental.as_ref()); db.update_dependency_manifest(configuration.dependency_manifest()); db.update_mdtest_rule_selection( preset, diff --git a/crates/ty_wasm/src/lib.rs b/crates/ty_wasm/src/lib.rs index d3e351ecf2..db3f7b763b 100644 --- a/crates/ty_wasm/src/lib.rs +++ b/crates/ty_wasm/src/lib.rs @@ -555,25 +555,33 @@ impl Workspace { let name = comp.label().to_string(); let kind = comp.kind.map(CompletionKind::from); let type_display = comp.ty.map(|ty| ty.display(&self.db, &env).to_string()); - let import_edit = comp.import.as_ref().map(|edit| { - let range = Range::from_text_range( - edit.range(), - &index, - &source, - self.position_encoding, - ); - TextEdit { - range, - new_text: edit.content().map(ToString::to_string).unwrap_or_default(), - } - }); + // an import is not the only edit a completion can need: a name + // completed inside a plain string also brings the `f` that + // makes the string an f-string + let edits: Vec = comp + .import + .iter() + .chain(comp.additional_edit.iter()) + .map(|edit| { + let range = Range::from_text_range( + edit.range(), + &index, + &source, + self.position_encoding, + ); + TextEdit { + range, + new_text: edit.content().map(ToString::to_string).unwrap_or_default(), + } + }) + .collect(); Completion { name, kind, detail: type_display, module_name: comp.module_name.map(ToString::to_string), insert_text: comp.insert.map(String::from), - additional_text_edits: import_edit.map(|edit| vec![edit]), + additional_text_edits: (!edits.is_empty()).then_some(edits), documentation: comp .documentation .map(|docstring| docstring.render_plaintext()), diff --git a/docs/basedpython/configuration.md b/docs/basedpython/configuration.md index 2a031e1e05..1be77a3ce9 100644 --- a/docs/basedpython/configuration.md +++ b/docs/basedpython/configuration.md @@ -25,6 +25,31 @@ override-raise = "ignore" a project has one configuration, not one per command: `by check`, `by run` and `by build` all read the same options +## experimental features + +a feature under `[experimental]` is still being designed. it is off unless the +project asks for it by name, and it may change or be withdrawn without the +deprecation period a settled feature gets — opting in says you would rather have +the feature than that guarantee: + +```toml +# basedpython.toml +[experimental] +module-api = true +``` + +| option | what it turns on | +| -------------- | ---------------------------------------------------------------- | +| `module-api` | [`implements` declarations](features/module-api.md) are enforced | +| `build-stamps` | a [`build:` block](features/build-stamps.md) declares stamps | + +the opt-in is per project, not per file: an experimental feature is a language +feature, and a module's meaning cannot depend on which file is asking about it. so +`[[overrides]]` does not read this section + +writing an experimental construct without opting in is an error rather than a +no-op, so nothing silently does nothing + ## the preset `type-checking-preset` supplies the defaults that `rules` and `analysis` start from: diff --git a/docs/basedpython/development/module-api-design.md b/docs/basedpython/development/module-api-design.md new file mode 100644 index 0000000000..b034a1a313 --- /dev/null +++ b/docs/basedpython/development/module-api-design.md @@ -0,0 +1,467 @@ +# module api enforcement + +## summary + +a module is already a structural value: ty models it as `Type::ModuleLiteral`, +and a module literal already satisfies a protocol through its public surface, so + +```by +backend: Backend = postgres +``` + +type-checks today when `postgres` has the members `Backend` asks for. what is +missing is a way to **attach that obligation to a module permanently**, so a +break is reported in the file that broke it rather than in whichever consumer +happened to assign it — or nowhere at all, when no consumer does + +one statement attaches one, and it has two forms: + +```by +implements Backend # this module answers `Backend` +implements Backend for ".*" # every submodule of this package does +``` + +the second form is the one that matters for a plugin directory. written in +`backends/__init__.by`, it holds for every module in `backends`, including one +added tomorrow by someone who never read the interface, and it cannot be dropped +by deleting a line in the module that fails it + +## what already works + +- a module literal answers protocol members through + `ModuleLiteralType::static_member` in `types.rs` +- a module-level function satisfies an instance or static method member — + `implementation_access` in `types/protocol_class.rs` deliberately skips the + class-side check for `Type::ModuleLiteral`, and `protocols.md` has the test + (*module objects with static-method protocol members*) +- `crates/ty_python_semantic/src/api_lockfile.rs` already owns the definition of + a module's *public* surface: `is_public_module_symbol`, `__all__` via + `dunder_all_names`, the public-by-convention dunder allowlist +- `ty_module_resolver::ModuleGlobSet` already owns module-name globs — `*` within + a component, `**` across components, `!` to exclude, last match wins. it is + what `allowed-unresolved-imports` matches with, and it is documented +- `Module::all_submodules` and `ruff_db::files::directory_listing` already + enumerate a package's contents as a tracked input, so "which modules does this + rule reach" is an ordinary incremental query rather than a filesystem walk + bolted onto the checker + +so the checking engine and the glob language both exist. what this feature adds +is a declaration, a way to find it from the module it governs, and a direction of +blame + +### where it falls short today + +- **no obligation.** a plugin module that stops matching its interface is only + caught if something assigns it to a protocol-typed place. a plugin loaded by + name never is +- **blame points the wrong way.** the error lands on the consumer's assignment, + which is often in a different package from the mistake +- **nothing is imposable.** an interface's author cannot state a requirement that + holds over a directory. every module has to volunteer + +## writing the interface + +no new syntax is required. a module's members are unbound, so the interface +spells them `static`, which basedpython already has: + +```by +protocol Backend: + name: str + static def connect(url: str) -> Connection +``` + +a plain `def connect(self, url: str) -> Connection` also works — instance access +strips `self`, so a module-level function matches the bound signature — but it +reads as a lie about a module, and it makes the protocol usable only in that +direction + +### optional sugar: `module protocol` + +`module protocol P:` is a protocol every one of whose members is `static`: + +```by +module protocol Backend: + name: str + def connect(url: str) -> Connection +``` + +it is *only* that desugaring. in particular it does not restrict what may +satisfy it: a class object satisfies a static-membered protocol just as a module +does, which is what lets a test substitute a fake: + +```by +class FakeBackend: + name = "fake" + static def connect(url: str) -> Connection: ... + +def run(backend: Backend): ... + +run(FakeBackend) # ok +``` + +the sugar is cheap (a flag on the existing `protocol_class` path) but it is not +load-bearing — it can come last, or never + +## the statement + +```by +# backends/__init__.by +from .api import Backend + +implements Backend for ".*", "!.base" +``` + +- **without `for`**, the statement obliges the module it is written in +- **with `for`**, it obliges the modules its patterns name, and says nothing + about the file it is written in + +a rule may name several protocols, and several rules may reach one module: +obligations accumulate, because two rules naming two protocols is the normal way +to say a module must answer both + +### the patterns + +they are `ModuleGlobSet` patterns — the existing matcher, the existing semantics, +the existing documentation — with a leading `.` marking them relative to the +declaring package, exactly as a relative import does. so in `app/__init__.by`: + +| pattern | reaches | +| --------------- | ------------------------------------------------------------------------ | +| `".*"` | every direct submodule — `app.home`, not `app.blog.home` | +| `".**"` | the whole subtree | +| `".pages.*"` | the direct submodules of `app.pages` | +| `".**.pages.*"` | a `pages` package at any depth — `app.pages.home`, `app.blog.pages.home` | +| `".handler_*"` | one component, matched by name | +| `"!.base"` | carves `app.base` back out | + +`*` matches within one component, `**` matches zero or more whole components and +must stand alone as one, and `!` excludes — all of that is the existing matcher's +behaviour and none of it is new here. `!` goes before the dot, because it negates +the pattern rather than being part of the path + +a pattern may not climb: `"..sibling"` is `invalid-module-api`. a rule that +reached outside its own package would break the ownership boundary, and it would +also be unfindable — a module looks for rules in its ancestors, and a rule +imposing sideways is not in one + +an absolute pattern (no leading dot) is rejected for the same reason. the +spelling is reserved rather than repurposed, so that a project-wide rule table, +if one is ever wanted, can use it + +### where a rule may live, and why + +**in a package's `__init__.by`, governing that package's subtree.** an absolute +pattern, or a rule in a file that is not a package `__init__`, is +`invalid-module-api` + +this is the whole design, so it is worth saying why it is not "anywhere". the +obligations of a module have to be discoverable **from that module**, or the +error will not appear in the file the author is editing (see *where the error +lands*). so something has to index rules by the module they govern, and the index +has to be cheap: + +- **anywhere in the project** means a project-wide scan, and then every file's + check depends on every file's declaration set. one edit anywhere + reinvalidates everything +- **through the import graph**, the way `extension` and conformance visibility + work, is the wrong relation here: the whole point is imposing on a module that + does not cooperate, and a module that does not cooperate does not import you +- **through containment** costs a bounded walk. the obligations of `a.b.c` are + found in `a/__init__.by` and `a/b/__init__.by`, two files the module resolver + already touches to resolve `a.b.c` at all + +containment also draws the ownership line in the right place: you may impose on +modules inside your own package, and you may not impose on someone else's. that +is the same boundary `sealed` already uses, and it means a dependency cannot +reach into your tree and add requirements to it + +### the private default + +`".*"` does not match a submodule whose name starts with `_`. a leading underscore +already means "not part of the surface" everywhere else in this project, and a +private helper module sitting next to the plugins is the common case, not the +exception. naming one exactly (`for "._special"`) still reaches it + +### rules that match nothing + +a typo'd pattern is a rule that silently enforces nothing, which is the worst +failure mode a checker can have. a rule that matches no module in its package is +reported at the rule, using `Module::all_submodules` on the declaring package — a +walk of the package, not of the project. + +with one deliberate hole: a package whose submodules the resolver cannot +enumerate is never accused. that is a namespace portion inside the package, whose +contents span directories and search paths and are not enumerated at all, so +"nothing matched" there means "nothing was visible" rather than "nothing +matched". accusing a rule that does enforce is worse than missing a typo. + +## what is checked + +the candidate member set is the module's public surface as `api_lockfile.rs` +already defines it, so the two features cannot drift: + +- a symbol whose simple name starts with `_` does not count, unless it is one of + the public-by-convention dunders +- `__all__`, when present, is the surface +- a submodule counts as a member only where an ordinary attribute access would + resolve it — the existing `available_submodule_attributes` rule, unchanged +- a re-export (`from .impl export Widget`) counts; a plain private import in a + stub does not. this is the existing distinction, not a new one + +nothing is checked about members the interface does not mention. a module may +expose whatever else it likes; `_`-prefixing and `__all__` are how a module says +a name is not part of its surface, and they are enough + +### `__getattr__` defeats it + +a module defining `__getattr__` answers every name, so every requirement would be +vacuously met. that is a silent pass, and a silent pass is worse than no check: +a module with a module-level `__getattr__` that carries an obligation is +`invalid-module-api`, however the obligation was attached + +### stubs + +when a module has a `.byi` stub, the stub is what importers see, so the stub is +the surface an obligation is about — and the file its declarations belong in. an +`implements` in an implementation file the stub shadows is `invalid-module-api`, +naming the stub. + +the alternative, checking each file against its own surface, is worse in three +ways at once, and all three were live before this rule existed: a module that +satisfies its stub but not its implementation reported nothing at all (the error +surfaced on a consumer instead); a module that satisfied its implementation but +not its stub reported an error nobody else could see the cause of; and a package +rule over a module with both files reported the same obligation twice. the same +shadowing also silently disabled every rule in a `__init__.by` sitting beside a +`__init__.byi`. + +## where the error lands + +in the module that failed, always — an obligation imposed from a package is still +the failing module's problem to fix. the anchor depends on what is available: + +- a member with the wrong type — on that member's own definition +- a missing member — there is nothing to point at, so a file-level primary + annotation (`Span::from(file)`, as `fixes.rs` already does) at the top of the + file +- a bare `implements` in the file itself — on the statement + +every such diagnostic carries a secondary annotation on the **rule that imposed +it**, in whichever `__init__.by` it was written. an error saying a module must +have `connect` is useless without saying who says so, and unlike a config file +the rule is a real declaration a reader can jump to + +## the demanding side + +three cases, in increasing order of how much new machinery they need — only the +first is part of this feature, see *what is left out* + +**1. a protocol-typed parameter.** works today, no new work: + +```by +def migrate(backend: Backend): ... +migrate(postgres) +``` + +**2. requiring an actual module.** `module T` as a third use-site type modifier +alongside `literal T` and `final T` (`types/restricted.rs`, `TypeModifier` in +`ruff_python_ast::helpers`), accepting only a `Type::ModuleLiteral` assignable to +`T`. this exists for the reflective cases — a registry keyed by `__name__`, a +reloader — and is not needed for ordinary use + +**3. loading a plugin by name.** the case the whole feature is for, since a +dynamically chosen module is the one thing no assignment can check: + +```by +def import_module[P](name: str, api: type[P]) -> P +``` + +- when `name` is a literal, resolve the module and check it statically, and the + call costs nothing at runtime beyond the import +- otherwise, the transpiler emits a witness check at the call — the module's + members are verified against `P`'s requirements once, at load — reusing the + existing `_soundness_check` boundary machinery rather than inventing a second + runtime check + +a package rule and a dynamic loader cover the two halves of the same problem: the +rule checks the plugins that are in the project, the loader checks the ones that +arrive from outside it + +## lowering + +the statement erases in both forms. the emitted python keeps the import that +named the protocol — it may be needed by annotations, and dropping it would +change the emitted module's own surface — so a package whose `__init__` carries +rules pays a real import at runtime for a purely static declaration. a +[lazy import](../features/lazy-imports.md) of the interface is the answer where +that matters, and the feature doc should say so rather than leaving it to be +discovered + +phase 3 must not see either form survive + +the reverse direction produces nothing: python has no idiom that means "this +module implements this interface", so `reverse_transforms` has no rule to add. a +heuristic — a module whose names happen to match some protocol — would invent an +obligation the author never wrote, which is exactly the kind of guess this +project does not make + +## implementation map + +| layer | work | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| parser | `implements` as a soft keyword at statement start, recognised only when followed by a name or dotted name, gated on `PySourceType::BasedPython` — the `extension` / `protocol` dispatch in `parser/statement.rs` is the model. the `for` clause takes string literals | +| ast | no new node: the declaration parses to a call to the synthetic `__implements__` marker, the encoding `newtype` and the modifier declarations already use. see *alternatives* | +| semantic index | each interface named is a **load** of that name, so an import used only by the declaration is not unused (`F401`) and an undefined one is `F821` | +| ty | `package_rules(db, file)` — syntactic, the rules an `__init__` declares — and `module_obligations(db, file)` — the ancestor walk plus the file's own bare statements — then the check over them | +| formatter | a printer for the node and a `.by` fixture. the formatter rebuilds from the ast, so a source-only form corrupts on reformat | +| linter | nothing, unless a name the declaration resolves becomes invisible to ruff's binder | +| lsp | semantic tokens for the keyword; the secondary annotation gives go-to-definition on the rule for free | +| api lockfile | one record per obligation, `:I=`, so an api contract is visible in the reviewed diff whichever way it was attached | +| docs | `docs/basedpython/features/module-api.md`, plus `features/index.md` and the `zensical.toml` nav (all three, `scripts/check_docs_nav.py` enforces it) | + +## salsa and cost + +- **the fast negative is syntactic.** `package_rules` reads the semantic index of + one `__init__` and needs no inference, so a package that declares nothing + answers immediately and the ancestor walk is a handful of interned lookups. + only a module that actually carries an obligation pays for resolving a + protocol +- **the check is a cycle.** resolving the protocol a rule names infers the + `__init__`'s module-level code, and an `__init__` routinely imports the very + submodules its rules govern. `module_obligations` takes + `cycle_initial = no obligations`, the same recovery the conformance registry + uses for the same reason +- **no lookup rule changes.** nothing here touches module member lookup, so no + cost lands on programs that use none of this + +## diagnostics + +| id | when | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `unmet-module-api` | an attached interface has a requirement the module does not answer, or answers with the wrong type. error, default on | +| `invalid-module-api` | the declaration itself is wrong: the name is not a protocol, an abstract class, an absolute pattern, a rule outside a package `__init__`, a module with `__getattr__`, a rule that matches nothing | + +the `unmet-module-api` message should say which member, what shape was required, +and what the module has instead — the three-part form `types/conformance.rs` +already uses for a conformance that fails to answer a requirement + +## the opt-in + +the feature is off unless a project names it under `[experimental]`. the gate sits +at the top of `check_module_api`, which is the whole of it: the parser still +parses a declaration and the transpiler still erases one, because a project that +turns the feature off after writing `implements` would otherwise emit python +carrying a call to a name that does not exist at runtime. + +what the gate does *not* do is ignore the declaration. a declaration written while +the feature is off is reported, with the opt-in in the help — an obligation +nothing checks is the exact failure this feature exists to remove, and introducing +one at the switch would be a poor joke. + +the flag is project-wide and is not part of `AnalysisSettings`, so +`[[overrides]]` cannot vary it: an analysis setting says how to read code that is +already understood, while this says whether a language feature exists, and a +module's meaning must not depend on which file is asking. + +## what is left out + +the enforcement itself — the declaration, the rule, and the checks — is the whole +of what is described above. two adjacent things the same vocabulary suggests are +deliberately not part of it: + +- **requiring an actual module.** `module T` as a third use-site type modifier + alongside `literal T` and `final T` (`types/restricted.rs`, `TypeModifier` in + `ruff_python_ast::helpers`), accepting only a `Type::ModuleLiteral` assignable + to `T`. it is for the reflective cases — a registry keyed by `__name__`, a + reloader — and an ordinary protocol-typed parameter already accepts a module + without it +- **the typed dynamic import**, `import_module[P](name, api=P)`, which would check + a literal module path statically and emit a runtime witness otherwise. it is + the other half of the plugin problem — this feature checks the plugins that + are in the project, that would check the ones that arrive from outside it — + but it lands in the transpiler's runtime rather than in the type checker, and + it is a feature of its own + +`module protocol P:` — sugar for a protocol whose members are all `static` — is +also left out. `static def` says it today, and the sugar buys nothing but the +keyword + +## alternatives considered + +- **rules in `pyproject.toml`**, `[[tool.ty.module-api]]` with a `protocol = "backends.api.Backend"` path and a module glob list. the config machinery is + all there — array-of-tables like `[[tool.ty.overrides]]`, module globs like + `allowed-unresolved-imports`, `RangedValue` for anchoring a diagnostic on the + line — and it can impose across package boundaries, which the source form + deliberately cannot. it was the earlier draft of this document and it is + rejected because the protocol becomes a **string**: no go-to-definition, no + rename, no import edge, no way for the checker to tell a typo from a module it + cannot see. an api contract belongs in the language, next to the interface it + names. left available as a later escape hatch if imposing across packages + turns out to be needed +- **`api P`, a lid on the module.** an earlier draft had a second statement that + both obliged the module and narrowed what importers could see, hiding public + members outside the protocol. it is not worth its weight: `_`-prefixing and + `__all__` already say "not part of my surface", a `.byi` stub already narrows + what importers see, and the lid would have put a new arm in module member + lookup — a hot path — to buy the difference between a name list and a type +- **checking from the rule site.** the rule enumerates its modules and reports + there, which needs no index at all. rejected because the squiggle then appears + in `backends/__init__.by` while the author is editing `backends/postgres.by`, + and in the language server it does not appear until the `__init__` is checked + again +- **a dedicated `StmtImplements` node.** the first draft of this document called + for one, on the grounds that a declaration is a statement and should be one in + the tree. what it missed is that the parser already has an encoding for exactly + this shape — a synthetic marker name with an `Invalid` expression context, + which `newtype`, `let` and the modifier declarations all use — and that the + marker being a *call* is what keeps the interfaces ordinary load expressions. + that is not a detail: it is what makes an import used only by a declaration + count as used, an interface that does not exist an undefined name, and ruff's + binder behave without being taught anything. a new node would have had to + re-earn all three +- **a marker-decorated `ClassDef`**, the trick `extension` and `protocol` use. a + declaration with no body is not a class in any sense, and every consumer — the + formatter, the semantic index, the ide — would have to un-tell the lie +- **an assignment instead of a statement**, `_: Backend = sys.modules[__name__]`. + it works today, which is the honest baseline. it also requires a runtime + import of `sys`, blames a line nobody reads as a declaration, cannot be + imposed from outside, and reads as dead code to every tool that does not know + better +- **a `.byi` stub as the contract.** a stub already narrows what importers see, + but it replaces the module's types wholesale and nothing checks the + implementation against it. that is a different tool: a stub describes, an + obligation checks +- **status quo, use-site checking only.** covered above under *where it falls + short* + +## a declaration cannot be wrapped + +the interfaces and patterns are a flat list with no bracket around them, so a long +declaration has nowhere to break and the formatter leaves it over the line width. +that is a consequence of the syntax rather than of the printer. it has not been a +problem — a rule with enough patterns to overflow a line is usually a rule that +wants splitting into two — but it is the reason the formatter has no wrapping +logic for the statement. + +## open questions + +- **namespace packages have no `__init__`**, so they cannot carry rules. giving + one an `__init__.by` is the answer, and is what a package that wants to impose + requirements on its contents should have anyway — but it is a real gap for a + plugin directory assembled across distributions, which is exactly where + namespace packages are used +- **is `".*"` the right default reach**, or should a bare `implements Backend for` with no patterns mean "every direct submodule"? the pattern list is one + token of ceremony against being explicit about a rule that imposes on files + its author may not have written +- **should a rule be able to demand a module exist?** a pattern matching nothing + is currently a diagnostic about the rule. for a plugin a package genuinely + requires, an error saying the module is missing might be what is wanted, which + is a different feature wearing the same syntax +- **write access.** a protocol data member is read-write, so a module satisfying + `name: str` must have a rebindable `name`. whether an immutable module-level + binding can answer a member declared read-only needs the same answer protocols + already give for `ReadOnly`, and should not get a second one +- **third-party modules.** a rule cannot reach outside its own package, and the + project does not check installed code anyway. the dynamic loader's runtime + witness is the only enforcement available for a plugin that arrives installed, + and that asymmetry should be stated in the feature doc rather than discovered diff --git a/docs/basedpython/features/build-stamps.md b/docs/basedpython/features/build-stamps.md new file mode 100644 index 0000000000..cf4dade388 --- /dev/null +++ b/docs/basedpython/features/build-stamps.md @@ -0,0 +1,172 @@ +# build stamps + +a `build:` block declares values the build settles when it produces the +artifact. each one is read as an attribute of `build`, at the type it declares: + +```by +build: + GIT_SHA: str + +def main(): + print(build.GIT_SHA) +``` + +```python +class build: + GIT_SHA: str = "e6f9ac1d4b2a7c3f9081be2d5a4c7e13f8b0d6a2" +def main(): + print(build.GIT_SHA) +if __name__ == "__main__": + main() +``` + +the value is fixed for the life of the artifact — every run of that build +reports the same commit — and `build.GIT_SHA` is a `str` to the checker, so a +misspelled stamp is an error rather than an attribute that is missing at +runtime + +the feature is [experimental](../configuration.md#experimental-features), so a +project asks for it by name: + +```toml +# basedpython.toml +[experimental] +build-stamps = true +``` + +a block written without opting in is an error rather than a no-op. it still +lowers, so a program that reads a stamp keeps working — which is why nothing at +the point of use would tell you the value was never settled + +## what the build supplies + +| stamp | type | is | +| ---------------- | ------ | -------------------------------------------------- | +| `GIT_SHA` | `str` | the commit `HEAD` names | +| `GIT_SHA_SHORT` | `str` | its first twelve characters | +| `GIT_BRANCH` | `str` | the branch checked out | +| `GIT_TAG` | `str` | the tag *on this commit*, empty when there is none | +| `GIT_DIRTY` | `bool` | whether the tree had uncommitted changes | +| `BUILT_AT` | `str` | when the build ran, RFC 3339 in UTC | +| `PYTHON_VERSION` | `str` | the python the output was lowered to | + +declare only what the program uses. a stamp the block does not name is not +computed into anything + +`GIT_DIRTY` is worth declaring wherever `GIT_SHA` is. a build from a tree with +uncommitted changes is not the commit it names, and a program that cannot say so +will eventually be asked to explain a traceback that does not match its source: + +```by +build: + GIT_SHA_SHORT: str + GIT_DIRTY: bool + +def version() -> str: + return build.GIT_SHA_SHORT + ("-dirty" if build.GIT_DIRTY else "") +``` + +## a stamp the build cannot supply + +a stamp with no default is a claim that the build has to produce it. when it +cannot — no `git` on the machine, or a source tree that is not a checkout, as an +exported tarball is — the transpile fails and says which stamp: + +```text +the build supplied no value for the stamp `GIT_SHA`, and it has no default +``` + +that failure is the point of writing the declaration down: nothing has to +remember to check, and no artifact goes out claiming a commit it does not have. +give the stamp a default where the program would rather carry on without it: + +```by +build: + GIT_SHA: str = "unreleased" +``` + +a default only stands in. when the build does supply a value, that value wins + +## stamps a project settles itself + +`--stamp NAME=VALUE` supplies one directly, and beats anything the build would +have worked out for itself — which is what a CI job that knows the commit it was +dispatched for wants, since the checkout it runs in may be shallow or headless: + +```sh +by build --stamp GIT_SHA=$GITHUB_SHA --stamp BUILD_NUMBER=$GITHUB_RUN_NUMBER +``` + +```by +build: + GIT_SHA: str + BUILD_NUMBER: int +``` + +a stamp is written to the program as text, so it can be declared `str`, `int` or +`bool`. a `bool` reads the spellings a shell produces — `true`/`1`/`yes`/`on` +against `false`/`0`/`no`/`off`, and an empty value is `false`. a value that is +not the type it was declared with is an error, not a coercion: + +```text +the stamp `RUN` is declared `int`, but the build supplied `later`, which is not one +``` + +## reproducible builds + +`BUILT_AT` is the one stamp that differs between two builds of the same source. +when `SOURCE_DATE_EPOCH` is set — the ecosystem's agreement on what "now" is for +a build that has to come out the same twice — it is used instead of the clock, +so a project with a `BUILT_AT` stamp still produces a byte-identical artifact: + +```sh +SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) by build +``` + +## the version is not a stamp + +there is deliberately no built-in `VERSION`. a wheel already carries its version +in its metadata, and the authoritative way to read it is to ask for it: + +```by +from importlib.metadata import version + +def about() -> str: + return version("app") +``` + +a stamped copy is a second answer that can disagree with the first — the wheel +says `1.4.0` and the program says `1.3.9` — and there is no way to tell from +inside which one is wrong. a project that wants one anyway can stamp it +explicitly with `--stamp VERSION=…` + +## a wheel built from a source distribution + +publishing runs the build twice: once to make the source distribution, and again +to make a wheel out of it — and the second one happens where there is no +checkout to read a commit from. + +that is not a problem here. what a source distribution carries is already +transpiled python, and the pyproject in it names an ordinary python backend, so +building a wheel from it never transpiles anything a second time. the stamps the +first build settled are simply what is in the file. a `pip install` from source +gets the same commit the sdist was made from + +## where the values come from + +the build settles the stamps once and hands them to the transpiler; the +transpiler never goes looking for them. that is what keeps the emitted python a +function of its source — the same source and the same stamps give the same file, +every time, which is what lets one file be recomputed into a tree an earlier +build wrote and still agree with the modules around it about what commit they are + +it also means the checker never sees a value. `by check` knows `build.GIT_SHA` +is a `str`; what that `str` holds is settled by whichever build runs + +## stamps are not a place for secrets + +a stamp is written into the artifact as a literal, so it is readable by anyone +who has the wheel — and a build that refuses one reports the value it was given, +which puts it in the build log too. that is the right behaviour for a commit +hash and the wrong place for a token. read a secret from the environment at +startup instead, where it stays out of both diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index a80821d3cf..0ab5704c17 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -52,12 +52,14 @@ at module level, `main` completes to the whole [entry point](main-function.md) definition, with `async main` beside it. once the module defines one, the name completes to it like any other -### keywords written as two words - -a construct spelled with more than one keyword is offered whole, wherever it is -valid: `async def`, `data class`, `frozen data class`, `enum class`, -`override def`, `static var`, and the rest of the -[modifiers](modifiers.md). the method modifiers only appear inside a class +### keywords + +every keyword a statement may open with is offered wherever it is valid, whether +it is spelled with one word or several. the single words are basedpython's own — +`extension`, `let`, `var`, and `init` inside a class body — and a construct +spelled with more than one keyword is offered whole: `async def`, `data class`, +`frozen data class`, `enum class`, `override def`, `static var`, and the rest of +the [modifiers](modifiers.md). the method modifiers only appear inside a class body, and `async for` / `async with` only inside an `async def` they are offered at the start of a statement only — after `async` the plain @@ -118,6 +120,62 @@ diagnostic writes the same import these are auto-imports, so turning off `ty.completions.autoImport` turns them off too +### unimported names + +a name the file has not imported still completes past the dot. `Asdf.` offers +the members of the `Asdf` an import would bind, and accepting one writes that +import: + +```py +from mod import Asdf + +Asdf.name +``` + +a module reads the same way — `mod.` offers what `import mod` reaches — and a +longer chain follows every step, so `mod.Asdf.` lands on those same members. a +package's submodules are among what it offers, and a submodule is imported from +its package, since `import pkg.sub` binds `pkg` rather than the `sub` the file +wrote: + +```py +from pkg import sub + +sub.Asdf +``` + +a name that two modules each define of their own is two offers, one per module, +each carrying its own import. one class that several modules re-export is a +single offer, since the copies would differ only by an import you cannot see + +a name the file already binds means what the file says it means and gets +nothing, and a chain that starts from anything but a name — `Asdf().` — has no +name for an import to bind. as with any auto-import, the name has to be one you +have begun to write: a bare `mod.` in a file that never imported `mod` offers +nothing, because every symbol spelled `mod` anywhere would qualify + +like the aliases above, these are auto-imports, and turning off +`ty.completions.autoImport` turns them off too + +### names inside a string + +a name written inside a plain string's braces completes as a name, and taking +one turns the string into the f-string that reads it — the `f` and the closing +brace are written for you: + +```py +name = "john" + +"hello {na" # → f"hello {name}" +``` + +the rest of the string is left exactly as it is written, so any other brace in +it starts meaning what an f-string reads it to mean once the prefix goes on + +a docstring, a `case` pattern and a `str.format` template are not offered the +conversion, and neither is a string written where a type belongs: an f-string +is a different thing in each of those places, or no longer legal at all + ### enum members and extensions a bare [enum member](enums.md) is offered where the expected type admits one — @@ -130,6 +188,13 @@ a: Color = Red attribute completions include the members any [`extension`](extensions.md) block in scope declares on the receiver, alongside the type's own +### callables + +accepting a completion for anything callable writes its parentheses and leaves +the caret between them, ready for the first argument. set +`ty.completions.completeFunctionParentheses` to `false` to insert the bare name +instead + ## inlay hints each kind of hint can be turned off on its own through the diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index 026c5ed3ff..42b8001d20 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -149,10 +149,12 @@ the forms a class, function or binding can take - [block scoping](block-scoping.md) - [based enums (`enum class`)](enums.md) - [sealed classes](sealed-classes.md) +- [module api enforcement (`implements`)](module-api.md) - [init method shorthand](init-method.md) - [properties](properties.md) - [empty declarations](empty-declarations.md) - [main function](main-function.md) +- [build stamps](build-stamps.md) - [`sentinel` declarations](sentinel.md) - [decorator keyword](decorator-keyword.md) - [decorated function parameters](decorated-parameters.md) @@ -192,6 +194,7 @@ syntax inside a function body - [repeated `_` parameters](repeated-underscore.md) - [lazy imports](lazy-imports.md) - [export imports](export-imports.md) — `from x export y` +- [static resources](static-resources.md) — `import "data/config.yaml" as config` - [extensions](extensions.md) — add members to an existing type, and declare that it conforms to an existing protocol - [conversions (`__from__` / `__into__` / `__of__`)](conversions.md) — and diff --git a/docs/basedpython/features/literal-types.md b/docs/basedpython/features/literal-types.md index 6f7a6a52bb..f3a4608f7b 100644 --- a/docs/basedpython/features/literal-types.md +++ b/docs/basedpython/features/literal-types.md @@ -28,6 +28,43 @@ the following literal forms are recognized in type contexts: - booleans: `True`, `False` - float and complex literals: `1.5`, `3.14j` +## float and complex literals + +python has no `Literal[...]` for a float or a complex: PEP 586 admits only +`None`, `int`, `bool`, `str`, `bytes` and enum members, and every checker +enforces it. So a float literal type is written as the type it is one of, and +the precision is lost at the boundary: + +```by +ratio: 1.5 +scale: int | 2.5 +``` + +→ + +```python +ratio: float +scale: int | float +``` + +a project that would rather keep the precision than keep the output checkable +can ask for the literal instead: + +```toml +[tool.ty.lowering] +float-literals = "literal" +``` + +→ + +```python +ratio: Literal[1.5] +scale: int | Literal[2.5] +``` + +`typing` does not check what it is handed, so this runs — but a checker reading +the output reports the argument as invalid + ## scope promotion fires only in syntactic type contexts: annotations, return types, diff --git a/docs/basedpython/features/module-api.md b/docs/basedpython/features/module-api.md new file mode 100644 index 0000000000..06859a2b10 --- /dev/null +++ b/docs/basedpython/features/module-api.md @@ -0,0 +1,141 @@ +# module api enforcement + +!!! warning "experimental" + + this feature is off unless the project asks for it: + + ```toml + # basedpython.toml + [experimental] + module-api = true + ``` + + an `implements` declaration written without that is reported rather than + quietly ignored. see [experimental features](../configuration.md#experimental-features) + +`implements` says a module answers an interface, and has the type checker hold it +to that: + +```by +# postgres.by +from .api import Backend + +implements Backend + +def connect(url: str) -> str: + return url +``` + +a module that stops answering `Backend` is an error in *that module*, whether or +not anything ever assigns it to a `Backend`-typed place — which matters most for +a plugin that is only ever loaded by name, since nothing else would ever check it + +## writing the interface + +a module's members are unbound, so the interface spells them `static`: + +```by +protocol Backend: + name: str + static def connect(url: str) -> str +``` + +a member like `name` is writable through the interface, so the module has to +declare its type — `name: str = "postgres"` rather than `name = "postgres"`, whose +type is the literal it was given + +nothing else about the protocol changes. a class object answers a static-membered +protocol just as a module does, which is what lets a test substitute a fake: + +```by +class FakeBackend: + name: str = "fake" + static def connect(url: str) -> str: + return url + +def run(backend: Backend) -> None: ... + +run(FakeBackend) +``` + +## imposing an interface on a package + +a `for` clause obliges other modules rather than the one it is written in. it +belongs in a package's `__init__`, and its patterns name what is inside that +package: + +```by +# backends/__init__.by +from .api import Backend + +implements Backend for ".*", "!.base" +``` + +every module in `backends` now answers `Backend` — including one added tomorrow +by someone who never read the interface, and including one nothing imports. the +obligation cannot be dropped by editing the module that fails it + +the error is reported in the failing module, with the rule that imposed it +pointed at as well: + +```text +error[unmet-module-api]: `backends.broken` does not answer `Backend` + --> backends/broken.by:1:1 + | +1 | def connect(url: int) -> str: + | ^ + | + ::: backends/__init__.by:3:1 + | +3 | implements Backend for ".*" + | ---------- required by this declaration +info: `connect` is `def connect(url: int) -> str`, but `Backend` declares it as `def connect(url: str) -> str` +``` + +### patterns + +a pattern starts with a `.`, marking it relative to the package it is written in, +as a relative import does. `*` matches inside one name, `**` matches any number of +levels, and `!` excludes: + +| pattern | reaches | +| --------------- | ------------------------------ | +| `".*"` | every submodule of the package | +| `".**"` | the whole subtree | +| `".pages.*"` | the submodules of `pages` | +| `".**.pages.*"` | a `pages` package at any depth | +| `".handler_*"` | a submodule matched by name | +| `"!.base"` | carves one back out | + +a module with an `_`-prefixed component is not reached by a pattern containing a +wildcard — a private helper sitting among the plugins is not a plugin, and +neither is anything inside a private package. a pattern that names one outright, +with no wildcard in it at all (`"._named"`), still reaches it + +a rule may only be written in a package's `__init__`, and its patterns may not +climb out of that package with `..`. a module finds the obligations imposed on it +by looking at the packages it is in, so a rule anywhere else would enforce +nothing, and a package cannot have requirements added to it from outside + +## stubs + +when a module has a `.byi` stub, the stub is what everything outside the module +reads, so the stub is what an obligation is about — and where its declarations +belong. an `implements` written in an implementation file that a stub shadows is +an error saying so, rather than a check of a surface nobody else can see + +## what a module has to answer + +only the members the interface names. everything else about the module is its own +business — a module may expose whatever else it likes, and `_`-prefixing and +`__all__` say what is not part of its surface, as they always did + +a module whose `__getattr__` answers every name cannot be checked at all, so +attaching an obligation to one is an error rather than a check that always passes + +## what it does not do + +nothing is enforced at runtime, and nothing is hidden: `getattr` still reaches +whatever the module defines, and python that was never type-checked sees the +module it always saw. this is a type-level contract, exactly like a class's +private members diff --git a/docs/basedpython/features/static-resources.md b/docs/basedpython/features/static-resources.md new file mode 100644 index 0000000000..22cea4a938 --- /dev/null +++ b/docs/basedpython/features/static-resources.md @@ -0,0 +1,172 @@ +# static resources + +a json, toml or yaml file can be imported by path, which binds its document to a +name with a type: + +```yaml +# data/config.yaml +a: + b: + - 1 + - 2 +``` + +```by +import "data/config.yaml" as config + +config.a.b[1] # 2 +``` + +the file is read while the program is built, and the document is written into +the module that imported it: + +```python +from typing import Final + + +class config: + class a: + b: Final = (1, 2) +``` + +nothing is opened at run time, and nothing has to be installed to read yaml or +toml — by the time the program runs, the document is python. + +## what a document becomes + +a mapping becomes a class, so its keys are attributes. a sequence becomes a +tuple, so an index reaches one element rather than the union of everything in +the collection. a scalar keeps the value it was written with: + +```json +{ "name": "ty", "port": 8080, "ratio": 0.5, "debug": true, "missing": null } +``` + +```by +import "settings.json" as settings + +reveal_type(settings.name) # "ty" +reveal_type(settings.port) # 8080 +reveal_type(settings.ratio) # float +reveal_type(settings.debug) # True +reveal_type(settings.missing) # None +``` + +because a sequence is a tuple, an index that is not there is an error rather +than a surprise at run time: + +```by +settings.ports[7] # error: index-out-of-bounds +``` + +and because every value is `final`, so is the document: + +```by +settings.port = 9000 # error: invalid-assignment +``` + +a mapping at the top of the document is the value itself. anything else — a +sequence, a scalar — is bound as it is: + +```json +[80, 443] +``` + +```by +import "ports.json" as ports + +ports[0] # 80 +``` + +## the path + +the path is relative to the file that imports it, and it is written with `/` +whichever platform the build runs on: + +```by +import "data/config.yaml" as config +import "../shared/defaults.toml" as defaults +``` + +an absolute path is [`invalid-static-resource`](#errors): it names a place on +one machine, and a program is not built on one machine. + +the name to bind has to be written. a path is not a name, so there is nothing +for `import "data/config.yaml"` to fall back on, and it is an error to leave the +`as` clause off. + +a resource is imported by a statement of its own — not beside a module import, +and not [lazily](lazy-imports.md), since there is nothing left to defer. + +## keys python cannot name + +a document is read through attributes, so a key that is not a valid python +identifier has no attribute to be read through. such a key is left out of the +value, and the import reports +[`unusable-resource-key`](#errors): + +```json +{ "build-backend": "hatchling.build", "root": "." } +``` + +```by +import "pyproject.json" as project # warning: unusable-resource-key + +project.root # "." +project.build_backend # error: unresolved-attribute — the key is `build-backend` +``` + +the same goes for a name with two leading underscores: python mangles `__x` +inside a class body, so the attribute a reader would write is not the one that +would exist, and `__x__` would collide with what a class carries of its own. two +names the rendering needs for itself are left out as well — `Final`, which the +values are annotated with, and anything beginning with `_by_`. + +the document still holds those keys. nothing in the program can reach them. + +## the formats + +| extension | notes | +| ---------------- | ------------------------------------------------------------- | +| `.json` | an integer too large for 64 bits is read as a float | +| `.toml` | a date or time is read as the text it was written with | +| `.yaml` / `.yml` | one document per file; an anchor is expanded where it is used | + +a key written twice is read as its last value, which is what json and yaml +themselves do. a yaml mapping key that is not a string is an error: a document +is read through its keys, and there would be nothing to call that one. + +## two importers, two objects + +the document is written into each module that imports it, so two modules +importing one file get two objects. they hold equal values and answer every +attribute the same way, but they are not the same object: + +```by +# a.by +import "data/config.yaml" as config + +# b.by +import "data/config.yaml" as config + +a.config.a === b.config.a # False +``` + +## errors + +| diagnostic | when | +| ------------------------- | --------------------------------------------------------------------------------------------- | +| `invalid-static-resource` | the path names nothing, is absolute, is not a resource format, or the document cannot be read | +| `unusable-resource-key` | a key in the document has no name python can spell | + +reading a document that cannot be parsed also fails the build: there is no value +to write into the module. + +## limits + +a static resource is a document, not a module, so python read back as +basedpython never turns a class tree into one — the classes come back as +classes. + +going to the definition of a value lands on the resource file rather than on the +line the key is written on. diff --git a/docs/basedpython/features/template-literal-types.md b/docs/basedpython/features/template-literal-types.md index 8abd611b88..c5dc1bf052 100644 --- a/docs/basedpython/features/template-literal-types.md +++ b/docs/basedpython/features/template-literal-types.md @@ -25,6 +25,18 @@ negative number, and never a leading zero, so `"a07b"` is not an `f"a{int}b"`. `Character` is one extended grapheme cluster. a type the reading does not model stands for any string at all, which never rejects anything +## a hole written as an alias + +a hole is read for the type it means, not for the name it was written with, so +an alias stands for whatever it names — however many aliases deep that is: + +```by +type Inner = "foo" | "bar" +type Outer = Inner | "baz" + +a: f"the {Outer}" # "the foo" | "the bar" | "the baz" +``` + ## a pattern that says something simpler is that simpler thing a hole that renders to exactly one string folds into the text, so a pattern with diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index c6ba8af6a0..7f17db613c 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -157,6 +157,10 @@ wheel-versions = ["3.12", "3.14"] the result. nothing reaches `dist/` unless the whole set built, because a release missing one of its wheels hands that interpreter an older one without saying so +the lowering flags apply to the whole release. `by build --wheels --soundness none` settles that once and every wheel in the set is lowered with it, so the +artifacts of one release are lowered alike rather than each wheel answering for +itself + `dist/` itself is checked too, since that is where a release is published *from*. an artifact of this release that this build did not produce — an untagged wheel from an earlier `uv build`, or a version no longer built — is refused, because diff --git a/python/basedpython-pygments/basedpython_pygments/__init__.py b/python/basedpython-pygments/basedpython_pygments/__init__.py index 1bd9d195ec..969aacf0c2 100644 --- a/python/basedpython-pygments/basedpython_pygments/__init__.py +++ b/python/basedpython-pygments/basedpython_pygments/__init__.py @@ -141,6 +141,10 @@ class BasedPythonLexer(PythonLexer): inherit, ], "root": [ + # `build:` opens the block of values the build settles. only at the + # margin, and only where the colon opens a block: `build: int` is an + # ordinary annotated assignment and `build.GIT_SHA` an ordinary read + (r"^build(?=:[ \t]*(?:#.*)?$)", Keyword), # a property accessor stands where a `def` would, so it only counts # at the head of a line. `d.get(k)` and `def get(self)` keep their # name token diff --git a/scripts/check_by_lexer.py b/scripts/check_by_lexer.py index 63c218107d..10cb9f3a09 100644 --- a/scripts/check_by_lexer.py +++ b/scripts/check_by_lexer.py @@ -49,6 +49,7 @@ ("b = a cast? int", "cast?"), ("sentinel MISSING", "sentinel"), ("extension list[int]:", "extension"), + ("build:", "build"), ("implementation Show for Point:", "implementation"), ("def f(x: protocol(a: int)): ...", "protocol"), ("class C[reified T]: ...", "reified"), @@ -89,6 +90,8 @@ ("literal: object | None", "literal"), ("x = open(path)", "open"), ("cast(int, x)", "cast"), + ("build: int = 3", "build"), + ("print(build.GIT_SHA)", "build"), ] diff --git a/ty.schema.json b/ty.schema.json index 978fd6d11f..dd9a35467a 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -46,6 +46,27 @@ } ] }, + "experimental": { + "anyOf": [ + { + "$ref": "#/definitions/ExperimentalOptions" + }, + { + "type": "null" + } + ] + }, + "lowering": { + "description": "Configures how basedpython spells constructs python has no spelling of its own for.", + "anyOf": [ + { + "$ref": "#/definitions/LoweringOptions" + }, + { + "type": "null" + } + ] + }, "overrides": { "description": "Override configurations for specific file patterns.\n\nEach override specifies include/exclude patterns and rule configurations\nthat apply to matching files. Multiple overrides can match the same file,\nwith later overrides taking precedence.", "anyOf": [ @@ -432,6 +453,44 @@ }, "additionalProperties": false }, + "ExperimentalOptions": { + "description": "Features that are still being designed, and are off unless the project asks\nfor them by name.\n\nAn experimental feature may change or be withdrawn without the deprecation\nperiod a stable one gets. Opting in says you would rather have it than that\nguarantee.", + "type": "object", + "properties": { + "build-stamps": { + "description": "Whether a `build:` block declares build stamps.\n\n`build:` declares the values a build settles when it produces the artifact\n— the commit it was built from, the time it was built at — and each is read\nas `build.NAME` at the type it declares. With this off the block still\nparses and still lowers, so a program that reads a stamp keeps working, but\nwriting one is reported: nothing settles a stamp the project has not asked\nfor, so it would silently stand for its default, or for nothing.", + "type": [ + "boolean", + "null" + ] + }, + "module-api": { + "description": "Whether an `implements` declaration is enforced.\n\n`implements Backend` obliges the module that writes it to answer the\nprotocol, and a `for` clause in a package's `__init__` imposes the same\nobligation on the modules its patterns name. With this off the declaration\nstill parses and still lowers, but nothing is checked against it — and a\ndeclaration written anyway is reported, rather than quietly doing nothing.", + "type": [ + "boolean", + "null" + ] + } + }, + "additionalProperties": false + }, + "FloatLiteralLowering": { + "description": "How a float or complex literal type reaches the transpiled python.", + "oneOf": [ + { + "title": "Nominal", + "description": "The type the literal is one of: `1.5` becomes `float`, `2j` becomes `complex`.", + "type": "string", + "const": "nominal" + }, + { + "title": "Literal", + "description": "The literal itself, inside `Literal[...]`. Runs, but no checker accepts it.", + "type": "string", + "const": "literal" + } + ] + }, "Level": { "oneOf": [ { @@ -454,6 +513,23 @@ } ] }, + "LoweringOptions": { + "type": "object", + "properties": { + "float-literals": { + "description": "How a float or complex literal type is spelled in the transpiled python.\n\nbasedpython reads `a: 1.5` as a literal type, and python has no spelling for one:\nPEP 586 admits only `None`, `int`, `bool`, `str`, `bytes` and enum members into\n`Literal[...]`.\n\n* `nominal` (the default) writes the type the literal is one of — `a: 1.5` becomes\n `a: float`, `a: 2j` becomes `a: complex`. The precision is lost, and every checker\n that reads the output accepts it.\n* `literal` keeps the literal, writing `a: Literal[1.5]`. The precision survives and\n the output still runs, because `typing` does not check what it is handed — but a\n checker reading it reports the argument as invalid.", + "anyOf": [ + { + "$ref": "#/definitions/FloatLiteralLowering" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, "OutputFormat": { "description": "The diagnostic output format.", "oneOf": [ @@ -1034,6 +1110,16 @@ } ] }, + "invalid-build-stamps": { + "title": "detects a basedpython `build:` block the project has not opted in to", + "description": "## What it does\nChecks that a `build:` block is one the project has asked for. Build\nstamps are experimental, so a project opts in by name:\n\n```toml\n# basedpython.toml\n[experimental]\nbuild-stamps = true\n```\n\n## Why is this bad?\nThe block parses and lowers whether or not the project opted in, because a\nprogram that reads `build.GIT_SHA` has to keep working when the feature is\nturned off — so nothing at the point of use says the value was never\nsettled. A stamp declared without a default fails the transpile, and one\nwith a default quietly stands for that default in an artifact that claims\nto know what commit it came from.\n\n## Example\n\n```by\nbuild: # error: `build` is an experimental feature, and is off\n GIT_SHA: str\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-conformance": { "title": "detects invalid basedpython conformance declarations", "description": "## What it does\nChecks for invalid basedpython conformance declarations\n(`extension str(A):`): an interface that is neither a protocol nor an\nabstract class, or a requirement nothing answers.\n\n## Why is this bad?\nA conformance states that an existing type satisfies an existing\ninterface, and registers a witness table so that a call through an\ninterface-typed receiver reaches it. Conforming to a concrete class would\nmean promising its fields, which a conformance has nowhere to store; and a\nrequirement neither the block, a default on the interface's own extension,\nnor the type itself answers is an `AttributeError` the first time anything\ndispatches through the conformance.\n\n## Example\n\n```by\nprotocol A:\n def bar(self)\n\nextension str(A): # error: `str` does not answer every member of `A`\n ...\n```", @@ -1264,6 +1350,16 @@ } ] }, + "invalid-module-api": { + "title": "detects invalid basedpython `implements` declarations", + "description": "## What it does\nChecks that an `implements` declaration is one a module can be held to:\nthat it names a protocol, that a `for` clause is written in a package's\n`__init__` with patterns relative to that package and reaching something,\nand that the module it obliges can actually be checked.\n\n## Why is this bad?\nAn obligation nothing can check is worse than no obligation: it reads, in\nreview and in the editor, as a promise that is being enforced. A rule\nwhose patterns match no module, or one written where no module will look\nfor it, enforces nothing at all.\n\nThis also covers a declaration written while the feature is off. `implements`\nis experimental, so a project opts in by name:\n\n```toml\n# basedpython.toml\n[experimental]\nmodule-api = true\n```\n\n## Example\n\n```by\nclass Backend:\n def connect(self) -> None: ...\n\nimplements Backend # error: `Backend` is not a protocol\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-named-tuple": { "title": "detects invalid `NamedTuple` class definitions", "description": "## What it does\n\nChecks for invalidly defined `NamedTuple` classes.\n\n## Why is this bad?\n\nAn invalidly defined `NamedTuple` class may lead to the type checker\ndrawing incorrect conclusions. It may also lead to `TypeError`s or\n`AttributeError`s at runtime.\n\n## Examples\n\nA class definition cannot combine `NamedTuple` with other base classes\nin multiple inheritance; doing so raises a `TypeError` at runtime. The sole\nexception to this rule is `Generic[]`, which can be used alongside `NamedTuple`\nin a class's bases list.\n\n```pycon\n>>> from typing import NamedTuple\n>>> class Foo(NamedTuple, object): ...\nTypeError: can only inherit from a NamedTuple type and Generic\n```\n\nFurther, `NamedTuple` field names cannot start with an underscore:\n\n```pycon\n>>> from typing import NamedTuple\n>>> class Foo(NamedTuple):\n... _bar: int\nValueError: Field names cannot start with an underscore: '_bar'\n```\n\n`NamedTuple` classes also have certain synthesized attributes (like `_asdict`, `_make`,\n`_replace`, etc.) that cannot be overwritten. Attempting to assign to these attributes\nwithout a type annotation will raise an `AttributeError` at runtime.\n\n```pycon\n>>> from typing import NamedTuple\n>>> class Foo(NamedTuple):\n... x: int\n... _asdict = 42\nAttributeError: Cannot overwrite NamedTuple attribute _asdict\n```\n\nFinally, `NamedTuple` field annotations cannot use the `ClassVar` or `Final` type\nqualifiers. These qualifiers also cause a runtime error when annotations are evaluated eagerly:\n\n```pycon\n>>> from typing import ClassVar, NamedTuple\n>>> class Foo(NamedTuple):\n... x: ClassVar[int]\nTypeError: typing.ClassVar[int] is not valid as type argument\n```", @@ -1424,6 +1520,16 @@ } ] }, + "invalid-static-resource": { + "title": "detects a static resource import that cannot be read", + "description": "## What it does\n\nChecks for basedpython static resource imports that cannot be read.\n\n## Why is this bad?\n\n`import \"data/config.yaml\" as config` says the file is part of the program. A\npath that names nothing, a path that names a place on one machine, a file in a\nformat that is not `.json`, `.toml`, `.yaml` or `.yml`, and a document the\nformat's own parser rejects all leave the import with no value to bind.\n\n## Examples\n\n`main.by`:\n\n```by\n# error: [invalid-static-resource]\nimport \"data/config.txt\" as config\n\n# error: [invalid-static-resource]\nimport \"/etc/hosts.json\" as hosts\n\n# error: [invalid-static-resource]\nimport \"data/missing.json\" as missing\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-super-argument": { "title": "detects invalid arguments for `super()`", "description": "## What it does\n\nDetects `super()` calls where:\n\n- the first argument is not a valid class literal, or\n- the second argument is not an instance or subclass of the first argument.\n\n## Why is this bad?\n\n`super(type, obj)` expects:\n\n- the first argument to be a class,\n- and the second argument to satisfy one of the following:\n - `isinstance(obj, type)` is `True`\n - `issubclass(obj, type)` is `True`\n\nViolating this relationship will raise a `TypeError` at runtime.\n\n## Examples\n\n```python\nclass A: ...\n\n\nclass B(A): ...\n\n\nsuper(A, B()) # it's okay! `A` satisfies `isinstance(B(), A)`\n\n# `A()` is not a class\nsuper(A(), B()) # error\n\n# `A()` does not satisfy `isinstance(A(), B)`\nsuper(B, A()) # error\n# `A` does not satisfy `issubclass(A, B)`\nsuper(B, A) # error\n```\n\n## References\n\n- [Python documentation: super()](https://docs.python.org/3/library/functions.html#super)", @@ -2334,6 +2440,16 @@ } ] }, + "unmet-module-api": { + "title": "detects a module that does not answer an interface it is obliged to", + "description": "## What it does\nChecks a module against the interfaces it is obliged to answer, whether it\ndeclared one itself with `implements` or a package it lives in imposed one\nwith `implements ... for`.\n\n## Why is this bad?\nA module that is meant to be used through an interface — a plugin, a\nbackend, a settings module — is otherwise only checked where something\nassigns it to an interface-typed place. A plugin loaded by name is never\nchecked at all, and an error that does surface lands on the consumer\nrather than on the module that broke.\n\n## Example\n\n```by\nprotocol Backend:\n static def connect(url: str) -> None\n\nimplements Backend # error: this module does not answer `Backend`\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "unresolved-attribute": { "title": "detects references to unresolved attributes", "description": "## What it does\n\nChecks for unresolved attributes.\n\n## Why is this bad?\n\nAccessing an unbound attribute will raise an `AttributeError` at runtime.\nAn unresolved attribute is not guaranteed to exist from the type alone,\nso this could also indicate that the object is not of the type that the user expects.\n\n## Examples\n\n```python\nclass A: ...\n\n\n# AttributeError: 'A' object has no attribute 'foo'\nA().foo # error\n```", @@ -2494,6 +2610,16 @@ } ] }, + "unusable-resource-key": { + "title": "detects a key in a static resource that python cannot name", + "description": "## What it does\n\nChecks for keys in an imported static resource that python cannot name.\n\n## Why is this bad?\n\nA static resource is read through attributes, so a key that is not a valid\npython identifier — `build-backend`, `class`, `2` — has no attribute to be read\nthrough, and is left out of the value the import binds. The document still holds\nit; nothing in the program can reach it.\n\nNames with two leading underscores are left out for the same reason: python\nmangles `__x` inside a class body, so the attribute the reader would write is\nnot the one that would exist.\n\n## Examples\n\n`data/project.json`:\n\n```json\n{ \"build-backend\": \"hatchling.build\", \"root\": \".\" }\n```\n\n`main.by`:\n\n```by\n# error: [unusable-resource-key]\nimport \"data/project.json\" as project\n\nreveal_type(project.root) # revealed: \".\"\n```", + "default": "warn", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "unused-awaitable": { "title": "detects awaitable objects that are used as expression statements without being awaited", "description": "## What it does\n\nChecks for awaitable objects (such as coroutines) used as expression\nstatements without being awaited.\n\n## Why is this bad?\n\nCalling an `async def` function returns a coroutine object. If the\ncoroutine is never awaited, the body of the async function will never\nexecute, which is almost always a bug. Python emits a\n`RuntimeWarning: coroutine was never awaited` at runtime in this case.\n\n## Examples\n\n```python\nasync def fetch_data() -> str:\n return \"data\"\n\n\nasync def main() -> None:\n # Warning: coroutine is not awaited\n fetch_data() # error\n await fetch_data() # OK\n```", diff --git a/zensical.toml b/zensical.toml index 0d4d8cd5ff..34b09cfed4 100644 --- a/zensical.toml +++ b/zensical.toml @@ -159,10 +159,12 @@ features = [ { "block scoping" = "features/block-scoping.md" }, { "based enums" = "features/enums.md" }, { "sealed classes" = "features/sealed-classes.md" }, + { "module api enforcement" = "features/module-api.md" }, { "init method shorthand" = "features/init-method.md" }, { "properties" = "features/properties.md" }, { "empty declarations" = "features/empty-declarations.md" }, { "main function" = "features/main-function.md" }, + { "build stamps" = "features/build-stamps.md" }, { "sentinel declarations" = "features/sentinel.md" }, { "decorator keyword" = "features/decorator-keyword.md" }, { "decorated function parameters" = "features/decorated-parameters.md" }, @@ -195,6 +197,7 @@ features = [ { "repeated _ parameters" = "features/repeated-underscore.md" }, { "lazy imports" = "features/lazy-imports.md" }, { "export imports" = "features/export-imports.md" }, + { "static resources" = "features/static-resources.md" }, { "extensions" = "features/extensions.md" }, { "conversions (__from__ / __into__ / __of__)" = "features/conversions.md" }, { "frozen container displays" = "features/frozen-displays.md" }, @@ -220,6 +223,7 @@ features = [ [[project.nav]] development = [ { "how transpilation works" = "development/how-transpilation-works.md" }, + { "module api enforcement" = "development/module-api-design.md" }, { "reverse transforms" = "development/reverse-transforms.md" }, { "sourcemaps" = "development/sourcemaps.md" }, { "type functions" = "development/type-def-design.md" },