diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 302d025764..347bbc29bd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1227,7 +1227,7 @@ jobs: echo '```console' > "$GITHUB_STEP_SUMMARY" # Enable color output for prek and remove it for the summary # Use --hook-stage=manual to enable slower hooks that are skipped by default - SKIP=rustfmt uv run --only-dev --locked prek run --all-files --show-diff-on-failure --color always --hook-stage manual | \ + SKIP=rustfmt,cargo-doc uv run --only-dev --locked prek run --all-files --show-diff-on-failure --color always --hook-stage manual | \ tee >(sed -E 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})*)?[mGK]//g' >> "$GITHUB_STEP_SUMMARY") >&1 exit_code="${PIPESTATUS[0]}" echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32d926f708..d33a431e1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,14 +79,15 @@ repos: - id: cargo-doc name: check `cargo doc` and `cargo test --doc` entry: > - bash -c 'RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items -p ty_python_semantic - -p ty_python_core -p ty_module_resolver -p ty_site_packages -p ty_combine -p ty_project -p ty_ide -p ty_wasm - -p ty_vendored -p ty_static -p ty -p ty_test -p ruff_db -p ruff_python_formatter - && cargo test --all-features --doc' + bash -c 'RUSTDOCFLAGS="-D warnings" cargo doc --all --no-deps + && RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --document-private-items -p ty_python_semantic + -p ty_python_core -p ty_module_resolver -p ty_site_packages -p ty_combine -p ty_project -p ty_ide + -p ty_wasm -p ty_vendored -p ty_static -p ty -p ty_test -p ruff_db -p ruff_python_formatter + && cargo test --all-features --doc' types: [rust] language: system pass_filenames: false - priority: 0 + stages: [pre-push] # Prettier - repo: https://github.com/rbubley/mirrors-prettier diff --git a/Cargo.lock b/Cargo.lock index 5fcc5eb587..9ca868479f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5151,6 +5151,7 @@ dependencies = [ "jod-thread", "libc", "lsp-server", + "rand 0.10.2", "regex", "ruff_db", "ruff_diagnostics", @@ -5179,6 +5180,7 @@ dependencies = [ "ty_project", "ty_python_core", "ty_python_semantic", + "ty_static", ] [[package]] diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index f3add74dd5..23ac643d2c 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -22,8 +22,18 @@ pub(crate) mod annotate; pub struct Artifact { /// the generated C, kept for `--annotate` and for debugging pub source: PathBuf, - /// the loadable extension - pub extension: PathBuf, + /// the header every generated `.c` includes, at the root of the tree + pub header: PathBuf, + /// the loadable extension, or `None` when only the C was emitted + /// + /// a caller that records what the build wrote — so that a later one can take + /// a stale file back — has to be told the difference. naming the extension an + /// `--emit-c-only` run *would* have built is not harmless: the name is + /// `.so`, which is not where any real build puts it (that is + /// `Toolchain::extension_path`, tagged with the interpreter's abi), so a + /// caller checking the file system for it finds nothing, records nothing, and + /// prunes the extension a previous real build left + pub extension: Option, /// the `--annotate` report, when one was asked for pub annotation: Option, } @@ -113,7 +123,8 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result } fs::create_dir_all(out_dir) .with_context(|| format!("could not create {}", out_dir.display()))?; - fs::write(out_dir.join(by_rt::BY_H_NAME), by_rt::BY_H)?; + let header = out_dir.join(by_rt::BY_H_NAME); + fs::write(&header, by_rt::BY_H)?; let source_path = out_dir.join(module.name.relative_path(".c")); create_parent(&source_path)?; @@ -123,7 +134,8 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result Ok(Built { artifact: Artifact { source: source_path, - extension: out_dir.join(module.name.relative_path(".so")), + header, + extension: None, annotation: write_annotation(module, out_dir, options)?, }, declined: module.declined.clone(), @@ -364,7 +376,8 @@ pub fn build_module(module: &ModuleIr, toolchain: &Toolchain, out_dir: &Path) -> Ok(Artifact { source, - extension, + header, + extension: Some(extension), annotation: None, }) } diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 8c9e20da2f..d9678fbcff 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -81,7 +81,10 @@ fn built(module: &ModuleIr, toolchain: &Toolchain, tag: &str) -> Option let _ = std::fs::remove_dir_all(&dir); match build_module(module, toolchain, &dir) { Ok(artifact) => { - assert!(artifact.extension.exists(), "the extension was written"); + assert!( + extension_of(&artifact).exists(), + "the extension was written" + ); Some(dir) } Err(error) => { @@ -218,6 +221,18 @@ fn fib_module() -> ModuleIr { } } +/// The extension a build wrote. +/// +/// `Artifact::extension` is `None` only for an `--emit-c-only` emit, which +/// nothing in this file does — every case here goes through `build_module` or +/// `build_lowered`, both of which invoke the C compiler. +fn extension_of(artifact: &by_build::Artifact) -> &std::path::Path { + artifact + .extension + .as_deref() + .expect("this build compiled an extension, so it has to name one") +} + #[test] fn integer_arithmetic_computes_the_same_answers_as_python() { let Some((python, toolchain)) = environment() else { @@ -931,7 +946,7 @@ fn an_unchanged_module_is_not_recompiled() { .and_then(|meta| meta.modified()) .expect("the artifact exists") }; - let before = stamp(&first.artifact.extension); + let before = stamp(extension_of(&first.artifact)); let second = build_source( source, @@ -941,7 +956,11 @@ fn an_unchanged_module_is_not_recompiled() { &Options::default(), ) .expect("the toolchain already worked"); - assert_eq!(stamp(&second.artifact.extension), before, "it recompiled"); + assert_eq!( + stamp(extension_of(&second.artifact)), + before, + "it recompiled" + ); // and a real change does rebuild let changed = "def double(a: int) -> int:\n return a * 3\n"; @@ -954,7 +973,7 @@ fn an_unchanged_module_is_not_recompiled() { ) .expect("the toolchain already worked"); assert_ne!( - stamp(&third.artifact.extension), + stamp(extension_of(&third.artifact)), before, "it skipped a change" ); @@ -980,7 +999,7 @@ fn a_stale_extension_is_rebuilt_even_when_the_c_is_unchanged() { eprintln!("skipping: no working C toolchain"); return; }; - std::fs::remove_file(&built.artifact.extension).expect("the artifact exists"); + std::fs::remove_file(extension_of(&built.artifact)).expect("the artifact exists"); let again = build_source( source, "by_e2e_rebuild_stale", @@ -989,7 +1008,7 @@ fn a_stale_extension_is_rebuilt_even_when_the_c_is_unchanged() { &Options::default(), ) .expect("the toolchain already worked"); - assert!(again.artifact.extension.exists()); + assert!(extension_of(&again.artifact).exists()); } /// a class on a base from outside the module, holding storage of its own @@ -1251,10 +1270,10 @@ fn a_package_is_built_as_a_tree_and_imports_under_its_dotted_names() { return; }; assert!( - built.artifact.extension.exists(), + extension_of(&built.artifact).exists(), "{} was written to {}", name.dotted(), - built.artifact.extension.display() + extension_of(&built.artifact).display() ); } diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 98877c7338..326bd1e454 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -14,11 +14,13 @@ //! build would have transpiled it, and the only way to guarantee that is for there //! to be one implementation of each. +use std::collections::HashMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use anyhow::Context; -use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; +use ruff_db::Db as _; +use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; /// Everything needed to build a project db a second time. @@ -308,7 +310,8 @@ pub(crate) fn project_sources( root: &Path, output: Option<&Path>, ) -> Vec<(PathBuf, ruff_db::files::File)> { - db.project() + let candidates: Vec<(PathBuf, ruff_db::files::File)> = db + .project() .files(db) .into_iter() .filter(|file| { @@ -328,9 +331,75 @@ pub(crate) fn project_sources( // this build is about to read, and reading those instead would build the // project into itself, one directory deeper each time .filter(|(path, _)| output.is_none_or(|output| !path.starts_with(output))) + .collect(); + // and nor is any *other* build's output. `--out` can name any directory, and + // a project can have several, so the one this run was given is not the only + // one that has to be turned away + let mut known = HashMap::new(); + candidates + .into_iter() + .filter(|(path, _)| !inside_build_output(db.system(), path, root, &mut known)) .collect() } +/// Whether `path` sits inside a tree that some build already wrote. +/// +/// A build output is self-describing: it holds a `.by-manifest` naming everything +/// that build put there. That marker is what identifies it, rather than the +/// directory's name. [`NON_SOURCE_DIRS`] already turns away the two default names, +/// but `--out` can say anything, and only the tree this run was *given* is known +/// to be an output from the arguments — a project that builds into two of them has +/// to have the other recognised on sight. +/// +/// It has to be turned away because the tree holds a *copy* of the project: every +/// hand-written `.py`, and every `.by` too when `build.sources` is on. Reading one +/// back means each of those modules is claimed twice — once where it was written +/// and once where it was copied — and the build stops, telling the author that two +/// files they never put side by side would compile to the same module. +/// +/// The project root is never itself an output, whatever it holds: a `.by-manifest` +/// left at the root would otherwise empty the project of every file it has. +fn inside_build_output( + system: &dyn System, + path: &Path, + root: &Path, + known: &mut HashMap, +) -> bool { + let Some(parent) = path.parent() else { + return false; + }; + let mut current = parent; + while current != root && current.starts_with(root) { + let is_output = match known.get(current) { + Some(answer) => *answer, + None => { + let answer = holds_manifest(system, current); + known.insert(current.to_path_buf(), answer); + answer + } + }; + if is_output { + return true; + } + match current.parent() { + Some(next) => current = next, + None => return false, + } + } + false +} + +/// Whether `directory` carries the manifest that marks it a build output. +/// +/// Asked of the db's [`System`] rather than of `std::fs`, so that it is the same +/// file system every other question in a build is asked — the language server's +/// re-stage runs this on a warm db, and an in-memory test system has to be able +/// to answer it. +pub(crate) fn holds_manifest(system: &dyn System, directory: &Path) -> bool { + SystemPath::from_std_path(directory) + .is_some_and(|path| system.is_file(&path.join(crate::staging::MANIFEST_FILENAME))) +} + /// The project's first-party module roots, longest first, as absolute paths. /// /// These are the directories a module name is resolved against — for a @@ -351,9 +420,60 @@ pub fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { #[cfg(test)] mod tests { - use super::is_hidden_within; + use super::{inside_build_output, is_hidden_within}; + use ruff_db::system::{OsSystem, SystemPath}; + use std::collections::HashMap; use std::path::Path; + /// `--out` can name any directory and a project can hold several, so a build + /// output is recognised by the manifest it carries rather than by its name. + #[test] + fn a_directory_holding_a_manifest_is_a_build_output() { + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path(); + let output = root.join("anything"); + std::fs::create_dir_all(output.join("pkg")).expect("create"); + std::fs::write(output.join(".by-manifest"), "").expect("write"); + let plain = root.join("src"); + std::fs::create_dir_all(&plain).expect("create"); + + let system = + OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); + let mut known = HashMap::new(); + assert!(inside_build_output( + &system, + &output.join("a.by"), + root, + &mut known + )); + assert!( + inside_build_output(&system, &output.join("pkg").join("a.by"), root, &mut known), + "a file deeper inside the output is inside it too" + ); + assert!( + !inside_build_output(&system, &plain.join("a.by"), root, &mut known), + "a directory with no manifest is ordinary source" + ); + } + + /// A manifest at the root would otherwise empty the project of every file. + #[test] + fn the_project_root_is_never_itself_an_output() { + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path(); + std::fs::write(root.join(".by-manifest"), "").expect("write"); + + let system = + OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); + let mut known = HashMap::new(); + assert!(!inside_build_output( + &system, + &root.join("a.by"), + root, + &mut known + )); + } + #[test] fn a_hidden_directory_is_not_project_source() { let root = Path::new("/p"); diff --git a/crates/by_stage/src/staging.rs b/crates/by_stage/src/staging.rs index 62c83a0c46..ea08eee095 100644 --- a/crates/by_stage/src/staging.rs +++ b/crates/by_stage/src/staging.rs @@ -20,13 +20,16 @@ use anyhow::Context; /// until one shadows a module that moved. The manifest is what makes the output a /// mirror rather than a pile: what the previous build wrote and this one did not /// is deleted. -const MANIFEST_FILENAME: &str = ".by-manifest"; +pub(crate) const MANIFEST_FILENAME: &str = ".by-manifest"; /// An output tree being written. pub struct Staging { out: PathBuf, /// relative destination -> the source it came from, for collision reporting written: BTreeMap>, + /// destinations another writer produced, held apart from `written` so a + /// source that would land on one is reported rather than overwriting it + recorded: BTreeSet, } impl Staging { @@ -34,6 +37,7 @@ impl Staging { Self { out: out.to_path_buf(), written: BTreeMap::new(), + recorded: BTreeSet::new(), } } @@ -78,6 +82,32 @@ impl Staging { .with_context(|| format!("could not write {}", destination.display())) } + /// Record a file another writer put into the tree at `relative`. + /// + /// `by compile` writes its extensions through `by_build`, which lays them out + /// itself rather than through a staging. They still belong in the manifest: + /// an extension left behind by a module that has since been deleted keeps + /// importing, and — because python's finder prefers an extension to source — + /// it goes on shadowing the `.py` a later `by build` writes in its place. + /// + /// A recorded path is also claimed, so a file the project keeps where an + /// artifact lands is reported rather than copied over it. The check runs in + /// both directions for whichever order the caller happens to use; today + /// `compile` records every artifact before it stages anything, so in practice + /// it is the staging side that reports. + pub fn record(&mut self, relative: &Path) -> anyhow::Result<()> { + if let Some((previous, source)) = self.written.get_key_value(relative) { + anyhow::bail!( + "the compiler wrote an artifact to `{}`, where {} is already staged \ + — one of them has to be renamed", + previous.display(), + claimant(source.as_deref()), + ); + } + self.recorded.insert(relative.to_path_buf()); + Ok(()) + } + /// Copy `source` to `relative` verbatim. pub(crate) fn copy(&mut self, relative: &Path, source: &Path) -> anyhow::Result<()> { self.claim(relative, Some(source))?; @@ -94,17 +124,27 @@ impl Staging { } fn claim(&mut self, relative: &Path, source: Option<&Path>) -> anyhow::Result<()> { + // a project that keeps a file of its own where an artifact lands — a + // hand-written `main.c` beside `main.by`, say. copying over it would + // leave a tree whose generated half is somebody else's file, and say + // nothing about it + if self.recorded.contains(relative) { + anyhow::bail!( + "{} would be carried over to `{}`, where the compiler \ + already wrote an artifact of that name — rename one of them, or \ + keep the file out of the build with `build.exclude`", + claimant(source), + relative.display(), + ); + } if let Some((previous, Some(previous_source))) = self.written.get_key_value(relative) && Some(previous_source.as_path()) != source { - let claimant = source.map_or_else( - || "the build".to_owned(), - |source| format!("`{}`", source.display()), - ); anyhow::bail!( - "`{}` and {claimant} both build to `{}` — \ + "`{}` and {} both build to `{}` — \ they are the same module, so one of them has to be renamed", previous_source.display(), + claimant(source), previous.display(), ); } @@ -118,7 +158,12 @@ impl Staging { pub fn finish(self) -> anyhow::Result<()> { let manifest = self.out.join(MANIFEST_FILENAME); let previous = read_manifest(&manifest); - let current: BTreeSet<&Path> = self.written.keys().map(PathBuf::as_path).collect(); + let current: BTreeSet<&Path> = self + .written + .keys() + .chain(self.recorded.iter()) + .map(PathBuf::as_path) + .collect(); let mut emptied: BTreeSet = BTreeSet::new(); for stale in &previous { @@ -156,6 +201,14 @@ impl Staging { } } +/// How a collision names the thing that wanted the destination. +fn claimant(source: Option<&Path>) -> String { + source.map_or_else( + || "the build".to_owned(), + |source| format!("`{}`", source.display()), + ) +} + fn create_parent(path: &Path) -> anyhow::Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) diff --git a/crates/by_stage/src/verbatim.rs b/crates/by_stage/src/verbatim.rs index d87ea88308..cab6374dcd 100644 --- a/crates/by_stage/src/verbatim.rs +++ b/crates/by_stage/src/verbatim.rs @@ -16,11 +16,12 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; +use ruff_db::Db as _; use ruff_db::system::SystemPath; use ty_project::{Db, ProjectDatabase}; use walkdir::WalkDir; -use crate::project::may_hold_build_content; +use crate::project::{holds_manifest, may_hold_build_content}; use crate::staging::{Staging, relative_destination}; /// One file carried into the output unchanged. @@ -56,6 +57,19 @@ fn verbatim_files( if entry.path() == out { return false; } + // nor is any *other* build's output. a project can have several — + // `by build --out one` beside `by build --out two` — and each holds a + // copy of every file this walk carries, so one that was not turned + // away would be copied into the next wholesale, a tree deep in a + // tree. it is recognised by the manifest it carries rather than by + // its name, because `--out` can say anything. the root is exempt: + // a manifest there would otherwise carry nothing over at all + if entry.file_type().is_dir() + && entry.path() != root + && holds_manifest(db.system(), entry.path()) + { + return false; + } if !may_hold_build_content(entry) { return false; } diff --git a/crates/by_transforms/src/reverse_transforms/identity_swap.rs b/crates/by_transforms/src/reverse_transforms/identity_swap.rs index d3616fec8c..23b9177827 100644 --- a/crates/by_transforms/src/reverse_transforms/identity_swap.rs +++ b/crates/by_transforms/src/reverse_transforms/identity_swap.rs @@ -7,12 +7,23 @@ //! basedpython's `is` is the instance check, so a python identity comparison //! round-trips to `===` / `!==` and an `isinstance` call round-trips to `is` //! -//! a literal right-hand side is left alone, mirroring the forward transform's -//! own literal guard: `x is None` is identity in both languages, so rewriting -//! it to `x === None` would churn idiomatic source for no change in meaning. -//! this is why the operator must be rewritten rather than skipped — leaving a -//! python `is not` in place re-reads it as `not isinstance(...)` on the way -//! back out +//! `x is None` is left alone: basedpython reads it as a type test against the +//! type `None`, which is the same runtime check, so rewriting it to +//! `x === None` would churn idiomatic source for no change in meaning. Every +//! *other* literal must still be rewritten — `x is 3` is identity in python but +//! a test for the type `Literal[3]` in basedpython, and those differ for an int +//! outside the interned range. +//! +//! This is also why the operator has to be rewritten rather than skipped: +//! leaving a python `is not` in place re-reads it as a type test on the way +//! back out. +//! +//! An `isinstance` call is only rewritten when its second argument is +//! something a type expression can say. A tuple of classes is `isinstance`'s +//! own spelling for a union and reads as a *tuple type* in a type expression, +//! so it is written back out as `A | B`; anything else — `type(y)`, a variable +//! holding classinfo — keeps the call it was, which basedpython runs just as +//! python does use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; @@ -45,9 +56,9 @@ impl<'src> IdentitySwapReverse<'src> { for (op, rhs) in c.ops.iter().zip(c.comparators.iter()) { let rhs_start = rhs.range().start(); let between = &self.source[usize::from(lhs_end)..usize::from(rhs_start)]; - // a literal rhs means identity in basedpython too, so the forward - // transform leaves it as `is` — mirror that and don't churn it - if !rhs.is_literal_expr() { + // `is None` reads the same in both languages; every other literal + // rhs changes meaning, so it takes the identity spelling + if !rhs.is_none_literal_expr() { let words: &[&str] = match op { CmpOp::Is => &["is"], CmpOp::IsNot => &["is", "not"], @@ -123,13 +134,13 @@ impl<'src> IdentitySwapReverse<'src> { let Some((x, y)) = isinstance_operands(call) else { return; }; - let (x_src, y_src) = ( - self.src(x.range()).to_owned(), - self.src(y.range()).to_owned(), - ); + let Some(target) = self.type_expression_target(y) else { + return; + }; + let x_src = self.src(x.range()).to_owned(); self.folded_into_not.push(call.range()); self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{x_src} is not {y_src}"), + format!("{x_src} is not {target}"), unary.range(), ))); } @@ -141,15 +152,45 @@ impl<'src> IdentitySwapReverse<'src> { let Some((x, y)) = isinstance_operands(call) else { return; }; - let (x_src, y_src) = ( - self.src(x.range()).to_owned(), - self.src(y.range()).to_owned(), - ); + let Some(target) = self.type_expression_target(y) else { + return; + }; + let x_src = self.src(x.range()).to_owned(); self.edits.push(Fix::safe_edit(Edit::range_replacement( - format!("{x_src} is {y_src}"), + format!("{x_src} is {target}"), call.range(), ))); } + + /// how an `isinstance` classinfo argument is written as the type expression + /// a type test takes, or `None` when its shape is not one a type expression + /// has and the call has to stay a call. + /// + /// A tuple is `isinstance`'s spelling for "any of these", which a type + /// expression spells `A | B` — writing the tuple back out would name the + /// *tuple type* instead, a different test entirely. + fn type_expression_target(&self, classinfo: &Expr) -> Option { + match classinfo { + Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_) => { + Some(self.src(classinfo.range()).to_owned()) + } + Expr::BinOp(binop) if binop.op == ruff_python_ast::Operator::BitOr => { + Some(self.src(classinfo.range()).to_owned()) + } + Expr::Tuple(tuple) => { + let arms: Option> = tuple + .elts + .iter() + .map(|element| self.type_expression_target(element)) + .collect(); + let arms = arms?; + // an empty `isinstance(x, ())` is always `False`; there is no + // union to write for it + (!arms.is_empty()).then(|| arms.join(" | ")) + } + _ => None, + } + } } /// the two operands of an `isinstance(x, y)` call. anything else — a keyword @@ -328,14 +369,43 @@ mod tests { ); } - /// a literal rhs is identity in both languages, so the forward transform - /// leaves it as `is` and the reverse must not churn it #[test] - fn literal_comparisons_left_alone() { + fn none_comparisons_left_alone() { + // basedpython reads `is None` as a test for the type `None`, which is + // the same runtime check python's identity performs check("y = a is None\n", "y = a is None\n"); check("y = a is not None\n", "y = a is not None\n"); - check("y = a is True\n", "y = a is True\n"); - check("y = a is not 1\n", "y = a is not 1\n"); + } + + #[test] + fn a_classinfo_tuple_becomes_a_union() { + // `isinstance`'s tuple means "any of these"; a type expression spells + // that `A | B`, and a tuple there is the tuple *type* + check("y = isinstance(a, (int, str))\n", "y = a is int | str\n"); + check( + "y = not isinstance(a, (int, (str, bytes)))\n", + "y = a is not int | str | bytes\n", + ); + } + + #[test] + fn a_classinfo_no_type_expression_can_say_keeps_the_call() { + // a call is not a type expression, and neither is the empty tuple — + // whose test is always `False` and has no union to write + check( + "y = isinstance(a, type(a))\n", + "y = isinstance(a, type(a))\n", + ); + check("y = isinstance(a, ())\n", "y = isinstance(a, ())\n"); + } + + #[test] + fn other_literal_comparisons_take_the_identity_operator() { + // `a is 1` is identity in python but a test for the type `Literal[1]` + // in basedpython, and those part company for an int outside the + // interned range — so the operator has to be written back out + check("y = a is True\n", "y = a === True\n"); + check("y = a is not 1\n", "y = a !== 1\n"); } /// the comment case cannot round-trip byte for byte — the operator's layout diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 9766d46ea2..add7ce5cc3 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -111,6 +111,17 @@ pub(crate) struct PassContext { /// statement materialized inside an expression is a syntax error, not a /// composition pub(crate) statement_inserts: Vec<(TextSize, Vec)>, + /// Sub-statement edits standing in for a construct the same pass re-emits + /// somewhere else: the `_MISSING` a mutable default leaves in the + /// signature, whose written value the body guard evaluates instead. + /// + /// Identical to a [`template_edits`](Self::template_edits) entry except + /// that it leads every other edit at its span. The others are rewrites of + /// the construct, and the construct is no longer here — they materialize + /// where the pass re-emits it. Without this they would be ordered against + /// the substitution by shape alone, which cannot tell two substitutions of + /// one span apart + pub(crate) relocating_edits: Vec<(TextRange, Vec)>, /// Hard transpile errors a pass surfaced — abort the pipeline rather /// than emit partial / invalid output. Each entry is a human-readable /// message suitable for showing the user @@ -211,6 +222,11 @@ enum SubPatch { /// [`PassContext::statement_inserts`]. Materializes exactly like /// [`SubPatch::Template`]; the difference is only in what may claim it Statement(Vec), + /// a template standing in for a construct the same pass re-emits somewhere + /// else — see [`PassContext::relocating_edits`]. Materializes exactly like + /// [`SubPatch::Template`]; the difference is only that it leads every other + /// edit at its span + Relocating(Vec), } /// The sub-edits a template materializes, in position order: those nested in @@ -325,7 +341,9 @@ fn apply_within( out.push_source(source, cursor, s); match &all[idx].2 { SubPatch::Text(t) => out.push_generated(t, s), - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { let inner: Vec = contained[k + 1..] .iter() .copied() @@ -637,9 +655,9 @@ pub(crate) fn run_against_source<'a>( // keep their source bytes and the lowerings inside them compose &destructure_pass, // text-edit-emitting passes first (read source ranges). - // type_is must run before identity_swap so type-position `a is T` - // wins the first-wins overlap dedup over identity_swap's - // value-context `isinstance(a, T)` rewrite + // type_is rewrites a `-> a is T` return guard into `TypeIs[T]`, which + // claims the whole guard; `parametric_is` would otherwise lower the + // same `is` pair inside it &type_is_pass, // `from x export y` → `from x import y as y`: two source edits inside // an import statement, independent of every other pass @@ -1078,6 +1096,13 @@ pub(crate) fn run_against_source<'a>( let at = usize::from(at); (at, at, SubPatch::Statement(frags)) })) + .chain(ctx.relocating_edits.into_iter().map(|(r, frags)| { + ( + usize::from(r.start()), + usize::from(r.end()), + SubPatch::Relocating(frags), + ) + })) .collect(); // start asc. tie-break by edit shape: // 1. zero-width insertions first — they don't consume bytes, so any @@ -1087,30 +1112,34 @@ pub(crate) fn run_against_source<'a>( // 2. then wider replacements before narrower ones — so a wider edit // wins over (or, for templates, absorbs) a narrow one nested inside // it - // 3. at one identical span, a *substitution* — plain text, or a template - // with no `Src` passthrough — ahead of a *rewrite*, a template that - // re-emits part of the span. a substitution says the construct does not - // appear here at all, which a rewrite of it cannot outrank: the pass - // that substitutes may be relocating the construct (default - // re-evaluation moves a parameter default into the body), and the - // rewrite still materializes wherever the passthrough re-emits it + // 3. at one identical span, a *relocating* edit leads: it says the + // construct has moved, and the pass that moved it re-emits the span + // itself, so every other edit there materializes at the new home + // 4. then a *substitution* — plain text, or a template with no `Src` + // passthrough — ahead of a *rewrite*, a template that re-emits part of + // the span. a substitution says the construct does not appear here at + // all, which a rewrite of it cannot outrank sub_edits.sort_by(|a, b| { let priority = |e: &(usize, usize, SubPatch)| { let rewrites = i64::from(match &e.2 { SubPatch::Text(_) => false, - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { frags.iter().any(|frag| matches!(frag, Fragment::Src(_))) } }); let statement = i64::from(!matches!(e.2, SubPatch::Statement(_))); + let relocating = i64::from(!matches!(e.2, SubPatch::Relocating(_))); // (start, is_replacement_not_insertion, statement-insert-first, - // neg_end-for-wider-first, substitution-before-rewrite) + // neg_end-for-wider-first, relocating-first, + // substitution-before-rewrite) if e.1 == e.0 { - (e.0, 0i64, statement, 0i64, rewrites) // insertion + (e.0, 0i64, statement, 0i64, relocating, rewrites) // insertion } else { #[allow(clippy::cast_possible_wrap)] let neg_end = -(e.1 as i64); - (e.0, 1i64, statement, neg_end, rewrites) + (e.0, 1i64, statement, neg_end, relocating, rewrites) } }; priority(a).cmp(&priority(b)) @@ -1134,7 +1163,7 @@ pub(crate) fn run_against_source<'a>( } let is_template = matches!( sub_edits[i].2, - SubPatch::Template(_) | SubPatch::Statement(_) + SubPatch::Template(_) | SubPatch::Statement(_) | SubPatch::Relocating(_) ); for (m, edit) in sub_edits.iter().enumerate() { if m == i || claimed[m] { @@ -1182,7 +1211,9 @@ pub(crate) fn run_against_source<'a>( if !claimed[j] { match &sub_edits[j].2 { SubPatch::Text(t) => combined.push_generated(t, start), - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { let contained = template_claimees(frags, &sub_edits, &claimed, j, None); materialize_fragments( &mut combined, @@ -1204,7 +1235,9 @@ pub(crate) fn run_against_source<'a>( let repl = match &sub_edits[i].2 { // a plain-text replacement wins over anything inside it SubPatch::Text(t) => Replacement::generated(t, start), - SubPatch::Template(frags) | SubPatch::Statement(frags) => { + SubPatch::Template(frags) + | SubPatch::Statement(frags) + | SubPatch::Relocating(frags) => { // the claimees nested in this span materialize inside the // template's `Src` passthrough fragments let contained = diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 24bb5c43d5..f26717d3af 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -20,12 +20,10 @@ //! still compose and it is evaluated exactly once. //! //! How a *checked* form validates is decided by the **same engine that decides -//! `x is T`** — [`build_predicate`], via [`TypeInfo::parametric_cast_plan`]. The -//! two forms ask one question (does this value satisfy this specialization at -//! runtime) and differ only in two parameters: +//! `x is T`** — [`build_predicate`], via [`TypeInfo::parametric_is_plan`]. The +//! two forms ask one question (does this value satisfy this type at runtime) +//! over the same kind of expression, and differ in one parameter: //! -//! - [`TargetPosition::Type`], because a cast's target is a *type* expression -//! while an `is`-rhs is a value expression, so ty infers it differently; //! - [`ProbeStrictness::Lenient`], because a cast is an assertion: arguments the //! runtime cannot see are not held against the value, keeping //! `[1, 2] cast! list[int]` legal. An `is`-test is strict — a `True` narrows, @@ -52,9 +50,7 @@ use ruff_python_ast::{CastKind, Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::parametric_is::{ - PARAMETRIC_IS_RUNTIME, PROTOCOL_IS_RUNTIME, ProbeStrictness, TargetPosition, build_predicate, -}; +use super::parametric_is::{PredicateRuntime, ProbeStrictness, build_predicate}; use crate::type_info::{CastCheck, SoundnessCheck, TypeInfo}; /// the lambda parameter a predicate-form cast binds its value to, so the value @@ -151,9 +147,8 @@ struct CastLower<'a> { types: &'a dyn TypeInfo, edits: Vec<(TextRange, Vec)>, used: BTreeSet, - needs_parametric: bool, - needs_protocol: bool, - needs_conformance: bool, + /// the runtime helpers the emitted predicates call + runtimes: BTreeSet, } impl<'a> CastLower<'a> { @@ -162,9 +157,7 @@ impl<'a> CastLower<'a> { types, edits: Vec::new(), used: BTreeSet::new(), - needs_parametric: false, - needs_protocol: false, - needs_conformance: false, + runtimes: BTreeSet::new(), } } @@ -189,13 +182,10 @@ impl<'a> CastLower<'a> { &value_ref, value_arg, type_arg, - TargetPosition::Type, ProbeStrictness::Lenient, ); if !needs.all_plain && !needs.erased { - self.needs_parametric |= needs.parametric_runtime; - self.needs_protocol |= needs.protocol_runtime; - self.needs_conformance |= needs.conformance_runtime; + self.runtimes.extend(PredicateRuntime::used(&needs)); let mut fragments = vec![Fragment::Lit(format!("lambda {CAST_VALUE_PARAM}: "))]; fragments.extend(predicate); return (helper.as_predicate(), fragments); @@ -292,15 +282,8 @@ impl TypeAwarePass for CheckedCastPass { } // `_parametric_is` / `_by_protocol_is` must precede the predicates that // call them - if inner.needs_parametric { - ctx.required_imports.push(PARAMETRIC_IS_RUNTIME.to_owned()); - } - if inner.needs_protocol { - ctx.required_imports.push(PROTOCOL_IS_RUNTIME.to_owned()); - } - if inner.needs_conformance { - ctx.required_imports - .push(super::conformance::WITNESS_RUNTIME.to_owned()); + for runtime in inner.runtimes { + ctx.required_imports.push(runtime.source().to_owned()); } for helper in &inner.used { ctx.required_imports.push(helper.runtime().to_owned()); @@ -371,9 +354,13 @@ mod tests { let out = check( "from typing import Literal\n\ndef f(a: object):\n b = a cast! Literal[\"x\", \"y\"]\n", ); + // each arm pins the class as well as the value: python's `1 == True` + // would otherwise let a `bool` satisfy `Literal[1]` assert!( out.contains( - "b = _checked_cast_pred(a, lambda _by_cast_value: _by_cast_value in (\"x\", \"y\"))" + "b = _checked_cast_pred(a, lambda _by_cast_value: \ + ((type(_by_cast_value) is str and _by_cast_value == \"x\") or \ + (type(_by_cast_value) is str and _by_cast_value == \"y\")))" ), "got:\n{out}" ); @@ -387,7 +374,9 @@ mod tests { ); assert!( out.contains( - "b = _try_cast_pred(a, lambda _by_cast_value: _by_cast_value in (\"x\", \"y\"))" + "b = _try_cast_pred(a, lambda _by_cast_value: \ + ((type(_by_cast_value) is str and _by_cast_value == \"x\") or \ + (type(_by_cast_value) is str and _by_cast_value == \"y\")))" ), "got:\n{out}" ); @@ -400,7 +389,9 @@ mod tests { let out = check("from typing import Literal\n\ndef f(a: object):\n b = a cast? Literal[7]\n"); assert!( - out.contains("lambda _by_cast_value: _by_cast_value in (7,)"), + out.contains( + "lambda _by_cast_value: (type(_by_cast_value) is int and _by_cast_value == 7)" + ), "got:\n{out}" ); } @@ -411,7 +402,10 @@ mod tests { "from typing import Literal\n\ndef f(a: object):\n b = a cast? Literal[True] | None\n", ); assert!( - out.contains("lambda _by_cast_value: _by_cast_value in (True, None)"), + out.contains( + "lambda _by_cast_value: ((type(_by_cast_value) is bool and \ + _by_cast_value == True) or _by_cast_value is None)" + ), "got:\n{out}" ); } @@ -488,7 +482,7 @@ mod tests { fn union_arms_are_decomposed() { let out = check("def f(a: object):\n b = a cast? list[int] | None\n"); assert!( - out.contains("b = _try_cast_pred(a, lambda _by_cast_value: _parametric_is_lenient(_by_cast_value, list[int], (0,)) or _by_cast_value is None)"), + out.contains("b = _try_cast_pred(a, lambda _by_cast_value: (_parametric_is_lenient(_by_cast_value, list[int], (0,)) or _by_cast_value is None))"), "got:\n{out}" ); } @@ -684,8 +678,8 @@ mod tests { assert!( out.contains( "b = _checked_cast_pred(a, lambda _by_cast_value: \ - _parametric_is_lenient(_by_cast_value, A[int], (0,)) \ - or isinstance(_by_cast_value, str))" + (_parametric_is_lenient(_by_cast_value, A[int], (0,)) \ + or isinstance(_by_cast_value, str)))" ), "each arm lowered by its own kind: {out}" ); diff --git a/crates/by_transforms/src/transforms/coalesce_chain.rs b/crates/by_transforms/src/transforms/coalesce_chain.rs index 25c1b5ea4d..d7c6898b02 100644 --- a/crates/by_transforms/src/transforms/coalesce_chain.rs +++ b/crates/by_transforms/src/transforms/coalesce_chain.rs @@ -384,6 +384,9 @@ fn compare_is_not_none(left: Expr) -> Expr { range: TextRange::default(), left: Box::new(left), ops: Box::new([CmpOp::IsNot]), + // python identity against `None`, written `is not` — the lowering emits + // python, so it never wants basedpython's `!==` spelling + identity_ops: None, comparators: Box::new([Expr::NoneLiteral(ExprNoneLiteral { node_index: AtomicNodeIndex::NONE, range: TextRange::default(), diff --git a/crates/by_transforms/src/transforms/context_sensitive.rs b/crates/by_transforms/src/transforms/context_sensitive.rs index dcfc4b656e..d08fc82175 100644 --- a/crates/by_transforms/src/transforms/context_sensitive.rs +++ b/crates/by_transforms/src/transforms/context_sensitive.rs @@ -209,9 +209,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] + Empty: ClassVar[_Shape_Empty] @final @dataclass(frozen=True, slots=True) @@ -308,9 +309,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] + Empty: ClassVar[_Shape_Empty] @final @dataclass(frozen=True, slots=True) diff --git a/crates/by_transforms/src/transforms/enums.rs b/crates/by_transforms/src/transforms/enums.rs index 08baf667da..635a54f835 100644 --- a/crates/by_transforms/src/transforms/enums.rs +++ b/crates/by_transforms/src/transforms/enums.rs @@ -414,6 +414,26 @@ fn emit_sealed_hierarchy( .unwrap_or_default(); out.push_gen(&format!("{vis}class {name}{params}:\n")); + // each variant is *declared* in the body and *assigned* below it. the + // assignment is what the runtime needs; the declaration is what lets a + // reader — and the checker — see that `Shape.Point` is a value and + // `Shape.Circle` a class, which an assignment from outside the body does + // not say. the annotations are strings (the lowering emits `from __future__ + // import annotations`), so naming a class defined further down costs + // nothing at run time + if !variants.is_empty() { + imports.add("typing", "ClassVar"); + } + for variant in variants { + let variant_class = format!("_{name}_{}", variant.name); + let declared = match variant.kind { + // a payload variant is the class its call constructs + VariantKind::Tuple => format!("type[{variant_class}]"), + // a unit variant is the one instance of its class + VariantKind::Unit => variant_class, + }; + out.push_gen(&format!(" {}: ClassVar[{declared}]\n", variant.name)); + } // ordinary members (methods, classmethods, constants) — copied verbatim, // already indented under the enum in the source. they may refer to variants // (`A.Foo`) freely: the references resolve lazily at call time, by which @@ -421,7 +441,7 @@ fn emit_sealed_hierarchy( for member in members { emit_member(out, source, member); } - if members.is_empty() { + if members.is_empty() && variants.is_empty() { out.push_gen(" pass\n"); } // variant subclasses, emitted at module level and attached to the enum @@ -722,9 +742,9 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class _Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] @final @dataclass(frozen=True, slots=True) @@ -751,7 +771,9 @@ mod tests { "}, indoc! {" from __future__ import annotations + from typing import ClassVar class E: + A: ClassVar[_E_A] MAX: int = 10 class _E_A(E): @@ -776,9 +798,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] + Point: ClassVar[_Shape_Point] @final @dataclass(frozen=True, slots=True) @@ -812,9 +835,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Rectangle: ClassVar[type[_Shape_Rectangle]] + Polygon: ClassVar[type[_Shape_Polygon]] @final @dataclass(frozen=True, slots=True) @@ -848,9 +872,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Value: - pass + Pair: ClassVar[type[_Value_Pair]] + Nothing: ClassVar[_Value_Nothing] @final @dataclass(frozen=True, slots=True) @@ -888,8 +913,9 @@ mod tests { from __future__ import annotations from ty_extensions import JustFloat from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: + Circle: ClassVar[type[_Shape_Circle]] def area(self) -> JustFloat: return 0.0 @@ -940,9 +966,9 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Shape: - pass + Circle: ClassVar[type[_Shape_Circle]] @final @dataclass(frozen=True) @@ -969,9 +995,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class E: - pass + A: ClassVar[type[_E_A]] + B: ClassVar[_E_B] @final @dataclass(frozen=True, slots=True) @@ -1008,11 +1035,12 @@ mod tests { from __future__ import annotations from typing import TypeVar, Generic from dataclasses import dataclass - from typing import final + from typing import final, ClassVar _T = TypeVar(\"_T\") _E = TypeVar(\"_E\") class Result(Generic[_T, _E]): - pass + Ok: ClassVar[type[_Result_Ok]] + Err: ClassVar[type[_Result_Err]] @final @dataclass(frozen=True, slots=True) @@ -1045,9 +1073,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Result[T, E]: - pass + Ok: ClassVar[type[_Result_Ok]] + Err: ClassVar[type[_Result_Err]] @final @dataclass(frozen=True, slots=True) @@ -1083,9 +1112,10 @@ mod tests { indoc! {" from __future__ import annotations from dataclasses import dataclass - from typing import final + from typing import final, ClassVar class Tree[T]: - pass + Leaf: ClassVar[_Tree_Leaf] + Node: ClassVar[type[_Tree_Node]] class _Tree_Leaf(Tree): __slots__ = () diff --git a/crates/by_transforms/src/transforms/identity_swap.rs b/crates/by_transforms/src/transforms/identity_swap.rs index e815ae55bb..4501643eb7 100644 --- a/crates/by_transforms/src/transforms/identity_swap.rs +++ b/crates/by_transforms/src/transforms/identity_swap.rs @@ -1,24 +1,18 @@ -//! `===` / `!==` are real python identity; `is` / `is not` mean -//! `isinstance` / `not isinstance`. parser flattens both spellings to -//! `CmpOp::Is`/`IsNot`, so disambiguation reads the operator text from -//! source. +//! `===` / `!==` are real python identity, which python spells `is` / `is not`. +//! basedpython gives the `is` keyword to the type test instead, and the parser +//! folds both spellings onto the same [`CmpOp`], recording which one it saw. +//! +//! This pass lowers only the identity spelling, by replacing the operator text. +//! Every type test belongs to [`parametric_is`](super::parametric_is), which +//! decides its lowering from the target's *type* rather than from the shape the +//! target was written in. use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{ - Arguments, AtomicNodeIndex, CmpOp, Expr, ExprCall, ExprContext, ExprName, ExprUnaryOp, - ModModule, Operator, Stmt, UnaryOp, name::Name, -}; +use ruff_python_ast::{CmpOp, Expr, ModModule, Stmt}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use super::ast_driver::{AstPass, PassContext, render_expr}; - -/// whether the type-aware `parametric_is` pass owns lowering this `is`-rhs, so -/// `identity_swap` must leave it alone. it owns a name / attribute / subscript -/// target (may name a specialization or alias) and a `|` union of them -fn owned_by_parametric_is(rhs: &Expr) -> bool { - matches!(rhs, Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_)) - || matches!(rhs, Expr::BinOp(binop) if binop.op == Operator::BitOr) -} +use super::ast_driver::{AstPass, PassContext}; pub(crate) struct IdentitySwap<'src> { source: &'src str, @@ -51,93 +45,64 @@ struct State<'src> { impl State<'_> { fn process_compare(&mut self, c: &ruff_python_ast::ExprCompare) { let mut lhs_end = c.left.range().end(); - let mut lhs: &Expr = c.left.as_ref(); - for (op, rhs) in c.ops.iter().zip(c.comparators.iter()) { - let rhs_start = rhs.range().start(); - let between = &self.source[usize::from(lhs_end)..usize::from(rhs_start)]; - let trimmed = between.trim(); - match op { - CmpOp::Is => { - if trimmed.starts_with("===") { - if let Some(pos) = between.find("===") { - let op_start = lhs_end + TextSize::try_from(pos).unwrap(); - let op_range = - TextRange::new(op_start, op_start + TextSize::from(3u32)); - self.edits.push((op_range, "is".to_owned())); - } - } else if trimmed == "is" - && !rhs.is_literal_expr() - && !owned_by_parametric_is(rhs) - { - let call = isinstance_call(lhs.clone(), rhs.clone(), false); - let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); - self.edits.push((pair_range, render_expr(&call))); - } + for (index, rhs) in c.comparators.iter().enumerate() { + let gap = TextRange::new(lhs_end, rhs.range().start()); + if c.is_identity_operator(index) { + let replacement = match c.ops.get(index) { + Some(CmpOp::Is) => "is", + Some(CmpOp::IsNot) => "is not", + _ => unreachable!("only `is` / `is not` carry the identity spelling"), + }; + if let Some(range) = identity_operator_range(self.source, gap) { + // `===` needs no space around it and `is` does, so an + // operator the source wrote tight against its operands + // (`a===b`) has to bring its own + let before = self.source[..usize::from(range.start())] + .ends_with(|c: char| c.is_whitespace()); + let after = self.source[usize::from(range.end())..] + .starts_with(|c: char| c.is_whitespace()); + let padded = format!( + "{}{replacement}{}", + if before { "" } else { " " }, + if after { "" } else { " " }, + ); + self.edits.push((range, padded)); } - CmpOp::IsNot => { - if trimmed.starts_with("!==") { - if let Some(pos) = between.find("!==") { - let op_start = lhs_end + TextSize::try_from(pos).unwrap(); - let op_range = - TextRange::new(op_start, op_start + TextSize::from(3u32)); - self.edits.push((op_range, "is not".to_owned())); - } - } else if !rhs.is_literal_expr() && !owned_by_parametric_is(rhs) { - let call = isinstance_call(lhs.clone(), rhs.clone(), true); - let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); - self.edits.push((pair_range, render_expr(&call))); - } - } - _ => {} } lhs_end = rhs.range().end(); - lhs = rhs; } } } -fn isinstance_call(lhs: Expr, rhs: Expr, negate: bool) -> Expr { - let call = Expr::Call(ExprCall { - node_index: AtomicNodeIndex::NONE, - range_start: ruff_text_size::TextSize::default(), - func: Box::new(Expr::Name(ExprName { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - id: Name::from("isinstance"), - ctx: ExprContext::Load, - })), - arguments: Arguments { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - args: Box::new([lhs, rhs]), - keywords: thin_vec::ThinVec::new(), - }, - cast_kind: None, - is_string_tag: false, - }); - if negate { - Expr::UnaryOp(ExprUnaryOp { - node_index: AtomicNodeIndex::NONE, - range: TextRange::default(), - op: UnaryOp::Not, - operand: Box::new(call), - }) - } else { - call - } +/// the source range of the `===` / `!==` written in `gap` — the span between +/// the two operands it joins. +/// +/// The operand ranges stop inside any parentheses wrapping them, and a comment +/// may sit in the gap of a bracketed expression, so the operator is found by +/// tokenizing rather than by searching for its text. +fn identity_operator_range(source: &str, gap: TextRange) -> Option { + let start = SimpleTokenizer::new(source, gap) + .skip_trivia() + .find(|token| token.kind() != SimpleTokenKind::RParen)? + .start(); + let rest = &source[usize::from(start)..]; + ["===", "!=="] + .into_iter() + .find(|symbol| rest.starts_with(symbol)) + .map(|_| TextRange::at(start, TextSize::from(3u32))) } impl<'ast> Visitor<'ast> for State<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + walk_stmt(self, stmt); + } + fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Compare(c) = expr { - self.process_compare(c); + if let Expr::Compare(compare) = expr { + self.process_compare(compare); } walk_expr(self, expr); } - - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - walk_stmt(self, stmt); - } } #[cfg(test)] @@ -167,6 +132,14 @@ mod tests { check("x is int\n", "isinstance(x, int)\n"); } + #[test] + fn parenthesized_operands_still_lower() { + // an operand's own range stops inside its parentheses, so the rewrite + // has to close what it swallowed and open what the source closes after + check("(x) is int\n", "(isinstance(x, int))\n"); + check("x is ( int )\n", "(isinstance(x, int) )\n"); + } + #[test] fn is_not_to_not_isinstance() { check("x is not int\n", "not isinstance(x, int)\n"); @@ -188,24 +161,21 @@ mod tests { } #[test] - fn is_bool_kept() { - check("a is True\n", "a is True\n"); - check("a is False\n", "a is False\n"); - } - - #[test] - fn is_number_kept() { - check("a is 0\n", "a is 0\n"); - } - - #[test] - fn is_string_kept() { - check("a is \"x\"\n", "a is \"x\"\n"); - } - - #[test] - fn is_ellipsis_kept() { - check("a is ...\n", "a is ...\n"); + fn is_literal_tests_the_literal_type() { + // a literal names a type holding exactly the values equal to it, and the + // class guard is what keeps python's `1 == True` from widening that + check( + "def f(a: object):\n return a is True\n", + "def f(a: object):\n return (type(a) is bool and a == True)\n", + ); + check( + "def f(a: object):\n return a is 0\n", + "def f(a: object):\n return (type(a) is int and a == 0)\n", + ); + check( + "def f(a: object):\n return a is \"x\"\n", + "def f(a: object):\n return (type(a) is str and a == \"x\")\n", + ); } #[test] @@ -218,6 +188,13 @@ mod tests { check("a !== None\n", "a is not None\n"); } + #[test] + fn an_unspaced_identity_operator_brings_its_own_spaces() { + // `===` needs no space around it; `is` does + check("c = a===b\n", "c = a is b\n"); + check("d = a!==b\n", "d = a is not b\n"); + } + #[test] fn python_eq_unchanged() { unchanged("a == b\n"); diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index 9f74d082f3..a22b68bf84 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -56,6 +56,7 @@ impl TypeAwarePass for InitMethod<'_> { float_literals: self.float_literals, edits: RefCell::new(Vec::new()), templates: RefCell::new(Vec::new()), + relocating: RefCell::new(Vec::new()), errors: RefCell::new(Vec::new()), needs_missing: RefCell::new(false), }; @@ -67,6 +68,7 @@ impl TypeAwarePass for InitMethod<'_> { } ctx.text_edits.extend(state.edits.into_inner()); ctx.template_edits.extend(state.templates.into_inner()); + ctx.relocating_edits.extend(state.relocating.into_inner()); ctx.errors.extend(state.errors.into_inner()); } } @@ -101,6 +103,8 @@ struct State<'src> { float_literals: FloatLiteralLowering, edits: RefCell>, templates: RefCell)>>, + /// the `_MISSING` substitutions, whose defaults the guards re-evaluate + relocating: RefCell)>>, errors: RefCell>, /// whether the `_MISSING` sentinel reached the output needs_missing: RefCell, @@ -302,7 +306,11 @@ impl State<'_> { // the whole body is written here, so the guards a defaulted or // relaxed-order parameter needs are written here too — `mutable_ // defaults` has no source statement to anchor them before - let (sentinels, guards) = parameter_guards(func, self.types); + let super::mutable_defaults::ParameterGuards { + sentinels, + written, + guards, + } = parameter_guards(func, self.types); let header_indent = self.line_indent(func.range.start()).to_owned(); let body_indent = format!("{header_indent} "); let mut frags = vec![Fragment::Lit(":".to_owned())]; @@ -319,7 +327,8 @@ impl State<'_> { } // an inherited default is a signature edit that needs no guard, so the sentinels // are written whether or not the body gains anything - self.templates.borrow_mut().extend(sentinels); + self.relocating.borrow_mut().extend(sentinels); + self.templates.borrow_mut().extend(written); let pos = func.range.end(); if guards.is_empty() { // no passthrough to carry, so emit plain text: a template diff --git a/crates/by_transforms/src/transforms/intersection.rs b/crates/by_transforms/src/transforms/intersection.rs index 488e163b81..b593ff1479 100644 --- a/crates/by_transforms/src/transforms/intersection.rs +++ b/crates/by_transforms/src/transforms/intersection.rs @@ -558,4 +558,99 @@ mod tests { "}, ); } + + // a handful of typing constructs spell a type through a call rather than through an + // annotation. those arguments are type expressions, so the surface syntax written in + // one lowers exactly as it does after a `:` — an unlowered `A & B` reaching the output + // is a runtime `A.__and__(B)` + + #[test] + fn typevar_bound_lowers() { + check( + "import typing\nT = typing.TypeVar(\"T\", bound=A & B)\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + T = typing.TypeVar("T", bound=Intersection[A, B]) + "#}, + ); + } + + #[test] + fn typevar_constraints_lower() { + check( + "import typing\nT = typing.TypeVar(\"T\", A & B, C or D)\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + T = typing.TypeVar("T", Intersection[A, B], C | D) + "#}, + ); + } + + #[test] + fn paramspec_default_elements_lower() { + check( + "import typing\nP = typing.ParamSpec(\"P\", default=[A & B, C])\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + P = typing.ParamSpec("P", default=[Intersection[A, B], C]) + "#}, + ); + } + + #[test] + fn newtype_base_lowers() { + check( + "import typing\nD = typing.NewType(\"D\", A or B)\n", + indoc! {r#" + import typing + D = typing.NewType("D", A | B) + "#}, + ); + } + + #[test] + fn functional_named_tuple_field_lowers() { + check( + "import typing\nNT = typing.NamedTuple(\"NT\", [(\"f\", A & B)])\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + NT = typing.NamedTuple("NT", [("f", Intersection[A, B])]) + "#}, + ); + } + + #[test] + fn functional_typed_dict_field_lowers() { + check( + "import typing\nTD = typing.TypedDict(\"TD\", {\"f\": A & B})\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + TD = typing.TypedDict("TD", {"f": Intersection[A, B]}) + "#}, + ); + } + + #[test] + fn type_alias_type_value_lowers() { + check_py312( + "import typing\nAL = typing.TypeAliasType(\"AL\", A & B)\n", + indoc! {r#" + from ty_extensions import Intersection + import typing + AL = typing.TypeAliasType("AL", Intersection[A, B]) + "#}, + ); + } + + // the keyword is only a type operator where the call really is one of those + // constructs; an ordinary call's arguments stay boolean expressions + #[test] + fn ordinary_call_argument_untouched() { + unchanged("f(A and B)\n"); + } } diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index 9f1d129c21..ccd776af42 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -88,6 +88,8 @@ struct MutableDefaults<'src> { source: &'src str, types: &'src dyn TypeInfo, edits: Vec<(TextRange, Vec)>, + /// the `_MISSING` substitutions, whose defaults the guards re-evaluate + relocating: Vec<(TextRange, Vec)>, /// the guard suites, anchored at the body statement they precede guards: Vec<(TextSize, Vec)>, used: bool, @@ -122,11 +124,21 @@ fn written_default(pw: &ParameterWithDefault, value: &str) -> (TextRange, Vec (Vec<(TextRange, Vec)>, Vec) { +pub(crate) struct ParameterGuards { + /// the `_MISSING` left where a written default stood. *relocating*: the + /// guard evaluates the default's own source instead, so at this span this + /// edit leads every lowering written inside the default + pub(crate) sentinels: Vec<(TextRange, Vec)>, + /// a default written into the signature that the source did not write — an + /// inherited one, or the `_MISSING` a relaxed-order parameter needs. these + /// relocate nothing; they add text at the end of a parameter + pub(crate) written: Vec<(TextRange, Vec)>, + pub(crate) guards: Vec, +} + +pub(crate) fn parameter_guards(f: &StmtFunctionDef, types: &dyn TypeInfo) -> ParameterGuards { let mut sentinels = Vec::new(); + let mut written = Vec::new(); let mut guards = Vec::new(); let params = f.parameters.as_ref(); // positional parameters: swap non-scalar defaults for the sentinel, and give @@ -150,10 +162,10 @@ pub(crate) fn parameter_guards( // the parameters after it "after a default", exactly as a written one would None if let Some(value) = types.inherited_parameter_default(pw) => { seen_default = true; - sentinels.push(written_default(pw, &value)); + written.push(written_default(pw, &value)); } None if seen_default => { - sentinels.push(written_default(pw, "_MISSING")); + written.push(written_default(pw, "_MISSING")); guards.push(Guard::Required { name: pw.parameter.name.id.to_string(), function: f.name.id.to_string(), @@ -174,12 +186,16 @@ pub(crate) fn parameter_guards( Some(_) => {} None => { if let Some(value) = types.inherited_parameter_default(pw) { - sentinels.push(written_default(pw, &value)); + written.push(written_default(pw, &value)); } } } } - (sentinels, guards) + ParameterGuards { + sentinels, + written, + guards, + } } /// Whether the *callee's* body could evaluate `default` at all. @@ -237,8 +253,13 @@ impl MutableDefaults<'_> { if is_bodyless_init_shorthand(f) { return; } - let (sentinels, guards) = parameter_guards(f, self.types); - self.edits.extend(sentinels); + let ParameterGuards { + sentinels, + written, + guards, + } = parameter_guards(f, self.types); + self.relocating.extend(sentinels); + self.edits.extend(written); if guards.is_empty() { return; } @@ -281,6 +302,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { source: self.source, types, edits: Vec::new(), + relocating: Vec::new(), guards: Vec::new(), used: false, unanchored: Vec::new(), @@ -303,6 +325,7 @@ impl TypeAwarePass for MutableDefaultsPass<'_> { ctx.required_imports.push("_MISSING = object()".to_owned()); } ctx.template_edits.extend(inner.edits); + ctx.relocating_edits.extend(inner.relocating); ctx.statement_inserts.extend(inner.guards); } } @@ -880,7 +903,8 @@ mod tests { fn default_lowerings_survive() { // the default is re-emitted in the body through a `Src` passthrough, so // the lowerings written inside it land in the guard rather than being - // dropped with the signature they came from + // dropped with the signature they came from. `1 is int` is a type test + // the checker settles, so what lands is the constant it settled on check( indoc! {" def f(x = [1 is int]): @@ -890,7 +914,7 @@ mod tests { _MISSING = object() def f(x = _MISSING): if x is _MISSING: - x = [isinstance(1, int)] + x = [True] return x "}, ); @@ -899,8 +923,9 @@ mod tests { #[test] fn default_lowering_spanning_the_whole_default_survives() { // the sentinel and the `is` lowering claim the *same* span. the sentinel - // substitutes it and the lowering rewrites it, so the substitution - // decides the signature and the rewrite materializes in the guard + // relocates the default, so it decides the signature and the lowering + // materializes in the guard — which holds even here, where the lowering + // settles to a constant and so replaces the span just as flatly check( indoc! {" def f(x = 1 is int): @@ -910,7 +935,7 @@ mod tests { _MISSING = object() def f(x = _MISSING): if x is _MISSING: - x = isinstance(1, int) + x = True return x "}, ); diff --git a/crates/by_transforms/src/transforms/parametric_is.rs b/crates/by_transforms/src/transforms/parametric_is.rs index d5667430ee..77be606b25 100644 --- a/crates/by_transforms/src/transforms/parametric_is.rs +++ b/crates/by_transforms/src/transforms/parametric_is.rs @@ -42,22 +42,20 @@ //! //! [`build_predicate`] is the core of this pass *and* of //! [`checked_cast`](super::checked_cast): both ask one question — does this -//! value satisfy this specialization at runtime — over the same -//! [`ParametricIsPlan`]. they differ only in two parameters: -//! -//! - [`TargetPosition`], because a cast's target is a type expression while an -//! `is`-rhs is a value expression, so ty infers the two differently -//! - [`ProbeStrictness`], because an `is`-test must *earn* a `True` (it narrows) -//! while a cast is an assertion that only holds the value to arguments the -//! runtime can actually see +//! value satisfy this type at runtime — over the same [`ParametricIsPlan`], +//! built from the same type expression. they differ in one parameter, +//! [`ProbeStrictness`]: an `is`-test must *earn* a `True` (it narrows) while a +//! cast is an assertion that only holds the value to arguments the runtime can +//! actually see +use std::collections::BTreeSet; use std::fmt::Write as _; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{self as ast, CmpOp, Expr, Stmt}; +use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange}; -use ty_python_semantic::reified::is_keyword_comparison; -use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck}; +use ty_python_semantic::{ArgVariance, ParametricIsPlan, ProtocolMemberCheck, TargetSpelling}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; use crate::type_info::TypeInfo; @@ -257,7 +255,7 @@ def _parametric_is_lenient(value, alias, variances): /// covariant → subtype, 2 contravariant → supertype, 3 bivariant → any). /// annotations are read with `typing.get_type_hints` (resolving string /// annotations and inherited members), falling back to a raw `__mro__` walk -pub(crate) const PROTOCOL_IS_RUNTIME: &str = "\ +const PROTOCOL_IS_RUNTIME: &str = "\ _by_proto_missing = object() def _by_member_annotation(klass, name): @@ -392,6 +390,20 @@ def _by_protocol_is(value, members): return True "; +/// matches a value against a template literal type — a pattern such as +/// `f"a{int}b"`, whose type is the set of strings it can produce. +/// +/// the regular expression comes from the checker, which builds it from the same +/// reading of the pattern's holes that decides the static answer, so the test +/// accepts exactly the strings the type contains. a non-`str` value is not one +/// of them +const PATTERN_IS_RUNTIME: &str = "\ +import re as _by_re + +def _by_pattern_is(value, pattern): + return isinstance(value, str) and _by_re.fullmatch(pattern, value) is not None +"; + /// the runtime variance code `_by_variance_ok` expects fn variance_code(variance: ArgVariance) -> u8 { match variance { @@ -437,17 +449,6 @@ fn protocol_members_literal(checks: &[ProtocolMemberCheck]) -> String { format!("[{entries}]") } -/// which inference position a target expression lives in. this is the *only* -/// difference between an `is`-test and a checked cast at the front end: an -/// `is`-rhs is a value expression (its type is the class object), a `cast` -/// target is a type expression (its type is the instance). both then classify -/// through the same engine -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum TargetPosition { - Value, - Type, -} - /// how a runtime probe treats a value that carries no reification #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum ProbeStrictness { @@ -474,6 +475,8 @@ pub(crate) struct PredicateNeeds { pub(crate) protocol_runtime: bool, /// the predicate calls `_by_conforms` pub(crate) conformance_runtime: bool, + /// the predicate calls `_by_pattern_is` + pub(crate) pattern_runtime: bool, /// no arm carried a parametric claim — a plain `isinstance` covers the whole /// target, so a caller may use its compact shallow form instead pub(crate) all_plain: bool, @@ -497,72 +500,21 @@ impl PredicateNeeds { } } -/// the plan for one `(value, target)` pair, resolved through the position's -/// inference rules -fn plan_for( - types: &dyn TypeInfo, - value_expr: &Expr, - target: &Expr, - position: TargetPosition, -) -> Option { - match position { - TargetPosition::Value => types.parametric_is_plan(value_expr, target), - TargetPosition::Type => types.parametric_cast_plan(value_expr, target), - } -} - /// the bare (non-negated) runtime predicate for one target arm, referencing the /// value through `value_ref` so the caller controls how it is bound. /// /// this is the shared core of `x is T` and `x cast T`: both ask the same -/// question — does this value satisfy this specialization at runtime — and -/// differ only in `position` (how the target is inferred) and `probe` (what an -/// unreified value means). +/// question — does this value satisfy this type at runtime — and differ only in +/// `probe`, which decides what an unreified value means. fn arm_predicate( types: &dyn TypeInfo, value_ref: &dyn Fn() -> Fragment, value_expr: &Expr, arm: &Expr, - position: TargetPosition, probe: ProbeStrictness, needs: &mut PredicateNeeds, ) -> Vec { - // a `None` arm (an `X | None` optional) is an identity check, not - // `isinstance(_, None)` — `None` is a value, not a class - if matches!(arm, Expr::NoneLiteral(_)) { - needs.all_true = false; - needs.references_value = true; - return vec![value_ref(), Fragment::Lit(" is None".to_owned())]; - } - // an interface something visibly conforms to cannot be answered by - // `isinstance`: a conforming type is not a subclass, and a protocol is not - // even a legal `isinstance` target. the registry answers first, and a value - // nothing registered falls back to carrying the requirements - if let Some(test) = types.conformance_test(arm) { - needs.all_plain = false; - needs.all_true = false; - needs.references_value = true; - needs.conformance_runtime = true; - let members = match &test.members { - Some(names) => { - let mut spelled = String::from(", ("); - for name in names { - let _ = write!(spelled, "\"{name}\", "); - } - spelled.push(')'); - spelled - } - None => String::new(), - }; - return vec![ - Fragment::Lit("_by_conforms(".to_owned()), - value_ref(), - Fragment::Lit(", ".to_owned()), - Fragment::Src(arm.range()), - Fragment::Lit(format!("{members})")), - ]; - } - let Some(plan) = plan_for(types, value_expr, arm, position) else { + let Some(plan) = types.parametric_is_plan(value_expr, arm) else { needs.all_true = false; needs.references_value = true; return vec![ @@ -573,9 +525,33 @@ fn arm_predicate( Fragment::Lit(")".to_owned()), ]; }; - needs.all_plain = false; - if !matches!(plan, ParametricIsPlan::Fold(true)) { - needs.all_true = false; + fragments_for_plan(types, plan, value_ref, arm, probe, needs) +} + +/// the fragments one already-classified plan lowers to. split out from +/// [`arm_predicate`] so a union plan can build its arms, which the source need +/// not spell separately — a PEP 695 alias names a whole union with one word +fn fragments_for_plan( + types: &dyn TypeInfo, + plan: ParametricIsPlan, + value_ref: &dyn Fn() -> Fragment, + arm: &Expr, + probe: ProbeStrictness, + needs: &mut PredicateNeeds, +) -> Vec { + // a union reports through its arms instead: a disjunction of plain + // `isinstance` calls is still a plain check, and it holds as soon as one arm + // does + if !matches!(plan, ParametricIsPlan::Union(_)) { + if !matches!( + plan, + ParametricIsPlan::Isinstance(_) | ParametricIsPlan::Unresolved + ) { + needs.all_plain = false; + } + if !matches!(plan, ParametricIsPlan::Fold(true)) { + needs.all_true = false; + } } match plan { // an erased arm can't be checked at runtime; ty reports the error (a @@ -597,7 +573,7 @@ fn arm_predicate( frags.push(Fragment::Lit(")".to_owned())); frags } - ParametricIsPlan::Probe(variances) => { + ParametricIsPlan::Probe { target, variances } => { needs.parametric_runtime = true; needs.references_value = true; let codes: Vec = variances.iter().copied().map(variance_code).collect(); @@ -605,13 +581,14 @@ fn arm_predicate( ProbeStrictness::Strict => "_parametric_is(", ProbeStrictness::Lenient => "_parametric_is_lenient(", }; - vec![ + let mut frags = vec![ Fragment::Lit(call.to_owned()), value_ref(), Fragment::Lit(", ".to_owned()), - Fragment::Src(arm.range()), - Fragment::Lit(format!(", {})", variance_tuple(&codes))), - ] + target_fragment(&target, arm), + ]; + frags.push(Fragment::Lit(format!(", {})", variance_tuple(&codes)))); + frags } ParametricIsPlan::ProtocolStructural(checks) => { needs.protocol_runtime = true; @@ -623,35 +600,192 @@ fn arm_predicate( Fragment::Lit(format!(", {members})")), ] } + // nothing is known about the target, so the source's own spelling is + // the only thing to test against — and the error that left it unknown + // is already reported + // an interface something visibly conforms to cannot be answered by + // `isinstance`: a conforming type is not a subclass. the registry + // answers first, and a value nothing registered falls back to carrying + // the requirements + ParametricIsPlan::Conformance { target, members } => { + needs.references_value = true; + needs.conformance_runtime = true; + let mut spelled = String::from(", ("); + for name in &members { + let _ = write!(spelled, "\"{name}\", "); + } + spelled.push(')'); + vec![ + Fragment::Lit("_by_conforms(".to_owned()), + value_ref(), + Fragment::Lit(", ".to_owned()), + target_fragment(&target, arm), + Fragment::Lit(format!("{spelled})")), + ] + } + // the checker could not read the target as a type. that is how a unit + // enum variant arrives: the enum lowering has already rewritten it into + // a singleton instance, so what the source named as a type names a + // value here — and identity is the test for a value. anything else + // keeps the plain instance check, and whatever left the target unknown + // is already reported where it is written + ParametricIsPlan::Unresolved => { + needs.references_value = true; + if types.is_plain_value(arm) { + return vec![ + value_ref(), + Fragment::Lit(" is ".to_owned()), + Fragment::Src(arm.range()), + ]; + } + vec![ + Fragment::Lit("isinstance(".to_owned()), + value_ref(), + Fragment::Lit(", ".to_owned()), + Fragment::Src(arm.range()), + Fragment::Lit(")".to_owned()), + ] + } + ParametricIsPlan::Isinstance(target) => { + needs.references_value = true; + let mut frags = vec![ + Fragment::Lit("isinstance(".to_owned()), + value_ref(), + Fragment::Lit(", ".to_owned()), + ]; + frags.push(target_fragment(&target, arm)); + frags.push(Fragment::Lit(")".to_owned())); + frags + } + // `None` is a value, not a class, so `isinstance` cannot take it — and + // there is only one `None`, which makes identity the whole test + // a bare `Callable` asks exactly what `callable()` answers + ParametricIsPlan::IsCallable => { + needs.references_value = true; + vec![ + Fragment::Lit("callable(".to_owned()), + value_ref(), + Fragment::Lit(")".to_owned()), + ] + } + ParametricIsPlan::IsNone => { + needs.references_value = true; + vec![value_ref(), Fragment::Lit(" is None".to_owned())] + } + // the class guard is not redundant: python's `1 == True` would let a + // `bool` satisfy `Literal[1]` without it + ParametricIsPlan::Equality { class, value } => { + needs.references_value = true; + vec![ + Fragment::Lit("(type(".to_owned()), + value_ref(), + Fragment::Lit(format!(") is {class} and ")), + value_ref(), + Fragment::Lit(" == ".to_owned()), + target_fragment(&value, arm), + Fragment::Lit(")".to_owned()), + ] + } + ParametricIsPlan::Identity(target) => { + needs.references_value = true; + vec![ + value_ref(), + Fragment::Lit(" is ".to_owned()), + target_fragment(&target, arm), + ] + } + ParametricIsPlan::Subclass(target) => { + needs.references_value = true; + let mut frags = vec![Fragment::Lit("(isinstance(".to_owned()), value_ref()]; + frags.push(Fragment::Lit(", type) and issubclass(".to_owned())); + frags.push(value_ref()); + frags.push(Fragment::Lit(", ".to_owned())); + frags.push(target_fragment(&target, arm)); + frags.push(Fragment::Lit("))".to_owned())); + frags + } + // a template literal type is a set of strings; the regular expression + // ty built spells exactly the strings it produces + ParametricIsPlan::Pattern(pattern) => { + needs.pattern_runtime = true; + needs.references_value = true; + vec![ + Fragment::Lit("_by_pattern_is(".to_owned()), + value_ref(), + Fragment::Lit(format!(", {})", python_string_literal(&pattern))), + ] + } + // a value satisfies a union as soon as one arm holds + ParametricIsPlan::Union(arms) => { + let mut frags = vec![Fragment::Lit("(".to_owned())]; + let mut any_true = false; + for (index, arm_plan) in arms.iter().enumerate() { + if index > 0 { + frags.push(Fragment::Lit(" or ".to_owned())); + } + any_true |= matches!(arm_plan, ParametricIsPlan::Fold(true)); + frags.extend(fragments_for_plan( + types, + arm_plan.clone(), + value_ref, + arm, + probe, + needs, + )); + } + frags.push(Fragment::Lit(")".to_owned())); + if any_true { + needs.all_true = true; + } + frags + } + } +} + +/// how a plan's target is written: its own spelling, or the source the target +/// was written as when the plan carries none +fn target_fragment(target: &TargetSpelling, arm: &Expr) -> Fragment { + match target { + TargetSpelling::Written => Fragment::Src(arm.range()), + TargetSpelling::Rebuilt(text) => Fragment::Lit(text.clone()), } } -/// the full runtime predicate for a target expression, splitting a union into -/// the disjunction of its arms. shared by `is` and `cast`. +/// a python string literal for `text`, escaped so the emitted source reads it +/// back byte for byte. only the characters that end or reinterpret a +/// single-quoted literal need escaping +fn python_string_literal(text: &str) -> String { + let mut out = String::with_capacity(text.len() + 2); + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + _ => out.push(ch), + } + } + out.push('"'); + out +} + +/// the full runtime predicate for a target expression. shared by `is` and +/// `cast`. +/// +/// A union target needs no splitting here: the target is a type expression, so +/// its type is the union and the plan carries one arm per member — including +/// the arms of a union the source never spelled as one. pub(crate) fn build_predicate( types: &dyn TypeInfo, value_ref: &dyn Fn() -> Fragment, value_expr: &Expr, target: &Expr, - position: TargetPosition, probe: ProbeStrictness, ) -> (Vec, PredicateNeeds) { let mut needs = PredicateNeeds::new(); - let Some(arms) = union_arms(target) else { - let frags = arm_predicate( - types, value_ref, value_expr, target, position, probe, &mut needs, - ); - return (frags, needs); - }; - let mut frags = Vec::new(); - for (index, arm) in arms.iter().enumerate() { - if index > 0 { - frags.push(Fragment::Lit(" or ".to_owned())); - } - frags.extend(arm_predicate( - types, value_ref, value_expr, arm, position, probe, &mut needs, - )); - } + let frags = arm_predicate(types, value_ref, value_expr, target, probe, &mut needs); (frags, needs) } @@ -664,9 +798,8 @@ struct ParametricIs<'src, 'ti> { source: &'src str, types: &'ti dyn TypeInfo, edits: Vec<(TextRange, Vec)>, - needs_probe: bool, - needs_protocol: bool, - needs_conformance: bool, + /// the runtime helpers the predicates emitted so far call + runtimes: BTreeSet, } impl ParametricIs<'_, '_> { @@ -687,18 +820,26 @@ impl ParametricIs<'_, '_> { /// predicate builder and then wrapped for this form: negation, and keeping /// an effectful lhs alive when the predicate doesn't mention it fn lower_pair(&mut self, lhs: &Expr, rhs: &Expr, negate: bool) -> Vec { - let value = || Fragment::Src(lhs.range()); - let (frags, needs) = build_predicate( - self.types, - &value, - lhs, - rhs, - TargetPosition::Value, - ProbeStrictness::Strict, - ); - self.needs_probe |= needs.parametric_runtime; - self.needs_protocol |= needs.protocol_runtime; - self.needs_conformance |= needs.conformance_runtime; + // a predicate may mention the value more than once — a union tests each + // arm, an equality also checks the class — so an effectful left operand + // is bound to a lambda parameter and evaluated once. counting the + // references is what tells the two cases apart: a single mention needs + // no binding, and the lambda would only obscure the output + let mentions = std::cell::Cell::new(0usize); + let counting = || { + mentions.set(mentions.get() + 1); + Fragment::Src(lhs.range()) + }; + let (frags, needs) = + build_predicate(self.types, &counting, lhs, rhs, ProbeStrictness::Strict); + let via_lambda = mentions.get() > 1 && !effect_free(lhs); + let (frags, needs) = if via_lambda { + let param = || Fragment::Lit(UNION_VALUE_PARAM.to_owned()); + build_predicate(self.types, ¶m, lhs, rhs, ProbeStrictness::Strict) + } else { + (frags, needs) + }; + self.runtimes.extend(PredicateRuntime::used(&needs)); // a predicate that folded to a constant inverts in place rather than // growing a `not` @@ -713,10 +854,34 @@ impl ParametricIs<'_, '_> { return Self::with_lhs_effects(lhs, vec![Fragment::Lit(value.to_owned())]); } + // `x is not None` rather than `not x is None`: the same test, and the + // one a reader (and every linter) expects. an identity predicate is the + // only shape python can negate in place, and it is spelled with the + // operator in a literal fragment of its own + if negate + && let [value, Fragment::Lit(operator), rest @ ..] = frags.as_slice() + && operator.starts_with(" is ") + { + let mut negated = vec![ + value.clone(), + Fragment::Lit(operator.replacen(" is ", " is not ", 1)), + ]; + negated.extend_from_slice(rest); + return negated; + } + let mut result = Vec::new(); if negate { result.push(Fragment::Lit("not ".to_owned())); } + if via_lambda { + result.push(Fragment::Lit(format!("(lambda {UNION_VALUE_PARAM}: "))); + result.extend(frags); + result.push(Fragment::Lit(")(".to_owned())); + result.push(Fragment::Src(lhs.range())); + result.push(Fragment::Lit(")".to_owned())); + return result; + } result.extend(frags); if needs.references_value { result @@ -725,122 +890,94 @@ impl ParametricIs<'_, '_> { } } - /// one arm of a union `is`-target, delegated to the shared predicate - /// builder so `is` and `cast` stay in lockstep - fn lower_arm(&mut self, value: &dyn Fn() -> Fragment, lhs: &Expr, arm: &Expr) -> Vec { - let mut needs = PredicateNeeds::new(); - let frags = arm_predicate( - self.types, - value, - lhs, - arm, - TargetPosition::Value, - ProbeStrictness::Strict, - &mut needs, - ); - self.needs_probe |= needs.parametric_runtime; - self.needs_protocol |= needs.protocol_runtime; - self.needs_conformance |= needs.conformance_runtime; - frags - } - - /// `lhs is (T1 | T2 | …)` — a test against a union type — is the disjunction - /// of the per-arm tests (`type(lhs) <: Ti` for any arm). the lhs is bound - /// once: referenced directly when it has no effects, else through a lambda - /// parameter so the arms share a single evaluation - fn lower_union(&mut self, lhs: &Expr, arms: &[&Expr], negate: bool) -> Vec { - let via_lambda = !effect_free(lhs); - let value = || { - if via_lambda { - Fragment::Lit(UNION_VALUE_PARAM.to_owned()) - } else { - Fragment::Src(lhs.range()) - } - }; - let mut inner: Vec = Vec::new(); - for (index, arm) in arms.iter().enumerate() { - if index > 0 { - inner.push(Fragment::Lit(" or ".to_owned())); - } - inner.extend(self.lower_arm(&value, lhs, arm)); - } - let mut frags = Vec::new(); - if via_lambda { - frags.push(Fragment::Lit(format!( - "{}(lambda {UNION_VALUE_PARAM}: ", - if negate { "not " } else { "" } - ))); - frags.extend(inner); - frags.push(Fragment::Lit(")(".to_owned())); - frags.push(Fragment::Src(lhs.range())); - frags.push(Fragment::Lit(")".to_owned())); - } else { - frags.push(Fragment::Lit(if negate { "not (" } else { "(" }.to_owned())); - frags.extend(inner); - frags.push(Fragment::Lit(")".to_owned())); - } - frags - } - fn process_compare(&mut self, compare: &ast::ExprCompare) { let mut lhs: &Expr = &compare.left; - for (op, rhs) in compare.ops.iter().zip(&compare.comparators) { - // identity_swap defers every non-literal name/attribute/subscript - // and union `is`-rhs to this pass, which owns the - // isinstance-vs-parametric decision (a bare class → isinstance, a - // specialization or an alias to one → a parametric test, a union → - // the disjunction of the arms) - if matches!(op, CmpOp::Is | CmpOp::IsNot) - && is_keyword_comparison(self.source, *op, lhs, rhs) - // a subscript that resolves to a plain value (`candidates[0]` - // holding an enum member) keeps python identity semantics, - // same as identity_swap's rule for unsubscripted rhs - && !self.types.is_keeps_identity(rhs) - { - let negate = matches!(op, CmpOp::IsNot); - let replacement = - if matches!(rhs, Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_)) { - Some(self.lower_pair(lhs, rhs, negate)) - } else { - union_arms(rhs).map(|arms| self.lower_union(lhs, &arms, negate)) - }; - if let Some(replacement) = replacement { - let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); - self.edits.push((pair_range, replacement)); - } + for (index, rhs) in compare.comparators.iter().enumerate() { + // every type test is lowered here, whatever shape its target was + // written in. the plan comes from the target's *type*, so a target + // the source spells as one word (`x is Alias`) and the union that + // word stands for reach the same lowering + if compare.is_type_test(index, PySourceType::BasedPython) { + let negate = compare.ops.get(index) == Some(&CmpOp::IsNot); + let (opening, closing) = paren_padding(self.source, lhs, rhs); + let mut replacement = Vec::new(); + replacement.push(Fragment::Lit("(".repeat(opening))); + replacement.extend(self.lower_pair(lhs, rhs, negate)); + replacement.push(Fragment::Lit(")".repeat(closing))); + let pair_range = TextRange::new(lhs.range().start(), rhs.range().end()); + self.edits.push((pair_range, replacement)); } lhs = rhs; } } } -/// the lambda parameter that binds an effectful union-test lhs for its arms. -/// unlikely to collide: an `is`-target arm is a type expression, and this name -/// would have to appear free inside one -const UNION_VALUE_PARAM: &str = "_by_is_value"; +/// a runtime helper one of the emitted predicates calls. the preamble emits one +/// definition per helper a pass actually used, in this order — `_parametric_is` +/// and `_by_protocol_is` must precede the predicates that call them +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum PredicateRuntime { + Parametric, + Protocol, + Conformance, + Pattern, +} -/// the flat arms of a union type expression (`A | B | C` → `[A, B, C]`), or -/// `None` when `expr` is not a `|` union -fn union_arms(expr: &Expr) -> Option> { - if !matches!(expr, Expr::BinOp(binop) if binop.op == ast::Operator::BitOr) { - return None; +impl PredicateRuntime { + /// the definitions this helper needs in the emitted module + pub(crate) fn source(self) -> &'static str { + match self { + Self::Parametric => PARAMETRIC_IS_RUNTIME, + Self::Protocol => PROTOCOL_IS_RUNTIME, + Self::Conformance => super::conformance::WITNESS_RUNTIME, + Self::Pattern => PATTERN_IS_RUNTIME, + } } - let mut arms = Vec::new(); - collect_union_arms(expr, &mut arms); - Some(arms) -} -fn collect_union_arms<'a>(expr: &'a Expr, arms: &mut Vec<&'a Expr>) { - if let Expr::BinOp(binop) = expr - && binop.op == ast::Operator::BitOr - { - collect_union_arms(&binop.left, arms); - collect_union_arms(&binop.right, arms); - } else { - arms.push(expr); + /// the helpers `needs` recorded, in preamble order + pub(crate) fn used(needs: &PredicateNeeds) -> impl Iterator { + [ + needs.parametric_runtime.then_some(Self::Parametric), + needs.protocol_runtime.then_some(Self::Protocol), + needs.conformance_runtime.then_some(Self::Conformance), + needs.pattern_runtime.then_some(Self::Pattern), + ] + .into_iter() + .flatten() } } +/// the lambda parameter that binds an effectful test value for a predicate that +/// mentions it more than once. unlikely to collide: an `is`-target is a type +/// expression, and this name would have to appear free inside one +const UNION_VALUE_PARAM: &str = "_by_is_value"; + +/// how many parentheses a replacement for one comparison pair must supply +/// itself, because the source wrote them around an operand rather than around +/// the pair. +/// +/// An operand's range stops inside its own parentheses, so the pair +/// `lhs.start() .. rhs.end()` of `(a) is str` swallows the `)` while leaving the +/// `(` outside it. Rather than guess which outer `(` that was — the `(` of an +/// enclosing call looks exactly the same — the replacement closes what it +/// swallowed and opens what the source will close after it. +fn paren_padding(source: &str, lhs: &Expr, rhs: &Expr) -> (usize, usize) { + let gap = TextRange::new(lhs.range().end(), rhs.range().start()); + let tokens: Vec<_> = SimpleTokenizer::new(source, gap).skip_trivia().collect(); + // parens closing around the left operand come first in the gap, parens + // opening around the right operand last; the operator sits between them + let closing = tokens + .iter() + .take_while(|token| token.kind() == SimpleTokenKind::RParen) + .count(); + let opening = tokens + .iter() + .rev() + .take_while(|token| token.kind() == SimpleTokenKind::LParen) + .count(); + (opening, closing) +} + impl<'ast> Visitor<'ast> for ParametricIs<'_, '_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { walk_stmt(self, stmt); @@ -870,9 +1007,7 @@ impl TypeAwarePass for ParametricIsPass<'_> { source: self.source, types, edits: Vec::new(), - needs_probe: false, - needs_protocol: false, - needs_conformance: false, + runtimes: BTreeSet::new(), }; for stmt in stmts { inner.visit_stmt(stmt); @@ -882,15 +1017,8 @@ impl TypeAwarePass for ParametricIsPass<'_> { // (`T == list[int]`), which is already restricted to 3.12+ by the // reified-generic requirement; a user-generic probe (`A[int]`) works // on any target - if inner.needs_probe { - ctx.required_imports.push(PARAMETRIC_IS_RUNTIME.to_owned()); - } - if inner.needs_protocol { - ctx.required_imports.push(PROTOCOL_IS_RUNTIME.to_owned()); - } - if inner.needs_conformance { - ctx.required_imports - .push(super::conformance::WITNESS_RUNTIME.to_owned()); + for runtime in inner.runtimes { + ctx.required_imports.push(runtime.source().to_owned()); } ctx.template_edits.extend(inner.edits); } @@ -1344,7 +1472,8 @@ mod tests { #[test] fn implicit_alias_target_probes_like_the_specialization() { // `X = A[int]` binds `X` to the specialization itself, so `y is X` - // resolves exactly as `y is A[int]` would — a probe here + // resolves exactly as `y is A[int]` would — and is written back out + // that way, which is what the runtime probe needs to unwind let out = out(indoc! {" class A[T]: def __init__(self, v: T): @@ -1354,30 +1483,30 @@ mod tests { return y is X "}); assert!( - out.contains("return _parametric_is(y, X, (0,))"), - "alias name probes through `X`: {out}" + out.contains("return _parametric_is(y, A[int], (0,))"), + "alias name probes against the specialization it binds: {out}" ); } #[test] fn implicit_alias_builtin_target_probes() { // an alias to a builtin specialization probes just like the direct form - // — the alias name `X` is passed through so the runtime unwinds it let out = out(indoc! {" X = list[int] def f(y: object) -> bool: return y is X "}); assert!( - out.contains("return _parametric_is(y, X, (0,))"), - "alias to a builtin probes through `X`: {out}" + out.contains("return _parametric_is(y, list[int], (0,))"), + "alias to a builtin probes against the specialization: {out}" ); } #[test] fn pep695_type_alias_target_probes_through_value() { - // `type W = A[int]` is a `TypeAliasType`; the probe unwraps `.__value__` - // at runtime, so `y is W` still resolves against `A[int]` + // `type W = A[int]` evaluates to a `TypeAliasType`, which `isinstance` + // would refuse — so the specialization it stands for is written out + // instead of the name let out = out(indoc! {" class A[T]: def __init__(self, v: T): @@ -1387,8 +1516,8 @@ mod tests { return y is W "}); assert!( - out.contains("return _parametric_is(y, W, (0,))"), - "type alias probes through `W`: {out}" + out.contains("return _parametric_is(y, A[int], (0,))"), + "type alias probes against its value: {out}" ); } @@ -1428,10 +1557,10 @@ mod tests { def __init__(self, v: T): self.v: list[T] = [v] def f(a: object) -> bool: - return a is A[int] | object + return a is A[int] | str "}); assert!( - out.contains("return (_parametric_is(a, A[int], (0,)) or isinstance(a, object))"), + out.contains("return (_parametric_is(a, A[int], (0,)) or isinstance(a, str))"), "each arm lowered by its own kind: {out}" ); } @@ -1488,8 +1617,8 @@ mod tests { "}); assert!( out.contains( - "return (lambda _by_is_value: isinstance(_by_is_value, int) or \ - isinstance(_by_is_value, str))(g())" + "return (lambda _by_is_value: (isinstance(_by_is_value, int) or \ + isinstance(_by_is_value, str)))(g())" ), "effectful lhs bound once: {out}" ); @@ -1620,25 +1749,24 @@ mod tests { } #[test] - fn a_target_outside_the_union_compares_the_cell() { - // no arm matches `list[bytes]`, so this can never be true. it used to - // fold to `False` statically; now the parameter carries a reified type - // parameter constrained to `(int, str)`, so it lowers to a cell - // comparison that is always false at runtime instead. same answer, one - // comparison rather than a constant — the fold is not recoverable from - // the token path, which carries source ranges rather than types + fn a_target_outside_the_union_can_never_hold() { + // no arm matches `list[bytes]`, so this can never be true — and the + // constraint `(int, str)` on the reified parameter says so statically, + // before any runtime residue is reached let out = out(indoc! {" def f(x: list[int] | list[str]) -> bool: return x is list[bytes] "}); assert!( - out.contains("return (__by_erased_0 == bytes)"), - "target outside the union compares the cell: {out}" + out.contains("return False"), + "a target outside the union can never hold: {out}" ); } #[test] - fn value_subscript_rhs_falls_back_to_isinstance() { + fn a_subscript_of_a_value_keeps_the_plain_instance_check() { + // `pair[0]` is not something a type expression can say, so the checker + // reports it and the lowering keeps the `isinstance` the source wrote let out = out(indoc! {" class A: ... pair = (A, A) @@ -1680,9 +1808,9 @@ mod tests { #[test] fn stdlib_enum_member_rhs_keeps_identity() { - // an enum member is a singleton instance, not a class, so - // `isinstance(x, Color.RED)` would be a runtime TypeError; the pair - // must keep `is` / `is not` + // `Color.RED` names the type `Literal[Color.RED]`, which holds exactly + // one object — so the test is identity, and `isinstance(x, Color.RED)` + // (a runtime `TypeError`) is never emitted let out = out(indoc! {" from enum import Enum @@ -1690,17 +1818,106 @@ mod tests { RED = 1 GREEN = 2 - print(Color.RED is Color.RED) - print(Color.RED is not Color.GREEN) + def f(c: Color) -> None: + print(c is Color.RED) + print(c is not Color.GREEN) "}); assert!( - out.contains("print(Color.RED is Color.RED)"), - "enum member rhs keeps identity: {out}" + out.contains("print(c is Color.RED)"), + "enum member rhs is an identity check: {out}" ); assert!( - out.contains("print(Color.RED is not Color.GREEN)"), - "enum member rhs keeps identity under is not: {out}" + out.contains("print(c is not Color.GREEN)"), + "enum member rhs is an identity check under `is not`: {out}" + ); + } + + #[test] + fn an_alias_to_the_enum_still_reaches_the_variant() { + // the target is decided from the *type* it names, so a binding standing + // in for the enum reaches the same variant. a match on the written name + // would miss this one and emit `isinstance` against a singleton + let out = out(indoc! {" + enum class Shape: + case Circle(radius: float) + case Point + + S = Shape + + def f(s: Shape) -> bool: + return s is S.Point + "}); + assert!( + out.contains("return s is S.Point"), + "an aliased enum's unit variant is still an identity check: {out}" + ); + } + + #[test] + fn a_binding_shadowing_the_enum_names_what_it_holds() { + // the same rule the other way: `Shape` here is the local class, whose + // `Point` is an ordinary class and therefore an instance check + let out = out(indoc! {" + enum class Shape: + case Circle(radius: float) + case Point + + class Other: + class Point: pass + + def f(x: object) -> bool: + Shape = Other + return x is Shape.Point + "}); + assert!( + out.contains("return isinstance(x, Shape.Point)"), + "a shadowed name names what the binding holds: {out}" + ); + } + + #[test] + fn a_literal_target_keeps_the_source_spelling() { + // the source already wrote a literal that is valid where it sits — + // rebuilding one would pick its own quote character, which a python + // before 3.12 forbids reusing inside an f-string + let out = out(indoc! {" + def f(x: object) -> str: + return f\"{x is 'q'}\" + "}); + assert!( + out.contains("x == 'q'"), + "the literal is re-emitted as written: {out}" + ); + } + + #[test] + fn an_effectful_value_is_evaluated_once_for_a_literal_target() { + // the equality check mentions the value twice — once for the class + // guard — so an effectful operand is bound rather than repeated + let out = out(indoc! {" + def g() -> object: + return 1 + + def f() -> bool: + return g() is 1 + "}); + assert_eq!(out.matches("g()").count(), 2, "single evaluation:\n{out}"); + } + + #[test] + fn a_pattern_target_escapes_what_the_source_cannot_carry() { + // a control character has no raw spelling in python source — CPython + // refuses a file containing a NUL outright — so the regex carries its + // escape instead + let out = out(indoc! {" + def f(s: str) -> bool: + return s is f\"a\\x00b{int}\" + "}); + assert!( + out.contains("\\\\x00") || out.contains("a\\\\x00b"), + "the pattern escapes the control character: {out}" ); + assert!(!out.contains('\0'), "no raw NUL reaches the output: {out}"); } #[test] @@ -1710,11 +1927,12 @@ mod tests { enum class Genre: case A, B - print(Genre.A is not Genre.B) + def f(g: Genre) -> None: + print(g is not Genre.B) "}); assert!( - out.contains("print(Genre.A is not Genre.B)"), - "caseless variant rhs keeps identity: {out}" + out.contains("print(g is not Genre.B)"), + "caseless variant rhs is an identity check: {out}" ); } @@ -1727,12 +1945,12 @@ mod tests { case Circle(radius: float) case Point - c = Shape.Circle(1.0) - print(c is Shape.Circle) - print(c is not Shape.Point) + def f(c: Shape) -> None: + print(c is Shape.Circle) + print(c is not Shape.Point) "}); assert!( - out.contains("print(isinstance(c, Shape.Circle))"), + out.contains("print(isinstance(c, _Shape_Circle))"), "payload variant rhs is a class and lowers to isinstance: {out}" ); assert!( diff --git a/crates/by_transforms/src/transforms/reified_class.rs b/crates/by_transforms/src/transforms/reified_class.rs index 51984d3f55..fe2ae49431 100644 --- a/crates/by_transforms/src/transforms/reified_class.rs +++ b/crates/by_transforms/src/transforms/reified_class.rs @@ -227,7 +227,7 @@ impl<'src> ReifiedClass<'src> { } fn specialize(&mut self, class: &StmtClassDef) { - let reads = reified_class_reads(self.source, PySourceType::BasedPython, class); + let reads = reified_class_reads(PySourceType::BasedPython, class); if reads.names.is_empty() { return; } diff --git a/crates/by_transforms/src/transforms/reified_generic.rs b/crates/by_transforms/src/transforms/reified_generic.rs index d845701762..e965deadd6 100644 --- a/crates/by_transforms/src/transforms/reified_generic.rs +++ b/crates/by_transforms/src/transforms/reified_generic.rs @@ -230,7 +230,7 @@ impl<'src> ReifiedGeneric<'src> { } fn wrap(&mut self, function: &StmtFunctionDef) { - if reified_type_param_names(self.source, PySourceType::BasedPython, function).is_empty() { + if reified_type_param_names(PySourceType::BasedPython, function).is_empty() { return; } if !self.supports_native_generics { diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index 101903c6a0..de6c993228 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -32,6 +32,11 @@ //! 9. `Annotated[T, meta…]` only the first arg //! 10. `Callable[[P1, P2], R]` parameter list elements + return type //! 12. class base list +//! 13. the type-expression arguments of a typing construct that spells a type through a +//! call — `NewType("D", int)`, `TypeVar("T", bound=int)`, the functional +//! `NamedTuple` and `TypedDict`. which arguments those are is the type checker's +//! answer rather than a list kept here, so a form ty learns to check as a type is +//! one this lowers without further change use ruff_python_ast::helpers::declaration_annotation_type; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; @@ -374,6 +379,36 @@ impl<'ast> Visitor<'ast> for TypePosWalker<'_> { } return; } + + // a typing construct that spells a type through a call + if let Some(types) = self.types { + let type_arguments = types.call_type_expression_arguments(c); + if !type_arguments.is_empty() { + // a form can hold its types inside a literal — a `NamedTuple`'s + // field list, a `TypedDict`'s field dict — so an argument counts as + // handled once it *contains* one, and is not walked again as a value + let holds_type_argument = |expr: &Expr| { + type_arguments + .iter() + .any(|argument| expr.range().contains_range(argument.range())) + }; + for argument in &type_arguments { + self.visit_type_expr(argument, TypePos::Root); + } + // everything else the call holds is an ordinary value + for arg in &c.arguments.args { + if !holds_type_argument(arg) { + self.visit_expr(arg); + } + } + for kw in &c.arguments.keywords { + if !holds_type_argument(&kw.value) { + self.visit_expr(&kw.value); + } + } + return; + } + } } // lambda parameter annotations (basedpython supports typed lambdas) if let Expr::Lambda(l) = expr { diff --git a/crates/by_transforms/src/transforms/type_is.rs b/crates/by_transforms/src/transforms/type_is.rs index 88eea4e5ea..49ddc402e0 100644 --- a/crates/by_transforms/src/transforms/type_is.rs +++ b/crates/by_transforms/src/transforms/type_is.rs @@ -8,11 +8,10 @@ //! identical to PEP 742 `TypeIs[T]`; the parameter name is lost in //! lowering since `TypeIs` doesn't carry it. //! -//! traversal is delegated to [`type_expr_walker`] (with `types = None` — -//! value-position `a is T` is *not* a type expression here; it's the -//! basedpython surface form for `isinstance(a, T)`, owned by -//! `identity_swap`). running before `identity_swap` in the `AstPass` list so -//! type-position rewrites win the first-wins overlap dedup +//! traversal is delegated to [`type_expr_walker`] (with `types = None`): the +//! `a is T` written in a *return guard* is what this rewrites, and the same +//! pair written in a body is a type test that [`parametric_is`] lowers. this +//! pass claims the guard first, so the two never rewrite the same span use ruff_python_ast::helpers::{ReturnGuardForm, return_guards}; use ruff_python_ast::visitor::{Visitor, walk_stmt}; diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index cf9df35f79..35da2c5acc 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -10,6 +10,7 @@ use ruff_python_stdlib::basedpython::IMPLICIT_TYPING_NAMES; use ruff_text_size::TextRange; use ty_python_core::scope::ScopeKind; use ty_python_core::{global_scope, place_table, semantic_index}; +use ty_python_semantic::types::call_type_forms::CallTypeForm; use ty_python_semantic::types::{ DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, }; @@ -91,6 +92,18 @@ pub(crate) trait TypeInfo { /// unresolved name should be treated as a runtime subscript, not a type) fn subscript_is_known_type_context(&self, value: &Expr) -> bool; + /// the arguments of `call` that are type expressions, because the call is one of the + /// typing constructs that spells a type through a call — `NewType("D", int)`, + /// `TypeVar("T", bound=int)`, the functional `NamedTuple` and `TypedDict` + /// + /// the type checker decides this, from the same + /// [`CallTypeForm`](ty_python_semantic::types::call_type_forms::CallTypeForm) it uses + /// to check those arguments as types. a form ty checks as a type and the transpiler + /// leaves alone is a silent miscompilation: the surface syntax survives into the + /// output as ordinary python, where `A & B` is a runtime `__and__` call and + /// `A and B or C` evaluates to `C` + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr>; + fn is_function(&self, name: &ExprName) -> bool; /// basedpython: how many entries the class pattern's class lists in @@ -154,22 +167,16 @@ pub(crate) trait TypeInfo { rhs: &Expr, ) -> Option; - /// [`Self::parametric_is_plan`] for a checked cast's `(value, target)` pair. - /// the same classification engine backs both; only the target's inference - /// position differs (a cast target is a type expression) - fn parametric_cast_plan( - &self, - value: &Expr, - target: &Expr, - ) -> Option; - - /// whether an `is`/`is not` comparison whose rhs is `expr` keeps python - /// identity semantics instead of lowering to `isinstance`: true when - /// `expr` resolves to a plain *value* — an enum member (`Color.RED`, a - /// based-enum unit variant like `Shape.Point`), another literal, or an - /// instance of a concrete non-type class — which `isinstance` would - /// reject as its classinfo argument at runtime - fn is_keeps_identity(&self, expr: &Expr) -> bool; + /// whether the *value* `expr` evaluates to is a plain value rather than a + /// class — an enum member, a literal, an instance of a concrete non-type + /// class. `isinstance` would reject such a value as its classinfo argument, + /// so the runtime test for it is identity. + /// + /// Only consulted for a target the checker could not read as a type at all. + /// The enum lowering rewrites a unit variant into a singleton instance + /// before this pass runs, so `s is Shape.Point` reaches here naming + /// something that is a type in the source and a value in what is emitted + fn is_plain_value(&self, expr: &Expr) -> bool; /// whether `expr` is a PEP 604 union standing where the runtime will /// evaluate it — `isinstance(x, int | str)`, a `cast` target, an alias @@ -222,11 +229,6 @@ pub(crate) trait TypeInfo { attribute: &ruff_python_ast::ExprAttribute, ) -> Option; - /// how `x is ` is answered when `target` is an interface something - /// visibly conforms to. `None` when the ordinary `isinstance` lowering is - /// still right - fn conformance_test(&self, target: &Expr) -> Option; - /// the conversions a statement's value needs: an annotated assignment, an /// attribute assignment, or a `return`. one wrap for a value that converts /// whole, or one per element for a collection literal @@ -626,6 +628,14 @@ impl TypeInfo for SemanticModel<'_> { } } + fn call_type_expression_arguments<'ast>(&self, call: &'ast ExprCall) -> Vec<&'ast Expr> { + call.func + .inferred_type(self) + .and_then(|callee| CallTypeForm::of(self.db(), callee)) + .map(|form| form.type_expressions(&call.arguments)) + .unwrap_or_default() + } + fn is_function(&self, name: &ExprName) -> bool { name.inferred_type(self) .is_some_and(|ty| ty.as_function_literal().is_some()) @@ -683,22 +693,8 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::parametric_is_plan(self, lhs, rhs) } - fn parametric_cast_plan( - &self, - value: &Expr, - target: &Expr, - ) -> Option { - SemanticModel::parametric_cast_plan(self, value, target) - } - - fn is_keeps_identity(&self, expr: &Expr) -> bool { - expr.inferred_type(self).is_some_and(|ty| { - ty_python_semantic::types::basedpython_is_keeps_identity( - self.db(), - &self.program_environment(), - ty, - ) - }) + fn is_plain_value(&self, expr: &Expr) -> bool { + SemanticModel::denotes_plain_value(self, expr) } fn is_runtime_union(&self, expr: &Expr) -> bool { @@ -752,10 +748,6 @@ impl TypeInfo for SemanticModel<'_> { SemanticModel::witness_dispatch(self, attribute) } - fn conformance_test(&self, target: &Expr) -> Option { - SemanticModel::conformance_test(self, target) - } - fn statement_conversions( &self, stmt: &Stmt, diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index 311b81f375..669147ee48 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -339,6 +339,111 @@ print("ok") /// Returns `None` (test skips) when none is found — PEP 695 class syntax is a /// hard requirement here. /// pep 695 type parameters — what [`PROGRAM`] needs (3.12+) +/// basedpython whose module-level `assert`s exercise the runtime check each +/// *non-parametric* target lowers to. The checker's own tests pin the type it +/// reads and the transpiler's pin the text it emits; only running the result +/// says whether the check accepts the values the type contains and no others. +const TARGETS_PROGRAM: &str = r#" +from typing import Callable, Literal, Protocol, runtime_checkable + +class A: ... +class B(A): ... + +# a class is the `isinstance` it looks like +assert (B() is A) is True, "a subclass instance is an A" +assert (A() is B) is False, "a base instance is not a B" + +# `type[C]` asks two things, and the runtime can answer both +assert (B is type[A]) is True, "a subclass is a type[A]" +assert (B() is type[A]) is False, "an instance is not a type[A]" +assert (int is type[A]) is False, "an unrelated class is not a type[A]" + +# a literal names the values equal to it, with the class pinned: `1 == True` in +# python, and `Literal[1]` does not contain `True` +def literal_one(v: object) -> bool: + return v is Literal[1] + +assert literal_one(1) is True, "1 is a Literal[1]" +assert literal_one(True) is False, "True is not a Literal[1]" +assert literal_one(2) is False, "2 is not a Literal[1]" + +def literal_true(v: object) -> bool: + return v is Literal[True] + +assert literal_true(True) is True, "True is a Literal[True]" +assert literal_true(1) is False, "1 is not a Literal[True]" + +# a union holds when any arm does, including one the source spells with a +# single word +type Key = int | str + +def key(v: object) -> bool: + return v is Key + +assert key(1) is True, "an int is a Key" +assert key("a") is True, "a str is a Key" +assert key(1.5) is False, "a float is not a Key" + +# a template literal type is the set of strings its pattern produces +def item(s: str) -> bool: + return s is f"item-{int}" + +assert item("item-12") is True, "item-12 matches" +assert item("item--3") is True, "a negative renders with its sign" +assert item("item-0") is True, "zero renders as one digit" +assert item("item--0") is False, "str(-0) is 0, so -0 renders no such string" +assert item("item-01") is False, "a leading zero is not an int rendering" +assert item("item-ab") is False, "a non-numeric tail does not match" +assert item("item-") is False, "an empty hole does not match" +assert (1 is f"item-{int}") is False, "a non-str is never one of the strings" + +# a pattern's literal text matches itself, metacharacters and all +def dotted(s: str) -> bool: + return s is f"a.b{int}" + +assert dotted("a.b1") is True, "the dot is literal text" +assert dotted("axb1") is False, "and does not stand for any character" + +# `None` is a value, so its test is the identity python already performs +def optional(v: object) -> bool: + return v is None + +assert optional(None) is True, "None is None" +assert optional(0) is False, "0 is not None" + +# a bare `Callable` asks exactly what `callable()` answers +def call(v: object) -> bool: + return v is Callable + +assert call(len) is True, "a builtin is callable" +assert call(A) is True, "a class is callable" +assert call(1) is False, "an int is not" + +# a `@runtime_checkable` protocol is the check python itself performs +@runtime_checkable +class HasName(Protocol): + name: str + +class Named: + def __init__(self) -> None: + self.name = "x" + +assert (Named() is HasName) is True, "a value with the member satisfies it" +assert (1 is HasName) is False, "one without does not" + +# `is not` is the negation, evaluated once +calls = [] + +def once() -> object: + calls.append(1) + return 1 + +assert (once() is not Literal[2]) is True, "the negation holds" +assert len(calls) == 1, "an effectful value is evaluated once" + +print("ok") +"#; + const PEP695_PROBE: &str = "type X[T] = T"; /// pep 696 defaults in native syntax plus the `has_default()` accessor the probe @@ -406,6 +511,19 @@ fn parametric_protocol_checks_run_correctly() { run_program(&python, PROGRAM); } +#[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn every_target_kind_checks_correctly_at_runtime() { + let Some(python) = python_supporting(PEP695_PROBE) else { + eprintln!("skipping type-test target runtime test: no PEP 695-capable interpreter found"); + return; + }; + run_program(&python, TARGETS_PROGRAM); +} + #[test] #[expect( clippy::print_stderr, diff --git a/crates/ruff_dev/src/generate_ty_env_vars_reference.rs b/crates/ruff_dev/src/generate_ty_env_vars_reference.rs index a9a474fe6f..fa404d6186 100644 --- a/crates/ruff_dev/src/generate_ty_env_vars_reference.rs +++ b/crates/ruff_dev/src/generate_ty_env_vars_reference.rs @@ -91,10 +91,12 @@ fn generate() -> String { output.push_str("# Environment variables\n\n"); - // Partition and sort environment variables into TY_ and external variables. + // Partition and sort environment variables into the ones ty defines and the ones it only + // reads. `BY_` as well as `TY_`, because basedpython's own variables are as much ty's as + // the rest — listing them beside `VIRTUAL_ENV` would say ty merely observes them. let (ty_vars, external_vars): (BTreeSet<_>, BTreeSet<_>) = EnvVars::metadata() .iter() - .partition(|(var, _)| var.starts_with("TY_")); + .partition(|(var, _)| var.starts_with("TY_") || var.starts_with("BY_")); output.push_str("ty defines and respects the following environment variables:\n\n"); diff --git a/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by b/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by index 623b7c8278..1f808f6ebb 100644 --- a/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by +++ b/crates/ruff_linter/resources/test/fixtures/basedpython/BY001.by @@ -26,12 +26,14 @@ def not_coalescing(a, b, c): # a falsiness test is not the same test _ = a if a else b - # not a `None` comparison - _ = a if a is not b else b + # not a `None` comparison. written with the identity operator because `is` + # takes a type, and `b` is a value + _ = a if a !== b else b _ = a if a == None else b - # a chained comparison guards more than one thing - _ = a if a is not None is not b else b + # a chained comparison guards more than one thing. it is written with the + # identity operator because a type test may not join a chain at all + _ = a if a !== None !== b else b # the left operand has to be evaluated twice to write it this way, so it # cannot be anything that runs code — a subscription included diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by b/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by new file mode 100644 index 0000000000..df94bfc0c5 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B007_basedpython.by @@ -0,0 +1,31 @@ +"""a destructuring loop binds its control variables in the header's pattern rather +than in its target, which holds only the synthetic binder the pattern takes apart""" + + +class Point: + __match_args__ = ("x", "y") + x: int + y: int + + +def unused_capture(points: list[Point]) -> None: + for Point(x, y) in points: # B007 on `y` + print(x) + + +def both_used(points: list[Point]) -> None: + for Point(x, y) in points: + print(x, y) + + +# an underscore-prefixed capture says the omission is deliberate, as it does for a +# tuple target +def deliberately_unused(points: list[Point]) -> None: + for Point(x, _y) in points: + print(x) + + +# nesting reaches the captures at every depth +def nested(pairs: list[tuple[Point, Point]]) -> None: + for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + print(a) diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by new file mode 100644 index 0000000000..15ce8cea79 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM222_basedpython.by @@ -0,0 +1,20 @@ +# `or` and `and` inside a type expression are basedpython's keyword spellings of +# union and intersection, so what looks like boolean logic there is type syntax: +# `A or B` is the type `A | B`, not an expression whose value is always `A` +type Names = "foo" or "bar" + +type Both = int and str + + +def annotated(a: str or bytes, b: "x" or "y") -> int or None: ... + + +def nested(handlers: list[int or str]) -> None: ... + + +# a boolean operator in a value expression is untouched +def values(a: int, b: int) -> None: + if a or True: # SIM222 + pass + if b and False: # SIM223 + pass diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py index 0745cc6108..b6c12cdd86 100644 --- a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py +++ b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300.py @@ -44,3 +44,18 @@ # https://github.com/astral-sh/ruff/issues/14761 {"": print(1)} == print(2) {0: 1, **print(2)} == print(4) + + +# A type parameter is a name the caller chooses, not a constant, however the +# one-letter convention spells it +def compare_type_param[T](value: T) -> None: + if T == int: + pass + if T > value: + pass + + +class Box[T]: + def check(self) -> None: + if T == int: + pass diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by new file mode 100644 index 0000000000..603b283e4d --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM300_basedpython.by @@ -0,0 +1,16 @@ +# a `type def` computes a type from its type parameters, so its body compares +# those parameters against types — none of which is a constant on the left +type def Widen[X]: + if X <= int: + return int + return str + + +def reified_cell[reified T](x: T) -> None: + assert T == int + + +# a genuine yoda condition still fires +def yoda(age: int) -> None: + if 42 == age: # SIM300 + pass diff --git a/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by index 44ef59d12f..ca0d0fe3b0 100644 --- a/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E70_basedpython.by @@ -14,3 +14,55 @@ class Members: # a class definition written on one line is still reported class OneLiner: x = 1 + + +# a match type's `case` arms are type expressions, so the `:` in one separates a +# pattern from a type and opens no suite +class A: ... + + +class B: ... + + +type Swap[*Ts] = match *Ts: + case (A, B): (B, A) + case _: () + + +def nested_alias[T]() -> None: + type Widen[X] = match X: + case int: float + case _: X + + # a `match` statement after the alias is a compound statement again + match T: + case int(): pass + case _: pass + + +# a statement-expression `match` really does hold statements +def statement_expression(v: object) -> int: + result = match v: + case int(): 1 + case _: 0 + return result + + +class Holder: + type Inner[X] = match X: + case int: float + case _: X + + def method(self, v: object) -> None: + match v: + case int(): pass + + +def guarded(flag: bool) -> None: + if flag: + type Guarded[X] = match X: + case int: float + case _: X + + match flag: + case True: pass diff --git a/crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by new file mode 100644 index 0000000000..8f5f5953c6 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pycodestyle/E721_basedpython.by @@ -0,0 +1,11 @@ +# `is` is basedpython's isinstance check and `===` its identity comparison, so +# the two operators this rule names are not the ones python's message names +def compare(a: object, b: object) -> None: + if type(a) == type(b): + pass + if type(a) == int: + pass + + +def reified[reified T](x: T) -> None: + assert T == int diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by new file mode 100644 index 0000000000..d57c6a2936 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F632_basedpython.by @@ -0,0 +1,36 @@ +# basedpython spells identity `===` / `!==`; a plain `is` is a parametric type +# test, which the parser flattens to the same operator. Only the identity +# spellings, and an `is` whose right-hand side is a literal, compare identity +def identity(x: object) -> None: + print(1 === 1) # F632 + print(x !== "a") # F632 + + +def literal_right_hand_side(x: object) -> None: + print(1 is 1) # F632 + print(x is not "a") # F632 + + +def type_tests[T](x: T, xs: list[int]) -> None: + print(1 is T) + print(xs is list[int]) + # this one lowers to `isinstance([], [])`, which raises at runtime — but the + # mistake in it is the target, not the operator, so it is ty's to report + print([] is []) + + +# which spelling was written is recorded on the comparison, not recovered from the +# source between the operands, so it survives everything that can sit there +def spelling_survives_what_sits_between(x: object) -> None: + a = (1) === (1) # F632 + b = 1 \ + === 1 # F632 + c = (1 === # a comment between the operands + 1) # F632 + d = 1===1 # F632 + + +# each operator of a chain answers for its own spelling +def chained(x: object) -> None: + e = x < 1 === 1 # F632 + f = 1 === 1 !== 2 # F632 diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by index 57e96f2e30..2e4bd0ebf2 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F841_basedpython.by @@ -45,3 +45,39 @@ def ordinary_capture(value: object) -> int: return 1 case _: return 0 + + +# a destructuring binder always binds, so its captures are the ones the +# equivalent python spelling makes: `for Point(x, y) in points` binds like +# `for x, y in points`, and `let Point(x, y) := p` like `x, y = p`. None of +# those are the plain assignment this rule reports +class Point: + __match_args__ = ("x", "y") + x: int + y: int + + +def destructuring_binders(points: list[Point], p: Point) -> None: + let Point(a, b) := p + print(a) + + for Point(c, d) in points: + print(c) + + with open("f") as Point(e, g): + print(e) + + +def destructuring_parameter(Point(h, i): Point) -> int: + return h + + +# a `match` case and an `if let` bind only when the pattern matches, which is +# what python's own capture pattern does, so an unused capture still fires +def matching_binders(p: Point) -> int: + if let Point(j, k) := p: # F841 on `k` + return j + match p: + case Point(l, m): # F841 on `m` + return l + return 0 diff --git a/crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by b/crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by new file mode 100644 index 0000000000..3e394d6bc4 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_basedpython.by @@ -0,0 +1,20 @@ +from __future__ import annotations + + +# basedpython reads a bare string in a type position as the literal type +# `Literal["c"]` rather than as a forward reference, so its quotes carry the +# meaning and removing them would name a variable instead +c: "c" = "b" + + +def parameter(mode: "read") -> "write": ... + + +def body() -> None: + local: "held" = "held" + print(local) + + +# a forward reference is written the way every other type is +class Node: + parent: Node | None diff --git a/crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by b/crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by new file mode 100644 index 0000000000..6f5b554ff6 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/ruff/RUF021_basedpython.by @@ -0,0 +1,21 @@ +# `A and B or C` in a type expression is the intersection of `A` and `B` unioned +# with `C`, which the keywords already spell unambiguously — there is no python +# precedence to make clear with parentheses +class A: ... + + +class B: ... + + +class C: ... + + +def annotated(x: A and B or C) -> None: ... + + +type Alias = A and B or C + + +# a boolean expression in a value position still wants the parentheses +def values(a: bool, b: bool, c: bool) -> bool: + return a and b or c diff --git a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs index 33c3cf36ed..5af4e1dc93 100644 --- a/crates/ruff_linter/src/checkers/ast/analyze/expression.rs +++ b/crates/ruff_linter/src/checkers/ast/analyze/expression.rs @@ -1689,13 +1689,14 @@ pub(crate) fn expression(expr: &Expr, checker: &Checker) { comparators, range: _, node_index: _, + identity_ops: _, }, ) => { if checker.any_rule_enabled(&[Rule::NoneComparison, Rule::TrueFalseComparison]) { pycodestyle::rules::literal_comparisons(checker, compare); } if checker.is_rule_enabled(Rule::IsLiteral) { - pyflakes::rules::invalid_literal_comparison(checker, left, ops, comparators, expr); + pyflakes::rules::invalid_literal_comparison(checker, compare); } if checker.is_rule_enabled(Rule::TypeComparison) { pycodestyle::rules::type_comparison(checker, compare); @@ -1966,6 +1967,12 @@ pub(crate) fn expression(expr: &Expr, checker: &Checker) { } } Expr::BoolOp(bool_op) => { + // basedpython spells union and intersection with `or` and `and` inside a + // type expression, so `A or B` there is the type `A | B` rather than a + // boolean expression. None of the rules below describe that operator + if checker.semantic().in_basedpython_type_expression() { + return; + } if checker.is_rule_enabled(Rule::BooleanChainedComparison) { pylint::rules::boolean_chained_comparison(checker, bool_op); } diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 40a469e076..87868f759a 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -39,7 +39,7 @@ use ruff_python_ast::identifier::Identifier; use ruff_python_ast::name::QualifiedName; use ruff_python_ast::str::Quote; use ruff_python_ast::token::Tokens; -use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_pattern}; +use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_pattern, walk_with_item}; use ruff_python_ast::{ self as ast, AnyParameterRef, ArgOrKeyword, Comprehension, ElifElseClause, ExceptHandler, Expr, ExprContext, ExprFString, ExprTString, InterpolatedStringElement, Keyword, MatchCase, @@ -194,6 +194,26 @@ impl ExpectedDocstringKind { } } +/// basedpython: how a pattern's captures are bound, which follows the binder the +/// pattern sits in rather than the pattern itself +/// +/// a `match` case or an `if let` binds only when the pattern matches, which is what +/// python's own capture pattern does. the destructuring binders bind unconditionally, +/// and each one has a python spelling whose bindings they should match: `for Point(x, +/// y) in points` binds `x` and `y` the way `for x, y in points` does, and `let Point(x, +/// y) := p` the way `x, y = p` does +#[derive(Copy, Clone, Debug, Default)] +enum PatternBinder { + /// `case :`, `if let := ...` — a conditional capture + #[default] + Match, + /// `for in ...` — a loop variable + Loop, + /// `let := ...`, `with ... as `, `def f(: T)` — the + /// captures come out of unpacking a value that is always there + Unpacking, +} + pub(crate) struct Checker<'a> { /// The [`Parsed`] output for the source code. parsed: &'a Parsed, @@ -250,6 +270,8 @@ pub(crate) struct Checker<'a> { semantic_checker: SemanticSyntaxChecker, /// Errors collected by the `semantic_checker`. semantic_errors: RefCell>, + /// basedpython: the binder whose pattern is currently being visited + pattern_binder: PatternBinder, context: &'a LintContext<'a>, } @@ -307,6 +329,7 @@ impl<'a> Checker<'a> { target_version, semantic_checker: SemanticSyntaxChecker::new(), semantic_errors: RefCell::default(), + pattern_binder: PatternBinder::default(), context, } } @@ -1703,6 +1726,19 @@ impl<'a> Visitor<'a> for Checker<'a> { self.visit_body(orelse); self.semantic.flags = flags_snapshot; } + // basedpython: `let := ` unpacks the subject, and falls + // through to the `else` block when the pattern does not match + Stmt::Let(ast::StmtLet { + pattern, + value, + orelse, + range: _, + node_index: _, + }) => { + self.visit_expr(value); + self.visit_pattern_binder(PatternBinder::Unpacking, pattern); + self.visit_body(orelse); + } Stmt::For(ast::StmtFor { node_index: _, range: _, @@ -1717,7 +1753,7 @@ impl<'a> Visitor<'a> for Checker<'a> { self.visit_expr(target); // basedpython: a destructuring loop binds the pattern's captures if let Some(pattern) = pattern { - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Loop, pattern); } self.visit_body(body); let flags_snapshot = self.semantic.flags; @@ -1739,7 +1775,7 @@ impl<'a> Visitor<'a> for Checker<'a> { // its subject against the pattern rather than testing it if let Some(pattern) = pattern { self.visit_expr(test); - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Match, pattern); } else { self.visit_boolean_test(test); } @@ -1822,15 +1858,21 @@ impl<'a> Visitor<'a> for Checker<'a> { && (self.semantic.in_annotation() || self.source_type.is_stub()) { if let Expr::StringLiteral(string_literal) = expr { - self.visit - .string_type_definitions - .push((string_literal, self.semantic.snapshot())); + // basedpython promotes bare string literals in type positions to + // `Literal["..."]` rather than treating them as forward references, + // so there is no annotation inside the quotes to defer + if !self.source_type.is_basedpython() { + self.visit + .string_type_definitions + .push((string_literal, self.semantic.snapshot())); + return; + } } else { self.visit .future_type_definitions .push((expr, self.semantic.snapshot())); + return; } - return; } self.semantic.push_node(expr); @@ -2515,13 +2557,21 @@ impl<'a> Visitor<'a> for Checker<'a> { // basedpython: a destructuring parameter binds its pattern's captures in // the same scope the parameter itself is bound in if let Some(pattern) = parameter.pattern.as_deref() { - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Unpacking, pattern); } // Step 4: Analysis analyze::parameter(parameter, self); } + fn visit_with_item(&mut self, with_item: &'a ast::WithItem) { + // basedpython: `with ctx() as Point(x, y):` unpacks the bound value. a with item + // holds no statements, so the binder can stand for the whole walk + let snapshot = std::mem::replace(&mut self.pattern_binder, PatternBinder::Unpacking); + walk_with_item(self, with_item); + self.pattern_binder = snapshot; + } + fn visit_pattern(&mut self, pattern: &'a Pattern) { // Step 1: Binding if let Pattern::MatchAs(ast::PatternMatchAs { @@ -2543,12 +2593,20 @@ impl<'a> Visitor<'a> for Checker<'a> { // `is_basedpython_transpile_resolved_name` makes applies: a name that // matches no variant at all stays an ordinary capture, and ty reports // one whose subject does not accept it - let flags = if self.semantic.is_based_enum_case_name(name.as_str()) { + let mut flags = if self.semantic.is_based_enum_case_name(name.as_str()) { BindingFlags::BASED_ENUM_CASE_NAME } else { BindingFlags::empty() }; - self.add_binding(name, name.range(), BindingKind::Assignment, flags); + let kind = match self.pattern_binder { + PatternBinder::Match => BindingKind::Assignment, + PatternBinder::Loop => BindingKind::LoopVar, + PatternBinder::Unpacking => { + flags |= BindingFlags::UNPACKED_ASSIGNMENT; + BindingKind::Assignment + } + }; + self.add_binding(name, name.range(), kind, flags); } // Step 2: Traversal @@ -2569,7 +2627,7 @@ impl<'a> Visitor<'a> for Checker<'a> { } fn visit_match_case(&mut self, match_case: &'a MatchCase) { - self.visit_pattern(&match_case.pattern); + self.visit_pattern_binder(PatternBinder::Match, &match_case.pattern); if let Some(expr) = &match_case.guard { self.visit_boolean_test(expr); } @@ -2870,12 +2928,20 @@ impl<'a> Checker<'a> { self.semantic.flags = snapshot; } + /// basedpython: visit a [`Pattern`] as the given binder's, so that its captures are + /// bound the way that binder's python spelling binds them + fn visit_pattern_binder(&mut self, binder: PatternBinder, pattern: &'a Pattern) { + let snapshot = std::mem::replace(&mut self.pattern_binder, binder); + self.visit_pattern(pattern); + self.pattern_binder = snapshot; + } + /// Visit an [`ElifElseClause`] fn visit_elif_else_clause(&mut self, clause: &'a ElifElseClause) { if let Some(test) = &clause.test { if let Some(pattern) = &clause.pattern { self.visit_expr(test); - self.visit_pattern(pattern); + self.visit_pattern_binder(PatternBinder::Match, pattern); } else { self.visit_boolean_test(test); } diff --git a/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs b/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs index 52ce7d36de..1384ab5f6c 100644 --- a/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs +++ b/crates/ruff_linter/src/rules/basedpython/rules/manual_isinstance.rs @@ -1,6 +1,6 @@ use ruff_diagnostics::Applicability; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::{self as ast, Expr, UnaryOp}; +use ruff_python_ast::{self as ast, Expr, Operator, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use crate::checkers::ast::Checker; @@ -43,7 +43,7 @@ use crate::{AlwaysFixableViolation, Edit, Fix}; /// than as several classes. /// /// ## References -/// - [basedpython documentation: identity and isinstance](https://docs.basedpython.org/features/identity-swap) +/// - [basedpython documentation: type tests and identity](https://docs.basedpython.org/features/identity-swap) #[derive(ViolationMetadata)] #[violation_metadata(stable_since = "0.0.1-a10", category = Category::Style)] pub(crate) struct ManualIsinstance; @@ -59,6 +59,24 @@ impl AlwaysFixableViolation for ManualIsinstance { } } +/// whether `classinfo` is something a type expression can name — a name, a +/// dotted name, a subscript of one, or a `|` union of those. +/// +/// `isinstance` takes a runtime value, and most of what can stand there is not +/// a type expression: a tuple means the tuple *type* rather than a choice of +/// classes, and `type(y)` or `registry["a"]` mean nothing there at all. +fn names_a_type(classinfo: &Expr) -> bool { + match classinfo { + Expr::Name(_) => true, + Expr::Attribute(attribute) => names_a_type(&attribute.value), + Expr::Subscript(subscript) => names_a_type(&subscript.value), + Expr::BinOp(binop) if binop.op == Operator::BitOr => { + names_a_type(&binop.left) && names_a_type(&binop.right) + } + _ => false, + } +} + /// BY003 pub(crate) fn manual_isinstance(checker: &Checker, call: &ast::ExprCall) { if !checker.source_type.is_basedpython() { @@ -76,9 +94,11 @@ pub(crate) fn manual_isinstance(checker: &Checker, call: &ast::ExprCall) { if !call.arguments.keywords.is_empty() { return; } - // a tuple in the class position reads as a tuple type, not as a choice of - // classes - if class.is_tuple_expr() { + // `is` takes a *type expression*, so the rewrite is only available when the + // classinfo argument is something one can say. a tuple reads as the tuple + // type rather than as a choice of classes; a call or a subscript of a value + // is not a type expression at all + if !names_a_type(class) { return; } diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs b/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs index 7a2cc0413b..06e879914b 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/mod.rs @@ -77,6 +77,7 @@ mod tests { #[test_case(Rule::UnintentionalTypeAnnotation, Path::new("B032.py"))] #[test_case(Rule::UnreliableCallableCheck, Path::new("B004.py"))] #[test_case(Rule::UnusedLoopControlVariable, Path::new("B007.py"))] + #[test_case(Rule::UnusedLoopControlVariable, Path::new("B007_basedpython.by"))] #[test_case(Rule::UselessComparison, Path::new("B015.ipynb"))] #[test_case(Rule::UselessComparison, Path::new("B015.py"))] #[test_case(Rule::UselessContextlibSuppress, Path::new("B022.py"))] diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs index c896454d8d..20398c64bf 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/unused_loop_control_variable.rs @@ -2,9 +2,10 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; use ruff_python_ast::helpers; use ruff_python_ast::helpers::{NameFinder, StoredNameFinder}; -use ruff_python_ast::visitor::Visitor; +use ruff_python_ast::visitor::{Visitor, walk_pattern}; use ruff_python_semantic::Binding; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashMap; use crate::checkers::ast::Checker; use crate::codes::Category; @@ -83,10 +84,26 @@ impl Violation for UnusedLoopControlVariable { /// B007 pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::StmtFor) { - let control_names = { + let control_names: FxHashMap<&str, TextRange> = { let mut finder = StoredNameFinder::default(); finder.visit_expr(stmt_for.target.as_ref()); - finder.names + let mut names: FxHashMap<&str, TextRange> = finder + .names + .into_iter() + .map(|(name, expr)| (name, expr.range())) + .collect(); + + // basedpython: a destructuring loop binds its control variables in the header's + // pattern rather than in its target, which holds only the synthetic binder the + // pattern takes apart. `for Point(x, y) in points` controls `x` and `y` exactly + // as `for x, y in points` does + if let Some(pattern) = &stmt_for.pattern { + let mut finder = PatternCaptureFinder::default(); + finder.visit_pattern(pattern); + names.extend(finder.names); + } + + names }; let used_names = { @@ -101,7 +118,7 @@ pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::St clippy::iter_over_hash_type, reason = "iteration order does not affect the diagnostics or fixes produced" )] - for (name, expr) in control_names { + for (name, range) in control_names { // Ignore names that are already underscore-prefixed. if checker.settings().ignores_unused_binding(name) { continue; @@ -137,7 +154,7 @@ pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::St rename: rename.clone(), certainty, }, - expr.range(), + range, ); if certainty == Certainty::Certain { @@ -152,19 +169,43 @@ pub(crate) fn unused_loop_control_variable(checker: &Checker, stmt_for: &ast::St if scope .get_all(name) .map(|binding_id| checker.semantic().binding(binding_id)) - .filter(|binding| binding.start() >= expr.start()) + .filter(|binding| binding.start() >= range.start()) .all(Binding::is_unused) { - diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement( - rename, - expr.range(), - ))); + diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement(rename, range))); } } } } } +/// A [`Visitor`] that collects the names a pattern captures, with the range of each. +/// +/// basedpython binds a destructuring loop's control variables here rather than in the +/// loop target, so the rule has to read them out of the pattern to see them at all. +#[derive(Default)] +struct PatternCaptureFinder<'a> { + names: Vec<(&'a str, TextRange)>, +} + +impl<'a> Visitor<'a> for PatternCaptureFinder<'a> { + fn visit_pattern(&mut self, pattern: &'a ast::Pattern) { + if let ast::Pattern::MatchAs(ast::PatternMatchAs { + name: Some(name), .. + }) + | ast::Pattern::MatchStar(ast::PatternMatchStar { + name: Some(name), .. + }) + | ast::Pattern::MatchMapping(ast::PatternMatchMapping { + rest: Some(name), .. + }) = pattern + { + self.names.push((name.as_str(), name.range())); + } + walk_pattern(self, pattern); + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum Certainty { Certain, diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap new file mode 100644 index 0000000000..3cbc372032 --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__unused-loop-control-variable_B007_basedpython.by.snap @@ -0,0 +1,73 @@ +--- +source: crates/ruff_linter/src/rules/flake8_bugbear/mod.rs +--- +B007 [*] Loop control variable `y` not used within loop body + --> B007_basedpython.by:12:18 + | +11 | def unused_capture(points: list[Point]) -> None: +12 | for Point(x, y) in points: # B007 on `y` + | ^ +13 | print(x) + | +help: Rename unused `y` to `_y` + | +11 | def unused_capture(points: list[Point]) -> None: + - for Point(x, y) in points: # B007 on `y` +12 + for Point(x, _y) in points: # B007 on `y` +13 | print(x) + | +note: This is an unsafe fix and may change runtime behavior + +B007 [*] Loop control variable `b` not used within loop body + --> B007_basedpython.by:30:19 + | +28 | # nesting reaches the captures at every depth +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: +30 | for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + | ^ +31 | print(a) + | +help: Rename unused `b` to `_b` + | +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: + - for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +30 + for (Point(a, _b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +31 | print(a) + | +note: This is an unsafe fix and may change runtime behavior + +B007 [*] Loop control variable `c` not used within loop body + --> B007_basedpython.by:30:29 + | +28 | # nesting reaches the captures at every depth +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: +30 | for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + | ^ +31 | print(a) + | +help: Rename unused `c` to `_c` + | +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: + - for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +30 + for (Point(a, b), Point(_c, d)) in pairs: # B007 on `b`, `c` and `d` +31 | print(a) + | +note: This is an unsafe fix and may change runtime behavior + +B007 [*] Loop control variable `d` not used within loop body + --> B007_basedpython.by:30:32 + | +28 | # nesting reaches the captures at every depth +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: +30 | for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` + | ^ +31 | print(a) + | +help: Rename unused `d` to `_d` + | +29 | def nested(pairs: list[tuple[Point, Point]]) -> None: + - for (Point(a, b), Point(c, d)) in pairs: # B007 on `b`, `c` and `d` +30 + for (Point(a, b), Point(c, _d)) in pairs: # B007 on `b`, `c` and `d` +31 | print(a) + | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs index 52261b2409..a0968cbf41 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs @@ -322,7 +322,7 @@ pub(crate) fn unittest_assertion( && checker.semantic().current_expression_parent().is_none() && !checker.comment_ranges().intersects(expr.range()) { - if let Ok(stmt) = unittest_assert.generate_assert(args, keywords) { + if let Ok(stmt) = unittest_assert.generate_assert(args, keywords, checker.source_type) { diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement( checker.generator().stmt(&stmt), parenthesized_range( diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs index 987184bcfa..01f0b69041 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/unittest_assert.rs @@ -1,7 +1,8 @@ use anyhow::{Result, anyhow, bail}; use ruff_python_ast::name::Name; use ruff_python_ast::{ - self as ast, Arguments, CmpOp, Expr, ExprContext, Identifier, Keyword, Stmt, UnaryOp, + self as ast, Arguments, CmpOp, Expr, ExprContext, Identifier, Keyword, PySourceType, Stmt, + UnaryOp, }; use ruff_text_size::TextRange; use rustc_hash::{FxBuildHasher, FxHashMap}; @@ -170,11 +171,26 @@ fn assert(expr: &Expr, msg: Option<&Expr>) -> Stmt { }) } -fn compare(left: &Expr, cmp_op: CmpOp, right: &Expr) -> Expr { +fn compare(left: &Expr, cmp_op: CmpOp, right: &Expr, source_type: PySourceType) -> Expr { Expr::Compare(ast::ExprCompare { left: Box::new(left.clone()), ops: Box::from([cmp_op]), comparators: Box::from([right.clone()]), + // `assertIs` asserts python *identity*, which basedpython writes `===` + // — its `is` keyword is a type test, a different assertion entirely. + // recording no spelling prints `is`, so this has to say so. + // + // `assertIsNone` is the exception: `is None` is a test for the type + // `None`, which holds one object, so it is the same test either way — + // and it is the spelling a reader expects + identity_ops: (source_type.is_basedpython() + && matches!(cmp_op, CmpOp::Is | CmpOp::IsNot) + && !right.is_none_literal_expr()) + .then(|| { + Box::new(ast::IdentityOperators { + ops: Box::from([true]), + }) + }), range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) @@ -278,7 +294,12 @@ impl UnittestAssert { Ok(args_map) } - pub(crate) fn generate_assert(self, args: &[Expr], keywords: &[Keyword]) -> Result { + pub(crate) fn generate_assert( + self, + args: &[Expr], + keywords: &[Keyword], + source_type: PySourceType, + ) -> Result { let args = self.args_map(args, keywords)?; match self { UnittestAssert::True @@ -339,7 +360,7 @@ impl UnittestAssert { UnittestAssert::IsNot => CmpOp::IsNot, _ => unreachable!(), }; - let expr = compare(first, cmp_op, second); + let expr = compare(first, cmp_op, second, source_type); Ok(assert(&expr, msg)) } UnittestAssert::In | UnittestAssert::NotIn => { @@ -355,7 +376,7 @@ impl UnittestAssert { } else { CmpOp::NotIn }; - let expr = compare(member, cmp_op, container); + let expr = compare(member, cmp_op, container, source_type); Ok(assert(&expr, msg)) } UnittestAssert::IsNone | UnittestAssert::IsNotNone => { @@ -372,7 +393,7 @@ impl UnittestAssert { range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); - let expr = compare(expr, cmp_op, &node); + let expr = compare(expr, cmp_op, &node, source_type); Ok(assert(&expr, msg)) } UnittestAssert::IsInstance | UnittestAssert::NotIsInstance => { diff --git a/crates/ruff_linter/src/rules/flake8_simplify/mod.rs b/crates/ruff_linter/src/rules/flake8_simplify/mod.rs index 209479f8be..6f94b4434e 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/mod.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/mod.rs @@ -49,7 +49,10 @@ mod tests { #[test_case(Rule::ExprOrNotExpr, Path::new("SIM221.py"))] #[test_case(Rule::ExprOrTrue, Path::new("SIM222.py"))] #[test_case(Rule::ExprAndFalse, Path::new("SIM223.py"))] + #[test_case(Rule::ExprOrTrue, Path::new("SIM222_basedpython.by"))] + #[test_case(Rule::ExprAndFalse, Path::new("SIM222_basedpython.by"))] #[test_case(Rule::YodaConditions, Path::new("SIM300.py"))] + #[test_case(Rule::YodaConditions, Path::new("SIM300_basedpython.by"))] #[test_case(Rule::IfElseBlockInsteadOfDictGet, Path::new("SIM401.py"))] #[test_case(Rule::SplitStaticString, Path::new("SIM905.py"))] #[test_case(Rule::DictGetWithNoneDefault, Path::new("SIM910.py"))] diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs index 9e5259bbb0..f0987d1871 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs @@ -466,6 +466,7 @@ fn match_eq_target(expr: &Expr) -> Option<(&Name, &Expr)> { comparators, range: _, node_index: _, + identity_ops: _, }) = expr else { return None; @@ -554,6 +555,7 @@ pub(crate) fn compare_with_tuple(checker: &Checker, expr: &Expr) { left: Box::new(node1.into()), ops: Box::from([CmpOp::In]), comparators: Box::from([node.into()]), + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs index 2bb0aa9dee..4c932cd679 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_unary_op.rs @@ -158,6 +158,7 @@ pub(crate) fn negation_with_equal_op(checker: &Checker, expr: &Expr, op: UnaryOp comparators, range: _, node_index: _, + identity_ops: _, }) = operand else { return; @@ -189,6 +190,7 @@ pub(crate) fn negation_with_equal_op(checker: &Checker, expr: &Expr, op: UnaryOp left: left.clone(), ops: Box::from([CmpOp::NotEq]), comparators: comparators.clone(), + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; @@ -214,6 +216,7 @@ pub(crate) fn negation_with_not_equal_op( comparators, range: _, node_index: _, + identity_ops: _, }) = operand else { return; @@ -245,6 +248,7 @@ pub(crate) fn negation_with_not_equal_op( left: left.clone(), ops: Box::from([CmpOp::Eq]), comparators: comparators.clone(), + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs index 41f3703a57..a484d18333 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_get.rs @@ -136,6 +136,7 @@ pub(crate) fn if_else_block_instead_of_dict_get(checker: &Checker, stmt_if: &ast comparators: test_dict, range: _, node_index: _, + identity_ops: _, }) = &**test else { return; @@ -268,6 +269,7 @@ pub(crate) fn if_exp_instead_of_dict_get( comparators: test_dict, range: _, node_index: _, + identity_ops: _, }) = test else { return; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs index 6d5d387002..bdfe4d7626 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_else_block_instead_of_dict_lookup.rs @@ -65,6 +65,7 @@ pub(crate) fn if_else_block_instead_of_dict_lookup(checker: &Checker, stmt_if: & comparators, range: _, node_index: _, + identity_ops: _, }) = test.as_ref() else { return; @@ -152,6 +153,7 @@ pub(crate) fn if_else_block_instead_of_dict_lookup(checker: &Checker, stmt_if: & comparators, range: _, node_index: _, + identity_ops: _, })) => { let Expr::Name(ast::ExprName { id, .. }) = left.as_ref() else { return; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs index 08db5e741a..1cdc999113 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/needless_bool.rs @@ -250,6 +250,7 @@ pub(crate) fn needless_bool(checker: &Checker, stmt: &Stmt) { ops, left, comparators, + identity_ops, .. }) if matches!( ops.as_ref(), @@ -269,6 +270,10 @@ pub(crate) fn needless_bool(checker: &Checker, stmt: &Stmt) { ops: Box::new([op.negate()]), left: left.clone(), comparators: Box::new([right.clone()]), + // negating `is` gives `is not`, which is still the + // basedpython type test the source spelled; dropping the + // flag would rewrite it to the `!==` identity operator + identity_ops: identity_ops.clone(), range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, })) diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs index 20bc7e42f8..d33cb98720 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/reimplemented_builtin.rs @@ -155,6 +155,7 @@ pub(crate) fn convert_for_loop_to_any_all(checker: &Checker, stmt: &Stmt) { comparators, range: _, node_index: _, + identity_ops, }) = &loop_.test { if let ([op], [comparator]) = (&**ops, &**comparators) { @@ -174,6 +175,9 @@ pub(crate) fn convert_for_loop_to_any_all(checker: &Checker, stmt: &Stmt) { left: left.clone(), ops: Box::from([op]), comparators: Box::from([comparator.clone()]), + // the negation keeps the spelling: `is` becomes + // `is not`, never the `!==` identity operator + identity_ops: identity_ops.clone(), range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs index 2cf102c870..8cd4af2eec 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/yoda_conditions.rs @@ -6,6 +6,7 @@ use libcst_native::CompOp; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::{self as ast, CmpOp, Expr, UnaryOp}; use ruff_python_codegen::Stylist; +use ruff_python_semantic::SemanticModel; use ruff_python_stdlib::str::{self}; use ruff_text_size::Ranged; @@ -84,48 +85,58 @@ enum ConstantLikelihood { Definitely = 2, } -impl From<&Expr> for ConstantLikelihood { +impl ConstantLikelihood { /// Determine the [`ConstantLikelihood`] of an expression. - fn from(expr: &Expr) -> Self { + fn of(expr: &Expr, semantic: &SemanticModel) -> Self { + let of = |expr: &Expr| ConstantLikelihood::of(expr, semantic); match expr { _ if expr.is_literal_expr() => ConstantLikelihood::Definitely, Expr::Attribute(ast::ExprAttribute { attr, .. }) => { ConstantLikelihood::from_identifier(attr) } - Expr::Name(ast::ExprName { id, .. }) => ConstantLikelihood::from_identifier(id), + Expr::Name(name) => { + // A type parameter is not a constant, however it is spelled: `T` in + // `def f[T](...)` names a type the caller chooses. The one-letter + // convention makes almost every one of them read as `SCREAMING_CASE`, + // so without this a comparison against a type parameter is a Yoda + // condition whose "fix" reverses a perfectly ordinary comparison. + if semantic + .resolve_name(name) + .is_some_and(|id| semantic.binding(id).kind.is_type_param()) + { + ConstantLikelihood::Unlikely + } else { + ConstantLikelihood::from_identifier(&name.id) + } + } Expr::Tuple(tuple) => tuple .iter() - .map(ConstantLikelihood::from) + .map(of) .min() .unwrap_or(ConstantLikelihood::Definitely), Expr::List(list) => list .iter() - .map(ConstantLikelihood::from) + .map(of) .min() .unwrap_or(ConstantLikelihood::Definitely), Expr::Dict(dict) => dict .items .iter() .flat_map(|item| std::iter::once(&item.value).chain(item.key.as_ref())) - .map(ConstantLikelihood::from) + .map(of) .min() .unwrap_or(ConstantLikelihood::Definitely), - Expr::BinOp(ast::ExprBinOp { left, right, .. }) => cmp::min( - ConstantLikelihood::from(&**left), - ConstantLikelihood::from(&**right), - ), + Expr::BinOp(ast::ExprBinOp { left, right, .. }) => cmp::min(of(left), of(right)), Expr::UnaryOp(ast::ExprUnaryOp { op: UnaryOp::UAdd | UnaryOp::USub | UnaryOp::Invert, operand, range: _, node_index: _, - }) => ConstantLikelihood::from(&**operand), + }) => of(operand), _ => ConstantLikelihood::Unlikely, } } -} -impl ConstantLikelihood { /// Determine the [`ConstantLikelihood`] of an identifier. fn from_identifier(identifier: &str) -> Self { if str::is_cased_uppercase(identifier) { @@ -223,7 +234,8 @@ pub(crate) fn yoda_conditions( return; } - if ConstantLikelihood::from(left) <= ConstantLikelihood::from(right) { + let semantic = checker.semantic(); + if ConstantLikelihood::of(left, semantic) <= ConstantLikelihood::of(right, semantic) { return; } diff --git a/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap new file mode 100644 index 0000000000..1add375dbe --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-and-false_SIM222_basedpython.by.snap @@ -0,0 +1,20 @@ +--- +source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs +--- +SIM223 [*] Use `False` instead of `... and False` + --> SIM222_basedpython.by:19:8 + | +17 | if a or True: # SIM222 +18 | pass +19 | if b and False: # SIM223 + | ^^^^^^^^^^^ +20 | pass + | +help: Replace with `False` + | +18 | pass + - if b and False: # SIM223 +19 + if False: # SIM223 +20 | pass + | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap new file mode 100644 index 0000000000..a09f0910fe --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__expr-or-true_SIM222_basedpython.by.snap @@ -0,0 +1,21 @@ +--- +source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs +--- +SIM222 [*] Use `True` instead of `... or True` + --> SIM222_basedpython.by:17:8 + | +15 | # a boolean operator in a value expression is untouched +16 | def values(a: int, b: int) -> None: +17 | if a or True: # SIM222 + | ^^^^^^^^^ +18 | pass +19 | if b and False: # SIM223 + | +help: Replace with `True` + | +16 | def values(a: int, b: int) -> None: + - if a or True: # SIM222 +17 + if True: # SIM222 +18 | pass + | +note: This is an unsafe fix and may change runtime behavior diff --git a/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap new file mode 100644 index 0000000000..08d2702906 --- /dev/null +++ b/crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__yoda-conditions_SIM300_basedpython.by.snap @@ -0,0 +1,19 @@ +--- +source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs +--- +SIM300 [*] Yoda condition detected + --> SIM300_basedpython.by:15:8 + | +13 | # a genuine yoda condition still fires +14 | def yoda(age: int) -> None: +15 | if 42 == age: # SIM300 + | ^^^^^^^^^ +16 | pass + | +help: Rewrite as `age == 42` + | +14 | def yoda(age: int) -> None: + - if 42 == age: # SIM300 +15 + if age == 42: # SIM300 +16 | pass + | diff --git a/crates/ruff_linter/src/rules/pycodestyle/mod.rs b/crates/ruff_linter/src/rules/pycodestyle/mod.rs index 937a024266..89f5afe288 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/mod.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/mod.rs @@ -67,6 +67,7 @@ mod tests { #[test_case(Rule::TrailingWhitespace, Path::new("W291.py"))] #[test_case(Rule::TrueFalseComparison, Path::new("E712.py"))] #[test_case(Rule::TypeComparison, Path::new("E721.py"))] + #[test_case(Rule::TypeComparison, Path::new("E721_basedpython.by"))] #[test_case(Rule::UselessSemicolon, Path::new("E70.py"))] #[test_case(Rule::UselessSemicolon, Path::new("E703.ipynb"))] #[test_case(Rule::WhitespaceAfterDecorator, Path::new("E204.py"))] diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs index dc9968d024..6c39e4e4b4 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs @@ -130,6 +130,14 @@ pub(crate) fn compound_statements( // This is used to allow `class C: ...`-style definitions in stubs. let mut allow_ellipsis = false; + // basedpython: `type Swap[*Ts] = match *Ts:` is a type whose `case` arms are type + // expressions rather than statements, so the `:` in `case (A, B): (B, A)` separates + // a pattern from a type and opens no suite. `type_match` holds the indent of the + // header line for as long as the arms it opened last, and `type_keyword` records the + // `type` that tells such a header from `x = match ...`, which really is a statement + let mut type_match: Option = None; + let mut type_keyword = false; + // Track indentation. let mut indent = 0u32; @@ -147,6 +155,9 @@ pub(crate) fn compound_statements( } TokenKind::Dedent => { indent = indent.saturating_sub(1); + if type_match.is_some_and(|header| indent <= header) { + type_match = None; + } } _ => {} } @@ -194,6 +205,7 @@ pub(crate) fn compound_statements( try_ = None; while_ = None; with = None; + type_keyword = false; } TokenKind::Colon => { if case.is_some() @@ -273,7 +285,12 @@ pub(crate) fn compound_statements( with = None; } TokenKind::Case => { - case = Some(token.range()); + if type_match.is_none() { + case = Some(token.range()); + } + } + TokenKind::Type => { + type_keyword = true; } TokenKind::If => { if_ = Some(token.range()); @@ -313,6 +330,9 @@ pub(crate) fn compound_statements( } TokenKind::Match => { match_ = Some(token.range()); + if source_type.is_basedpython() && type_keyword { + type_match = Some(indent); + } } _ => {} } diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs index c7b0cde1a1..6ee27706ff 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs @@ -403,6 +403,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) &compare.left, &ops, &compare.comparators, + compare.identity_ops.as_deref(), compare.into(), tokens, source, @@ -413,6 +414,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) &compare.left, &ops, &compare.comparators, + compare.identity_ops.as_deref(), compare.into(), tokens, source, diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs index 2f1eb95385..2d37ba4038 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs @@ -92,6 +92,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { comparators, range: _, node_index: _, + identity_ops, }) = unary_op.operand.as_ref() else { return; @@ -106,6 +107,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { left, &[CmpOp::NotIn], comparators, + identity_ops.as_deref(), unary_op.into(), checker.tokens(), checker.source(), @@ -124,6 +126,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { left, &[CmpOp::IsNot], comparators, + identity_ops.as_deref(), unary_op.into(), checker.tokens(), checker.source(), diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs index 2835593119..e889c2dc7a 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/type_comparison.rs @@ -51,13 +51,21 @@ use crate::codes::Category; /// ``` #[derive(ViolationMetadata)] #[violation_metadata(stable_since = "v0.0.39", category = Category::Pedantic)] -pub(crate) struct TypeComparison; +pub(crate) struct TypeComparison { + basedpython: bool, +} impl Violation for TypeComparison { #[derive_message_formats] fn message(&self) -> String { - "Use `is` and `is not` for type comparisons, or `isinstance()` for isinstance checks" - .to_string() + // basedpython spells identity `===` and an isinstance check `x is C`, so the + // python wording names the wrong two operators there + if self.basedpython { + "Use `===` and `!==` for type comparisons, or `is` for isinstance checks".to_string() + } else { + "Use `is` and `is not` for type comparisons, or `isinstance()` for isinstance checks" + .to_string() + } } } @@ -78,7 +86,12 @@ pub(crate) fn type_comparison(checker: &Checker, compare: &ast::ExprCompare) { } // Disallow the comparison. - checker.report_diagnostic(TypeComparison, compare.range()); + checker.report_diagnostic( + TypeComparison { + basedpython: checker.source_type.is_basedpython(), + }, + compare.range(), + ); } } } diff --git a/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap index 11aae13b0f..4f8f16da18 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__multiple-statements-on-one-line-colon_E70_basedpython.by.snap @@ -7,3 +7,57 @@ E701 Multiple statements on one line (colon) 15 | # a class definition written on one line is still reported 16 | class OneLiner: x = 1 | ^ + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:39:19 + | +37 | # a `match` statement after the alias is a compound statement again +38 | match T: +39 | case int(): pass + | ^ +40 | case _: pass + | + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:40:15 + | +38 | match T: +39 | case int(): pass +40 | case _: pass + | ^ + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:46:19 + | +44 | def statement_expression(v: object) -> int: +45 | result = match v: +46 | case int(): 1 + | ^ +47 | case _: 0 +48 | return result + | + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:47:15 + | +45 | result = match v: +46 | case int(): 1 +47 | case _: 0 + | ^ +48 | return result + | + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:58:23 + | +56 | def method(self, v: object) -> None: +57 | match v: +58 | case int(): pass + | ^ + +E701 Multiple statements on one line (colon) + --> E70_basedpython.by:68:18 + | +67 | match flag: +68 | case True: pass + | ^ diff --git a/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap new file mode 100644 index 0000000000..51126e3190 --- /dev/null +++ b/crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__type-comparison_E721_basedpython.by.snap @@ -0,0 +1,30 @@ +--- +source: crates/ruff_linter/src/rules/pycodestyle/mod.rs +--- +E721 Use `===` and `!==` for type comparisons, or `is` for isinstance checks + --> E721_basedpython.by:4:8 + | +2 | # the two operators this rule names are not the ones python's message names +3 | def compare(a: object, b: object) -> None: +4 | if type(a) == type(b): + | ^^^^^^^^^^^^^^^^^^ +5 | pass +6 | if type(a) == int: + | + +E721 Use `===` and `!==` for type comparisons, or `is` for isinstance checks + --> E721_basedpython.by:6:8 + | +4 | if type(a) == type(b): +5 | pass +6 | if type(a) == int: + | ^^^^^^^^^^^^^^ +7 | pass + | + +E721 Use `===` and `!==` for type comparisons, or `is` for isinstance checks + --> E721_basedpython.by:11:12 + | +10 | def reified[reified T](x: T) -> None: +11 | assert T == int + | ^^^^^^^^ diff --git a/crates/ruff_linter/src/rules/pyflakes/mod.rs b/crates/ruff_linter/src/rules/pyflakes/mod.rs index 6cda6ca437..2ad8b998e6 100644 --- a/crates/ruff_linter/src/rules/pyflakes/mod.rs +++ b/crates/ruff_linter/src/rules/pyflakes/mod.rs @@ -91,6 +91,7 @@ mod tests { #[test_case(Rule::MultipleStarredExpressions, Path::new("F622.py"))] #[test_case(Rule::AssertTuple, Path::new("F631.py"))] #[test_case(Rule::IsLiteral, Path::new("F632.py"))] + #[test_case(Rule::IsLiteral, Path::new("F632_basedpython.by"))] #[test_case(Rule::InvalidPrintSyntax, Path::new("F633.py"))] #[test_case(Rule::IfTuple, Path::new("F634.py"))] // basedpython: a pattern clause's subject is a value, not a test diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs index 6a8b9b164e..cc33093673 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs @@ -3,7 +3,7 @@ use anyhow::{Error, bail}; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers; use ruff_python_ast::token::{TokenKind, Tokens}; -use ruff_python_ast::{CmpOp, Expr}; +use ruff_python_ast::{self as ast, CmpOp}; use ruff_text_size::{Ranged, TextRange}; use crate::checkers::ast::Checker; @@ -55,6 +55,8 @@ use crate::{AlwaysFixableViolation, Edit, Fix}; #[violation_metadata(stable_since = "v0.0.39", category = Category::Suspicious)] pub(crate) struct IsLiteral { cmp_op: IsCmpOp, + /// Whether the operator was written with basedpython's `===` / `!==` + spells_identity: bool, } impl AlwaysFixableViolation for IsLiteral { @@ -67,41 +69,57 @@ impl AlwaysFixableViolation for IsLiteral { } fn fix_title(&self) -> String { - let title = match self.cmp_op { - IsCmpOp::Is => "Replace `is` with `==`", - IsCmpOp::IsNot => "Replace `is not` with `!=`", + // basedpython's `===` / `!==` are the identity operators this rule replaces there + let title = match (self.cmp_op, self.spells_identity) { + (IsCmpOp::Is, false) => "Replace `is` with `==`", + (IsCmpOp::Is, true) => "Replace `===` with `==`", + (IsCmpOp::IsNot, false) => "Replace `is not` with `!=`", + (IsCmpOp::IsNot, true) => "Replace `!==` with `!=`", }; title.to_string() } } /// F632 -pub(crate) fn invalid_literal_comparison( - checker: &Checker, - left: &Expr, - ops: &[CmpOp], - comparators: &[Expr], - expr: &Expr, -) { +pub(crate) fn invalid_literal_comparison(checker: &Checker, compare: &ast::ExprCompare) { + // basedpython keeps `is` as python identity only where its right-hand side is a + // literal; anywhere else `is` is a type test, and `===` is the spelling that always + // compares identity. the parser records which of the two was written, so the tokens + // are needed only to place a fix let mut lazy_located = None; - let mut left = left; - for (index, (op, right)) in ops.iter().zip(comparators).enumerate() { + let mut left = &*compare.left; + for (index, (op, right)) in compare.ops.iter().zip(&compare.comparators).enumerate() { + let spells_identity = compare.is_identity_operator(index); + if matches!(op, CmpOp::Is | CmpOp::IsNot) && (helpers::is_constant_non_singleton(left) || helpers::is_constant_non_singleton(right) || helpers::is_mutable_iterable_initializer(left) || helpers::is_mutable_iterable_initializer(right)) + && (!checker.source_type.is_basedpython() || spells_identity || right.is_literal_expr()) { - let mut diagnostic = - checker.report_diagnostic(IsLiteral { cmp_op: op.into() }, expr.range()); + let mut diagnostic = checker.report_diagnostic( + IsLiteral { + cmp_op: op.into(), + spells_identity, + }, + compare.range(), + ); if lazy_located.is_none() { - lazy_located = Some(locate_cmp_ops(expr, checker.tokens())); + lazy_located = Some(locate_cmp_ops(compare.range(), checker.tokens())); } diagnostic.try_set_optional_fix(|| { - if let Some(located_op) = - lazy_located.as_ref().and_then(|located| located.get(index)) - { - assert_eq!(located_op.op, *op); + let located_op = lazy_located.as_ref().and_then(|located| located.get(index)); + // the tokens and the operators come from one parse, so they line up in + // either language: basedpython's `===` and `!==` scan to the same + // `CmpOp` the parser recorded for them. a dropped fix rather than a + // panic if that ever stops holding + debug_assert!( + located_op.is_none_or(|located_op| located_op.op == *op), + "located `{:?}` where the comparison has `{op:?}`", + located_op.map(|located_op| located_op.op) + ); + if let Some(located_op) = located_op.filter(|located_op| located_op.op == *op) { if let Ok(content) = match located_op.op { CmpOp::Is => Ok::("==".to_string()), CmpOp::IsNot => Ok("!=".to_string()), @@ -145,9 +163,9 @@ impl From<&CmpOp> for IsCmpOp { /// /// This method iterates over the token stream and re-identifies [`CmpOp`] nodes, annotating them /// with valid ranges. -fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { +fn locate_cmp_ops(range: TextRange, tokens: &Tokens) -> Vec { let mut tok_iter = tokens - .in_range(expr.range()) + .in_range(range) .iter() .filter(|token| !token.kind().is_trivia()) .peekable(); @@ -197,6 +215,15 @@ fn locate_cmp_ops(expr: &Expr, tokens: &Tokens) -> Vec { }; ops.push(op); } + // basedpython's identity operators, which parse to the same `CmpOp` as + // the `is` keyword — scanned so the operators keep lining up with the + // comparison's own, whichever spelling the source used + TokenKind::EqEqEqual => { + ops.push(LocatedCmpOp::new(token.range(), CmpOp::Is)); + } + TokenKind::BangEqEqual => { + ops.push(LocatedCmpOp::new(token.range(), CmpOp::IsNot)); + } TokenKind::NotEqual => { ops.push(LocatedCmpOp::new(token.range(), CmpOp::NotEq)); } @@ -242,13 +269,13 @@ mod tests { use ruff_python_ast::CmpOp; use ruff_python_parser::parse_expression; - use ruff_text_size::TextSize; + use ruff_text_size::{Ranged, TextSize}; use super::{LocatedCmpOp, locate_cmp_ops}; fn extract_cmp_op_locations(source: &str) -> Result> { let parsed = parse_expression(source)?; - Ok(locate_cmp_ops(parsed.expr(), parsed.tokens())) + Ok(locate_cmp_ops(parsed.expr().range(), parsed.tokens())) } #[test] diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap new file mode 100644 index 0000000000..b6e703c28f --- /dev/null +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__is-literal_F632_basedpython.by.snap @@ -0,0 +1,182 @@ +--- +source: crates/ruff_linter/src/rules/pyflakes/mod.rs +--- +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:5:11 + | +3 | # spellings, and an `is` whose right-hand side is a literal, compare identity +4 | def identity(x: object) -> None: +5 | print(1 === 1) # F632 + | ^^^^^^^ +6 | print(x !== "a") # F632 + | +help: Replace `===` with `==` + | +4 | def identity(x: object) -> None: + - print(1 === 1) # F632 +5 + print(1 == 1) # F632 +6 | print(x !== "a") # F632 + | + +F632 [*] Use `!=` to compare constant literals + --> F632_basedpython.by:6:11 + | +4 | def identity(x: object) -> None: +5 | print(1 === 1) # F632 +6 | print(x !== "a") # F632 + | ^^^^^^^^^ +help: Replace `!==` with `!=` + | +5 | print(1 === 1) # F632 + - print(x !== "a") # F632 +6 + print(x != "a") # F632 +7 | + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:10:11 + | + 9 | def literal_right_hand_side(x: object) -> None: +10 | print(1 is 1) # F632 + | ^^^^^^ +11 | print(x is not "a") # F632 + | +help: Replace `is` with `==` + | +9 | def literal_right_hand_side(x: object) -> None: + - print(1 is 1) # F632 +10 + print(1 == 1) # F632 +11 | print(x is not "a") # F632 + | + +F632 [*] Use `!=` to compare constant literals + --> F632_basedpython.by:11:11 + | + 9 | def literal_right_hand_side(x: object) -> None: +10 | print(1 is 1) # F632 +11 | print(x is not "a") # F632 + | ^^^^^^^^^^^^ +help: Replace `is not` with `!=` + | +10 | print(1 is 1) # F632 + - print(x is not "a") # F632 +11 + print(x != "a") # F632 +12 | + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:25:9 + | +23 | # source between the operands, so it survives everything that can sit there +24 | def spelling_survives_what_sits_between(x: object) -> None: +25 | a = (1) === (1) # F632 + | ^^^^^^^^^^^ +26 | b = 1 \ +27 | === 1 # F632 + | +help: Replace `===` with `==` + | +24 | def spelling_survives_what_sits_between(x: object) -> None: + - a = (1) === (1) # F632 +25 + a = (1) == (1) # F632 +26 | b = 1 \ + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:26:9 + | +24 | def spelling_survives_what_sits_between(x: object) -> None: +25 | a = (1) === (1) # F632 +26 | b = 1 \ + | _________^ +27 | | === 1 # F632 + | |_____________^ +28 | c = (1 === # a comment between the operands +29 | 1) # F632 + | +help: Replace `===` with `==` + | +26 | b = 1 \ + - === 1 # F632 +27 + == 1 # F632 +28 | c = (1 === # a comment between the operands + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:28:10 + | +26 | b = 1 \ +27 | === 1 # F632 +28 | c = (1 === # a comment between the operands + | __________^ +29 | | 1) # F632 + | |__________^ +30 | d = 1===1 # F632 + | +help: Replace `===` with `==` + | +27 | === 1 # F632 + - c = (1 === # a comment between the operands +28 + c = (1 == # a comment between the operands +29 | 1) # F632 + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:30:9 + | +28 | c = (1 === # a comment between the operands +29 | 1) # F632 +30 | d = 1===1 # F632 + | ^^^^^ +help: Replace `===` with `==` + | +29 | 1) # F632 + - d = 1===1 # F632 +30 + d = 1==1 # F632 +31 | + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:35:9 + | +33 | # each operator of a chain answers for its own spelling +34 | def chained(x: object) -> None: +35 | e = x < 1 === 1 # F632 + | ^^^^^^^^^^^ +36 | f = 1 === 1 !== 2 # F632 + | +help: Replace `===` with `==` + | +34 | def chained(x: object) -> None: + - e = x < 1 === 1 # F632 +35 + e = x < 1 == 1 # F632 +36 | f = 1 === 1 !== 2 # F632 + | + +F632 [*] Use `==` to compare constant literals + --> F632_basedpython.by:36:9 + | +34 | def chained(x: object) -> None: +35 | e = x < 1 === 1 # F632 +36 | f = 1 === 1 !== 2 # F632 + | ^^^^^^^^^^^^^ +help: Replace `===` with `==` + | +35 | e = x < 1 === 1 # F632 + - f = 1 === 1 !== 2 # F632 +36 + f = 1 == 1 !== 2 # F632 + | + +F632 [*] Use `!=` to compare constant literals + --> F632_basedpython.by:36:9 + | +34 | def chained(x: object) -> None: +35 | e = x < 1 === 1 # F632 +36 | f = 1 === 1 !== 2 # F632 + | ^^^^^^^^^^^^^ +help: Replace `!==` with `!=` + | +35 | e = x < 1 === 1 # F632 + - f = 1 === 1 !== 2 # F632 +36 + f = 1 === 1 != 2 # F632 + | diff --git a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap index da1004479d..7365f7df87 100644 --- a/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap +++ b/crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__unused-variable_F841_basedpython.by.snap @@ -29,3 +29,27 @@ F841 Local variable `Unmatched` is assigned to but never used 46 | case _: | help: Remove assignment to unused variable `Unmatched` + +F841 Local variable `k` is assigned to but never used + --> F841_basedpython.by:78:21 + | +76 | # what python's own capture pattern does, so an unused capture still fires +77 | def matching_binders(p: Point) -> int: +78 | if let Point(j, k) := p: # F841 on `k` + | ^ +79 | return j +80 | match p: + | +help: Remove assignment to unused variable `k` + +F841 Local variable `m` is assigned to but never used + --> F841_basedpython.by:81:23 + | +79 | return j +80 | match p: +81 | case Point(l, m): # F841 on `m` + | ^ +82 | return l +83 | return 0 + | +help: Remove assignment to unused variable `m` diff --git a/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs b/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs index 3dc4a2a1a5..a63cdd0d73 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs @@ -212,6 +212,7 @@ pub(crate) fn repeated_equality_comparison(checker: &Checker, bool_op: &ast::Exp BoolOp::And => Box::from([CmpOp::NotIn]), }, comparators: Box::from([comparator]), + identity_ops: None, range: bool_op.range(), node_index: AtomicNodeIndex::NONE, }))) diff --git a/crates/ruff_linter/src/rules/pyupgrade/mod.rs b/crates/ruff_linter/src/rules/pyupgrade/mod.rs index dc0abeeb9c..6f952c11e2 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/mod.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/mod.rs @@ -65,6 +65,7 @@ mod tests { #[test_case(Rule::QuotedAnnotation, Path::new("UP037_0.py"))] #[test_case(Rule::QuotedAnnotation, Path::new("UP037_1.py"))] #[test_case(Rule::QuotedAnnotation, Path::new("UP037_2.pyi"))] + #[test_case(Rule::QuotedAnnotation, Path::new("UP037_basedpython.by"))] #[test_case(Rule::QuotedAnnotation, Path::new("UP037_3.py"))] #[test_case(Rule::RedundantOpenModes, Path::new("UP015.py"))] #[test_case(Rule::RedundantOpenModes, Path::new("UP015_1.py"))] diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs index 10e9cd34f2..acbb3ca041 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/outdated_version_block.rs @@ -97,6 +97,7 @@ pub(crate) fn outdated_version_block(checker: &Checker, stmt_if: &StmtIf) { comparators, range: _, node_index: _, + identity_ops: _, }) = &branch.test else { continue; diff --git a/crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap b/crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap new file mode 100644 index 0000000000..2bacb5d540 --- /dev/null +++ b/crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_basedpython.by.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff_linter/src/rules/pyupgrade/mod.rs +--- + diff --git a/crates/ruff_linter/src/rules/refurb/helpers.rs b/crates/ruff_linter/src/rules/refurb/helpers.rs index f6375410a3..a48766f3a1 100644 --- a/crates/ruff_linter/src/rules/refurb/helpers.rs +++ b/crates/ruff_linter/src/rules/refurb/helpers.rs @@ -72,6 +72,10 @@ pub(super) fn replace_with_identity_check( left: left.clone().into(), ops: [op].into(), comparators: [ast::ExprNoneLiteral::default().into()].into(), + // basedpython writes this test as `is None`, whose target is the type + // `None`: the same runtime check, and the spelling a reader expects. + // recording no `===` is what prints it that way in both languages + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); diff --git a/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs b/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs index feceeb5c89..1b58817e2e 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs @@ -95,6 +95,8 @@ pub(crate) fn single_item_membership_test( left, &[membership_test.replacement_op()], std::slice::from_ref(item), + // only `in` / `not in`, which have one spelling + None, expr.into(), checker.tokens(), checker.source(), diff --git a/crates/ruff_linter/src/rules/ruff/mod.rs b/crates/ruff_linter/src/rules/ruff/mod.rs index 6635c112dd..5770e3ddd9 100644 --- a/crates/ruff_linter/src/rules/ruff/mod.rs +++ b/crates/ruff_linter/src/rules/ruff/mod.rs @@ -62,6 +62,7 @@ mod tests { #[test_case(Rule::UnnecessaryKeyCheck, Path::new("RUF019.py"))] #[test_case(Rule::NeverUnion, Path::new("RUF020.py"))] #[test_case(Rule::ParenthesizeChainedOperators, Path::new("RUF021.py"))] + #[test_case(Rule::ParenthesizeChainedOperators, Path::new("RUF021_basedpython.by"))] #[test_case(Rule::UnsortedDunderAll, Path::new("RUF022.py"))] #[test_case(Rule::UnsortedDunderSlots, Path::new("RUF023.py"))] #[test_case(Rule::MutableFromkeysValue, Path::new("RUF024.py"))] diff --git a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs index 3342a3bb37..41ee0cd368 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_regular_expression.rs @@ -334,6 +334,8 @@ impl<'a> ReFunc<'a> { left: Box::new(left.clone()), ops: Box::new([op]), comparators: Box::new([right.clone()]), + // only ever `==` / `!=` here, which have one spelling + identity_ops: None, range: TextRange::default(), node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) diff --git a/crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap b/crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap new file mode 100644 index 0000000000..5c673b2ec4 --- /dev/null +++ b/crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__parenthesize-chained-operators_RUF021_basedpython.by.snap @@ -0,0 +1,16 @@ +--- +source: crates/ruff_linter/src/rules/ruff/mod.rs +--- +RUF021 [*] Parenthesize `a and b` expressions when chaining `and` and `or` together, to make the precedence clear + --> RUF021_basedpython.by:21:12 + | +19 | # a boolean expression in a value position still wants the parentheses +20 | def values(a: bool, b: bool, c: bool) -> bool: +21 | return a and b or c + | ^^^^^^^ +help: Parenthesize the `and` subexpression + | +20 | def values(a: bool, b: bool, c: bool) -> bool: + - return a and b or c +21 + return (a and b) or c + | diff --git a/crates/ruff_python_ast/ast.toml b/crates/ruff_python_ast/ast.toml index 0c6859a902..2f385f33f1 100644 --- a/crates/ruff_python_ast/ast.toml +++ b/crates/ruff_python_ast/ast.toml @@ -485,6 +485,20 @@ fields = [ { name = "left", type = "Expr" }, { name = "ops", type = "Box<[CmpOp]>" }, { name = "comparators", type = "Box<[Expr]>" }, + { name = "identity_ops", type = "Box?", skip_visit = true, doc = """basedpython: which operators were written `===` / `!==`. + +basedpython gives the `is` keyword to the type test and spells Python's identity +comparison `===` / `!==`. Both parse to [`CmpOp::Is`](crate::CmpOp::Is) / [`CmpOp::IsNot`](crate::CmpOp::IsNot) so the +AST keeps Python's shape, and the spelling is recorded here instead. `None` when +nothing was written `===` / `!==`, which is every `.py` file — and boxed, because +paying two more words on every comparison would widen `Expr` itself. + +The lexer emits `===` / `!==` for any source that spells them, so a renderer can +print them back from this field alone without knowing which language it is +emitting — a `.py` file that contains them was never valid Python anyway. Read it +through [`ExprCompare::is_identity_operator`], and ask +[`ExprCompare::is_type_test`] for the other half of the question, which is the +one that needs the source type.""" }, ] # The fields must be visited simultaneously custom_source_order = true diff --git a/crates/ruff_python_ast/src/comparable.rs b/crates/ruff_python_ast/src/comparable.rs index dca4ec069b..765c7cb2aa 100644 --- a/crates/ruff_python_ast/src/comparable.rs +++ b/crates/ruff_python_ast/src/comparable.rs @@ -971,6 +971,10 @@ pub struct ExprCompare<'a> { left: Box>, ops: Vec, comparators: Vec>, + /// basedpython: `a is int` and `a === int` share an operator but not a + /// meaning, so two comparisons are only equal when they were spelled the + /// same way + identity_ops: Vec, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -1282,12 +1286,17 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> { left, ops, comparators, + identity_ops, range: _, node_index: _, }) => Self::Compare(ExprCompare { left: left.into(), ops: ops.iter().copied().map(Into::into).collect(), comparators: comparators.iter().map(Into::into).collect(), + identity_ops: identity_ops + .as_ref() + .map(|identity| identity.ops.to_vec()) + .unwrap_or_default(), }), ast::Expr::Call(ast::ExprCall { func, diff --git a/crates/ruff_python_ast/src/generated.rs b/crates/ruff_python_ast/src/generated.rs index 65886f1a10..086f1078ba 100644 --- a/crates/ruff_python_ast/src/generated.rs +++ b/crates/ruff_python_ast/src/generated.rs @@ -10440,6 +10440,21 @@ pub struct ExprCompare { pub left: Box, pub ops: Box<[crate::CmpOp]>, pub comparators: Box<[Expr]>, + /// basedpython: which operators were written `===` / `!==`. + /// + /// basedpython gives the `is` keyword to the type test and spells Python's identity + /// comparison `===` / `!==`. Both parse to [`CmpOp::Is`](crate::CmpOp::Is) / [`CmpOp::IsNot`](crate::CmpOp::IsNot) so the + /// AST keeps Python's shape, and the spelling is recorded here instead. `None` when + /// nothing was written `===` / `!==`, which is every `.py` file — and boxed, because + /// paying two more words on every comparison would widen `Expr` itself. + /// + /// The lexer emits `===` / `!==` for any source that spells them, so a renderer can + /// print them back from this field alone without knowing which language it is + /// emitting — a `.py` file that contains them was never valid Python anyway. Read it + /// through [`ExprCompare::is_identity_operator`], and ask + /// [`ExprCompare::is_type_test`] for the other half of the question, which is the + /// one that needs the source type. + pub identity_ops: Option>, } /// A call expression whose end offset is derived from its arguments. diff --git a/crates/ruff_python_ast/src/helpers.rs b/crates/ruff_python_ast/src/helpers.rs index 9733d556bc..165e1f08ad 100644 --- a/crates/ruff_python_ast/src/helpers.rs +++ b/crates/ruff_python_ast/src/helpers.rs @@ -2766,6 +2766,7 @@ pub fn generate_comparison( left: &Expr, ops: &[CmpOp], comparators: &[Expr], + identity_ops: Option<&crate::IdentityOperators>, parent: AnyNodeRef, tokens: &Tokens, source: &str, @@ -2779,7 +2780,14 @@ pub fn generate_comparison( &source[parenthesized_range(left.into(), parent, tokens).unwrap_or(left.range())], ); - for (op, comparator) in ops.iter().zip(comparators) { + for (index, (op, comparator)) in ops.iter().zip(comparators).enumerate() { + // basedpython writes python's identity comparison `===` / `!==` and + // gives the `is` keyword to a type test. printing `is` for an operator + // the source wrote `===` would rewrite one into the other + let identity = identity_ops + .and_then(|identity| identity.ops.get(index)) + .copied() + .unwrap_or(false); // Add the operator. contents.push_str(match op { CmpOp::Eq => " == ", @@ -2790,6 +2798,8 @@ pub fn generate_comparison( CmpOp::GtE => " >= ", CmpOp::In => " in ", CmpOp::NotIn => " not in ", + CmpOp::Is if identity => " === ", + CmpOp::IsNot if identity => " !== ", CmpOp::Is => " is ", CmpOp::IsNot => " is not ", }); diff --git a/crates/ruff_python_ast/src/node.rs b/crates/ruff_python_ast/src/node.rs index 929e51c464..34716b388f 100644 --- a/crates/ruff_python_ast/src/node.rs +++ b/crates/ruff_python_ast/src/node.rs @@ -96,6 +96,7 @@ impl ast::ExprCompare { left, ops, comparators, + identity_ops: _, range: _, node_index: _, } = self; diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs index a6adbd4aad..9c1e506098 100644 --- a/crates/ruff_python_ast/src/nodes.rs +++ b/crates/ruff_python_ast/src/nodes.rs @@ -2799,6 +2799,83 @@ impl ExprNamed { } } +/// basedpython: which operators of an [`ExprCompare`](crate::ExprCompare) were +/// written `===` / `!==`, python's identity comparison. +/// +/// basedpython gives the `is` keyword to the type test and spells identity +/// `===` / `!==`. Both parse to the same [`CmpOp`](crate::CmpOp), so the AST +/// keeps this alongside to say which was written. +/// +/// Boxed behind an `Option` on the node: a comparison written any other way +/// carries none of this, and paying two words for it on every comparison would +/// widen `Expr` itself. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))] +pub struct IdentityOperators { + /// one entry per operator, in the same order as + /// [`ExprCompare::ops`](crate::ExprCompare::ops) + pub ops: Box<[bool]>, +} + +impl IdentityOperators { + /// the side table to store on a node, or `None` when no operator was + /// written `===` / `!==`. + /// + /// `operators` is the comparison's own operator count: an entry past it + /// would never be read, and one short of it would answer `false` for an + /// operator that was written `===`. + pub fn into_stored(ops: Vec, operators: usize) -> Option> { + debug_assert!( + ops.len() <= operators, + "identity spellings outnumber the operators they belong to" + ); + ops.iter().any(|identity| *identity).then(|| { + Box::new(Self { + ops: ops.into_boxed_slice(), + }) + }) + } +} + +impl crate::ExprCompare { + /// basedpython: whether the operator at `index` was written `===` / `!==`, + /// which is Python's identity comparison. + /// + /// `false` for every operator that is not an identity comparison and for an + /// index past the end, so a caller never has to check the encoding or the + /// length first. A renderer can print `===` from this answer alone, without + /// knowing which language it is emitting: the lexer produces the token for + /// any source that spells it, and a `.py` file that does was never valid + /// Python. + pub fn is_identity_operator(&self, index: usize) -> bool { + self.identity_ops + .as_ref() + .and_then(|identity| identity.ops.get(index)) + .copied() + .unwrap_or(false) + } + + /// basedpython: whether the operator at `index` is a *type test* — `is` or + /// `is not` spelled with the keyword, which asks whether the left operand + /// has the type its right-hand side names. + /// + /// The source type is required because the same operator means Python's + /// identity comparison in a `.py` file, where there is no type test at all. + pub fn is_type_test(&self, index: usize, source_type: crate::PySourceType) -> bool { + source_type.is_basedpython() + && matches!( + self.ops.get(index), + Some(crate::CmpOp::Is | crate::CmpOp::IsNot) + ) + && !self.is_identity_operator(index) + } + + /// basedpython: whether any operator of this comparison is a type test. + pub fn has_type_test(&self, source_type: crate::PySourceType) -> bool { + (0..self.ops.len()).any(|index| self.is_type_test(index, source_type)) + } +} + impl ExprList { pub fn iter(&self) -> std::slice::Iter<'_, Expr> { self.elts.iter() @@ -4552,7 +4629,10 @@ mod tests { assert_eq!(std::mem::size_of::(), 16); assert_eq!(std::mem::size_of::(), 48); assert_eq!(std::mem::size_of::(), 64); - assert_eq!(std::mem::size_of::(), 56); + // basedpython: a comparison carries which of its operators were written + // `===` / `!==`, boxed so this stays inside `ExprCall`'s width and + // `Expr` does not grow + assert_eq!(std::mem::size_of::(), 64); assert_eq!(std::mem::size_of::(), 40); assert_eq!(std::mem::size_of::(), 56); assert_eq!(std::mem::size_of::(), 12); diff --git a/crates/ruff_python_ast/src/visitor.rs b/crates/ruff_python_ast/src/visitor.rs index caac8b36ee..b22b57ca74 100644 --- a/crates/ruff_python_ast/src/visitor.rs +++ b/crates/ruff_python_ast/src/visitor.rs @@ -569,6 +569,7 @@ pub fn walk_expr<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, expr: &'a Expr) { left, ops, comparators, + identity_ops: _, range: _, node_index: _, }) => { diff --git a/crates/ruff_python_ast/src/visitor/transformer.rs b/crates/ruff_python_ast/src/visitor/transformer.rs index 9b3221a7af..f189d53375 100644 --- a/crates/ruff_python_ast/src/visitor/transformer.rs +++ b/crates/ruff_python_ast/src/visitor/transformer.rs @@ -556,6 +556,7 @@ pub fn walk_expr(visitor: &V, expr: &mut Expr) { left, ops, comparators, + identity_ops: _, range: _, node_index: _, }) => { diff --git a/crates/ruff_python_codegen/src/generator.rs b/crates/ruff_python_codegen/src/generator.rs index f56a6f497e..887832b328 100644 --- a/crates/ruff_python_codegen/src/generator.rs +++ b/crates/ruff_python_codegen/src/generator.rs @@ -1478,17 +1478,26 @@ impl<'a> Generator<'a> { self.unparse_expr(value, precedence::MAX); }); } - Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - range: _, - node_index: _, - }) => { + Expr::Compare(compare) => { + let ast::ExprCompare { + left, + ops, + comparators, + identity_ops: _, + range: _, + node_index: _, + } = compare; group_if!(precedence::CMP, { let new_lvl = precedence::CMP + 1; self.unparse_expr(left, new_lvl); - for (op, cmp) in ops.iter().zip(comparators) { + for (index, (op, cmp)) in ops.iter().zip(comparators).enumerate() { + // basedpython gives the `is` keyword to the type test and + // spells python's identity comparison `===` / `!==`. both + // parse to the same operator, so printing `is` for one + // the source wrote `===` would quietly turn an identity + // check into a type test. only basedpython records the + // spelling, so this needs no mode of its own + let identity = compare.is_identity_operator(index); let op = match op { CmpOp::Eq => " == ", CmpOp::NotEq => " != ", @@ -1496,6 +1505,8 @@ impl<'a> Generator<'a> { CmpOp::LtE => " <= ", CmpOp::Gt => " > ", CmpOp::GtE => " >= ", + CmpOp::Is if identity => " === ", + CmpOp::IsNot if identity => " !== ", CmpOp::Is => " is ", CmpOp::IsNot => " is not ", CmpOp::In => " in ", diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by index c10a156483..566905d4c4 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/identity_operators.by @@ -11,8 +11,10 @@ b = xs !== xs c = xs is list[int] d = xs is not list[int] -# chained comparisons mix the spellings freely -e = xs === xs !== None is list[int] is not list[str] +# a chain of identity comparisons is ordinary python, and each operator keeps +# the spelling it was written with. a type test may not join a chain at all — +# python's chaining rule would ask whether the *class* has the next type +e = xs === xs !== None # mixed with value comparisons f = 1 === 1 == 1 @@ -21,6 +23,16 @@ f = 1 === 1 == 1 g = (xs) === (xs) h = (xs) !== (xs) +# a comment between the operands, and a line continuation — the two the old +# source-gap scan read wrong +i = ( + xs + # which one is this + === xs +) +j = xs \ + === xs + # inside a condition if xs === xs: pass diff --git a/crates/ruff_python_formatter/src/expression/binary_like.rs b/crates/ruff_python_formatter/src/expression/binary_like.rs index ec658172be..77738c02aa 100644 --- a/crates/ruff_python_formatter/src/expression/binary_like.rs +++ b/crates/ruff_python_formatter/src/expression/binary_like.rs @@ -8,7 +8,7 @@ use ruff_python_ast::{ Expr, ExprAttribute, ExprBinOp, ExprBoolOp, ExprCompare, ExprUnaryOp, StringLike, UnaryOp, }; use ruff_python_trivia::{SimpleToken, SimpleTokenKind, SimpleTokenizer, TriviaRanges}; -use ruff_text_size::{Ranged, TextLen, TextRange}; +use ruff_text_size::{Ranged, TextRange}; use crate::comments::{Comments, SourceComment, leading_comments, trailing_comments}; use crate::expression::OperatorPrecedence; @@ -65,20 +65,16 @@ impl<'a> BinaryLike<'a> { if let Some((last_expression, middle_expressions)) = compare.comparators.split_last() { let (last_operator, middle_operators) = compare.ops.split_last().unwrap(); - // the source gap preceding each operator, so a basedpython - // `===` / `!==` can be told apart from the `is` / `is not` it - // shares a `CmpOp` with - let mut gap_start = compare.left.end(); - - for (operator, expression) in middle_operators.iter().zip(middle_expressions) { + for (index, (operator, expression)) in + middle_operators.iter().zip(middle_expressions).enumerate() + { parts.push(OperandOrOperator::Operator(Operator { symbol: OperatorSymbol::Comparator( *operator, - TextRange::new(gap_start, expression.start()), + compare.is_identity_operator(index), ), trailing_comments: &[], })); - gap_start = expression.end(); rec(Operand::Middle { expression }, comments, trivia, parts); } @@ -86,7 +82,7 @@ impl<'a> BinaryLike<'a> { parts.push(OperandOrOperator::Operator(Operator { symbol: OperatorSymbol::Comparator( *last_operator, - TextRange::new(gap_start, last_expression.start()), + compare.is_identity_operator(middle_operators.len()), ), trailing_comments: &[], })); @@ -1022,12 +1018,11 @@ impl Format> for Operator<'_> { #[derive(Copy, Clone, Debug)] enum OperatorSymbol { Binary(ruff_python_ast::Operator), - /// The comparison operator together with the source gap that precedes it — - /// the span between the end of the left operand and the start of the right - /// one. basedpython spells identity as `===` / `!==` but parses both to the - /// `CmpOp` of `is` / `is not` (whose surface form is instead a parametric - /// type test), so the spelling can only be recovered from that source. - Comparator(ruff_python_ast::CmpOp, TextRange), + /// The comparison operator, and whether it was written `===` / `!==`. + /// basedpython spells python's identity comparison that way and parses it + /// to the same `CmpOp` as the `is` keyword, so the operator alone does not + /// say which one to print back. + Comparator(ruff_python_ast::CmpOp, bool), Bool(ruff_python_ast::BoolOp), } @@ -1045,40 +1040,33 @@ impl OperatorSymbol { } } -/// The source range of a basedpython `===` / `!==` identity operator written in -/// `gap` — the span between the two operands it joins — or `None` when the -/// operator is spelled `is` / `is not`, or is not an identity comparison at all. -fn identity_operator_range( - context: &PyFormatContext, +/// The basedpython spelling of an identity comparison — `===` / `!==` — or +/// `None` when the operator is written `is` / `is not`, or is not an identity +/// comparison at all. +/// +/// Only basedpython records the spelling, so a `.py` file always takes the +/// `None` arm and prints python's `is`. +fn identity_operator_symbol( operator: ruff_python_ast::CmpOp, - gap: TextRange, -) -> Option { - let symbol = match operator { - ruff_python_ast::CmpOp::Is => "===", - ruff_python_ast::CmpOp::IsNot => "!==", - _ => return None, - }; - - let source = context.source(); - // the operator is the first token in the gap that is neither trivia nor a - // closing parenthesis of the left operand (`(a) === b`) - let start = SimpleTokenizer::new(source, gap) - .skip_trivia() - .find(|token| token.kind() != SimpleTokenKind::RParen)? - .start(); - - source[usize::from(start)..] - .starts_with(symbol) - .then(|| TextRange::at(start, symbol.text_len())) + identity: bool, +) -> Option<&'static str> { + if !identity { + return None; + } + match operator { + ruff_python_ast::CmpOp::Is => Some("==="), + ruff_python_ast::CmpOp::IsNot => Some("!=="), + _ => None, + } } impl Format> for OperatorSymbol { fn fmt(&self, f: &mut Formatter>) -> FormatResult<()> { match self { OperatorSymbol::Binary(operator) => operator.format().fmt(f), - OperatorSymbol::Comparator(operator, gap) => { - match identity_operator_range(f.context(), *operator, *gap) { - Some(range) => source_text_slice(range).fmt(f), + OperatorSymbol::Comparator(operator, identity) => { + match identity_operator_symbol(*operator, *identity) { + Some(symbol) => token(symbol).fmt(f), None => operator.format().fmt(f), } } diff --git a/crates/ruff_python_formatter/src/expression/mod.rs b/crates/ruff_python_formatter/src/expression/mod.rs index a88e740e26..39f21dfd33 100644 --- a/crates/ruff_python_formatter/src/expression/mod.rs +++ b/crates/ruff_python_formatter/src/expression/mod.rs @@ -743,6 +743,7 @@ impl<'input> CanOmitOptionalParenthesesVisitor<'input> { left: _, ops, comparators: _, + identity_ops: _, }) => { self.update_max_precedence_with_count( OperatorPrecedence::Comparator, diff --git a/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap index 098d728222..7605789ec4 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@identity_operators.by.snap @@ -17,8 +17,10 @@ b = xs !== xs c = xs is list[int] d = xs is not list[int] -# chained comparisons mix the spellings freely -e = xs === xs !== None is list[int] is not list[str] +# a chain of identity comparisons is ordinary python, and each operator keeps +# the spelling it was written with. a type test may not join a chain at all — +# python's chaining rule would ask whether the *class* has the next type +e = xs === xs !== None # mixed with value comparisons f = 1 === 1 == 1 @@ -27,6 +29,16 @@ f = 1 === 1 == 1 g = (xs) === (xs) h = (xs) !== (xs) +# a comment between the operands, and a line continuation — the two the old +# source-gap scan read wrong +i = ( + xs + # which one is this + === xs +) +j = xs \ + === xs + # inside a condition if xs === xs: pass @@ -53,8 +65,10 @@ b = xs !== xs c = xs is list[int] d = xs is not list[int] -# chained comparisons mix the spellings freely -e = xs === xs !== None is list[int] is not list[str] +# a chain of identity comparisons is ordinary python, and each operator keeps +# the spelling it was written with. a type test may not join a chain at all — +# python's chaining rule would ask whether the *class* has the next type +e = xs === xs !== None # mixed with value comparisons f = 1 === 1 == 1 @@ -63,6 +77,15 @@ f = 1 === 1 == 1 g = (xs) === (xs) h = (xs) !== (xs) +# a comment between the operands, and a line continuation — the two the old +# source-gap scan read wrong +i = ( + xs + # which one is this + === xs +) +j = xs === xs + # inside a condition if xs === xs: pass diff --git a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py b/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py index e456a03c87..e460bb77b6 100644 --- a/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py +++ b/crates/ruff_python_parser/resources/valid/expressions/identity_compare.py @@ -1,7 +1,10 @@ -# basedpython identity-comparison operators +# basedpython identity-comparison operators. this fixture parses in python +# mode, where the lexer still produces the tokens — a `.py` file spelling them +# was never valid python, and the parser's job here is only to keep the shape x === y x !== y -# chained with other comparisons -a === b is not c +# chained with each other. a chain mixing in the `is` keyword is rejected in +# basedpython mode, and has its own test in `parser::tests` +a === b !== c a !== b diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 30f86d7a28..91a33e91ff 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -139,6 +139,8 @@ pub enum ParseErrorType { DuplicateTypeParamSeparator(&'static str), /// basedpython: a bound range `T: Lower..Upper` was missing one of its ends. IncompleteTypeParamBoundRange, + /// basedpython: an `is` type test appeared in a chained comparison. + ChainedTypeTest, /// An unparenthesized named expression was found where it is not allowed. UnparenthesizedNamedExpression, @@ -308,6 +310,10 @@ impl std::fmt::Display for ParseErrorType { "Type parameter list cannot have two `{separator}` separators" ) } + ParseErrorType::ChainedTypeTest => f.write_str( + "`is` type test cannot be chained with another comparison; \ + split it into separate tests joined with `and`", + ), ParseErrorType::IncompleteTypeParamBoundRange => f.write_str( "Type parameter bound range requires both a lower and an upper bound, as in `T: int..object`", ), diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 9d111825ad..ca974a91d0 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2259,6 +2259,17 @@ impl<'src> Parser<'src> { op: CmpOp, context: ExpressionContext, ) -> ast::ExprCompare { + // built only once an operator is actually written `===` / `!==`, so an + // ordinary comparison — every one in a `.py` file — allocates nothing + let mut identity_ops: Vec = Vec::new(); + let record_identity = |identity_ops: &mut Vec, index: usize, identity: bool| { + if identity_ops.is_empty() && !identity { + return; + } + identity_ops.resize(index, false); + identity_ops.push(identity); + }; + record_identity(&mut identity_ops, 0, self.at_identity_operator(op)); self.bump_cmp_op(op); let comparators_snapshot = self.expr_scratch.snapshot(); @@ -2288,19 +2299,58 @@ impl<'src> Parser<'src> { break; }; + record_identity( + &mut identity_ops, + operators.len(), + self.at_identity_operator(next_op), + ); self.bump_cmp_op(next_op); operators.push(next_op); } + let range = self.node_range(start); + // a type test's right-hand side is a type expression, and python's + // chaining rule makes each operand the *next* comparison's left operand + // — so `a is int is str` becomes `a is int and int is str`, where the + // class `int` is asked whether it has the type `str`. only a type test + // that something follows creates that conflict; a trailing one + // (`a < b is None`) hands nothing on, and chains as it always did + if self.options.is_basedpython + && operators + .iter() + .enumerate() + .take(operators.len().saturating_sub(1)) + .any(|(index, op)| { + matches!(op, CmpOp::Is | CmpOp::IsNot) + && !identity_ops.get(index).copied().unwrap_or(false) + }) + { + self.add_error(ParseErrorType::ChainedTypeTest, range); + } + let operator_count = operators.len(); ast::ExprCompare { left: Box::new(lhs), ops: operators.into_boxed_slice(), comparators: self.expr_scratch.take(comparators_snapshot), - range: self.node_range(start), + // the common case — a `.py` file, or basedpython written with the + // `is` keyword — stores nothing at all + identity_ops: ast::IdentityOperators::into_stored(identity_ops, operator_count), + range, node_index: AtomicNodeIndex::NONE, } } + /// basedpython: whether the comparison operator the parser is positioned at + /// was written `===` / `!==` — python's identity comparison, which the lexer + /// emits as its own token and which shares a [`CmpOp`] with the `is` keyword. + fn at_identity_operator(&self, op: CmpOp) -> bool { + matches!(op, CmpOp::Is | CmpOp::IsNot) + && self.at_ts(TokenSet::new([ + TokenKind::EqEqEqual, + TokenKind::BangEqEqual, + ])) + } + /// Parses all kinds of strings and implicitly concatenated strings. /// /// # Panics diff --git a/crates/ruff_python_parser/src/parser/helpers.rs b/crates/ruff_python_parser/src/parser/helpers.rs index e48886b996..05074e6ac9 100644 --- a/crates/ruff_python_parser/src/parser/helpers.rs +++ b/crates/ruff_python_parser/src/parser/helpers.rs @@ -44,14 +44,13 @@ pub(super) const fn token_kind_to_cmp_op( (TokenKind::In, _) => CmpOp::In, (TokenKind::EqEqual, _) => CmpOp::Eq, // basedpython: `===` is the identity comparison (Python's `is`). - // we map it to CmpOp::Is so the AST matches Python semantics; the - // forward transpile distinguishes it from the basedpython `is` - // keyword by looking at the source text + // we map it to CmpOp::Is so the AST matches Python semantics, and the + // parser records the spelling in `ExprCompare::identity_ops` so nothing + // downstream has to read it back out of the source (TokenKind::EqEqEqual, _) => CmpOp::Is, // basedpython: `!==` is the negated identity comparison (Python's - // `is not`). mapped to CmpOp::IsNot; the forward transpile - // distinguishes it from the basedpython `is not` keyword pair by - // looking at the source text + // `is not`). mapped to CmpOp::IsNot, with the spelling recorded the + // same way (TokenKind::BangEqEqual, _) => CmpOp::IsNot, (TokenKind::NotEqual, _) => CmpOp::NotEq, (TokenKind::Less, _) => CmpOp::Lt, diff --git a/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap index b5b94a8af2..99c2c06bcf 100644 --- a/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap +++ b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap @@ -194,6 +194,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 82f693be85..c9e126d609 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -39,6 +39,73 @@ fn parse_basedpython_module_with_errors(source: &str) -> Parsed { .unwrap() } +/// the identity spelling recorded for each operator of the single comparison in +/// `source`, parsed as basedpython +fn identity_spellings(source: &str) -> Vec { + let parsed = parse_basedpython_module(source); + let [Stmt::Expr(statement)] = parsed.syntax().body.as_slice() else { + panic!("expected a single expression statement: {source}"); + }; + let Expr::Compare(compare) = statement.value.as_ref() else { + panic!("expected a comparison: {source}"); + }; + (0..compare.ops.len()) + .map(|index| compare.is_identity_operator(index)) + .collect() +} + +#[test] +fn identity_operators_are_recorded_per_operator() { + // the two spellings share a `CmpOp`, so the parser records which it saw — + // and records nothing at all when the answer is "none of them" + assert_eq!(identity_spellings("a === b"), vec![true]); + assert_eq!(identity_spellings("a !== b"), vec![true]); + assert_eq!(identity_spellings("a is b"), vec![false]); + assert_eq!(identity_spellings("a == b"), vec![false]); + // a spelling recorded for one operator does not shift the others + assert_eq!(identity_spellings("a == b === c"), vec![false, true]); + assert_eq!(identity_spellings("a === b == c"), vec![true, false]); + assert_eq!( + identity_spellings("a === b !== c === d"), + vec![true, true, true] + ); +} + +#[test] +fn a_type_test_may_not_be_chained() { + // python chains `a is int is str` into `a is int and int is str`, whose + // second half asks whether the class `int` has the type `str` + let parsed = parse_basedpython_module_with_errors("a is int is str\n"); + let errors = parsed.errors(); + assert!( + errors + .iter() + .any(|error| matches!(error.error, ParseErrorType::ChainedTypeTest)), + "expected a chained-type-test error, got {errors:?}" + ); + // the reported range covers the whole comparison, which is what has to change + let error = errors + .iter() + .find(|error| matches!(error.error, ParseErrorType::ChainedTypeTest)) + .unwrap(); + assert_eq!(&"a is int is str\n"[error.location], "a is int is str"); +} + +#[test] +fn a_trailing_type_test_still_chains() { + // nothing follows it, so no type expression becomes the next comparison's + // left operand — the conflict the rejection exists for + let parsed = parse_basedpython_module_with_errors("a < b is None\n"); + assert!( + !parsed + .errors() + .iter() + .any(|error| matches!(error.error, ParseErrorType::ChainedTypeTest)), + "a trailing type test is not a chained one: {:?}", + parsed.errors() + ); +} + /// The keyword-argument names of the single call statement in `source`, each /// with the way it was spelled. fn keyword_names(source: &str) -> Vec<(String, ruff_python_ast::KeywordKey)> { diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap index 7a9edf207f..9b727a1956 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap @@ -49,6 +49,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap index 3ab964282e..499e7dd3d5 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap @@ -49,6 +49,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap index c7baa8a513..70f5bae3ff 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap @@ -48,6 +48,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap index bf1180fe07..b8c3e457ae 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap @@ -48,6 +48,7 @@ expression: suite }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap index 17ed357836..cfd2db5e3d 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_order.py.snap @@ -46,6 +46,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -90,6 +91,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap index 642c1d4402..5506702509 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__invalid_rhs_expression.py.snap @@ -76,6 +76,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -117,6 +118,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap index b40badc953..5b39fbc339 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_0.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap index a9019dc09e..95295d193a 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__missing_rhs_2.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap index 7b1a47a58d..691867e8b2 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__named_expression.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -110,6 +111,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap index 4b546bf98c..053c2434fa 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__compare__starred_expression.py.snap @@ -46,6 +46,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -86,6 +87,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -123,6 +125,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -163,6 +166,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap index d8ba224fd8..e15f5a3659 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__double_star.py.snap @@ -442,6 +442,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -486,6 +487,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -530,6 +532,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap index 52c1ef0432..429b8d5fdf 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap @@ -86,6 +86,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap index 617832fc69..b5ff137f49 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__generator.py.snap @@ -90,6 +90,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap index 3dcdb1c7b9..4593ae60d8 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap @@ -48,6 +48,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -90,6 +91,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -720,6 +722,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, @@ -762,6 +765,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap index 5059784a2b..f4ab2b2c11 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap @@ -85,6 +85,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Load, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap index 6da0d01b1f..4597ac6bc8 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap @@ -307,6 +307,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap index ffcc5f43b8..4fcc9fb2a7 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap @@ -40,6 +40,7 @@ Module( }, ), ], + identity_ops: None, }, ), pattern: None, @@ -98,6 +99,7 @@ Module( }, ), ], + identity_ops: None, }, ), pattern: None, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap index d5ef96f286..b198947628 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap @@ -56,6 +56,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -125,6 +126,7 @@ Module( }, ), ], + identity_ops: None, }, ), arguments: Arguments { @@ -193,6 +195,7 @@ Module( }, ), ], + identity_ops: None, }, ), pattern: None, @@ -256,6 +259,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( @@ -336,6 +340,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( @@ -411,6 +416,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap index d9762e8efe..477401a8d4 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_assignment_targets.py.snap @@ -853,6 +853,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap index 84ed5916fe..6cc5335145 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@statements__invalid_augmented_assignment_target.py.snap @@ -732,6 +732,7 @@ Module( }, ), ], + identity_ops: None, }, ), op: Add, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap index 9ad6e62132..1555b0189c 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@while_stmt_missing_colon.py.snap @@ -40,6 +40,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: [ diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap index 279f8ac0b9..dac46093f8 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@ambiguous_lpar_with_items_binary_expr.py.snap @@ -98,6 +98,7 @@ Module( }, ), ], + identity_ops: None, }, ), optional_vars: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap index ce65176739..f2057516eb 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__await.py.snap @@ -390,6 +390,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap index b81211f86c..db6cc2c01e 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__compare.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -72,6 +73,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -105,6 +107,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -138,6 +141,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -171,6 +175,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -204,6 +209,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -237,6 +243,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -270,6 +277,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -303,6 +311,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -336,6 +345,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -405,6 +415,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -492,6 +503,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -530,6 +542,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -585,6 +598,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( @@ -639,6 +653,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -679,6 +694,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -784,6 +800,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap index f97cfa5d14..f0d0213c9f 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__dictionary_comprehension.py.snap @@ -336,6 +336,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -515,6 +516,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -645,6 +647,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap index f289d34a95..4b981ac072 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__f_string.py.snap @@ -370,6 +370,7 @@ Module( }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap index 3ff8b3044d..1087249e6a 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__generator.py.snap @@ -162,6 +162,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -316,6 +317,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -437,6 +439,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap index 82a85e4db9..cae10a971b 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__identity_compare.py.snap @@ -8,20 +8,20 @@ input_file: crates/ruff_python_parser/resources/valid/expressions/identity_compa Module( ModModule { node_index: NodeIndex(None), - range: 0..119, + range: 0..410, body: [ Expr( StmtExpr { node_index: NodeIndex(None), - range: 44..51, + range: 234..241, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 44..51, + range: 234..241, left: Name( ExprName { node_index: NodeIndex(None), - range: 44..45, + range: 234..235, id: Name("x"), ctx: Load, }, @@ -33,12 +33,19 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 50..51, + range: 240..241, id: Name("y"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + ], + }, + ), }, ), }, @@ -46,15 +53,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 52..59, + range: 242..249, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 52..59, + range: 242..249, left: Name( ExprName { node_index: NodeIndex(None), - range: 52..53, + range: 242..243, id: Name("x"), ctx: Load, }, @@ -66,12 +73,19 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 58..59, + range: 248..249, id: Name("y"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + ], + }, + ), }, ), }, @@ -79,15 +93,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 94..110, + range: 388..401, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 94..110, + range: 388..401, left: Name( ExprName { node_index: NodeIndex(None), - range: 94..95, + range: 388..389, id: Name("a"), ctx: Load, }, @@ -100,7 +114,7 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 100..101, + range: 394..395, id: Name("b"), ctx: Load, }, @@ -108,12 +122,20 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 109..110, + range: 400..401, id: Name("c"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + true, + ], + }, + ), }, ), }, @@ -121,15 +143,15 @@ Module( Expr( StmtExpr { node_index: NodeIndex(None), - range: 111..118, + range: 402..409, value: Compare( ExprCompare { node_index: NodeIndex(None), - range: 111..118, + range: 402..409, left: Name( ExprName { node_index: NodeIndex(None), - range: 111..112, + range: 402..403, id: Name("a"), ctx: Load, }, @@ -141,12 +163,19 @@ Module( Name( ExprName { node_index: NodeIndex(None), - range: 117..118, + range: 408..409, id: Name("b"), ctx: Load, }, ), ], + identity_ops: Some( + IdentityOperators { + ops: [ + true, + ], + }, + ), }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap index c321bfbe65..0c796aca94 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__if.py.snap @@ -186,6 +186,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: BinOp( @@ -325,6 +326,7 @@ Module( }, ), ], + identity_ops: None, }, ), orelse: Name( diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap index f39ea8650c..7f4b3f51ae 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list.py.snap @@ -964,6 +964,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap index b5e3275e91..02ee04f199 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__list_comprehension.py.snap @@ -228,6 +228,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -381,6 +382,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -501,6 +503,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -565,6 +568,7 @@ Module( }, ), ], + identity_ops: None, }, ), ifs: [], @@ -781,6 +785,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap index c6ba397fb6..7d0e3fa7bc 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__set_comprehension.py.snap @@ -116,6 +116,7 @@ Module( }, ), ], + identity_ops: None, }, ), BoolOp( @@ -269,6 +270,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], @@ -389,6 +391,7 @@ Module( }, ), ], + identity_ops: None, }, ), ], diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap index dc3f7f79a3..78f3e21ac2 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__t_string.py.snap @@ -353,6 +353,7 @@ Module( }, ), ], + identity_ops: None, }, ), debug_text: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap index 7613e48055..5c7a19142a 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield.py.snap @@ -404,6 +404,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap index 4eb78e54a3..fc81e1c3da 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@expressions__yield_from.py.snap @@ -330,6 +330,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap index 788be1a067..a4a71d6e70 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap @@ -52,6 +52,7 @@ Module( }, ), ], + identity_ops: None, }, ), ctx: Store, @@ -119,6 +120,7 @@ Module( }, ), ], + identity_ops: None, }, ), slice: NumberLiteral( @@ -195,6 +197,7 @@ Module( }, ), ], + identity_ops: None, }, ), attr: Identifier { diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap index 7eb5caad29..1be7df9bcd 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_1.py.snap @@ -39,6 +39,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap index 616aeb61bb..3d0d7d6abf 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_classify_as_identifier_2.py.snap @@ -53,6 +53,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -367,6 +368,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap index fc65621d8a..1a3feff163 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__ambiguous_lpar_with_items.py.snap @@ -625,6 +625,7 @@ Module( }, ), ], + identity_ops: None, }, ), optional_vars: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap index 23d81b8c5a..a0ad80056d 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__assert.py.snap @@ -41,6 +41,7 @@ Module( }, ), ], + identity_ops: None, }, ), msg: None, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap index f57cf2cb45..2aac57fe17 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__for.py.snap @@ -272,6 +272,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: [ diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap index 7adfc6f3f9..1cf47f0ccd 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__if.py.snap @@ -176,6 +176,7 @@ Module( }, ), ], + identity_ops: None, }, ), body: [ diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap index b3c84a2be2..1f38fe8c7d 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__match.py.snap @@ -3366,6 +3366,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), @@ -8077,6 +8078,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), @@ -9050,6 +9052,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap index ee2161cd6d..f4fafe203f 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__raise.py.snap @@ -106,6 +106,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), @@ -378,6 +379,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap index cf64239bc1..b0af82ddd0 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__return.py.snap @@ -195,6 +195,7 @@ Module( }, ), ], + identity_ops: None, }, ), ), diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap index 985fe73d59..a2a5b67daf 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__type.py.snap @@ -3201,6 +3201,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, @@ -3345,6 +3346,7 @@ Module( }, ), ], + identity_ops: None, }, ), }, diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap index fffc87aa97..577ba2b3bc 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@statement__while.py.snap @@ -75,6 +75,7 @@ Module( }, ), ], + identity_ops: None, }, ), Name( diff --git a/crates/ruff_python_semantic/src/model.rs b/crates/ruff_python_semantic/src/model.rs index 369830232e..0aa5ffb3ab 100644 --- a/crates/ruff_python_semantic/src/model.rs +++ b/crates/ruff_python_semantic/src/model.rs @@ -892,6 +892,18 @@ impl<'a> SemanticModel<'a> { self.in_basedpython_file() && self.in_trailing_lambda_block() } + /// True when the model is inside a type expression in a basedpython file, + /// where `or` and `and` are the + /// [keyword spellings](https://docs.basedpython.org/features/or-and-types) + /// of union and intersection. + /// + /// `A or B` there is the type `A | B`, not a boolean expression whose value + /// is `A` whenever `A` is truthy, so a rule that reasons about python's + /// boolean operators has nothing to say about it. + pub fn in_basedpython_type_expression(&self) -> bool { + self.in_basedpython_file() && self.in_type_definition() + } + /// True when the innermost function scope is a /// [trailing-lambda](https://docs.basedpython.org/features/trailing-lambdas) /// block, whose receiver the parser binds outside the source's reach diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index b01c3d6d6e..aa7953501a 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -76,6 +76,9 @@ over all configuration files.

--ignore rule

Disables the rule. Can be specified multiple times. Use 'all' to apply to all rules.

--no-progress

Hide all progress outputs.

For example, spinners or progress bars.

+
--no-server

Check the project from scratch instead of asking a running language server.

+

A server holding this project has already parsed and inferred it, and answers a check out of that in a fraction of the time. It only answers when the two agree about the configuration and there is nothing unsaved, so this is a way to rule the server out rather than a way to get a different answer.

+

BY_NO_PROJECT_SERVER does the same, and also stops a server from listening.

--output-format output-format

The format to use for printing diagnostic messages

May also be set with the TY_OUTPUT_FORMAT environment variable.

Possible values:

    @@ -305,7 +308,7 @@ by build [OPTIONS]
    --help, -h

    Print help (see a summary with '-h')

    --min-version version

    minimum Python version the output must run on [default: the project's configured python version]

    --no-unique-loop-bindings

    leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

    -
    --out, -o dir

    Where to write the output [default: out, or dist with --wheels]

    +
    --out, -o dir

    Where to write the output [default: build, or dist with --wheels]

    --print-manifest

    Report what the build read and produced, as <kind> <value> lines.

    input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

    --runtime-raises-checks

    wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

    @@ -367,8 +370,9 @@ by compile [OPTIONS] [FILE]...
    --no-any

    Fail the build when a function cannot be compiled because a type is gradual, instead of leaving it to its interpreted definition.

    A contract about predictability rather than a speed switch: a gradual type is the commonest reason a function silently stays interpreted.

    --no-unique-loop-bindings

    leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

    -
    --output, -o dir

    Where to write the generated C and the extension modules

    -

    [default: out]

    --require-native

    Fail the build if any function is left to its interpreted definition, whatever the reason.

    +
    --out, --output, -o dir

    Where to write the generated C and the extension modules.

    +

    The same directory by build writes, and the same flag names it — --output is kept because it was the only spelling this command took.

    +

    [default: build]

    --require-native

    Fail the build if any function is left to its interpreted definition, whatever the reason.

    Stricter than --no-any, and a different question: --no-any asks whether the module is fully typed, this asks whether it compiles entirely.

    --runtime-raises-checks

    wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

    --soundness spec

    which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

    diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 54bead8781..fce846bba4 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -2,6 +2,17 @@ ty defines and respects the following environment variables: +### `BY_NO_PROJECT_SERVER` + +Disable the project server: the language server answering `by` command lines. + +A server started for a project holds it parsed and inferred, and a `by check` in that +project asks the server rather than doing the work again. Set this to `1` to switch +that off — on a command line, so it checks for itself, and on a server, so it does +not open the socket that makes it reachable. + +Equivalent to the `--no-server` command-line argument. + ### `TY_CONFIG_FILE` Path to a `ty.toml` configuration file to use. diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 308fe6e800..838a845903 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -146,7 +146,7 @@ pub(crate) enum Command { /// `uv`, which does the packaging. #[arg(long, conflicts_with_all = ["min_version", "print_manifest"])] wheels: bool, - /// Where to write the output [default: `out`, or `dist` with `--wheels`] + /// Where to write the output [default: `build`, or `dist` with `--wheels`] #[arg(short = 'o', long, value_name = "DIR")] out: Option, /// Report what the build read and produced, as ` ` lines. @@ -196,7 +196,16 @@ pub(crate) enum Command { #[arg(value_name = "FILE")] files: Vec, /// Where to write the generated C and the extension modules. - #[arg(short = 'o', long, value_name = "DIR", default_value = "out")] + /// + /// The same directory `by build` writes, and the same flag names it — + /// `--output` is kept because it was the only spelling this command took. + #[arg( + short = 'o', + long = "out", + visible_alias = "output", + value_name = "DIR", + default_value = "build" + )] output: PathBuf, /// Report every function that was not lowered natively, with the reason. #[arg(long)] @@ -433,6 +442,17 @@ pub(crate) struct CheckCommand { #[arg(long, short = 'W')] pub(crate) watch: bool, + /// Check the project from scratch instead of asking a running language server. + /// + /// A server holding this project has already parsed and inferred it, and answers a + /// check out of that in a fraction of the time. It only answers when the two agree + /// about the configuration and there is nothing unsaved, so this is a way to rule the + /// server out rather than a way to get a different answer. + /// + /// `BY_NO_PROJECT_SERVER` does the same, and also stops a server from listening. + #[arg(long, help_heading = "Global options")] + pub(crate) no_server: bool, + /// Respect file exclusions via `.gitignore` and other standard ignore files. /// Use `--no-respect-ignore-files` to disable. #[arg( diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index e0089b50d3..3de90be6a1 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -846,14 +846,7 @@ pub(crate) fn cmd_build( let target = config.min_version.to_string(); crate::by_stamps::fill_discovered(&mut config.stamps, &cwd, Some(&target)); - // the output directory is settled before the project is read, because it is - // the one directory the project must not be read *from*: it holds a copy of - // every source this build is about to write. canonical, because that is what - // the paths it is compared against are — creating it first is what makes - // canonicalizing it possible - let out = cwd.join(out); - fs::create_dir_all(&out).with_context(|| format!("could not create {}", out.display()))?; - let out = fs::canonicalize(&out).unwrap_or(out); + let out = settled_output_dir(&cwd, out); let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, Some(&out))?; if handles.is_empty() { @@ -863,7 +856,7 @@ pub(crate) fn cmd_build( let file_count = handles.len(); let roots = module_roots(&db, &root); let mut staging = Staging::new(&out); - // `out/` outlives the build that wrote it — it is what a test runner, a + // `build/` outlives the build that wrote it — it is what a test runner, a // debugger or an editor plugin sees — so the sourcemap goes with it. this is // the directory where a `.by` really can be saved after the transpile, which // is what the digests beside the map are for @@ -889,12 +882,12 @@ pub(crate) fn cmd_build( }, ); // the sourcemap and the package markers describe the tree that was written, - // so they are staged whether or not something was reported — an `out/` a + // so they are staged whether or not something was reported — a `build/` a // debugger cannot read is worse than one built from a partial check stage_verbatim(&db, &root, &roots, &mut staging)?; stage_by_typed_markers(&db, &mut staging, &roots, &root)?; write_sourcemap_module(&mut staging, &entries)?; - // `out/` outlives the build that wrote it and is what a debugger, a test + // `build/` outlives the build that wrote it and is what a debugger, a test // runner or an editor plugin later reads, so it carries the same record a // `run` tree does. no entry module: a build is not pointed at one stage_build_record( @@ -946,6 +939,33 @@ fn print_build_manifest( Ok(()) } +/// Where a build's output goes, settled before the project is read. +/// +/// It is the one directory the project must not be read *from*: it holds a copy +/// of every source the build is about to write, so a second build that walked it +/// would take its own output for more of the project. Canonical, because the +/// paths it is compared against are. +/// +/// Nothing is created here. An output directory conjured before the command has +/// anything to put in it outlives every way the command can fail early, and a +/// project that has just been told it has no sources should not be left holding +/// an empty `build/` it did not ask for. A directory that does not exist yet +/// cannot be canonicalized, so its parent is, which answers the same question: +/// two paths naming this directory compare equal. +fn settled_output_dir(cwd: &Path, out: &Path) -> PathBuf { + let out = cwd.join(out); + if let Ok(canonical) = fs::canonicalize(&out) { + return canonical; + } + match (out.parent(), out.file_name()) { + (Some(parent), Some(name)) => match fs::canonicalize(parent) { + Ok(canonical) => canonical.join(name), + Err(_) => out, + }, + _ => out, + } +} + // ── compile ───────────────────────────────────────────────────────────────── /// How `by compile` was invoked. @@ -990,6 +1010,20 @@ pub(crate) fn cmd_compile( // to say which, and gets told so rather than told the wrong one crate::by_stamps::fill_discovered(&mut fallback.stamps, &cwd, None); options.fallback = Some(fallback); + // the python this writes into the tree is `by build`'s python, derived the + // way `build` derives it: from the version the *project* declares. + // + // it must not come from the interpreter the extensions are built against. + // the two commands share an output directory, so the module `build` writes + // and the module `compile` writes are the same file — and a project that + // declares `>=3.9` would get a tree lowered for 3.9 or for 3.13 depending on + // which command ran last. `build/` is also where an editable install points, + // so the losing half of that is code the project says it supports and no + // longer runs on + let mut tree_config = version_config(None, &cwd)?; + lowering.apply_for_build(&mut tree_config, &cwd)?; + let target = tree_config.min_version.to_string(); + crate::by_stamps::fill_discovered(&mut tree_config.stamps, &cwd, Some(&target)); let sources: Vec = if files.is_empty() { compilable_files(&cwd) } else { @@ -1009,17 +1043,20 @@ pub(crate) fn cmd_compile( // context that used to sit here misreported a version refusal as a missing header let toolchain = by_build::Toolchain::probe(&python)?; - let out_dir = cwd.join(output); + let out_dir = settled_output_dir(&cwd, output); let mut compiled = 0usize; let mut declined_total = 0usize; + // the output tree. `by_build` lays the artefacts out itself, so they are + // recorded rather than written through this — but they belong in the same + // manifest as everything else, or nothing ever takes a stale one back + let mut staging = Staging::new(&out_dir); // one database for the whole project, so a type imported from a sibling module // resolves. lowering each file on its own is sound — an unresolved class // degrades to the object protocol — but it makes every imported type look // gradual, and `--no-any` would then fail on noise - // `compile` embeds fallback source produced by the untyped transpile, which - // takes no db, so the rebuilder the other commands thread through is unused here - let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES, None)?; + let (db, project, rebuilder, root) = + build_project_db(&cwd, COMPILABLE_SOURCES, Some(&out_dir))?; // the database holds the whole project so a type imported from a sibling // resolves, but only the files that were *asked for* are checked and emitted. @@ -1076,6 +1113,47 @@ pub(crate) fn cmd_compile( render_diagnostics(&db, &diagnostics)?; } + // `compile` writes a superset of what `build` writes: the whole project as + // importable python, and native extensions for the modules that were asked + // for. two things follow from it, and neither is optional. + // + // the tree is a program rather than a heap of artefacts — a module nobody + // named still imports, from the python `by build` would have written, so + // `by compile one.by` no longer produces a tree that can import `one` and + // nothing else written in basedpython. + // + // and the manifest becomes sound. `finish` deletes what the last run wrote + // and this one did not, which is only a correct thing to do when this run + // authored the whole tree. while `compile` wrote artefacts alone, it deleted + // the sourcemap and build record a `by build` into the same directory had + // left — and `by restage` then refused that tree, which took the language + // server's single-file re-stage down with it + let roots = module_roots(&db, &root); + let transpilable: Vec<(PathBuf, ruff_db::files::File)> = project + .iter() + .filter(|(path, _)| { + path.extension() + .and_then(OsStr::to_str) + .is_some_and(|extension| BY_SOURCES.contains(&extension)) + }) + .cloned() + .collect(); + let mut entries: Vec = Vec::new(); + let mut requirements = by_transforms::RuntimeRequirements::default(); + let transpiled = render_check_and_transpile( + &db, + &transpilable, + &tree_config, + CheckGate::ParseErrorsOnly, + &rebuilder, + &mut requirements, + |emitted| { + let relative = transpiled_destination(&roots, &root, emitted.by_path); + entries.push(stage_module(&mut staging, &relative, emitted)?); + Ok(()) + }, + ); + // what each source will be compiled as, worked out before anything is written: // two sources that land on the same artefact used to leave only the second, and // nothing said so @@ -1109,63 +1187,139 @@ pub(crate) fn cmd_compile( planned.push((handle, name)); } - for ((path, file), name) in planned { - let source = fs::read_to_string(path) - .with_context(|| format!("could not read {}", path.display()))?; - - let program_file = ty_python_semantic::Db::program_file(&db, *file); - let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); - let model = ty_python_semantic::SemanticModel::new(&db, program_file); - // a `.py` source needs no transpiling to be its own interpreted fallback - let mut options = options.clone(); - if path.extension().is_some_and(|x| x == "py") { - options.language = by_irbuild::Language::Python; - } - let mut lowered = by_irbuild::build_module( - &db, - &model.program_environment(), - &model, - parsed.suite(), - name, - options.language, - ); - // the real path, so a `#line` in the generated C resolves for a debugger - let absolute = std::fs::canonicalize(path).unwrap_or_else(|_| path.clone()); - lowered.lines = Some(by_ir::function::LineTable::new( - absolute.display().to_string(), - &source, - )); + // the tree is finished whatever happens in here, so a run that gives up + // half way still leaves a manifest describing what is actually on disk. a + // manifest left describing the run before it would have the *next* run prune + // against a tree that no longer exists + let compiling: anyhow::Result<()> = 'compiling: { + for ((path, file), name) in planned { + let source = match fs::read_to_string(path) + .with_context(|| format!("could not read {}", path.display())) + { + Ok(source) => source, + Err(error) => break 'compiling Err(error), + }; - let built = if emit_c_only { - by_build::emit_lowered(lowered, &source, Some(&toolchain), &out_dir, &options) - } else { - by_build::build_lowered(lowered, &source, &toolchain, &out_dir, &options).inspect( - |built| { - eprintln!( - "{} -> {}", - path.display(), - built.artifact.extension.display() - ); - }, - ) + let program_file = ty_python_semantic::Db::program_file(&db, *file); + let parsed = + ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); + let model = ty_python_semantic::SemanticModel::new(&db, program_file); + // a `.py` source needs no transpiling to be its own interpreted fallback + let mut options = options.clone(); + if path.extension().is_some_and(|x| x == "py") { + options.language = by_irbuild::Language::Python; + } + let mut lowered = by_irbuild::build_module( + &db, + &model.program_environment(), + &model, + parsed.suite(), + name, + options.language, + ); + // the real path, so a `#line` in the generated C resolves for a debugger + let absolute = std::fs::canonicalize(path).unwrap_or_else(|_| path.clone()); + lowered.lines = Some(by_ir::function::LineTable::new( + absolute.display().to_string(), + &source, + )); + + let built = match if emit_c_only { + by_build::emit_lowered(lowered, &source, Some(&toolchain), &out_dir, &options) + } else { + by_build::build_lowered(lowered, &source, &toolchain, &out_dir, &options).inspect( + |built| { + if let Some(extension) = &built.artifact.extension { + eprintln!("{} -> {}", path.display(), extension.display()); + } + }, + ) + } + .with_context(|| format!("could not compile {}", path.display())) + { + Ok(built) => built, + Err(error) => break 'compiling Err(error), + }; + + // exactly what `by_build` says it wrote — no guessing at names, and no + // asking the file system, which cannot tell an artefact this run + // produced from one a previous run left + for produced in [&built.artifact.source, &built.artifact.header] + .into_iter() + .chain(built.artifact.extension.as_ref()) + .chain(built.artifact.annotation.as_ref()) + { + if let Ok(relative) = produced.strip_prefix(&out_dir) + && let Err(error) = staging.record(relative) + { + break 'compiling Err(error); + } + } + if let Some(annotation) = &built.artifact.annotation { + eprintln!(" annotated {}", annotation.display()); + } + declined_total += built.declined.len(); + if verbose { + // a decline is the compiler's report on the code it did *not* take, + // so it points at that code the way every other diagnostic does + let diagnostics: Vec = built + .declined + .iter() + .map(|declined| declined_diagnostic(*file, declined)) + .collect(); + if let Err(error) = render_diagnostics(&db, &diagnostics) { + break 'compiling Err(error); + } + } + compiled += 1; } - .with_context(|| format!("could not compile {}", path.display()))?; - if let Some(annotation) = &built.artifact.annotation { - eprintln!(" annotated {}", annotation.display()); + // the rest of the project: the data a module reads relative to itself, + // the hand-written `.py` beside it, the markers and the record that let a + // tool read the tree back. the same calls `build` makes, in the same + // order, because this is the same tree. + // + // inside the block with the compile, because each of these can fail and + // the tree has to be finished either way — `stage_verbatim` in particular + // is where a file the project keeps on top of an artefact is reported, + // which is a failure this change deliberately introduced + if let Err(error) = stage_verbatim(&db, &root, &roots, &mut staging) { + break 'compiling Err(error); } - declined_total += built.declined.len(); - if verbose { - // a decline is the compiler's report on the code it did *not* take, so - // it points at that code the way every other diagnostic does - let diagnostics: Vec = built - .declined - .iter() - .map(|declined| declined_diagnostic(*file, declined)) - .collect(); - render_diagnostics(&db, &diagnostics)?; + if let Err(error) = stage_by_typed_markers(&db, &mut staging, &roots, &root) { + break 'compiling Err(error); + } + if let Err(error) = write_sourcemap_module(&mut staging, &entries) { + break 'compiling Err(error); } - compiled += 1; + // whether the tree actually *holds* extensions, which is the question a + // reader of the record is asking — `--emit-c-only` writes the generated C + // and stops, so its tree is as replaceable one file at a time as a + // `by build`'s. the generated `.c` beside it is then a description of the + // source at the time it was written, and a later re-stage does not + // regenerate it; nothing loads it, so the two are allowed to drift + let holds_extensions = !emit_c_only && compiled > 0; + if let Err(error) = stage_build_record( + &mut staging, + &BuildRecord::new(&root, &roots, None, holds_extensions, &tree_config), + ) { + break 'compiling Err(error); + } + Ok(()) + }; + + // whatever happened above, the manifest is brought into line with what is + // actually on disk. a manifest left describing the run before would have the + // *next* run prune against a tree that no longer exists, leaving behind files + // no source produces — an importable module nobody wrote + staging.finish()?; + + compiling?; + // the transpile's verdict is answered after the tree is finished, the way + // `build` answers it: a diagnostic must not cost the tree, because a `build/` + // a debugger cannot read is worse than one built from a partial check + if !transpiled? { + return Ok(ExitStatus::Failure); } eprintln!("\ncompiled {compiled} module(s)"); @@ -1415,7 +1569,7 @@ fn reverse_dir_converting( /// Forward-transpile every `.by` under `dir` into a `.py` next to it, using one /// shared project db so cross-module types resolve (the same path as `by -/// build`, but written in place rather than to `out/`). +/// build`, but written in place rather than to `build/`). #[allow(clippy::print_stderr)] fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES, None)?; diff --git a/crates/ty/src/by_project_server.rs b/crates/ty/src/by_project_server.rs new file mode 100644 index 0000000000..b152d0dcac --- /dev/null +++ b/crates/ty/src/by_project_server.rs @@ -0,0 +1,192 @@ +//! Checking a project by asking the language server that is already holding it. +//! +//! `by check` builds a project database, resolves an environment, and parses and infers +//! every file in the project — and then exits, and throws all of it away. An editor with the +//! language server running has done that work already and has been keeping it current ever +//! since. So before doing it again, ask. +//! +//! What comes back has to be what this process would have produced on its own, which is why +//! most of what is here is the deciding rather than the asking. The server refuses whenever +//! it might answer differently — a different build, a different configuration, an unsaved +//! buffer — and this side refuses for every invocation whose answer is not simply "the +//! diagnostics for this project": a fix, a watch, a subset of paths. +//! +//! The fallback is not a failure mode, it is the normal path. Nothing here is required to +//! work, and a caller that gets nothing goes on to check for itself. + +use std::io::Write; + +use ruff_db::Db as _; +use ruff_db::diagnostic::Severity; +use ruff_db::system::SystemPath; +use ty_project::{Db as _, ProjectDatabase}; +use ty_server::project_server; +use ty_server::project_server::client; +use ty_server::project_server::protocol; +use ty_server::project_server::protocol::{Build, CheckRequest, CheckResponse, SeverityLevel}; + +use crate::printer::Printer; +use crate::{ExitStatus, exit_status_from_summary, write_summary}; + +/// Why one `by check` is not a check a server may answer. +/// +/// A server answers one question — the diagnostics for a whole project, as configured — so +/// anything that asks a narrower or a different one is ruled out here. Every one of these +/// would otherwise be a silent change in what the command does. +#[derive(Debug, Clone, Copy)] +pub(crate) enum Ineligible { + /// The caller passed `--no-server`. + /// + /// `BY_NO_PROJECT_SERVER` rules it out too, but later and separately: it is read through + /// the same [`System`](ruff_db::system::System) the server reads it through, which this + /// side does not have until it has a database. + Disabled, + + /// The check re-runs on every change, so everything after the first run is warm already. + Watch, + + /// The check rewrites the files it reports on, which a server does not do. + Fixing, + + /// The check was pointed at part of the project rather than all of it. + Paths, + + /// The caller asked about a database that this path never builds. + MemoryReport, +} + +impl std::fmt::Display for Ineligible { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Ineligible::Disabled => "the project server is disabled", + Ineligible::Watch => "the check is in watch mode", + Ineligible::Fixing => "the check rewrites the files it reports on", + Ineligible::Paths => "the check was given paths rather than the whole project", + Ineligible::MemoryReport => "a memory report was requested", + }) + } +} + +/// Asks a running server to check the project `db` was built for, and prints what it says. +/// +/// `None` when there was no answer, for any reason at all. The caller then checks the project +/// itself, which is what it would have done had none of this existed. +/// +/// `db` rather than the metadata it came from because the comparison that decides whether the +/// server may answer is over what this process *resolved* — its search paths, its interpreter, +/// its merged configuration — and resolving that is the database's job. Building one is cheap +/// next to checking with it, which is the part this exists to skip. +pub(crate) fn check( + db: &ProjectDatabase, + working_directory: &SystemPath, + printer: Printer, + ineligible: Option, +) -> Option { + if let Some(ineligible) = ineligible { + tracing::debug!("Not asking a project server: {ineligible}"); + return None; + } + + if project_server::disabled(db.system()) { + tracing::debug!("Not asking a project server: disabled by the environment"); + return None; + } + + // a platform with nowhere for a server to have announced itself + let directory = project_server::discovery::default_directory(db.system())?; + + let project = db.project(); + let options = protocol::configuration(project.metadata(db).to_merged_options().options()) + .inspect_err(|error| tracing::debug!("Failed to serialize the resolved options: {error}")) + .ok()?; + + let response = client::check( + &directory, + CheckRequest { + project_root: project.root(db).to_path_buf(), + working_directory: working_directory.to_path_buf(), + options, + environment: project_server::environment(db), + force_exclude: project.force_exclude(db), + verbose: project.verbose(db), + color: colored::control::SHOULD_COLORIZE.should_colorize(), + }, + &Build::current(ruff_db::program_version()?), + )?; + + Some(report(&response, printer)) +} + +/// Prints an answer, and says what the command should exit with. +/// +/// Separate from asking for it because from here on there is no falling back: the answer has +/// reached the caller's terminal, and checking again would print it twice. A stream that +/// cannot be written to is the same failure it is on the cold path, and is reported the same +/// way rather than being turned into "no server answered". +fn report(response: &CheckResponse, printer: Printer) -> ExitStatus { + // said before the diagnostics, and on stderr: a reader piping `--output-format=json` + // somewhere is still owed the explanation of why the answer arrived so fast, and is not + // owed it in the middle of their json + if printer.shows_general_messages() { + writeln!(std::io::stderr(), "using project server information").ok(); + } + + let written = write_answer(response, printer); + + // the project has already been checked and the answer is already partly on its way out, + // so a broken stream changes what the caller sees but not what the check found. the cold + // path reports the same failure the same way and returns the same status + if let Err(error) = written { + tracing::warn!("Failed to write the diagnostics: {error}"); + } + + exit_status_from_summary( + response.max_severity.map(severity), + response.io_error, + response.error_on_warning, + ) +} + +fn write_answer(response: &CheckResponse, printer: Printer) -> std::io::Result<()> { + { + let stdout = printer.stream_for_details().lock(); + if stdout.is_enabled() { + let mut stdout = std::io::BufWriter::new(stdout); + write!(stdout, "{}", response.rendered)?; + stdout.flush()?; + } + } + + // the cold path warns about both of these, and a caller that cannot tell which path + // answered it is a caller for whom the two paths are the same command + if response.empty_project { + tracing::warn!("No python files found under the given path(s)"); + } + if response.fatal { + tracing::warn!( + "A fatal error occurred while checking some files. \ + Not all project files were analyzed. \ + See the diagnostics list above for details." + ); + } + + // never cancelled: a check the server could not finish comes back as a refusal, and a + // refusal never reaches this far + write_summary( + printer, + response.diagnostics, + response.human_readable, + None, + false, + ) + .map_err(std::io::Error::other) +} + +fn severity(level: SeverityLevel) -> Severity { + match level { + SeverityLevel::Info => Severity::Info, + SeverityLevel::Warning => Severity::Warning, + SeverityLevel::Error => Severity::Error, + SeverityLevel::Fatal => Severity::Fatal, + } +} diff --git a/crates/ty/src/cli-reference.md b/crates/ty/src/cli-reference.md index d939c24546..a3ec9abc58 100644 --- a/crates/ty/src/cli-reference.md +++ b/crates/ty/src/cli-reference.md @@ -24,8 +24,8 @@ by run main --min-version 3.12 ## `by build` -transpile every `.by` file in the project to `out/`, mirroring the module -layout — a src-layout project's source root is stripped, so `out/` is +transpile every `.by` file in the project to `build/`, mirroring the module +layout — a src-layout project's source root is stripped, so `build/` is importable as it stands: ```sh @@ -33,9 +33,9 @@ by build ``` ```text -main.by -> out/main.py -utils.by -> out/utils.py -src/package_name/main.by -> out/package_name/main.py +main.by -> build/main.py +utils.by -> build/utils.py +src/package_name/main.by -> build/package_name/main.py ``` generated `.py` files are ordinary Python — run them with any Python tool diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index c81874bde3..9f1a763d88 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -2,6 +2,7 @@ mod args; mod by_commands; mod by_init; mod by_lowering; +mod by_project_server; mod by_source_encoding; mod by_stamps; mod by_wheels; @@ -162,7 +163,7 @@ fn run_command(command: Command) -> anyhow::Result { by_commands::cmd_build( min_version.as_deref(), &lowering, - out.as_deref().unwrap_or(Path::new("out")), + out.as_deref().unwrap_or(Path::new("build")), print_manifest, ) } @@ -362,6 +363,8 @@ fn run_check(args: CheckCommand) -> anyhow::Result { .map(|path| SystemPath::absolute(path, &cwd)) .collect(); + let check_paths_given = !check_paths.is_empty(); + let mode = if args.fix { MainLoopMode::Fix(FixMode::ApplyFixes) } else if args.add_ignore { @@ -394,8 +397,12 @@ fn run_check(args: CheckCommand) -> anyhow::Result { project_metadata.apply_configuration_files(&system)?; + let no_server = args.no_server; project_metadata.apply_override_options(args.into_options()); + // the answer this command exists to produce may already exist, in a server that has been + // holding this project open. what comes back is only ever the answer this process would + // have computed — see `by_project_server` for what that costs and what it refuses let mut db = ProjectDatabase::fallible(project_metadata, system)?; // the project's django, which the type checker does not read: its templates are @@ -413,6 +420,31 @@ fn run_check(args: CheckCommand) -> anyhow::Result { project.set_included_paths(&mut db, check_paths); } + // the answer this command exists to produce may already exist, in a server that has been + // holding this project open. asked here rather than earlier because what decides whether + // the server may answer is what *this* database resolved — see `by_project_server` + let ineligible = if no_server { + Some(by_project_server::Ineligible::Disabled) + } else if watch { + Some(by_project_server::Ineligible::Watch) + } else if !matches!(mode, MainLoopMode::Check) { + Some(by_project_server::Ineligible::Fixing) + } else if check_paths_given { + Some(by_project_server::Ineligible::Paths) + } else if memory_report.is_some() { + Some(by_project_server::Ineligible::MemoryReport) + } else { + None + }; + + if let Some(exit_status) = by_project_server::check(&db, &cwd, printer, ineligible) { + return Ok(if exit_zero { + ExitStatus::Success + } else { + exit_status + }); + } + // Disabling LRU only assumes that the database is short-lived; unlike freezing below, it does // not require immutable inputs. if !watch { @@ -851,60 +883,89 @@ impl MainLoop { let terminal_settings = db.project().settings(db).terminal(); let is_human_readable = terminal_settings.output_format.is_human_readable(); - match diagnostics { - [] if is_human_readable && fixed_diagnostics.is_none_or(|fixed| fixed == 0) => { - writeln!( - self.printer.stream_for_success_summary(), + { + let stdout = self.printer.stream_for_details().lock(); + + // Only render diagnostics if they're going to be displayed, since doing + // so is expensive. + if stdout.is_enabled() { + let mut stdout = BufWriter::new(stdout); + let display_config = DisplayDiagnosticConfig::new("ty") + .format(terminal_settings.output_format.into()) + .color(colored::control::SHOULD_COLORIZE.should_colorize()) + .with_cancellation_token(Some(self.cancellation_token.clone())) + .context(0); + + write!( + stdout, "{}", - "All checks passed!".green().bold() + DisplayDiagnostics::new(db, &display_config, diagnostics) )?; + stdout.flush()?; } - diagnostics => { - let diagnostics_count = diagnostics.len(); - - let stdout = self.printer.stream_for_details().lock(); - - // Only render diagnostics if they're going to be displayed, since doing - // so is expensive. - if stdout.is_enabled() { - let mut stdout = BufWriter::new(stdout); - let display_config = DisplayDiagnosticConfig::new("ty") - .format(terminal_settings.output_format.into()) - .color(colored::control::SHOULD_COLORIZE.should_colorize()) - .with_cancellation_token(Some(self.cancellation_token.clone())) - .context(0); - - write!( - stdout, - "{}", - DisplayDiagnostics::new(db, &display_config, diagnostics) - )?; - stdout.flush()?; - } + } - if !self.cancellation_token.is_cancelled() && is_human_readable { - if let Some(fixed) = fixed_diagnostics { - let total = fixed + diagnostics_count; - writeln!( - self.printer.stream_for_failure_summary(), - "Found {total} diagnostic{} \ - ({fixed} fixed, {diagnostics_count} remaining).", - if total == 1 { "" } else { "s" } - )?; - } else { - writeln!( - self.printer.stream_for_failure_summary(), - "Found {} diagnostic{}", - diagnostics_count, - if diagnostics_count > 1 { "s" } else { "" } - )?; - } - } - } + write_summary( + self.printer, + diagnostics.len(), + is_human_readable, + fixed_diagnostics, + self.cancellation_token.is_cancelled(), + ) + } +} + +/// Writes the line that says how a check came out, after its diagnostics. +/// +/// Shared with the [project server](crate::by_project_server) path, which renders its +/// diagnostics elsewhere but has to summarize them the same way — the wording of "how did +/// that go" belongs to the command, not to whoever computed the answer. +/// +/// `cancelled` suppresses the count but not the sentence for a project with nothing wrong +/// with it: a run interrupted before it finished counting cannot say how many there were, but +/// a fix run that removed everything and was then interrupted still removed everything. +pub(crate) fn write_summary( + printer: Printer, + diagnostics: usize, + is_human_readable: bool, + fixed_diagnostics: Option, + cancelled: bool, +) -> anyhow::Result<()> { + // nothing left and nothing fixed, which is the only outcome with its own sentence + if diagnostics == 0 && fixed_diagnostics.is_none_or(|fixed| fixed == 0) { + if is_human_readable { + writeln!( + printer.stream_for_success_summary(), + "{}", + "All checks passed!".green().bold() + )?; } + return Ok(()); + } - Ok(()) + // a check that was interrupted did not finish counting, so it has no total to report + if cancelled || !is_human_readable { + return Ok(()); } + + if let Some(fixed) = fixed_diagnostics { + let total = fixed + diagnostics; + writeln!( + printer.stream_for_failure_summary(), + "Found {total} diagnostic{} \ + ({fixed} fixed, {diagnostics} remaining).", + if total == 1 { "" } else { "s" } + )?; + } else { + writeln!( + printer.stream_for_failure_summary(), + "Found {} diagnostic{}", + diagnostics, + if diagnostics > 1 { "s" } else { "" } + )?; + } + + Ok(()) } #[derive(Copy, Clone, Debug)] @@ -949,18 +1010,32 @@ fn exit_status_from_diagnostics( diagnostics: &[Diagnostic], terminal_settings: &TerminalSettings, ) -> ExitStatus { - if diagnostics.is_empty() { - return ExitStatus::Success; - } - - let mut max_severity = Severity::Info; + let mut max_severity = None; let mut io_error = false; for diagnostic in diagnostics { - max_severity = max_severity.max(diagnostic.severity()); + max_severity = max_severity.max(Some(diagnostic.severity())); io_error = io_error || matches!(diagnostic.id(), DiagnosticId::Io); } + exit_status_from_summary(max_severity, io_error, terminal_settings.error_on_warning) +} + +/// The status a check exits with, from what it found. +/// +/// Separate from the diagnostics themselves so that the [project +/// server](crate::by_project_server) path decides it here too: that path never holds the +/// diagnostics — they are rendered where they were computed — but the rules for turning them +/// into an exit code belong to the command either way. +pub(crate) fn exit_status_from_summary( + max_severity: Option, + io_error: bool, + error_on_warning: bool, +) -> ExitStatus { + let Some(max_severity) = max_severity else { + return ExitStatus::Success; + }; + if !max_severity.is_fatal() && io_error { return ExitStatus::Error; } @@ -968,7 +1043,7 @@ fn exit_status_from_diagnostics( match max_severity { Severity::Info => ExitStatus::Success, Severity::Warning => { - if terminal_settings.error_on_warning { + if error_on_warning { ExitStatus::Failure } else { ExitStatus::Success diff --git a/crates/ty/src/printer.rs b/crates/ty/src/printer.rs index d72e1ebb5a..842a9492e6 100644 --- a/crates/ty/src/printer.rs +++ b/crates/ty/src/printer.rs @@ -102,6 +102,11 @@ impl Printer { self.stdout_general() } + /// Whether messages that are neither diagnostics nor a summary should be shown. + pub(crate) fn shows_general_messages(self) -> bool { + self.stdout_general().is_enabled() + } + pub(crate) fn clear_screen() -> Result<()> { clearscreen::clear()?; Ok(()) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 00993a4658..0fe72c3909 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -2,6 +2,7 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use ty_static::EnvVars; /// a temp directory of this process's own /// @@ -58,6 +59,398 @@ fn run_transpile(source: &str, extra_args: &[&str]) -> String { String::from_utf8(output.stdout).unwrap() } +/// A project whose program reads a data file sitting beside it. +/// +/// Written by each `compile` staging test, which then differ only in what they do +/// to the tree afterwards. +fn resource_project(name: &str) -> PathBuf { + let dir = cli_root().join(name); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("data")).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write( + dir.join("data").join("config.json"), + "{\"greeting\": \"hi\"}\n", + ) + .unwrap(); + fs::write( + dir.join("helper.py"), + "def helper() -> int:\n return 1\n", + ) + .unwrap(); + fs::write( + dir.join("main.by"), + "from pathlib import Path\n\n\ndef go() -> str:\n \ + return (Path(__file__).parent / \"data\" / \"config.json\").read_text()\n", + ) + .unwrap(); + dir +} + +/// Whether `by` refused because this host's interpreter is below the native floor. +/// +/// `by compile` probes an interpreter before it does anything, and +/// `by_build::MINIMUM_PYTHON` refuses one older than 3.11 by name. On a host whose +/// ambient `python3` is older that is not a failure of the code under test, and a +/// test that asserted its way through it reported a wall of unrelated noise — so +/// every `compile` test here skips on it, the way `by_build`'s own suite does. +/// Pin `PYTHON` to a 3.11+ interpreter to actually run them. +#[allow( + clippy::print_stderr, + reason = "skip notices belong on the harness's stderr" +)] +fn refused_for_python_version(result: &std::process::Output) -> bool { + let refused = String::from_utf8_lossy(&result.stderr) + .contains("a native build needs python 3.11 or later"); + if refused { + eprintln!("skipping: this host's python is below the native compilation floor"); + } + refused +} + +/// Run `by compile` in `dir`, or `None` when this host cannot compile natively. +fn compile_in(dir: &Path) -> Option { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only"]) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return None; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + Some(result) +} + +#[test] +fn compile_carries_the_rest_of_the_project_into_the_output_tree() { + // a compiled module is only half of a project. `main.by` reads its data file + // relative to itself, and the extension's `__file__` is its own place in the + // output tree — so a tree holding artefacts and nothing else fails on the + // first `open`, with a `FileNotFoundError` naming a path in a directory the + // author never wrote anything to. `compile` writes everything `build` writes + // and the extensions as well, so a compiled module finds its data exactly + // where its interpreted twin would + let dir = resource_project("by_cli_compile_resources"); + if compile_in(&dir).is_none() { + return; + } + + let out = dir.join("build"); + assert!(out.join("main.c").exists(), "the module is compiled"); + assert!( + out.join("data").join("config.json").exists(), + "a data file lands at the same relative place it had in the source" + ); + assert!( + out.join("helper.py").exists(), + "a hand-written python module beside the source is carried over too" + ); + assert!( + out.join("main.py").exists(), + "the `.by` is transpiled too, so the module imports whether or not its \ + extension was built" + ); + assert!( + out.join("_by_sourcemap.py").exists() && out.join("_by_build.json").exists(), + "the tree describes itself, the way a `by build` tree does" + ); +} + +#[test] +fn compiling_one_module_leaves_the_others_importable() { + // the docs offer `by compile app.hot` as "compile one module, leave the rest + // interpreted". while `compile` wrote artefacts alone that was not what + // happened: the modules nobody named reached the tree as `.by`, which python + // cannot import, and the second invocation's `finish` took the first's + // artefact back as well — so `compile a` then `compile b` left a tree that + // could import neither + let dir = resource_project("by_cli_compile_one_of_many"); + fs::write(dir.join("other.by"), "def other() -> int:\n return 2\n").unwrap(); + + let compile_one = |name: &str| -> bool { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only", name]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return false; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + true + }; + let out = dir.join("build"); + + if !compile_one("main.by") { + return; + } + assert!(out.join("main.c").exists(), "the named module is compiled"); + assert!( + out.join("other.py").exists(), + "a module nobody named still reaches the tree as importable python" + ); + + assert!(compile_one("other.by"), "the second compile ran"); + assert!( + out.join("other.c").exists(), + "the newly named one is compiled" + ); + assert!( + out.join("main.py").exists(), + "and the previously named one is still importable" + ); +} + +#[test] +fn a_compile_leaves_a_build_tree_readable() { + // `compile` and `build` write to the same directory by design. while + // `compile` wrote artefacts alone it took the sourcemap and the build record + // with it, and `by restage` — the language server's single-file re-stage — + // then refused the tree for having no `_by_build.json`, so a compile silently + // disabled the editor plugin against it + let dir = resource_project("by_cli_compile_then_restage"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!(result.status.success()); + if compile_in(&dir).is_none() { + return; + } + + let restaged = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["restage", "build", "main.by"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + let answer = String::from_utf8_lossy(&restaged.stdout); + assert!( + answer.contains("\"generated\""), + "the tree is still one a re-stage can read:\n{answer}" + ); +} + +#[test] +fn a_build_with_nothing_to_write_leaves_no_output_directory() { + // `by build` created the output directory before it knew whether it had + // anything to put in it, so a project with no `.by` files — one whose sources + // are all python, say — was left holding an empty `build/` it never asked for + let dir = cli_root().join("by_cli_build_no_litter"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.join("only.py"), "x = 1\n").unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&result.stderr); + assert!(result.status.success(), "by build failed:\n{stderr}"); + assert!( + stderr.contains("no .by files found"), + "the build found nothing to do:\n{stderr}" + ); + assert!( + !dir.join("build").exists(), + "a build that wrote nothing left a directory behind:\n{stderr}" + ); +} + +#[test] +fn a_second_output_directory_is_not_carried_into_the_first() { + // a project can have more than one output tree — `by build --out one` beside + // `by build --out two`. only the directory *this* run was given is known to + // be an output from its arguments; the other is recognised by the + // `.by-manifest` it carries, and without that it is carried over as though it + // were source, putting a whole copy of one tree inside the other + let dir = cli_root().join("by_cli_two_outputs"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.join("main.by"), "x = 1\n").unwrap(); + fs::write(dir.join("data.json"), "{}\n").unwrap(); + + let build_into = |name: &str| { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--out", name]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + }; + build_into("one"); + build_into("two"); + + assert!(dir.join("two").join("main.py").exists(), "the build ran"); + assert!( + !dir.join("two").join("one").exists(), + "the first output tree was copied into the second" + ); +} + +#[test] +fn compile_takes_back_what_the_previous_compile_wrote() { + // the output tree is a mirror rather than a pile: a resource deleted from the + // source is deleted from the tree. without the manifest it would keep being + // read, and a wheel built from the same tree would ship it + let dir = resource_project("by_cli_compile_stale"); + if compile_in(&dir).is_none() { + return; + } + let out = dir.join("build"); + assert!(out.join("data").join("config.json").exists()); + + fs::remove_file(dir.join("data").join("config.json")).unwrap(); + assert!(compile_in(&dir).is_some(), "the second compile ran"); + assert!( + !out.join("data").join("config.json").exists(), + "a resource the source no longer has is taken back out of the tree" + ); +} + +#[test] +fn a_build_takes_back_the_artifacts_a_compile_left() { + // `compile` and `build` write to the same directory by default, and python's + // finder prefers an extension to source. an artefact left behind by an + // earlier `compile` would therefore go on shadowing the `.py` this `build` + // writes in its place — which is why `compile` records what it produced even + // though `by_build` is what laid it out + let dir = resource_project("by_cli_compile_then_build"); + if compile_in(&dir).is_none() { + return; + } + let out = dir.join("build"); + assert!(out.join("main.c").exists()); + + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(out.join("main.py").exists(), "the build wrote the module"); + assert!( + !out.join("main.c").exists(), + "what the compile produced is taken back" + ); +} + +#[test] +fn compile_refuses_to_carry_a_file_over_an_artifact_it_wrote() { + // a project can keep a `main.c` of its own beside `main.by` — and the + // compiler writes its generated C under that same name. carrying the + // hand-written one over would leave a tree whose generated half is somebody + // else's file, with nothing said about it, so it is reported the way two + // sources claiming one module are + let dir = resource_project("by_cli_compile_artifact_collision"); + fs::write(dir.join("main.c"), "/* hand-written */\n").unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } + let stderr = String::from_utf8_lossy(&result.stderr); + assert!(!result.status.success(), "by should have failed:\n{stderr}"); + assert!( + stderr.contains("already wrote an artifact of that name"), + "the collision is named:\n{stderr}" + ); + assert!( + !fs::read_to_string(dir.join("build").join("main.c")) + .unwrap() + .contains("hand-written"), + "the generated C is left as the compiler wrote it" + ); +} + +#[test] +fn compile_does_not_read_the_tree_it_writes() { + // the output holds a copy of every resource the build carried over, including + // the project's `.py` modules. a second `compile` that walked them would + // compile each module twice — once from the source and once from the copy — + // and the two would claim the same artifact + // + // the directory is deliberately *not* one of the names the project walk skips + // by default (`build`, `out`, `target`, …). those are turned away whoever + // asks, so a tree written to one of them would pass this test even if the + // build never told the database where its own output was going. + // + // two mechanisms keep `generated/` out — the output this run was given, and + // the `.by-manifest` any output carries — and either alone is enough here. + // the manifest is the only one that covers an output this run was *not* + // given, which `a_second_output_directory_is_not_carried_into_the_first` + // is for + let dir = resource_project("by_cli_compile_not_own_input"); + let compile = || -> Option { + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "--emit-c-only", "-o", "generated"]) + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return None; + } + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + // the count `compile` reports, not the per-artifact lines: `--emit-c-only` + // does not print those, so counting them compares nothing to nothing + let stderr = String::from_utf8_lossy(&result.stderr); + let reported = stderr + .lines() + .find_map(|line| line.strip_prefix("compiled ")?.strip_suffix(" module(s)")) + .and_then(|count| count.parse::().ok()); + Some(reported.unwrap_or_else(|| panic!("no module count in:\n{stderr}"))) + }; + let Some(first) = compile() else { + return; + }; + assert!(first > 0, "the first compile compiled something"); + assert_eq!( + Some(first), + compile(), + "the second compile sees the same sources as the first" + ); +} + #[test] fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { // `by compile a.py` used to compile every source in the project and ignore the @@ -94,14 +487,18 @@ fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { ) .unwrap(); - let out = dir.join("out"); + let out = dir.join("build"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "wanted.py", "-o"]) .arg(&out) .arg("--emit-c-only") .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } assert!( result.status.success(), "by exited with error:\n{}", @@ -159,12 +556,16 @@ fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { let out = dir.join("o"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "-o"]) .arg(&out) .arg("--emit-c-only") .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } assert!( result.status.success(), "by exited with error:\n{}", @@ -215,12 +616,16 @@ fn compile_refuses_two_sources_that_would_write_the_same_artifact() { let out = dir.join("o"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "-o"]) .arg(&out) .arg("--emit-c-only") .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } assert!(!result.status.success(), "the clash is refused"); let stderr = String::from_utf8_lossy(&result.stderr); assert!( @@ -253,12 +658,16 @@ fn compile_declines_a_package_body_whose_package_has_no_importable_name() { let out = dir.join("o"); let result = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile", "-o"]) .arg(&out) .arg("--emit-c-only") .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&result) { + return; + } let stderr = String::from_utf8_lossy(&result.stderr); // declining one source is not a failed build — the rest of the project is // compiled, and what was left out is said rather than silently produced @@ -287,9 +696,10 @@ async def total(s: str, n: int) -> int: let file = dir.join("sound.by"); std::fs::write(&file, source).unwrap(); - let emitted = |spec: &str| -> String { + let emitted = |spec: &str| -> Option { let out = dir.join(spec); let status = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["compile"]) .arg(&file) .arg("-o") @@ -298,20 +708,26 @@ async def total(s: str, n: int) -> int: .current_dir(&dir) .output() .expect("failed to spawn by"); + if refused_for_python_version(&status) { + return None; + } assert!( status.status.success(), "by exited with error:\n{}", String::from_utf8_lossy(&status.stderr) ); - std::fs::read_to_string(out.join("sound.c")).expect("the C is readable") + Some(std::fs::read_to_string(out.join("sound.c")).expect("the C is readable")) }; + let Some(all) = emitted("all") else { + return; + }; assert!( - emitted("all").contains("_soundness_check"), + all.contains("_soundness_check"), "`all` puts the entry checks in the fallback" ); assert!( - !emitted("none").contains("_soundness_check"), + !emitted("none").is_some_and(|none| none.contains("_soundness_check")), "`none` leaves them out, so the flag is what made the difference" ); } @@ -322,6 +738,7 @@ fn run_executes_module() { fs::write(dir.path().join("main.by"), "print('hello from by run')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -357,6 +774,7 @@ fn run_names_the_by_source_for_file_and_argv() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -407,6 +825,7 @@ def main(): .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -435,6 +854,7 @@ fn run_enters_a_package_through_its_main_module() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "app"]) .current_dir(dir.path()) .output() @@ -464,6 +884,7 @@ fn run_still_rewrites_traceback_frames_to_by_lines() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -509,6 +930,7 @@ main() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -554,6 +976,7 @@ fn run_checks_a_deeply_nested_expression() { fs::write(dir.path().join("main.by"), format!("print({terms})\n")).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -575,6 +998,7 @@ fn run_force_unwrap_yields_inner_value() { fs::write(dir.path().join("main.by"), "x = Some(5)\nprint(x! + 1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -600,6 +1024,7 @@ fn run_invokes_top_level_main() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -624,6 +1049,7 @@ fn run_invokes_async_main_via_asyncio() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -648,6 +1074,7 @@ fn run_uses_the_configured_entry_point() { fs::write(dir.path().join("app.by"), "print('ran the entry point')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -677,6 +1104,7 @@ fn run_reads_the_entry_point_from_pyproject() { fs::write(dir.path().join("pkg/cli.by"), "print('ran pkg.cli')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -704,6 +1132,7 @@ fn run_reads_the_entry_point_from_basedpython_toml() { fs::write(dir.path().join("app.by"), "print('ran the entry point')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -733,6 +1162,7 @@ fn run_reads_the_entry_point_from_the_basedpython_section() { fs::write(dir.path().join("pkg/cli.by"), "print('ran pkg.cli')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -757,6 +1187,7 @@ fn run_prefers_an_explicit_module_over_the_configured_entry_point() { fs::write(dir.path().join("other.by"), "print('explicit')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "other"]) .current_dir(dir.path()) .output() @@ -783,6 +1214,7 @@ fn run_forwards_arguments_to_the_named_entry_point() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "app", "--name", "asdf"]) .current_dir(dir.path()) .output() @@ -802,6 +1234,7 @@ fn run_without_a_module_or_entry_point_reports_both_ways_out() { fs::write(dir.path().join("main.by"), "print('unreached')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -823,6 +1256,7 @@ fn run_main_with_args(source: &str, args: &[&str]) -> (String, String, i32) { fs::write(dir.path().join("main.by"), source).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .args(args) .current_dir(dir.path()) @@ -848,6 +1282,7 @@ fn run_main_stamped(source: &str, stamps: &[&str]) -> (String, String, i32) { fs::write(dir.path().join("basedpython.toml"), OPT_IN_TO_STAMPS).unwrap(); let mut command = Command::new(env!("CARGO_BIN_EXE_by")); + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command.arg("run"); for stamp in stamps { command.args(["--stamp", stamp]); @@ -1009,6 +1444,7 @@ def main(): fs::write(dir.path().join("basedpython.toml"), OPT_IN_TO_STAMPS).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "--compiled", "--python"]) .arg(&python) .args([ @@ -1185,6 +1621,7 @@ Grid()[(1, 2)] .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1214,6 +1651,7 @@ print(A.__sealed_members__ == (B, C)) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1247,6 +1685,7 @@ print(shout(\"moon\", greeting=\"good night\")) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1288,6 +1727,7 @@ print(\"quiet\".shouty) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1349,6 +1789,7 @@ print(b not in w) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1395,6 +1836,7 @@ print(Widget().kind) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1431,6 +1873,7 @@ extension str: .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1465,6 +1908,7 @@ print(Holder.value) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1502,6 +1946,7 @@ print(xs.second()) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1585,6 +2030,7 @@ print(Color.Green.name) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1623,6 +2069,7 @@ main() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1697,6 +2144,7 @@ print('verified', checked) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1741,6 +2189,7 @@ boom() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -1782,6 +2231,7 @@ fn build_skips_a_source_it_cannot_read() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--min-version", "3.12"]) .current_dir(dir.path()) .output() @@ -1797,11 +2247,11 @@ fn build_skips_a_source_it_cannot_read() { "the skipped file must be reported:\n{stderr}" ); assert_eq!( - fs::read_to_string(dir.path().join("out/good.py")).unwrap(), + fs::read_to_string(dir.path().join("build/good.py")).unwrap(), "x = 1\n" ); assert!( - !dir.path().join("out/bad.py").exists(), + !dir.path().join("build/bad.py").exists(), "an unreadable source must not be emitted as an empty module" ); } @@ -1887,6 +2337,7 @@ fn transpile_renders_parse_error_with_location() { fs::write(&by_path, "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .arg(&by_path) .output() @@ -1943,6 +2394,7 @@ fn transpile_malformed_inputs_never_panic() { ]; for src in inputs { let mut child = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -1977,6 +2429,7 @@ fn run_renders_parse_error_and_aborts() { fs::write(dir.path().join("main.by"), "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2004,6 +2457,7 @@ fn build_renders_parse_error_and_aborts() { fs::write(dir.path().join("bad.by"), "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2016,7 +2470,7 @@ fn build_renders_parse_error_and_aborts() { "stderr should include invalid-syntax diagnostic:\n{stderr}" ); assert!( - !dir.path().join("out").join("bad.py").exists(), + !dir.path().join("build").join("bad.py").exists(), "build should not emit output when parse error present" ); } @@ -2035,6 +2489,7 @@ fn run_refuses_to_execute_on_check_errors() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -2075,6 +2530,7 @@ fn run_min_version_newer_than_interpreter_errors() { fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "--min-version", "3.99", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2110,6 +2566,7 @@ fn run_honors_explicit_min_version() { fs::write(dir.path().join("main.by"), "print('versioned')\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "--min-version", "3.9", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2135,6 +2592,7 @@ fn build_skips_hidden_directories() { fs::write(hidden.join("junk.by"), "a b\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2146,9 +2604,9 @@ fn build_skips_hidden_directories() { !stderr.contains("junk"), "hidden-directory file must not be checked:\n{stderr}" ); - assert!(dir.path().join("out").join("main.py").exists()); + assert!(dir.path().join("build").join("main.py").exists()); assert!( - !dir.path().join("out").join(".claude").exists(), + !dir.path().join("build").join(".claude").exists(), "hidden-directory file must not be emitted" ); } @@ -2174,6 +2632,7 @@ fn build_writes_what_the_project_exports_into_its_marker() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2182,7 +2641,7 @@ fn build_writes_what_the_project_exports_into_its_marker() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert_eq!( - fs::read_to_string(dir.path().join("out").join("my_lib").join("by.typed")).unwrap(), + fs::read_to_string(dir.path().join("build").join("my_lib").join("by.typed")).unwrap(), "exported-dependencies = [\"numpy\"]\n" ); } @@ -2197,6 +2656,7 @@ fn build_writes_a_marker_for_a_project_that_exports_nothing() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2204,14 +2664,14 @@ fn build_writes_a_marker_for_a_project_that_exports_nothing() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let marker = dir.path().join("out").join("my_lib").join("by.typed"); - assert!(marker.exists(), "expected out/my_lib/by.typed:\n{stderr}"); + let marker = dir.path().join("build").join("my_lib").join("by.typed"); + assert!(marker.exists(), "expected build/my_lib/by.typed:\n{stderr}"); assert_eq!(fs::read_to_string(marker).unwrap(), ""); } /// a src-layout project's `src/pkg/main.by` is the module `pkg.main`, so the /// emitted tree has to be rooted at `src` — mirroring the directory instead -/// emits `out/src/pkg/main.py`, whose module is `src.pkg.main`, a name nothing +/// emits `build/src/pkg/main.py`, whose module is `src.pkg.main`, a name nothing /// imports and `run.main` cannot sensibly be set to #[test] fn build_mirrors_the_module_tree_not_the_directory_tree() { @@ -2228,6 +2688,7 @@ fn build_mirrors_the_module_tree_not_the_directory_tree() { fs::write(package.join("main.by"), "print(\"src layout\")\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2235,10 +2696,10 @@ fn build_mirrors_the_module_tree_not_the_directory_tree() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out"); + let out = dir.path().join("build"); assert!( out.join("package_name").join("main.py").exists(), - "expected out/package_name/main.py:\n{stderr}" + "expected build/package_name/main.py:\n{stderr}" ); assert!( !out.join("src").exists(), @@ -2246,7 +2707,7 @@ fn build_mirrors_the_module_tree_not_the_directory_tree() { ); } -/// `out/` outlives the build that wrote it — a test runner, a debugger or an +/// `build/` outlives the build that wrote it — a test runner, a debugger or an /// editor reads it later — so it is the tree where a `.by` really can be saved /// after the transpile, and the one that needs the digests to say so #[test] @@ -2255,6 +2716,7 @@ fn build_writes_a_sourcemap_beside_the_generated_python() { fs::write(dir.path().join("main.by"), "print(\"built\")\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2262,7 +2724,7 @@ fn build_writes_a_sourcemap_beside_the_generated_python() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out"); + let out = dir.path().join("build"); let map = fs::read_to_string(out.join("_by_sourcemap.py")).expect("sourcemap module"); // read the keys out of the file rather than rebuilding them: the build @@ -2310,6 +2772,7 @@ fn sourcemap_table_for(source: &str) -> (Vec>, String) { fs::write(dir.path().join("main.by"), source).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2320,7 +2783,7 @@ fn sourcemap_table_for(source: &str) -> (Vec>, String) { String::from_utf8_lossy(&output.stderr) ); - let out = dir.path().join("out"); + let out = dir.path().join("build"); let map = fs::read_to_string(out.join("_by_sourcemap.py")).expect("sourcemap module"); let generated = fs::read_to_string(out.join("main.py")).expect("generated module"); @@ -2417,6 +2880,7 @@ fn run_resolves_a_src_layout_entry_point() { fs::write(package.join("main.by"), "print(\"src layout\")\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(dir.path()) .output() @@ -2445,6 +2909,7 @@ fn build_targets_the_configured_python_version() { fs::write(dir.path().join("main.by"), "type X = int\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2452,7 +2917,7 @@ fn build_targets_the_configured_python_version() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let emitted = fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + let emitted = fs::read_to_string(dir.path().join("build/main.py")).unwrap(); assert!( !emitted.contains("typing_extensions"), "a 3.13 target needs no shim:\n{emitted}" @@ -2470,6 +2935,7 @@ fn build_emits_every_file_it_can_past_a_broken_one() { fs::write(dir.path().join("broken.by"), "x = (\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2477,7 +2943,7 @@ fn build_emits_every_file_it_can_past_a_broken_one() { let stderr = String::from_utf8_lossy(&output.stderr); assert!( - dir.path().join("out/good.py").exists(), + dir.path().join("build/good.py").exists(), "the parseable file must still be emitted:\n{stderr}" ); assert!( @@ -2504,6 +2970,7 @@ fn build_honours_src_exclude() { fs::write(negative.join("bad.by"), "def f(:\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -2515,8 +2982,8 @@ fn build_honours_src_exclude() { !stderr.contains("bad.by"), "excluded file checked:\n{stderr}" ); - assert!(dir.path().join("out/main.py").exists()); - assert!(!dir.path().join("out/tests").exists()); + assert!(dir.path().join("build/main.py").exists()); + assert!(!dir.path().join("build/tests").exists()); } #[test] @@ -2528,6 +2995,7 @@ fn transpile_proceeds_past_non_syntax_errors() { fs::write(&by_path, "x: int = \"string\"\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .arg(&by_path) .output() @@ -2567,6 +3035,7 @@ fn transpile_directory_reverses_in_place() { fs::write(root.join(".venv/dep.py"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("transpile") .arg("--reverse") .arg(root) @@ -2604,6 +3073,7 @@ fn transpile_directory_round_trips_through_build() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(root) .output() @@ -2614,7 +3084,7 @@ fn transpile_directory_round_trips_through_build() { String::from_utf8_lossy(&output.stderr) ); - let built = fs::read_to_string(root.join("out/pkg/models.py")).unwrap(); + let built = fs::read_to_string(root.join("build/pkg/models.py")).unwrap(); assert!( built.contains("x if x is not None else 0"), "coalesce lowered back to python:\n{built}" @@ -2728,6 +3198,7 @@ a.f() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", &python) .current_dir(dir.path()) @@ -2799,6 +3270,7 @@ fn declared_reified_generic_runs() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2840,6 +3312,7 @@ fn reified_generic_infers_specialization_from_arguments() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2902,6 +3375,7 @@ Box().kind[float]() .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -2989,6 +3463,7 @@ print(sorted(s)) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3067,6 +3542,7 @@ x(A(True)) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -3140,6 +3616,7 @@ print(h(A(\"x\"))) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3207,6 +3684,7 @@ con(Con[object]()) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3285,6 +3763,7 @@ print(object() is Sequence[int]) .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -3331,6 +3810,7 @@ def main(): .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main", "--runtime-raises-checks"]) .current_dir(dir.path()) .output() @@ -3380,6 +3860,7 @@ def main(): .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main", "--runtime-raises-checks"]) .current_dir(dir.path()) .output() @@ -3425,6 +3906,7 @@ def main(): let check = || { Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("check") .current_dir(dir.path()) .output() @@ -3498,6 +3980,7 @@ fn a_lazy_from_import_resolves_a_submodule_and_refuses_a_missing_name_as_python_ .unwrap(); let transpiled = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["transpile", "main.by"]) .env("PYTHON", python) .current_dir(dir.path()) @@ -3564,6 +4047,7 @@ fn build_carries_a_python_module_into_the_output() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3572,7 +4056,7 @@ fn build_carries_a_python_module_into_the_output() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert_eq!( - fs::read_to_string(dir.path().join("out/helper.py")).unwrap(), + fs::read_to_string(dir.path().join("build/helper.py")).unwrap(), "def shout(text: str) -> str:\n return text.upper()\n", "a hand-written python module belongs in the output verbatim" ); @@ -3591,6 +4075,7 @@ fn build_carries_data_files_into_the_output() { fs::write(package.join("template.html"), "

    hi

    \n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3598,7 +4083,7 @@ fn build_carries_data_files_into_the_output() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out").join("app"); + let out = dir.path().join("build").join("app"); assert_eq!( fs::read_to_string(out.join("settings.json")).unwrap(), "{\"key\": 1}\n" @@ -3617,6 +4102,7 @@ fn build_writes_a_stub_as_a_stub() { fs::write(dir.path().join("shapes.byi"), "def area() -> int: ...\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3625,11 +4111,11 @@ fn build_writes_a_stub_as_a_stub() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert!( - dir.path().join("out/shapes.pyi").exists(), + dir.path().join("build/shapes.pyi").exists(), "a `.byi` builds to a `.pyi`:\n{stderr}" ); assert!( - !dir.path().join("out/shapes.py").exists(), + !dir.path().join("build/shapes.py").exists(), "a stub emitted as a module shadows the implementation" ); } @@ -3644,6 +4130,7 @@ fn build_refuses_two_sources_that_are_one_module() { fs::write(dir.path().join("thing.py"), "x = 2\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3671,6 +4158,7 @@ fn build_deletes_output_the_project_no_longer_has() { let build = || { let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3683,14 +4171,14 @@ fn build_deletes_output_the_project_no_longer_has() { }; build(); - assert!(dir.path().join("out/removed.py").exists()); + assert!(dir.path().join("build/removed.py").exists()); fs::remove_file(dir.path().join("removed.by")).unwrap(); build(); - assert!(dir.path().join("out/kept.py").exists()); + assert!(dir.path().join("build/kept.py").exists()); assert!( - !dir.path().join("out/removed.py").exists(), + !dir.path().join("build/removed.py").exists(), "output for a source that is gone must not survive the next build" ); } @@ -3701,10 +4189,11 @@ fn build_deletes_output_the_project_no_longer_has() { fn build_leaves_output_it_never_wrote_alone() { let dir = tempfile::tempdir().expect("tempdir"); fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); - fs::create_dir_all(dir.path().join("out")).unwrap(); - fs::write(dir.path().join("out/theirs.txt"), "hands off\n").unwrap(); + fs::create_dir_all(dir.path().join("build")).unwrap(); + fs::write(dir.path().join("build/theirs.txt"), "hands off\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3715,7 +4204,7 @@ fn build_leaves_output_it_never_wrote_alone() { "by build failed:\n{}", String::from_utf8_lossy(&output.stderr) ); - assert!(dir.path().join("out/theirs.txt").exists()); + assert!(dir.path().join("build/theirs.txt").exists()); } #[test] @@ -3724,6 +4213,7 @@ fn build_writes_where_out_says() { fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--out", "elsewhere"]) .current_dir(dir.path()) .output() @@ -3732,7 +4222,7 @@ fn build_writes_where_out_says() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert!(dir.path().join("elsewhere/main.py").exists()); - assert!(!dir.path().join("out").exists()); + assert!(!dir.path().join("build").exists()); } /// the output directory is not an input to itself, wherever it is put @@ -3743,6 +4233,7 @@ fn build_does_not_read_its_own_output() { for _ in 0..2 { let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--out", "elsewhere"]) .current_dir(dir.path()) .output() @@ -3772,6 +4263,7 @@ fn build_reports_what_it_read_and_what_it_produced() { fs::write(package.join("helper.py"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .current_dir(dir.path()) .output() @@ -3828,6 +4320,7 @@ fn build_does_not_ship_what_lives_outside_the_source_root() { fs::write(tests.join("test_it.py"), "def test_x(): pass\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .current_dir(dir.path()) .output() @@ -3847,9 +4340,9 @@ fn build_does_not_ship_what_lives_outside_the_source_root() { ); // it is still built, because it is still the project — running the tests out // of the output tree is the point of building them - assert!(dir.path().join("out/tests/test_it.py").exists()); + assert!(dir.path().join("build/tests/test_it.py").exists()); assert!( - !dir.path().join("out/tests/by.typed").exists(), + !dir.path().join("build/tests/by.typed").exists(), "a marker only speaks for what the project ships" ); } @@ -3865,6 +4358,7 @@ fn build_marks_a_package_as_carrying_its_sources() { fs::write(package.join("deep.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3872,7 +4366,7 @@ fn build_marks_a_package_as_carrying_its_sources() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out").join("app"); + let out = dir.path().join("build").join("app"); assert!( out.join("by.typed").exists(), "expected a marker:\n{stderr}" @@ -3898,6 +4392,7 @@ fn build_ships_python_only_when_the_project_says_so() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3905,7 +4400,7 @@ fn build_ships_python_only_when_the_project_says_so() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - let out = dir.path().join("out").join("app"); + let out = dir.path().join("build").join("app"); assert!(out.join("__init__.py").exists()); assert!( !out.join("__init__.by").exists(), @@ -3935,6 +4430,7 @@ fn build_honours_the_configured_exclusions() { fs::write(dir.path().join("public.json"), "{}\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3942,9 +4438,9 @@ fn build_honours_the_configured_exclusions() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - assert!(dir.path().join("out/public.json").exists()); + assert!(dir.path().join("build/public.json").exists()); assert!( - !dir.path().join("out/secrets.json").exists(), + !dir.path().join("build/secrets.json").exists(), "an excluded file must not reach the output" ); } @@ -3968,6 +4464,7 @@ fn build_carries_a_directory_a_negated_exclude_takes_back() { fs::write(generated.join("kept.json"), "{}\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -3976,11 +4473,11 @@ fn build_carries_a_directory_a_negated_exclude_takes_back() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); assert!( - dir.path().join("out/dist/kept.py").exists(), + dir.path().join("build/dist/kept.py").exists(), "the re-included `.by` builds:\n{stderr}" ); assert!( - dir.path().join("out/dist/kept.json").exists(), + dir.path().join("build/dist/kept.json").exists(), "and so does everything beside it:\n{stderr}" ); } @@ -4003,6 +4500,7 @@ fn build_ships_a_source_directory_that_is_itself_a_package() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .current_dir(dir.path()) .output() @@ -4018,7 +4516,7 @@ fn build_ships_a_source_directory_that_is_itself_a_package() { stdout.lines().any(|line| line == "package src"), "`src.mymod` is the module, so `src` is the package:\n{stdout}" ); - assert!(dir.path().join("out/src/mymod/__init__.py").exists()); + assert!(dir.path().join("build/src/mymod/__init__.py").exists()); } /// lowering for an older python can put a name in the output that only @@ -4044,6 +4542,7 @@ fn build_reports_what_lowering_needs_at_run_time() { let manifest = |extra: &[&str]| -> String { let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--print-manifest"]) .args(extra) .current_dir(dir.path()) @@ -4088,6 +4587,7 @@ fn building_wheels_without_a_frontend_says_what_is_missing() { fs::write(package.join("__init__.by"), "").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels"]) .current_dir(dir.path()) // an empty `PATH` is the only way to be sure this machine's `uv` is not @@ -4142,8 +4642,9 @@ fn build_emitting(source: &str, settled: Option<&str>) -> String { fs::write(dir.path().join("main.by"), source).unwrap(); let mut command = Command::new(env!("CARGO_BIN_EXE_by")); + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command - .args(["build", "--out", "out"]) + .args(["build", "--out", "build"]) .current_dir(dir.path()); match settled { Some(settled) => command.env("BY_BUILD_LOWERING", settled), @@ -4154,7 +4655,7 @@ fn build_emitting(source: &str, settled: Option<&str>) -> String { let output = command.output().expect("failed to spawn by"); let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!(output.status.success(), "by build failed:\n{stderr}"); - fs::read_to_string(dir.path().join("out").join("main.py")).expect("emitted python") + fs::read_to_string(dir.path().join("build").join("main.py")).expect("emitted python") } /// the builds inside a `--wheels` release are the ones that transpile, so every @@ -4234,6 +4735,7 @@ fn only_a_build_takes_its_lowering_from_the_environment() { fs::write(dir.path().join("main.by"), LOWERED_THREE_WAYS).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["transpile", "main.by"]) .current_dir(dir.path()) .env( @@ -4261,6 +4763,7 @@ fn building_wheels_refuses_a_soundness_spec_it_cannot_parse() { fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels", "--soundness", "nonsense"]) .current_dir(dir.path()) // refused before the frontend is even looked for, so this holds on a @@ -4312,6 +4815,7 @@ fn a_release_hands_each_build_the_stamps_and_the_lowering() { fs::set_permissions(&uv, fs::Permissions::from_mode(0o755)).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels", "--soundness", "none"]) .current_dir(dir.path()) // the stub is the only `uv` reachable, so this cannot accidentally @@ -4348,6 +4852,7 @@ fn building_wheels_refuses_a_single_target_version() { fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["build", "--wheels", "--min-version", "3.12"]) .current_dir(dir.path()) .output() @@ -4381,6 +4886,7 @@ fn run_imports_a_python_module_beside_the_transpiled_ones() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -4406,6 +4912,7 @@ fn run_reads_a_data_file_beside_the_program() { fs::write(dir.path().join("greeting.txt"), "read from disk\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -4448,6 +4955,7 @@ fn run_refuses_an_interpreter_older_than_the_project_targets() { fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .env_remove("PYTHON") @@ -4596,6 +5104,7 @@ fn run_uses_the_environment_the_project_configures() { fs::write(dir.path().join("main.by"), REPORTS_ITS_INTERPRETER).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .env_remove("PYTHON") @@ -4635,6 +5144,7 @@ fn run_from_a_subdirectory_is_still_the_project() { fs::create_dir_all(&elsewhere).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(&elsewhere) .env_remove("PYTHON") @@ -4665,6 +5175,7 @@ fn build_from_a_subdirectory_builds_the_project() { fs::create_dir_all(&elsewhere).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(&elsewhere) .output() @@ -4674,7 +5185,7 @@ fn build_from_a_subdirectory_builds_the_project() { assert!(output.status.success(), "by build failed:\n{stderr}"); assert!( elsewhere - .join("out") + .join("build") .join("app") .join("__init__.py") .exists(), @@ -4704,6 +5215,7 @@ fn run_prefers_the_project_environment_to_the_python_variable() { fs::write(dir.path().join("main.by"), REPORTS_ITS_INTERPRETER).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .env("PYTHON", named.interpreter()) @@ -4741,6 +5253,7 @@ fn run_falls_back_to_the_python_variable() { fs::write(project.join("main.by"), REPORTS_ITS_INTERPRETER).unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(&project) .env("PYTHON", elsewhere.interpreter()) @@ -4772,6 +5285,7 @@ fn run_refuses_a_configured_environment_that_is_not_one() { fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -4795,6 +5309,7 @@ fn run_refuses_a_project_file_that_collides_with_its_shim() { fs::write(dir.path().join("_by_runner.py"), "x = 1\n").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) .current_dir(dir.path()) .output() @@ -4818,6 +5333,7 @@ fn build_does_not_carry_a_compilers_output_directory() { fs::write(artifacts.join("blob"), "an enormous binary").unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("build") .current_dir(dir.path()) .output() @@ -4825,9 +5341,9 @@ fn build_does_not_carry_a_compilers_output_directory() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(output.status.success(), "by build failed:\n{stderr}"); - assert!(dir.path().join("out").join("main.py").exists()); + assert!(dir.path().join("build").join("main.py").exists()); assert!( - !dir.path().join("out").join("target").exists(), + !dir.path().join("build").join("target").exists(), "a build directory must not be carried into the build:\n{stderr}" ); } @@ -4842,6 +5358,7 @@ fn init_writes_a_project_that_builds_and_runs() { let (major, minor) = running_python_version(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args([ "init", "demo", @@ -4860,6 +5377,7 @@ fn init_writes_a_project_that_builds_and_runs() { assert!(project.join("src/demo/__init__.by").exists()); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("run") .current_dir(&project) .output() @@ -4885,6 +5403,7 @@ fn init_refuses_to_write_over_a_project() { .unwrap(); let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .arg("init") .current_dir(dir.path()) .output() diff --git a/crates/ty/tests/by_restage.rs b/crates/ty/tests/by_restage.rs index 061ff2769b..954d9f613e 100644 --- a/crates/ty/tests/by_restage.rs +++ b/crates/ty/tests/by_restage.rs @@ -24,7 +24,7 @@ fn write_project(dir: &Path) { fn build(dir: &Path) { let status = Command::new(env!("CARGO_BIN_EXE_by")) - .args(["build", "--out", "out"]) + .args(["build", "--out", "build"]) .current_dir(dir) .status() .expect("`by build` should run"); @@ -33,7 +33,7 @@ fn build(dir: &Path) { fn restage(dir: &Path, file: &str) -> (bool, serde_json::Value) { let out = Command::new(env!("CARGO_BIN_EXE_by")) - .args(["restage", "out", file]) + .args(["restage", "build", file]) .current_dir(dir) .output() .expect("`by restage` should run"); @@ -78,13 +78,13 @@ build-stamps = true .unwrap(); let status = Command::new(env!("CARGO_BIN_EXE_by")) - .args(["build", "--out", "out", "--stamp", "GIT_SHA=abc123"]) + .args(["build", "--out", "build", "--stamp", "GIT_SHA=abc123"]) .current_dir(dir.path()) .status() .expect("`by build` should run"); assert!(status.success(), "`by build` failed"); - let on_disk = std::fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + let on_disk = std::fs::read_to_string(dir.path().join("build/main.py")).unwrap(); assert!( on_disk.contains(r#"GIT_SHA: str = "abc123""#), "the build should have stamped the value:\n{on_disk}" @@ -111,7 +111,7 @@ fn restaging_a_file_nobody_edited_reproduces_the_build_exactly() { write_project(dir.path()); build(dir.path()); - let on_disk = std::fs::read_to_string(dir.path().join("out/main.py")).unwrap(); + let on_disk = std::fs::read_to_string(dir.path().join("build/main.py")).unwrap(); let (ok, answer) = restage(dir.path(), "main.by"); assert!(ok, "an unedited file should re-stage: {answer}"); @@ -137,7 +137,7 @@ fn the_generated_path_is_absolute_even_for_a_relative_build_directory() { generated.display() ); assert!( - generated.ends_with("out/main.py"), + generated.ends_with("build/main.py"), "{}", generated.display() ); @@ -284,7 +284,7 @@ fn a_directory_that_is_not_a_build_is_refused() { let dir = tempfile::tempdir().unwrap(); write_project(dir.path()); build(dir.path()); - std::fs::remove_file(dir.path().join("out/_by_build.json")).unwrap(); + std::fs::remove_file(dir.path().join("build/_by_build.json")).unwrap(); let (ok, answer) = restage(dir.path(), "main.by"); @@ -306,7 +306,7 @@ fn a_tree_built_by_another_by_is_refused() { write_project(dir.path()); build(dir.path()); - let record = dir.path().join("out/_by_build.json"); + let record = dir.path().join("build/_by_build.json"); let mut parsed: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&record).unwrap()).unwrap(); parsed["byVersion"] = serde_json::Value::String("0.0.0+somethingelse".to_owned()); diff --git a/crates/ty/tests/cli/main.rs b/crates/ty/tests/cli/main.rs index d073a599a9..8cb08a2f1d 100644 --- a/crates/ty/tests/cli/main.rs +++ b/crates/ty/tests/cli/main.rs @@ -5,6 +5,7 @@ mod django; mod exit_code; mod file_selection; mod fixes; +mod project_server; mod python_environment; mod rule; mod rule_selection; @@ -23,6 +24,7 @@ use std::{ process::Command, }; use tempfile::TempDir; +use ty_static::EnvVars; #[test] fn test_quiet_output() -> anyhow::Result<()> { @@ -1097,6 +1099,9 @@ impl CliTest { user_config_directory_env_var(), self.user_config_directory(), ); + // These tests assert on exact output, and a language server running on this machine + // for a directory above the test's own would otherwise be entitled to answer them. + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command } @@ -1111,6 +1116,7 @@ impl CliTest { user_config_directory_env_var(), self.user_config_directory(), ); + command.env(EnvVars::BY_NO_PROJECT_SERVER, "1"); command } diff --git a/crates/ty/tests/cli/project_server.rs b/crates/ty/tests/cli/project_server.rs new file mode 100644 index 0000000000..e5c0128476 --- /dev/null +++ b/crates/ty/tests/cli/project_server.rs @@ -0,0 +1,65 @@ +//! `by check`'s side of the [project server](../../../src/by_project_server.rs). +//! +//! What a server would say, and whether it is asked at all, is covered against a real server +//! in `ty_server`'s end-to-end tests. What is left here is the command line's own half: that +//! the switch exists, that it does not change the answer, and that these tests are not +//! quietly being answered by whatever server the person running them has open. + +use anyhow::Result; +use insta_cmd::assert_cmd_snapshot; + +use crate::CliTest; + +/// The answer is the answer either way. `--no-server` chooses how it is arrived at, and a +/// flag that changed what came out would be a different command rather than a faster one. +#[test] +fn no_server_does_not_change_the_answer() -> Result<()> { + let case = CliTest::with_file( + "main.py", + r#" +def f() -> str: + return 42 +"#, + )?; + + let with_server = case.command().output()?; + assert_cmd_snapshot!(case.command().arg("--no-server"), @r" + success: false + exit_code: 1 + ----- stdout ----- + error[invalid-return-type]: Return type does not match returned value + --> main.py:3:12 + | + 2 | def f() -> str: + | --- Expected `str` because of return type + 3 | return 42 + | ^^ expected `str`, found `Literal[42]` + + Found 1 diagnostic + + ----- stderr ----- + "); + + assert_eq!( + String::from_utf8(with_server.stdout)?, + String::from_utf8(case.command().arg("--no-server").output()?.stdout)? + ); + + Ok(()) +} + +/// These tests assert on exact output, so none of them may be answered out of a language +/// server that happens to be running on this machine. `CliTest` sets the kill switch for +/// every command it builds; this is the assertion that it still does. +#[test] +fn the_tests_never_ask_a_server() -> Result<()> { + let case = CliTest::with_file("main.py", "x: int = 1\n")?; + + let output = case.command().output()?; + assert!( + !String::from_utf8(output.stderr)?.contains("using project server information"), + "a cli test was answered by a project server" + ); + + Ok(()) +} diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 33ffc07e98..f0265da138 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -1337,18 +1337,16 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// basedpython: hint `reified` on each type parameter of `function` that a /// value-position use in the body reifies without saying so. fn add_inferred_reification(&mut self, function: &ast::StmtFunctionDef) { - let inferred = |hints: &mut Self| { - inferred_reified_type_param_names(hints.source, hints.source_type, function) - }; + let inferred = + |hints: &mut Self| inferred_reified_type_param_names(hints.source_type, function); self.add_inferred_reification_of(function.type_params.as_deref(), inferred); } /// basedpython: the same hint on a class, which reifies a type parameter its /// methods read through their receiver. fn add_inferred_class_reification(&mut self, class: &ast::StmtClassDef) { - let inferred = |hints: &mut Self| { - inferred_reified_class_type_param_names(hints.source, hints.source_type, class) - }; + let inferred = + |hints: &mut Self| inferred_reified_class_type_param_names(hints.source_type, class); self.add_inferred_reification_of(class.type_params.as_deref(), inferred); } diff --git a/crates/ty_project/src/db.rs b/crates/ty_project/src/db.rs index 9f1d0d782e..ed556a03ad 100644 --- a/crates/ty_project/src/db.rs +++ b/crates/ty_project/src/db.rs @@ -267,6 +267,11 @@ impl ProjectDatabase { self.checker = Some(checker); } + /// The set of files this database checks. + pub fn check_mode(&self) -> CheckMode { + self.project().check_mode(self) + } + /// Set the check mode for the project. pub fn set_check_mode(&mut self, mode: CheckMode) { if self.project().check_mode(self) != mode { diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index e8c1992002..b9f0907709 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -675,7 +675,7 @@ impl Project { } } - fn verbose(self, db: &dyn Db) -> bool { + pub fn verbose(self, db: &dyn Db) -> bool { self.verbose_flag(db) } @@ -685,7 +685,7 @@ impl Project { } } - fn force_exclude(self, db: &dyn Db) -> bool { + pub fn force_exclude(self, db: &dyn Db) -> bool { self.force_exclude_flag(db) } @@ -1281,6 +1281,43 @@ mod tests { Ok(()) } + /// A diagnostic names its types in the syntax of the file it is reported in, and + /// that has to hold however the inference behind it was reached. Checking the `.py` + /// importer first forces the `.by` module's inference, and the message salsa caches + /// there is the one every later reader of that file sees. + #[test] + fn basedpython_diagnostic_keeps_its_spelling_when_a_python_importer_is_checked_first() { + let root = SystemPathBuf::from("/project"); + let project = ProjectMetadata::new("test", root.clone()); + let mut db = TestDb::new(project); + + db.write_files([ + ( + root.join("lib.by"), + "def g():\n c: \"c\" = \"b\"\n return c\n", + ), + (root.join("main.py"), "from lib import g\n\ng()\n"), + ]) + .unwrap(); + + let main = system_path_to_file(&db, root.join("main.py")).unwrap(); + let lib = system_path_to_file(&db, root.join("lib.by")).unwrap(); + + // inferring `g`'s return type infers its body, diagnostics and all + check_file_impl(&db, db.program_file(main)).unwrap(); + + let messages: Vec = check_file_impl(&db, db.program_file(lib)) + .unwrap() + .iter() + .map(|diagnostic| diagnostic.concise_message().to_string()) + .collect(); + + assert_eq!( + messages, + vec![r#"Object of type `"b"` is not assignable to `"c"`"#.to_string()] + ); + } + #[test] fn explicit_nested_included_file_is_a_literal_match() { let root = SystemPathBuf::from("/project"); diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md b/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md index ae3affd265..712801c996 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_enums.md @@ -596,34 +596,33 @@ c: Color.RED = Color.RED reveal_type(c) # revealed: Color.RED ``` -## `is` / `is not` between members keeps identity at runtime +## `is` / `is not` against a member is identity at runtime -a payload-less variant is a singleton *instance*, not a class, so the `is`/`is not` keyword pair -keeps python identity semantics for it — the `isinstance` lowering only fires when the rhs resolves -to a variant *class*. this block is checker-clean, so the divergence harness executes it and pins -the runtime contract +a payload-less variant names the type holding exactly one object, so the test for it is identity — +and a payload-less variant is a singleton *instance*, so identity is also all the runtime has to +compare. a payload variant names a *class* instead, and its test is the `isinstance` that asks. this +block is checker-clean, so the divergence harness executes it and pins the runtime contract + +the values come from a call rather than being written inline, so the tests are not settled +statically — a settled one would fold to a constant and stop exercising the lowering ```by enum class Genre: case A, B -assert Genre.A is Genre.A -assert Genre.A is not Genre.B +def pick() -> Genre: + return Genre.A -g: Genre = Genre.A -assert g is Genre.A -assert g is not Genre.B +assert pick() is Genre.A +assert pick() is not Genre.B enum class Shape: case Circle(radius: float) case Point -assert Shape.Point is Shape.Point -p = Shape.Point -assert p is Shape.Point +def shape() -> Shape: + return Shape.Circle(1.0) -# a payload variant is a class, so the rhs of `is` lowers to `isinstance` -c = Shape.Circle(1.0) -assert c is Shape.Circle -assert c is not Shape.Point +assert shape() is Shape.Circle +assert shape() is not Shape.Point ``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md index 2a41101fa4..162812015c 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_escaping_local.md @@ -262,7 +262,7 @@ def f(fn: (local Resource) -> None): f: tmp = it - print(tmp is None) + print(tmp === None) ``` ## a `once` block's fresh binding still escapes diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md b/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md index cb286776b7..b14f1a8a1f 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_identity_narrow.md @@ -1,23 +1,23 @@ -# basedpython: `is` / `is not` keyword narrowing +# basedpython: `is` / `is not` narrowing -In basedpython, the `is` and `is not` keyword pair perform instance checks (they transpile to -`isinstance(...)` / `not isinstance(...)`). The `===` and `!==` operators retain Python's identity -comparison semantics. Narrowing in `.by` files mirrors this swap. +In basedpython the `is` and `is not` keyword pair is a *type test*: its right-hand side is a type +expression, and the test asks whether the value has that type. The `===` and `!==` operators keep +Python's identity comparison. Narrowing mirrors that split. -## `is not` narrows to negation of the instance type +## `is` narrows to the type named ```by def f(a: object): - if a is not int: - reveal_type(a) # revealed: not int + if a is int: + reveal_type(a) # revealed: int ``` -## `is` narrows to the instance type +## `is not` narrows to the negation of it ```by def f(a: object): - if a is int: - reveal_type(a) # revealed: int + if a is not int: + reveal_type(a) # revealed: not int ``` ## `!==` keeps Python identity semantics @@ -36,11 +36,11 @@ def f(a: int | None): reveal_type(a) # revealed: None ``` -## `is` with literal RHS keeps Python identity semantics +## A literal names the type holding exactly that value -`isinstance(x, None)` is invalid at runtime, so `is`/`is not` against literal singletons (`None`, -`True`/`False`, numbers, strings, bytes, `...`) must transpile as Python `is`/`is not` rather than -`isinstance`. +`None`, `True`/`False` and a number are all type expressions, so a test against one narrows to the +literal type it names. The runtime check that comes out is the equality — or, for `None`, the +identity — that decides membership of that type. ```by def f(a: int | None): @@ -59,16 +59,17 @@ def f(a: bool | int): ``` ```by -def f(a: int | None): - if a is ...: - reveal_type(a) # revealed: Never +def f(a: int | str): + if a is 1: + reveal_type(a) # revealed: 1 + if a is "x": + reveal_type(a) # revealed: "x" ``` -## `is` with an enum member RHS keeps Python identity semantics +## An enum member names the type holding exactly that member -An enum member is a singleton *instance*, not a class — `isinstance(x, Color.RED)` would be a -runtime `TypeError` — so `is`/`is not` against a member keeps Python identity semantics and narrows -by identity, the same as literal singletons. +An enum member is a singleton, so `Literal[Color.RED]` holds one object and the test for it is +identity — which is also what the runtime compares. ```by import enum @@ -97,11 +98,11 @@ def g(x: Genre): reveal_type(x) # revealed: Literal[Genre.B] ``` -## An instance check yields `bool`, never an identity fold +## An undecidable test is `bool` -The keyword form is an instance check, so Python's identity folds (an instance is never identical to -a class object, so plain Python would type `x is int` as `Literal[False]`) must not apply — -otherwise everything after `assert x is int` would be unreachable. +The identity folds Python applies to the same operator have no place here: the right-hand side names +a type rather than the class object the same source spells as a value, so `x is int` is not "an +instance compared to a class" and must not collapse to `Literal[False]`. ```by def f(x: object): @@ -111,10 +112,21 @@ def f(x: object): reveal_type(x) # revealed: int ``` +## A test the types settle is its answer + +Where the value's type decides the question, the test *is* that answer — which is what lets a reader +see that the branch it guards is already decided. + +```by +def f(x: int): + reveal_type(x is int) # revealed: True + reveal_type(x is not int) # revealed: False +``` + ## A test against a disjoint type is reported -An instance check whose value can never have the tested type is a constant: `is` never holds and -`is not` always does. Either the guarded branch is dead or the wrong type was named. +A test whose value can never have the type named is a constant: `is` never holds and `is not` always +does. Either the guarded branch is dead or the wrong type was named. ```by def f(x: None): @@ -189,11 +201,30 @@ def g(x: int): ## Identity comparisons are left alone -The `===` operators and the literal/enum-member forms keep Python identity semantics, where an -always-`False` comparison is already typed `Literal[False]`. +`===` and `!==` keep Python identity semantics, where an always-`False` comparison is already typed +`Literal[False]`. ```by def f(x: None): b = x === 1 reveal_type(b) # revealed: False ``` + +## A chained type test is rejected + +Python chains `a is int is str` into `a is int and int is str`, whose second half asks whether the +*class* `int` has the type `str`. That is never what the writer meant, so the chain is refused +rather than given a meaning. + +```by +def f(a: object): + # error: [invalid-syntax] "`is` type test cannot be chained with another comparison; split it into separate tests joined with `and`" + b = a is int is str +``` + +A chain of identity comparisons is ordinary Python and stays legal. + +```by +def f(a: object, b: object): + c = a === b !== None +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md b/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md index 0d4f6d2b4d..40ea39566f 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_inferred_narrowing.md @@ -384,8 +384,8 @@ def f(x: str): ## a parameter the body puts something else in is not a guard A guard names the argument a call passed, so it says nothing once the body puts something else where -that argument was. `rebound` hands back `True` whatever it is given, and reading that as a claim -about the argument would narrow it to `Never`. +that argument was. `rebound` hands back `True` whatever it is given — the test is settled by the `1` +the body assigned — and reading that as a claim about the argument would narrow it to `Never`. ```by def rebound(a: object): @@ -393,7 +393,7 @@ def rebound(a: object): return a is int def f(x: str): - reveal_type(rebound) # revealed: def rebound(a: object) -> bool + reveal_type(rebound) # revealed: def rebound(a: object) -> True if rebound(x): reveal_type(x) # revealed: str ``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md b/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md index 0123f058cf..f49c972148 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_intersection.md @@ -55,3 +55,49 @@ class C: ... def f(x: A & B & C) -> None: reveal_type(x) # revealed: A & B & C ``` + +## intersection in a type spelled through a call + +A few typing constructs name a type in a call argument rather than after a `:` — `NewType` names its +base, `TypeVar` names its bound and its constraints, and the functional `NamedTuple` and `TypedDict` +name their field types inside a literal. Those arguments are type expressions, so `&` means the same +thing in one as it does in an annotation, and the transpiler has to lower it there too: an `A & B` +that survived into the emitted python would be a runtime `A.__and__(B)`. + +```by +import typing + +class A: ... + +class B: ... + +T = typing.TypeVar("T", bound=A & B) +U = typing.TypeVar("U", A & B, int) +Pair = typing.NamedTuple("Pair", [("left", A & B), ("right", int)]) +Row = typing.TypedDict("Row", {"cell": A & B}) + +def f(x: Pair, y: Row) -> None: + reveal_type(x.left) # revealed: A & B + reveal_type(y["cell"]) # revealed: A & B +``` + +## an ordinary call is not one of those constructs + +Only a call that resolves to one of those constructs reads its arguments as types. Anywhere else +`and` is python's boolean operator, and the value it produces is the one the call receives. + +```by +import typing + +class A: ... + +class B(A): ... + +Both = typing.NewType("Both", B) + +def pick(value: object) -> object: + return value + +chosen = pick(A and B) +reveal_type(chosen) # revealed: object +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md b/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md index ed0ad3d5b9..a39a7731cc 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_parametric_is.md @@ -384,7 +384,7 @@ class C: a: bool def f(c: C) -> bool: - reveal_type(c is HasA[bool]) # revealed: bool + reveal_type(c is HasA[bool]) # revealed: True return c is HasA[bool] ``` @@ -840,3 +840,205 @@ def outer(data: list[int] | list[str]) -> str: print(outer(list[int]())) # ints print(outer(list[str]())) # strs ``` + +## the target is a type expression + +The right-hand side of a type test is a type expression, so it names its target the way an +annotation does — and an unusable target is reported the same way too. + +```by +import os + +def fn() -> None: ... + +def f(v: object) -> None: + # error: [invalid-type-form] "Module `os` is not valid in a type expression" + print(v is os) + # error: [invalid-type-form] "Function `fn` is not valid in a type expression" + print(v is fn) +``` + +A bare generic class is complete without arguments: the test asks whether the value is one, and the +arguments are what the runtime cannot see either way. + +```by +def f(v: object) -> None: + print(v is list) + print(v is dict) +``` + +A tuple names the tuple *type*, not the "any of these" that `isinstance` spells with one. Write a +union for that. + +```by +def f(v: int) -> None: + # error: [non-overlapping-type-test] "`int` and `(int, str)` are non-overlapping types, so this test is always `False`" + print(v is (int, str)) + print(v is int | str) +``` + +## a target with no runtime form is rejected + +A test must *earn* its `True` — it narrows — so a target the runtime can only partly check is +rejected rather than approximated. `Any` admits every value, and a test that can only answer `True` +narrows nothing. + +```by +from typing import Any + +def f(v: object) -> None: + # error: [erased-type-check] "`is Any` cannot be checked at runtime: `Any` admits every value, so there is nothing for the test to look for" + print(v is Any) +``` + +A callable type's parameter and return types are not recorded on the value, so a runtime check would +assume the signature rather than test it. A bare `Callable` asks only what the runtime does record, +and is the test `callable()` performs. + +```by +from typing import Callable + +def f(v: object) -> None: + # error: [erased-type-check] "`is Callable[[], int]` cannot be checked at runtime: a callable's parameter and return types are not recorded on the value" + print(v is Callable[[], int]) + print(v is Callable) +``` + +A `float` or `complex` literal names a type equality cannot decide — `0.0 == -0.0` and `nan != nan` +— and `LiteralString` is a property of how a value was written rather than of the value. + +```by +from typing import LiteralString + +def f(v: object) -> None: + # error: [erased-type-check] "`is 1.5` cannot be checked at runtime: equality does not decide membership of `1.5`" + print(v is 1.5) + # error: [erased-type-check] "`is LiteralString` cannot be checked at runtime: equality does not decide membership of `LiteralString`" + print(v is LiteralString) +``` + +An intersection has no single runtime form, and `issubclass` cannot be asked what a `type[Any]` is +parameterized by. + +```by +from typing import Any + +class A: ... + +class B: ... + +def f(v: object) -> None: + # error: [erased-type-check] "`is A & B` cannot be checked at runtime: an intersection has no single runtime form" + print(v is A & B) + # error: [erased-type-check] "`is type[Any]` cannot be checked at runtime: `issubclass` cannot take what `type[Any]` is parameterized by" + print(v is type[Any]) +``` + +A `TypedDict`'s instances are plain dicts, so the test could only ask whether the value is a `dict` +and assume every key. + +```by +from typing import TypedDict + +class Movie(TypedDict): + title: str + +def f(v: dict[str, str]) -> None: + # error: [erased-type-check] "`is Movie` cannot be checked at runtime: a `TypedDict`'s instances are plain dicts" + print(v is Movie) +``` + +A protocol whose members all have a runtime spelling is checked against the value's own reified +annotations, member by member — `isinstance` could not, since a conforming class need not be a +subclass. One with a member the emitted python cannot name leaves nothing to check against. + +```by +from typing import Callable, Protocol + +class Handler(Protocol): + on_event: Callable[[], int] + +def f(v: object) -> None: + # error: [erased-type-check] "`is Handler` cannot be checked at runtime: `Handler` has a member with no runtime spelling" + print(v is Handler) +``` + +## a `@runtime_checkable` protocol target is an ordinary instance test + +Decorating the protocol is the author's own statement that `isinstance` may take it, so the test is +the check python itself performs — that the members are present. + +```by +from typing import Protocol, runtime_checkable + +@runtime_checkable +class HasName(Protocol): + name: str + +def f(v: object) -> None: + print(v is HasName) +``` + +## a literal target tests the value, not its class + +A literal names the type holding exactly the values equal to it, so the runtime check is that +equality — with the class pinned, because python's `1 == True` would otherwise let a `bool` satisfy +`Literal[1]`. + +```by +from typing import Literal + +def f(v: object) -> None: + print(v is Literal[1]) + print(v is Literal["a", "b"]) +``` + +An alias to a union of literals is the same test, arm by arm — the source spells the union with one +word, and `isinstance` could take neither that word nor the literals it stands for. + +```by +from typing import Literal + +type Small = Literal[1, 2] + +def f(v: object) -> None: + print(v is Small) +``` + +## a `type[C]` target tests the class + +`type[C]` is one of the few parameterized types the runtime can check in full: the value must be a +class, and a subclass of `C`. + +```by +class A: ... + +def f(v: object) -> None: + print(v is type[A]) +``` + +## a template literal type tests the string + +A template literal type is the set of strings its pattern produces, and a value is one of them or it +is not — so a test against it is decided by the pattern. + +```by +def f(s: str) -> None: + print(s is f"item-{int}") + +reveal_type("item-12" is f"item-{int}") # revealed: True +# error: [non-overlapping-type-test] "`"item-ab"` and `f"item-{int}"` are non-overlapping types, so this test is always `False`" +reveal_type("item-ab" is f"item-{int}") # revealed: False +``` + +## a union target the source spells with one word + +A PEP 695 alias names a whole union with a single identifier, which `isinstance` cannot take — the +test is the disjunction of the arms all the same. + +```by +type Key = int | str + +def f(v: object) -> None: + print(v is Key) +``` diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index a7ad4c4a4b..d012b52aeb 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -46,9 +46,7 @@ pub use ty_site_packages::{ SitePackagesDiscoveryError, SitePackagesPaths, SysPrefixPathOrigin, }; pub use types::conformance::declares_conformances; -pub use types::conformance::{ - ConformanceRegistration, ConformanceTest, WitnessDispatch, WitnessKind, -}; +pub use types::conformance::{ConformanceRegistration, WitnessDispatch, WitnessKind}; pub use types::conversions::{ ConversionImport, ConversionInfo, ConversionRuntime, DISCARD_ADAPTER, }; @@ -64,6 +62,7 @@ pub use types::ide_support::{ pub use types::implicit_names::implicit_names; pub use types::reified_infer::{ ArgVariance, ErasedTargetReason, ErasedUnion, ParametricIsPlan, ProtocolMemberCheck, + TargetSpelling, }; pub use types::static_resource::{ResourceError, render_as, resolve_static_resource}; pub use types::template::finite_string_set; @@ -481,11 +480,7 @@ pub fn check_file_with( /// hint — should go through this, or it will spell types in a syntax the file /// cannot be written in. pub fn with_display_for_file(db: &dyn Db, file: File, body: impl FnOnce() -> R) -> R { - if file.source_type(db).is_basedpython() { - crate::types::display::with_basedpython_display(body) - } else { - body() - } + crate::types::display::with_basedpython_display(file.source_type(db).is_basedpython(), body) } fn check_file_inner( diff --git a/crates/ty_python_semantic/src/reified.rs b/crates/ty_python_semantic/src/reified.rs index 0af235c034..00ec3545c0 100644 --- a/crates/ty_python_semantic/src/reified.rs +++ b/crates/ty_python_semantic/src/reified.rs @@ -17,7 +17,7 @@ use ruff_python_ast::name::Name; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{self as ast, CmpOp, Expr, PySourceType, Stmt}; +use ruff_python_ast::{self as ast, Expr, PySourceType, Stmt}; use ruff_text_size::{Ranged, TextRange}; use rustc_hash::{FxHashMap, FxHashSet}; @@ -30,7 +30,6 @@ use rustc_hash::{FxHashMap, FxHashSet}; /// `ParamSpec`, a parameter list with no runtime object to bind — so /// `source_type` decides whether it is a candidate pub fn reified_type_param_names( - source: &str, source_type: PySourceType, function: &ast::StmtFunctionDef, ) -> Vec { @@ -68,7 +67,7 @@ pub fn reified_type_param_names( shadow_bound_names(&function.body, &mut active); let param_typevars = param_annotation_typevars(&function.parameters, &active); let mut finder = ValueUseFinder { - source, + source_type, active, param_typevars, found: Vec::new(), @@ -147,11 +146,10 @@ pub enum UnansweredReason { /// a `**Kwargs` keyword pack is never a candidate. a class writes its /// specialization as a subscript, and a subscript takes no keyword arguments, /// so there is no way to supply one -pub fn reified_class_reads<'ast>( - source: &str, +pub fn reified_class_reads( source_type: PySourceType, - class: &'ast ast::StmtClassDef, -) -> ReifiedClassReads<'ast> { + class: &ast::StmtClassDef, +) -> ReifiedClassReads<'_> { let Some(type_params) = class.type_params.as_deref() else { return ReifiedClassReads::default(); }; @@ -199,7 +197,7 @@ pub fn reified_class_reads<'ast>( // wherever the body writes the `def` — guarded by a version check, say — // while a read in the class body itself, in a method's header, or inside a // class nested in the class body has no method around it - let mut finder = ValueUseFinder::new(source, active); + let mut finder = ValueUseFinder::new(source_type, active); for stmt in &class.body { finder.visit_stmt(stmt); } @@ -347,18 +345,16 @@ fn body_span(function: &ast::StmtFunctionDef) -> Option { /// names of the class's type parameters that are reified, in declaration order pub(crate) fn reified_class_type_param_names( - source: &str, source_type: PySourceType, class: &ast::StmtClassDef, ) -> Vec { - reified_class_reads(source, source_type, class).names + reified_class_reads(source_type, class).names } /// names of the class's type parameters that are reified *only* because the /// class reads them in a value position, in declaration order. this is what an /// editor hints, where the keyword would be written pub fn inferred_reified_class_type_param_names( - source: &str, source_type: PySourceType, class: &ast::StmtClassDef, ) -> Vec { @@ -370,7 +366,7 @@ pub fn inferred_reified_class_type_param_names( .filter(|param| param.is_reified()) .map(|param| param.name().id.as_str()) .collect(); - let mut names = reified_class_type_param_names(source, source_type, class); + let mut names = reified_class_type_param_names(source_type, class); names.retain(|name| !declared.contains(name.as_str())); names } @@ -419,7 +415,6 @@ fn own_bindings(function: &ast::StmtFunctionDef) -> Vec<&str> { /// [`reified_type_param_names`] finds that does not already say so itself. /// this is what an editor hints, where the keyword would be written pub fn inferred_reified_type_param_names( - source: &str, source_type: PySourceType, function: &ast::StmtFunctionDef, ) -> Vec { @@ -431,24 +426,11 @@ pub fn inferred_reified_type_param_names( .filter(|param| param.is_reified()) .map(|param| param.name().id.as_str()) .collect(); - let mut names = reified_type_param_names(source, source_type, function); + let mut names = reified_type_param_names(source_type, function); names.retain(|name| !declared.contains(name.as_str())); names } -/// whether the `is` / `is not` between two compare operands is the keyword -/// form (isinstance semantics) rather than the `===` / `!==` identity -/// operators, which the parser flattens to the same ast -pub fn is_keyword_comparison(source: &str, op: CmpOp, lhs: &Expr, rhs: &Expr) -> bool { - let between = &source[usize::from(lhs.range().end())..usize::from(rhs.range().start())]; - let trimmed = between.trim(); - match op { - CmpOp::Is => trimmed == "is", - CmpOp::IsNot => !trimmed.starts_with("!=="), - _ => false, - } -} - /// parameter name → the still-active type-param names its annotation /// mentions. a parametric `is` test on such a parameter lowers to an equality /// check of those params' reified cells, so the test is a value-position use @@ -557,7 +539,7 @@ impl<'a> Visitor<'a> for StoredNames<'a> { } struct ValueUseFinder<'a> { - source: &'a str, + source_type: PySourceType, active: FxHashSet<&'a str>, /// parameters of the innermost enclosing def whose annotations mention /// active type params — parametric `is` tests on them reify those params @@ -570,9 +552,9 @@ struct ValueUseFinder<'a> { impl<'a> ValueUseFinder<'a> { /// a finder for a region that binds no parameters of its own, so no /// annotation can carry a parametric type test into it - fn new(source: &'a str, active: FxHashSet<&'a str>) -> Self { + fn new(source_type: PySourceType, active: FxHashSet<&'a str>) -> Self { Self { - source, + source_type, active, param_typevars: FxHashMap::default(), found: Vec::new(), @@ -602,11 +584,8 @@ impl<'a> ValueUseFinder<'a> { /// reified cell against the target's type arguments fn check_parametric_tests(&mut self, compare: &'a ast::ExprCompare) { let mut lhs: &Expr = &compare.left; - for (op, rhs) in compare.ops.iter().zip(&compare.comparators) { - if matches!(op, CmpOp::Is | CmpOp::IsNot) - && matches!(rhs, Expr::Subscript(_)) - && is_keyword_comparison(self.source, *op, lhs, rhs) - { + for (index, rhs) in compare.comparators.iter().enumerate() { + if matches!(rhs, Expr::Subscript(_)) && compare.is_type_test(index, self.source_type) { self.reify_tested_param(lhs); } lhs = rhs; diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 07c8e61d07..814507a729 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -348,35 +348,6 @@ impl<'db> SemanticModel<'db> { }) } - /// basedpython: how `x is ` is answered once conformances are in - /// play. `None` when the target is not an interface anything conforms to - /// here, where the ordinary `isinstance` lowering is still right - pub fn conformance_test( - &self, - target: &ast::Expr, - ) -> Option { - use crate::types::conformance; - - let db = self.db; - if !self.file.file(db).source_type(db).is_basedpython() { - return None; - } - let interface = target.inferred_type(self)?.to_class_type(db)?; - if !conformance::visible_conformances(db, self.file.file(db)) - .iter() - .any(|(_, declared)| declared.class_literal(db) == interface.class_literal(db)) - { - return None; - } - let members = Some( - conformance::interface_requirements(db, interface) - .iter() - .map(ToString::to_string) - .collect(), - ); - Some(conformance::ConformanceTest { members }) - } - /// basedpython: when an attribute access resolves to an `extension` /// member (this module's, or one from a module imported with a plain /// `import mod`), the backing-function rewrite the transpiler applies. @@ -1003,57 +974,72 @@ impl<'db> SemanticModel<'db> { .is_some_and(|ty| crate::types::trailing_lambda::callee_callback_is_once(self.db, ty)) } - /// basedpython: how the parametric type test `lhs is rhs` (keyword form) - /// resolves, from the operands' inferred types. `rhs` may name the target - /// specialization directly (`list[int]`) or through an alias — an implicit - /// alias whose value is a specialization (`X = list[int]`) or a PEP 695 - /// `type` alias. `None` when `rhs` does not resolve to a specialization — - /// the test is then an ordinary isinstance lowering - pub fn parametric_is_plan( + /// basedpython: whether `target`, read as a *value*, denotes a plain value + /// rather than a class — an enum member, a literal, an instance of a + /// concrete non-type class. + /// + /// A type test's target is a type expression, so its inferred type is the + /// type it names and the value it evaluates to is not recorded. That is + /// almost always the same thing, and where it is not, the type reading is + /// the one that matters. The exception is a target the type reading cannot + /// make sense of at all: the enum lowering rewrites a unit variant into a + /// singleton instance before the transpiler's passes run, so `s is + /// Shape.Point` reaches them naming a type in the source and a value in + /// what is emitted — and the test for a value is identity. + /// + /// Resolved here rather than read from the inference store, which holds the + /// type reading. Only a name or a dotted name is resolved; anything else + /// answers `false`, which leaves the ordinary instance check in place. + pub fn denotes_plain_value(&self, target: &ast::Expr) -> bool { + let env = self.program_environment(); + self.value_of(target, &env) + .is_some_and(|ty| crate::types::basedpython_is_plain_value(self.db, &env, ty)) + } + + /// the type of the value `expr` evaluates to, for a name or a dotted name + fn value_of( &self, - lhs: &ast::Expr, - rhs: &ast::Expr, - ) -> Option { - let env = &self.program_environment(); - let alias = crate::types::reified_infer::parametric_is_target( - self.db, - env, - rhs.inferred_type(self)?, - )?; - let lhs_ty = lhs.inferred_type(self)?; - Some(crate::types::reified_infer::classify_parametric_is( - self.db, - env, - self.file(), - lhs_ty, - alias, - rhs, - )) + expr: &ast::Expr, + env: &crate::types::context::ProgramEnvironment<'db>, + ) -> Option> { + match expr { + ast::Expr::Name(name) => crate::place::global_symbol( + self.db, + self.db.program_file(self.file()), + name.id.as_str(), + ) + .place + .ignore_possibly_undefined(), + ast::Expr::Attribute(attribute) => { + let base = self.value_of(&attribute.value, env)?; + base.member(self.db, env, attribute.attr.id.as_str()) + .place + .ignore_possibly_undefined() + } + _ => None, + } } - /// basedpython: [`Self::parametric_is_plan`] for a checked cast - /// (`value cast T`). The same classification engine decides both — only the - /// target's inference position differs, since a cast's target is a *type* - /// expression while an `is`-rhs is a value expression. - pub fn parametric_cast_plan( + /// basedpython: how the type test `lhs is rhs` resolves, from the value's + /// inferred type and the type its right-hand side names. + /// + /// The right-hand side is a type expression, so `rhs` may name its target + /// however a type expression can: directly (`list[int]`), through an + /// implicit alias (`X = list[int]`), or through a PEP 695 `type` alias. + /// `None` only when inference has no type for one of the operands. + pub fn parametric_is_plan( &self, - value: &ast::Expr, - target: &ast::Expr, + lhs: &ast::Expr, + rhs: &ast::Expr, ) -> Option { let env = &self.program_environment(); - let alias = crate::types::reified_infer::parametric_cast_target( - self.db, - env, - target.inferred_type(self)?, - )?; - let value_ty = value.inferred_type(self)?; - Some(crate::types::reified_infer::classify_parametric_is( + Some(crate::types::reified_infer::type_test_plan( self.db, env, self.file(), - value_ty, - alias, - target, + lhs.inferred_type(self)?, + rhs.inferred_type(self)?, + Some(rhs), )) } diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 40c4f3bf42..d7ebc881e9 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -94,7 +94,7 @@ use crate::types::diagnostic::{ report_bad_dunder_get_call, report_bad_import_call, }; pub use crate::types::display::{DisplaySettings, SourceSpelling, TypeDetail, TypeDisplayDetails}; -pub use crate::types::enums::basedpython_is_keeps_identity; +pub(crate) use crate::types::enums::basedpython_is_plain_value; pub(crate) use crate::types::enums::{EnumClassLiteral, EnumComplementType, enum_metadata}; pub(crate) use crate::types::equality::{ComparisonSoundnessPolicy, equality_truthiness}; use crate::types::function::{ @@ -173,6 +173,7 @@ mod bool; mod bound_super; mod build_stamps; mod call; +pub mod call_type_forms; mod callable; pub mod character; mod class; diff --git a/crates/ty_python_semantic/src/types/call_type_forms.rs b/crates/ty_python_semantic/src/types/call_type_forms.rs new file mode 100644 index 0000000000..1ca77a7640 --- /dev/null +++ b/crates/ty_python_semantic/src/types/call_type_forms.rs @@ -0,0 +1,239 @@ +//! The typing constructs that spell a type through a *call* rather than through an +//! annotation. +//! +//! Most type expressions sit somewhere a reader can point at syntactically — after a +//! `:`, after a `->`, on the right of a `type` alias. These do not: `NewType("D", int)` +//! names its base in an argument, `TypeVar("T", bound=int)` names its bound in a +//! keyword, and the functional `NamedTuple` and `TypedDict` name their field types +//! inside a list or dict literal. Nothing about the call syntax says so — the answer +//! comes from what the callee resolves to. +//! +//! Two consumers need that answer. Type inference needs it to check the arguments as +//! type expressions, and the basedpython transpiler needs it to lower the surface syntax +//! written in them: `TypeVar("T", bound=A & B)` has to reach the emitted python as +//! `bound=Intersection[A, B]`, exactly as `x: A & B` does. Stating it once, here, is +//! what keeps a form the type checker accepts from being one the transpiler leaves +//! behind — which is a silent miscompilation, because the unlowered expression is +//! perfectly good python that means something else at runtime. + +use ruff_python_ast as ast; + +use crate::Db; +use crate::types::{KnownClass, SpecialFormType, Type, TypingModule}; + +/// A typing construct whose call arguments hold type expressions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CallTypeForm { + /// `NewType("D", int)` — the second argument is the base type + NewType, + /// `TypeAliasType("X", int | str)` — the second argument is the alias value + TypeAliasType, + /// `NamedTuple("NT", [("f", int)])` — the second argument holds `(name, type)` pairs + NamedTuple, + /// `TypedDict("TD", {"f": int})` — the second argument holds `key: type` items + TypedDict, + /// `TypeVar("T", int, str, bound=…, default=…)` — the constraints, the bound and the + /// default are all type expressions + TypeVar, + /// `ParamSpec("P", default=[int, str])` — the default is a parameter list + ParamSpec, + /// `TypeVarTuple("Ts", default=Unpack[tuple[int, ...]])` — the default is a type + TypeVarTuple, +} + +impl CallTypeForm { + /// Which construct, if any, `callee` is. `callee` is the inferred type of the + /// expression in call position, so an unresolved or shadowed name answers `None` + /// rather than being matched by spelling. + pub fn of<'db>(db: &'db dyn Db, callee: Type<'db>) -> Option { + if callee == Type::SpecialForm(SpecialFormType::NamedTuple) { + return Some(Self::NamedTuple); + } + if TypingModule::from_typed_dict_type(db, callee).is_some() { + return Some(Self::TypedDict); + } + match callee.as_class_literal()?.known(db)? { + KnownClass::NewType => Some(Self::NewType), + KnownClass::TypeVar | KnownClass::ExtensionsTypeVar => Some(Self::TypeVar), + KnownClass::ParamSpec | KnownClass::ExtensionsParamSpec => Some(Self::ParamSpec), + KnownClass::TypeVarTuple | KnownClass::ExtensionsTypeVarTuple => { + Some(Self::TypeVarTuple) + } + known_class if TypingModule::from_type_alias_class(known_class).is_some() => { + Some(Self::TypeAliasType) + } + _ => None, + } + } + + /// The sub-expressions of `arguments` that this form reads as type expressions. + /// + /// Each one is a complete type expression, so a caller that walks type expressions + /// can hand them to the same traversal it uses for an annotation. Arguments a form + /// reads as ordinary values — every form's leading name, a `TypedDict`'s field keys, + /// `TypeAliasType`'s `type_params` — are not returned. + pub fn type_expressions(self, arguments: &ast::Arguments) -> Vec<&ast::Expr> { + let mut found = Vec::new(); + match self { + Self::NewType | Self::TypeAliasType => found.extend(arguments.args.get(1)), + Self::NamedTuple => { + // `[("f", int), ("g", str)]`, or the same as a tuple, each field written + // as either a list or a tuple of its name and its type + if let Some(fields) = arguments.args.get(1) { + for field in sequence_elements(fields).into_iter().flatten() { + found.extend(sequence_elements(field).and_then(|pair| pair.get(1))); + } + } + } + Self::TypedDict => { + if let Some(ast::Expr::Dict(fields)) = arguments.args.get(1) { + found.extend( + fields + .items + .iter() + .filter(|item| item.key.is_some()) + .map(|item| &item.value), + ); + } + found.extend( + arguments + .find_keyword("extra_items") + .map(|keyword| &keyword.value), + ); + } + Self::TypeVar => { + // every positional argument after the name is a constraint + found.extend(arguments.args.iter().skip(1)); + found.extend(arguments.find_keyword("bound").map(|kw| &kw.value)); + found.extend(arguments.find_keyword("default").map(|kw| &kw.value)); + } + Self::TypeVarTuple => { + found.extend(arguments.find_keyword("default").map(|kw| &kw.value)); + } + Self::ParamSpec => { + // a `ParamSpec` default is a parameter list, `[int, str]`, whose elements + // are the type expressions — or `...`, which holds none + if let Some(default) = arguments.find_keyword("default") { + match &default.value { + ast::Expr::List(list) => found.extend(&list.elts), + value => found.push(value), + } + } + } + } + found + } +} + +/// The elements of a list or tuple literal, or `None` for anything else. +fn sequence_elements(expr: &ast::Expr) -> Option<&[ast::Expr]> { + match expr { + ast::Expr::List(list) => Some(&list.elts), + ast::Expr::Tuple(tuple) => Some(&tuple.elts), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ruff_python_parser::parse_expression; + use ruff_text_size::Ranged; + + /// The source text of each argument `form` reads as a type expression in `call`. + fn type_expressions(form: CallTypeForm, call: &str) -> Vec { + let parsed = parse_expression(call).expect("a call expression"); + let ast::Expr::Call(call_expr) = parsed.expr() else { + panic!("expected a call expression"); + }; + form.type_expressions(&call_expr.arguments) + .into_iter() + .map(|expr| call[expr.range()].to_string()) + .collect() + } + + #[test] + fn new_type_reads_its_base() { + assert_eq!( + type_expressions(CallTypeForm::NewType, r#"NewType("D", int | str)"#), + ["int | str"] + ); + } + + #[test] + fn type_alias_type_reads_its_value_but_not_its_type_params() { + assert_eq!( + type_expressions( + CallTypeForm::TypeAliasType, + r#"TypeAliasType("X", list[T], type_params=(T,))"# + ), + ["list[T]"] + ); + } + + #[test] + fn type_var_reads_its_constraints_bound_and_default() { + assert_eq!( + type_expressions( + CallTypeForm::TypeVar, + r#"TypeVar("T", int, str, bound=object, default=int, covariant=True)"# + ), + ["int", "str", "object", "int"] + ); + } + + #[test] + fn param_spec_reads_the_elements_of_its_default() { + assert_eq!( + type_expressions( + CallTypeForm::ParamSpec, + r#"ParamSpec("P", default=[int, str])"# + ), + ["int", "str"] + ); + assert_eq!( + type_expressions(CallTypeForm::ParamSpec, r#"ParamSpec("P", default=...)"#), + ["..."] + ); + } + + #[test] + fn type_var_tuple_reads_its_default() { + assert_eq!( + type_expressions( + CallTypeForm::TypeVarTuple, + r#"TypeVarTuple("Ts", default=Unpack[tuple[int, ...]])"# + ), + ["Unpack[tuple[int, ...]]"] + ); + } + + #[test] + fn named_tuple_reads_each_field_type_but_not_its_name() { + assert_eq!( + type_expressions( + CallTypeForm::NamedTuple, + r#"NamedTuple("NT", [("f", int), ["g", str]])"# + ), + ["int", "str"] + ); + } + + #[test] + fn named_tuple_reads_nothing_from_a_field_list_it_cannot_see_into() { + assert!( + type_expressions(CallTypeForm::NamedTuple, r#"NamedTuple("NT", fields)"#).is_empty() + ); + } + + #[test] + fn typed_dict_reads_each_field_type_and_extra_items() { + assert_eq!( + type_expressions( + CallTypeForm::TypedDict, + r#"TypedDict("TD", {"f": int, **rest}, total=False, extra_items=str)"# + ), + ["int", "str"] + ); + } +} diff --git a/crates/ty_python_semantic/src/types/class/static_literal.rs b/crates/ty_python_semantic/src/types/class/static_literal.rs index 317db9c390..7f8b9841e5 100644 --- a/crates/ty_python_semantic/src/types/class/static_literal.rs +++ b/crates/ty_python_semantic/src/types/class/static_literal.rs @@ -664,13 +664,8 @@ impl<'db> StaticClassLiteral<'db> { return Box::default(); } let module = parsed_module(db, self.python_file(db)).load(db); - let source = ruff_db::source::source_text(db, file); - crate::reified::reified_class_type_param_names( - source.as_str(), - source_type, - self.node(db, &module), - ) - .into_boxed_slice() + crate::reified::reified_class_type_param_names(source_type, self.node(db, &module)) + .into_boxed_slice() } /// basedpython: whether this class or anything it inherits from reifies a diff --git a/crates/ty_python_semantic/src/types/conformance.rs b/crates/ty_python_semantic/src/types/conformance.rs index 9799d1e6ac..5e178d29d3 100644 --- a/crates/ty_python_semantic/src/types/conformance.rs +++ b/crates/ty_python_semantic/src/types/conformance.rs @@ -398,16 +398,6 @@ pub enum WitnessKind { Property, } -/// how `x is ` is answered at runtime once conformances are in play -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ConformanceTest { - /// the requirement names, for a protocol target: with no registered - /// conformance for the value's class, answering the test means checking that - /// the value carries them. `None` for an abstract class, where `isinstance` - /// is already the whole answer - pub members: Option>, -} - /// declaration-site validation for a conformance extension, run from the /// post-inference static-class checks alongside the ordinary extension checks pub(crate) fn validate_conformance_declaration<'db>( diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 6e74b900e9..3566d45248 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -920,15 +920,26 @@ fn basedpython_display_enabled() -> bool { BASEDPYTHON_DISPLAY.with(std::cell::Cell::get) } -/// Run `f` with basedpython-style type display enabled. Used by -/// diagnostic emission for `.by` files -pub(crate) fn with_basedpython_display(f: impl FnOnce() -> R) -> R { - BASEDPYTHON_DISPLAY.with(|cell| { - let prev = cell.replace(true); - let result = f(); - cell.set(prev); - result - }) +/// Run `f` with basedpython-style type display set to `enabled`. +/// +/// Both directions matter: inferring a `.by` file reaches definitions in `.py` files +/// and vice versa, and each file's own diagnostics have to be spelled in that file's +/// syntax rather than in whichever one the outermost caller happened to be reading. +pub(crate) fn with_basedpython_display(enabled: bool, f: impl FnOnce() -> R) -> R { + /// Restores the previous setting however `f` ends. Type inference is where this is + /// set, and salsa cancels an inference by unwinding through it — a `f()` that never + /// returns would otherwise leave the thread spelling every later type in the syntax + /// of the file whose inference was cancelled. + struct Restore(bool); + + impl Drop for Restore { + fn drop(&mut self) { + BASEDPYTHON_DISPLAY.with(|cell| cell.set(self.0)); + } + } + + let _restore = BASEDPYTHON_DISPLAY.with(|cell| Restore(cell.replace(enabled))); + f() } /// Format a file location suffix for disambiguation (e.g., " @ path:line:column") diff --git a/crates/ty_python_semantic/src/types/enums.rs b/crates/ty_python_semantic/src/types/enums.rs index bf273f4f35..1fdcbb2b90 100644 --- a/crates/ty_python_semantic/src/types/enums.rs +++ b/crates/ty_python_semantic/src/types/enums.rs @@ -1735,17 +1735,19 @@ pub(crate) fn is_enum_class<'db>(db: &'db dyn Db, ty: Type<'db>) -> bool { } } -/// shared checker/transpiler contract for the basedpython `is`/`is not` -/// keyword pair: whether a comparison whose rhs has type `ty` keeps python -/// identity semantics instead of lowering to `isinstance`. true when the rhs -/// is statically a plain *value* — an enum member (`Color.RED`, a based-enum -/// unit variant like `Shape.Point`), another literal, or an instance of a -/// concrete non-type class — which `isinstance` would reject as its classinfo -/// argument at runtime. a class-like rhs (a class literal, `type[...]`, an -/// instance of a metaclass), a tuple (a valid multi-target classinfo -/// spelling), a bare `object` (which admits classes), and anything dynamic or -/// unresolved lower to `isinstance` as usual -pub fn basedpython_is_keeps_identity<'db>( +/// basedpython: whether a value of type `ty` is a plain *value* rather than a +/// class — an enum member (`Color.RED`, a based-enum unit variant), another +/// literal, or an instance of a concrete non-type class. `isinstance` rejects +/// such a value as its classinfo argument at runtime, so the test for one is +/// identity. +/// +/// This answers a lowering question, not a checking one. A type test's target +/// is a type expression, and the checker reads it as one — but the enum +/// lowering rewrites a unit variant into a singleton instance before the +/// transpiler's own pass runs, so a target that names a type in the source can +/// name a value in what is emitted. That is the case this decides, from the +/// type rather than from the shape the target was written in. +pub(crate) fn basedpython_is_plain_value<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, @@ -1754,13 +1756,13 @@ pub fn basedpython_is_keeps_identity<'db>( Type::Union(union) => union .elements(db) .iter() - .all(|element| basedpython_is_keeps_identity(db, env, *element)), + .all(|element| basedpython_is_plain_value(db, env, *element)), // literal values (enum members included) are never classes Type::LiteralValue(_) | Type::EnumComplement(_) => true, // a use-site modifier says nothing about whether the value is a class: // a unit enum variant is a `final _Shape_Point`, still an instance Type::Restricted(restricted) => { - basedpython_is_keeps_identity(db, env, restricted.value_type(db)) + basedpython_is_plain_value(db, env, restricted.value_type(db)) } Type::NominalInstance(instance) => { !instance.has_known_class(db, KnownClass::Object) diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 49bff5c7ba..3b80ade284 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -573,9 +573,7 @@ impl<'db> OverloadLiteral<'db> { // below belongs to whichever one the body scope was built for let module = parsed_module(db, self.python_file(db)).load(db); let node = self.body_scope(db).node(db).expect_function().node(&module); - let source = source_text(db, file); - crate::reified::reified_type_param_names(source.as_str(), source_type, node) - .into_boxed_slice() + crate::reified::reified_type_param_names(source_type, node).into_boxed_slice() } /// basedpython: reified type parameters that a call must supply a value diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index a6787ab202..7536bf20aa 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -2540,6 +2540,15 @@ bitflags::bitflags! { /// means something: it bounds the pack as a whole rather than field by field. const IN_PACK_BOUND = 1 << 18; + /// basedpython: set while inferring the right-hand side of a type test — the + /// type expression `x is T` tests against. + /// + /// A bare generic class is complete there: the test asks whether the value is + /// one, and the type arguments are exactly what the runtime cannot see. So + /// `missing-type-argument`, which asks an annotation to say more, has nothing + /// to ask for here. + const IN_TYPE_TEST_TARGET = 1 << 19; + /// Whether the current method's explicit receiver annotation is incompatible with `Self`. const HAS_INCOMPATIBLE_SELF_RECEIVER = 1 << 15; } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index efc4a1c48d..e8f87f036a 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -61,6 +61,7 @@ use crate::types::call::bind::{ requires_overload_evaluation, }; use crate::types::call::{Argument, Binding, Bindings, CallArguments, CallError, CallErrorKind}; +use crate::types::call_type_forms::CallTypeForm; use crate::types::callable::CallableTypeKind; use crate::types::class::{ ClassLiteral, CodeGeneratorKind, DynamicClassScopeOffset, DynamicNamedTupleAnchor, @@ -161,16 +162,15 @@ use crate::types::visibility::{private_constructor, scope_is_within_class}; use crate::types::{ BindingContext, BoundTypeVarInstance, CallDunderError, CallableBinding, CallableType, CallableTypes, ClassType, DeferredOperation, DeferredType, DynamicType, GeneratorTypeMode, - InferenceFlags, InstanceProjection, InternedConstraintSet, InternedType, IntersectionBuilder, - IntersectionType, KnownBoundMethodType, KnownClass, KnownInstanceType, KnownUnion, - LiteralValueType, LiteralValueTypeKind, MemberLookupPolicy, ParamSpecAttrKind, Parameter, - Parameters, ProgramEnvironment, PropertyDeprecations, RestrictedType, SentinelInstance, - Signature, SpecialFormType, SubclassOfType, Type, TypeAliasType, TypeAndQualifiers, - TypeContext, TypeQualifiers, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, - TypedDictType, TypingModule, UnionAccumulator, UnionBuilder, UnionType, any_over_type, - binding_type, extract_fixed_length_iterable_element_types, infer_complete_scope_types, - infer_scope_types, is_discarded_dict_key_assignment, report_iteration_over_character, - todo_type, + InferenceFlags, InternedConstraintSet, InternedType, IntersectionBuilder, IntersectionType, + KnownBoundMethodType, KnownClass, KnownInstanceType, KnownUnion, LiteralValueType, + LiteralValueTypeKind, MemberLookupPolicy, ParamSpecAttrKind, Parameter, Parameters, + ProgramEnvironment, PropertyDeprecations, RestrictedType, SentinelInstance, Signature, + SpecialFormType, SubclassOfType, Type, TypeAliasType, TypeAndQualifiers, TypeContext, + TypeQualifiers, TypeVarBoundOrConstraints, TypeVarKind, TypeVarVariance, TypedDictType, + TypingModule, UnionAccumulator, UnionBuilder, UnionType, any_over_type, binding_type, + extract_fixed_length_iterable_element_types, infer_complete_scope_types, infer_scope_types, + is_discarded_dict_key_assignment, report_iteration_over_character, todo_type, }; use crate::{AnalysisSettings, Db, DisplaySettings, FxIndexSet, FxOrderSet, SemanticModel}; use fluid::FluidTimeline; @@ -1278,6 +1278,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { /// Infers types in the given [`InferenceRegion`]. fn infer_region(&mut self) { + // a diagnostic raised here is written for the file being inferred, so the types + // it names have to be spelled in that file's own syntax. the spelling is taken + // from the file rather than from whoever asked for the inference because this + // work is cached: were it taken from the caller, the first reader of a `.by` + // file would decide how every later reader sees the messages salsa kept + crate::with_display_for_file(self.db(), self.file(), || self.infer_region_inner()); + } + + fn infer_region_inner(&mut self) { match self.region { InferenceRegion::Statement(statement) => self.infer_region_statement(statement), InferenceRegion::Scope(scope, tcx) => self.infer_region_scope(scope, tcx), @@ -4773,35 +4782,40 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let func_ty = self .try_expression_type(func) .unwrap_or_else(|| self.infer_expression(func, TypeContext::default())); - if func_ty == Type::SpecialForm(SpecialFormType::NamedTuple) { - // Only the `fields` argument is deferred for `NamedTuple`; - // other arguments are inferred eagerly. - self.infer_typing_namedtuple_fields(&arguments.args[1]); - return; - } let known_class = func_ty .as_class_literal() .and_then(|cls| cls.known(self.db())); - match (known_class, self.region) { - (Some(KnownClass::NewType), _) => { + // which construct this is, and so which of its arguments are type expressions, is + // decided in one place — the basedpython transpiler asks the same question of the + // same call, because a form whose arguments are checked as types is one whose + // arguments have to be lowered as types too + match (CallTypeForm::of(self.db(), func_ty), self.region) { + (Some(CallTypeForm::NamedTuple), _) => { + // Only the `fields` argument is deferred for `NamedTuple`; + // other arguments are inferred eagerly. + self.infer_typing_namedtuple_fields(&arguments.args[1]); + return; + } + (Some(CallTypeForm::NewType), _) => { self.infer_newtype_assignment_deferred(arguments); return; } - ( - Some(KnownClass::TypeAliasType | KnownClass::ExtensionsTypeAliasType), - InferenceRegion::Deferred(definition), - ) => { + (Some(CallTypeForm::TypeAliasType), InferenceRegion::Deferred(definition)) => { self.infer_typealiastype_assignment_deferred(definition, target, arguments); return; } - (Some(KnownClass::Type), InferenceRegion::Deferred(definition)) => { - self.infer_builtins_type_deferred(definition, value); + (Some(CallTypeForm::TypedDict), _) => { + self.infer_functional_typeddict_deferred(arguments); return; } _ => {} } - if TypingModule::from_typed_dict_type(self.db(), func_ty).is_some() { - self.infer_functional_typeddict_deferred(arguments); + // `type("C", bases, ns)` and `new_class` build a class rather than name a type, so + // neither is a form whose arguments are type expressions + if let (Some(KnownClass::Type), InferenceRegion::Deferred(definition)) = + (known_class, self.region) + { + self.infer_builtins_type_deferred(definition, value); return; } if let InferenceRegion::Deferred(definition) = self.region @@ -11008,7 +11022,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.file(), value_ty, alias, - type_arg, + Some(type_arg), ), crate::types::reified_infer::ParametricIsPlan::TokenEq(_) | crate::types::reified_infer::ParametricIsPlan::Fold(_) @@ -15813,6 +15827,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let ast::ExprCompare { range: _, node_index: _, + identity_ops: _, left, ops, comparators, @@ -15832,6 +15847,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // A chain like `a == True == b` is two comparisons over one literal: reporting each pair // would double up on that `True`, and "test the operand" is not the fix for the chain. let single_comparison = ops.len() == 1; + // a chained type test is a parse error; nothing here re-reports it + let chained_type_test = !single_comparison && compare.has_type_test(self.source_type()); let ChainedBooleanResult { value_type: ty, preceding_truthiness, @@ -15841,32 +15858,45 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { std::iter::once(&**left) .chain(comparators) .tuple_windows::<(_, _)>() - .zip(ops), + .zip(ops.iter().enumerate()), |_| false, - |builder, ((left, right), op), _peer_ty| { + |builder, ((left, right), (index, op)), _peer_ty| { let left_ty = builder.expression_type(left); - let right_ty = builder.infer_expression(right, TypeContext::default()); - let range = TextRange::new(left.start(), right.end()); + // basedpython: the right-hand side of a type test is a *type* + // expression — `x is list[int]` names the type `list[int]`, not + // the class object the same source spells as a value. inferring + // it that way is what makes an annotation and a type test agree + // about which targets exist, and about which ones are rejected. + // + // a chain carrying one is rejected outright by the parser, and + // checking its pairs anyway would pile a second report on the + // same mistake + if !chained_type_test && compare.is_type_test(index, builder.source_type()) { + let previously_in_target = builder + .context + .inference_flags + .replace(InferenceFlags::IN_TYPE_TEST_TARGET, true); + let target = builder.infer_type_expression(right); + builder + .context + .inference_flags + .set(InferenceFlags::IN_TYPE_TEST_TARGET, previously_in_target); + return ( + builder.check_type_test(left, right, left_ty, target, *op), + range, + ); + } + + let right_ty = builder.infer_expression(right, TypeContext::default()); + if single_comparison { builder.check_redundant_boolean_comparison( left, right, left_ty, right_ty, *op, range, ); } - // a basedpython keyword-form `is`/`is not` whose rhs is a - // class (or a parametric test like `x is list[int]`) is an - // instance check, not python identity: it always yields a - // `bool`, and its reachability is decided by narrowing (not - // by the instance-vs-class-object disjointness that would - // otherwise type it `Literal[False]` and kill a live branch) - if let Some(ty) = - builder.check_basedpython_is_test(left, right, left_ty, right_ty, *op) - { - return (ty, range); - } - let ty = comparisons::infer_binary_type_comparison( &builder.context, left_ty, @@ -15936,71 +15966,85 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ty } - /// basedpython: type a keyword-form `is`/`is not` pair that performs an - /// *instance check* rather than python identity, and check it. Returns - /// `Some(bool)` — the runtime result type — for any such pair, so the - /// identity folds (disjointness → `Literal[False]`) never apply to it; - /// reachability is decided by narrowing instead - fn check_basedpython_is_test( + /// basedpython: check a type test — `x is T` / `x is not T` — and give it + /// the `bool` its runtime check produces. + /// + /// `target` is the *type* the right-hand side names, already inferred as a + /// type expression, so an invalid target has been reported as + /// `invalid-type-form` before this runs. What is left to decide is whether + /// that type can be tested at runtime at all, and whether the test could + /// ever hold. Reachability is decided by narrowing rather than by the + /// identity folds, so this never returns anything but `bool`. + fn check_type_test( &mut self, left: &ast::Expr, right: &ast::Expr, left_ty: Type<'db>, - right_ty: Type<'db>, + target: Type<'db>, op: ast::CmpOp, - ) -> Option> { - let (bool_ty, decision) = - self.classify_basedpython_is_test(left, right, left_ty, right_ty, op)?; - self.report_non_overlapping_type_test(left, right, left_ty, right_ty, op, decision); - Some(bool_ty) - } - - /// The type a keyword-form `is`/`is not` asks its left operand to have: the - /// instance type of the class on the right, or the union of the arms' instance - /// types for a union target. `None` for a target with no instance form. - fn is_test_target_instance(&self, right: &ast::Expr, right_ty: Type<'db>) -> Option> { - let env = self.program_environment(); - // an over-approximating projection is the safe direction here: a wider - // target can only overlap more, so it never invents a disjointness + ) -> Type<'db> { let db = self.db(); - let Some(arms) = union_target_arms(right) else { - return right_ty - .to_instance(db, env) - .map(InstanceProjection::into_inner); - }; - let mut instances = Vec::with_capacity(arms.len()); - for arm in arms { - instances.push(self.expression_type(arm).to_instance(db, env)?.into_inner()); + let env = self.program_environment(); + let plan = crate::types::reified_infer::type_test_plan( + db, + env, + self.file(), + left_ty, + target, + Some(right), + ); + if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { + let source = ruff_db::source::source_text(db, self.file()); + self.report_erased_type_check( + TextRange::new(left.start(), right.end()), + &source[right.range()], + reason, + ); + } else { + // a target with no runtime test has already been rejected, and + // saying it can also never hold would pile a second report on one + // mistake + self.report_non_overlapping_type_test( + left, + right, + left_ty, + target, + op, + plan.never_holds(), + ); + } + // a test the static types settle *is* its answer, and saying so is what + // lets a reader — and the editor's data-flow view — see that the branch + // it guards is decided. an undecidable test stays `bool`: the identity + // folds python would apply to the same operator have no place here, + // since the right-hand side names a type rather than the class object + // the same source spells as a value + match plan { + crate::types::reified_infer::ParametricIsPlan::Fold(holds) => { + Type::bool_literal(holds == (op == ast::CmpOp::Is)) + } + _ => KnownClass::Bool.to_instance(db, env), } - Some(UnionType::from_elements(db, env, instances)) } - /// Warn when a keyword-form `is`/`is not` tests a value against a type it can - /// never have. The test is then a constant — `is` never holds and `is not` - /// always does — so either the guarded branch is dead or the wrong type was - /// named. `Any`/`Unknown` overlap everything, so those never fire. + /// Warn when a type test asks whether a value has a type it can never have. + /// The test is then a constant — `is` never holds and `is not` always does — + /// so either the guarded branch is dead or the wrong type was named. + /// `Any`/`Unknown` overlap everything, so those never fire. fn report_non_overlapping_type_test( &self, left: &ast::Expr, right: &ast::Expr, left_ty: Type<'db>, - right_ty: Type<'db>, + target: Type<'db>, op: ast::CmpOp, - decision: IsTestDecision, + never_holds: bool, ) { - let env = self.program_environment(); - let db = self.db(); - let Some(target) = self.is_test_target_instance(right, right_ty) else { - return; - }; - let never_holds = match decision { - IsTestDecision::Instance => left_ty.is_disjoint_from(db, env, target), - IsTestDecision::ParametricNeverHolds => true, - IsTestDecision::Undecided => false, - }; if !never_holds { return; } + let db = self.db(); + let env = self.program_environment(); let range = TextRange::new(left.start(), right.end()); let Some(builder) = self.context.report_lint(&NON_OVERLAPPING_TYPE_TEST, range) else { return; @@ -16017,118 +16061,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); } - /// Decide whether this pair is an instance check, erroring when the pair is a - /// parametric test (`x is list[int]`) against a builtin collection whose - /// runtime instances erase their type arguments, so no runtime probe of the - /// value can ever confirm the specialization. `None` when the pair keeps - /// python identity semantics (`===` spelling, a literal or other plain-value - /// rhs such as an enum member — mirroring the transpiler's lowering), so the - /// caller keeps its usual comparison typing - fn classify_basedpython_is_test( - &mut self, - left: &ast::Expr, - right: &ast::Expr, - left_ty: Type<'db>, - right_ty: Type<'db>, - op: ast::CmpOp, - ) -> Option<(Type<'db>, IsTestDecision)> { - let env = self.program_environment(); - if !matches!(op, ast::CmpOp::Is | ast::CmpOp::IsNot) || !self.is_basedpython_file() { - return None; - } - // a literal rhs (`x is None`, `x is 0`) keeps python identity - // semantics; the transpiler leaves the operator untouched - if right.is_literal_expr() { - return None; - } - let source = ruff_db::source::source_text(self.db(), self.file()); - if !crate::reified::is_keyword_comparison(source.as_str(), op, left, right) { - return None; - } - let bool_ty = KnownClass::Bool.to_instance(self.db(), env); - - // a union target `a is T1 | T2` tests each arm (`type(a) <: Ti` for any - // arm). an erased arm can't be checked at runtime and, unlike a - // standalone erased target, may not fold to a constant inside the - // disjunction — that would be unsound — so it is rejected per arm - if let Some(arms) = union_target_arms(right) { - let mut decision = IsTestDecision::Instance; - for arm in arms { - let Some(alias) = crate::types::reified_infer::parametric_is_target( - self.db(), - env, - self.expression_type(arm), - ) else { - continue; - }; - // a parametric arm is decided by the engine rather than by - // disjointness, and the test holds as soon as *any* arm does, so - // one such arm puts the whole disjunction out of the lint's reach - decision = IsTestDecision::Undecided; - if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = - crate::types::reified_infer::classify_parametric_is( - self.db(), - env, - self.file(), - left_ty, - alias, - arm, - ) - { - self.report_erased_type_check(arm.range(), &source[arm.range()], reason); - } - } - return Some((bool_ty, decision)); - } - - // a plain-value rhs (an enum member, an instance of a non-type class) - // keeps python identity semantics — the transpiler leaves `is`/`is not` - // untouched, so ty types it as an ordinary identity comparison too - if crate::types::basedpython_is_keeps_identity(self.db(), env, right_ty) { - return None; - } - - let Some(alias) = - crate::types::reified_infer::parametric_is_target(self.db(), env, right_ty) - else { - // a bare class / dynamic rhs (`x is int`, `x is SomeClass`) is an - // instance check that lowers to `isinstance`, so it always yields a - // `bool` — the identity folds (disjointness → `Literal[False]`) - // must not apply - return Some((bool_ty, IsTestDecision::Instance)); - }; - let plan = crate::types::reified_infer::classify_parametric_is( - self.db(), - env, - self.file(), - left_ty, - alias, - right, - ); - // only a probe against a runtime-erased target is an error; every - // other plan (fold, reified-cell equality, witness, or a probe of a - // user generic that carries `__orig_class__`) is a valid test - if let crate::types::reified_infer::ParametricIsPlan::ErasedTarget(reason) = plan { - self.report_erased_type_check( - TextRange::new(left.start(), right.end()), - &source[right.range()], - reason, - ); - } - let decision = if plan == crate::types::reified_infer::ParametricIsPlan::Fold(false) { - IsTestDecision::ParametricNeverHolds - } else { - IsTestDecision::Undecided - }; - Some((bool_ty, decision)) - } - - /// report an `erased-type-check` for a parametric `is`-target (or one arm - /// of a union target) that has no runtime residue — either because the - /// target records no specialization to probe, or because the target cannot - /// be spelled at runtime at all. every other concrete class records its - /// specialization on the instance or across its mro, so the runtime probe - /// unwinds it instead fn report_erased_type_check( &self, primary: TextRange, @@ -16177,6 +16109,95 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { whose `__orig_bases__` the probe can unwind" )); } + ErasedTargetReason::Dynamic => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` admits every value, \ + so there is nothing for the test to look for" + )); + diagnostic.info(format_args!( + "a test that can only answer `True` narrows nothing; drop it, or name the \ + type the value is expected to have" + )); + } + ErasedTargetReason::Callable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a callable's parameter and \ + return types are not recorded on the value" + )); + diagnostic.info(format_args!( + "the runtime can see that a value is callable and nothing more, so the \ + signature would be assumed rather than checked" + )); + diagnostic.info(format_args!( + "test against a bare `Callable`, which asks only what the runtime records, \ + or take the value's type from the call site" + )); + } + ErasedTargetReason::TypedDict => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: a `TypedDict`'s instances are \ + plain dicts" + )); + diagnostic.info(format_args!( + "the test could only ask whether the value is a `dict`, assuming every key \ + and value type `{target}` declares" + )); + diagnostic.info(format_args!( + "`isinstance` refuses a `TypedDict` for the same reason" + )); + } + ErasedTargetReason::NonRuntimeCheckableProtocol => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has a member with no \ + runtime spelling" + )); + diagnostic.info(format_args!( + "a member whose type the emitted python cannot name leaves nothing to check \ + the value's own annotation against" + )); + diagnostic.info(format_args!( + "decorate `{target}` with `@typing.runtime_checkable` for the check python \ + itself performs, which asks only that the members are present" + )); + } + ErasedTargetReason::Intersection => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: an intersection has no single \ + runtime form" + )); + diagnostic.info(format_args!( + "test each part separately and join the tests with `and`" + )); + } + ErasedTargetReason::Subclass => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `issubclass` cannot take what \ + `{target}` is parameterized by" + )); + diagnostic.info(format_args!( + "the test can see that the value is a class and nothing more; name the class \ + it must be a subclass of" + )); + } + ErasedTargetReason::UncomparableLiteral => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: equality does not decide \ + membership of `{target}`" + )); + diagnostic.info(format_args!( + "`0.0 == -0.0` and `nan != nan`, so a `float` or `complex` literal names a \ + type the runtime cannot be asked about; `LiteralString` is a property of how \ + a value was written rather than of the value" + )); + } + ErasedTargetReason::Unspellable => { + let mut diagnostic = builder.into_diagnostic(format_args!( + "`is {target}` cannot be checked at runtime: `{target}` has no runtime form" + )); + diagnostic.info(format_args!( + "the emitted python has no expression to evaluate for this target" + )); + } } } @@ -17296,44 +17317,6 @@ fn is_collection_literal(expression: &ast::Expr) -> bool { ) } -/// the flat arms of a `|` union type expression (`A | B | C` → `[A, B, C]`), or -/// `None` when `expr` is not a union — used to test each arm of a parametric -/// `is`-target union independently -/// basedpython: how a keyword-form `is`/`is not` instance check is decided, which -/// is what the `non-overlapping-type-test` lint reads to know whether the test can -/// ever hold. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum IsTestDecision { - /// a bare class target (`x is int`): ordinary disjointness decides it - Instance, - /// a parametric target (`x is list[int]`) the parametric engine folded to - /// `False`. That engine is asked rather than disjointness directly because it - /// also honours a use-site variance projection (`a is A[out int]`) - ParametricNeverHolds, - /// a parametric target left to a runtime probe, or a union with such an arm — - /// nothing the lint can call constant - Undecided, -} - -fn union_target_arms(expr: &ast::Expr) -> Option> { - fn collect<'a>(expr: &'a ast::Expr, arms: &mut Vec<&'a ast::Expr>) { - if let ast::Expr::BinOp(binop) = expr - && binop.op == ast::Operator::BitOr - { - collect(&binop.left, arms); - collect(&binop.right, arms); - } else { - arms.push(expr); - } - } - if !matches!(expr, ast::Expr::BinOp(binop) if binop.op == ast::Operator::BitOr) { - return None; - } - let mut arms = Vec::new(); - collect(expr, &mut arms); - Some(arms) -} - /// Returns `true` if `expression` is a link of a basedpython optional chain: a `?.` access, or a /// trailer applied to one. /// diff --git a/crates/ty_python_semantic/src/types/infer/builder/class.rs b/crates/ty_python_semantic/src/types/infer/builder/class.rs index f038cc78ec..fb8e6c5c50 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/class.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/class.rs @@ -15,7 +15,6 @@ use crate::types::{ }, special_form::TypeQualifier, }; -use ruff_db::source::source_text; use ruff_python_ast::{self as ast, helpers::any_over_expr}; use ty_module_resolver::{ImportingFile, KnownModule, file_to_module}; use ty_python_core::{definition::Definition, scope::NodeWithScopeRef}; @@ -35,8 +34,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { if !source_type.is_basedpython() { return; } - let source = source_text(self.db(), self.file()); - let reads = reified_class_reads(source.as_str(), source_type, class); + let reads = reified_class_reads(source_type, class); // reification fixes the variance, so a declaration saying anything else // is a contradiction rather than a refinement — and the variance it diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index b5f73395da..08601f8f2c 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -1013,12 +1013,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { || is_implicit_classmethod(&name.id)) && let Some(type_params) = function.type_params.as_deref() { - let source = ruff_db::source::source_text(db, self.file()); - let reified = crate::reified::reified_type_param_names( - source.as_str(), - self.file().source_type(db), - function, - ); + let reified = + crate::reified::reified_type_param_names(self.file().source_type(db), function); if let Some(first) = reified.first() && let Some(builder) = self.context.report_lint(&REIFIED_CLASSMETHOD, type_params) { diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index 7b03151fe6..ad82b4dcd6 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -208,7 +208,20 @@ impl<'db> TypeInferenceBuilder<'db, '_> { if self.is_basedpython_file() && ty.as_enum_literal().is_some() { return ty; } - report_missing_type_arguments(&self.context, ty, annotation); + // a type test's target is complete without type arguments: it asks + // whether the value is one of these, and the arguments are what the + // runtime cannot see either way. that holds for the target itself, not + // for a bare generic *inside* it — `x is A[list]` really does leave an + // argument the probe compares + let bare_target_is_complete = self + .inference_flags() + .contains(InferenceFlags::IN_TYPE_TEST_TARGET) + && !self + .inference_flags() + .contains(InferenceFlags::IN_NESTED_TYPE_EXPRESSION); + if !bare_target_is_complete { + report_missing_type_arguments(&self.context, ty, annotation); + } let result_ty = ty .default_specialize(db, env) .in_type_expression( diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index 450d32c1fd..d8816bdae1 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -21,8 +21,8 @@ use crate::types::{ CallableType, ClassBase, ClassLiteral, ClassPatternPositionalSource, ClassType, IntersectionBuilder, IntersectionType, KnownClass, KnownInstanceType, LiteralValueTypeKind, Parameter, Parameters, Signature, SpecialFormType, SubclassOfInner, SubclassOfType, Truthiness, - Type, TypeContext, TypeVarBoundOrConstraints, UnionBuilder, basedpython_is_keeps_identity, - binding_type, callable_pattern_type, class_pattern_positional_sources, + Type, TypeContext, TypeVarBoundOrConstraints, UnionBuilder, binding_type, + callable_pattern_type, class_pattern_positional_sources, definite_match_pattern_type_for_subject, exact_sequence_pattern_type, infer_expression_types, mapping_pattern_type, pattern_binding_fallthrough_type, sequence_pattern_type_builder, singleton_pattern_type, starred_sequence_pattern_type, typed_dict_matches_class_pattern, @@ -4230,6 +4230,7 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { let ast::ExprCompare { range: _, node_index: _, + identity_ops: _, left, ops, comparators, @@ -4543,19 +4544,18 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { }; let mut last_rhs_ty: Option = None; - // basedpython: in `.by` files, the `is`/`is not` keyword form - // performs isinstance-style narrowing on the lhs (the `===`/`!==` - // operators retain Python's identity-narrowing semantics). The - // parser flattens both spellings to the same `CmpOp`, so we look - // at the original source text between the two operands to tell - // them apart - let file = expression.file(self.db); - let basedpython_keyword_form = file - .source_type(self.db) - .is_basedpython() - .then(|| ruff_db::source::source_text(self.db, file)); - - for (op, (left, right)) in std::iter::zip(&**ops, comparator_tuples) { + // basedpython: in `.by` files, the `is`/`is not` keyword form is a type + // test and narrows the way `isinstance` does; `===` / `!==` keep + // python's identity narrowing. the parser folds both spellings onto one + // `CmpOp` and records which it saw + let source_type = expression.file(self.db).source_type(self.db); + // a chain carrying a type test is a parse error, and the inference + // builder falls back to reading the whole chain as ordinary + // comparisons. narrowing has to read it the same way, or the body of a + // branch it guards disappears on top of the syntax error + let chained_type_test = ops.len() > 1 && expr_compare.has_type_test(source_type); + + for (index, (op, (left, right))) in std::iter::zip(&**ops, comparator_tuples).enumerate() { let lhs_ty = last_rhs_ty.unwrap_or_else(|| expression_type(left, &self.env)); let rhs_ty = expression_type(right, &self.env); let lhs_narrowing_rhs_ty = if matches!(op, ast::CmpOp::In | ast::CmpOp::NotIn) { @@ -4565,23 +4565,8 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { rhs_ty }; - // literal rhs (None, True/False, numbers, strings, bytes, `...`) keeps - // Python identity semantics — `isinstance(x, None)` would be invalid. - // the same holds for any rhs that resolves to a plain value (an enum - // member like `Color.RED`, a based unit variant, an instance of a - // non-type class): it is not a class, so the transpiler keeps `is` - // and narrowing must mirror that - let basedpython_is_keyword = basedpython_keyword_form.as_ref().is_some_and(|src| { - use ruff_text_size::Ranged; - matches!(op, ast::CmpOp::Is | ast::CmpOp::IsNot) && !right.is_literal_expr() && { - let between = - &src[usize::from(left.range().end())..usize::from(right.range().start())]; - let trimmed = between.trim(); - !trimmed.starts_with("===") && !trimmed.starts_with("!==") - } - }) && !basedpython_is_keeps_identity( - self.db, &env, rhs_ty, - ); + let basedpython_is_keyword = + !chained_type_test && expr_compare.is_type_test(index, source_type); // Narrowing for: // - `if type(x) is Y` @@ -4644,15 +4629,25 @@ impl<'db> NarrowingConstraintsBuilder<'db, '_> { && let Some(narrowable) = PlaceExpr::try_from_expr(left) { let positive = is_positive == matches!(op, ast::CmpOp::Is); - // a parametric test (`x is list[int]`) verifies the exact - // specialization, so the positive branch narrows to it. no - // negative narrowing: an unreified or witness-less (empty) - // value answers `False` even when it *is* one statically - let constraint = if let Type::GenericAlias(alias) = rhs_ty { - positive.then(|| Type::instance(self.db, &env, ClassType::Generic(alias))) + // the right-hand side of a type test is a type expression, so + // its inferred type *is* the type being tested for — there is + // no class object to unwrap. the negative branch narrows only + // where the runtime check is exact: a parametric probe reads the + // arguments a value happens to record, and answers `False` for + // one that records none even though it is a match + let constraint = if positive { + Some(rhs_ty) } else { - ClassInfoConstraintFunction::IsInstance - .generate_constraint(self.db, &env, rhs_ty, positive, false) + crate::types::reified_infer::type_test_plan( + self.db, + &env, + expression.file(self.db), + lhs_ty, + rhs_ty, + Some(right), + ) + .narrows_negatively() + .then_some(rhs_ty) }; if let Some(constraint_ty) = constraint { let place = self.expect_place(&narrowable); diff --git a/crates/ty_python_semantic/src/types/reified_infer.rs b/crates/ty_python_semantic/src/types/reified_infer.rs index 8189cab160..577693b7b5 100644 --- a/crates/ty_python_semantic/src/types/reified_infer.rs +++ b/crates/ty_python_semantic/src/types/reified_infer.rs @@ -16,6 +16,8 @@ //! classes, exotic type forms — has no spelling and the bare call stays an //! error +use std::fmt::Write as _; + use itertools::Itertools; use ruff_db::files::File; use ruff_db::parsed::parsed_module; @@ -696,7 +698,16 @@ pub enum ParametricIsPlan { /// generic whose instances carry `__orig_class__` — probe it at runtime, /// matching each argument by the target's effective variance (one entry /// per type parameter). a legitimate, unwarned runtime test - Probe(Box<[ArgVariance]>), + Probe { + /// the target's runtime spelling (`A[int]`), or `None` when the source + /// already spells it and should be passed through — which is the more + /// robust of the two, since a name the source wrote is in scope by + /// construction while a rebuilt spelling needs the origin class to be + /// nameable where the test is written. `Some` where nothing in the + /// source spells this target on its own: one arm of a union + target: TargetSpelling, + variances: Box<[ArgVariance]>, + }, /// basedpython: not decidable from static types, and the target is a /// protocol — but every data member's specialized type has a runtime /// spelling, so the value's reified annotations can be checked structurally @@ -706,6 +717,114 @@ pub enum ParametricIsPlan { /// a usable `__orig_class__`, so no sound runtime probe exists — the test /// is an error. the reason picks the diagnostic wording ErasedTarget(ErasedTargetReason), + /// `isinstance(value, )` — the target is a plain class, which is + /// exactly what `isinstance` was built to answer. `None` where the source + /// spells the target itself and should be passed through, which names even + /// a class no module global does (an enum's `Shape.Circle`) + Isinstance(TargetSpelling), + /// `callable(value)` — a `Callable` with no signature, which asks only what + /// the runtime records + IsCallable, + /// `value is None`. `None` is a value, not a class, so `isinstance` cannot + /// take it, and identity is the whole test — there is only one `None` + IsNone, + /// `type(value) is and value == ` — a literal target such as + /// `Literal[3]`, whose type holds exactly one value. the class guard is not + /// redundant: python's `1 == True` would otherwise let a `bool` satisfy + /// `Literal[1]` + Equality { + class: String, + /// the value to compare against. the source's own literal where it + /// wrote one — which is already spelled correctly for wherever it sits, + /// including inside an f-string on a python that forbids reusing the + /// outer quote + value: TargetSpelling, + }, + /// `value is ` — an enum member, which is a singleton, so + /// identity is both exact and what the runtime compares anyway. `None` + /// passes the source through, as for [`Self::Isinstance`] + Identity(TargetSpelling), + /// `isinstance(value, type) and issubclass(value, )` — a + /// `type[C]` target, which the runtime can check in full. `None` passes the + /// source through, as for [`Self::Isinstance`] + Subclass(TargetSpelling), + /// a template literal type: the value must be a `str` matching this + /// regular expression, which spells the same language `matches_str` decides + Pattern(String), + /// the target is an interface something in scope visibly *conforms* to, so + /// the conformance registry answers the test. a conforming type is not a + /// subclass, so `isinstance` could never see the relationship + Conformance { + /// how the interface is written into the emitted python + target: TargetSpelling, + /// the interface's required member names, for the runtime check to look + /// for on a value nothing registered + members: Vec, + }, + /// nothing is known about the target — an error elsewhere left `Unknown` + /// behind. the test is lowered as the plain `isinstance` the source spells + /// and reported by whatever produced the `Unknown`, which is the sharper + /// report and the only one + Unresolved, + /// the disjunction of these plans — the target is a union, and a value + /// satisfies it as soon as one arm holds. the arms are carried as plans of + /// their own because a union's arms need not be spelled in the source: a + /// PEP 695 alias names one with a single identifier + Union(Box<[ParametricIsPlan]>), +} + +impl ParametricIsPlan { + /// whether a `False` from this test proves the value does *not* have the + /// type, so the negative branch may narrow. + /// + /// An exact check answers the question the type asks. The parametric ones + /// do not: a runtime probe reads the arguments a value happens to record, + /// and a value that records none answers `False` even where the static + /// types say it is a match — narrowing on that would remove a type the + /// value really has. + pub(crate) fn narrows_negatively(&self) -> bool { + match self { + Self::Fold(_) + | Self::Isinstance(_) + | Self::IsCallable + | Self::IsNone + | Self::Equality { .. } + | Self::Identity(_) + | Self::Subclass(_) + | Self::Pattern(_) => true, + Self::Union(arms) => arms.iter().all(Self::narrows_negatively), + Self::TokenEq(_) + | Self::Probe { .. } + | Self::ProtocolStructural(_) + | Self::Conformance { .. } + | Self::Unresolved + | Self::ErasedTarget(_) => false, + } + } + + /// whether this plan proves the test can never hold, so the guarded branch + /// is dead. only a static fold proves that; every runtime residue leaves + /// the answer to the value + pub(crate) fn never_holds(&self) -> bool { + match self { + Self::Fold(holds) => !holds, + Self::Union(arms) => arms.iter().all(Self::never_holds), + _ => false, + } + } +} + +/// basedpython: how a type test's target is written into the emitted python +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetSpelling { + /// the source spells the target itself, so its own text is passed through. + /// this is the more robust of the two: a name the source wrote is in scope + /// by construction, while a rebuilt spelling needs the class to be nameable + /// where the test is written + Written, + /// nothing in the source spells this target on its own — one arm of a union + /// the source named with a single word — so it is rebuilt + Rebuilt(String), } /// basedpython: one protocol member a parametric `is`-test checks structurally @@ -736,9 +855,36 @@ pub enum ProtocolMemberCheck { }, } -/// why a parametric test's target cannot be probed at runtime +/// why a type test's target cannot be checked at runtime #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ErasedTargetReason { + /// the target says nothing a runtime check could look for — `Any` and + /// `Unknown` admit every value, so the test has no content + Dynamic, + /// a callable type: the runtime sees that a value is callable and nothing + /// more, so its parameter and return types would be assumed rather than + /// checked + Callable, + /// a `TypedDict`: its instances are plain dicts, so a runtime check can + /// only ask whether the value is a `dict` and would assume every key + TypedDict, + /// an intersection of types has no single runtime form to test against + Intersection, + /// a protocol that is not `@runtime_checkable` and has a member with no + /// runtime spelling, so neither python's presence check nor a structural + /// one against the value's reified annotations can answer it + NonRuntimeCheckableProtocol, + /// a `type[…]` whose argument `issubclass` cannot take — `Any`, or a + /// protocol with a data member + Subclass, + /// a literal whose value the runtime cannot be asked to compare: a `float` + /// or `complex`, whose equality does not decide the type (`0.0 == -0.0`), + /// or `LiteralString`, which is a property of how a value was written and + /// not of the value + UncomparableLiteral, + /// the target has no runtime spelling at all — a type the checker can name + /// but the emitted python cannot evaluate + Unspellable, /// a builtin collection (`list` / `dict` / `set` / `frozenset` / `tuple`) /// erases its type arguments — its C-level instances reject /// `__orig_class__` entirely @@ -906,7 +1052,7 @@ fn erased_target_reason<'db>( /// (`X = list[int]`); a PEP 695 `type` alias is unwrapped to the same. `None` /// when the rhs is not a specialization — a bare class or value — so the caller /// keeps the ordinary `isinstance` lowering. -pub(crate) fn parametric_is_target<'db>( +fn parametric_is_target<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, rhs_ty: Type<'db>, @@ -969,11 +1115,11 @@ pub(crate) fn classify_parametric_is<'db>( file: File, lhs_ty: Type<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, - rhs_node: &ast::Expr, + rhs_node: Option<&ast::Expr>, ) -> ParametricIsPlan { let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); let target_args_ast: Vec<&ast::Expr> = match rhs_node { - ast::Expr::Subscript(subscript) => match subscript.slice.as_ref() { + Some(ast::Expr::Subscript(subscript)) => match subscript.slice.as_ref() { ast::Expr::Tuple(tuple) => tuple.elts.iter().collect(), single => vec![single], }, @@ -982,8 +1128,8 @@ pub(crate) fn classify_parametric_is<'db>( let plan = classify_value( db, env, + file, lhs_ty.promote(db, env), - target_origin, rhs_alias, &target_args_ast, rhs_node, @@ -997,7 +1143,7 @@ pub(crate) fn classify_parametric_is<'db>( // spellable data members can still be checked structurally against those // annotations. only a protocol that also has a method member (unrecoverable // from an annotation) stays an error - if let ParametricIsPlan::Probe(_) = plan + if let ParametricIsPlan::Probe { .. } = plan && let Some(ErasedTargetReason::Protocol) = erased_target_reason(db, target_origin) { return protocol_structural_members(db, env, file, ClassType::Generic(rhs_alias)) @@ -1015,7 +1161,7 @@ pub(crate) fn classify_parametric_is<'db>( // `x is Sequence[int]` — which runs perfectly well — into an error if matches!( plan, - ParametricIsPlan::Probe(_) | ParametricIsPlan::TokenEq(_) + ParametricIsPlan::Probe { .. } | ParametricIsPlan::TokenEq(_) ) && runtime_subscript(db, env, target_origin) == RuntimeSubscript::Unsupported { return ParametricIsPlan::ErasedTarget(ErasedTargetReason::NotSubscriptable); @@ -1023,6 +1169,517 @@ pub(crate) fn classify_parametric_is<'db>( plan } +/// basedpython: how a type test — `value is Target` — resolves, for *any* +/// target type. +/// +/// The right-hand side of a type test is a type expression, so an unusable +/// target has already been reported as `invalid-type-form` by the time this +/// runs. What is left is a narrower question: does the type it named have a +/// runtime form? A test must *earn* its `True` — it narrows — so a target the +/// runtime can only partly check is rejected rather than approximated. That +/// rules out a `TypedDict` (its instances are plain dicts), a callable type +/// (the runtime sees only that a value is callable), and `Any`. +/// +/// `target_node` is the source the target was written as, which the +/// specialization plans use to spell type arguments back out. +/// +/// `value_ty` is taken exactly as the value has it — a construction's `final A` +/// is disjoint from an unrelated class in a way a plain `A` is not, and that is +/// the whole point of tracking it. Only the specialization path widens, where a +/// literal argument would otherwise decide a test the runtime cannot. +pub(crate) fn type_test_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, +) -> ParametricIsPlan { + type_test_plan_seen( + db, + env, + file, + value_ty, + target, + target_node, + &mut Vec::new(), + ) +} + +/// [`type_test_plan`] carrying the aliases already opened on the way here. +/// +/// A `type` alias may name itself — `type A = int | B` with `type B = str | A` +/// — and the plan for one is the plan for its value, so following that without +/// a record would not terminate. +fn type_test_plan_seen<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, + open: &mut Vec>, +) -> ParametricIsPlan { + // an `Unknown` target is not evidence of anything: every value is a subtype + // of it, and folding on that would answer `True` for a test the source got + // wrong somewhere else + if target.is_dynamic() { + return runtime_test_plan(db, env, file, value_ty, target, target_node, open); + } + // a target that resolves statically needs no runtime residue at all, and + // answering it here keeps every unspellable-but-decidable target working: + // `x is Never` is `False` without the runtime ever seeing `Never` + if value_ty.is_subtype_of(db, env, target) { + return ParametricIsPlan::Fold(true); + } + if value_ty.is_disjoint_from(db, env, target) { + return ParametricIsPlan::Fold(false); + } + runtime_test_plan(db, env, file, value_ty, target, target_node, open) +} + +/// The runtime residue of a type test whose answer the static types do not +/// already give. Split from [`type_test_plan`] so a union arm can be planned +/// without re-asking the static question the whole test already answered. +fn runtime_test_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, + open: &mut Vec>, +) -> ParametricIsPlan { + if target.is_none(db) { + return ParametricIsPlan::IsNone; + } + match target { + // an alias stands for its value, and a use-site modifier (`final T`) + // constrains how the value may be used rather than what it is at + // runtime — neither adds anything a runtime check could look for + Type::TypeAlias(alias) => { + // an alias that names itself has no value to resolve to, and the + // definition is reported where it is written + if open.contains(&target) { + return ParametricIsPlan::Unresolved; + } + open.push(target); + let plan = runtime_test_plan(db, env, file, value_ty, alias.value_type(db), None, open); + open.pop(); + plan + } + // a use-site modifier is not python, so the source no longer spells + // what is left once it is dropped + Type::Restricted(restricted) => runtime_test_plan( + db, + env, + file, + value_ty, + restricted.value_type(db), + None, + open, + ), + + // a value satisfies a union as soon as it satisfies one arm. the arms + // are planned separately because a union need not be spelled as one: + // `type AU = int | str` names it with a single identifier, and + // `isinstance` cannot take the alias object that identifier evaluates to + Type::Union(union) => { + let mut arms = Vec::with_capacity(union.elements(db).len()); + for element in union.elements(db) { + // the source spells the union, not this arm, so the arm's own + // plan may not read type arguments back out of it + let arm = type_test_plan_seen(db, env, file, value_ty, *element, None, open); + // one unusable arm makes the whole disjunction unusable: it may + // not quietly fold to `False`, since that would answer `False` + // for a value the arm would have accepted + if let ParametricIsPlan::ErasedTarget(reason) = arm { + return ParametricIsPlan::ErasedTarget(reason); + } + // an arm the checker could not read has no spelling of its own, + // and the source spells the union rather than the arm — so the + // whole test falls back to what the source wrote + if matches!(arm, ParametricIsPlan::Unresolved) { + return ParametricIsPlan::Unresolved; + } + arms.push(arm); + } + ParametricIsPlan::Union(arms.into_boxed_slice()) + } + + // a literal type holds exactly the values equal to it, which is what + // the runtime compares. an enum member is a literal too, and its + // equality is identity + Type::LiteralValue(literal) => literal_target_plan(db, env, file, literal, target_node), + + // `type[C]`: the runtime can check this one in full — the value must be + // a class, and a subclass of `C` + Type::SubclassOf(subclass) => match subclass.subclass_of().into_class(db, env) { + // `type[C]` is written as a subscript, so the source spells the + // subscript rather than `C` — the spelling has to be rebuilt + Some(class) => match spell_class(db, env, file, class) { + Some(spelling) => ParametricIsPlan::Subclass(TargetSpelling::Rebuilt(spelling)), + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + }, + // `type[Any]`, or `type[]` — the value must be a class, + // and `issubclass` has nothing it can ask beyond that + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Subclass), + }, + + Type::NominalInstance(_) | Type::ProtocolInstance(_) => { + class_target_plan(db, env, file, value_ty, target, target_node) + } + + // the value's type is carried by a reified type parameter on the + // *target* side — `x is T` where `T` is reified spells `T` itself + Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) => { + ParametricIsPlan::Isinstance(TargetSpelling::Rebuilt( + bound_typevar.name(db).to_string(), + )) + } + + // `Any` really does admit every value and is worth rejecting. `Unknown` + // is what an already-reported error leaves behind, and a second report + // on the same target would only repeat it + Type::Dynamic(crate::types::DynamicType::Any) => { + ParametricIsPlan::ErasedTarget(ErasedTargetReason::Dynamic) + } + Type::Dynamic(_) => ParametricIsPlan::Unresolved, + // a bare `Callable` is exactly what `callable()` answers; only a + // *signature* asks for something the value does not record + Type::Callable(callable) + if callable.signatures(db).iter().all(is_unannotated_signature) => + { + ParametricIsPlan::IsCallable + } + Type::Callable(_) => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Callable), + Type::Intersection(_) => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Intersection), + // a `TypedDict`'s inhabitants are plain dicts: nothing at runtime + // records which one a dict was built as + Type::TypedDict(_) => ParametricIsPlan::ErasedTarget(ErasedTargetReason::TypedDict), + _ => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + } +} + +/// The runtime test for a literal target. A `TypedDict`-like erasure is not +/// good enough here: the type holds exactly the values equal to the literal, so +/// the check is that equality. +fn literal_target_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + literal: crate::types::LiteralValueType<'db>, + target_node: Option<&ast::Expr>, +) -> ParametricIsPlan { + let unspellable = ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + // the guard names a builtin, whose spelling is fixed and always in scope + let written = target_node + .is_some_and(|node| node.is_literal_expr() || matches!(node, ast::Expr::UnaryOp(_))); + let equality = |class: &str, value: String| ParametricIsPlan::Equality { + class: class.to_owned(), + value: if written { + TargetSpelling::Written + } else { + TargetSpelling::Rebuilt(value) + }, + }; + match literal.kind() { + // a pattern is a set of strings, and the regular expression that spells + // it decides exactly the language `matches_str` decides + LiteralValueTypeKind::Template(template) => match template_pattern(db, env, template) { + Some(pattern) => ParametricIsPlan::Pattern(pattern), + None => unspellable, + }, + LiteralValueTypeKind::Bool(value) => { + equality("bool", (if value { "True" } else { "False" }).to_owned()) + } + LiteralValueTypeKind::Int(value) => equality("int", value.as_i64().to_string()), + LiteralValueTypeKind::String(value) => equality("str", python_str_literal(value.value(db))), + LiteralValueTypeKind::Bytes(value) => { + equality("bytes", python_bytes_literal(value.value(db))) + } + // an enum member is a singleton, so identity is exact — and it is the + // comparison `Enum.__eq__` performs anyway + LiteralValueTypeKind::Enum(member) => { + match written_or_spelled(target_node, false, || { + spell_class_literal(db, env, file, member.enum_class(db)) + .map(|class| format!("{class}.{}", member.name(db))) + }) { + Some(spelling) => ParametricIsPlan::Identity(spelling), + None => unspellable, + } + } + // `LiteralString` is the *property* of having been written as a literal, + // which nothing about a value at runtime records; `float` and `complex` + // literals are values the checker tracks but does not promise are + // distinguishable from equal ones + LiteralValueTypeKind::LiteralString + | LiteralValueTypeKind::Float(_) + | LiteralValueTypeKind::Complex(_) => { + ParametricIsPlan::ErasedTarget(ErasedTargetReason::UncomparableLiteral) + } + } +} + +/// how a target is written into the emitted python: the spelling rebuilt from +/// the type, or the source's own text where that is what the runtime evaluates. +/// +/// The rebuilt spelling is preferred, because the source names a *type* and the +/// emitted check needs a *value*, and the two part company far more often than +/// they look like they do: a PEP 695 alias evaluates to a `TypeAliasType`, +/// `Literal[…]` and `Annotated[…]` to special forms, `list[Any]` to a +/// subscripted generic — none of which `isinstance` will take. Falling back to +/// the source is for the one case rebuilding cannot express: a class the +/// emitting module cannot name as a global, such as an enum's attached variant +/// or a class imported under another name. Only a plain dotted name qualifies, +/// and only a [`Probe`](ParametricIsPlan::Probe) also accepts a subscript, +/// whose value is the specialization it unwinds. +fn written_or_spelled( + target_node: Option<&ast::Expr>, + subscript_evaluates: bool, + spell: impl FnOnce() -> Option, +) -> Option { + if let Some(spelling) = spell() { + return Some(TargetSpelling::Rebuilt(spelling)); + } + target_node + .filter(|node| evaluates_to_its_target(node, subscript_evaluates)) + .map(|_| TargetSpelling::Written) +} + +/// whether the runtime value of `node` is the thing the type it names denotes +fn evaluates_to_its_target(node: &ast::Expr, subscript_evaluates: bool) -> bool { + match node { + ast::Expr::Name(_) => true, + ast::Expr::Attribute(attribute) => evaluates_to_its_target(&attribute.value, false), + ast::Expr::Subscript(subscript) if subscript_evaluates => { + evaluates_to_its_target(&subscript.value, false) + } + _ => false, + } +} + +/// a python string literal for `value`, escaped so the emitted source reads it +/// back character for character. +fn python_str_literal(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + ch if ch.is_control() || ch == '"' => push_escaped_char(ch, &mut out), + ch => out.push(ch), + } + } + out.push('"'); + out +} + +/// a python bytes literal for `value`, written one `\xNN` escape per byte so +/// every byte round-trips whatever it is +fn python_bytes_literal(value: &[u8]) -> String { + let mut out = String::with_capacity(value.len() * 4 + 3); + out.push_str("b\""); + for byte in value { + let _ = write!(out, "\\x{byte:02x}"); + } + out.push('"'); + out +} + +/// The runtime test for an instance target — the common case, and the one the +/// parametric engine already answered for a specialization. +fn class_target_plan<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: File, + value_ty: Type<'db>, + target: Type<'db>, + target_node: Option<&ast::Expr>, +) -> ParametricIsPlan { + let Some(class) = target_class(db, env, target) else { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + }; + let literal = class.class_literal(db); + // a `TypedDict`'s instances are plain dicts, so the only runtime question + // is whether the value is a `dict` — every key and value type would be + // assumed. a test that must earn its `True` cannot assume them + if literal.is_typed_dict(db) { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::TypedDict); + } + match class { + // a bare generic class in a type expression means every specialization + // of it, which is the class itself — and `isinstance` answers exactly + // that. only *written* arguments give the parametric engine something + // to check + ClassType::Generic(alias) + if alias + .specialization(db) + .types(db) + .iter() + .all(Type::is_dynamic) => + { + match written_or_spelled(target_node, false, || { + spell_class_literal(db, env, file, ClassLiteral::Static(alias.origin(db))) + }) { + Some(spelling) => ParametricIsPlan::Isinstance(spelling), + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + } + } + // a specialization keeps the engine it always had: reified cells, + // an `__orig_class__` probe, or a structural protocol check + ClassType::Generic(alias) => { + classify_parametric_is(db, env, file, value_ty, alias, target_node) + } + ClassType::NonGeneric(_) => { + // an interface something visibly conforms to is answered by the + // registry rather than by the class hierarchy, so it is checkable + // even though a conforming type is not a subclass + if let Some(members) = conformance_members(db, file, class) { + return match written_or_spelled(target_node, false, || { + spell_class(db, env, file, class) + }) { + Some(target) => ParametricIsPlan::Conformance { target, members }, + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + }; + } + if let Some(protocol) = class.into_protocol_class(db) { + // `@runtime_checkable` is the author's own statement that + // `isinstance` may take the class, and it is what python then + // checks — that the members are present + if !protocol.is_runtime_checkable(db) { + // basedpython reifies class annotations, so a protocol whose + // members all have a runtime spelling can be checked against + // them member by member — a stricter answer than presence, + // and the only one available without the decorator + return match protocol_structural_members(db, env, file, class) { + Some(checks) => { + ParametricIsPlan::ProtocolStructural(checks.into_boxed_slice()) + } + None => ParametricIsPlan::ErasedTarget( + ErasedTargetReason::NonRuntimeCheckableProtocol, + ), + }; + } + } + match written_or_spelled(target_node, false, || spell_class(db, env, file, class)) { + Some(spelling) => ParametricIsPlan::Isinstance(spelling), + None => ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable), + } + } + } +} + +/// The required member names of `class` when something in `file`'s scope +/// visibly conforms to it, or `None` when it is not a conformance interface. +fn conformance_members<'db>( + db: &'db dyn Db, + file: File, + class: ClassType<'db>, +) -> Option> { + let conformed = crate::types::conformance::visible_conformances(db, file) + .iter() + .any(|(_, declared)| declared.class_literal(db) == class.class_literal(db)); + conformed.then(|| { + crate::types::conformance::interface_requirements(db, class) + .iter() + .map(ToString::to_string) + .collect() + }) +} + +/// whether a callable signature says nothing beyond "this is callable" — the +/// gradual form `Callable[..., Any]` the bare `Callable` denotes +fn is_unannotated_signature(signature: &crate::types::signatures::Signature<'_>) -> bool { + signature.parameters().is_gradual() && signature.return_ty.is_dynamic() +} + +/// The class an instance target names, for both the nominal and the protocol +/// spelling of one. +fn target_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + target: Type<'db>, +) -> Option> { + match target { + Type::NominalInstance(instance) => Some(instance.class(db, env)), + Type::ProtocolInstance(instance) => match instance.inner { + crate::types::instance::Protocol::FromClass(class) => Some(*class), + crate::types::instance::Protocol::Materialized(_) + | crate::types::instance::Protocol::Synthesized(_) => None, + }, + _ => None, + } +} + +/// The regular expression that matches exactly the strings a template literal +/// type produces, for `re.fullmatch`, or `None` when one of its holes has no +/// regular-expression spelling. +/// +/// The holes are read through the same [`HoleShape`](crate::types::template::HoleShape) +/// classification +/// `matches_str` uses, so the runtime test and the static one decide the same +/// language rather than two that happen to agree on the cases anyone tried. +fn template_pattern<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + template: crate::types::template::TemplateLiteralType<'db>, +) -> Option { + let mut pattern = String::new(); + for part in template.parts(db) { + match part { + crate::types::template::TemplatePart::Text(text) => { + escape_regex(text.as_str(), &mut pattern); + } + crate::types::template::TemplatePart::Hole(hole) => { + pattern.push_str(crate::types::template::HoleShape::of(db, env, *hole).regex()?); + } + } + } + Some(pattern) +} + +/// Append `text` to `pattern` with every character python's `re` gives a +/// meaning escaped, so the text matches itself. +/// +/// A character the emitted source cannot carry — anything the `str` escaping +/// below would have to spell — is written as its own `\\xNN` / `\\uNNNN` +/// escape, which `re` reads as that character. Passing one through raw would +/// put a literal control byte in the python, and CPython refuses to compile a +/// source containing a NUL at all. +fn escape_regex(text: &str, pattern: &mut String) { + for ch in text.chars() { + if "\\.^$*+?()[]{}|-#&~".contains(ch) { + pattern.push('\\'); + pattern.push(ch); + } else if ch.is_control() || ch == '"' { + push_escaped_char(ch, pattern); + } else { + pattern.push(ch); + } + } +} + +/// Write `ch` as the escape a python string literal reads back as that +/// character. Used for anything the emitted source cannot carry raw. +fn push_escaped_char(ch: char, out: &mut String) { + match ch { + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '"' => out.push_str("\\\""), + ch if (ch as u32) < 0x100 => { + let _ = write!(out, "\\x{:02x}", ch as u32); + } + ch if (ch as u32) < 0x1_0000 => { + let _ = write!(out, "\\u{:04x}", ch as u32); + } + ch => { + let _ = write!(out, "\\U{:08x}", ch as u32); + } + } +} + /// basedpython: the structural runtime check for a protocol target whose data /// members can all be verified against a value's reified class annotations, or /// `None` when the protocol has a member that can't be — a method (its shape @@ -1163,12 +1820,13 @@ fn is_object_instance<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: T fn classify_value<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, + file: File, value_ty: Type<'db>, - target_origin: ClassLiteral<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, target_args_ast: &[&ast::Expr], - rhs_node: &ast::Expr, + rhs_node: Option<&ast::Expr>, ) -> ParametricIsPlan { + let target_origin = ClassLiteral::Static(rhs_alias.origin(db)); // when the value's type is carried by a reified type parameter, the answer // lives in a runtime cell rather than the static type — extract the cell // comparisons before falling back to static subtyping @@ -1196,7 +1854,17 @@ fn classify_value<'db>( } else { // undecidable statically; `classify_parametric_is` turns this into a // runtime probe (user generic) or an erased-target error (builtin) - ParametricIsPlan::Probe(target_variances(db, rhs_alias)) + // the source's own spelling is preferred: a name it wrote is in scope, + // and the runtime probe unwraps a `TypeAliasType` for itself + let Some(target) = written_or_spelled(rhs_node, true, || { + spell_class(db, env, file, ClassType::Generic(rhs_alias)) + }) else { + return ParametricIsPlan::ErasedTarget(ErasedTargetReason::Unspellable); + }; + ParametricIsPlan::Probe { + target, + variances: target_variances(db, rhs_alias), + } } } @@ -1211,7 +1879,7 @@ fn try_token_eq<'db>( target_origin: ClassLiteral<'db>, rhs_alias: crate::types::class::GenericAlias<'db>, target_args_ast: &[&ast::Expr], - rhs_node: &ast::Expr, + rhs_node: Option<&ast::Expr>, ) -> Option { match value_ty { // `x: T is ` compares the reified `T` cell against the target @@ -1221,11 +1889,11 @@ fn try_token_eq<'db>( // `TypeAliasType` wrapper), so it falls through to the static resolution Type::TypeVar(bound_typevar) if is_reified_function_typevar(db, bound_typevar) - && matches!(rhs_node, ast::Expr::Subscript(_)) => + && matches!(rhs_node, Some(ast::Expr::Subscript(_))) => { Some(ParametricIsPlan::TokenEq(vec![( bound_typevar.name(db).clone(), - rhs_node.range(), + rhs_node.expect("guarded above").range(), )])) } Type::NominalInstance(instance) => { @@ -1453,8 +2121,7 @@ fn is_reified_function_typevar<'db>( return false; }; let node = function.node(&module); - let source = ruff_db::source::source_text(db, def_file); - crate::reified::reified_type_param_names(source.as_str(), def_file.source_type(db), node) + crate::reified::reified_type_param_names(def_file.source_type(db), node) .iter() .any(|name| name == bound_typevar.name(db)) } diff --git a/crates/ty_python_semantic/src/types/template.rs b/crates/ty_python_semantic/src/types/template.rs index 9211971fc9..6fb16675a9 100644 --- a/crates/ty_python_semantic/src/types/template.rs +++ b/crates/ty_python_semantic/src/types/template.rs @@ -92,7 +92,7 @@ pub(crate) enum HoleShape { } impl HoleShape { - fn of<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, hole: Type<'db>) -> Self { + pub(crate) fn of<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, hole: Type<'db>) -> Self { let Some(class) = hole.nominal_class(db, env) else { return Self::Anything; }; @@ -123,6 +123,22 @@ impl HoleShape { fn admits_empty(self) -> bool { self == Self::Anything } + + /// the regular expression matching exactly this shape's strings, or `None` + /// when it has no spelling python's `re` can evaluate. + /// + /// A grapheme is a cluster of code points rather than one, and `re` has no + /// way to say that, so a pattern with a grapheme hole has no runtime test — + /// which is why the type test rejects one rather than approximating it + pub(crate) fn regex(self) -> Option<&'static str> { + match self { + Self::Anything => Some("(?s:.*)"), + // exactly the renderings `is_int_rendering` accepts — `str(-0)` is + // `"0"`, so the sign belongs to the non-zero alternative alone + Self::Int => Some("(?:0|-?[1-9][0-9]*)"), + Self::Grapheme => None, + } + } } /// whether `value` is what `str()` produces for some `int` diff --git a/crates/ty_server/Cargo.toml b/crates/ty_server/Cargo.toml index bbd5988940..38791450c7 100644 --- a/crates/ty_server/Cargo.toml +++ b/crates/ty_server/Cargo.toml @@ -32,6 +32,7 @@ ty_ide = { workspace = true } ty_module_resolver = { workspace = true } ty_project = { workspace = true } ty_python_semantic = { workspace = true } +ty_static = { workspace = true } anyhow = { workspace = true } bitflags = { workspace = true } @@ -39,6 +40,7 @@ crossbeam = { workspace = true } jod-thread = { workspace = true } lsp-server = { workspace = true } lsp-types = { workspace = true } +rand = { workspace = true, features = ["sys_rng"] } rustc-hash = { workspace = true } salsa = { workspace = true } serde = { workspace = true } diff --git a/crates/ty_server/src/document.rs b/crates/ty_server/src/document.rs index 1daccb3786..37583d26d2 100644 --- a/crates/ty_server/src/document.rs +++ b/crates/ty_server/src/document.rs @@ -86,6 +86,14 @@ impl DocumentKey { } } + /// The path this document has on disk, or `None` for one that has none. + pub(crate) const fn file_path(&self) -> Option<&SystemPathBuf> { + match self { + Self::File(path) => Some(path), + Self::Opaque(_) => None, + } + } + /// Returns the corresponding [`AnySystemPath`] for this document key. /// /// Note, calling this method on a `DocumentKey::Opaque` representing a cell document diff --git a/crates/ty_server/src/lib.rs b/crates/ty_server/src/lib.rs index da4f138448..25ae32cfc1 100644 --- a/crates/ty_server/src/lib.rs +++ b/crates/ty_server/src/lib.rs @@ -15,6 +15,7 @@ mod capabilities; mod db; mod document; mod logging; +pub mod project_server; mod server; mod session; mod system; @@ -51,9 +52,24 @@ pub fn run_server() -> anyhow::Result<()> { // This is to complement the `LSPSystem` if the document is not available in the index. let fallback_system = Arc::new(OsSystem::new(cwd)); - let server_result = Server::new(worker_threads, connection, fallback_system, false) - .context("Failed to start server")? - .run(); + // a server publishes itself for `by` command lines to find, unless the user has said + // not to. see `project_server` + let project_server_directory = if project_server::disabled(&*fallback_system) { + tracing::debug!("Not publishing this server: disabled by the environment"); + None + } else { + project_server::discovery::default_directory(&*fallback_system) + }; + + let server_result = Server::new( + worker_threads, + connection, + fallback_system, + false, + project_server_directory, + ) + .context("Failed to start server")? + .run(); let io_result = io_threads.join(); diff --git a/crates/ty_server/src/project_server/check.rs b/crates/ty_server/src/project_server/check.rs new file mode 100644 index 0000000000..207ce80c36 --- /dev/null +++ b/crates/ty_server/src/project_server/check.rs @@ -0,0 +1,320 @@ +//! Answering a command line's check out of the session the editor is already keeping warm. +//! +//! The whole of the saving is that this runs on a database somebody else built and has been +//! feeding changes into. What is left is the part a cold run cannot skip either: deciding +//! whether this database is allowed to answer for this caller, and rendering. + +use std::path::{Path, PathBuf}; + +use ruff_db::Db as _; +use ruff_db::diagnostic::{ + DiagnosticId, DisplayDiagnosticConfig, DisplayDiagnostics, FileResolver, Input, Severity, + UnifiedFile, +}; +use ruff_db::files::File; +use ruff_db::system::SystemPath; +use ruff_notebook::NotebookIndex; +use ty_project::{CheckMode, CollectReporter, Db as _, ProjectDatabase}; + +use super::Outcome; +use super::protocol; +use super::protocol::{CheckRequest, CheckResponse, Refusal, Response, SeverityLevel}; +use crate::session::{OpenDocument, SessionSnapshot}; + +/// Runs `request` against `snapshot`, or explains why it did not. +/// +/// A [`Refusal`] is a normal outcome, not an error. The caller falls back to checking for +/// itself, which is slower and always correct, so every uncertainty here resolves to one. +pub(super) fn run(snapshot: &SessionSnapshot, request: &CheckRequest, rescanned: bool) -> Outcome { + let db = match gate(snapshot, request) { + Ok(db) => db, + Err(refusal) => return Outcome::Answered(Response::Refused { reason: refusal }), + }; + + // everything that could have refused has agreed, so this request is going to be answered + // — and now it is worth going to the file system, which is the one thing that decides + // whether the answer is about the project as it is now + if !rescanned { + return Outcome::NeedsRescan; + } + + Outcome::Answered(answer(db, request)) +} + +/// Everything that has to be true before this session may answer for `request`. +/// +/// Separate from [`answer`] because it is the cheap half, and because the main loop runs it +/// first on its own: re-reading the file system is expensive and interrupts the editor, and +/// there is no reason to do either for a request that was never going to be answered. +fn gate<'a>( + snapshot: &'a SessionSnapshot, + request: &CheckRequest, +) -> Result<&'a ProjectDatabase, Refusal> { + let Some(db) = project_database(snapshot, request) else { + return Err(Refusal::UnknownProject); + }; + + // a server that has been told to diagnose only what is open checks only what is open, + // and would answer a whole project's check with a handful of files' diagnostics. the + // editor's `diagnostic-mode` decides this, and nothing here may quietly override it: it + // is a salsa input on a project the editor is also using + if db.check_mode() != CheckMode::AllFiles { + return Err(Refusal::CheckMode); + } + + if let Some(refusal) = disagreement(db, request) { + return Err(refusal); + } + + // this database reads an open file out of the editor's buffer, so for as long as a + // buffer differs from the file, the two sides are checking different programs + let unsaved = unsaved_documents(db, snapshot); + if !unsaved.is_empty() { + return Err(Refusal::Unsaved { files: unsaved }); + } + + Ok(db) +} + +/// Checks the project and renders what it found. +fn answer(db: &ProjectDatabase, request: &CheckRequest) -> Response { + let diagnostics = match salsa::Cancelled::catch(|| { + let mut reporter = CollectReporter::default(); + db.check_with_reporter(&mut reporter); + reporter.into_sorted(db) + }) { + Ok(diagnostics) => diagnostics, + Err(cancelled) => { + tracing::debug!("A command-line check was cancelled: {cancelled:?}"); + return Response::Refused { + reason: Refusal::Cancelled, + }; + } + }; + + let terminal = db.project().settings(db).terminal(); + let config = DisplayDiagnosticConfig::new("ty") + .format(terminal.output_format.into()) + .color(request.color) + .context(0); + let resolver = RelativeTo { + db, + working_directory: working_directory(db, request), + }; + + let mut max_severity = None; + let mut io_error = false; + for diagnostic in &diagnostics { + max_severity = max_severity.max(Some(diagnostic.severity())); + io_error = io_error || matches!(diagnostic.id(), DiagnosticId::Io); + } + + Response::Check(CheckResponse { + rendered: DisplayDiagnostics::new(&resolver, &config, &diagnostics).to_string(), + diagnostics: diagnostics.len(), + human_readable: terminal.output_format.is_human_readable(), + max_severity: max_severity.map(SeverityLevel::from), + io_error, + error_on_warning: terminal.error_on_warning, + empty_project: db.project().files(db).is_empty(), + fatal: max_severity == Some(Severity::Fatal), + }) +} + +/// Where the caller is standing, spelled the way this database spells it. +/// +/// Both sides reach the same tree, and neither necessarily by the same name — see +/// [`super::canonical`]. Paths are relativized by stripping a prefix, so the caller's own +/// spelling of its directory would strip nothing off the files this database holds. What +/// makes it strip is putting the caller's position *inside* the project onto the root this +/// database uses. +/// +/// A caller standing outside the project — `by check --project ../elsewhere` — has no +/// position inside it, and gets its own directory back. Nothing is stripped then, which +/// leaves absolute paths, which is what a cold check prints from there too. +fn working_directory(db: &ProjectDatabase, request: &CheckRequest) -> PathBuf { + let root = db.project().root(db); + let relative = super::canonical(&request.working_directory) + .strip_prefix(super::canonical(root)) + .map(SystemPath::to_path_buf); + + match relative { + Ok(relative) => root.join(relative).as_std_path().to_path_buf(), + Err(_) => request.working_directory.as_std_path().to_path_buf(), + } +} + +/// The database, rendering paths from where the caller is standing rather than from where the +/// server is. +/// +/// Everything a rendering needs comes from the database, except the one thing that does not +/// belong to it: which directory the reader will read the answer in. A server's own is +/// wherever its editor was started, which is nowhere in particular. +struct RelativeTo<'db> { + db: &'db ProjectDatabase, + working_directory: PathBuf, +} + +impl FileResolver for RelativeTo<'_> { + fn path(&self, file: File) -> &str { + self.db.path(file) + } + + fn input(&self, file: File) -> Input { + self.db.input(file) + } + + fn notebook_index(&self, file: &UnifiedFile) -> Option { + self.db.notebook_index(file) + } + + fn is_notebook(&self, file: &UnifiedFile) -> bool { + self.db.is_notebook(file) + } + + fn current_directory(&self) -> &Path { + &self.working_directory + } +} + +impl From for SeverityLevel { + fn from(severity: Severity) -> Self { + match severity { + Severity::Info => SeverityLevel::Info, + Severity::Warning => SeverityLevel::Warning, + Severity::Error => SeverityLevel::Error, + Severity::Fatal => SeverityLevel::Fatal, + } + } +} + +/// Whatever this database and the caller disagree about, if anything. +/// +/// Both sides resolved the same project from the same files on disk, and they have to have +/// resolved it to the same thing before one can answer for the other. What is compared is +/// what each side actually ended up with rather than what it was told, because most of the +/// ways they can differ are not written down anywhere: a flag on the command line, a setting +/// the editor contributed, an interpreter discovered out of one process's environment and not +/// the other's. +fn disagreement(db: &ProjectDatabase, request: &CheckRequest) -> Option { + let options = + match protocol::configuration(db.project().metadata(db).to_merged_options().options()) { + Ok(options) => options, + Err(error) => { + tracing::debug!("Failed to serialize the project's options: {error}"); + return Some(Refusal::UnknownProject); + } + }; + if options != request.options { + return Some(Refusal::Options { + server: Box::new(options), + }); + } + + let environment = environment(db); + if environment != request.environment { + return Some(Refusal::Environment { + server: environment, + }); + } + + if request.verbose { + return Some(Refusal::Verbose); + } + + let force_exclude = db.project().force_exclude(db); + if force_exclude != request.force_exclude { + return Some(Refusal::ForceExclude { + server: force_exclude, + }); + } + + None +} + +/// A project's resolved python version, platform and search paths, as one comparable string. +/// +/// The `Debug` rendering rather than a hand-built one because it is already the curated set: +/// it prints the four kinds of search path and deliberately leaves out the typeshed version +/// map, which is thousands of lines and identical wherever the typeshed path is. +/// +/// The project's own directory is written out of it first. Two processes reach the same tree +/// by different names — a `by check` takes its root from the working directory, which the +/// operating system hands over with symlinks resolved, while an editor sends whatever it was +/// opened with — and every first-party search path, and every path under the project such as +/// a `.venv` beside it, inherits that difference. What is left is what the two would actually +/// disagree about: a different interpreter, a different typeshed, a different version. +pub fn environment(db: &ProjectDatabase) -> String { + let rendered = format!("{:?}", db.project().program_settings(db)); + let root = db.project().root(db); + + // both spellings, because which one this side happens to hold is the whole problem + let canonical = super::canonical(root); + let mut rendered = rendered.replace(root.as_str(), PROJECT_ROOT); + if canonical.as_str() != root.as_str() { + rendered = rendered.replace(canonical.as_str(), PROJECT_ROOT); + } + rendered +} + +/// What the project's own directory is called in an [`environment`] rendering. +const PROJECT_ROOT: &str = ""; + +/// The database rooted exactly at the project the caller resolved, if this session holds one. +/// +/// Exactly, not enclosing: a caller inside a workspace member resolved that member as its +/// project, and a database rooted at the repository above it checks a different set of files +/// under different settings. +fn project_database<'a>( + snapshot: &'a SessionSnapshot, + request: &CheckRequest, +) -> Option<&'a ProjectDatabase> { + let wanted = super::canonical(&request.project_root); + + snapshot + .projects() + .iter() + .find(|db| super::canonical(db.project().root(*db)) == wanted) +} + +/// The open documents this project might read whose content is not what is on disk. +/// +/// `db` first because everything about the question is the database's: which paths are in +/// scope, and what the file system says a path holds. +/// +/// In scope is the project's own tree, plus anything this database has already interned a +/// file for — an editable install, a stub package, a file on an extra search path. The second +/// half matters because [`LSPSystem`](crate::system::LSPSystem) lays the editor's buffers +/// over the whole file system, not over the project: a check that resolves an import into a +/// file the editor happens to be holding reads the buffer. What is deliberately *not* in +/// scope is a document from some unrelated workspace in the same editor window, which this +/// project would never read and which should not cost a caller its answer. +fn unsaved_documents(db: &ProjectDatabase, snapshot: &SessionSnapshot) -> Vec { + let root = super::canonical(db.project().root(db)); + + snapshot + .open_documents() + .filter(|(path, _)| { + super::canonical(path).starts_with(&root) || db.files().try_system(db, path).is_some() + }) + .filter(|(path, document)| match *document { + // a file the file system cannot produce at all — one the editor holds for a file + // that has since been deleted — differs like any other + OpenDocument::Text(contents) => { + snapshot.read_from_disk(path).as_deref() != Some(contents) + } + OpenDocument::Notebook => true, + }) + .map(|(path, _)| { + // the prefix to strip is the canonical root, so what it is stripped from has to + // be the canonical path — a file in scope only because the database interned it + // may still sit outside the root, and keeps its own spelling + let canonical = super::canonical(path); + canonical + .strip_prefix(&root) + .unwrap_or(path) + .as_str() + .to_string() + }) + .collect() +} diff --git a/crates/ty_server/src/project_server/client.rs b/crates/ty_server/src/project_server/client.rs new file mode 100644 index 0000000000..4fb02b66ed --- /dev/null +++ b/crates/ty_server/src/project_server/client.rs @@ -0,0 +1,158 @@ +//! Asking a running server for an answer, from the `by` command line. +//! +//! Everything in here fails quietly. A caller uses this the way it would use a cache: it +//! asks, and if it does not get an answer it does the work. So there is no error type — a +//! failure is `None` and a line in the log, and the caller carries on into the check it was +//! always able to run. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::time::Duration; + +use ruff_db::system::SystemPath; + +use super::discovery; +use super::protocol::{ + Answer, Build, CheckRequest, CheckResponse, PROTOCOL, Payload, Refusal, Request, Response, +}; + +/// How long the server has to accept a connection. +/// +/// It is on this machine, and it is either listening or it is a record left behind by a +/// process that has died. There is no third case worth waiting on. +const CONNECT_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long the server has to answer once it has accepted. +/// +/// Long, because the answer is a real check: the usual case is a database that has the result +/// already, but a server that has just started, or one whose project changed under it, does +/// the same work the caller would have done. Not unbounded, because a server whose main loop +/// has wedged would otherwise leave the caller waiting for something that is never coming, +/// with a check it could have run itself sitting there the whole time. +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(120); + +const SEND_TIMEOUT: Duration = Duration::from_secs(10); + +/// The most an answer may be. +/// +/// A project's rendered diagnostics can genuinely be megabytes, so this is generous — but the +/// port a record names is free for anything to bind once its server dies, and a reply from +/// something that is not a server should not be able to fill memory before the token that +/// would have caught it is read. +const MAX_RESPONSE: u64 = 256 * 1024 * 1024; + +/// Asks a server holding `request.project_root` to check it. +/// +/// `None` when there is no server to ask, when the ones there are will not answer for this +/// caller, or when anything at all goes wrong on the way. +pub fn check( + directory: &SystemPath, + request: CheckRequest, + build: &Build, +) -> Option { + let project_root = request.project_root.clone(); + let payload = Payload::Check(request); + + // two editors can hold the same project, and one of them may be in a state that stops it + // answering while the other is not — so a refusal moves on to the next rather than ending + // the search. every refusal is a server that did nothing, so the request is still ours to + // send again + for candidate in discovery::candidates(directory, &project_root) { + match ask(&candidate, &payload, build) { + Some(Response::Check(response)) => return Some(response), + Some(Response::Refused { reason }) => { + tracing::debug!("A project server did not answer: {reason}"); + if let Refusal::Options { server } = &reason { + tracing::debug!("It resolved: {server}"); + } + } + None => {} + } + } + + tracing::debug!("No project server answered for `{project_root}`"); + None +} + +/// One round trip, or `None` if there wasn't one. +/// +/// `payload` is borrowed rather than moved because every outcome here leaves the request +/// still ours to make: a server that refuses did nothing, and a server that could not be +/// reached never saw it. +fn ask(candidate: &discovery::Candidate, payload: &Payload, build: &Build) -> Option { + let record = &candidate.record; + + // both of these are in the record, so they are settled before a connection is opened + // rather than after a round trip + if record.protocol != PROTOCOL { + tracing::debug!( + "Ignoring a project server speaking protocol {} (this build speaks {PROTOCOL})", + record.protocol + ); + return None; + } + if !build.is(&record.build) { + tracing::debug!( + "Ignoring a project server from a different build ({})", + record.build.version + ); + return None; + } + + let address = SocketAddr::from((Ipv4Addr::LOCALHOST, record.port)); + let Ok(mut connection) = TcpStream::connect_timeout(&address, CONNECT_TIMEOUT) else { + // a listening server always accepts, so this is a record whose server is gone + tracing::debug!("No project server is listening on {address}; forgetting its record"); + discovery::forget(candidate); + return None; + }; + + let request = Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: build.clone(), + payload: serde_json::to_value(payload) + .inspect_err(|error| tracing::debug!("Failed to serialize the request: {error}")) + .ok()?, + }; + + match round_trip(&mut connection, &request, &record.token) { + Ok(response) => Some(response), + Err(error) => { + tracing::debug!("The project server on {address} did not answer: {error}"); + None + } + } +} + +fn round_trip( + connection: &mut TcpStream, + request: &Request, + token: &str, +) -> anyhow::Result { + connection.set_read_timeout(Some(RESPONSE_TIMEOUT))?; + connection.set_write_timeout(Some(SEND_TIMEOUT))?; + + let mut line = serde_json::to_vec(request)?; + line.push(b'\n'); + connection.write_all(&line)?; + connection.flush()?; + + let mut response = String::new(); + BufReader::new(connection) + .take(MAX_RESPONSE) + .read_line(&mut response)?; + if response.is_empty() { + anyhow::bail!("the server closed the connection without answering"); + } + + let answer: Answer = serde_json::from_str(&response)?; + + // the record's port was free to be taken by anything once the server that published it + // died, so an answer only counts if it came from something that had read the record + if answer.token != token { + anyhow::bail!("something other than the project server answered on its port"); + } + + Ok(answer.response) +} diff --git a/crates/ty_server/src/project_server/discovery.rs b/crates/ty_server/src/project_server/discovery.rs new file mode 100644 index 0000000000..b334de7e2c --- /dev/null +++ b/crates/ty_server/src/project_server/discovery.rs @@ -0,0 +1,194 @@ +//! How a command line finds a server that is already holding this project. +//! +//! A server is started by an editor, over stdin and stdout, and nothing about that says +//! where it is or that it exists. So a listening server leaves a small record of itself in +//! a per-user directory, and a command line reads the directory. +//! +//! The record is the capability: it carries the port *and* the token, so being able to +//! reach a server means being able to read a file that only this user can read. + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use ruff_db::system::{System, SystemPath, SystemPathBuf}; + +use super::protocol::Build; + +/// What one listening server publishes about itself. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Record { + pub protocol: u32, + pub build: Build, + pub port: u16, + pub token: String, + + /// The workspace roots this server was started for. + /// + /// A caller keeps a record whose roots contain the project it resolved. That is a + /// filter, not a promise — whether a database is actually rooted at that project is + /// something only the server can answer, and it does. + pub roots: Vec, +} + +/// Where records live for this user, or `None` where this platform has no home to put them. +/// +/// Under ty's own cache directory rather than a directory of this module's choosing, so that +/// there is one answer to "where does ty keep per-user state" and it is the same answer on +/// every platform. +/// +/// Callers pass the directory in rather than reaching for this, so that a server started for +/// a test publishes somewhere a real `by check` will never look. +pub fn default_directory(system: &dyn System) -> Option { + Some(system.cache_dir()?.join("project-servers")) +} + +/// A record on disk, removed when the server that wrote it lets go of this. +#[derive(Debug)] +pub(crate) struct Registration { + path: PathBuf, +} + +impl Drop for Registration { + fn drop(&mut self) { + if let Err(error) = std::fs::remove_file(&self.path) { + tracing::debug!( + "Failed to remove the project server record `{}`: {error}", + self.path.display() + ); + } + } +} + +/// Publishes `record`, replacing whatever this process published before. +/// +/// Keyed by process id, so a server that died without cleaning up is overwritten by +/// whatever the operating system next gives that number to, and in the meantime is +/// recognised as dead by [`live_records`] failing to connect to it. +pub(crate) fn publish(directory: &SystemPath, record: &Record) -> std::io::Result { + let directory = directory.as_std_path(); + std::fs::create_dir_all(directory)?; + + let path = directory.join(format!("{}.json", std::process::id())); + let mut file = std::fs::File::create(&path)?; + restrict_to_owner(&file)?; + file.write_all(&serde_json::to_vec(record)?)?; + file.flush()?; + + Ok(Registration { path }) +} + +/// The token is the whole of the access control, so nobody else may read the file. +#[cfg(unix)] +fn restrict_to_owner(file: &std::fs::File) -> std::io::Result<()> { + file.set_permissions(std::fs::Permissions::from_mode(0o600)) +} + +/// Windows has no mode bits; the containing directory is already per-user. +#[cfg(not(unix))] +fn restrict_to_owner(_file: &std::fs::File) -> std::io::Result<()> { + Ok(()) +} + +/// A published record, and the file it was published in. +/// +/// The path travels with it so that a caller who finds nothing listening can take the record +/// away — see `forget`. +#[derive(Debug, Clone)] +pub struct Candidate { + pub record: Record, + path: PathBuf, +} + +/// Removes a record whose server is not there any more. +/// +/// A server takes its own record away as it exits, but a server that was killed never gets +/// to. Nothing else would ever remove those, and each one costs every later `by check` a +/// connection attempt. +/// +/// The file is re-read first. Records are named by process id, so between reading one and +/// failing to reach it a new server can have been given that id and published over it — +/// deleting that one would leave a live server undiscoverable for the rest of its life. +pub(super) fn forget(candidate: &Candidate) { + match read_record(&candidate.path) { + Some(current) if current.port != candidate.record.port => { + tracing::debug!( + "Leaving `{}` alone: another server has published over it", + candidate.path.display() + ); + return; + } + _ => {} + } + + if let Err(error) = std::fs::remove_file(&candidate.path) { + tracing::debug!( + "Failed to remove the stale project server record `{}`: {error}", + candidate.path.display() + ); + } +} + +/// Every published record whose roots contain `project_root`, nearest root first. +/// +/// Both `project_root` and each record's roots go through `super::canonical` before they are +/// compared: the two reach the same tree from different processes, and neither necessarily by +/// the same name. +/// +/// Nearest first because a workspace opened at a repository root and one opened at the +/// package inside it are both candidates, and the inner one is the one whose settings were +/// resolved for this project. +pub fn candidates(directory: &SystemPath, project_root: &SystemPath) -> Vec { + let Ok(entries) = std::fs::read_dir(directory.as_std_path()) else { + return Vec::new(); + }; + + let project_root = super::canonical(project_root); + let mut candidates: Vec<(usize, Candidate)> = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter_map(|path| Some((read_record(&path)?, path))) + .filter_map(|(record, path)| { + let depth = record + .roots + .iter() + // both sides, because a root reaches this file from whichever process wrote + // it and a project root reaches this function from whichever asked — see + // [`super::canonical`] + .map(|root| super::canonical(root)) + .filter(|root| project_root.starts_with(root)) + .map(|root| root.components().count()) + .max()?; + Some((depth, Candidate { record, path })) + }) + .collect(); + + candidates.sort_by(|(left, _), (right, _)| right.cmp(left)); + candidates + .into_iter() + .map(|(_, candidate)| candidate) + .collect() +} + +fn read_record(path: &Path) -> Option { + if path.extension()? != "json" { + return None; + } + let contents = std::fs::read(path).ok()?; + match serde_json::from_slice(&contents) { + Ok(record) => Some(record), + // a record written by a build that has since changed the shape. it is not this + // process's to delete — the server that wrote it still owns the file, and will + // remove it when it exits + Err(error) => { + tracing::debug!( + "Ignoring unreadable project server record `{}`: {error}", + path.display() + ); + None + } + } +} diff --git a/crates/ty_server/src/project_server/listener.rs b/crates/ty_server/src/project_server/listener.rs new file mode 100644 index 0000000000..7ac58a1257 --- /dev/null +++ b/crates/ty_server/src/project_server/listener.rs @@ -0,0 +1,242 @@ +//! The side channel a `by` command line reaches the server on. +//! +//! The language server's own connection is stdin and stdout, and it belongs to the editor +//! that started it. So requests from a command line arrive somewhere else: a loopback socket, +//! opened once at startup, announced through a [record](super::discovery::Record) only this +//! user can read. +//! +//! Nothing here touches the session. A connection is read, checked against the two things +//! that can be settled without it — the protocol and the build — and then handed to the main +//! loop, which is the only place a database may be looked at. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream}; +use std::time::Duration; + +use ruff_db::system::{SystemPath, SystemPathBuf}; + +use super::discovery::{self, Record, Registration}; +use super::protocol::{Answer, Build, PROTOCOL, Payload, Refusal, Request, Response}; +use crate::server::{Event, MainLoopSender}; + +/// How long a client is given to send its request once it has connected. +/// +/// A connection that opens and says nothing holds a thread, and the port is reachable by +/// anything running on this machine. +const READ_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long the server will spend trying to hand an answer back. +/// +/// A client that stops reading part way through would otherwise pin a worker thread for as +/// long as it cared to. +const WRITE_TIMEOUT: Duration = Duration::from_secs(30); + +/// The most a request may be. +/// +/// Nothing legitimate comes close: the largest field is a project's merged configuration, +/// which is kilobytes. The cap is here because reading a line from a socket that anyone on +/// this machine can connect to is otherwise an invitation to fill memory, and the token that +/// would stop them is not read until after the line is. +const MAX_REQUEST: u64 = 4 * 1024 * 1024; + +/// A request that reached the main loop, and the connection waiting for its answer. +#[derive(Debug)] +pub(crate) struct Incoming { + pub(crate) payload: Payload, + pub(crate) connection: TcpStream, + + /// The secret this request arrived with, to be repeated in the answer. + pub(crate) token: String, + + /// Whether the file system has been re-read for this request yet. + /// + /// The re-read is the expensive part and it disrupts the editor, so it happens only once + /// everything cheap about the request has been agreed — see the main loop. + pub(crate) rescanned: bool, + + /// How many times this has been through the main loop. + /// + /// A check runs on a snapshot, and an edit in the editor cancels it. Re-queueing takes a + /// fresh snapshot, which is the only way to retry; giving up after a few is what keeps a + /// caller from waiting on a session that is being typed into. + pub(crate) attempts: u8, +} + +/// Everything the server holds open for as long as it accepts command-line requests. +/// +/// Dropping it takes the record off disk. The listener thread ends when the process does: it +/// is blocked in `accept`, and there is nothing worth waking it for. +#[derive(Debug)] +pub(crate) struct Listener { + _registration: Registration, +} + +impl Listener { + /// Starts accepting command-line requests for `roots`, announcing itself in `directory`. + /// + /// `Ok(None)` when there is nothing to announce: a server with no workspace folders holds + /// no project any caller could ask about, so a record for it could only ever be a port to + /// connect to and be refused by. + /// + /// An `Err` is the listener failing to start, which is not fatal: a server that cannot be + /// reached this way is a server every command line checks for itself, which is what they + /// all did before. + pub(crate) fn spawn( + sender: MainLoopSender, + directory: &SystemPath, + roots: Vec, + version: &str, + ) -> anyhow::Result> { + if roots.is_empty() { + tracing::debug!("Not accepting command-line requests: this server has no workspaces"); + return Ok(None); + } + + let listener = TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))?; + let port = listener.local_addr()?.port(); + let token = super::token()?; + + let build = Build::current(version); + let registration = discovery::publish( + directory, + &Record { + protocol: PROTOCOL, + build: build.clone(), + port, + token: token.clone(), + roots, + }, + )?; + + std::thread::Builder::new() + .name("ty:project-server".to_owned()) + .spawn(move || accept_loop(&listener, &sender, &token, &build))?; + + tracing::info!("Accepting command-line requests on 127.0.0.1:{port}"); + + Ok(Some(Self { + _registration: registration, + })) + } +} + +fn accept_loop(listener: &TcpListener, sender: &MainLoopSender, token: &str, build: &Build) { + std::thread::scope(|scope| { + for connection in listener.incoming() { + let Ok(connection) = connection.inspect_err(|error| { + tracing::debug!("Failed to accept a command-line connection: {error}"); + }) else { + continue; + }; + + // one thread per connection, because reading one is the only part of this that + // waits on somebody else. a single slow caller — or a local process that connects + // and says nothing until its five seconds are up — would otherwise be enough to + // stop every other caller being heard + let spawned = std::thread::Builder::new() + .name("ty:project-server-request".to_owned()) + .spawn_scoped(scope, || { + if let Err(error) = accept(connection, sender, token, build) { + tracing::debug!("Failed to read a command-line request: {error}"); + } + }); + + if let Err(error) = spawned { + tracing::debug!("Failed to start a thread for a command-line request: {error}"); + } + } + }); +} + +fn accept( + mut connection: TcpStream, + sender: &MainLoopSender, + token: &str, + build: &Build, +) -> anyhow::Result<()> { + connection.set_read_timeout(Some(READ_TIMEOUT))?; + connection.set_write_timeout(Some(WRITE_TIMEOUT))?; + + let mut line = String::new(); + BufReader::new(connection.try_clone()?) + .take(MAX_REQUEST) + .read_line(&mut line)?; + let request: Request = serde_json::from_str(&line)?; + + // an unreadable request and a wrong token get the same treatment, which is none: the + // caller learns whether it guessed the token by whether it is answered at all + if !constant_time_eq(&request.token, token) { + tracing::warn!("Rejecting a command-line request with the wrong token"); + return Ok(()); + } + + if request.protocol != PROTOCOL { + return respond( + &mut connection, + token, + &Response::Refused { + reason: Refusal::Protocol { server: PROTOCOL }, + }, + ); + } + + // same protocol is not the same checker. two builds of `by` disagree about what this + // project's diagnostics are as readily as they disagree about anything else, and the + // caller asked for its own answer + if !build.is(&request.client) { + return respond( + &mut connection, + token, + &Response::Refused { + reason: Refusal::Version { + server: Box::new(build.clone()), + }, + }, + ); + } + + // the payload is only parsed once the two agree on how to read it. a newer client's + // request would fail here, having already been told that its build is not this one + let payload: Payload = serde_json::from_value(request.payload)?; + + sender + .send(Event::ProjectServer(Box::new(Incoming { + payload, + connection, + token: request.token, + rescanned: false, + attempts: 0, + }))) + .map_err(|_| anyhow::anyhow!("the main loop is gone")) +} + +pub(crate) fn respond( + connection: &mut TcpStream, + token: &str, + response: &Response, +) -> anyhow::Result<()> { + let mut line = serde_json::to_vec(&Answer { + token: token.to_owned(), + response: response.clone(), + })?; + line.push(b'\n'); + connection.write_all(&line)?; + connection.flush()?; + Ok(()) +} + +/// Whether two secrets are equal, in time that does not depend on where they first differ. +/// +/// The channel is loopback and the attack is not a practical one, but a comparison that +/// leaks is not cheaper to write than one that does not. +fn constant_time_eq(left: &str, right: &str) -> bool { + let (left, right) = (left.as_bytes(), right.as_bytes()); + if left.len() != right.len() { + return false; + } + + left.iter() + .zip(right) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 +} diff --git a/crates/ty_server/src/project_server/mod.rs b/crates/ty_server/src/project_server/mod.rs new file mode 100644 index 0000000000..499bdd328f --- /dev/null +++ b/crates/ty_server/src/project_server/mod.rs @@ -0,0 +1,195 @@ +//! Answering `by` command lines out of a server that is already holding the project. +//! +//! A `by check` builds a project database, resolves an environment, parses every file and +//! infers every one of them, and then exits and throws all of it away. An editor with the +//! language server running has that same work already done and kept up to date. This is the +//! way across: a loopback socket the server announces in a per-user +//! [record](discovery::Record), a [request](protocol::Request) that names the project and +//! the configuration the caller resolved, and an answer rendered out of the warm database. +//! +//! Two rules shape everything here. +//! +//! The hot answer has to be the cold answer. A server checks what the editor is holding, +//! under settings the editor contributed to, in a build that may be months older than the +//! `by` on the `PATH` — so the answer only crosses when none of that is true, and every +//! other case is a [refusal](protocol::Refusal) the caller silently checks past. A refusal +//! costs the time it would have cost anyway. A wrong answer costs trust in the command. +//! +//! And it is never the only way to get an answer. Nothing here is required to work: no +//! server, a stale record, a refused connection and a garbled reply all lead to the same +//! place, which is the check the caller would have run if none of this existed. + +use std::fmt::Write; + +use rand::TryRng; +use ruff_db::system::{System, SystemPath, SystemPathBuf}; +use ty_static::EnvVars; + +mod check; +pub mod client; +pub mod discovery; +mod listener; +pub mod protocol; + +pub use check::environment; +pub(crate) use listener::{Incoming, Listener, respond}; + +/// Whether [`EnvVars::BY_NO_PROJECT_SERVER`] has switched the project server off. +/// +/// One switch for both halves, because a user who does not want a command line reaching into +/// their editor's process does not want the port open either. +/// +/// Read through `system` rather than the process environment, which is what the rest of the +/// server does: tests run concurrently in one process, so a test that wanted to exercise this +/// could not set the variable without setting it for every other test at the same time. +pub fn disabled(system: &dyn System) -> bool { + system + .env_var(EnvVars::BY_NO_PROJECT_SERVER) + .is_ok_and(|value| { + !matches!( + value.trim().to_ascii_lowercase().as_str(), + "" | "0" | "false" | "no" | "n" | "off" + ) + }) +} + +/// `path` with every symlink resolved, or `path` itself where it cannot be. +/// +/// The two sides reach the same project by different names. A `by check` takes its project +/// root from the working directory, which the operating system hands over already resolved; +/// an editor sends whatever it was opened with. On macOS that alone is enough to disagree — +/// a temporary directory is `/var/…` to one and `/private/var/…` to the other — and a user +/// whose project is behind a symlink disagrees everywhere. +/// +/// So both roots are put through this before either is compared to the other, and a path +/// that cannot be resolved at all is left alone rather than dropped: it still compares equal +/// to itself, which is the case where both sides spell it the same way. +fn canonical(path: &SystemPath) -> SystemPathBuf { + // Two things make a resolved path stop comparing against an unresolved one. Resolving + // only succeeds for a path that exists, so a directory and a file under it that has not + // been written yet would come back spelled differently; and on Windows the answer is a + // verbatim `\\?\C:\…` path, which no path the editor or the caller sends is spelled + // like. So the longest ancestor that does exist is resolved, the rest is put back on, + // and the verbatim prefix is dropped the way `System::canonicalize_path` drops it — + // everything under one root then comes back in one spelling, created or not. + let mut unresolved = Vec::new(); + let mut current = path; + loop { + if let Ok(resolved) = std::fs::canonicalize(current.as_std_path()) + && let Ok(resolved) = SystemPathBuf::from_path_buf(resolved) + { + let mut resolved = resolved.simplified().to_path_buf(); + for component in unresolved.iter().rev() { + resolved.push(component); + } + return resolved; + } + let (Some(parent), Some(name)) = (current.parent(), current.file_name()) else { + return path.to_path_buf(); + }; + unresolved.push(name); + current = parent; + } +} + +/// A fresh secret for one server's lifetime. +/// +/// The listener is on loopback, which every user on this machine can reach; the token is +/// what makes reaching it depend on being able to read a file that only this user can. +fn token() -> anyhow::Result { + let mut bytes = [0u8; 32]; + rand::rngs::SysRng + .try_fill_bytes(&mut bytes) + .map_err(|error| anyhow::anyhow!("failed to read random bytes: {error}"))?; + + let mut token = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(token, "{byte:02x}")?; + } + Ok(token) +} + +/// What a pass over a request produced. +#[derive(Debug)] +pub(crate) enum Outcome { + /// Nothing stands in the way, and the file system has not been re-read yet. + /// + /// The re-read has to happen on the main loop, because it mutates the session — so a + /// request that gets this far goes back there and comes round once more. + NeedsRescan, + + Answered(protocol::Response), +} + +/// Runs a request against `snapshot`. +/// +/// `rescanned` says whether the file system has already been re-read for this request. Until +/// it has, the most this can do is agree that the request is answerable. +/// +/// A panic in the checker is caught here rather than left to the request-handling hook, +/// because the caller on the other end of the socket is waiting for one of two shapes and a +/// server that says nothing at all leaves it there until its own timeout. +pub(crate) fn run( + snapshot: &crate::session::SessionSnapshot, + payload: &protocol::Payload, + rescanned: bool, +) -> Outcome { + // nothing borrowed here is read again after an unwind: the outcome is built from scratch + // on either path, so asserting unwind safety costs nothing + let checked = ruff_db::panic::catch_unwind(std::panic::AssertUnwindSafe(|| match payload { + protocol::Payload::Check(request) => check::run(snapshot, request, rescanned), + })); + + match checked { + Ok(outcome) => outcome, + Err(panic) => { + tracing::error!("A command-line request panicked: {panic}"); + Outcome::Answered(protocol::Response::Refused { + reason: protocol::Refusal::Panicked, + }) + } + } +} + +/// Whether a fresh snapshot could turn `response` into an answer. +/// +/// A check runs against a snapshot, and typing in the editor cancels it. Retrying means +/// going back through the main loop for a snapshot of what the session is now. +pub(crate) fn is_retryable(response: &protocol::Response) -> bool { + matches!( + response, + protocol::Response::Refused { + reason: protocol::Refusal::Cancelled + } + ) +} + +#[cfg(test)] +mod tests { + use super::canonical; + use ruff_db::system::SystemPath; + use tempfile::TempDir; + + /// The two sides compare a root against a path under it, and a path is only ever + /// compared to another that went through [`canonical`] too. What breaks that is one + /// spelling for a path that exists and another for one that does not: a temporary + /// directory reached through a symlink resolves elsewhere, and on Windows a resolved + /// path carries a verbatim prefix an unresolved one never has. + #[test] + fn a_path_that_does_not_exist_yet_is_spelled_like_the_root_it_is_under() { + let directory = TempDir::new().expect("a temporary directory"); + let root = SystemPath::from_std_path(directory.path()).expect("a UTF-8 path"); + + let canonical_root = canonical(root); + let nested = canonical(&root.join("not-written-yet")); + + assert!( + nested.starts_with(&canonical_root), + "`{nested}` should be inside `{canonical_root}`" + ); + assert_eq!( + nested.strip_prefix(&canonical_root).map(SystemPath::as_str), + Ok("not-written-yet") + ); + } +} diff --git a/crates/ty_server/src/project_server/protocol.rs b/crates/ty_server/src/project_server/protocol.rs new file mode 100644 index 0000000000..f7a2c6b35d --- /dev/null +++ b/crates/ty_server/src/project_server/protocol.rs @@ -0,0 +1,377 @@ +//! What a `by` command line and a running server say to each other. +//! +//! One JSON object per message, terminated by a newline. `serde_json` escapes newlines +//! inside strings, so a rendered diagnostic — which is full of them — still arrives as a +//! single line. + +use ruff_db::system::SystemPathBuf; + +/// Bumped whenever the shape below changes. +/// +/// A client and a server that disagree on it stop talking, which matters because the two +/// are separate builds of separate binaries: the server in the editor may be months older +/// than the `by` on the `PATH`. +pub const PROTOCOL: u32 = 1; + +/// A project's merged configuration, in the one shape both sides will produce for it. +/// +/// Serializing the options is not enough on its own. A command line builds its layer of them +/// out of its arguments, and does so by filling in every group whether or not anything went +/// into it — so a `by check` with no flags at all resolves `{"environment": {}, "terminal": +/// {}, …}` where a server resolves `{}`. Those say the same thing, and a comparison that +/// called them different would refuse every invocation there is. +/// +/// So a group with nothing in it is removed, recursively. An empty *list* is left alone: it +/// is a real setting, and `exclude = []` does not mean the same as no `exclude` at all. +pub fn configuration( + options: &T, +) -> Result { + fn prune(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(fields) => serde_json::Value::Object( + fields + .into_iter() + .map(|(name, value)| (name, prune(value))) + .filter(|(_, value)| match value { + serde_json::Value::Null => false, + serde_json::Value::Object(fields) => !fields.is_empty(), + _ => true, + }) + .collect(), + ), + serde_json::Value::Array(items) => { + serde_json::Value::Array(items.into_iter().map(prune).collect()) + } + value => value, + } + } + + serde_json::to_value(options).map(prune) +} + +/// The part of a request that is read before anything is decided. +/// +/// Deliberately lenient about its payload, which stays JSON until the protocol and the build +/// have been agreed. A stricter envelope would fail to parse a newer client's request and +/// close the connection without saying why, which is the one case the version fields exist to +/// explain. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Request { + pub protocol: u32, + + /// The secret from the server's own discovery record. + /// + /// The listener is on loopback, so anything running as any user on this machine can + /// reach the port. The record file it has to read first is not readable by them. + pub token: String, + + pub client: Build, + + pub payload: serde_json::Value, +} + +/// Enough about a build of `by` to tell it apart from another one. +/// +/// The version alone is not enough, and the case it misses is the one that matters most: two +/// builds of the same commit with different uncommitted changes report the same version, and +/// that is the ordinary state of an editor holding a server while its `by` is rebuilt. So the +/// executable is identified as well — the same file, unmodified, is the same build, which is +/// the only direction this needs to be sure about. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Build { + pub version: String, + + /// The executable, and what the file system last said about it. + /// + /// `None` where the executable cannot be identified at all, which never compares equal to + /// anything — including another `None`. + pub executable: Option, +} + +impl Build { + /// This process's build. + pub fn current(version: &str) -> Self { + Self { + version: version.to_owned(), + executable: Executable::current(), + } + } + + /// Whether `other` is known to be the same build as this one. + /// + /// Never assumes: two builds it cannot tell apart are reported as different, because the + /// cost of that is a check the caller was always able to run. + pub(crate) fn is(&self, other: &Self) -> bool { + self.version == other.version + && match (&self.executable, &other.executable) { + (Some(left), Some(right)) => left == right, + _ => false, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Executable { + pub path: String, + pub modified: u64, + pub size: u64, +} + +impl Executable { + fn current() -> Option { + let path = std::env::current_exe().ok()?; + let metadata = std::fs::metadata(&path).ok()?; + Some(Self { + path: path.to_str()?.to_owned(), + modified: metadata + .modified() + .ok()? + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(), + size: metadata.len(), + }) + } +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum Payload { + Check(CheckRequest), +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CheckRequest { + /// The project the command line resolved for itself. + /// + /// A server holds a database per workspace, and a workspace is not always a project — + /// so the root is matched against the databases rather than assumed. + pub project_root: SystemPathBuf, + + /// The command line's configuration, merged across every layer, as JSON. + /// + /// The two sides must agree about what checking this project *means* before one can + /// answer for the other. Merged, because the layers are where the disagreements are: the + /// caller's flags and the editor's own settings both arrive as layers over the same + /// configuration file, and comparing the file alone compares the one thing that could + /// never have differed. + /// + /// Sending the whole thing rather than a hash of it costs a few kilobytes once and buys a + /// diffable explanation when the two disagree. + pub options: serde_json::Value, + + /// The python version, platform and search paths the command line resolved. + /// + /// Not derivable from the options: an environment is discovered as much as configured, + /// out of `VIRTUAL_ENV`, `CONDA_PREFIX`, a `.venv` beside the project, uv's answer, or the + /// interpreter the running executable sits next to — and a server started by an editor + /// discovers it from the editor's environment, not from the caller's shell. Two processes + /// that agree about every option can still be checking against different site-packages. + /// + /// A rendering rather than the settings themselves, because what has to cross is an + /// identity to compare and a difference to print. + pub environment: String, + + /// Whether exclusions apply to paths named on the command line. + /// + /// A database input rather than an option, so it is not in the merged configuration and + /// has to travel on its own. + pub force_exclude: bool, + + /// Whether every diagnostic should say where its rule was turned on. + /// + /// Also a database input, and one a server never sets: an editor has its own way of + /// showing where a rule came from. So a verbose check is always refused — which is the + /// point of comparing it rather than declining to ask, because a refusal says so in the + /// log where a decision not to ask would say nothing. + pub verbose: bool, + + /// The directory the command was run from. + /// + /// Diagnostics name their file relative to where the reader is standing, and the server + /// is standing wherever the editor started it. Without this every path in the answer + /// would be absolute, which is a visible difference from the same check run cold. + pub working_directory: SystemPathBuf, + + /// Whether the command line's output is going somewhere that renders ANSI colour. + /// + /// The server cannot see the caller's terminal, and its own stdout is the LSP + /// connection. + pub color: bool, +} + +/// A response, and the proof that it came from the server the caller meant. +/// +/// The token travels back because a record outlives the process that wrote it: a server that +/// was killed leaves its port behind, and any local process that binds that port next +/// receives the request. Without this, such a process could answer "All checks passed!" and +/// the caller would exit zero on a project it never looked at. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Answer { + pub token: String, + pub response: Response, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum Response { + Check(CheckResponse), + + /// The server will not answer, and the caller should check for itself. + /// + /// Never an error: every refusal is a case where the hot answer might not have been + /// the answer a cold run would give. + Refused { + reason: Refusal, + }, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +// four flags, because a rendering is the one thing that crosses and everything the caller +// still has to decide has to cross beside it. they are not a state machine; they are four +// independent facts about one check +#[expect(clippy::struct_excessive_bools)] +pub struct CheckResponse { + /// The diagnostics, already rendered in the project's configured output format. + /// + /// Rendering happens on the server because it needs the database — the source text + /// behind every span, and the roots that make a path relative. Sending the diagnostics + /// themselves would mean the caller reopening every file they point into. + pub rendered: String, + + pub diagnostics: usize, + + /// Whether the configured output format is one a summary line belongs under. + pub human_readable: bool, + + /// The worst severity reported, or `None` when nothing was. + /// + /// The exit status is the caller's to decide — it is the process that carries it — so + /// what crosses is what the decision reads rather than the decision. + pub max_severity: Option, + + pub io_error: bool, + pub error_on_warning: bool, + + /// Whether the project turned out to hold no python at all. + /// + /// The cold path warns about this, and about [`Self::fatal`], on stderr. Neither is a + /// diagnostic, so neither is in the rendering, and a caller that did not hear them would + /// be quietly getting less than the command normally gives it. + pub empty_project: bool, + + /// Whether anything failed so badly that the check is incomplete. + pub fatal: bool, +} + +/// [`ruff_db::diagnostic::Severity`], in a shape that survives the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SeverityLevel { + Info, + Warning, + Error, + Fatal, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum Refusal { + Protocol { + server: u32, + }, + + /// The two are different builds of `by`. + /// + /// Same protocol, different checker: the answers could differ for any reason at all. + Version { + server: Box, + }, + + UnknownProject, + + /// The server resolved a different configuration for this project than the caller did. + Options { + server: Box, + }, + + /// The server resolved a different environment for this project than the caller did. + Environment { + server: String, + }, + + /// The two disagree about whether exclusions apply to paths named on the command line. + ForceExclude { + server: bool, + }, + + /// The caller asked for diagnostics that say where each rule was turned on. + Verbose, + + /// The server is only checking the files the editor has open. + /// + /// Its answer would be the diagnostics for those files, and a `by check` asked for the + /// project's. + CheckMode, + + /// A file open in the editor does not match what is on disk. + /// + /// The server's view of an open file is the editor's buffer, so answering from it would + /// report on source the caller cannot see. + Unsaved { + files: Vec, + }, + + /// The database changed under the check often enough that it never finished. + Cancelled, + + /// Already logged on the server. + Panicked, +} + +impl std::fmt::Display for Refusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Refusal::Protocol { server } => { + write!( + f, + "the server speaks protocol {server}, this build speaks {PROTOCOL}" + ) + } + Refusal::Version { server } => { + write!(f, "the server is a different build ({})", server.version) + } + Refusal::UnknownProject => f.write_str("the server has no database for this project"), + Refusal::CheckMode => f.write_str( + "the server is only checking open files — set the editor's diagnostic mode to \ + `workspace` for it to answer for the whole project", + ), + Refusal::Options { .. } => { + f.write_str("the server resolved a different configuration for this project") + } + Refusal::Environment { .. } => { + f.write_str("the server resolved a different environment for this project") + } + Refusal::Verbose => f.write_str( + "a verbose check explains where each rule was turned on, and the server's \ + diagnostics do not", + ), + Refusal::ForceExclude { server } => write!( + f, + "the server checks with `force-exclude` {}", + if *server { "on" } else { "off" } + ), + Refusal::Unsaved { files } => { + write!(f, "unsaved editor changes in {}", files.join(", ")) + } + Refusal::Cancelled => f.write_str("the server was too busy to finish the check"), + Refusal::Panicked => f.write_str("the check panicked on the server"), + } + } +} diff --git a/crates/ty_server/src/server.rs b/crates/ty_server/src/server.rs index 10930cebec..bc977dbf66 100644 --- a/crates/ty_server/src/server.rs +++ b/crates/ty_server/src/server.rs @@ -10,7 +10,7 @@ use lsp_types::{ ClientCapabilities, InitializeParams, MessageType, Uri, WorkspaceFolders, WorkspaceFoldersInitializeParams, }; -use ruff_db::system::System; +use ruff_db::system::{System, SystemPathBuf}; use std::num::NonZeroUsize; use std::panic::{PanicHookInfo, RefUnwindSafe}; use std::sync::Arc; @@ -21,6 +21,7 @@ mod main_loop; mod schedule; mod script_progress; +use crate::project_server::Listener as ProjectServerListener; use crate::session::client::Client; pub(crate) use api::Error; pub(crate) use api::{ @@ -39,6 +40,12 @@ pub struct Server { main_loop_receiver: MainLoopReceiver, main_loop_sender: MainLoopSender, session: Session, + + /// The side channel `by` command lines reach this session on. + /// + /// `None` when it is switched off or failed to start. Held rather than used: dropping + /// it takes the server's discovery record off disk. + _project_server_listener: Option, } impl Server { @@ -47,6 +54,7 @@ impl Server { connection: Connection, native_system: Arc, in_test: bool, + project_server_directory: Option, ) -> crate::Result { let (id, init_value) = connection.initialize_start()?; @@ -157,11 +165,28 @@ impl Server { ) })?; + let project_server_listener = project_server_directory.and_then(|directory| { + ProjectServerListener::spawn( + main_loop_sender.clone(), + &directory, + workspace_roots(&workspace_urls), + version, + ) + // not being reachable from a command line is not a reason to fail to start: it + // costs `by check` the warm answer, which it was never entitled to + .inspect_err(|error| { + tracing::warn!("Failed to listen for command-line requests: {error}"); + }) + .ok() + .flatten() + }); + Ok(Self { connection, worker_threads, main_loop_receiver, main_loop_sender, + _project_server_listener: project_server_listener, session: Session::new( resolved_client_capabilities, position_encoding, @@ -200,6 +225,17 @@ impl Server { } } +/// The workspace URIs as paths, dropping any that does not name one. +/// +/// Only for the discovery record, which is a filter on which servers a command line bothers +/// to ask. A URI that is not a file path names nothing a command line could be running in. +fn workspace_roots(workspace_urls: &[Uri]) -> Vec { + workspace_urls + .iter() + .filter_map(|uri| SystemPathBuf::from_path_buf(uri.to_file_path().ok()?).ok()) + .collect() +} + type PanicHook = Box) + 'static + Sync + Send>; struct ServerPanicHookHandler { diff --git a/crates/ty_server/src/server/main_loop.rs b/crates/ty_server/src/server/main_loop.rs index 814eecd39d..2413ae3c3c 100644 --- a/crates/ty_server/src/server/main_loop.rs +++ b/crates/ty_server/src/server/main_loop.rs @@ -1,4 +1,6 @@ -use crate::server::schedule::Scheduler; +use crate::Session; +use crate::project_server; +use crate::server::schedule::{BackgroundSchedule, Scheduler, Task}; use crate::server::{Server, api}; use crate::session::client::{Client, ClientResponseHandler}; use crate::session::{ClientOptions, SuspendedWorkspaceDiagnosticRequest}; @@ -7,6 +9,16 @@ use lsp_server::Message; use lsp_types::Notification; use lsp_types::Uri; use ruff_db::system::SystemPathBuf; +use std::panic::AssertUnwindSafe; +use ty_project::watch::ChangeEvent; + +/// How many fresh snapshots a command-line request gets before it is told to check for +/// itself. +/// +/// Every retry is a check that ran and was cancelled, so this is bounded by patience rather +/// than by correctness: a session being typed into cancels checks faster than they finish, +/// and the caller has a cold path that always works. +const RETRY_LIMIT: u8 = 3; pub(crate) type ConnectionSender = crossbeam::channel::Sender; pub(crate) type MainLoopSender = crossbeam::channel::Sender; @@ -165,6 +177,9 @@ impl Server { // self.try_register_file_watcher(&client); } }, + Event::ProjectServer(incoming) => { + self.answer_project_server_request(incoming, &mut scheduler, client); + } Event::PollUvEnvironments { project_root } => { self.session.poll_uv_sync(&client, &project_root); } @@ -200,12 +215,87 @@ impl Server { uv_sync.select(&self.connection.receiver, &self.main_loop_receiver) } + /// Schedules a `by` command line's request against a snapshot of this session. + /// + /// Twice, in the ordinary case. The first pass decides only whether this session may + /// answer at all, and the second produces the answer — with a re-read of the file system + /// in between, on this thread, because that is what makes the answer the caller's own: + /// this session's picture of the file system is whatever the editor's watcher reported, + /// and the caller is a process that can see the disk directly, so anything the watcher + /// missed — a `git checkout` most of all — would otherwise be answered out of a tree that + /// is no longer there. + /// + /// The order is the point. Re-reading walks every project in the session and makes the + /// editor re-pull its diagnostics, which is far too much to spend on a request that was + /// going to be refused for a configuration the two never shared. + fn answer_project_server_request( + &mut self, + mut incoming: Box, + scheduler: &mut Scheduler, + client: Client, + ) { + if incoming.rescanned { + api::changes::apply(&mut self.session, &client, &[ChangeEvent::Rescan]); + } + + let sender = self.main_loop_sender.clone(); + let task = Task::background(BackgroundSchedule::Worker, move |session: &Session| { + // the snapshot is not read after an unwind: `project_server::run` catches the + // panic and builds its answer without it + let snapshot = AssertUnwindSafe(session.snapshot_session()); + + Box::new(move |_client: &Client| { + let _span = tracing::debug_span!("project server request").entered(); + let snapshot = snapshot.0; + let outcome = project_server::run(&snapshot, &incoming.payload, incoming.rescanned); + + let response = match outcome { + // back to the main loop to be re-read and then answered. the flag is set + // here rather than there so that the pass which set it is the pass that + // agreed the request was worth it + project_server::Outcome::NeedsRescan => { + incoming.rescanned = true; + requeue(&sender, incoming); + return; + } + project_server::Outcome::Answered(response) => response, + }; + + if project_server::is_retryable(&response) && incoming.attempts < RETRY_LIMIT { + incoming.attempts += 1; + requeue(&sender, incoming); + return; + } + + let token = incoming.token.clone(); + if let Err(error) = + project_server::respond(&mut incoming.connection, &token, &response) + { + tracing::debug!("Failed to answer a command-line request: {error}"); + } + }) + }); + + scheduler.dispatch(task, &mut self.session, client); + } + fn initialize(&mut self, client: &Client) { self.session .request_uninitialized_workspace_folder_configurations(client); } } +/// Sends a request back to the main loop for another pass. +/// +/// A failure means the main loop is gone and the process is on its way out; the caller sees +/// the connection close and checks for itself, which is the same thing every other failure +/// here leads to. +fn requeue(sender: &MainLoopSender, incoming: Box) { + if sender.send(Event::ProjectServer(incoming)).is_err() { + tracing::debug!("Dropping a command-line request: the main loop is gone"); + } +} + /// An action that should be performed on the main loop. #[derive(Debug)] pub(crate) enum Action { @@ -235,6 +325,9 @@ pub(crate) enum Event { Action(Action), + /// A request from a `by` command line, over the server's side channel. + ProjectServer(Box), + PollUvEnvironments { project_root: SystemPathBuf, }, diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index c4eb70d62d..663cf211ce 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -1349,6 +1349,7 @@ impl Session { resolved_client_capabilities: self.resolved_client_capabilities, revision: self.revision, client_name: self.client_name, + native_system: self.native_system.clone(), } } @@ -1675,6 +1676,12 @@ pub(crate) struct SessionSnapshot { revision: u64, client_name: ClientName, + /// The file system underneath the editor's buffers. + /// + /// Held so that a snapshot can ask what a file says on disk, which is a different + /// question from what [`Self::open_documents`] answers. + native_system: Arc, + /// IMPORTANT: It's important that the databases come last, or at least, /// after any `Arc` that we try to extract or mutate in-place using `Arc::into_inner` /// and that relies on Salsa's cancellation to guarantee that there's now only a @@ -1696,6 +1703,32 @@ impl SessionSnapshot { &self.index } + /// Every file-backed document the editor has open, with the path it stands for. + /// + /// Notebooks are included, because the whole point of asking is whether the editor is + /// holding something the file system is not. Notebook *cells* are not: a cell's + /// file-level representation is the notebook that contains it, which is already here. + /// + /// A document behind a non-`file` URI has no path on disk at all, and is left out — there + /// is nothing to compare it to, and a check that reads it is reading something a caller on + /// the command line could never have seen either. + pub(crate) fn open_documents(&self) -> impl Iterator)> { + self.index + .keyed_file_documents() + .filter_map(|(key, document)| { + let document = match document { + Document::Text(text) => OpenDocument::Text(text.contents()), + Document::Notebook(_) => OpenDocument::Notebook, + }; + Some((key.file_path()?.as_path(), document)) + }) + } + + /// What `path` says on disk, past whatever the editor is holding for it. + pub(crate) fn read_from_disk(&self, path: &SystemPath) -> Option { + self.native_system.read_to_string(path).ok() + } + pub(crate) fn global_settings(&self) -> &GlobalSettings { &self.global_settings } @@ -1721,6 +1754,18 @@ impl SessionSnapshot { } } +/// What the editor is holding for one file. +/// +/// A notebook carries no text to compare: the editor holds it as cells, and what is on disk +/// is a serialization of them whose formatting is the writer's choice. So the two are only +/// ever known to be the same when neither has been touched, and this says which case it is +/// rather than pretending to answer. +#[derive(Debug, Clone, Copy)] +pub(crate) enum OpenDocument<'a> { + Text(&'a str), + Notebook, +} + /// Represents the client (editor) that's connected to the language server. #[derive(Debug, Clone, Copy)] pub(crate) enum ClientName { diff --git a/crates/ty_server/src/session/index.rs b/crates/ty_server/src/session/index.rs index 81815a929a..b41730dd37 100644 --- a/crates/ty_server/src/session/index.rs +++ b/crates/ty_server/src/session/index.rs @@ -43,6 +43,16 @@ impl Index { }) } + /// The same documents as [`Self::file_documents`], each with the key it is stored under. + pub(super) fn keyed_file_documents(&self) -> impl Iterator { + self.documents + .iter() + .filter(|(_, document)| match document { + Document::Text(text_document) => text_document.notebook().is_none(), + Document::Notebook(_) => true, + }) + } + pub(crate) fn document_handle( &self, uri: &lsp_types::Uri, diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 9fd0f83d54..5f366522c8 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -42,6 +42,7 @@ mod initialize; mod injections; mod inlay_hints; mod notebook; +mod project_server; mod publish_diagnostics; mod pull_diagnostics; mod rename; @@ -226,6 +227,7 @@ impl TestServer { capabilities: ClientCapabilities, initialization_options: Option, env_vars: Vec<(String, Option)>, + project_server_directory: Option, ) -> Self { setup_tracing(); @@ -252,7 +254,13 @@ impl TestServer { // TODO: This should probably be configurable to test concurrency issues let worker_threads = NonZeroUsize::new(1).unwrap(); - match Server::new(worker_threads, server_connection, test_system, true) { + match Server::new( + worker_threads, + server_connection, + test_system, + true, + project_server_directory, + ) { Ok(server) => { if let Err(err) = server.run() { panic!("Server stopped with error: {err:?}"); @@ -1279,9 +1287,18 @@ pub(crate) struct TestServerBuilder { initialization_options: Option, client_capabilities: ClientCapabilities, env_vars: Vec<(String, Option)>, + project_server_directory: Option, } impl TestServerBuilder { + /// The environment variables a test server is started without. + /// + /// Removed so that a test's answer does not depend on the machine it runs on — most of + /// all on which interpreter happens to be on the `PATH`. A test that builds a database of + /// its own alongside the server has to remove them too, or the two resolve different + /// environments and the server rightly refuses to answer for it. + pub(crate) const CLEARED_ENV_VARS: &'static [&'static str] = &["HOME", "PATH", "VIRTUAL_ENV"]; + /// Create a new builder pub(crate) fn new() -> Result { // Default client capabilities for the test server: @@ -1311,11 +1328,11 @@ impl TestServerBuilder { test_context: TestContext::new()?, initialization_options: None, client_capabilities, - env_vars: vec![ - ("HOME".into(), None), - ("PATH".into(), None), - ("VIRTUAL_ENV".into(), None), - ], + env_vars: Self::CLEARED_ENV_VARS + .iter() + .map(|name| ((*name).to_string(), None)) + .collect(), + project_server_directory: None, }) } @@ -1611,6 +1628,16 @@ impl TestServerBuilder { Ok(self) } + /// Let `by` command lines reach this server, announcing it in `directory`. + /// + /// Off by default, and never the user's own directory: a test server that published + /// itself there would be found and asked for answers by whatever `by check` the person + /// running the tests has going in another terminal. + pub(crate) fn with_project_server(mut self, directory: &SystemPath) -> Self { + self.project_server_directory = Some(directory.to_path_buf()); + self + } + /// Build the test server pub(crate) fn build(self) -> TestServer { TestServer::new( @@ -1619,6 +1646,7 @@ impl TestServerBuilder { self.client_capabilities, self.initialization_options, self.env_vars, + self.project_server_directory, ) } } diff --git a/crates/ty_server/tests/e2e/project_server.rs b/crates/ty_server/tests/e2e/project_server.rs new file mode 100644 index 0000000000..56021ecd1a --- /dev/null +++ b/crates/ty_server/tests/e2e/project_server.rs @@ -0,0 +1,646 @@ +//! The side channel a `by` command line reaches a running server on. +//! +//! Exercised through a real server, because everything worth testing about it is about the +//! session: which database answers, what the editor's open buffers do to the answer, and +//! whether the configuration and environment the caller resolved are the ones the server did. +//! None of that exists in the library the handler calls. +//! +//! Requests are built out of a real cold [`ProjectDatabase`], the way the `by` command line +//! builds them, rather than by hand. That is the whole point of [`answers_what_a_cold_check +//!_would_have`]: a request assembled to suit the server would agree with the server about +//! anything, including the things it gets wrong. +//! +//! The server under test publishes itself into a temporary directory rather than the user's +//! own, so that a `by check` running elsewhere on this machine never finds it. + +use std::io::{BufRead, BufReader, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use ruff_db::diagnostic::{DisplayDiagnosticConfig, DisplayDiagnostics}; +use ruff_db::system::{OsSystem, SystemPath, TestSystem}; +use ruff_ranged_value::{ValueSource, ValueSourceGuard}; +use tempfile::TempDir; +use ty_project::{Db as _, ProjectDatabase, ProjectMetadata}; +use ty_server::project_server::protocol; +use ty_server::project_server::protocol::{ + Answer, Build, CheckRequest, PROTOCOL, Payload, Refusal, Request, Response, +}; +use ty_server::project_server::{discovery, environment}; +use ty_server::{ClientOptions, DiagnosticMode}; + +use crate::{TestServer, TestServerBuilder}; + +/// A file whose one error is easy to recognise in rendered output. +const MAIN: &str = "def f() -> str:\n return 42\n"; + +/// How long a test will wait for an answer. +/// +/// Bounded so that a change which stops the server answering — a dropped request, a wedged +/// main loop — fails the suite rather than hanging it. +const TIMEOUT: Duration = Duration::from_secs(60); + +/// Where a test server publishes itself: never the directory a real `by check` reads. +fn published(directory: &TempDir) -> &SystemPath { + SystemPath::from_std_path(directory.path()).expect("a temporary directory to be utf-8") +} + +/// A server holding `src`, reachable by a command line, checking the whole project. +/// +/// Workspace diagnostics because that is the precondition: a server diagnosing only what is +/// open has not checked the project and refuses outright — see +/// [`refuses_while_checking_only_open_files`]. +fn server(directory: &TempDir, files: &[(&SystemPath, &str)]) -> Result { + let mut builder = TestServerBuilder::new()? + .with_project_server(published(directory)) + .with_workspace( + SystemPath::new("src"), + Some(ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace)), + )?; + for (path, content) in files { + builder = builder.with_file(path, content)?; + } + Ok(builder.build().wait_until_workspaces_are_initialized()) +} + +/// The database a `by check` in `project_root` would build for itself. +fn cold(project_root: &SystemPath) -> Result { + cold_with(project_root, None) +} + +/// The same, for a `by check` that was given flags. +/// +/// The flags arrive as a layer over the configuration file, the way `--python-version` and +/// the rest do, rather than by editing the file: the layering is exactly what this has to +/// exercise. +fn cold_with(project_root: &SystemPath, overrides: Option<&str>) -> Result { + // the same environment the server under test was given. an interpreter discovered out of + // one process's environment and not the other's is a real disagreement, and one this + // refuses over — see [`refuses_an_environment_it_does_not_share`] — so a test that wants + // an answer has to stand where the server stands + let system = TestSystem::new(OsSystem::new(project_root)); + for name in TestServerBuilder::CLEARED_ENV_VARS { + system.remove_env_var(*name); + } + + let mut metadata = ProjectMetadata::discover(project_root, &system)?; + metadata.apply_configuration_files(&system)?; + if let Some(overrides) = overrides { + // the same source a flag's value carries, which is what decides whether a range is + // expected alongside it + let _guard = ValueSourceGuard::new(ValueSource::Cli, false); + metadata.apply_override_options(serde_json::from_str(overrides)?); + } + + let mut db = ProjectDatabase::fallible(metadata, system)?; + db.set_checker(Arc::new(ty_ide::DjangoChecker)); + Ok(db) +} + +/// The request that command line would send, resolved from its own database. +fn request_for(db: &ProjectDatabase) -> Result { + let project = db.project(); + Ok(CheckRequest { + project_root: project.root(db).to_path_buf(), + working_directory: project.root(db).to_path_buf(), + options: protocol::configuration(project.metadata(db).to_merged_options().options())?, + environment: environment(db), + force_exclude: project.force_exclude(db), + verbose: project.verbose(db), + color: false, + }) +} + +/// One round trip on the side channel, bypassing the client so that a test can send something +/// a client never would. +fn ask( + directory: &TempDir, + project_root: &SystemPath, + request: impl FnOnce(&discovery::Record) -> Result, +) -> Result> { + let candidates = discovery::candidates(published(directory), project_root); + let Some(record) = candidates.first().map(|candidate| &candidate.record) else { + anyhow::bail!("the server published no record covering `{project_root}`"); + }; + + let mut connection = TcpStream::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, record.port)))?; + connection.set_read_timeout(Some(TIMEOUT))?; + connection.set_write_timeout(Some(TIMEOUT))?; + + let mut line = serde_json::to_vec(&request(record)?)?; + line.push(b'\n'); + connection.write_all(&line)?; + connection.flush()?; + + let mut response = String::new(); + BufReader::new(&connection).read_line(&mut response)?; + if response.is_empty() { + return Ok(None); + } + + let answer: Answer = serde_json::from_str(&response)?; + // the caller has no way to know it is talking to the server unless the server proves it + // read the record, so every test asserts it along the way + assert_eq!(answer.token, record.token); + Ok(Some(answer.response)) +} + +/// A well-formed request, which the tests below then spoil one field at a time. +fn check_request(record: &discovery::Record, db: &ProjectDatabase) -> Result { + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request_for(db)?))?, + }) +} + +/// Asks the usual way, from a database built the way the command line builds one. +fn check(directory: &TempDir, db: &ProjectDatabase) -> Result> { + ask(directory, db.project().root(db), |record| { + check_request(record, db) + }) +} + +/// The claim the whole design rests on: what the server answers is what this process would +/// have produced on its own, character for character. +#[test] +fn answers_what_a_cold_check_would_have() -> Result<()> { + let directory = TempDir::new()?; + let server = server( + &directory, + &[ + (SystemPath::new("src/main.py"), MAIN), + ( + SystemPath::new("src/other.py"), + "import collections\n\nx: int = collections\n", + ), + ], + )?; + + let db = cold(&server.file_path(SystemPath::new("src")))?; + let Some(Response::Check(response)) = check(&directory, &db)? else { + panic!("the server did not answer"); + }; + + let diagnostics = db.check(); + let config = DisplayDiagnosticConfig::new("ty") + .format(db.project().settings(&db).terminal().output_format.into()) + .color(false) + .context(0); + let expected = DisplayDiagnostics::new(&db, &config, &diagnostics).to_string(); + + assert_eq!(response.rendered, expected); + assert_eq!(response.diagnostics, diagnostics.len()); + assert!(response.human_readable); + assert!(!response.empty_project); + + Ok(()) +} + +/// A change made behind the server's back is a change the answer has to include. The editor's +/// watcher is what feeds this session, and nothing tells it about a file written by a `git +/// checkout`, a code generator, or this test. +#[test] +fn answers_from_the_file_system_rather_than_from_the_watcher() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let project_root = server.file_path(SystemPath::new("src")); + + // no `didChangeWatchedFiles`, because the test client is not watching anything + std::fs::write(project_root.join("added.py").as_std_path(), "y: str = 1\n")?; + + let db = cold(&project_root)?; + let Some(Response::Check(response)) = check(&directory, &db)? else { + panic!("the server did not answer"); + }; + + assert!( + response.rendered.contains("added.py"), + "the answer did not see a file written behind the server's back:\n{}", + response.rendered + ); + assert_eq!(response.diagnostics, db.check().len()); + + Ok(()) +} + +/// A flag on the command line lands in the merged configuration, and moves what the check +/// reports. Comparing the configuration file alone would compare the one thing that could +/// never have differed. +#[test] +fn refuses_a_flag_that_changes_the_answer() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + + let db = cold_with( + &server.file_path(SystemPath::new("src")), + Some(r#"{"rules": {"invalid-return-type": "ignore"}}"#), + )?; + + let response = check(&directory, &db)?; + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Options { .. } + }) + ), + "expected a refusal over the configuration, got {response:?}" + ); + + Ok(()) +} + +/// Two processes that agree about every option can still be checking against different +/// site-packages: an environment is discovered as much as configured, and a server discovers +/// it from the editor's environment rather than the caller's shell. +#[test] +fn refuses_an_environment_it_does_not_share() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = request_for(&db)?; + request.environment.push_str(" (somewhere else)"); + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request))?, + }) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Environment { .. } + }) + ), + "expected a refusal over the environment, got {response:?}" + ); + + Ok(()) +} + +/// A verbose check adds a note to every diagnostic saying where its rule was turned on, and a +/// server's database never does. Refused rather than declined to ask, so that the reason is +/// in the log where somebody looking for it can find it. +#[test] +fn refuses_a_verbose_check() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = request_for(&db)?; + request.verbose = true; + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request))?, + }) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Verbose + }) + ), + "expected a refusal over verbosity, got {response:?}" + ); + + Ok(()) +} + +/// The editor is holding text that is not in the file, so the server is checking a program +/// the caller cannot see. +#[test] +fn refuses_while_a_buffer_is_unsaved() -> Result<()> { + let directory = TempDir::new()?; + let main = SystemPath::new("src/main.py"); + let mut server = server(&directory, &[(main, MAIN)])?; + + server.open_text_document(main, "x: int = 1\n", 1); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + match check(&directory, &db)? { + Some(Response::Refused { + reason: Refusal::Unsaved { files }, + }) => assert_eq!(files, vec!["main.py".to_string()]), + other => panic!("expected a refusal over the open buffer, got {other:?}"), + } + + Ok(()) +} + +/// A notebook the editor is holding carries no text to compare against the file, because what +/// is on disk is a serialization whose formatting is the writer's choice. Refused, rather than +/// skipped for want of a way to compare it. +#[test] +fn refuses_while_a_notebook_is_open() -> Result<()> { + let directory = TempDir::new()?; + let notebook = SystemPath::new("src/notes.ipynb"); + let mut server = server( + &directory, + &[ + (SystemPath::new("src/main.py"), MAIN), + (notebook, EMPTY_NOTEBOOK), + ], + )?; + + server.send_notification::( + lsp_types::DidOpenNotebookDocumentParams { + notebook_document: lsp_types::NotebookDocument { + uri: server.file_uri(notebook), + notebook_type: "jupyter-notebook".to_string(), + version: 0, + metadata: None, + cells: Vec::new(), + }, + cell_text_documents: Vec::new(), + }, + ); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + match check(&directory, &db)? { + Some(Response::Refused { + reason: Refusal::Unsaved { files }, + }) => assert_eq!(files, vec!["notes.ipynb".to_string()]), + other => panic!("expected a refusal over the open notebook, got {other:?}"), + } + + Ok(()) +} + +/// An open document that still matches its file is not a reason to refuse. Otherwise the +/// feature would be unavailable to anyone who had the project open, which is everyone it is +/// for. +#[test] +fn answers_while_a_buffer_is_open_and_saved() -> Result<()> { + let directory = TempDir::new()?; + let main = SystemPath::new("src/main.py"); + let mut server = server(&directory, &[(main, MAIN)])?; + + server.open_text_document(main, MAIN, 1); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + let response = check(&directory, &db)?; + assert!( + matches!(response, Some(Response::Check(_))), + "an open, saved buffer should not stop the server answering, got {response:?}\nclient: {}", + environment(&db) + ); + + Ok(()) +} + +/// The default editor setting diagnoses only what is open. A server in that mode has not +/// checked the project and must not answer as though it had. +#[test] +fn refuses_while_checking_only_open_files() -> Result<()> { + let directory = TempDir::new()?; + let server = TestServerBuilder::new()? + .with_project_server(published(&directory)) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.py"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + + let db = cold(&server.file_path(SystemPath::new("src")))?; + let response = check(&directory, &db)?; + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::CheckMode + }) + ), + "expected a refusal over the check mode, got {response:?}" + ); + + Ok(()) +} + +/// A server holds a database per workspace, and is asked about a project rather than about a +/// workspace. One it does not hold, it does not answer for. +#[test] +fn refuses_a_project_it_does_not_hold() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = request_for(&db)?; + request.project_root = request.project_root.join("nested"); + Ok(Request { + protocol: PROTOCOL, + token: record.token.clone(), + client: record.build.clone(), + payload: serde_json::to_value(Payload::Check(request))?, + }) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::UnknownProject + }) + ), + "expected a refusal over the project, got {response:?}" + ); + + Ok(()) +} + +/// Two builds of `by` disagree about a project's diagnostics as readily as they disagree +/// about anything else, and the caller asked for its own build's answer. +#[test] +fn refuses_a_different_build() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + request.client.version = "0.0.0-not-this-one".to_owned(); + Ok(request) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Version { .. } + }) + ), + "expected a refusal over the build, got {response:?}" + ); + + Ok(()) +} + +/// Two builds of the same commit report the same version — which is the ordinary state of an +/// editor holding a server while its `by` is rebuilt — so the executable settles it. +#[test] +fn refuses_a_rebuild_of_the_same_version() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + if let Some(executable) = request.client.executable.as_mut() { + executable.modified += 1; + } + Ok(request) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Version { .. } + }) + ), + "expected a refusal over the executable, got {response:?}" + ); + + Ok(()) +} + +/// A protocol the server does not speak is said so, rather than left to fail to parse. +#[test] +fn refuses_a_protocol_it_does_not_speak() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + request.protocol = PROTOCOL + 1; + // a later protocol's payload is not this one's, and must not have to be + request.payload = serde_json::json!({ "kind": "SomethingLater" }); + Ok(request) + })?; + + assert!( + matches!( + response, + Some(Response::Refused { + reason: Refusal::Protocol { .. } + }) + ), + "expected a refusal over the protocol, got {response:?}" + ); + + Ok(()) +} + +/// The port is on loopback, which every user on this machine can reach; the token is what +/// makes reaching it depend on being able to read a file that only this user can. +#[test] +fn says_nothing_without_the_token() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let db = cold(&server.file_path(SystemPath::new("src")))?; + + let response = ask(&directory, db.project().root(&db), |record| { + let mut request = check_request(record, &db)?; + request.token = "not the token".to_owned(); + Ok(request) + })?; + + assert!( + response.is_none(), + "a request without the token was answered with {response:?}" + ); + + Ok(()) +} + +/// A record only rules a server *in*. Whether the database is really rooted there is the +/// server's to answer, but a server for an unrelated tree is not worth connecting to at all. +#[test] +fn a_record_is_found_only_from_inside_its_roots() -> Result<()> { + let directory = TempDir::new()?; + let server = server(&directory, &[(SystemPath::new("src/main.py"), MAIN)])?; + let published = published(&directory); + let project_root = server.file_path(SystemPath::new("src")); + + assert_eq!(discovery::candidates(published, &project_root).len(), 1); + assert_eq!( + discovery::candidates(published, &project_root.join("nested")).len(), + 1, + "a directory inside a root is inside it" + ); + assert_eq!( + discovery::candidates(published, SystemPath::new("/somewhere/else")).len(), + 0 + ); + + Ok(()) +} + +/// A record whose server has gone is taken away by whoever discovers that, or it would cost +/// every later `by check` a connection attempt for the life of the machine. +/// +/// No server here at all: a record is a file, and this is about what happens to a file whose +/// process is not there any more. +#[test] +fn a_record_outlives_a_server_only_until_somebody_tries_it() -> Result<()> { + let project = TempDir::new()?; + let project_root = SystemPath::from_std_path(project.path()) + .unwrap() + .to_path_buf(); + std::fs::write(project_root.join("main.py").as_std_path(), MAIN)?; + + let directory = TempDir::new()?; + let published = published(&directory); + let build = Build::current(ruff_db::program_version().unwrap_or("test")); + + // a port nothing is listening on: bound to learn a free number, then let go + let port = std::net::TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)))? + .local_addr()? + .port(); + std::fs::write( + directory.path().join("stale.json"), + serde_json::to_vec(&discovery::Record { + protocol: PROTOCOL, + build: build.clone(), + port, + token: "irrelevant".to_owned(), + roots: vec![project_root.clone()], + })?, + )?; + + let db = cold(&project_root)?; + assert_eq!(discovery::candidates(published, &project_root).len(), 1); + assert!( + ty_server::project_server::client::check(published, request_for(&db)?, &build).is_none(), + "a record whose server is gone should not produce an answer" + ); + assert_eq!( + discovery::candidates(published, &project_root).len(), + 0, + "the stale record should have been taken away" + ); + + Ok(()) +} + +/// The smallest notebook the server will accept as one. +const EMPTY_NOTEBOOK: &str = r#"{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +}"#; diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 88c6b4110f..e67a9723e5 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -61,6 +61,16 @@ impl EnvVars { /// Accepts the same values as the `--output-format` command-line argument. pub const TY_OUTPUT_FORMAT: &'static str = "TY_OUTPUT_FORMAT"; + /// Disable the project server: the language server answering `by` command lines. + /// + /// A server started for a project holds it parsed and inferred, and a `by check` in that + /// project asks the server rather than doing the work again. Set this to `1` to switch + /// that off — on a command line, so it checks for itself, and on a server, so it does + /// not open the socket that makes it reachable. + /// + /// Equivalent to the `--no-server` command-line argument. + pub const BY_NO_PROJECT_SERVER: &'static str = "BY_NO_PROJECT_SERVER"; + /// Enable uv integration. /// /// When set to `"1"` or `"true"`, ty invokes `uv workspace metadata` to discover the workspace diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 17573f651c..0fbd139d0f 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -14,7 +14,7 @@ in addition to the cli provided by `ty`, `by` includes: | command | what it does | | ------------------- | ------------------------------------------------------------------ | | `run` | transpile and run a module with `python -m ` | -| `build` | transpile every `.by`/`.byi` file and write to `out/` | +| `build` | transpile every `.by`/`.byi` file and write to `build/` | | `compile` | compile `.by` files to native CPython extension modules | | `generate-api-file` | write a public-api lockfile (see [api-lock](features/api-lock.md)) | | `transpile` | transpile a single file to stdout (reads stdin if no file) | @@ -104,18 +104,20 @@ reach for when one file is mid-edit artifacts and exit status answer different questions. everything that could be emitted is emitted, sourcemap and package markers included; the exit status says whether anything was *reported*. so a build that prints an error exits 1 while -still leaving a usable `out/`, and `by build && pytest out/tests` runs the tests +still leaving a usable `build/`, and `by build && pytest build/tests` runs the tests only against a tree the checker had nothing to say about -writes the transpiled python to `./out/` mirroring the *module* tree. a +writes the transpiled python to `./build/` mirroring the *module* tree. a src-layout project's `src/package_name/main.by` is the module -`package_name.main`, so it lands at `out/package_name/main.py` — `out/` is a +`package_name.main`, so it lands at `build/package_name/main.py` — `build/` is a directory you can put on `sys.path` as it stands, and `run.main` names a module -the same way an import does. the `out/` directory is **not** considered -first-party source for `by check` or `by generate-api-file` — it is regenerated -on every build +the same way an import does. `by generate-api-file` does not read `build/` as +first-party source — it is regenerated on every build. `by check` still walks it, +though, so a project that keeps its output beside its sources will see +diagnostics reported against generated python; exclude it with +`src.exclude = ["build"]` if that is in the way -alongside the python it writes `out/_by_sourcemap.py`, mapping each generated +alongside the python it writes `build/_by_sourcemap.py`, mapping each generated line back to the `.by` line it came from, with a digest of both files so a tool reading it can tell whether it still describes what is on disk — see [sourcemaps](development/sourcemaps.md) @@ -127,9 +129,9 @@ reading it can tell whether it still describes what is on disk — see > definition — see [native compilation](development/compilation/index.md) ```sh -by compile # every .by file under the project root → out/ +by compile # every .by file under the project root → build/ by compile hot.by # one file -by compile -o build hot.by # a different output directory +by compile -o native hot.by # a different output directory by compile --verbose # report every function left interpreted, and why by compile --emit-c-only # write the generated C without compiling it by compile --no-any # refuse to leave a gradual-typed function interpreted @@ -137,11 +139,56 @@ by compile --require-native # refuse to leave *any* function interpreted ``` the output directory mirrors the *module* tree, the way `by build`'s does: the -package member `pkg/sub/dup.py` lands at `out/pkg/sub/dup.cpython-313-darwin.so`, +package member `pkg/sub/dup.py` lands at `build/pkg/sub/dup.cpython-313-darwin.so`, and the package `pkg/sub/__init__.py` at -`out/pkg/sub/__init__.cpython-313-darwin.so` — so `out/` can go on `sys.path` as +`build/pkg/sub/__init__.cpython-313-darwin.so` — so `build/` can go on `sys.path` as it stands and every module imports under the dotted name it was compiled as +**`by compile` writes everything `by build` writes, and the extensions as well.** +a compiled module is not a program on its own: it reads its templates and +fixtures relative to itself, and imports the modules beside it. so the tree holds +the whole project as importable python — every `.by` transpiled, every +hand-written `.py`, `py.typed`, the data files, the sourcemap — with a native +extension beside each module that was compiled. python's finder prefers an +extension to source, so those modules load natively and the rest are interpreted, +which is what makes naming files a speed decision rather than a correctness one: + +```sh +by compile hot.by # hot is native, every other module is interpreted +``` + +naming files decides what is *compiled*, not what is written: the whole project +is checked and transpiled either way, so `by compile hot.by` costs what a +`by build` costs plus the one module's native compile. that is the price of the +tree being a program rather than a heap. + +`build/` is a mirror rather than a pile. what the previous run wrote and this one +did not is taken back, which matters most for extensions: python prefers one to +source, so an extension left behind by a deleted or renamed module would go on +shadowing the `.py` written in its place. the artefacts therefore describe the +*last* invocation — `by compile a.by` then `by compile b.by` leaves `b` native and +`a` interpreted, and `--emit-c-only`, which stops before the C compiler, leaves a +tree with no extensions at all. in every case each module still imports, so this +costs speed rather than correctness; `by compile` with no arguments compiles the +whole project. + +the directory is shared with `by build` on purpose — the same tree, built two +ways — and the two take each other's output back accordingly. it is also a name +setuptools and `python -m build` use, so a project using both writes them into one +directory; nothing is destroyed, because the manifest only ever takes back what +`by` itself wrote. + +a directory holding a `.by-manifest` is a build output, whoever wrote it, and is +never read as source or carried into another tree — a project that builds into +two directories would otherwise put a copy of each inside the other. that is by +the marker rather than by the directory's name, because `--out` can say anything. +a project that deliberately ships a directory containing one has to rename the +marker or keep the directory outside its source roots. + +> upgrading from a version that wrote `out/`: nothing migrates it, and nothing +> reads it any more. delete it — a stale `out/` holds importable python that no +> build refreshes. + `--no-any` buys no speed on its own — it is a **predictability contract**. a gradual type is the commonest reason a function silently stays interpreted, and a decline is invisible unless you look for it, so a module that means to be fully @@ -177,7 +224,7 @@ compiled functions are installed over the top ```console $ by compile hot.by --verbose -hot.by -> out/hot.cpython-313-darwin.so +hot.by -> build/hot.cpython-313-darwin.so declined describe: `list[int]` has no native representation yet compiled 1 module(s) diff --git a/docs/basedpython/development/compilation/index.md b/docs/basedpython/development/compilation/index.md index ad4b486810..3c857afac1 100644 --- a/docs/basedpython/development/compilation/index.md +++ b/docs/basedpython/development/compilation/index.md @@ -9,11 +9,22 @@ transpiled python it replaces: ```sh -by compile # compile the whole project to ./out/ +by compile # compile the whole project to ./build/ by compile app.hot # compile one module, leave the rest interpreted python -c "import app" # the extension is picked up ahead of the .py ``` +`by compile` writes everything `by build` writes and the extensions as well, so +`build/` holds the whole project as importable python — every `.by` transpiled, +every hand-written `.py`, the markers, the data files, the sourcemap — with a +native extension beside each module that was compiled. two things depend on that. +a compiled module reads its fixtures relative to itself the way its interpreted +twin does, so the twin property below is a claim about two modules in the *same* +project, not about one of them running somewhere the other's data never reached. +and the tree's manifest only means anything while one command authors the whole +tree: deleting what the last run wrote and this one did not is right for a mirror +and wrong for a heap. + the observable behaviour of a compiled module and its interpreted twin must be identical. that is not an aspiration, it is the property the entire test strategy is built on ([plan](plan.md#differential-testing)) @@ -88,7 +99,7 @@ AST and ty's inferred types │ │ │ cc + ld (platform) ▼ ▼ - out/*.py out/*.cpython-*.so + build/*.py build/*.cpython-*.so ``` ordinary python enters the same front end. `by_irbuild` lowers the `.py` AST diff --git a/docs/basedpython/development/compilation/technology.md b/docs/basedpython/development/compilation/technology.md index 6aa6ff60fa..9333b8663c 100644 --- a/docs/basedpython/development/compilation/technology.md +++ b/docs/basedpython/development/compilation/technology.md @@ -299,7 +299,7 @@ instead, three entry points over one core: ### the CLI ```sh -by compile # whole project → out/ +by compile # whole project → build/ by compile app.hot app.parse # a subset; the rest stays interpreted by compile --tier=1 # open world (see index.md) by compile --annotate # emit C next to the .by that produced it diff --git a/docs/basedpython/development/sourcemaps.md b/docs/basedpython/development/sourcemaps.md index 616f94412b..e8500b017b 100644 --- a/docs/basedpython/development/sourcemaps.md +++ b/docs/basedpython/development/sourcemaps.md @@ -33,7 +33,7 @@ a correct sourcemap is the single primitive all of these share span back to its `.by` line for a source-annotated diagnostic - `_by_sourcemap.py` carries two tables keyed by the generated `.py` path: `SOURCEMAP`, the `.by` path and its line table, and `DIGESTS`, the sha-256 of - both files that entry describes. `by build` writes it into `out/` beside the + both files that entry describes. `by build` writes it into `build/` beside the python it describes, and `by run` into the temporary tree it executes — where it lives only as long as that run. see [staleness](#staleness) below @@ -90,7 +90,7 @@ never read is invisible to them the traceback shim is the first consumer, and it shows what refusing looks like: when either digest disagrees it leaves the frame in the generated python and writes a note saying which file no longer matches. a frame pointing at -`out/main.py` is a worse answer, but a frame quoting a `.by` that has been +`build/main.py` is a worse answer, but a frame quoting a `.by` that has been rewritten since is a false one both tables are keyed by the generated path exactly as the map spells it. the diff --git a/docs/basedpython/features/api-lock.md b/docs/basedpython/features/api-lock.md index ebf577782d..f69a12d023 100644 --- a/docs/basedpython/features/api-lock.md +++ b/docs/basedpython/features/api-lock.md @@ -71,7 +71,7 @@ record. fields are colon-separated, and the records are sorted lexicographically - any symbol whose simple name starts with `_` (unless it's one of the conventional public dunders above) - stdlib, site-packages, and other non-first-party modules -- output from `by build` — the `out/` directory is not considered first-party +- output from `by build` — the `build/` directory is not considered first-party source for lockfile purposes ## determinism and stability diff --git a/docs/basedpython/features/differences-from-python.md b/docs/basedpython/features/differences-from-python.md index 05e0c5c507..0a2b060927 100644 --- a/docs/basedpython/features/differences-from-python.md +++ b/docs/basedpython/features/differences-from-python.md @@ -32,7 +32,7 @@ the same source, running, does something else the compiler doesn't always do `isinstance`, for example `x is None` will become `x is None` in python, this is because "type of x is None" and "value of x is None" have identical meanings -see [identity and isinstance](identity-swap.md) +see [type tests and identity](identity-swap.md) ### a mutable default is re-evaluated per call diff --git a/docs/basedpython/features/enums.md b/docs/basedpython/features/enums.md index cd575279d4..c11421e27e 100644 --- a/docs/basedpython/features/enums.md +++ b/docs/basedpython/features/enums.md @@ -94,9 +94,10 @@ variants lower to **subclasses** of the enum attached as class attributes, so they are reached qualified through the enum name — `Shape.Circle(2.0)`, `Shape.Point` — everywhere: inside the enum body, in pattern contexts (`case Shape.Circle(r):`), and at module scope. variant constructors are real -classes at runtime, so `x is Shape.Circle` works (recall `is` is basedpython's -[`isinstance`](identity-swap.md); use `type(x) === Shape.Circle` for an -exact-class check). because variants are qualified, the same variant name may +classes at runtime, so `x is Shape.Circle` [tests](identity-swap.md) whether `x` +is one; use `type(x) === Shape.Circle` for an exact-class check. a payload-less +variant names the type holding exactly that one object, so `x is Shape.Point` is +the identity it comes down to. because variants are qualified, the same variant name may appear in two different enums (`A.Same` vs `B.Same`) without collision where the expected type is the enum, a variant may also be written diff --git a/docs/basedpython/features/identity-swap.md b/docs/basedpython/features/identity-swap.md index 712acae1bb..f04b625307 100644 --- a/docs/basedpython/features/identity-swap.md +++ b/docs/basedpython/features/identity-swap.md @@ -1,7 +1,7 @@ -# identity and isinstance +# type tests and identity -basedpython swaps the surface syntax for identity comparison and `isinstance` -checks: `===` is identity and `is` is an instance check +basedpython swaps the surface syntax for identity comparison and type tests: +`===` is identity, and `is` asks whether a value has a type ```by if x === y: @@ -31,8 +31,12 @@ if not isinstance(x, str): | ------------ | ---------------------- | | `x === y` | `x is y` | | `x !== y` | `x is not y` | -| `x is y` | `isinstance(x, y)` | -| `x is not y` | `not isinstance(x, y)` | +| `x is T` | `isinstance(x, T)` | +| `x is not T` | `not isinstance(x, T)` | + +`T` is a *type*, and the check that comes out is whatever decides membership of +it — `isinstance` for a class, but an equality for a literal and an identity for +`None`. the sections below say which ## why @@ -40,26 +44,62 @@ if not isinstance(x, str): rare outside of `is None`. basedpython promotes the common case to a keyword and demotes identity to a triple-equals operator borrowed from JavaScript -## checking against `None` +## the right-hand side is a type + +`is` takes a *type expression*, the same thing an annotation takes. anything an +annotation can name, a test can test against: + +```by +from typing import Literal + +def f(v: object): + if v is int | str: ... # a union + if v is list: ... # a class, arguments and all + if v is Literal[1, 2]: ... # a set of values + if v is type[int]: ... # a class object + if v is f"item-{int}": ... # a string pattern +``` + +and anything an annotation rejects, a test rejects with the same message — +`v is os` names a module, which is not a type in either place + +this is why `is` does not mean `isinstance`'s tuple: `isinstance(v, (int, str))` +spells "any of these", while `(int, str)` in a type expression is the *tuple +type*. write `v is int | str` for the first + +## checking against `None` and other values -`a is None` and `a is not None` stay as python identity checks. `a === None` -spells the same thing +`None`, `True`, a number, a string and an enum member are all type expressions — +each names the type holding exactly that one value. so the check that comes out +is the identity or equality that decides membership of that type: + +```by +if a is None: ... # → a is None +if flag is True: ... # → type(flag) is bool and flag == True +if g is Genre.A: ... # → g is Genre.A +``` -## checking against values +the class guard on a literal is not redundant: python's `1 == True` would +otherwise let a `bool` satisfy `Literal[1]` -`isinstance` requires a class as its second argument, so a rhs that resolves -to a plain *value* keeps python identity semantics. this covers literal -singletons (`None`, `True`/`False`, numbers, strings, `...`), enum members, -and any other rhs whose static type is an instance rather than a class: +## a target with no runtime form -```py -enum class Genre: - case A, B +a test narrows, so it has to *earn* its `True`. a target the runtime could only +partly check is rejected rather than approximated, by `erased-type-check` — and +the emitted python answers `False`, since a test that cannot be made is one +nothing satisfies: -Genre.A is not Genre.B # stays `is not` — members are singleton instances +```by +if v is Any: ... # error: admits every value +if v is Callable[[], int]: ... # error: the signature is not recorded on the value +if v is Movie: ... # error: a TypedDict's instances are plain dicts ``` -a payload-bearing variant (`Shape.Circle`) *is* a class, so `x is Shape.Circle` lowers to `isinstance(x, Shape.Circle)` as usual +a protocol is checkable when basedpython can see enough to check it: a +`@runtime_checkable` one gets the presence check python itself performs, and any +other is checked member by member against the value's +[reified annotations](reified-generics.md). one with a member the emitted python +cannot name is rejected ## a test that can never hold @@ -86,8 +126,30 @@ subclass of `A` either. a [parametric target](parametric-type-tests.md) is judged by the same fold that decides the test, so a use-site variance projection (`a is A[out int]`) that -makes the test possible keeps it quiet. a union target is never reported: any arm -matching makes the whole test hold +makes the test possible keeps it quiet. a union target is reported only when no +arm can hold: any arm matching makes the whole test hold + +## a settled test is its answer + +where the value's type decides the question, the test *is* that answer, and the +branch it guards is decided with it: + +```by +def f(x: int): + reveal_type(x is int) # revealed: True +``` + +an undecidable test is `bool`. the identity folds python applies to the same +operator have no place here: the right-hand side names a type rather than the +class object the same source spells as a value + +## chaining + +a type test may not join a chained comparison. python chains `a is int is str` +into `a is int and int is str`, whose second half asks whether the *class* `int` +has the type `str` — never what the writer meant, so it is a syntax error. split +it into separate tests joined with `and`. a chain of `===` / `!==` is ordinary +python and stays legal ## interaction with `==` @@ -96,7 +158,6 @@ and `===` are remapped ## scope -the swap applies to every comparison in source, with the value-rhs exemption -above decided from static types. there is no opt-out at the statement level — -write `===` / `!==` whenever you mean identity. ty understands both forms -when type-checking `.by` files +the swap applies to every comparison in source. there is no opt-out at the +statement level — write `===` / `!==` whenever you mean identity. ty understands +both forms when type-checking `.by` files diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index 42b8001d20..806039e161 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -45,6 +45,8 @@ features that apply to a project rather than to a file language is written inside it - [linting](linter.md) — the `BY` rules, and how ruff's own rules read `.by` source +- [the project server](project-server.md) — `by check` answered out of the + language server's warm state @@ -170,7 +172,7 @@ syntax inside a function body
    - [context-sensitive resolution](context-sensitive-resolution.md) — `a: Color = Red` -- [identity and isinstance (`===` / `!==` / `is`)](identity-swap.md) +- [type tests and identity (`is` / `===` / `!==`)](identity-swap.md) - [optional chaining (`?.`)](optional-chaining.md) - [none-coalesce operator (`??`)](none-coalesce.md) - [postfix await (`.await`)](await-attribute.md) diff --git a/docs/basedpython/features/linter.md b/docs/basedpython/features/linter.md index 276e6ae5f5..3e8550f015 100644 --- a/docs/basedpython/features/linter.md +++ b/docs/basedpython/features/linter.md @@ -78,12 +78,34 @@ depending on a line that basedpython does not need a rule that suggests a replacement suggests the python one. `SIM108` above is the case where that composes; where it does not, the suggestion is still valid -`.by`, just not the shortest way to write it - -nothing in ruff's rule set is known to report a construct that is correct -basedpython. `F821` used to: an unqualified builder inside a +`.by`, just not the shortest way to write it. `E721` is the exception, and names +basedpython's operators instead, because python's advice there — reach for `is` +— spells [a type test](parametric-type-tests.md) in `.by` rather than the +identity comparison it means in python + +a rule that reads a construct as the python it resembles is a false positive, +and each one is answered where the misreading is, not by silencing the rule: + +- `or` and `and` inside a type expression are + [union and intersection](or-and-types.md), so the boolean rules — `SIM222`, + `SIM223`, `RUF021` — say nothing there +- a bare string in a type position is + [the literal type](literal-types.md), not a forward reference, so `UP037` + does not offer to take its quotes off +- `is` is a type test unless its right-hand side is a literal, so `F632` reports + only what really compares identity: the `===` and `!==` spellings, and an + `is` against a literal +- a [match type](match-types.md)'s `case` arms are types, so `E701` does not + read the `:` in one as opening a suite +- a [destructuring](destructuring.md) binder always binds, so its captures are + reported the way the equivalent python unpacking's are — `B007` for a loop, + `F841` never +- a type parameter is not a constant, however the one-letter convention spells + it, so `assert T == int` is not a `SIM300` yoda condition + +`F821` is the older instance of the same thing: an unqualified builder inside a [trailing-lambda](trailing-lambdas.md) block resolves against the block's [implicit receiver](implicit-receivers.md), and the linter cannot see receiver -types, so it now defers every unresolved name inside a block to `by check`. the +types, so it defers every unresolved name inside a block to `by check`. the same deferral covers `self` and an [enum variant](context-sensitive-resolution.md) written bare diff --git a/docs/basedpython/features/project-server.md b/docs/basedpython/features/project-server.md new file mode 100644 index 0000000000..9bad89ec83 --- /dev/null +++ b/docs/basedpython/features/project-server.md @@ -0,0 +1,82 @@ +# the project server + +`by check` asks a running [language server](editor.md) for the project's diagnostics before +checking for itself, and says so when it does: + +```console +$ by check +using project server information +All checks passed! +``` + +a server that has the project open has already parsed and inferred it, and has been keeping +that current ever since. on a project of about twelve thousand files that is the difference +between four seconds and under one + +## what it needs + +the server has to be checking the whole project, which is not what an editor asks for by +default — the default is only the files you have open. a server in that mode has never looked +at the rest of the project, so it has no answer to give and does not pretend to: + +```json +{ "ty.diagnosticMode": "workspace" } +``` + +this is the same setting that decides whether errors in files you have not opened appear in +your editor's problems list, so it is one you want anyway if you want this + +## it is the same answer, or it is no answer + +what comes back is what your own process would have computed. the server refuses whenever +that might not be true, and a refusal is not a failure — the check simply runs the way it +always did, and you see no message + +it refuses when the two are different builds of `by`, when they resolved different +configuration, when they resolved different environments, when a file open in the editor has +unsaved changes, and when the session is being typed into so fast that the check keeps being +cancelled + +**any flag that changes the check is a different check.** `--python-version`, +`--error-on-warning`, `--output-format` and the rest all land in the configuration the two +compare, so a `by check` carrying one of them is always run in full. so is a `-v` check, whose +diagnostics carry an explanation of where each rule was turned on that a server's do not. and +`by check` does not ask at all when the invocation is not a whole-project check: `--watch`, +`--fix`, `--add-ignore`, or a check pointed at particular paths + +the answer describes the project as it is on disk, not as your editor last noticed it — a +`git checkout` your editor missed does not go unseen + +### finding out which one it was + +every refusal is logged. `-v` will not show you, because a verbose check is itself a reason to +refuse; ask for the log directly instead: + +```console +$ TY_LOG=ty=debug by check +``` + +## turning it off + +`--no-server` checks from scratch: + +```console +$ by check --no-server +``` + +`BY_NO_PROJECT_SERVER=1` does the same, and does it on both halves: a command line with it set +checks for itself, and a server started with it set does not open the socket that makes it +reachable at all + +```console +$ BY_NO_PROJECT_SERVER=1 by check +``` + +## what it opens + +a server listens on a loopback port and writes a record of itself — the port and a random +secret — into a per-user directory under ty's cache, readable only by you. a request without +the secret is not answered, and an answer that does not repeat the secret back is not believed + +the record is removed when the server shuts down. one whose server was killed outright stays +behind until the next `by check` finds nothing listening and clears it away diff --git a/docs/basedpython/frameworks/pytest.md b/docs/basedpython/frameworks/pytest.md index 5d8b56b32d..e450bdbbd9 100644 --- a/docs/basedpython/frameworks/pytest.md +++ b/docs/basedpython/frameworks/pytest.md @@ -131,7 +131,7 @@ custom collection configs in `pytest.ini` or `pyproject.toml` aren't read yet, s ```sh by build -pytest out/ +pytest build/ ``` ## see also diff --git a/docs/basedpython/getting-started.md b/docs/basedpython/getting-started.md index 956eee282d..0b19c34a85 100644 --- a/docs/basedpython/getting-started.md +++ b/docs/basedpython/getting-started.md @@ -8,11 +8,9 @@ minutes basedpython ships as the `basedpython` package, which installs two executables: `by`, the type checker and transpiler, and `buff`, the linter and formatter -=== "uv" - - ```sh - uv add --dev basedpython - ``` +```sh +uv add --dev basedpython +``` verify it works: @@ -36,8 +34,7 @@ by run main ``` `by run main` finds `main.by` in the current directory, transpiles it (and all -other `.by` files in the project) to a temporary directory, then executes -`python -m main` from there +other `.by` files in the project), then executes it with your python interpreter !!! note "`by run` takes a module, not a path" @@ -63,30 +60,29 @@ see [configuration](configuration.md) for everything that can go in there ## building -`by build` writes the project to `out/` as python: +`by build` writes the project to `build/` as python: ```sh by build ``` ```text -main.by -> out/main.py -utils.by -> out/utils.py +main.by -> build/main.py +utils.by -> build/utils.py build complete (2 files) ``` that is the whole project, not only its `.by` files — a hand-written `.py` module, a `py.typed`, a data file the program reads are all carried across to -the same place, so `out/` runs the way the source does +the same place, so `build/` runs the way the source does the generated `.py` files are ordinary python. run them with any python tool: ```sh -python out/main.py -pytest out/ -mypy out/ -ruff check out/ +python build/main.py +pytest build/ +mypy build/ ``` to ship the project rather than run it, build a wheel — see @@ -105,7 +101,7 @@ uv build by build - name: Test - run: pytest out/ + run: pytest build/ ``` ## converting python to basedpython @@ -194,7 +190,7 @@ echo 'a = b ?? 1' | by transpile output goes to stdout - redirect it to a file if you want to keep it (`by transpile hello.by > hello.py`). use `by build` to transpile a whole -project into `out/` +project into `build/` ## forward references diff --git a/docs/basedpython/index.md b/docs/basedpython/index.md index 2296f5010f..a4d62e3d94 100644 --- a/docs/basedpython/index.md +++ b/docs/basedpython/index.md @@ -9,9 +9,9 @@ files any python tool can read - **a python type checker with [framework support](frameworks/index.md)** — pydantic, sqlalchemy, pytest and django are modelled directly, so the magic they do at runtime checks like ordinary code -- **a build system** — write code against the latest version of python, and ship wheels that are compatible with old ones, no more waiting for 5 years to use something -- **basedpython, a python-like language that builds into python wheels** — `uv build`, and see [packaging](packaging.md) -- **compiles into high performance python extension modules** +- **polyfill and transpilation** — write code against the latest version of python, and ship wheels that are compatible with old ones, no more waiting for 5 years to use new features +- **basedpython-language** — based on Python, powerful and modern. fully backwards compatible +- **compiles into high performance python extension modules** — or ordinary Python - **a language server, formatter and linter** — high performance and feature rich tooling
    @@ -60,7 +60,7 @@ def main(): > > \- Guido van Rossum -Python and it's type system are held back due to an inability to make breaking changes and a +Python and its type system are held back due to an inability to make breaking changes and a hesitation to introduce new syntax other languages have indulged in modern features, powerful type systems, and integrated tooling. @@ -77,7 +77,7 @@ we want to close that gap `by build` writes ordinary `.py` files. pytest, mypy, ruff and everything else in your stack keep working, because what they see is python - [:octicons-arrow-right-24: how transpilation works](development/how-transpilation-works.md) + [:octicons-arrow-right-24: how to build](cli-reference.md) - :lucide-shapes:{ .lg .middle } **syntax python doesn't have** diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index 7f17db613c..dc80217a4c 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -39,21 +39,21 @@ leave off `--lib` and you also get a `main.by` and a configured entry point, so ## what a build produces -`by build` writes the project to `out/` as python. that is the whole project, +`by build` writes the project to `build/` as python. that is the whole project, not only its `.by` files: ```text -src/app/main.by -> out/app/main.py -src/app/helper.py -> out/app/helper.py -src/app/settings.json -> out/app/settings.json -src/app/py.typed -> out/app/py.typed +src/app/main.by -> build/app/main.py +src/app/helper.py -> build/app/helper.py +src/app/settings.json -> build/app/settings.json +src/app/py.typed -> build/app/py.typed ``` a `.by` file is transpiled; everything else is carried across unchanged, to the same place. the one rearrangement is the source root — `src/app/main.by` is the module `app.main`, so it lands at `app/main.py` and not at `src/app/main.py` -`out/` is a mirror, not a pile: what a previous build wrote and this one did not +`build/` is a mirror, not a pile: what a previous build wrote and this one did not is deleted, so a module you renamed does not go on being importable a stub stays a stub. `a.byi` builds to `a.pyi`, never to `a.py` @@ -230,8 +230,8 @@ checker guessing uv sync ``` -installs the project pointing at `out/`, so `by build` is what refreshes an -editable install. run it after editing, the same way any compiled language +installs the project pointing at `build/` — `by build`'s own output directory, so +a plain `by build` is what refreshes an editable install. run it after editing, the same way any compiled language rebuilds before its changes are visible ## a single-module project diff --git a/python/basedpython/build.py b/python/basedpython/build.py index e07d880833..cb424a66d5 100644 --- a/python/basedpython/build.py +++ b/python/basedpython/build.py @@ -55,8 +55,10 @@ # where `build_editable` stages the project. it is `by build`'s own default # output directory on purpose: an editable install points python at this tree, so -# a plain `by build` is what refreshes an editable install -EDITABLE_STAGING_DIRECTORY = "out" +# a plain `by build` is what refreshes an editable install. the two have to be +# renamed together — an install left pointing at a directory `by build` no longer +# writes goes stale with nothing said about it +EDITABLE_STAGING_DIRECTORY = "build" class BuildError(Exception): diff --git a/scripts/check_ecosystem_roundtrip.py b/scripts/check_ecosystem_roundtrip.py index 24a8ba97fe..d26a68a4d5 100755 --- a/scripts/check_ecosystem_roundtrip.py +++ b/scripts/check_ecosystem_roundtrip.py @@ -14,7 +14,7 @@ `by` commands:: by transpile --reverse # python -> basedpython (in place) - by build # basedpython -> python (-> out/) + by build # basedpython -> python (-> build/) `by build` uses one shared project db, so cross-module types resolve — the same path real `.by` projects take. This is far cheaper than spawning `by` per file: @@ -219,7 +219,7 @@ class ProjectOutcome(NamedTuple): """The result of round-trip-building a project with a single binary.""" error: str | None - # relpath under out/ -> built python; empty when error is set + # relpath under build/ -> built python; empty when error is set outputs: dict[str, bytes] @@ -333,7 +333,7 @@ async def roundtrip_project( build_mem_limit_bytes: int | None, build_timeout: float | None, ) -> ProjectOutcome: - """Reverse the whole project (py->by) then build it (by->py via out/).""" + """Reverse the whole project (py->by) then build it (by->py via build/).""" rc, _, err = await _run( by, ["transpile", "--reverse", "--min-version", ROUNDTRIP_MIN_VERSION, str(root)], @@ -356,10 +356,10 @@ async def roundtrip_project( mem_limit_bytes=build_mem_limit_bytes, timeout=build_timeout, ) - outputs = collect_outputs(root / "out") + outputs = collect_outputs(root / "build") # a non-zero exit from ty diagnostics (e.g. unresolved third-party imports — # the corpus is cloned source-only, so those are expected and unavoidable) - # is not a round-trip failure: the transpile still emitted `out/`. only a + # is not a round-trip failure: the transpile still emitted `build/`. only a # build the watchdog killed (137), or one that produced nothing at all, is a # genuine failure if rc == 137 or (rc != 0 and not outputs): @@ -944,7 +944,7 @@ async def setup_and_run(name: str) -> ProjectDiff | ProjectErrors: logger.warning("project %s failed setup: %s", name, e) return skipped_result(name, f"setup failed: {e}") finally: - # free the clone (source + generated `.by` + `out/`) so disk + # free the clone (source + generated `.by` + `build/`) so disk # doesn't accumulate across the shard's projects. --checkout # is for reuse, so only clean the temp-dir mode if args.checkout is None: diff --git a/scripts/check_project_server.py b/scripts/check_project_server.py new file mode 100644 index 0000000000..c35ccce258 --- /dev/null +++ b/scripts/check_project_server.py @@ -0,0 +1,293 @@ +# /// script +# requires-python = ">=3.11" +# /// +"""Check that `by check` gives the same answer through a running server as it does alone. + +Everything else that covers the project server runs the server and the caller in one +process, where the two share a build, a working directory and a view of the file system by +construction. This runs them the way a user does — a real `by server` over stdin and stdout, +a real `by check` finding it — and compares the two answers byte for byte. + + uv run --no-project scripts/check_project_server.py path/to/by [--modules N] + +`--modules` sizes the generated project, which is also what makes the timings mean anything: +the point of the feature is a project big enough that checking it twice is worth avoiding. +""" + +from __future__ import annotations + +import argparse +import json +import os +import queue +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path +from typing import Any + + +class Server: + """A `by server` on the other end of a pipe, with its output drained. + + Draining is not optional. This asks for pushed diagnostics over a project of a few + thousand files, which is more than a pipe holds — a client that only reads when it wants + something blocks the server inside a write and never gets there. + """ + + def __init__(self, by: Path, root: Path): + self.process = subprocess.Popen( + [by, "server"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + cwd=root, + ) + # `Popen` types the pipes as optional because asking for them is optional; they + # were asked for just above, so bind them once rather than at every use + stdin, stdout = self.process.stdin, self.process.stdout + if stdin is None or stdout is None: + raise RuntimeError("the server started without the pipes it was given") + self.stdin, self.stdout = stdin, stdout + self.messages: queue.Queue[dict[str, Any]] = queue.Queue() + self.reader = threading.Thread(target=self._read_forever, daemon=True) + self.reader.start() + + def _read_forever(self): + while True: + length = None + while True: + line = self.stdout.readline() + if not line: + return + line = line.strip() + if not line: + break + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + if length is None: + return + self.messages.put(json.loads(self.stdout.read(length))) + + def send(self, message: dict[str, Any]): + body = json.dumps(message).encode() + self.stdin.write(f"Content-Length: {len(body)}\r\n\r\n".encode() + body) + self.stdin.flush() + + def await_message(self, matches, timeout: float) -> dict[str, Any] | None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + message = self.messages.get(timeout=deadline - time.monotonic()) + except queue.Empty: + return None + # the server asks the client for things during startup, and a request it never + # gets an answer to leaves it waiting rather than working + if message.get("method") == "workspace/configuration": + self.send( + { + "jsonrpc": "2.0", + "id": message["id"], + "result": [None] * len(message["params"]["items"]), + } + ) + elif "id" in message and "method" in message: + self.send({"jsonrpc": "2.0", "id": message["id"], "result": None}) + if matches(message): + return message + return None + + +def write_project(root: Path, modules: int): + """A project with one error in it, and `modules` files of work around that error. + + The work has to be work. A few thousand files that each declare a dataclass and add up a + list infer almost instantly, and a measurement over those measures process startup — the + part this feature cannot save. What makes a real check slow is inference that has + somewhere to go: generics that get solved, overloads that get picked between, and long + chains where each step's type depends on the last. + """ + (root / "pyproject.toml").write_text('[project]\nname = "sample"\nversion = "0"\n') + (root / "main.py").write_text("def f() -> str:\n return 42\n") + for index in range(modules): + previous = ( + f"from module_{index - 1} import pipeline as previous\n" if index else "" + ) + seed = "previous(rows)" if index else "rows" + (root / f"module_{index}.py").write_text( + f"from collections.abc import Callable, Iterable, Mapping, Sequence\n" + f"from dataclasses import dataclass, field\n" + f"from typing import Generic, TypeVar, overload\n" + f"{previous}\n" + f"T = TypeVar('T')\n" + f"U = TypeVar('U')\n" + f"\n" + f"@dataclass\n" + f"class Row{index}(Generic[T]):\n" + f" key: str\n" + f" value: T\n" + f" tags: dict[str, list[tuple[int, str]]] = field(default_factory=dict)\n" + f"\n" + f"class Box{index}(Generic[T]):\n" + f" def __init__(self, inner: T) -> None:\n" + f" self.inner = inner\n" + f"\n" + f" def map(self, f: Callable[[T], U]) -> 'Box{index}[U]':\n" + f" return Box{index}(f(self.inner))\n" + f"\n" + f"@overload\n" + f"def widen(value: int) -> float: ...\n" + f"@overload\n" + f"def widen(value: str) -> str: ...\n" + f"def widen(value: int | str) -> float | str:\n" + f" return value + 0 if isinstance(value, int) else value\n" + f"\n" + f"def pipeline(rows: Sequence[Row{index}[int]]) -> Sequence[Row{index}[int]]:\n" + f" seeded = {seed}\n" + f" boxed = [Box{index}(row.value).map(widen).map(str).map(len) for row in seeded]\n" + f" grouped: Mapping[str, list[int]] = {{\n" + f" row.key: [box.inner for box in boxed] for row in seeded\n" + f" }}\n" + f" ordered: Iterable[tuple[str, list[int]]] = sorted(\n" + f" grouped.items(), key=lambda pair: (len(pair[1]), pair[0])\n" + f" )\n" + f" return [Row{index}(key, sum(values)) for key, values in ordered]\n" + ) + + +def run_check( + by: Path, root: Path, *args: str, env: dict[str, str] | None = None +) -> tuple[float, subprocess.CompletedProcess[str]]: + # a clean switch position rather than whatever the caller exported, so that a developer + # with the kill switch set in their shell does not get a confusing failure here + environment = {**os.environ, "BY_NO_PROJECT_SERVER": "0", **(env or {})} + started = time.monotonic() + done = subprocess.run( + [by, "check", *args], cwd=root, capture_output=True, text=True, env=environment + ) + return time.monotonic() - started, done + + +USED_THE_SERVER = "using project server information" + +# every one of these changes what a check reports or how it reports it, and none of them is +# something a server was asked to do — so each has to send the caller back to checking for +# itself rather than being quietly ignored +FLAGS_THAT_MUST_REFUSE = [ + ("--python-version", "3.9"), + ("--error-on-warning",), + ("--output-format", "concise"), + ("--no-server",), + ("-vv",), +] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("by", type=Path, help="the `by` binary to exercise") + parser.add_argument("--modules", type=int, default=200) + parser.add_argument( + "--project", + type=Path, + help="an existing project to measure instead of a generated one. a real project is " + "the only corpus that says anything about the timings: what this feature saves is " + "inference, and generated files have almost none to save", + ) + args = parser.parse_args() + + by = args.by.resolve() + generated = args.project is None + if generated: + root = Path(tempfile.mkdtemp(prefix="by-project-server-")) + write_project(root, args.modules) + else: + root = args.project.resolve() + + server = Server(by, root) + try: + server.send( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "processId": os.getpid(), + # pull diagnostics, which is how an editor in workspace mode asks a + # server to check a project. without it the server checks nothing, and a + # measurement against it measures the server doing the whole check inside + # the request — which is the thing this is supposed to avoid + "capabilities": { + "textDocument": {"diagnostic": {}}, + "workspace": {"diagnostics": {"refreshSupport": False}}, + }, + "workspaceFolders": [{"uri": root.as_uri(), "name": "sample"}], + # the server has to be checking the whole project before it can answer for + # one, and this is the setting that decides that + "initializationOptions": {"diagnosticMode": "workspace"}, + }, + } + ) + _ = server.await_message(lambda message: message.get("id") == 1, timeout=30) + server.send({"jsonrpc": "2.0", "method": "initialized", "params": {}}) + + warming = time.monotonic() + server.send( + { + "jsonrpc": "2.0", + "id": 3, + "method": "workspace/diagnostic", + "params": {"previousResultIds": []}, + } + ) + if server.await_message(lambda m: m.get("id") == 3, timeout=900) is None: + print("the server never answered the workspace diagnostic", file=sys.stderr) + return 1 + print(f"warming the server took {time.monotonic() - warming:.2f}s") + + hot_elapsed, hot = run_check(by, root) + cold_elapsed, cold = run_check(by, root, env={"BY_NO_PROJECT_SERVER": "1"}) + flagged = { + flags: run_check(by, root, *flags)[1] for flags in FLAGS_THAT_MUST_REFUSE + } + finally: + server.send({"jsonrpc": "2.0", "id": 2, "method": "shutdown", "params": None}) + _ = server.await_message(lambda message: message.get("id") == 2, timeout=30) + server.send({"jsonrpc": "2.0", "method": "exit", "params": None}) + server.process.wait(timeout=10) + if generated: + shutil.rmtree(root, ignore_errors=True) + + print(f"hot: {hot_elapsed:6.2f}s exit {hot.returncode}") + print(f"cold: {cold_elapsed:6.2f}s exit {cold.returncode}") + + failures = [] + if USED_THE_SERVER not in hot.stderr: + failures.append( + "the check did not use the server. every reason it might have refused is logged, " + "so run `TY_LOG=ty=debug by check` in the project to see which one it was — not " + f"`-v`, which is itself a reason to refuse.\nstderr was:\n{hot.stderr}" + ) + for flags, done in flagged.items(): + if USED_THE_SERVER in done.stderr: + failures.append( + f"`by check {' '.join(flags)}` was answered by the server, which cannot have " + "resolved the same check" + ) + if hot.stdout != cold.stdout: + failures.append( + f"the answers differ.\nhot:\n{hot.stdout}\ncold:\n{cold.stdout}" + ) + if hot.returncode != cold.returncode: + failures.append( + f"the exit statuses differ: {hot.returncode} hot, {cold.returncode} cold" + ) + + for failure in failures: + print(f"\nFAILED: {failure}", file=sys.stderr) + return 1 if failures else 0 + + +sys.exit(main()) diff --git a/scripts/check_project_server.py.lock b/scripts/check_project_server.py.lock new file mode 100644 index 0000000000..5951180dc2 --- /dev/null +++ b/scripts/check_project_server.py.lock @@ -0,0 +1,15 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P7D" + +[options.exclude-newer-package] +astral-dev-toolchain-hyperfine = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-insta = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-shear = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-codspeed = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-nextest = "2026-08-27T00:00:00Z" +astral-dev-toolchain-cargo-fuzz = "2026-09-01T00:00:00Z" diff --git a/zensical.toml b/zensical.toml index 51f1db1b39..807c233347 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,6 +93,7 @@ features = [ { "editor support" = "features/editor.md" }, { "language injection" = "features/language-injection.md" }, { "linting" = "features/linter.md" }, + { "the project server" = "features/project-server.md" }, ] }, { "standard library" = [ { "typeshed improvements" = "features/typeshed.md" }, @@ -173,7 +174,7 @@ features = [ ] }, { "expressions and statements" = [ { "context-sensitive resolution" = "features/context-sensitive-resolution.md" }, - { "identity and isinstance" = "features/identity-swap.md" }, + { "type tests and identity" = "features/identity-swap.md" }, { "optional chaining (?.)" = "features/optional-chaining.md" }, { "none-coalesce operator (??)" = "features/none-coalesce.md" }, { "postfix await (.await)" = "features/await-attribute.md" },